book3s_hv_uvmem.c 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Secure pages management: Migration of pages between normal and secure
  4. * memory of KVM guests.
  5. *
  6. * Copyright 2018 Bharata B Rao, IBM Corp. <bharata@linux.ibm.com>
  7. */
  8. /*
  9. * A pseries guest can be run as secure guest on Ultravisor-enabled
  10. * POWER platforms. On such platforms, this driver will be used to manage
  11. * the movement of guest pages between the normal memory managed by
  12. * hypervisor (HV) and secure memory managed by Ultravisor (UV).
  13. *
  14. * The page-in or page-out requests from UV will come to HV as hcalls and
  15. * HV will call back into UV via ultracalls to satisfy these page requests.
  16. *
  17. * Private ZONE_DEVICE memory equal to the amount of secure memory
  18. * available in the platform for running secure guests is hotplugged.
  19. * Whenever a page belonging to the guest becomes secure, a page from this
  20. * private device memory is used to represent and track that secure page
  21. * on the HV side. Some pages (like virtio buffers, VPA pages etc) are
  22. * shared between UV and HV. However such pages aren't represented by
  23. * device private memory and mappings to shared memory exist in both
  24. * UV and HV page tables.
  25. */
  26. /*
  27. * Notes on locking
  28. *
  29. * kvm->arch.uvmem_lock is a per-guest lock that prevents concurrent
  30. * page-in and page-out requests for the same GPA. Concurrent accesses
  31. * can either come via UV (guest vCPUs requesting for same page)
  32. * or when HV and guest simultaneously access the same page.
  33. * This mutex serializes the migration of page from HV(normal) to
  34. * UV(secure) and vice versa. So the serialization points are around
  35. * migrate_vma routines and page-in/out routines.
  36. *
  37. * Per-guest mutex comes with a cost though. Mainly it serializes the
  38. * fault path as page-out can occur when HV faults on accessing secure
  39. * guest pages. Currently UV issues page-in requests for all the guest
  40. * PFNs one at a time during early boot (UV_ESM uvcall), so this is
  41. * not a cause for concern. Also currently the number of page-outs caused
  42. * by HV touching secure pages is very very low. If an when UV supports
  43. * overcommitting, then we might see concurrent guest driven page-outs.
  44. *
  45. * Locking order
  46. *
  47. * 1. kvm->srcu - Protects KVM memslots
  48. * 2. kvm->mm->mmap_lock - find_vma, migrate_vma_pages and helpers, ksm_madvise
  49. * 3. kvm->arch.uvmem_lock - protects read/writes to uvmem slots thus acting
  50. * as sync-points for page-in/out
  51. */
  52. /*
  53. * Notes on page size
  54. *
  55. * Currently UV uses 2MB mappings internally, but will issue H_SVM_PAGE_IN
  56. * and H_SVM_PAGE_OUT hcalls in PAGE_SIZE(64K) granularity. HV tracks
  57. * secure GPAs at 64K page size and maintains one device PFN for each
  58. * 64K secure GPA. UV_PAGE_IN and UV_PAGE_OUT calls by HV are also issued
  59. * for 64K page at a time.
  60. *
  61. * HV faulting on secure pages: When HV touches any secure page, it
  62. * faults and issues a UV_PAGE_OUT request with 64K page size. Currently
  63. * UV splits and remaps the 2MB page if necessary and copies out the
  64. * required 64K page contents.
  65. *
  66. * Shared pages: Whenever guest shares a secure page, UV will split and
  67. * remap the 2MB page if required and issue H_SVM_PAGE_IN with 64K page size.
  68. *
  69. * HV invalidating a page: When a regular page belonging to secure
  70. * guest gets unmapped, HV informs UV with UV_PAGE_INVAL of 64K
  71. * page size. Using 64K page size is correct here because any non-secure
  72. * page will essentially be of 64K page size. Splitting by UV during sharing
  73. * and page-out ensures this.
  74. *
  75. * Page fault handling: When HV handles page fault of a page belonging
  76. * to secure guest, it sends that to UV with a 64K UV_PAGE_IN request.
  77. * Using 64K size is correct here too as UV would have split the 2MB page
  78. * into 64k mappings and would have done page-outs earlier.
  79. *
  80. * In summary, the current secure pages handling code in HV assumes
  81. * 64K page size and in fact fails any page-in/page-out requests of
  82. * non-64K size upfront. If and when UV starts supporting multiple
  83. * page-sizes, we need to break this assumption.
  84. */
  85. #include <linux/pagemap.h>
  86. #include <linux/migrate.h>
  87. #include <linux/kvm_host.h>
  88. #include <linux/ksm.h>
  89. #include <asm/ultravisor.h>
  90. #include <asm/mman.h>
  91. #include <asm/kvm_ppc.h>
  92. #include <asm/kvm_book3s_uvmem.h>
  93. static struct dev_pagemap kvmppc_uvmem_pgmap;
  94. static unsigned long *kvmppc_uvmem_bitmap;
  95. static DEFINE_SPINLOCK(kvmppc_uvmem_bitmap_lock);
  96. /*
  97. * States of a GFN
  98. * ---------------
  99. * The GFN can be in one of the following states.
  100. *
  101. * (a) Secure - The GFN is secure. The GFN is associated with
  102. * a Secure VM, the contents of the GFN is not accessible
  103. * to the Hypervisor. This GFN can be backed by a secure-PFN,
  104. * or can be backed by a normal-PFN with contents encrypted.
  105. * The former is true when the GFN is paged-in into the
  106. * ultravisor. The latter is true when the GFN is paged-out
  107. * of the ultravisor.
  108. *
  109. * (b) Shared - The GFN is shared. The GFN is associated with a
  110. * a secure VM. The contents of the GFN is accessible to
  111. * Hypervisor. This GFN is backed by a normal-PFN and its
  112. * content is un-encrypted.
  113. *
  114. * (c) Normal - The GFN is a normal. The GFN is associated with
  115. * a normal VM. The contents of the GFN is accesible to
  116. * the Hypervisor. Its content is never encrypted.
  117. *
  118. * States of a VM.
  119. * ---------------
  120. *
  121. * Normal VM: A VM whose contents are always accessible to
  122. * the hypervisor. All its GFNs are normal-GFNs.
  123. *
  124. * Secure VM: A VM whose contents are not accessible to the
  125. * hypervisor without the VM's consent. Its GFNs are
  126. * either Shared-GFN or Secure-GFNs.
  127. *
  128. * Transient VM: A Normal VM that is transitioning to secure VM.
  129. * The transition starts on successful return of
  130. * H_SVM_INIT_START, and ends on successful return
  131. * of H_SVM_INIT_DONE. This transient VM, can have GFNs
  132. * in any of the three states; i.e Secure-GFN, Shared-GFN,
  133. * and Normal-GFN. The VM never executes in this state
  134. * in supervisor-mode.
  135. *
  136. * Memory slot State.
  137. * -----------------------------
  138. * The state of a memory slot mirrors the state of the
  139. * VM the memory slot is associated with.
  140. *
  141. * VM State transition.
  142. * --------------------
  143. *
  144. * A VM always starts in Normal Mode.
  145. *
  146. * H_SVM_INIT_START moves the VM into transient state. During this
  147. * time the Ultravisor may request some of its GFNs to be shared or
  148. * secured. So its GFNs can be in one of the three GFN states.
  149. *
  150. * H_SVM_INIT_DONE moves the VM entirely from transient state to
  151. * secure-state. At this point any left-over normal-GFNs are
  152. * transitioned to Secure-GFN.
  153. *
  154. * H_SVM_INIT_ABORT moves the transient VM back to normal VM.
  155. * All its GFNs are moved to Normal-GFNs.
  156. *
  157. * UV_TERMINATE transitions the secure-VM back to normal-VM. All
  158. * the secure-GFN and shared-GFNs are tranistioned to normal-GFN
  159. * Note: The contents of the normal-GFN is undefined at this point.
  160. *
  161. * GFN state implementation:
  162. * -------------------------
  163. *
  164. * Secure GFN is associated with a secure-PFN; also called uvmem_pfn,
  165. * when the GFN is paged-in. Its pfn[] has KVMPPC_GFN_UVMEM_PFN flag
  166. * set, and contains the value of the secure-PFN.
  167. * It is associated with a normal-PFN; also called mem_pfn, when
  168. * the GFN is pagedout. Its pfn[] has KVMPPC_GFN_MEM_PFN flag set.
  169. * The value of the normal-PFN is not tracked.
  170. *
  171. * Shared GFN is associated with a normal-PFN. Its pfn[] has
  172. * KVMPPC_UVMEM_SHARED_PFN flag set. The value of the normal-PFN
  173. * is not tracked.
  174. *
  175. * Normal GFN is associated with normal-PFN. Its pfn[] has
  176. * no flag set. The value of the normal-PFN is not tracked.
  177. *
  178. * Life cycle of a GFN
  179. * --------------------
  180. *
  181. * --------------------------------------------------------------
  182. * | | Share | Unshare | SVM |H_SVM_INIT_DONE|
  183. * | |operation |operation | abort/ | |
  184. * | | | | terminate | |
  185. * -------------------------------------------------------------
  186. * | | | | | |
  187. * | Secure | Shared | Secure |Normal |Secure |
  188. * | | | | | |
  189. * | Shared | Shared | Secure |Normal |Shared |
  190. * | | | | | |
  191. * | Normal | Shared | Secure |Normal |Secure |
  192. * --------------------------------------------------------------
  193. *
  194. * Life cycle of a VM
  195. * --------------------
  196. *
  197. * --------------------------------------------------------------------
  198. * | | start | H_SVM_ |H_SVM_ |H_SVM_ |UV_SVM_ |
  199. * | | VM |INIT_START|INIT_DONE|INIT_ABORT |TERMINATE |
  200. * | | | | | | |
  201. * --------- ----------------------------------------------------------
  202. * | | | | | | |
  203. * | Normal | Normal | Transient|Error |Error |Normal |
  204. * | | | | | | |
  205. * | Secure | Error | Error |Error |Error |Normal |
  206. * | | | | | | |
  207. * |Transient| N/A | Error |Secure |Normal |Normal |
  208. * --------------------------------------------------------------------
  209. */
  210. #define KVMPPC_GFN_UVMEM_PFN (1UL << 63)
  211. #define KVMPPC_GFN_MEM_PFN (1UL << 62)
  212. #define KVMPPC_GFN_SHARED (1UL << 61)
  213. #define KVMPPC_GFN_SECURE (KVMPPC_GFN_UVMEM_PFN | KVMPPC_GFN_MEM_PFN)
  214. #define KVMPPC_GFN_FLAG_MASK (KVMPPC_GFN_SECURE | KVMPPC_GFN_SHARED)
  215. #define KVMPPC_GFN_PFN_MASK (~KVMPPC_GFN_FLAG_MASK)
  216. struct kvmppc_uvmem_slot {
  217. struct list_head list;
  218. unsigned long nr_pfns;
  219. unsigned long base_pfn;
  220. unsigned long *pfns;
  221. };
  222. struct kvmppc_uvmem_page_pvt {
  223. struct kvm *kvm;
  224. unsigned long gpa;
  225. bool skip_page_out;
  226. bool remove_gfn;
  227. };
  228. bool kvmppc_uvmem_available(void)
  229. {
  230. /*
  231. * If kvmppc_uvmem_bitmap != NULL, then there is an ultravisor
  232. * and our data structures have been initialized successfully.
  233. */
  234. return !!kvmppc_uvmem_bitmap;
  235. }
  236. int kvmppc_uvmem_slot_init(struct kvm *kvm, const struct kvm_memory_slot *slot)
  237. {
  238. struct kvmppc_uvmem_slot *p;
  239. p = kzalloc(sizeof(*p), GFP_KERNEL);
  240. if (!p)
  241. return -ENOMEM;
  242. p->pfns = vzalloc(array_size(slot->npages, sizeof(*p->pfns)));
  243. if (!p->pfns) {
  244. kfree(p);
  245. return -ENOMEM;
  246. }
  247. p->nr_pfns = slot->npages;
  248. p->base_pfn = slot->base_gfn;
  249. mutex_lock(&kvm->arch.uvmem_lock);
  250. list_add(&p->list, &kvm->arch.uvmem_pfns);
  251. mutex_unlock(&kvm->arch.uvmem_lock);
  252. return 0;
  253. }
  254. /*
  255. * All device PFNs are already released by the time we come here.
  256. */
  257. void kvmppc_uvmem_slot_free(struct kvm *kvm, const struct kvm_memory_slot *slot)
  258. {
  259. struct kvmppc_uvmem_slot *p, *next;
  260. mutex_lock(&kvm->arch.uvmem_lock);
  261. list_for_each_entry_safe(p, next, &kvm->arch.uvmem_pfns, list) {
  262. if (p->base_pfn == slot->base_gfn) {
  263. vfree(p->pfns);
  264. list_del(&p->list);
  265. kfree(p);
  266. break;
  267. }
  268. }
  269. mutex_unlock(&kvm->arch.uvmem_lock);
  270. }
  271. static void kvmppc_mark_gfn(unsigned long gfn, struct kvm *kvm,
  272. unsigned long flag, unsigned long uvmem_pfn)
  273. {
  274. struct kvmppc_uvmem_slot *p;
  275. list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
  276. if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
  277. unsigned long index = gfn - p->base_pfn;
  278. if (flag == KVMPPC_GFN_UVMEM_PFN)
  279. p->pfns[index] = uvmem_pfn | flag;
  280. else
  281. p->pfns[index] = flag;
  282. return;
  283. }
  284. }
  285. }
  286. /* mark the GFN as secure-GFN associated with @uvmem pfn device-PFN. */
  287. static void kvmppc_gfn_secure_uvmem_pfn(unsigned long gfn,
  288. unsigned long uvmem_pfn, struct kvm *kvm)
  289. {
  290. kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_UVMEM_PFN, uvmem_pfn);
  291. }
  292. /* mark the GFN as secure-GFN associated with a memory-PFN. */
  293. static void kvmppc_gfn_secure_mem_pfn(unsigned long gfn, struct kvm *kvm)
  294. {
  295. kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_MEM_PFN, 0);
  296. }
  297. /* mark the GFN as a shared GFN. */
  298. static void kvmppc_gfn_shared(unsigned long gfn, struct kvm *kvm)
  299. {
  300. kvmppc_mark_gfn(gfn, kvm, KVMPPC_GFN_SHARED, 0);
  301. }
  302. /* mark the GFN as a non-existent GFN. */
  303. static void kvmppc_gfn_remove(unsigned long gfn, struct kvm *kvm)
  304. {
  305. kvmppc_mark_gfn(gfn, kvm, 0, 0);
  306. }
  307. /* return true, if the GFN is a secure-GFN backed by a secure-PFN */
  308. static bool kvmppc_gfn_is_uvmem_pfn(unsigned long gfn, struct kvm *kvm,
  309. unsigned long *uvmem_pfn)
  310. {
  311. struct kvmppc_uvmem_slot *p;
  312. list_for_each_entry(p, &kvm->arch.uvmem_pfns, list) {
  313. if (gfn >= p->base_pfn && gfn < p->base_pfn + p->nr_pfns) {
  314. unsigned long index = gfn - p->base_pfn;
  315. if (p->pfns[index] & KVMPPC_GFN_UVMEM_PFN) {
  316. if (uvmem_pfn)
  317. *uvmem_pfn = p->pfns[index] &
  318. KVMPPC_GFN_PFN_MASK;
  319. return true;
  320. } else
  321. return false;
  322. }
  323. }
  324. return false;
  325. }
  326. /*
  327. * starting from *gfn search for the next available GFN that is not yet
  328. * transitioned to a secure GFN. return the value of that GFN in *gfn. If a
  329. * GFN is found, return true, else return false
  330. *
  331. * Must be called with kvm->arch.uvmem_lock held.
  332. */
  333. static bool kvmppc_next_nontransitioned_gfn(const struct kvm_memory_slot *memslot,
  334. struct kvm *kvm, unsigned long *gfn)
  335. {
  336. struct kvmppc_uvmem_slot *p;
  337. bool ret = false;
  338. unsigned long i;
  339. list_for_each_entry(p, &kvm->arch.uvmem_pfns, list)
  340. if (*gfn >= p->base_pfn && *gfn < p->base_pfn + p->nr_pfns)
  341. break;
  342. if (!p)
  343. return ret;
  344. /*
  345. * The code below assumes, one to one correspondence between
  346. * kvmppc_uvmem_slot and memslot.
  347. */
  348. for (i = *gfn; i < p->base_pfn + p->nr_pfns; i++) {
  349. unsigned long index = i - p->base_pfn;
  350. if (!(p->pfns[index] & KVMPPC_GFN_FLAG_MASK)) {
  351. *gfn = i;
  352. ret = true;
  353. break;
  354. }
  355. }
  356. return ret;
  357. }
  358. static int kvmppc_memslot_page_merge(struct kvm *kvm,
  359. const struct kvm_memory_slot *memslot, bool merge)
  360. {
  361. unsigned long gfn = memslot->base_gfn;
  362. unsigned long end, start = gfn_to_hva(kvm, gfn);
  363. int ret = 0;
  364. struct vm_area_struct *vma;
  365. int merge_flag = (merge) ? MADV_MERGEABLE : MADV_UNMERGEABLE;
  366. if (kvm_is_error_hva(start))
  367. return H_STATE;
  368. end = start + (memslot->npages << PAGE_SHIFT);
  369. mmap_write_lock(kvm->mm);
  370. do {
  371. vma = find_vma_intersection(kvm->mm, start, end);
  372. if (!vma) {
  373. ret = H_STATE;
  374. break;
  375. }
  376. ret = ksm_madvise(vma, vma->vm_start, vma->vm_end,
  377. merge_flag, &vma->vm_flags);
  378. if (ret) {
  379. ret = H_STATE;
  380. break;
  381. }
  382. start = vma->vm_end;
  383. } while (end > vma->vm_end);
  384. mmap_write_unlock(kvm->mm);
  385. return ret;
  386. }
  387. static void __kvmppc_uvmem_memslot_delete(struct kvm *kvm,
  388. const struct kvm_memory_slot *memslot)
  389. {
  390. uv_unregister_mem_slot(kvm->arch.lpid, memslot->id);
  391. kvmppc_uvmem_slot_free(kvm, memslot);
  392. kvmppc_memslot_page_merge(kvm, memslot, true);
  393. }
  394. static int __kvmppc_uvmem_memslot_create(struct kvm *kvm,
  395. const struct kvm_memory_slot *memslot)
  396. {
  397. int ret = H_PARAMETER;
  398. if (kvmppc_memslot_page_merge(kvm, memslot, false))
  399. return ret;
  400. if (kvmppc_uvmem_slot_init(kvm, memslot))
  401. goto out1;
  402. ret = uv_register_mem_slot(kvm->arch.lpid,
  403. memslot->base_gfn << PAGE_SHIFT,
  404. memslot->npages * PAGE_SIZE,
  405. 0, memslot->id);
  406. if (ret < 0) {
  407. ret = H_PARAMETER;
  408. goto out;
  409. }
  410. return 0;
  411. out:
  412. kvmppc_uvmem_slot_free(kvm, memslot);
  413. out1:
  414. kvmppc_memslot_page_merge(kvm, memslot, true);
  415. return ret;
  416. }
  417. unsigned long kvmppc_h_svm_init_start(struct kvm *kvm)
  418. {
  419. struct kvm_memslots *slots;
  420. struct kvm_memory_slot *memslot, *m;
  421. int ret = H_SUCCESS;
  422. int srcu_idx;
  423. kvm->arch.secure_guest = KVMPPC_SECURE_INIT_START;
  424. if (!kvmppc_uvmem_bitmap)
  425. return H_UNSUPPORTED;
  426. /* Only radix guests can be secure guests */
  427. if (!kvm_is_radix(kvm))
  428. return H_UNSUPPORTED;
  429. /* NAK the transition to secure if not enabled */
  430. if (!kvm->arch.svm_enabled)
  431. return H_AUTHORITY;
  432. srcu_idx = srcu_read_lock(&kvm->srcu);
  433. /* register the memslot */
  434. slots = kvm_memslots(kvm);
  435. kvm_for_each_memslot(memslot, slots) {
  436. ret = __kvmppc_uvmem_memslot_create(kvm, memslot);
  437. if (ret)
  438. break;
  439. }
  440. if (ret) {
  441. slots = kvm_memslots(kvm);
  442. kvm_for_each_memslot(m, slots) {
  443. if (m == memslot)
  444. break;
  445. __kvmppc_uvmem_memslot_delete(kvm, memslot);
  446. }
  447. }
  448. srcu_read_unlock(&kvm->srcu, srcu_idx);
  449. return ret;
  450. }
  451. /*
  452. * Provision a new page on HV side and copy over the contents
  453. * from secure memory using UV_PAGE_OUT uvcall.
  454. * Caller must held kvm->arch.uvmem_lock.
  455. */
  456. static int __kvmppc_svm_page_out(struct vm_area_struct *vma,
  457. unsigned long start,
  458. unsigned long end, unsigned long page_shift,
  459. struct kvm *kvm, unsigned long gpa)
  460. {
  461. unsigned long src_pfn, dst_pfn = 0;
  462. struct migrate_vma mig;
  463. struct page *dpage, *spage;
  464. struct kvmppc_uvmem_page_pvt *pvt;
  465. unsigned long pfn;
  466. int ret = U_SUCCESS;
  467. memset(&mig, 0, sizeof(mig));
  468. mig.vma = vma;
  469. mig.start = start;
  470. mig.end = end;
  471. mig.src = &src_pfn;
  472. mig.dst = &dst_pfn;
  473. mig.pgmap_owner = &kvmppc_uvmem_pgmap;
  474. mig.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
  475. /* The requested page is already paged-out, nothing to do */
  476. if (!kvmppc_gfn_is_uvmem_pfn(gpa >> page_shift, kvm, NULL))
  477. return ret;
  478. ret = migrate_vma_setup(&mig);
  479. if (ret)
  480. return -1;
  481. spage = migrate_pfn_to_page(*mig.src);
  482. if (!spage || !(*mig.src & MIGRATE_PFN_MIGRATE))
  483. goto out_finalize;
  484. if (!is_zone_device_page(spage))
  485. goto out_finalize;
  486. dpage = alloc_page_vma(GFP_HIGHUSER, vma, start);
  487. if (!dpage) {
  488. ret = -1;
  489. goto out_finalize;
  490. }
  491. lock_page(dpage);
  492. pvt = spage->zone_device_data;
  493. pfn = page_to_pfn(dpage);
  494. /*
  495. * This function is used in two cases:
  496. * - When HV touches a secure page, for which we do UV_PAGE_OUT
  497. * - When a secure page is converted to shared page, we *get*
  498. * the page to essentially unmap the device page. In this
  499. * case we skip page-out.
  500. */
  501. if (!pvt->skip_page_out)
  502. ret = uv_page_out(kvm->arch.lpid, pfn << page_shift,
  503. gpa, 0, page_shift);
  504. if (ret == U_SUCCESS)
  505. *mig.dst = migrate_pfn(pfn) | MIGRATE_PFN_LOCKED;
  506. else {
  507. unlock_page(dpage);
  508. __free_page(dpage);
  509. goto out_finalize;
  510. }
  511. migrate_vma_pages(&mig);
  512. out_finalize:
  513. migrate_vma_finalize(&mig);
  514. return ret;
  515. }
  516. static inline int kvmppc_svm_page_out(struct vm_area_struct *vma,
  517. unsigned long start, unsigned long end,
  518. unsigned long page_shift,
  519. struct kvm *kvm, unsigned long gpa)
  520. {
  521. int ret;
  522. mutex_lock(&kvm->arch.uvmem_lock);
  523. ret = __kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa);
  524. mutex_unlock(&kvm->arch.uvmem_lock);
  525. return ret;
  526. }
  527. /*
  528. * Drop device pages that we maintain for the secure guest
  529. *
  530. * We first mark the pages to be skipped from UV_PAGE_OUT when there
  531. * is HV side fault on these pages. Next we *get* these pages, forcing
  532. * fault on them, do fault time migration to replace the device PTEs in
  533. * QEMU page table with normal PTEs from newly allocated pages.
  534. */
  535. void kvmppc_uvmem_drop_pages(const struct kvm_memory_slot *slot,
  536. struct kvm *kvm, bool skip_page_out)
  537. {
  538. int i;
  539. struct kvmppc_uvmem_page_pvt *pvt;
  540. struct page *uvmem_page;
  541. struct vm_area_struct *vma = NULL;
  542. unsigned long uvmem_pfn, gfn;
  543. unsigned long addr;
  544. mmap_read_lock(kvm->mm);
  545. addr = slot->userspace_addr;
  546. gfn = slot->base_gfn;
  547. for (i = slot->npages; i; --i, ++gfn, addr += PAGE_SIZE) {
  548. /* Fetch the VMA if addr is not in the latest fetched one */
  549. if (!vma || addr >= vma->vm_end) {
  550. vma = find_vma_intersection(kvm->mm, addr, addr+1);
  551. if (!vma) {
  552. pr_err("Can't find VMA for gfn:0x%lx\n", gfn);
  553. break;
  554. }
  555. }
  556. mutex_lock(&kvm->arch.uvmem_lock);
  557. if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
  558. uvmem_page = pfn_to_page(uvmem_pfn);
  559. pvt = uvmem_page->zone_device_data;
  560. pvt->skip_page_out = skip_page_out;
  561. pvt->remove_gfn = true;
  562. if (__kvmppc_svm_page_out(vma, addr, addr + PAGE_SIZE,
  563. PAGE_SHIFT, kvm, pvt->gpa))
  564. pr_err("Can't page out gpa:0x%lx addr:0x%lx\n",
  565. pvt->gpa, addr);
  566. } else {
  567. /* Remove the shared flag if any */
  568. kvmppc_gfn_remove(gfn, kvm);
  569. }
  570. mutex_unlock(&kvm->arch.uvmem_lock);
  571. }
  572. mmap_read_unlock(kvm->mm);
  573. }
  574. unsigned long kvmppc_h_svm_init_abort(struct kvm *kvm)
  575. {
  576. int srcu_idx;
  577. struct kvm_memory_slot *memslot;
  578. /*
  579. * Expect to be called only after INIT_START and before INIT_DONE.
  580. * If INIT_DONE was completed, use normal VM termination sequence.
  581. */
  582. if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
  583. return H_UNSUPPORTED;
  584. if (kvm->arch.secure_guest & KVMPPC_SECURE_INIT_DONE)
  585. return H_STATE;
  586. srcu_idx = srcu_read_lock(&kvm->srcu);
  587. kvm_for_each_memslot(memslot, kvm_memslots(kvm))
  588. kvmppc_uvmem_drop_pages(memslot, kvm, false);
  589. srcu_read_unlock(&kvm->srcu, srcu_idx);
  590. kvm->arch.secure_guest = 0;
  591. uv_svm_terminate(kvm->arch.lpid);
  592. return H_PARAMETER;
  593. }
  594. /*
  595. * Get a free device PFN from the pool
  596. *
  597. * Called when a normal page is moved to secure memory (UV_PAGE_IN). Device
  598. * PFN will be used to keep track of the secure page on HV side.
  599. *
  600. * Called with kvm->arch.uvmem_lock held
  601. */
  602. static struct page *kvmppc_uvmem_get_page(unsigned long gpa, struct kvm *kvm)
  603. {
  604. struct page *dpage = NULL;
  605. unsigned long bit, uvmem_pfn;
  606. struct kvmppc_uvmem_page_pvt *pvt;
  607. unsigned long pfn_last, pfn_first;
  608. pfn_first = kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT;
  609. pfn_last = pfn_first +
  610. (range_len(&kvmppc_uvmem_pgmap.range) >> PAGE_SHIFT);
  611. spin_lock(&kvmppc_uvmem_bitmap_lock);
  612. bit = find_first_zero_bit(kvmppc_uvmem_bitmap,
  613. pfn_last - pfn_first);
  614. if (bit >= (pfn_last - pfn_first))
  615. goto out;
  616. bitmap_set(kvmppc_uvmem_bitmap, bit, 1);
  617. spin_unlock(&kvmppc_uvmem_bitmap_lock);
  618. pvt = kzalloc(sizeof(*pvt), GFP_KERNEL);
  619. if (!pvt)
  620. goto out_clear;
  621. uvmem_pfn = bit + pfn_first;
  622. kvmppc_gfn_secure_uvmem_pfn(gpa >> PAGE_SHIFT, uvmem_pfn, kvm);
  623. pvt->gpa = gpa;
  624. pvt->kvm = kvm;
  625. dpage = pfn_to_page(uvmem_pfn);
  626. dpage->zone_device_data = pvt;
  627. get_page(dpage);
  628. lock_page(dpage);
  629. return dpage;
  630. out_clear:
  631. spin_lock(&kvmppc_uvmem_bitmap_lock);
  632. bitmap_clear(kvmppc_uvmem_bitmap, bit, 1);
  633. out:
  634. spin_unlock(&kvmppc_uvmem_bitmap_lock);
  635. return NULL;
  636. }
  637. /*
  638. * Alloc a PFN from private device memory pool. If @pagein is true,
  639. * copy page from normal memory to secure memory using UV_PAGE_IN uvcall.
  640. */
  641. static int kvmppc_svm_page_in(struct vm_area_struct *vma,
  642. unsigned long start,
  643. unsigned long end, unsigned long gpa, struct kvm *kvm,
  644. unsigned long page_shift,
  645. bool pagein)
  646. {
  647. unsigned long src_pfn, dst_pfn = 0;
  648. struct migrate_vma mig;
  649. struct page *spage;
  650. unsigned long pfn;
  651. struct page *dpage;
  652. int ret = 0;
  653. memset(&mig, 0, sizeof(mig));
  654. mig.vma = vma;
  655. mig.start = start;
  656. mig.end = end;
  657. mig.src = &src_pfn;
  658. mig.dst = &dst_pfn;
  659. mig.flags = MIGRATE_VMA_SELECT_SYSTEM;
  660. ret = migrate_vma_setup(&mig);
  661. if (ret)
  662. return ret;
  663. if (!(*mig.src & MIGRATE_PFN_MIGRATE)) {
  664. ret = -1;
  665. goto out_finalize;
  666. }
  667. dpage = kvmppc_uvmem_get_page(gpa, kvm);
  668. if (!dpage) {
  669. ret = -1;
  670. goto out_finalize;
  671. }
  672. if (pagein) {
  673. pfn = *mig.src >> MIGRATE_PFN_SHIFT;
  674. spage = migrate_pfn_to_page(*mig.src);
  675. if (spage) {
  676. ret = uv_page_in(kvm->arch.lpid, pfn << page_shift,
  677. gpa, 0, page_shift);
  678. if (ret)
  679. goto out_finalize;
  680. }
  681. }
  682. *mig.dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
  683. migrate_vma_pages(&mig);
  684. out_finalize:
  685. migrate_vma_finalize(&mig);
  686. return ret;
  687. }
  688. static int kvmppc_uv_migrate_mem_slot(struct kvm *kvm,
  689. const struct kvm_memory_slot *memslot)
  690. {
  691. unsigned long gfn = memslot->base_gfn;
  692. struct vm_area_struct *vma;
  693. unsigned long start, end;
  694. int ret = 0;
  695. mmap_read_lock(kvm->mm);
  696. mutex_lock(&kvm->arch.uvmem_lock);
  697. while (kvmppc_next_nontransitioned_gfn(memslot, kvm, &gfn)) {
  698. ret = H_STATE;
  699. start = gfn_to_hva(kvm, gfn);
  700. if (kvm_is_error_hva(start))
  701. break;
  702. end = start + (1UL << PAGE_SHIFT);
  703. vma = find_vma_intersection(kvm->mm, start, end);
  704. if (!vma || vma->vm_start > start || vma->vm_end < end)
  705. break;
  706. ret = kvmppc_svm_page_in(vma, start, end,
  707. (gfn << PAGE_SHIFT), kvm, PAGE_SHIFT, false);
  708. if (ret) {
  709. ret = H_STATE;
  710. break;
  711. }
  712. /* relinquish the cpu if needed */
  713. cond_resched();
  714. }
  715. mutex_unlock(&kvm->arch.uvmem_lock);
  716. mmap_read_unlock(kvm->mm);
  717. return ret;
  718. }
  719. unsigned long kvmppc_h_svm_init_done(struct kvm *kvm)
  720. {
  721. struct kvm_memslots *slots;
  722. struct kvm_memory_slot *memslot;
  723. int srcu_idx;
  724. long ret = H_SUCCESS;
  725. if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
  726. return H_UNSUPPORTED;
  727. /* migrate any unmoved normal pfn to device pfns*/
  728. srcu_idx = srcu_read_lock(&kvm->srcu);
  729. slots = kvm_memslots(kvm);
  730. kvm_for_each_memslot(memslot, slots) {
  731. ret = kvmppc_uv_migrate_mem_slot(kvm, memslot);
  732. if (ret) {
  733. /*
  734. * The pages will remain transitioned.
  735. * Its the callers responsibility to
  736. * terminate the VM, which will undo
  737. * all state of the VM. Till then
  738. * this VM is in a erroneous state.
  739. * Its KVMPPC_SECURE_INIT_DONE will
  740. * remain unset.
  741. */
  742. ret = H_STATE;
  743. goto out;
  744. }
  745. }
  746. kvm->arch.secure_guest |= KVMPPC_SECURE_INIT_DONE;
  747. pr_info("LPID %d went secure\n", kvm->arch.lpid);
  748. out:
  749. srcu_read_unlock(&kvm->srcu, srcu_idx);
  750. return ret;
  751. }
  752. /*
  753. * Shares the page with HV, thus making it a normal page.
  754. *
  755. * - If the page is already secure, then provision a new page and share
  756. * - If the page is a normal page, share the existing page
  757. *
  758. * In the former case, uses dev_pagemap_ops.migrate_to_ram handler
  759. * to unmap the device page from QEMU's page tables.
  760. */
  761. static unsigned long kvmppc_share_page(struct kvm *kvm, unsigned long gpa,
  762. unsigned long page_shift)
  763. {
  764. int ret = H_PARAMETER;
  765. struct page *uvmem_page;
  766. struct kvmppc_uvmem_page_pvt *pvt;
  767. unsigned long pfn;
  768. unsigned long gfn = gpa >> page_shift;
  769. int srcu_idx;
  770. unsigned long uvmem_pfn;
  771. srcu_idx = srcu_read_lock(&kvm->srcu);
  772. mutex_lock(&kvm->arch.uvmem_lock);
  773. if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
  774. uvmem_page = pfn_to_page(uvmem_pfn);
  775. pvt = uvmem_page->zone_device_data;
  776. pvt->skip_page_out = true;
  777. /*
  778. * do not drop the GFN. It is a valid GFN
  779. * that is transitioned to a shared GFN.
  780. */
  781. pvt->remove_gfn = false;
  782. }
  783. retry:
  784. mutex_unlock(&kvm->arch.uvmem_lock);
  785. pfn = gfn_to_pfn(kvm, gfn);
  786. if (is_error_noslot_pfn(pfn))
  787. goto out;
  788. mutex_lock(&kvm->arch.uvmem_lock);
  789. if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, &uvmem_pfn)) {
  790. uvmem_page = pfn_to_page(uvmem_pfn);
  791. pvt = uvmem_page->zone_device_data;
  792. pvt->skip_page_out = true;
  793. pvt->remove_gfn = false; /* it continues to be a valid GFN */
  794. kvm_release_pfn_clean(pfn);
  795. goto retry;
  796. }
  797. if (!uv_page_in(kvm->arch.lpid, pfn << page_shift, gpa, 0,
  798. page_shift)) {
  799. kvmppc_gfn_shared(gfn, kvm);
  800. ret = H_SUCCESS;
  801. }
  802. kvm_release_pfn_clean(pfn);
  803. mutex_unlock(&kvm->arch.uvmem_lock);
  804. out:
  805. srcu_read_unlock(&kvm->srcu, srcu_idx);
  806. return ret;
  807. }
  808. /*
  809. * H_SVM_PAGE_IN: Move page from normal memory to secure memory.
  810. *
  811. * H_PAGE_IN_SHARED flag makes the page shared which means that the same
  812. * memory in is visible from both UV and HV.
  813. */
  814. unsigned long kvmppc_h_svm_page_in(struct kvm *kvm, unsigned long gpa,
  815. unsigned long flags,
  816. unsigned long page_shift)
  817. {
  818. unsigned long start, end;
  819. struct vm_area_struct *vma;
  820. int srcu_idx;
  821. unsigned long gfn = gpa >> page_shift;
  822. int ret;
  823. if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
  824. return H_UNSUPPORTED;
  825. if (page_shift != PAGE_SHIFT)
  826. return H_P3;
  827. if (flags & ~H_PAGE_IN_SHARED)
  828. return H_P2;
  829. if (flags & H_PAGE_IN_SHARED)
  830. return kvmppc_share_page(kvm, gpa, page_shift);
  831. ret = H_PARAMETER;
  832. srcu_idx = srcu_read_lock(&kvm->srcu);
  833. mmap_read_lock(kvm->mm);
  834. start = gfn_to_hva(kvm, gfn);
  835. if (kvm_is_error_hva(start))
  836. goto out;
  837. mutex_lock(&kvm->arch.uvmem_lock);
  838. /* Fail the page-in request of an already paged-in page */
  839. if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
  840. goto out_unlock;
  841. end = start + (1UL << page_shift);
  842. vma = find_vma_intersection(kvm->mm, start, end);
  843. if (!vma || vma->vm_start > start || vma->vm_end < end)
  844. goto out_unlock;
  845. if (kvmppc_svm_page_in(vma, start, end, gpa, kvm, page_shift,
  846. true))
  847. goto out_unlock;
  848. ret = H_SUCCESS;
  849. out_unlock:
  850. mutex_unlock(&kvm->arch.uvmem_lock);
  851. out:
  852. mmap_read_unlock(kvm->mm);
  853. srcu_read_unlock(&kvm->srcu, srcu_idx);
  854. return ret;
  855. }
  856. /*
  857. * Fault handler callback that gets called when HV touches any page that
  858. * has been moved to secure memory, we ask UV to give back the page by
  859. * issuing UV_PAGE_OUT uvcall.
  860. *
  861. * This eventually results in dropping of device PFN and the newly
  862. * provisioned page/PFN gets populated in QEMU page tables.
  863. */
  864. static vm_fault_t kvmppc_uvmem_migrate_to_ram(struct vm_fault *vmf)
  865. {
  866. struct kvmppc_uvmem_page_pvt *pvt = vmf->page->zone_device_data;
  867. if (kvmppc_svm_page_out(vmf->vma, vmf->address,
  868. vmf->address + PAGE_SIZE, PAGE_SHIFT,
  869. pvt->kvm, pvt->gpa))
  870. return VM_FAULT_SIGBUS;
  871. else
  872. return 0;
  873. }
  874. /*
  875. * Release the device PFN back to the pool
  876. *
  877. * Gets called when secure GFN tranistions from a secure-PFN
  878. * to a normal PFN during H_SVM_PAGE_OUT.
  879. * Gets called with kvm->arch.uvmem_lock held.
  880. */
  881. static void kvmppc_uvmem_page_free(struct page *page)
  882. {
  883. unsigned long pfn = page_to_pfn(page) -
  884. (kvmppc_uvmem_pgmap.range.start >> PAGE_SHIFT);
  885. struct kvmppc_uvmem_page_pvt *pvt;
  886. spin_lock(&kvmppc_uvmem_bitmap_lock);
  887. bitmap_clear(kvmppc_uvmem_bitmap, pfn, 1);
  888. spin_unlock(&kvmppc_uvmem_bitmap_lock);
  889. pvt = page->zone_device_data;
  890. page->zone_device_data = NULL;
  891. if (pvt->remove_gfn)
  892. kvmppc_gfn_remove(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
  893. else
  894. kvmppc_gfn_secure_mem_pfn(pvt->gpa >> PAGE_SHIFT, pvt->kvm);
  895. kfree(pvt);
  896. }
  897. static const struct dev_pagemap_ops kvmppc_uvmem_ops = {
  898. .page_free = kvmppc_uvmem_page_free,
  899. .migrate_to_ram = kvmppc_uvmem_migrate_to_ram,
  900. };
  901. /*
  902. * H_SVM_PAGE_OUT: Move page from secure memory to normal memory.
  903. */
  904. unsigned long
  905. kvmppc_h_svm_page_out(struct kvm *kvm, unsigned long gpa,
  906. unsigned long flags, unsigned long page_shift)
  907. {
  908. unsigned long gfn = gpa >> page_shift;
  909. unsigned long start, end;
  910. struct vm_area_struct *vma;
  911. int srcu_idx;
  912. int ret;
  913. if (!(kvm->arch.secure_guest & KVMPPC_SECURE_INIT_START))
  914. return H_UNSUPPORTED;
  915. if (page_shift != PAGE_SHIFT)
  916. return H_P3;
  917. if (flags)
  918. return H_P2;
  919. ret = H_PARAMETER;
  920. srcu_idx = srcu_read_lock(&kvm->srcu);
  921. mmap_read_lock(kvm->mm);
  922. start = gfn_to_hva(kvm, gfn);
  923. if (kvm_is_error_hva(start))
  924. goto out;
  925. end = start + (1UL << page_shift);
  926. vma = find_vma_intersection(kvm->mm, start, end);
  927. if (!vma || vma->vm_start > start || vma->vm_end < end)
  928. goto out;
  929. if (!kvmppc_svm_page_out(vma, start, end, page_shift, kvm, gpa))
  930. ret = H_SUCCESS;
  931. out:
  932. mmap_read_unlock(kvm->mm);
  933. srcu_read_unlock(&kvm->srcu, srcu_idx);
  934. return ret;
  935. }
  936. int kvmppc_send_page_to_uv(struct kvm *kvm, unsigned long gfn)
  937. {
  938. unsigned long pfn;
  939. int ret = U_SUCCESS;
  940. pfn = gfn_to_pfn(kvm, gfn);
  941. if (is_error_noslot_pfn(pfn))
  942. return -EFAULT;
  943. mutex_lock(&kvm->arch.uvmem_lock);
  944. if (kvmppc_gfn_is_uvmem_pfn(gfn, kvm, NULL))
  945. goto out;
  946. ret = uv_page_in(kvm->arch.lpid, pfn << PAGE_SHIFT, gfn << PAGE_SHIFT,
  947. 0, PAGE_SHIFT);
  948. out:
  949. kvm_release_pfn_clean(pfn);
  950. mutex_unlock(&kvm->arch.uvmem_lock);
  951. return (ret == U_SUCCESS) ? RESUME_GUEST : -EFAULT;
  952. }
  953. int kvmppc_uvmem_memslot_create(struct kvm *kvm, const struct kvm_memory_slot *new)
  954. {
  955. int ret = __kvmppc_uvmem_memslot_create(kvm, new);
  956. if (!ret)
  957. ret = kvmppc_uv_migrate_mem_slot(kvm, new);
  958. return ret;
  959. }
  960. void kvmppc_uvmem_memslot_delete(struct kvm *kvm, const struct kvm_memory_slot *old)
  961. {
  962. __kvmppc_uvmem_memslot_delete(kvm, old);
  963. }
  964. static u64 kvmppc_get_secmem_size(void)
  965. {
  966. struct device_node *np;
  967. int i, len;
  968. const __be32 *prop;
  969. u64 size = 0;
  970. /*
  971. * First try the new ibm,secure-memory nodes which supersede the
  972. * secure-memory-ranges property.
  973. * If we found some, no need to read the deprecated ones.
  974. */
  975. for_each_compatible_node(np, NULL, "ibm,secure-memory") {
  976. prop = of_get_property(np, "reg", &len);
  977. if (!prop)
  978. continue;
  979. size += of_read_number(prop + 2, 2);
  980. }
  981. if (size)
  982. return size;
  983. np = of_find_compatible_node(NULL, NULL, "ibm,uv-firmware");
  984. if (!np)
  985. goto out;
  986. prop = of_get_property(np, "secure-memory-ranges", &len);
  987. if (!prop)
  988. goto out_put;
  989. for (i = 0; i < len / (sizeof(*prop) * 4); i++)
  990. size += of_read_number(prop + (i * 4) + 2, 2);
  991. out_put:
  992. of_node_put(np);
  993. out:
  994. return size;
  995. }
  996. int kvmppc_uvmem_init(void)
  997. {
  998. int ret = 0;
  999. unsigned long size;
  1000. struct resource *res;
  1001. void *addr;
  1002. unsigned long pfn_last, pfn_first;
  1003. size = kvmppc_get_secmem_size();
  1004. if (!size) {
  1005. /*
  1006. * Don't fail the initialization of kvm-hv module if
  1007. * the platform doesn't export ibm,uv-firmware node.
  1008. * Let normal guests run on such PEF-disabled platform.
  1009. */
  1010. pr_info("KVMPPC-UVMEM: No support for secure guests\n");
  1011. goto out;
  1012. }
  1013. res = request_free_mem_region(&iomem_resource, size, "kvmppc_uvmem");
  1014. if (IS_ERR(res)) {
  1015. ret = PTR_ERR(res);
  1016. goto out;
  1017. }
  1018. kvmppc_uvmem_pgmap.type = MEMORY_DEVICE_PRIVATE;
  1019. kvmppc_uvmem_pgmap.range.start = res->start;
  1020. kvmppc_uvmem_pgmap.range.end = res->end;
  1021. kvmppc_uvmem_pgmap.nr_range = 1;
  1022. kvmppc_uvmem_pgmap.ops = &kvmppc_uvmem_ops;
  1023. /* just one global instance: */
  1024. kvmppc_uvmem_pgmap.owner = &kvmppc_uvmem_pgmap;
  1025. addr = memremap_pages(&kvmppc_uvmem_pgmap, NUMA_NO_NODE);
  1026. if (IS_ERR(addr)) {
  1027. ret = PTR_ERR(addr);
  1028. goto out_free_region;
  1029. }
  1030. pfn_first = res->start >> PAGE_SHIFT;
  1031. pfn_last = pfn_first + (resource_size(res) >> PAGE_SHIFT);
  1032. kvmppc_uvmem_bitmap = kcalloc(BITS_TO_LONGS(pfn_last - pfn_first),
  1033. sizeof(unsigned long), GFP_KERNEL);
  1034. if (!kvmppc_uvmem_bitmap) {
  1035. ret = -ENOMEM;
  1036. goto out_unmap;
  1037. }
  1038. pr_info("KVMPPC-UVMEM: Secure Memory size 0x%lx\n", size);
  1039. return ret;
  1040. out_unmap:
  1041. memunmap_pages(&kvmppc_uvmem_pgmap);
  1042. out_free_region:
  1043. release_mem_region(res->start, size);
  1044. out:
  1045. return ret;
  1046. }
  1047. void kvmppc_uvmem_free(void)
  1048. {
  1049. if (!kvmppc_uvmem_bitmap)
  1050. return;
  1051. memunmap_pages(&kvmppc_uvmem_pgmap);
  1052. release_mem_region(kvmppc_uvmem_pgmap.range.start,
  1053. range_len(&kvmppc_uvmem_pgmap.range));
  1054. kfree(kvmppc_uvmem_bitmap);
  1055. }