test_hmm.c 27 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * This is a module to test the HMM (Heterogeneous Memory Management)
  4. * mirror and zone device private memory migration APIs of the kernel.
  5. * Userspace programs can register with the driver to mirror their own address
  6. * space and can use the device to read/write any valid virtual address.
  7. */
  8. #include <linux/init.h>
  9. #include <linux/fs.h>
  10. #include <linux/mm.h>
  11. #include <linux/module.h>
  12. #include <linux/kernel.h>
  13. #include <linux/cdev.h>
  14. #include <linux/device.h>
  15. #include <linux/mutex.h>
  16. #include <linux/rwsem.h>
  17. #include <linux/sched.h>
  18. #include <linux/slab.h>
  19. #include <linux/highmem.h>
  20. #include <linux/delay.h>
  21. #include <linux/pagemap.h>
  22. #include <linux/hmm.h>
  23. #include <linux/vmalloc.h>
  24. #include <linux/swap.h>
  25. #include <linux/swapops.h>
  26. #include <linux/sched/mm.h>
  27. #include <linux/platform_device.h>
  28. #include "test_hmm_uapi.h"
  29. #define DMIRROR_NDEVICES 2
  30. #define DMIRROR_RANGE_FAULT_TIMEOUT 1000
  31. #define DEVMEM_CHUNK_SIZE (256 * 1024 * 1024U)
  32. #define DEVMEM_CHUNKS_RESERVE 16
  33. static const struct dev_pagemap_ops dmirror_devmem_ops;
  34. static const struct mmu_interval_notifier_ops dmirror_min_ops;
  35. static dev_t dmirror_dev;
  36. struct dmirror_device;
  37. struct dmirror_bounce {
  38. void *ptr;
  39. unsigned long size;
  40. unsigned long addr;
  41. unsigned long cpages;
  42. };
  43. #define DPT_XA_TAG_WRITE 3UL
  44. /*
  45. * Data structure to track address ranges and register for mmu interval
  46. * notifier updates.
  47. */
  48. struct dmirror_interval {
  49. struct mmu_interval_notifier notifier;
  50. struct dmirror *dmirror;
  51. };
  52. /*
  53. * Data attached to the open device file.
  54. * Note that it might be shared after a fork().
  55. */
  56. struct dmirror {
  57. struct dmirror_device *mdevice;
  58. struct xarray pt;
  59. struct mmu_interval_notifier notifier;
  60. struct mutex mutex;
  61. };
  62. /*
  63. * ZONE_DEVICE pages for migration and simulating device memory.
  64. */
  65. struct dmirror_chunk {
  66. struct dev_pagemap pagemap;
  67. struct dmirror_device *mdevice;
  68. };
  69. /*
  70. * Per device data.
  71. */
  72. struct dmirror_device {
  73. struct cdev cdevice;
  74. struct hmm_devmem *devmem;
  75. unsigned int devmem_capacity;
  76. unsigned int devmem_count;
  77. struct dmirror_chunk **devmem_chunks;
  78. struct mutex devmem_lock; /* protects the above */
  79. unsigned long calloc;
  80. unsigned long cfree;
  81. struct page *free_pages;
  82. spinlock_t lock; /* protects the above */
  83. };
  84. static struct dmirror_device dmirror_devices[DMIRROR_NDEVICES];
  85. static int dmirror_bounce_init(struct dmirror_bounce *bounce,
  86. unsigned long addr,
  87. unsigned long size)
  88. {
  89. bounce->addr = addr;
  90. bounce->size = size;
  91. bounce->cpages = 0;
  92. bounce->ptr = vmalloc(size);
  93. if (!bounce->ptr)
  94. return -ENOMEM;
  95. return 0;
  96. }
  97. static void dmirror_bounce_fini(struct dmirror_bounce *bounce)
  98. {
  99. vfree(bounce->ptr);
  100. }
  101. static int dmirror_fops_open(struct inode *inode, struct file *filp)
  102. {
  103. struct cdev *cdev = inode->i_cdev;
  104. struct dmirror *dmirror;
  105. int ret;
  106. /* Mirror this process address space */
  107. dmirror = kzalloc(sizeof(*dmirror), GFP_KERNEL);
  108. if (dmirror == NULL)
  109. return -ENOMEM;
  110. dmirror->mdevice = container_of(cdev, struct dmirror_device, cdevice);
  111. mutex_init(&dmirror->mutex);
  112. xa_init(&dmirror->pt);
  113. ret = mmu_interval_notifier_insert(&dmirror->notifier, current->mm,
  114. 0, ULONG_MAX & PAGE_MASK, &dmirror_min_ops);
  115. if (ret) {
  116. kfree(dmirror);
  117. return ret;
  118. }
  119. filp->private_data = dmirror;
  120. return 0;
  121. }
  122. static int dmirror_fops_release(struct inode *inode, struct file *filp)
  123. {
  124. struct dmirror *dmirror = filp->private_data;
  125. mmu_interval_notifier_remove(&dmirror->notifier);
  126. xa_destroy(&dmirror->pt);
  127. kfree(dmirror);
  128. return 0;
  129. }
  130. static struct dmirror_device *dmirror_page_to_device(struct page *page)
  131. {
  132. return container_of(page->pgmap, struct dmirror_chunk,
  133. pagemap)->mdevice;
  134. }
  135. static int dmirror_do_fault(struct dmirror *dmirror, struct hmm_range *range)
  136. {
  137. unsigned long *pfns = range->hmm_pfns;
  138. unsigned long pfn;
  139. for (pfn = (range->start >> PAGE_SHIFT);
  140. pfn < (range->end >> PAGE_SHIFT);
  141. pfn++, pfns++) {
  142. struct page *page;
  143. void *entry;
  144. /*
  145. * Since we asked for hmm_range_fault() to populate pages,
  146. * it shouldn't return an error entry on success.
  147. */
  148. WARN_ON(*pfns & HMM_PFN_ERROR);
  149. WARN_ON(!(*pfns & HMM_PFN_VALID));
  150. page = hmm_pfn_to_page(*pfns);
  151. WARN_ON(!page);
  152. entry = page;
  153. if (*pfns & HMM_PFN_WRITE)
  154. entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
  155. else if (WARN_ON(range->default_flags & HMM_PFN_WRITE))
  156. return -EFAULT;
  157. entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
  158. if (xa_is_err(entry))
  159. return xa_err(entry);
  160. }
  161. return 0;
  162. }
  163. static void dmirror_do_update(struct dmirror *dmirror, unsigned long start,
  164. unsigned long end)
  165. {
  166. unsigned long pfn;
  167. void *entry;
  168. /*
  169. * The XArray doesn't hold references to pages since it relies on
  170. * the mmu notifier to clear page pointers when they become stale.
  171. * Therefore, it is OK to just clear the entry.
  172. */
  173. xa_for_each_range(&dmirror->pt, pfn, entry, start >> PAGE_SHIFT,
  174. end >> PAGE_SHIFT)
  175. xa_erase(&dmirror->pt, pfn);
  176. }
  177. static bool dmirror_interval_invalidate(struct mmu_interval_notifier *mni,
  178. const struct mmu_notifier_range *range,
  179. unsigned long cur_seq)
  180. {
  181. struct dmirror *dmirror = container_of(mni, struct dmirror, notifier);
  182. /*
  183. * Ignore invalidation callbacks for device private pages since
  184. * the invalidation is handled as part of the migration process.
  185. */
  186. if (range->event == MMU_NOTIFY_MIGRATE &&
  187. range->migrate_pgmap_owner == dmirror->mdevice)
  188. return true;
  189. if (mmu_notifier_range_blockable(range))
  190. mutex_lock(&dmirror->mutex);
  191. else if (!mutex_trylock(&dmirror->mutex))
  192. return false;
  193. mmu_interval_set_seq(mni, cur_seq);
  194. dmirror_do_update(dmirror, range->start, range->end);
  195. mutex_unlock(&dmirror->mutex);
  196. return true;
  197. }
  198. static const struct mmu_interval_notifier_ops dmirror_min_ops = {
  199. .invalidate = dmirror_interval_invalidate,
  200. };
  201. static int dmirror_range_fault(struct dmirror *dmirror,
  202. struct hmm_range *range)
  203. {
  204. struct mm_struct *mm = dmirror->notifier.mm;
  205. unsigned long timeout =
  206. jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
  207. int ret;
  208. while (true) {
  209. if (time_after(jiffies, timeout)) {
  210. ret = -EBUSY;
  211. goto out;
  212. }
  213. range->notifier_seq = mmu_interval_read_begin(range->notifier);
  214. mmap_read_lock(mm);
  215. ret = hmm_range_fault(range);
  216. mmap_read_unlock(mm);
  217. if (ret) {
  218. if (ret == -EBUSY)
  219. continue;
  220. goto out;
  221. }
  222. mutex_lock(&dmirror->mutex);
  223. if (mmu_interval_read_retry(range->notifier,
  224. range->notifier_seq)) {
  225. mutex_unlock(&dmirror->mutex);
  226. continue;
  227. }
  228. break;
  229. }
  230. ret = dmirror_do_fault(dmirror, range);
  231. mutex_unlock(&dmirror->mutex);
  232. out:
  233. return ret;
  234. }
  235. static int dmirror_fault(struct dmirror *dmirror, unsigned long start,
  236. unsigned long end, bool write)
  237. {
  238. struct mm_struct *mm = dmirror->notifier.mm;
  239. unsigned long addr;
  240. unsigned long pfns[64];
  241. struct hmm_range range = {
  242. .notifier = &dmirror->notifier,
  243. .hmm_pfns = pfns,
  244. .pfn_flags_mask = 0,
  245. .default_flags =
  246. HMM_PFN_REQ_FAULT | (write ? HMM_PFN_REQ_WRITE : 0),
  247. .dev_private_owner = dmirror->mdevice,
  248. };
  249. int ret = 0;
  250. /* Since the mm is for the mirrored process, get a reference first. */
  251. if (!mmget_not_zero(mm))
  252. return 0;
  253. for (addr = start; addr < end; addr = range.end) {
  254. range.start = addr;
  255. range.end = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
  256. ret = dmirror_range_fault(dmirror, &range);
  257. if (ret)
  258. break;
  259. }
  260. mmput(mm);
  261. return ret;
  262. }
  263. static int dmirror_do_read(struct dmirror *dmirror, unsigned long start,
  264. unsigned long end, struct dmirror_bounce *bounce)
  265. {
  266. unsigned long pfn;
  267. void *ptr;
  268. ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
  269. for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
  270. void *entry;
  271. struct page *page;
  272. void *tmp;
  273. entry = xa_load(&dmirror->pt, pfn);
  274. page = xa_untag_pointer(entry);
  275. if (!page)
  276. return -ENOENT;
  277. tmp = kmap(page);
  278. memcpy(ptr, tmp, PAGE_SIZE);
  279. kunmap(page);
  280. ptr += PAGE_SIZE;
  281. bounce->cpages++;
  282. }
  283. return 0;
  284. }
  285. static int dmirror_read(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
  286. {
  287. struct dmirror_bounce bounce;
  288. unsigned long start, end;
  289. unsigned long size = cmd->npages << PAGE_SHIFT;
  290. int ret;
  291. start = cmd->addr;
  292. end = start + size;
  293. if (end < start)
  294. return -EINVAL;
  295. ret = dmirror_bounce_init(&bounce, start, size);
  296. if (ret)
  297. return ret;
  298. while (1) {
  299. mutex_lock(&dmirror->mutex);
  300. ret = dmirror_do_read(dmirror, start, end, &bounce);
  301. mutex_unlock(&dmirror->mutex);
  302. if (ret != -ENOENT)
  303. break;
  304. start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
  305. ret = dmirror_fault(dmirror, start, end, false);
  306. if (ret)
  307. break;
  308. cmd->faults++;
  309. }
  310. if (ret == 0) {
  311. if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
  312. bounce.size))
  313. ret = -EFAULT;
  314. }
  315. cmd->cpages = bounce.cpages;
  316. dmirror_bounce_fini(&bounce);
  317. return ret;
  318. }
  319. static int dmirror_do_write(struct dmirror *dmirror, unsigned long start,
  320. unsigned long end, struct dmirror_bounce *bounce)
  321. {
  322. unsigned long pfn;
  323. void *ptr;
  324. ptr = bounce->ptr + ((start - bounce->addr) & PAGE_MASK);
  325. for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++) {
  326. void *entry;
  327. struct page *page;
  328. void *tmp;
  329. entry = xa_load(&dmirror->pt, pfn);
  330. page = xa_untag_pointer(entry);
  331. if (!page || xa_pointer_tag(entry) != DPT_XA_TAG_WRITE)
  332. return -ENOENT;
  333. tmp = kmap(page);
  334. memcpy(tmp, ptr, PAGE_SIZE);
  335. kunmap(page);
  336. ptr += PAGE_SIZE;
  337. bounce->cpages++;
  338. }
  339. return 0;
  340. }
  341. static int dmirror_write(struct dmirror *dmirror, struct hmm_dmirror_cmd *cmd)
  342. {
  343. struct dmirror_bounce bounce;
  344. unsigned long start, end;
  345. unsigned long size = cmd->npages << PAGE_SHIFT;
  346. int ret;
  347. start = cmd->addr;
  348. end = start + size;
  349. if (end < start)
  350. return -EINVAL;
  351. ret = dmirror_bounce_init(&bounce, start, size);
  352. if (ret)
  353. return ret;
  354. if (copy_from_user(bounce.ptr, u64_to_user_ptr(cmd->ptr),
  355. bounce.size)) {
  356. ret = -EFAULT;
  357. goto fini;
  358. }
  359. while (1) {
  360. mutex_lock(&dmirror->mutex);
  361. ret = dmirror_do_write(dmirror, start, end, &bounce);
  362. mutex_unlock(&dmirror->mutex);
  363. if (ret != -ENOENT)
  364. break;
  365. start = cmd->addr + (bounce.cpages << PAGE_SHIFT);
  366. ret = dmirror_fault(dmirror, start, end, true);
  367. if (ret)
  368. break;
  369. cmd->faults++;
  370. }
  371. fini:
  372. cmd->cpages = bounce.cpages;
  373. dmirror_bounce_fini(&bounce);
  374. return ret;
  375. }
  376. static bool dmirror_allocate_chunk(struct dmirror_device *mdevice,
  377. struct page **ppage)
  378. {
  379. struct dmirror_chunk *devmem;
  380. struct resource *res;
  381. unsigned long pfn;
  382. unsigned long pfn_first;
  383. unsigned long pfn_last;
  384. void *ptr;
  385. devmem = kzalloc(sizeof(*devmem), GFP_KERNEL);
  386. if (!devmem)
  387. return false;
  388. res = request_free_mem_region(&iomem_resource, DEVMEM_CHUNK_SIZE,
  389. "hmm_dmirror");
  390. if (IS_ERR(res))
  391. goto err_devmem;
  392. devmem->pagemap.type = MEMORY_DEVICE_PRIVATE;
  393. devmem->pagemap.range.start = res->start;
  394. devmem->pagemap.range.end = res->end;
  395. devmem->pagemap.nr_range = 1;
  396. devmem->pagemap.ops = &dmirror_devmem_ops;
  397. devmem->pagemap.owner = mdevice;
  398. mutex_lock(&mdevice->devmem_lock);
  399. if (mdevice->devmem_count == mdevice->devmem_capacity) {
  400. struct dmirror_chunk **new_chunks;
  401. unsigned int new_capacity;
  402. new_capacity = mdevice->devmem_capacity +
  403. DEVMEM_CHUNKS_RESERVE;
  404. new_chunks = krealloc(mdevice->devmem_chunks,
  405. sizeof(new_chunks[0]) * new_capacity,
  406. GFP_KERNEL);
  407. if (!new_chunks)
  408. goto err_release;
  409. mdevice->devmem_capacity = new_capacity;
  410. mdevice->devmem_chunks = new_chunks;
  411. }
  412. ptr = memremap_pages(&devmem->pagemap, numa_node_id());
  413. if (IS_ERR(ptr))
  414. goto err_release;
  415. devmem->mdevice = mdevice;
  416. pfn_first = devmem->pagemap.range.start >> PAGE_SHIFT;
  417. pfn_last = pfn_first + (range_len(&devmem->pagemap.range) >> PAGE_SHIFT);
  418. mdevice->devmem_chunks[mdevice->devmem_count++] = devmem;
  419. mutex_unlock(&mdevice->devmem_lock);
  420. pr_info("added new %u MB chunk (total %u chunks, %u MB) PFNs [0x%lx 0x%lx)\n",
  421. DEVMEM_CHUNK_SIZE / (1024 * 1024),
  422. mdevice->devmem_count,
  423. mdevice->devmem_count * (DEVMEM_CHUNK_SIZE / (1024 * 1024)),
  424. pfn_first, pfn_last);
  425. spin_lock(&mdevice->lock);
  426. for (pfn = pfn_first; pfn < pfn_last; pfn++) {
  427. struct page *page = pfn_to_page(pfn);
  428. page->zone_device_data = mdevice->free_pages;
  429. mdevice->free_pages = page;
  430. }
  431. if (ppage) {
  432. *ppage = mdevice->free_pages;
  433. mdevice->free_pages = (*ppage)->zone_device_data;
  434. mdevice->calloc++;
  435. }
  436. spin_unlock(&mdevice->lock);
  437. return true;
  438. err_release:
  439. mutex_unlock(&mdevice->devmem_lock);
  440. release_mem_region(devmem->pagemap.range.start, range_len(&devmem->pagemap.range));
  441. err_devmem:
  442. kfree(devmem);
  443. return false;
  444. }
  445. static struct page *dmirror_devmem_alloc_page(struct dmirror_device *mdevice)
  446. {
  447. struct page *dpage = NULL;
  448. struct page *rpage;
  449. /*
  450. * This is a fake device so we alloc real system memory to store
  451. * our device memory.
  452. */
  453. rpage = alloc_page(GFP_HIGHUSER);
  454. if (!rpage)
  455. return NULL;
  456. spin_lock(&mdevice->lock);
  457. if (mdevice->free_pages) {
  458. dpage = mdevice->free_pages;
  459. mdevice->free_pages = dpage->zone_device_data;
  460. mdevice->calloc++;
  461. spin_unlock(&mdevice->lock);
  462. } else {
  463. spin_unlock(&mdevice->lock);
  464. if (!dmirror_allocate_chunk(mdevice, &dpage))
  465. goto error;
  466. }
  467. dpage->zone_device_data = rpage;
  468. get_page(dpage);
  469. lock_page(dpage);
  470. return dpage;
  471. error:
  472. __free_page(rpage);
  473. return NULL;
  474. }
  475. static void dmirror_migrate_alloc_and_copy(struct migrate_vma *args,
  476. struct dmirror *dmirror)
  477. {
  478. struct dmirror_device *mdevice = dmirror->mdevice;
  479. const unsigned long *src = args->src;
  480. unsigned long *dst = args->dst;
  481. unsigned long addr;
  482. for (addr = args->start; addr < args->end; addr += PAGE_SIZE,
  483. src++, dst++) {
  484. struct page *spage;
  485. struct page *dpage;
  486. struct page *rpage;
  487. if (!(*src & MIGRATE_PFN_MIGRATE))
  488. continue;
  489. /*
  490. * Note that spage might be NULL which is OK since it is an
  491. * unallocated pte_none() or read-only zero page.
  492. */
  493. spage = migrate_pfn_to_page(*src);
  494. dpage = dmirror_devmem_alloc_page(mdevice);
  495. if (!dpage)
  496. continue;
  497. rpage = dpage->zone_device_data;
  498. if (spage)
  499. copy_highpage(rpage, spage);
  500. else
  501. clear_highpage(rpage);
  502. /*
  503. * Normally, a device would use the page->zone_device_data to
  504. * point to the mirror but here we use it to hold the page for
  505. * the simulated device memory and that page holds the pointer
  506. * to the mirror.
  507. */
  508. rpage->zone_device_data = dmirror;
  509. *dst = migrate_pfn(page_to_pfn(dpage)) |
  510. MIGRATE_PFN_LOCKED;
  511. if ((*src & MIGRATE_PFN_WRITE) ||
  512. (!spage && args->vma->vm_flags & VM_WRITE))
  513. *dst |= MIGRATE_PFN_WRITE;
  514. }
  515. }
  516. static int dmirror_migrate_finalize_and_map(struct migrate_vma *args,
  517. struct dmirror *dmirror)
  518. {
  519. unsigned long start = args->start;
  520. unsigned long end = args->end;
  521. const unsigned long *src = args->src;
  522. const unsigned long *dst = args->dst;
  523. unsigned long pfn;
  524. /* Map the migrated pages into the device's page tables. */
  525. mutex_lock(&dmirror->mutex);
  526. for (pfn = start >> PAGE_SHIFT; pfn < (end >> PAGE_SHIFT); pfn++,
  527. src++, dst++) {
  528. struct page *dpage;
  529. void *entry;
  530. if (!(*src & MIGRATE_PFN_MIGRATE))
  531. continue;
  532. dpage = migrate_pfn_to_page(*dst);
  533. if (!dpage)
  534. continue;
  535. /*
  536. * Store the page that holds the data so the page table
  537. * doesn't have to deal with ZONE_DEVICE private pages.
  538. */
  539. entry = dpage->zone_device_data;
  540. if (*dst & MIGRATE_PFN_WRITE)
  541. entry = xa_tag_pointer(entry, DPT_XA_TAG_WRITE);
  542. entry = xa_store(&dmirror->pt, pfn, entry, GFP_ATOMIC);
  543. if (xa_is_err(entry)) {
  544. mutex_unlock(&dmirror->mutex);
  545. return xa_err(entry);
  546. }
  547. }
  548. mutex_unlock(&dmirror->mutex);
  549. return 0;
  550. }
  551. static int dmirror_migrate(struct dmirror *dmirror,
  552. struct hmm_dmirror_cmd *cmd)
  553. {
  554. unsigned long start, end, addr;
  555. unsigned long size = cmd->npages << PAGE_SHIFT;
  556. struct mm_struct *mm = dmirror->notifier.mm;
  557. struct vm_area_struct *vma;
  558. unsigned long src_pfns[64];
  559. unsigned long dst_pfns[64];
  560. struct dmirror_bounce bounce;
  561. struct migrate_vma args;
  562. unsigned long next;
  563. int ret;
  564. start = cmd->addr;
  565. end = start + size;
  566. if (end < start)
  567. return -EINVAL;
  568. /* Since the mm is for the mirrored process, get a reference first. */
  569. if (!mmget_not_zero(mm))
  570. return -EINVAL;
  571. mmap_read_lock(mm);
  572. for (addr = start; addr < end; addr = next) {
  573. vma = find_vma(mm, addr);
  574. if (!vma || addr < vma->vm_start ||
  575. !(vma->vm_flags & VM_READ)) {
  576. ret = -EINVAL;
  577. goto out;
  578. }
  579. next = min(end, addr + (ARRAY_SIZE(src_pfns) << PAGE_SHIFT));
  580. if (next > vma->vm_end)
  581. next = vma->vm_end;
  582. args.vma = vma;
  583. args.src = src_pfns;
  584. args.dst = dst_pfns;
  585. args.start = addr;
  586. args.end = next;
  587. args.pgmap_owner = dmirror->mdevice;
  588. args.flags = MIGRATE_VMA_SELECT_SYSTEM;
  589. ret = migrate_vma_setup(&args);
  590. if (ret)
  591. goto out;
  592. dmirror_migrate_alloc_and_copy(&args, dmirror);
  593. migrate_vma_pages(&args);
  594. dmirror_migrate_finalize_and_map(&args, dmirror);
  595. migrate_vma_finalize(&args);
  596. }
  597. mmap_read_unlock(mm);
  598. mmput(mm);
  599. /* Return the migrated data for verification. */
  600. ret = dmirror_bounce_init(&bounce, start, size);
  601. if (ret)
  602. return ret;
  603. mutex_lock(&dmirror->mutex);
  604. ret = dmirror_do_read(dmirror, start, end, &bounce);
  605. mutex_unlock(&dmirror->mutex);
  606. if (ret == 0) {
  607. if (copy_to_user(u64_to_user_ptr(cmd->ptr), bounce.ptr,
  608. bounce.size))
  609. ret = -EFAULT;
  610. }
  611. cmd->cpages = bounce.cpages;
  612. dmirror_bounce_fini(&bounce);
  613. return ret;
  614. out:
  615. mmap_read_unlock(mm);
  616. mmput(mm);
  617. return ret;
  618. }
  619. static void dmirror_mkentry(struct dmirror *dmirror, struct hmm_range *range,
  620. unsigned char *perm, unsigned long entry)
  621. {
  622. struct page *page;
  623. if (entry & HMM_PFN_ERROR) {
  624. *perm = HMM_DMIRROR_PROT_ERROR;
  625. return;
  626. }
  627. if (!(entry & HMM_PFN_VALID)) {
  628. *perm = HMM_DMIRROR_PROT_NONE;
  629. return;
  630. }
  631. page = hmm_pfn_to_page(entry);
  632. if (is_device_private_page(page)) {
  633. /* Is the page migrated to this device or some other? */
  634. if (dmirror->mdevice == dmirror_page_to_device(page))
  635. *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_LOCAL;
  636. else
  637. *perm = HMM_DMIRROR_PROT_DEV_PRIVATE_REMOTE;
  638. } else if (is_zero_pfn(page_to_pfn(page)))
  639. *perm = HMM_DMIRROR_PROT_ZERO;
  640. else
  641. *perm = HMM_DMIRROR_PROT_NONE;
  642. if (entry & HMM_PFN_WRITE)
  643. *perm |= HMM_DMIRROR_PROT_WRITE;
  644. else
  645. *perm |= HMM_DMIRROR_PROT_READ;
  646. if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PMD_SHIFT)
  647. *perm |= HMM_DMIRROR_PROT_PMD;
  648. else if (hmm_pfn_to_map_order(entry) + PAGE_SHIFT == PUD_SHIFT)
  649. *perm |= HMM_DMIRROR_PROT_PUD;
  650. }
  651. static bool dmirror_snapshot_invalidate(struct mmu_interval_notifier *mni,
  652. const struct mmu_notifier_range *range,
  653. unsigned long cur_seq)
  654. {
  655. struct dmirror_interval *dmi =
  656. container_of(mni, struct dmirror_interval, notifier);
  657. struct dmirror *dmirror = dmi->dmirror;
  658. if (mmu_notifier_range_blockable(range))
  659. mutex_lock(&dmirror->mutex);
  660. else if (!mutex_trylock(&dmirror->mutex))
  661. return false;
  662. /*
  663. * Snapshots only need to set the sequence number since any
  664. * invalidation in the interval invalidates the whole snapshot.
  665. */
  666. mmu_interval_set_seq(mni, cur_seq);
  667. mutex_unlock(&dmirror->mutex);
  668. return true;
  669. }
  670. static const struct mmu_interval_notifier_ops dmirror_mrn_ops = {
  671. .invalidate = dmirror_snapshot_invalidate,
  672. };
  673. static int dmirror_range_snapshot(struct dmirror *dmirror,
  674. struct hmm_range *range,
  675. unsigned char *perm)
  676. {
  677. struct mm_struct *mm = dmirror->notifier.mm;
  678. struct dmirror_interval notifier;
  679. unsigned long timeout =
  680. jiffies + msecs_to_jiffies(HMM_RANGE_DEFAULT_TIMEOUT);
  681. unsigned long i;
  682. unsigned long n;
  683. int ret = 0;
  684. notifier.dmirror = dmirror;
  685. range->notifier = &notifier.notifier;
  686. ret = mmu_interval_notifier_insert(range->notifier, mm,
  687. range->start, range->end - range->start,
  688. &dmirror_mrn_ops);
  689. if (ret)
  690. return ret;
  691. while (true) {
  692. if (time_after(jiffies, timeout)) {
  693. ret = -EBUSY;
  694. goto out;
  695. }
  696. range->notifier_seq = mmu_interval_read_begin(range->notifier);
  697. mmap_read_lock(mm);
  698. ret = hmm_range_fault(range);
  699. mmap_read_unlock(mm);
  700. if (ret) {
  701. if (ret == -EBUSY)
  702. continue;
  703. goto out;
  704. }
  705. mutex_lock(&dmirror->mutex);
  706. if (mmu_interval_read_retry(range->notifier,
  707. range->notifier_seq)) {
  708. mutex_unlock(&dmirror->mutex);
  709. continue;
  710. }
  711. break;
  712. }
  713. n = (range->end - range->start) >> PAGE_SHIFT;
  714. for (i = 0; i < n; i++)
  715. dmirror_mkentry(dmirror, range, perm + i, range->hmm_pfns[i]);
  716. mutex_unlock(&dmirror->mutex);
  717. out:
  718. mmu_interval_notifier_remove(range->notifier);
  719. return ret;
  720. }
  721. static int dmirror_snapshot(struct dmirror *dmirror,
  722. struct hmm_dmirror_cmd *cmd)
  723. {
  724. struct mm_struct *mm = dmirror->notifier.mm;
  725. unsigned long start, end;
  726. unsigned long size = cmd->npages << PAGE_SHIFT;
  727. unsigned long addr;
  728. unsigned long next;
  729. unsigned long pfns[64];
  730. unsigned char perm[64];
  731. char __user *uptr;
  732. struct hmm_range range = {
  733. .hmm_pfns = pfns,
  734. .dev_private_owner = dmirror->mdevice,
  735. };
  736. int ret = 0;
  737. start = cmd->addr;
  738. end = start + size;
  739. if (end < start)
  740. return -EINVAL;
  741. /* Since the mm is for the mirrored process, get a reference first. */
  742. if (!mmget_not_zero(mm))
  743. return -EINVAL;
  744. /*
  745. * Register a temporary notifier to detect invalidations even if it
  746. * overlaps with other mmu_interval_notifiers.
  747. */
  748. uptr = u64_to_user_ptr(cmd->ptr);
  749. for (addr = start; addr < end; addr = next) {
  750. unsigned long n;
  751. next = min(addr + (ARRAY_SIZE(pfns) << PAGE_SHIFT), end);
  752. range.start = addr;
  753. range.end = next;
  754. ret = dmirror_range_snapshot(dmirror, &range, perm);
  755. if (ret)
  756. break;
  757. n = (range.end - range.start) >> PAGE_SHIFT;
  758. if (copy_to_user(uptr, perm, n)) {
  759. ret = -EFAULT;
  760. break;
  761. }
  762. cmd->cpages += n;
  763. uptr += n;
  764. }
  765. mmput(mm);
  766. return ret;
  767. }
  768. static long dmirror_fops_unlocked_ioctl(struct file *filp,
  769. unsigned int command,
  770. unsigned long arg)
  771. {
  772. void __user *uarg = (void __user *)arg;
  773. struct hmm_dmirror_cmd cmd;
  774. struct dmirror *dmirror;
  775. int ret;
  776. dmirror = filp->private_data;
  777. if (!dmirror)
  778. return -EINVAL;
  779. if (copy_from_user(&cmd, uarg, sizeof(cmd)))
  780. return -EFAULT;
  781. if (cmd.addr & ~PAGE_MASK)
  782. return -EINVAL;
  783. if (cmd.addr >= (cmd.addr + (cmd.npages << PAGE_SHIFT)))
  784. return -EINVAL;
  785. cmd.cpages = 0;
  786. cmd.faults = 0;
  787. switch (command) {
  788. case HMM_DMIRROR_READ:
  789. ret = dmirror_read(dmirror, &cmd);
  790. break;
  791. case HMM_DMIRROR_WRITE:
  792. ret = dmirror_write(dmirror, &cmd);
  793. break;
  794. case HMM_DMIRROR_MIGRATE:
  795. ret = dmirror_migrate(dmirror, &cmd);
  796. break;
  797. case HMM_DMIRROR_SNAPSHOT:
  798. ret = dmirror_snapshot(dmirror, &cmd);
  799. break;
  800. default:
  801. return -EINVAL;
  802. }
  803. if (ret)
  804. return ret;
  805. if (copy_to_user(uarg, &cmd, sizeof(cmd)))
  806. return -EFAULT;
  807. return 0;
  808. }
  809. static int dmirror_fops_mmap(struct file *file, struct vm_area_struct *vma)
  810. {
  811. unsigned long addr;
  812. for (addr = vma->vm_start; addr < vma->vm_end; addr += PAGE_SIZE) {
  813. struct page *page;
  814. int ret;
  815. page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  816. if (!page)
  817. return -ENOMEM;
  818. ret = vm_insert_page(vma, addr, page);
  819. if (ret) {
  820. __free_page(page);
  821. return ret;
  822. }
  823. put_page(page);
  824. }
  825. return 0;
  826. }
  827. static const struct file_operations dmirror_fops = {
  828. .open = dmirror_fops_open,
  829. .release = dmirror_fops_release,
  830. .mmap = dmirror_fops_mmap,
  831. .unlocked_ioctl = dmirror_fops_unlocked_ioctl,
  832. .llseek = default_llseek,
  833. .owner = THIS_MODULE,
  834. };
  835. static void dmirror_devmem_free(struct page *page)
  836. {
  837. struct page *rpage = page->zone_device_data;
  838. struct dmirror_device *mdevice;
  839. if (rpage)
  840. __free_page(rpage);
  841. mdevice = dmirror_page_to_device(page);
  842. spin_lock(&mdevice->lock);
  843. mdevice->cfree++;
  844. page->zone_device_data = mdevice->free_pages;
  845. mdevice->free_pages = page;
  846. spin_unlock(&mdevice->lock);
  847. }
  848. static vm_fault_t dmirror_devmem_fault_alloc_and_copy(struct migrate_vma *args,
  849. struct dmirror *dmirror)
  850. {
  851. const unsigned long *src = args->src;
  852. unsigned long *dst = args->dst;
  853. unsigned long start = args->start;
  854. unsigned long end = args->end;
  855. unsigned long addr;
  856. for (addr = start; addr < end; addr += PAGE_SIZE,
  857. src++, dst++) {
  858. struct page *dpage, *spage;
  859. spage = migrate_pfn_to_page(*src);
  860. if (!spage || !(*src & MIGRATE_PFN_MIGRATE))
  861. continue;
  862. spage = spage->zone_device_data;
  863. dpage = alloc_page_vma(GFP_HIGHUSER_MOVABLE, args->vma, addr);
  864. if (!dpage)
  865. continue;
  866. lock_page(dpage);
  867. xa_erase(&dmirror->pt, addr >> PAGE_SHIFT);
  868. copy_highpage(dpage, spage);
  869. *dst = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
  870. if (*src & MIGRATE_PFN_WRITE)
  871. *dst |= MIGRATE_PFN_WRITE;
  872. }
  873. return 0;
  874. }
  875. static vm_fault_t dmirror_devmem_fault(struct vm_fault *vmf)
  876. {
  877. struct migrate_vma args;
  878. unsigned long src_pfns;
  879. unsigned long dst_pfns;
  880. struct page *rpage;
  881. struct dmirror *dmirror;
  882. vm_fault_t ret;
  883. /*
  884. * Normally, a device would use the page->zone_device_data to point to
  885. * the mirror but here we use it to hold the page for the simulated
  886. * device memory and that page holds the pointer to the mirror.
  887. */
  888. rpage = vmf->page->zone_device_data;
  889. dmirror = rpage->zone_device_data;
  890. /* FIXME demonstrate how we can adjust migrate range */
  891. args.vma = vmf->vma;
  892. args.start = vmf->address;
  893. args.end = args.start + PAGE_SIZE;
  894. args.src = &src_pfns;
  895. args.dst = &dst_pfns;
  896. args.pgmap_owner = dmirror->mdevice;
  897. args.flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE;
  898. if (migrate_vma_setup(&args))
  899. return VM_FAULT_SIGBUS;
  900. ret = dmirror_devmem_fault_alloc_and_copy(&args, dmirror);
  901. if (ret)
  902. return ret;
  903. migrate_vma_pages(&args);
  904. /*
  905. * No device finalize step is needed since
  906. * dmirror_devmem_fault_alloc_and_copy() will have already
  907. * invalidated the device page table.
  908. */
  909. migrate_vma_finalize(&args);
  910. return 0;
  911. }
  912. static const struct dev_pagemap_ops dmirror_devmem_ops = {
  913. .page_free = dmirror_devmem_free,
  914. .migrate_to_ram = dmirror_devmem_fault,
  915. };
  916. static int dmirror_device_init(struct dmirror_device *mdevice, int id)
  917. {
  918. dev_t dev;
  919. int ret;
  920. dev = MKDEV(MAJOR(dmirror_dev), id);
  921. mutex_init(&mdevice->devmem_lock);
  922. spin_lock_init(&mdevice->lock);
  923. cdev_init(&mdevice->cdevice, &dmirror_fops);
  924. mdevice->cdevice.owner = THIS_MODULE;
  925. ret = cdev_add(&mdevice->cdevice, dev, 1);
  926. if (ret)
  927. return ret;
  928. /* Build a list of free ZONE_DEVICE private struct pages */
  929. dmirror_allocate_chunk(mdevice, NULL);
  930. return 0;
  931. }
  932. static void dmirror_device_remove(struct dmirror_device *mdevice)
  933. {
  934. unsigned int i;
  935. if (mdevice->devmem_chunks) {
  936. for (i = 0; i < mdevice->devmem_count; i++) {
  937. struct dmirror_chunk *devmem =
  938. mdevice->devmem_chunks[i];
  939. memunmap_pages(&devmem->pagemap);
  940. release_mem_region(devmem->pagemap.range.start,
  941. range_len(&devmem->pagemap.range));
  942. kfree(devmem);
  943. }
  944. kfree(mdevice->devmem_chunks);
  945. }
  946. cdev_del(&mdevice->cdevice);
  947. }
  948. static int __init hmm_dmirror_init(void)
  949. {
  950. int ret;
  951. int id;
  952. ret = alloc_chrdev_region(&dmirror_dev, 0, DMIRROR_NDEVICES,
  953. "HMM_DMIRROR");
  954. if (ret)
  955. goto err_unreg;
  956. for (id = 0; id < DMIRROR_NDEVICES; id++) {
  957. ret = dmirror_device_init(dmirror_devices + id, id);
  958. if (ret)
  959. goto err_chrdev;
  960. }
  961. pr_info("HMM test module loaded. This is only for testing HMM.\n");
  962. return 0;
  963. err_chrdev:
  964. while (--id >= 0)
  965. dmirror_device_remove(dmirror_devices + id);
  966. unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
  967. err_unreg:
  968. return ret;
  969. }
  970. static void __exit hmm_dmirror_exit(void)
  971. {
  972. int id;
  973. for (id = 0; id < DMIRROR_NDEVICES; id++)
  974. dmirror_device_remove(dmirror_devices + id);
  975. unregister_chrdev_region(dmirror_dev, DMIRROR_NDEVICES);
  976. }
  977. module_init(hmm_dmirror_init);
  978. module_exit(hmm_dmirror_exit);
  979. MODULE_LICENSE("GPL");