device.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481
  1. // SPDX-License-Identifier: GPL-2.0
  2. /* Copyright(c) 2016-2018 Intel Corporation. All rights reserved. */
  3. #include <linux/memremap.h>
  4. #include <linux/pagemap.h>
  5. #include <linux/module.h>
  6. #include <linux/device.h>
  7. #include <linux/pfn_t.h>
  8. #include <linux/cdev.h>
  9. #include <linux/slab.h>
  10. #include <linux/dax.h>
  11. #include <linux/fs.h>
  12. #include <linux/mm.h>
  13. #include <linux/mman.h>
  14. #include "dax-private.h"
  15. #include "bus.h"
  16. static int check_vma(struct dev_dax *dev_dax, struct vm_area_struct *vma,
  17. const char *func)
  18. {
  19. struct device *dev = &dev_dax->dev;
  20. unsigned long mask;
  21. if (!dax_alive(dev_dax->dax_dev))
  22. return -ENXIO;
  23. /* prevent private mappings from being established */
  24. if ((vma->vm_flags & VM_MAYSHARE) != VM_MAYSHARE) {
  25. dev_info_ratelimited(dev,
  26. "%s: %s: fail, attempted private mapping\n",
  27. current->comm, func);
  28. return -EINVAL;
  29. }
  30. mask = dev_dax->align - 1;
  31. if (vma->vm_start & mask || vma->vm_end & mask) {
  32. dev_info_ratelimited(dev,
  33. "%s: %s: fail, unaligned vma (%#lx - %#lx, %#lx)\n",
  34. current->comm, func, vma->vm_start, vma->vm_end,
  35. mask);
  36. return -EINVAL;
  37. }
  38. if (!vma_is_dax(vma)) {
  39. dev_info_ratelimited(dev,
  40. "%s: %s: fail, vma is not DAX capable\n",
  41. current->comm, func);
  42. return -EINVAL;
  43. }
  44. return 0;
  45. }
  46. /* see "strong" declaration in tools/testing/nvdimm/dax-dev.c */
  47. __weak phys_addr_t dax_pgoff_to_phys(struct dev_dax *dev_dax, pgoff_t pgoff,
  48. unsigned long size)
  49. {
  50. int i;
  51. for (i = 0; i < dev_dax->nr_range; i++) {
  52. struct dev_dax_range *dax_range = &dev_dax->ranges[i];
  53. struct range *range = &dax_range->range;
  54. unsigned long long pgoff_end;
  55. phys_addr_t phys;
  56. pgoff_end = dax_range->pgoff + PHYS_PFN(range_len(range)) - 1;
  57. if (pgoff < dax_range->pgoff || pgoff > pgoff_end)
  58. continue;
  59. phys = PFN_PHYS(pgoff - dax_range->pgoff) + range->start;
  60. if (phys + size - 1 <= range->end)
  61. return phys;
  62. break;
  63. }
  64. return -1;
  65. }
  66. static vm_fault_t __dev_dax_pte_fault(struct dev_dax *dev_dax,
  67. struct vm_fault *vmf, pfn_t *pfn)
  68. {
  69. struct device *dev = &dev_dax->dev;
  70. phys_addr_t phys;
  71. unsigned int fault_size = PAGE_SIZE;
  72. if (check_vma(dev_dax, vmf->vma, __func__))
  73. return VM_FAULT_SIGBUS;
  74. if (dev_dax->align > PAGE_SIZE) {
  75. dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
  76. dev_dax->align, fault_size);
  77. return VM_FAULT_SIGBUS;
  78. }
  79. if (fault_size != dev_dax->align)
  80. return VM_FAULT_SIGBUS;
  81. phys = dax_pgoff_to_phys(dev_dax, vmf->pgoff, PAGE_SIZE);
  82. if (phys == -1) {
  83. dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", vmf->pgoff);
  84. return VM_FAULT_SIGBUS;
  85. }
  86. *pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP);
  87. return vmf_insert_mixed(vmf->vma, vmf->address, *pfn);
  88. }
  89. static vm_fault_t __dev_dax_pmd_fault(struct dev_dax *dev_dax,
  90. struct vm_fault *vmf, pfn_t *pfn)
  91. {
  92. unsigned long pmd_addr = vmf->address & PMD_MASK;
  93. struct device *dev = &dev_dax->dev;
  94. phys_addr_t phys;
  95. pgoff_t pgoff;
  96. unsigned int fault_size = PMD_SIZE;
  97. if (check_vma(dev_dax, vmf->vma, __func__))
  98. return VM_FAULT_SIGBUS;
  99. if (dev_dax->align > PMD_SIZE) {
  100. dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
  101. dev_dax->align, fault_size);
  102. return VM_FAULT_SIGBUS;
  103. }
  104. if (fault_size < dev_dax->align)
  105. return VM_FAULT_SIGBUS;
  106. else if (fault_size > dev_dax->align)
  107. return VM_FAULT_FALLBACK;
  108. /* if we are outside of the VMA */
  109. if (pmd_addr < vmf->vma->vm_start ||
  110. (pmd_addr + PMD_SIZE) > vmf->vma->vm_end)
  111. return VM_FAULT_SIGBUS;
  112. pgoff = linear_page_index(vmf->vma, pmd_addr);
  113. phys = dax_pgoff_to_phys(dev_dax, pgoff, PMD_SIZE);
  114. if (phys == -1) {
  115. dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
  116. return VM_FAULT_SIGBUS;
  117. }
  118. *pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP);
  119. return vmf_insert_pfn_pmd(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
  120. }
  121. #ifdef CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD
  122. static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
  123. struct vm_fault *vmf, pfn_t *pfn)
  124. {
  125. unsigned long pud_addr = vmf->address & PUD_MASK;
  126. struct device *dev = &dev_dax->dev;
  127. phys_addr_t phys;
  128. pgoff_t pgoff;
  129. unsigned int fault_size = PUD_SIZE;
  130. if (check_vma(dev_dax, vmf->vma, __func__))
  131. return VM_FAULT_SIGBUS;
  132. if (dev_dax->align > PUD_SIZE) {
  133. dev_dbg(dev, "alignment (%#x) > fault size (%#x)\n",
  134. dev_dax->align, fault_size);
  135. return VM_FAULT_SIGBUS;
  136. }
  137. if (fault_size < dev_dax->align)
  138. return VM_FAULT_SIGBUS;
  139. else if (fault_size > dev_dax->align)
  140. return VM_FAULT_FALLBACK;
  141. /* if we are outside of the VMA */
  142. if (pud_addr < vmf->vma->vm_start ||
  143. (pud_addr + PUD_SIZE) > vmf->vma->vm_end)
  144. return VM_FAULT_SIGBUS;
  145. pgoff = linear_page_index(vmf->vma, pud_addr);
  146. phys = dax_pgoff_to_phys(dev_dax, pgoff, PUD_SIZE);
  147. if (phys == -1) {
  148. dev_dbg(dev, "pgoff_to_phys(%#lx) failed\n", pgoff);
  149. return VM_FAULT_SIGBUS;
  150. }
  151. *pfn = phys_to_pfn_t(phys, PFN_DEV|PFN_MAP);
  152. return vmf_insert_pfn_pud(vmf, *pfn, vmf->flags & FAULT_FLAG_WRITE);
  153. }
  154. #else
  155. static vm_fault_t __dev_dax_pud_fault(struct dev_dax *dev_dax,
  156. struct vm_fault *vmf, pfn_t *pfn)
  157. {
  158. return VM_FAULT_FALLBACK;
  159. }
  160. #endif /* !CONFIG_HAVE_ARCH_TRANSPARENT_HUGEPAGE_PUD */
  161. static vm_fault_t dev_dax_huge_fault(struct vm_fault *vmf,
  162. enum page_entry_size pe_size)
  163. {
  164. struct file *filp = vmf->vma->vm_file;
  165. unsigned long fault_size;
  166. vm_fault_t rc = VM_FAULT_SIGBUS;
  167. int id;
  168. pfn_t pfn;
  169. struct dev_dax *dev_dax = filp->private_data;
  170. dev_dbg(&dev_dax->dev, "%s: %s (%#lx - %#lx) size = %d\n", current->comm,
  171. (vmf->flags & FAULT_FLAG_WRITE) ? "write" : "read",
  172. vmf->vma->vm_start, vmf->vma->vm_end, pe_size);
  173. id = dax_read_lock();
  174. switch (pe_size) {
  175. case PE_SIZE_PTE:
  176. fault_size = PAGE_SIZE;
  177. rc = __dev_dax_pte_fault(dev_dax, vmf, &pfn);
  178. break;
  179. case PE_SIZE_PMD:
  180. fault_size = PMD_SIZE;
  181. rc = __dev_dax_pmd_fault(dev_dax, vmf, &pfn);
  182. break;
  183. case PE_SIZE_PUD:
  184. fault_size = PUD_SIZE;
  185. rc = __dev_dax_pud_fault(dev_dax, vmf, &pfn);
  186. break;
  187. default:
  188. rc = VM_FAULT_SIGBUS;
  189. }
  190. if (rc == VM_FAULT_NOPAGE) {
  191. unsigned long i;
  192. pgoff_t pgoff;
  193. /*
  194. * In the device-dax case the only possibility for a
  195. * VM_FAULT_NOPAGE result is when device-dax capacity is
  196. * mapped. No need to consider the zero page, or racing
  197. * conflicting mappings.
  198. */
  199. pgoff = linear_page_index(vmf->vma, vmf->address
  200. & ~(fault_size - 1));
  201. for (i = 0; i < fault_size / PAGE_SIZE; i++) {
  202. struct page *page;
  203. page = pfn_to_page(pfn_t_to_pfn(pfn) + i);
  204. if (page->mapping)
  205. continue;
  206. page->mapping = filp->f_mapping;
  207. page->index = pgoff + i;
  208. }
  209. }
  210. dax_read_unlock(id);
  211. return rc;
  212. }
  213. static vm_fault_t dev_dax_fault(struct vm_fault *vmf)
  214. {
  215. return dev_dax_huge_fault(vmf, PE_SIZE_PTE);
  216. }
  217. static int dev_dax_split(struct vm_area_struct *vma, unsigned long addr)
  218. {
  219. struct file *filp = vma->vm_file;
  220. struct dev_dax *dev_dax = filp->private_data;
  221. if (!IS_ALIGNED(addr, dev_dax->align))
  222. return -EINVAL;
  223. return 0;
  224. }
  225. static unsigned long dev_dax_pagesize(struct vm_area_struct *vma)
  226. {
  227. struct file *filp = vma->vm_file;
  228. struct dev_dax *dev_dax = filp->private_data;
  229. return dev_dax->align;
  230. }
  231. static const struct vm_operations_struct dax_vm_ops = {
  232. .fault = dev_dax_fault,
  233. .huge_fault = dev_dax_huge_fault,
  234. .split = dev_dax_split,
  235. .pagesize = dev_dax_pagesize,
  236. };
  237. static int dax_mmap(struct file *filp, struct vm_area_struct *vma)
  238. {
  239. struct dev_dax *dev_dax = filp->private_data;
  240. int rc, id;
  241. dev_dbg(&dev_dax->dev, "trace\n");
  242. /*
  243. * We lock to check dax_dev liveness and will re-check at
  244. * fault time.
  245. */
  246. id = dax_read_lock();
  247. rc = check_vma(dev_dax, vma, __func__);
  248. dax_read_unlock(id);
  249. if (rc)
  250. return rc;
  251. vma->vm_ops = &dax_vm_ops;
  252. vma->vm_flags |= VM_HUGEPAGE;
  253. return 0;
  254. }
  255. /* return an unmapped area aligned to the dax region specified alignment */
  256. static unsigned long dax_get_unmapped_area(struct file *filp,
  257. unsigned long addr, unsigned long len, unsigned long pgoff,
  258. unsigned long flags)
  259. {
  260. unsigned long off, off_end, off_align, len_align, addr_align, align;
  261. struct dev_dax *dev_dax = filp ? filp->private_data : NULL;
  262. if (!dev_dax || addr)
  263. goto out;
  264. align = dev_dax->align;
  265. off = pgoff << PAGE_SHIFT;
  266. off_end = off + len;
  267. off_align = round_up(off, align);
  268. if ((off_end <= off_align) || ((off_end - off_align) < align))
  269. goto out;
  270. len_align = len + align;
  271. if ((off + len_align) < off)
  272. goto out;
  273. addr_align = current->mm->get_unmapped_area(filp, addr, len_align,
  274. pgoff, flags);
  275. if (!IS_ERR_VALUE(addr_align)) {
  276. addr_align += (off - addr_align) & (align - 1);
  277. return addr_align;
  278. }
  279. out:
  280. return current->mm->get_unmapped_area(filp, addr, len, pgoff, flags);
  281. }
  282. static const struct address_space_operations dev_dax_aops = {
  283. .set_page_dirty = noop_set_page_dirty,
  284. .invalidatepage = noop_invalidatepage,
  285. };
  286. static int dax_open(struct inode *inode, struct file *filp)
  287. {
  288. struct dax_device *dax_dev = inode_dax(inode);
  289. struct inode *__dax_inode = dax_inode(dax_dev);
  290. struct dev_dax *dev_dax = dax_get_private(dax_dev);
  291. dev_dbg(&dev_dax->dev, "trace\n");
  292. inode->i_mapping = __dax_inode->i_mapping;
  293. inode->i_mapping->host = __dax_inode;
  294. inode->i_mapping->a_ops = &dev_dax_aops;
  295. filp->f_mapping = inode->i_mapping;
  296. filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
  297. filp->f_sb_err = file_sample_sb_err(filp);
  298. filp->private_data = dev_dax;
  299. inode->i_flags = S_DAX;
  300. return 0;
  301. }
  302. static int dax_release(struct inode *inode, struct file *filp)
  303. {
  304. struct dev_dax *dev_dax = filp->private_data;
  305. dev_dbg(&dev_dax->dev, "trace\n");
  306. return 0;
  307. }
  308. static const struct file_operations dax_fops = {
  309. .llseek = noop_llseek,
  310. .owner = THIS_MODULE,
  311. .open = dax_open,
  312. .release = dax_release,
  313. .get_unmapped_area = dax_get_unmapped_area,
  314. .mmap = dax_mmap,
  315. .mmap_supported_flags = MAP_SYNC,
  316. };
  317. static void dev_dax_cdev_del(void *cdev)
  318. {
  319. cdev_del(cdev);
  320. }
  321. static void dev_dax_kill(void *dev_dax)
  322. {
  323. kill_dev_dax(dev_dax);
  324. }
  325. int dev_dax_probe(struct dev_dax *dev_dax)
  326. {
  327. struct dax_device *dax_dev = dev_dax->dax_dev;
  328. struct device *dev = &dev_dax->dev;
  329. struct dev_pagemap *pgmap;
  330. struct inode *inode;
  331. struct cdev *cdev;
  332. void *addr;
  333. int rc, i;
  334. pgmap = dev_dax->pgmap;
  335. if (dev_WARN_ONCE(dev, pgmap && dev_dax->nr_range > 1,
  336. "static pgmap / multi-range device conflict\n"))
  337. return -EINVAL;
  338. if (!pgmap) {
  339. pgmap = devm_kzalloc(dev, sizeof(*pgmap) + sizeof(struct range)
  340. * (dev_dax->nr_range - 1), GFP_KERNEL);
  341. if (!pgmap)
  342. return -ENOMEM;
  343. pgmap->nr_range = dev_dax->nr_range;
  344. }
  345. for (i = 0; i < dev_dax->nr_range; i++) {
  346. struct range *range = &dev_dax->ranges[i].range;
  347. if (!devm_request_mem_region(dev, range->start,
  348. range_len(range), dev_name(dev))) {
  349. dev_warn(dev, "mapping%d: %#llx-%#llx could not reserve range\n",
  350. i, range->start, range->end);
  351. return -EBUSY;
  352. }
  353. /* don't update the range for static pgmap */
  354. if (!dev_dax->pgmap)
  355. pgmap->ranges[i] = *range;
  356. }
  357. pgmap->type = MEMORY_DEVICE_GENERIC;
  358. addr = devm_memremap_pages(dev, pgmap);
  359. if (IS_ERR(addr))
  360. return PTR_ERR(addr);
  361. inode = dax_inode(dax_dev);
  362. cdev = inode->i_cdev;
  363. cdev_init(cdev, &dax_fops);
  364. if (dev->class) {
  365. /* for the CONFIG_DEV_DAX_PMEM_COMPAT case */
  366. cdev->owner = dev->parent->driver->owner;
  367. } else
  368. cdev->owner = dev->driver->owner;
  369. cdev_set_parent(cdev, &dev->kobj);
  370. rc = cdev_add(cdev, dev->devt, 1);
  371. if (rc)
  372. return rc;
  373. rc = devm_add_action_or_reset(dev, dev_dax_cdev_del, cdev);
  374. if (rc)
  375. return rc;
  376. run_dax(dax_dev);
  377. return devm_add_action_or_reset(dev, dev_dax_kill, dev_dax);
  378. }
  379. EXPORT_SYMBOL_GPL(dev_dax_probe);
  380. static int dev_dax_remove(struct dev_dax *dev_dax)
  381. {
  382. /* all probe actions are unwound by devm */
  383. return 0;
  384. }
  385. static struct dax_device_driver device_dax_driver = {
  386. .probe = dev_dax_probe,
  387. .remove = dev_dax_remove,
  388. .match_always = 1,
  389. };
  390. static int __init dax_init(void)
  391. {
  392. return dax_driver_register(&device_dax_driver);
  393. }
  394. static void __exit dax_exit(void)
  395. {
  396. dax_driver_unregister(&device_dax_driver);
  397. }
  398. MODULE_AUTHOR("Intel Corporation");
  399. MODULE_LICENSE("GPL v2");
  400. module_init(dax_init);
  401. module_exit(dax_exit);
  402. MODULE_ALIAS_DAX_DEVICE(0);