dfl-afu-dma-region.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Driver for FPGA Accelerated Function Unit (AFU) DMA Region Management
  4. *
  5. * Copyright (C) 2017-2018 Intel Corporation, Inc.
  6. *
  7. * Authors:
  8. * Wu Hao <hao.wu@intel.com>
  9. * Xiao Guangrong <guangrong.xiao@linux.intel.com>
  10. */
  11. #include <linux/dma-mapping.h>
  12. #include <linux/sched/signal.h>
  13. #include <linux/uaccess.h>
  14. #include <linux/mm.h>
  15. #include "dfl-afu.h"
  16. void afu_dma_region_init(struct dfl_feature_platform_data *pdata)
  17. {
  18. struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
  19. afu->dma_regions = RB_ROOT;
  20. }
  21. /**
  22. * afu_dma_pin_pages - pin pages of given dma memory region
  23. * @pdata: feature device platform data
  24. * @region: dma memory region to be pinned
  25. *
  26. * Pin all the pages of given dfl_afu_dma_region.
  27. * Return 0 for success or negative error code.
  28. */
  29. static int afu_dma_pin_pages(struct dfl_feature_platform_data *pdata,
  30. struct dfl_afu_dma_region *region)
  31. {
  32. int npages = region->length >> PAGE_SHIFT;
  33. struct device *dev = &pdata->dev->dev;
  34. int ret, pinned;
  35. ret = account_locked_vm(current->mm, npages, true);
  36. if (ret)
  37. return ret;
  38. region->pages = kcalloc(npages, sizeof(struct page *), GFP_KERNEL);
  39. if (!region->pages) {
  40. ret = -ENOMEM;
  41. goto unlock_vm;
  42. }
  43. pinned = pin_user_pages_fast(region->user_addr, npages, FOLL_WRITE,
  44. region->pages);
  45. if (pinned < 0) {
  46. ret = pinned;
  47. goto free_pages;
  48. } else if (pinned != npages) {
  49. ret = -EFAULT;
  50. goto unpin_pages;
  51. }
  52. dev_dbg(dev, "%d pages pinned\n", pinned);
  53. return 0;
  54. unpin_pages:
  55. unpin_user_pages(region->pages, pinned);
  56. free_pages:
  57. kfree(region->pages);
  58. unlock_vm:
  59. account_locked_vm(current->mm, npages, false);
  60. return ret;
  61. }
  62. /**
  63. * afu_dma_unpin_pages - unpin pages of given dma memory region
  64. * @pdata: feature device platform data
  65. * @region: dma memory region to be unpinned
  66. *
  67. * Unpin all the pages of given dfl_afu_dma_region.
  68. * Return 0 for success or negative error code.
  69. */
  70. static void afu_dma_unpin_pages(struct dfl_feature_platform_data *pdata,
  71. struct dfl_afu_dma_region *region)
  72. {
  73. long npages = region->length >> PAGE_SHIFT;
  74. struct device *dev = &pdata->dev->dev;
  75. unpin_user_pages(region->pages, npages);
  76. kfree(region->pages);
  77. account_locked_vm(current->mm, npages, false);
  78. dev_dbg(dev, "%ld pages unpinned\n", npages);
  79. }
  80. /**
  81. * afu_dma_check_continuous_pages - check if pages are continuous
  82. * @region: dma memory region
  83. *
  84. * Return true if pages of given dma memory region have continuous physical
  85. * address, otherwise return false.
  86. */
  87. static bool afu_dma_check_continuous_pages(struct dfl_afu_dma_region *region)
  88. {
  89. int npages = region->length >> PAGE_SHIFT;
  90. int i;
  91. for (i = 0; i < npages - 1; i++)
  92. if (page_to_pfn(region->pages[i]) + 1 !=
  93. page_to_pfn(region->pages[i + 1]))
  94. return false;
  95. return true;
  96. }
  97. /**
  98. * dma_region_check_iova - check if memory area is fully contained in the region
  99. * @region: dma memory region
  100. * @iova: address of the dma memory area
  101. * @size: size of the dma memory area
  102. *
  103. * Compare the dma memory area defined by @iova and @size with given dma region.
  104. * Return true if memory area is fully contained in the region, otherwise false.
  105. */
  106. static bool dma_region_check_iova(struct dfl_afu_dma_region *region,
  107. u64 iova, u64 size)
  108. {
  109. if (!size && region->iova != iova)
  110. return false;
  111. return (region->iova <= iova) &&
  112. (region->length + region->iova >= iova + size);
  113. }
  114. /**
  115. * afu_dma_region_add - add given dma region to rbtree
  116. * @pdata: feature device platform data
  117. * @region: dma region to be added
  118. *
  119. * Return 0 for success, -EEXIST if dma region has already been added.
  120. *
  121. * Needs to be called with pdata->lock heold.
  122. */
  123. static int afu_dma_region_add(struct dfl_feature_platform_data *pdata,
  124. struct dfl_afu_dma_region *region)
  125. {
  126. struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
  127. struct rb_node **new, *parent = NULL;
  128. dev_dbg(&pdata->dev->dev, "add region (iova = %llx)\n",
  129. (unsigned long long)region->iova);
  130. new = &afu->dma_regions.rb_node;
  131. while (*new) {
  132. struct dfl_afu_dma_region *this;
  133. this = container_of(*new, struct dfl_afu_dma_region, node);
  134. parent = *new;
  135. if (dma_region_check_iova(this, region->iova, region->length))
  136. return -EEXIST;
  137. if (region->iova < this->iova)
  138. new = &((*new)->rb_left);
  139. else if (region->iova > this->iova)
  140. new = &((*new)->rb_right);
  141. else
  142. return -EEXIST;
  143. }
  144. rb_link_node(&region->node, parent, new);
  145. rb_insert_color(&region->node, &afu->dma_regions);
  146. return 0;
  147. }
  148. /**
  149. * afu_dma_region_remove - remove given dma region from rbtree
  150. * @pdata: feature device platform data
  151. * @region: dma region to be removed
  152. *
  153. * Needs to be called with pdata->lock heold.
  154. */
  155. static void afu_dma_region_remove(struct dfl_feature_platform_data *pdata,
  156. struct dfl_afu_dma_region *region)
  157. {
  158. struct dfl_afu *afu;
  159. dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
  160. (unsigned long long)region->iova);
  161. afu = dfl_fpga_pdata_get_private(pdata);
  162. rb_erase(&region->node, &afu->dma_regions);
  163. }
  164. /**
  165. * afu_dma_region_destroy - destroy all regions in rbtree
  166. * @pdata: feature device platform data
  167. *
  168. * Needs to be called with pdata->lock heold.
  169. */
  170. void afu_dma_region_destroy(struct dfl_feature_platform_data *pdata)
  171. {
  172. struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
  173. struct rb_node *node = rb_first(&afu->dma_regions);
  174. struct dfl_afu_dma_region *region;
  175. while (node) {
  176. region = container_of(node, struct dfl_afu_dma_region, node);
  177. dev_dbg(&pdata->dev->dev, "del region (iova = %llx)\n",
  178. (unsigned long long)region->iova);
  179. rb_erase(node, &afu->dma_regions);
  180. if (region->iova)
  181. dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
  182. region->iova, region->length,
  183. DMA_BIDIRECTIONAL);
  184. if (region->pages)
  185. afu_dma_unpin_pages(pdata, region);
  186. node = rb_next(node);
  187. kfree(region);
  188. }
  189. }
  190. /**
  191. * afu_dma_region_find - find the dma region from rbtree based on iova and size
  192. * @pdata: feature device platform data
  193. * @iova: address of the dma memory area
  194. * @size: size of the dma memory area
  195. *
  196. * It finds the dma region from the rbtree based on @iova and @size:
  197. * - if @size == 0, it finds the dma region which starts from @iova
  198. * - otherwise, it finds the dma region which fully contains
  199. * [@iova, @iova+size)
  200. * If nothing is matched returns NULL.
  201. *
  202. * Needs to be called with pdata->lock held.
  203. */
  204. struct dfl_afu_dma_region *
  205. afu_dma_region_find(struct dfl_feature_platform_data *pdata, u64 iova, u64 size)
  206. {
  207. struct dfl_afu *afu = dfl_fpga_pdata_get_private(pdata);
  208. struct rb_node *node = afu->dma_regions.rb_node;
  209. struct device *dev = &pdata->dev->dev;
  210. while (node) {
  211. struct dfl_afu_dma_region *region;
  212. region = container_of(node, struct dfl_afu_dma_region, node);
  213. if (dma_region_check_iova(region, iova, size)) {
  214. dev_dbg(dev, "find region (iova = %llx)\n",
  215. (unsigned long long)region->iova);
  216. return region;
  217. }
  218. if (iova < region->iova)
  219. node = node->rb_left;
  220. else if (iova > region->iova)
  221. node = node->rb_right;
  222. else
  223. /* the iova region is not fully covered. */
  224. break;
  225. }
  226. dev_dbg(dev, "region with iova %llx and size %llx is not found\n",
  227. (unsigned long long)iova, (unsigned long long)size);
  228. return NULL;
  229. }
  230. /**
  231. * afu_dma_region_find_iova - find the dma region from rbtree by iova
  232. * @pdata: feature device platform data
  233. * @iova: address of the dma region
  234. *
  235. * Needs to be called with pdata->lock held.
  236. */
  237. static struct dfl_afu_dma_region *
  238. afu_dma_region_find_iova(struct dfl_feature_platform_data *pdata, u64 iova)
  239. {
  240. return afu_dma_region_find(pdata, iova, 0);
  241. }
  242. /**
  243. * afu_dma_map_region - map memory region for dma
  244. * @pdata: feature device platform data
  245. * @user_addr: address of the memory region
  246. * @length: size of the memory region
  247. * @iova: pointer of iova address
  248. *
  249. * Map memory region defined by @user_addr and @length, and return dma address
  250. * of the memory region via @iova.
  251. * Return 0 for success, otherwise error code.
  252. */
  253. int afu_dma_map_region(struct dfl_feature_platform_data *pdata,
  254. u64 user_addr, u64 length, u64 *iova)
  255. {
  256. struct dfl_afu_dma_region *region;
  257. int ret;
  258. /*
  259. * Check Inputs, only accept page-aligned user memory region with
  260. * valid length.
  261. */
  262. if (!PAGE_ALIGNED(user_addr) || !PAGE_ALIGNED(length) || !length)
  263. return -EINVAL;
  264. /* Check overflow */
  265. if (user_addr + length < user_addr)
  266. return -EINVAL;
  267. region = kzalloc(sizeof(*region), GFP_KERNEL);
  268. if (!region)
  269. return -ENOMEM;
  270. region->user_addr = user_addr;
  271. region->length = length;
  272. /* Pin the user memory region */
  273. ret = afu_dma_pin_pages(pdata, region);
  274. if (ret) {
  275. dev_err(&pdata->dev->dev, "failed to pin memory region\n");
  276. goto free_region;
  277. }
  278. /* Only accept continuous pages, return error else */
  279. if (!afu_dma_check_continuous_pages(region)) {
  280. dev_err(&pdata->dev->dev, "pages are not continuous\n");
  281. ret = -EINVAL;
  282. goto unpin_pages;
  283. }
  284. /* As pages are continuous then start to do DMA mapping */
  285. region->iova = dma_map_page(dfl_fpga_pdata_to_parent(pdata),
  286. region->pages[0], 0,
  287. region->length,
  288. DMA_BIDIRECTIONAL);
  289. if (dma_mapping_error(dfl_fpga_pdata_to_parent(pdata), region->iova)) {
  290. dev_err(&pdata->dev->dev, "failed to map for dma\n");
  291. ret = -EFAULT;
  292. goto unpin_pages;
  293. }
  294. *iova = region->iova;
  295. mutex_lock(&pdata->lock);
  296. ret = afu_dma_region_add(pdata, region);
  297. mutex_unlock(&pdata->lock);
  298. if (ret) {
  299. dev_err(&pdata->dev->dev, "failed to add dma region\n");
  300. goto unmap_dma;
  301. }
  302. return 0;
  303. unmap_dma:
  304. dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
  305. region->iova, region->length, DMA_BIDIRECTIONAL);
  306. unpin_pages:
  307. afu_dma_unpin_pages(pdata, region);
  308. free_region:
  309. kfree(region);
  310. return ret;
  311. }
  312. /**
  313. * afu_dma_unmap_region - unmap dma memory region
  314. * @pdata: feature device platform data
  315. * @iova: dma address of the region
  316. *
  317. * Unmap dma memory region based on @iova.
  318. * Return 0 for success, otherwise error code.
  319. */
  320. int afu_dma_unmap_region(struct dfl_feature_platform_data *pdata, u64 iova)
  321. {
  322. struct dfl_afu_dma_region *region;
  323. mutex_lock(&pdata->lock);
  324. region = afu_dma_region_find_iova(pdata, iova);
  325. if (!region) {
  326. mutex_unlock(&pdata->lock);
  327. return -EINVAL;
  328. }
  329. if (region->in_use) {
  330. mutex_unlock(&pdata->lock);
  331. return -EBUSY;
  332. }
  333. afu_dma_region_remove(pdata, region);
  334. mutex_unlock(&pdata->lock);
  335. dma_unmap_page(dfl_fpga_pdata_to_parent(pdata),
  336. region->iova, region->length, DMA_BIDIRECTIONAL);
  337. afu_dma_unpin_pages(pdata, region);
  338. kfree(region);
  339. return 0;
  340. }