nouveau_dmem.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704
  1. /*
  2. * Copyright 2018 Red Hat Inc.
  3. *
  4. * Permission is hereby granted, free of charge, to any person obtaining a
  5. * copy of this software and associated documentation files (the "Software"),
  6. * to deal in the Software without restriction, including without limitation
  7. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  8. * and/or sell copies of the Software, and to permit persons to whom the
  9. * Software is furnished to do so, subject to the following conditions:
  10. *
  11. * The above copyright notice and this permission notice shall be included in
  12. * all copies or substantial portions of the Software.
  13. *
  14. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  15. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  16. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  17. * THE COPYRIGHT HOLDER(S) OR AUTHOR(S) BE LIABLE FOR ANY CLAIM, DAMAGES OR
  18. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  19. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  20. * OTHER DEALINGS IN THE SOFTWARE.
  21. */
  22. #include "nouveau_dmem.h"
  23. #include "nouveau_drv.h"
  24. #include "nouveau_chan.h"
  25. #include "nouveau_dma.h"
  26. #include "nouveau_mem.h"
  27. #include "nouveau_bo.h"
  28. #include "nouveau_svm.h"
  29. #include <nvif/class.h>
  30. #include <nvif/object.h>
  31. #include <nvif/push906f.h>
  32. #include <nvif/if000c.h>
  33. #include <nvif/if500b.h>
  34. #include <nvif/if900b.h>
  35. #include <nvif/if000c.h>
  36. #include <nvhw/class/cla0b5.h>
  37. #include <linux/sched/mm.h>
  38. #include <linux/hmm.h>
  39. /*
  40. * FIXME: this is ugly right now we are using TTM to allocate vram and we pin
  41. * it in vram while in use. We likely want to overhaul memory management for
  42. * nouveau to be more page like (not necessarily with system page size but a
  43. * bigger page size) at lowest level and have some shim layer on top that would
  44. * provide the same functionality as TTM.
  45. */
  46. #define DMEM_CHUNK_SIZE (2UL << 20)
  47. #define DMEM_CHUNK_NPAGES (DMEM_CHUNK_SIZE >> PAGE_SHIFT)
  48. enum nouveau_aper {
  49. NOUVEAU_APER_VIRT,
  50. NOUVEAU_APER_VRAM,
  51. NOUVEAU_APER_HOST,
  52. };
  53. typedef int (*nouveau_migrate_copy_t)(struct nouveau_drm *drm, u64 npages,
  54. enum nouveau_aper, u64 dst_addr,
  55. enum nouveau_aper, u64 src_addr);
  56. typedef int (*nouveau_clear_page_t)(struct nouveau_drm *drm, u32 length,
  57. enum nouveau_aper, u64 dst_addr);
  58. struct nouveau_dmem_chunk {
  59. struct list_head list;
  60. struct nouveau_bo *bo;
  61. struct nouveau_drm *drm;
  62. unsigned long callocated;
  63. struct dev_pagemap pagemap;
  64. };
  65. struct nouveau_dmem_migrate {
  66. nouveau_migrate_copy_t copy_func;
  67. nouveau_clear_page_t clear_func;
  68. struct nouveau_channel *chan;
  69. };
  70. struct nouveau_dmem {
  71. struct nouveau_drm *drm;
  72. struct nouveau_dmem_migrate migrate;
  73. struct list_head chunks;
  74. struct mutex mutex;
  75. struct page *free_pages;
  76. spinlock_t lock;
  77. };
  78. static struct nouveau_dmem_chunk *nouveau_page_to_chunk(struct page *page)
  79. {
  80. return container_of(page->pgmap, struct nouveau_dmem_chunk, pagemap);
  81. }
  82. static struct nouveau_drm *page_to_drm(struct page *page)
  83. {
  84. struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
  85. return chunk->drm;
  86. }
  87. unsigned long nouveau_dmem_page_addr(struct page *page)
  88. {
  89. struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
  90. unsigned long off = (page_to_pfn(page) << PAGE_SHIFT) -
  91. chunk->pagemap.range.start;
  92. return chunk->bo->offset + off;
  93. }
  94. static void nouveau_dmem_page_free(struct page *page)
  95. {
  96. struct nouveau_dmem_chunk *chunk = nouveau_page_to_chunk(page);
  97. struct nouveau_dmem *dmem = chunk->drm->dmem;
  98. spin_lock(&dmem->lock);
  99. page->zone_device_data = dmem->free_pages;
  100. dmem->free_pages = page;
  101. WARN_ON(!chunk->callocated);
  102. chunk->callocated--;
  103. /*
  104. * FIXME when chunk->callocated reach 0 we should add the chunk to
  105. * a reclaim list so that it can be freed in case of memory pressure.
  106. */
  107. spin_unlock(&dmem->lock);
  108. }
  109. static void nouveau_dmem_fence_done(struct nouveau_fence **fence)
  110. {
  111. if (fence) {
  112. nouveau_fence_wait(*fence, true, false);
  113. nouveau_fence_unref(fence);
  114. } else {
  115. /*
  116. * FIXME wait for channel to be IDLE before calling finalizing
  117. * the hmem object.
  118. */
  119. }
  120. }
  121. static vm_fault_t nouveau_dmem_fault_copy_one(struct nouveau_drm *drm,
  122. struct vm_fault *vmf, struct migrate_vma *args,
  123. dma_addr_t *dma_addr)
  124. {
  125. struct device *dev = drm->dev->dev;
  126. struct page *dpage, *spage;
  127. struct nouveau_svmm *svmm;
  128. spage = migrate_pfn_to_page(args->src[0]);
  129. if (!spage || !(args->src[0] & MIGRATE_PFN_MIGRATE))
  130. return 0;
  131. dpage = alloc_page_vma(GFP_HIGHUSER, vmf->vma, vmf->address);
  132. if (!dpage)
  133. return VM_FAULT_SIGBUS;
  134. lock_page(dpage);
  135. *dma_addr = dma_map_page(dev, dpage, 0, PAGE_SIZE, DMA_BIDIRECTIONAL);
  136. if (dma_mapping_error(dev, *dma_addr))
  137. goto error_free_page;
  138. svmm = spage->zone_device_data;
  139. mutex_lock(&svmm->mutex);
  140. nouveau_svmm_invalidate(svmm, args->start, args->end);
  141. if (drm->dmem->migrate.copy_func(drm, 1, NOUVEAU_APER_HOST, *dma_addr,
  142. NOUVEAU_APER_VRAM, nouveau_dmem_page_addr(spage)))
  143. goto error_dma_unmap;
  144. mutex_unlock(&svmm->mutex);
  145. args->dst[0] = migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
  146. return 0;
  147. error_dma_unmap:
  148. mutex_unlock(&svmm->mutex);
  149. dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
  150. error_free_page:
  151. __free_page(dpage);
  152. return VM_FAULT_SIGBUS;
  153. }
  154. static vm_fault_t nouveau_dmem_migrate_to_ram(struct vm_fault *vmf)
  155. {
  156. struct nouveau_drm *drm = page_to_drm(vmf->page);
  157. struct nouveau_dmem *dmem = drm->dmem;
  158. struct nouveau_fence *fence;
  159. unsigned long src = 0, dst = 0;
  160. dma_addr_t dma_addr = 0;
  161. vm_fault_t ret;
  162. struct migrate_vma args = {
  163. .vma = vmf->vma,
  164. .start = vmf->address,
  165. .end = vmf->address + PAGE_SIZE,
  166. .src = &src,
  167. .dst = &dst,
  168. .pgmap_owner = drm->dev,
  169. .flags = MIGRATE_VMA_SELECT_DEVICE_PRIVATE,
  170. };
  171. /*
  172. * FIXME what we really want is to find some heuristic to migrate more
  173. * than just one page on CPU fault. When such fault happens it is very
  174. * likely that more surrounding page will CPU fault too.
  175. */
  176. if (migrate_vma_setup(&args) < 0)
  177. return VM_FAULT_SIGBUS;
  178. if (!args.cpages)
  179. return 0;
  180. ret = nouveau_dmem_fault_copy_one(drm, vmf, &args, &dma_addr);
  181. if (ret || dst == 0)
  182. goto done;
  183. nouveau_fence_new(dmem->migrate.chan, false, &fence);
  184. migrate_vma_pages(&args);
  185. nouveau_dmem_fence_done(&fence);
  186. dma_unmap_page(drm->dev->dev, dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
  187. done:
  188. migrate_vma_finalize(&args);
  189. return ret;
  190. }
  191. static const struct dev_pagemap_ops nouveau_dmem_pagemap_ops = {
  192. .page_free = nouveau_dmem_page_free,
  193. .migrate_to_ram = nouveau_dmem_migrate_to_ram,
  194. };
  195. static int
  196. nouveau_dmem_chunk_alloc(struct nouveau_drm *drm, struct page **ppage)
  197. {
  198. struct nouveau_dmem_chunk *chunk;
  199. struct resource *res;
  200. struct page *page;
  201. void *ptr;
  202. unsigned long i, pfn_first;
  203. int ret;
  204. chunk = kzalloc(sizeof(*chunk), GFP_KERNEL);
  205. if (chunk == NULL) {
  206. ret = -ENOMEM;
  207. goto out;
  208. }
  209. /* Allocate unused physical address space for device private pages. */
  210. res = request_free_mem_region(&iomem_resource, DMEM_CHUNK_SIZE,
  211. "nouveau_dmem");
  212. if (IS_ERR(res)) {
  213. ret = PTR_ERR(res);
  214. goto out_free;
  215. }
  216. chunk->drm = drm;
  217. chunk->pagemap.type = MEMORY_DEVICE_PRIVATE;
  218. chunk->pagemap.range.start = res->start;
  219. chunk->pagemap.range.end = res->end;
  220. chunk->pagemap.nr_range = 1;
  221. chunk->pagemap.ops = &nouveau_dmem_pagemap_ops;
  222. chunk->pagemap.owner = drm->dev;
  223. ret = nouveau_bo_new(&drm->client, DMEM_CHUNK_SIZE, 0,
  224. NOUVEAU_GEM_DOMAIN_VRAM, 0, 0, NULL, NULL,
  225. &chunk->bo);
  226. if (ret)
  227. goto out_release;
  228. ret = nouveau_bo_pin(chunk->bo, NOUVEAU_GEM_DOMAIN_VRAM, false);
  229. if (ret)
  230. goto out_bo_free;
  231. ptr = memremap_pages(&chunk->pagemap, numa_node_id());
  232. if (IS_ERR(ptr)) {
  233. ret = PTR_ERR(ptr);
  234. goto out_bo_unpin;
  235. }
  236. mutex_lock(&drm->dmem->mutex);
  237. list_add(&chunk->list, &drm->dmem->chunks);
  238. mutex_unlock(&drm->dmem->mutex);
  239. pfn_first = chunk->pagemap.range.start >> PAGE_SHIFT;
  240. page = pfn_to_page(pfn_first);
  241. spin_lock(&drm->dmem->lock);
  242. for (i = 0; i < DMEM_CHUNK_NPAGES - 1; ++i, ++page) {
  243. page->zone_device_data = drm->dmem->free_pages;
  244. drm->dmem->free_pages = page;
  245. }
  246. *ppage = page;
  247. chunk->callocated++;
  248. spin_unlock(&drm->dmem->lock);
  249. NV_INFO(drm, "DMEM: registered %ldMB of device memory\n",
  250. DMEM_CHUNK_SIZE >> 20);
  251. return 0;
  252. out_bo_unpin:
  253. nouveau_bo_unpin(chunk->bo);
  254. out_bo_free:
  255. nouveau_bo_ref(NULL, &chunk->bo);
  256. out_release:
  257. release_mem_region(chunk->pagemap.range.start, range_len(&chunk->pagemap.range));
  258. out_free:
  259. kfree(chunk);
  260. out:
  261. return ret;
  262. }
  263. static struct page *
  264. nouveau_dmem_page_alloc_locked(struct nouveau_drm *drm)
  265. {
  266. struct nouveau_dmem_chunk *chunk;
  267. struct page *page = NULL;
  268. int ret;
  269. spin_lock(&drm->dmem->lock);
  270. if (drm->dmem->free_pages) {
  271. page = drm->dmem->free_pages;
  272. drm->dmem->free_pages = page->zone_device_data;
  273. chunk = nouveau_page_to_chunk(page);
  274. chunk->callocated++;
  275. spin_unlock(&drm->dmem->lock);
  276. } else {
  277. spin_unlock(&drm->dmem->lock);
  278. ret = nouveau_dmem_chunk_alloc(drm, &page);
  279. if (ret)
  280. return NULL;
  281. }
  282. get_page(page);
  283. lock_page(page);
  284. return page;
  285. }
  286. static void
  287. nouveau_dmem_page_free_locked(struct nouveau_drm *drm, struct page *page)
  288. {
  289. unlock_page(page);
  290. put_page(page);
  291. }
  292. void
  293. nouveau_dmem_resume(struct nouveau_drm *drm)
  294. {
  295. struct nouveau_dmem_chunk *chunk;
  296. int ret;
  297. if (drm->dmem == NULL)
  298. return;
  299. mutex_lock(&drm->dmem->mutex);
  300. list_for_each_entry(chunk, &drm->dmem->chunks, list) {
  301. ret = nouveau_bo_pin(chunk->bo, NOUVEAU_GEM_DOMAIN_VRAM, false);
  302. /* FIXME handle pin failure */
  303. WARN_ON(ret);
  304. }
  305. mutex_unlock(&drm->dmem->mutex);
  306. }
  307. void
  308. nouveau_dmem_suspend(struct nouveau_drm *drm)
  309. {
  310. struct nouveau_dmem_chunk *chunk;
  311. if (drm->dmem == NULL)
  312. return;
  313. mutex_lock(&drm->dmem->mutex);
  314. list_for_each_entry(chunk, &drm->dmem->chunks, list)
  315. nouveau_bo_unpin(chunk->bo);
  316. mutex_unlock(&drm->dmem->mutex);
  317. }
  318. void
  319. nouveau_dmem_fini(struct nouveau_drm *drm)
  320. {
  321. struct nouveau_dmem_chunk *chunk, *tmp;
  322. if (drm->dmem == NULL)
  323. return;
  324. mutex_lock(&drm->dmem->mutex);
  325. list_for_each_entry_safe(chunk, tmp, &drm->dmem->chunks, list) {
  326. nouveau_bo_unpin(chunk->bo);
  327. nouveau_bo_ref(NULL, &chunk->bo);
  328. list_del(&chunk->list);
  329. memunmap_pages(&chunk->pagemap);
  330. release_mem_region(chunk->pagemap.range.start,
  331. range_len(&chunk->pagemap.range));
  332. kfree(chunk);
  333. }
  334. mutex_unlock(&drm->dmem->mutex);
  335. }
  336. static int
  337. nvc0b5_migrate_copy(struct nouveau_drm *drm, u64 npages,
  338. enum nouveau_aper dst_aper, u64 dst_addr,
  339. enum nouveau_aper src_aper, u64 src_addr)
  340. {
  341. struct nvif_push *push = drm->dmem->migrate.chan->chan.push;
  342. u32 launch_dma = 0;
  343. int ret;
  344. ret = PUSH_WAIT(push, 13);
  345. if (ret)
  346. return ret;
  347. if (src_aper != NOUVEAU_APER_VIRT) {
  348. switch (src_aper) {
  349. case NOUVEAU_APER_VRAM:
  350. PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE,
  351. NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, LOCAL_FB));
  352. break;
  353. case NOUVEAU_APER_HOST:
  354. PUSH_IMMD(push, NVA0B5, SET_SRC_PHYS_MODE,
  355. NVDEF(NVA0B5, SET_SRC_PHYS_MODE, TARGET, COHERENT_SYSMEM));
  356. break;
  357. default:
  358. return -EINVAL;
  359. }
  360. launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, SRC_TYPE, PHYSICAL);
  361. }
  362. if (dst_aper != NOUVEAU_APER_VIRT) {
  363. switch (dst_aper) {
  364. case NOUVEAU_APER_VRAM:
  365. PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
  366. NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB));
  367. break;
  368. case NOUVEAU_APER_HOST:
  369. PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
  370. NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM));
  371. break;
  372. default:
  373. return -EINVAL;
  374. }
  375. launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL);
  376. }
  377. PUSH_MTHD(push, NVA0B5, OFFSET_IN_UPPER,
  378. NVVAL(NVA0B5, OFFSET_IN_UPPER, UPPER, upper_32_bits(src_addr)),
  379. OFFSET_IN_LOWER, lower_32_bits(src_addr),
  380. OFFSET_OUT_UPPER,
  381. NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)),
  382. OFFSET_OUT_LOWER, lower_32_bits(dst_addr),
  383. PITCH_IN, PAGE_SIZE,
  384. PITCH_OUT, PAGE_SIZE,
  385. LINE_LENGTH_IN, PAGE_SIZE,
  386. LINE_COUNT, npages);
  387. PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma |
  388. NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) |
  389. NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) |
  390. NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) |
  391. NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) |
  392. NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) |
  393. NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) |
  394. NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, TRUE) |
  395. NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, FALSE) |
  396. NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING));
  397. return 0;
  398. }
  399. static int
  400. nvc0b5_migrate_clear(struct nouveau_drm *drm, u32 length,
  401. enum nouveau_aper dst_aper, u64 dst_addr)
  402. {
  403. struct nvif_push *push = drm->dmem->migrate.chan->chan.push;
  404. u32 launch_dma = 0;
  405. int ret;
  406. ret = PUSH_WAIT(push, 12);
  407. if (ret)
  408. return ret;
  409. switch (dst_aper) {
  410. case NOUVEAU_APER_VRAM:
  411. PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
  412. NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, LOCAL_FB));
  413. break;
  414. case NOUVEAU_APER_HOST:
  415. PUSH_IMMD(push, NVA0B5, SET_DST_PHYS_MODE,
  416. NVDEF(NVA0B5, SET_DST_PHYS_MODE, TARGET, COHERENT_SYSMEM));
  417. break;
  418. default:
  419. return -EINVAL;
  420. }
  421. launch_dma |= NVDEF(NVA0B5, LAUNCH_DMA, DST_TYPE, PHYSICAL);
  422. PUSH_MTHD(push, NVA0B5, SET_REMAP_CONST_A, 0,
  423. SET_REMAP_CONST_B, 0,
  424. SET_REMAP_COMPONENTS,
  425. NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_X, CONST_A) |
  426. NVDEF(NVA0B5, SET_REMAP_COMPONENTS, DST_Y, CONST_B) |
  427. NVDEF(NVA0B5, SET_REMAP_COMPONENTS, COMPONENT_SIZE, FOUR) |
  428. NVDEF(NVA0B5, SET_REMAP_COMPONENTS, NUM_DST_COMPONENTS, TWO));
  429. PUSH_MTHD(push, NVA0B5, OFFSET_OUT_UPPER,
  430. NVVAL(NVA0B5, OFFSET_OUT_UPPER, UPPER, upper_32_bits(dst_addr)),
  431. OFFSET_OUT_LOWER, lower_32_bits(dst_addr));
  432. PUSH_MTHD(push, NVA0B5, LINE_LENGTH_IN, length >> 3);
  433. PUSH_MTHD(push, NVA0B5, LAUNCH_DMA, launch_dma |
  434. NVDEF(NVA0B5, LAUNCH_DMA, DATA_TRANSFER_TYPE, NON_PIPELINED) |
  435. NVDEF(NVA0B5, LAUNCH_DMA, FLUSH_ENABLE, TRUE) |
  436. NVDEF(NVA0B5, LAUNCH_DMA, SEMAPHORE_TYPE, NONE) |
  437. NVDEF(NVA0B5, LAUNCH_DMA, INTERRUPT_TYPE, NONE) |
  438. NVDEF(NVA0B5, LAUNCH_DMA, SRC_MEMORY_LAYOUT, PITCH) |
  439. NVDEF(NVA0B5, LAUNCH_DMA, DST_MEMORY_LAYOUT, PITCH) |
  440. NVDEF(NVA0B5, LAUNCH_DMA, MULTI_LINE_ENABLE, FALSE) |
  441. NVDEF(NVA0B5, LAUNCH_DMA, REMAP_ENABLE, TRUE) |
  442. NVDEF(NVA0B5, LAUNCH_DMA, BYPASS_L2, USE_PTE_SETTING));
  443. return 0;
  444. }
  445. static int
  446. nouveau_dmem_migrate_init(struct nouveau_drm *drm)
  447. {
  448. switch (drm->ttm.copy.oclass) {
  449. case PASCAL_DMA_COPY_A:
  450. case PASCAL_DMA_COPY_B:
  451. case VOLTA_DMA_COPY_A:
  452. case TURING_DMA_COPY_A:
  453. drm->dmem->migrate.copy_func = nvc0b5_migrate_copy;
  454. drm->dmem->migrate.clear_func = nvc0b5_migrate_clear;
  455. drm->dmem->migrate.chan = drm->ttm.chan;
  456. return 0;
  457. default:
  458. break;
  459. }
  460. return -ENODEV;
  461. }
  462. void
  463. nouveau_dmem_init(struct nouveau_drm *drm)
  464. {
  465. int ret;
  466. /* This only make sense on PASCAL or newer */
  467. if (drm->client.device.info.family < NV_DEVICE_INFO_V0_PASCAL)
  468. return;
  469. if (!(drm->dmem = kzalloc(sizeof(*drm->dmem), GFP_KERNEL)))
  470. return;
  471. drm->dmem->drm = drm;
  472. mutex_init(&drm->dmem->mutex);
  473. INIT_LIST_HEAD(&drm->dmem->chunks);
  474. mutex_init(&drm->dmem->mutex);
  475. spin_lock_init(&drm->dmem->lock);
  476. /* Initialize migration dma helpers before registering memory */
  477. ret = nouveau_dmem_migrate_init(drm);
  478. if (ret) {
  479. kfree(drm->dmem);
  480. drm->dmem = NULL;
  481. }
  482. }
  483. static unsigned long nouveau_dmem_migrate_copy_one(struct nouveau_drm *drm,
  484. struct nouveau_svmm *svmm, unsigned long src,
  485. dma_addr_t *dma_addr, u64 *pfn)
  486. {
  487. struct device *dev = drm->dev->dev;
  488. struct page *dpage, *spage;
  489. unsigned long paddr;
  490. spage = migrate_pfn_to_page(src);
  491. if (!(src & MIGRATE_PFN_MIGRATE))
  492. goto out;
  493. dpage = nouveau_dmem_page_alloc_locked(drm);
  494. if (!dpage)
  495. goto out;
  496. paddr = nouveau_dmem_page_addr(dpage);
  497. if (spage) {
  498. *dma_addr = dma_map_page(dev, spage, 0, page_size(spage),
  499. DMA_BIDIRECTIONAL);
  500. if (dma_mapping_error(dev, *dma_addr))
  501. goto out_free_page;
  502. if (drm->dmem->migrate.copy_func(drm, 1,
  503. NOUVEAU_APER_VRAM, paddr, NOUVEAU_APER_HOST, *dma_addr))
  504. goto out_dma_unmap;
  505. } else {
  506. *dma_addr = DMA_MAPPING_ERROR;
  507. if (drm->dmem->migrate.clear_func(drm, page_size(dpage),
  508. NOUVEAU_APER_VRAM, paddr))
  509. goto out_free_page;
  510. }
  511. dpage->zone_device_data = svmm;
  512. *pfn = NVIF_VMM_PFNMAP_V0_V | NVIF_VMM_PFNMAP_V0_VRAM |
  513. ((paddr >> PAGE_SHIFT) << NVIF_VMM_PFNMAP_V0_ADDR_SHIFT);
  514. if (src & MIGRATE_PFN_WRITE)
  515. *pfn |= NVIF_VMM_PFNMAP_V0_W;
  516. return migrate_pfn(page_to_pfn(dpage)) | MIGRATE_PFN_LOCKED;
  517. out_dma_unmap:
  518. dma_unmap_page(dev, *dma_addr, PAGE_SIZE, DMA_BIDIRECTIONAL);
  519. out_free_page:
  520. nouveau_dmem_page_free_locked(drm, dpage);
  521. out:
  522. *pfn = NVIF_VMM_PFNMAP_V0_NONE;
  523. return 0;
  524. }
  525. static void nouveau_dmem_migrate_chunk(struct nouveau_drm *drm,
  526. struct nouveau_svmm *svmm, struct migrate_vma *args,
  527. dma_addr_t *dma_addrs, u64 *pfns)
  528. {
  529. struct nouveau_fence *fence;
  530. unsigned long addr = args->start, nr_dma = 0, i;
  531. for (i = 0; addr < args->end; i++) {
  532. args->dst[i] = nouveau_dmem_migrate_copy_one(drm, svmm,
  533. args->src[i], dma_addrs + nr_dma, pfns + i);
  534. if (!dma_mapping_error(drm->dev->dev, dma_addrs[nr_dma]))
  535. nr_dma++;
  536. addr += PAGE_SIZE;
  537. }
  538. nouveau_fence_new(drm->dmem->migrate.chan, false, &fence);
  539. migrate_vma_pages(args);
  540. nouveau_dmem_fence_done(&fence);
  541. nouveau_pfns_map(svmm, args->vma->vm_mm, args->start, pfns, i);
  542. while (nr_dma--) {
  543. dma_unmap_page(drm->dev->dev, dma_addrs[nr_dma], PAGE_SIZE,
  544. DMA_BIDIRECTIONAL);
  545. }
  546. migrate_vma_finalize(args);
  547. }
  548. int
  549. nouveau_dmem_migrate_vma(struct nouveau_drm *drm,
  550. struct nouveau_svmm *svmm,
  551. struct vm_area_struct *vma,
  552. unsigned long start,
  553. unsigned long end)
  554. {
  555. unsigned long npages = (end - start) >> PAGE_SHIFT;
  556. unsigned long max = min(SG_MAX_SINGLE_ALLOC, npages);
  557. dma_addr_t *dma_addrs;
  558. struct migrate_vma args = {
  559. .vma = vma,
  560. .start = start,
  561. .pgmap_owner = drm->dev,
  562. .flags = MIGRATE_VMA_SELECT_SYSTEM,
  563. };
  564. unsigned long i;
  565. u64 *pfns;
  566. int ret = -ENOMEM;
  567. if (drm->dmem == NULL)
  568. return -ENODEV;
  569. args.src = kcalloc(max, sizeof(*args.src), GFP_KERNEL);
  570. if (!args.src)
  571. goto out;
  572. args.dst = kcalloc(max, sizeof(*args.dst), GFP_KERNEL);
  573. if (!args.dst)
  574. goto out_free_src;
  575. dma_addrs = kmalloc_array(max, sizeof(*dma_addrs), GFP_KERNEL);
  576. if (!dma_addrs)
  577. goto out_free_dst;
  578. pfns = nouveau_pfns_alloc(max);
  579. if (!pfns)
  580. goto out_free_dma;
  581. for (i = 0; i < npages; i += max) {
  582. args.end = start + (max << PAGE_SHIFT);
  583. ret = migrate_vma_setup(&args);
  584. if (ret)
  585. goto out_free_pfns;
  586. if (args.cpages)
  587. nouveau_dmem_migrate_chunk(drm, svmm, &args, dma_addrs,
  588. pfns);
  589. args.start = args.end;
  590. }
  591. ret = 0;
  592. out_free_pfns:
  593. nouveau_pfns_free(pfns);
  594. out_free_dma:
  595. kfree(dma_addrs);
  596. out_free_dst:
  597. kfree(args.dst);
  598. out_free_src:
  599. kfree(args.src);
  600. out:
  601. return ret;
  602. }