nouveau_bo.c 35 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412
  1. /*
  2. * Copyright 2007 Dave Airlied
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining a
  6. * copy of this software and associated documentation files (the "Software"),
  7. * to deal in the Software without restriction, including without limitation
  8. * the rights to use, copy, modify, merge, publish, distribute, sublicense,
  9. * and/or sell copies of the Software, and to permit persons to whom the
  10. * Software is furnished to do so, subject to the following conditions:
  11. *
  12. * The above copyright notice and this permission notice (including the next
  13. * paragraph) shall be included in all copies or substantial portions of the
  14. * Software.
  15. *
  16. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  17. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  18. * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL
  19. * VA LINUX SYSTEMS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM, DAMAGES OR
  20. * OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE,
  21. * ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
  22. * OTHER DEALINGS IN THE SOFTWARE.
  23. */
  24. /*
  25. * Authors: Dave Airlied <airlied@linux.ie>
  26. * Ben Skeggs <darktama@iinet.net.au>
  27. * Jeremy Kolb <jkolb@brandeis.edu>
  28. */
  29. #include <linux/dma-mapping.h>
  30. #include <linux/swiotlb.h>
  31. #include "nouveau_drv.h"
  32. #include "nouveau_chan.h"
  33. #include "nouveau_fence.h"
  34. #include "nouveau_bo.h"
  35. #include "nouveau_ttm.h"
  36. #include "nouveau_gem.h"
  37. #include "nouveau_mem.h"
  38. #include "nouveau_vmm.h"
  39. #include <nvif/class.h>
  40. #include <nvif/if500b.h>
  41. #include <nvif/if900b.h>
  42. static int nouveau_ttm_tt_bind(struct ttm_bo_device *bdev, struct ttm_tt *ttm,
  43. struct ttm_resource *reg);
  44. /*
  45. * NV10-NV40 tiling helpers
  46. */
  47. static void
  48. nv10_bo_update_tile_region(struct drm_device *dev, struct nouveau_drm_tile *reg,
  49. u32 addr, u32 size, u32 pitch, u32 flags)
  50. {
  51. struct nouveau_drm *drm = nouveau_drm(dev);
  52. int i = reg - drm->tile.reg;
  53. struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
  54. struct nvkm_fb_tile *tile = &fb->tile.region[i];
  55. nouveau_fence_unref(&reg->fence);
  56. if (tile->pitch)
  57. nvkm_fb_tile_fini(fb, i, tile);
  58. if (pitch)
  59. nvkm_fb_tile_init(fb, i, addr, size, pitch, flags, tile);
  60. nvkm_fb_tile_prog(fb, i, tile);
  61. }
  62. static struct nouveau_drm_tile *
  63. nv10_bo_get_tile_region(struct drm_device *dev, int i)
  64. {
  65. struct nouveau_drm *drm = nouveau_drm(dev);
  66. struct nouveau_drm_tile *tile = &drm->tile.reg[i];
  67. spin_lock(&drm->tile.lock);
  68. if (!tile->used &&
  69. (!tile->fence || nouveau_fence_done(tile->fence)))
  70. tile->used = true;
  71. else
  72. tile = NULL;
  73. spin_unlock(&drm->tile.lock);
  74. return tile;
  75. }
  76. static void
  77. nv10_bo_put_tile_region(struct drm_device *dev, struct nouveau_drm_tile *tile,
  78. struct dma_fence *fence)
  79. {
  80. struct nouveau_drm *drm = nouveau_drm(dev);
  81. if (tile) {
  82. spin_lock(&drm->tile.lock);
  83. tile->fence = (struct nouveau_fence *)dma_fence_get(fence);
  84. tile->used = false;
  85. spin_unlock(&drm->tile.lock);
  86. }
  87. }
  88. static struct nouveau_drm_tile *
  89. nv10_bo_set_tiling(struct drm_device *dev, u32 addr,
  90. u32 size, u32 pitch, u32 zeta)
  91. {
  92. struct nouveau_drm *drm = nouveau_drm(dev);
  93. struct nvkm_fb *fb = nvxx_fb(&drm->client.device);
  94. struct nouveau_drm_tile *tile, *found = NULL;
  95. int i;
  96. for (i = 0; i < fb->tile.regions; i++) {
  97. tile = nv10_bo_get_tile_region(dev, i);
  98. if (pitch && !found) {
  99. found = tile;
  100. continue;
  101. } else if (tile && fb->tile.region[i].pitch) {
  102. /* Kill an unused tile region. */
  103. nv10_bo_update_tile_region(dev, tile, 0, 0, 0, 0);
  104. }
  105. nv10_bo_put_tile_region(dev, tile, NULL);
  106. }
  107. if (found)
  108. nv10_bo_update_tile_region(dev, found, addr, size, pitch, zeta);
  109. return found;
  110. }
  111. static void
  112. nouveau_bo_del_ttm(struct ttm_buffer_object *bo)
  113. {
  114. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  115. struct drm_device *dev = drm->dev;
  116. struct nouveau_bo *nvbo = nouveau_bo(bo);
  117. WARN_ON(nvbo->pin_refcnt > 0);
  118. nouveau_bo_del_io_reserve_lru(bo);
  119. nv10_bo_put_tile_region(dev, nvbo->tile, NULL);
  120. /*
  121. * If nouveau_bo_new() allocated this buffer, the GEM object was never
  122. * initialized, so don't attempt to release it.
  123. */
  124. if (bo->base.dev)
  125. drm_gem_object_release(&bo->base);
  126. kfree(nvbo);
  127. }
  128. static inline u64
  129. roundup_64(u64 x, u32 y)
  130. {
  131. x += y - 1;
  132. do_div(x, y);
  133. return x * y;
  134. }
  135. static void
  136. nouveau_bo_fixup_align(struct nouveau_bo *nvbo, int *align, u64 *size)
  137. {
  138. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  139. struct nvif_device *device = &drm->client.device;
  140. if (device->info.family < NV_DEVICE_INFO_V0_TESLA) {
  141. if (nvbo->mode) {
  142. if (device->info.chipset >= 0x40) {
  143. *align = 65536;
  144. *size = roundup_64(*size, 64 * nvbo->mode);
  145. } else if (device->info.chipset >= 0x30) {
  146. *align = 32768;
  147. *size = roundup_64(*size, 64 * nvbo->mode);
  148. } else if (device->info.chipset >= 0x20) {
  149. *align = 16384;
  150. *size = roundup_64(*size, 64 * nvbo->mode);
  151. } else if (device->info.chipset >= 0x10) {
  152. *align = 16384;
  153. *size = roundup_64(*size, 32 * nvbo->mode);
  154. }
  155. }
  156. } else {
  157. *size = roundup_64(*size, (1 << nvbo->page));
  158. *align = max((1 << nvbo->page), *align);
  159. }
  160. *size = roundup_64(*size, PAGE_SIZE);
  161. }
  162. struct nouveau_bo *
  163. nouveau_bo_alloc(struct nouveau_cli *cli, u64 *size, int *align, u32 domain,
  164. u32 tile_mode, u32 tile_flags)
  165. {
  166. struct nouveau_drm *drm = cli->drm;
  167. struct nouveau_bo *nvbo;
  168. struct nvif_mmu *mmu = &cli->mmu;
  169. struct nvif_vmm *vmm = cli->svm.cli ? &cli->svm.vmm : &cli->vmm.vmm;
  170. int i, pi = -1;
  171. if (!*size) {
  172. NV_WARN(drm, "skipped size %016llx\n", *size);
  173. return ERR_PTR(-EINVAL);
  174. }
  175. nvbo = kzalloc(sizeof(struct nouveau_bo), GFP_KERNEL);
  176. if (!nvbo)
  177. return ERR_PTR(-ENOMEM);
  178. INIT_LIST_HEAD(&nvbo->head);
  179. INIT_LIST_HEAD(&nvbo->entry);
  180. INIT_LIST_HEAD(&nvbo->vma_list);
  181. nvbo->bo.bdev = &drm->ttm.bdev;
  182. /* This is confusing, and doesn't actually mean we want an uncached
  183. * mapping, but is what NOUVEAU_GEM_DOMAIN_COHERENT gets translated
  184. * into in nouveau_gem_new().
  185. */
  186. if (domain & NOUVEAU_GEM_DOMAIN_COHERENT) {
  187. /* Determine if we can get a cache-coherent map, forcing
  188. * uncached mapping if we can't.
  189. */
  190. if (!nouveau_drm_use_coherent_gpu_mapping(drm))
  191. nvbo->force_coherent = true;
  192. }
  193. if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI) {
  194. nvbo->kind = (tile_flags & 0x0000ff00) >> 8;
  195. if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
  196. kfree(nvbo);
  197. return ERR_PTR(-EINVAL);
  198. }
  199. nvbo->comp = mmu->kind[nvbo->kind] != nvbo->kind;
  200. } else
  201. if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
  202. nvbo->kind = (tile_flags & 0x00007f00) >> 8;
  203. nvbo->comp = (tile_flags & 0x00030000) >> 16;
  204. if (!nvif_mmu_kind_valid(mmu, nvbo->kind)) {
  205. kfree(nvbo);
  206. return ERR_PTR(-EINVAL);
  207. }
  208. } else {
  209. nvbo->zeta = (tile_flags & 0x00000007);
  210. }
  211. nvbo->mode = tile_mode;
  212. nvbo->contig = !(tile_flags & NOUVEAU_GEM_TILE_NONCONTIG);
  213. /* Determine the desirable target GPU page size for the buffer. */
  214. for (i = 0; i < vmm->page_nr; i++) {
  215. /* Because we cannot currently allow VMM maps to fail
  216. * during buffer migration, we need to determine page
  217. * size for the buffer up-front, and pre-allocate its
  218. * page tables.
  219. *
  220. * Skip page sizes that can't support needed domains.
  221. */
  222. if (cli->device.info.family > NV_DEVICE_INFO_V0_CURIE &&
  223. (domain & NOUVEAU_GEM_DOMAIN_VRAM) && !vmm->page[i].vram)
  224. continue;
  225. if ((domain & NOUVEAU_GEM_DOMAIN_GART) &&
  226. (!vmm->page[i].host || vmm->page[i].shift > PAGE_SHIFT))
  227. continue;
  228. /* Select this page size if it's the first that supports
  229. * the potential memory domains, or when it's compatible
  230. * with the requested compression settings.
  231. */
  232. if (pi < 0 || !nvbo->comp || vmm->page[i].comp)
  233. pi = i;
  234. /* Stop once the buffer is larger than the current page size. */
  235. if (*size >= 1ULL << vmm->page[i].shift)
  236. break;
  237. }
  238. if (WARN_ON(pi < 0))
  239. return ERR_PTR(-EINVAL);
  240. /* Disable compression if suitable settings couldn't be found. */
  241. if (nvbo->comp && !vmm->page[pi].comp) {
  242. if (mmu->object.oclass >= NVIF_CLASS_MMU_GF100)
  243. nvbo->kind = mmu->kind[nvbo->kind];
  244. nvbo->comp = 0;
  245. }
  246. nvbo->page = vmm->page[pi].shift;
  247. nouveau_bo_fixup_align(nvbo, align, size);
  248. return nvbo;
  249. }
  250. int
  251. nouveau_bo_init(struct nouveau_bo *nvbo, u64 size, int align, u32 domain,
  252. struct sg_table *sg, struct dma_resv *robj)
  253. {
  254. int type = sg ? ttm_bo_type_sg : ttm_bo_type_device;
  255. size_t acc_size;
  256. int ret;
  257. acc_size = ttm_bo_dma_acc_size(nvbo->bo.bdev, size, sizeof(*nvbo));
  258. nvbo->bo.mem.num_pages = size >> PAGE_SHIFT;
  259. nouveau_bo_placement_set(nvbo, domain, 0);
  260. INIT_LIST_HEAD(&nvbo->io_reserve_lru);
  261. ret = ttm_bo_init(nvbo->bo.bdev, &nvbo->bo, size, type,
  262. &nvbo->placement, align >> PAGE_SHIFT, false,
  263. acc_size, sg, robj, nouveau_bo_del_ttm);
  264. if (ret) {
  265. /* ttm will call nouveau_bo_del_ttm if it fails.. */
  266. return ret;
  267. }
  268. return 0;
  269. }
  270. int
  271. nouveau_bo_new(struct nouveau_cli *cli, u64 size, int align,
  272. uint32_t domain, uint32_t tile_mode, uint32_t tile_flags,
  273. struct sg_table *sg, struct dma_resv *robj,
  274. struct nouveau_bo **pnvbo)
  275. {
  276. struct nouveau_bo *nvbo;
  277. int ret;
  278. nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
  279. tile_flags);
  280. if (IS_ERR(nvbo))
  281. return PTR_ERR(nvbo);
  282. ret = nouveau_bo_init(nvbo, size, align, domain, sg, robj);
  283. if (ret)
  284. return ret;
  285. *pnvbo = nvbo;
  286. return 0;
  287. }
  288. static void
  289. set_placement_list(struct nouveau_drm *drm, struct ttm_place *pl, unsigned *n,
  290. uint32_t domain, uint32_t flags)
  291. {
  292. *n = 0;
  293. if (domain & NOUVEAU_GEM_DOMAIN_VRAM) {
  294. struct nvif_mmu *mmu = &drm->client.mmu;
  295. pl[*n].mem_type = TTM_PL_VRAM;
  296. pl[*n].flags = flags & ~TTM_PL_FLAG_CACHED;
  297. /* Some BARs do not support being ioremapped WC */
  298. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
  299. mmu->type[drm->ttm.type_vram].type & NVIF_MEM_UNCACHED)
  300. pl[*n].flags &= ~TTM_PL_FLAG_WC;
  301. (*n)++;
  302. }
  303. if (domain & NOUVEAU_GEM_DOMAIN_GART) {
  304. pl[*n].mem_type = TTM_PL_TT;
  305. pl[*n].flags = flags;
  306. if (drm->agp.bridge)
  307. pl[*n].flags &= ~TTM_PL_FLAG_CACHED;
  308. (*n)++;
  309. }
  310. if (domain & NOUVEAU_GEM_DOMAIN_CPU) {
  311. pl[*n].mem_type = TTM_PL_SYSTEM;
  312. pl[(*n)++].flags = flags;
  313. }
  314. }
  315. static void
  316. set_placement_range(struct nouveau_bo *nvbo, uint32_t domain)
  317. {
  318. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  319. u32 vram_pages = drm->client.device.info.ram_size >> PAGE_SHIFT;
  320. unsigned i, fpfn, lpfn;
  321. if (drm->client.device.info.family == NV_DEVICE_INFO_V0_CELSIUS &&
  322. nvbo->mode && (domain & NOUVEAU_GEM_DOMAIN_VRAM) &&
  323. nvbo->bo.mem.num_pages < vram_pages / 4) {
  324. /*
  325. * Make sure that the color and depth buffers are handled
  326. * by independent memory controller units. Up to a 9x
  327. * speed up when alpha-blending and depth-test are enabled
  328. * at the same time.
  329. */
  330. if (nvbo->zeta) {
  331. fpfn = vram_pages / 2;
  332. lpfn = ~0;
  333. } else {
  334. fpfn = 0;
  335. lpfn = vram_pages / 2;
  336. }
  337. for (i = 0; i < nvbo->placement.num_placement; ++i) {
  338. nvbo->placements[i].fpfn = fpfn;
  339. nvbo->placements[i].lpfn = lpfn;
  340. }
  341. for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
  342. nvbo->busy_placements[i].fpfn = fpfn;
  343. nvbo->busy_placements[i].lpfn = lpfn;
  344. }
  345. }
  346. }
  347. void
  348. nouveau_bo_placement_set(struct nouveau_bo *nvbo, uint32_t domain,
  349. uint32_t busy)
  350. {
  351. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  352. struct ttm_placement *pl = &nvbo->placement;
  353. uint32_t flags = (nvbo->force_coherent ? TTM_PL_FLAG_UNCACHED :
  354. TTM_PL_MASK_CACHING) |
  355. (nvbo->pin_refcnt ? TTM_PL_FLAG_NO_EVICT : 0);
  356. pl->placement = nvbo->placements;
  357. set_placement_list(drm, nvbo->placements, &pl->num_placement,
  358. domain, flags);
  359. pl->busy_placement = nvbo->busy_placements;
  360. set_placement_list(drm, nvbo->busy_placements, &pl->num_busy_placement,
  361. domain | busy, flags);
  362. set_placement_range(nvbo, domain);
  363. }
  364. int
  365. nouveau_bo_pin(struct nouveau_bo *nvbo, uint32_t domain, bool contig)
  366. {
  367. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  368. struct ttm_buffer_object *bo = &nvbo->bo;
  369. bool force = false, evict = false;
  370. int ret;
  371. ret = ttm_bo_reserve(bo, false, false, NULL);
  372. if (ret)
  373. return ret;
  374. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA &&
  375. domain == NOUVEAU_GEM_DOMAIN_VRAM && contig) {
  376. if (!nvbo->contig) {
  377. nvbo->contig = true;
  378. force = true;
  379. evict = true;
  380. }
  381. }
  382. if (nvbo->pin_refcnt) {
  383. bool error = evict;
  384. switch (bo->mem.mem_type) {
  385. case TTM_PL_VRAM:
  386. error |= !(domain & NOUVEAU_GEM_DOMAIN_VRAM);
  387. break;
  388. case TTM_PL_TT:
  389. error |= !(domain & NOUVEAU_GEM_DOMAIN_GART);
  390. default:
  391. break;
  392. }
  393. if (error) {
  394. NV_ERROR(drm, "bo %p pinned elsewhere: "
  395. "0x%08x vs 0x%08x\n", bo,
  396. bo->mem.mem_type, domain);
  397. ret = -EBUSY;
  398. }
  399. nvbo->pin_refcnt++;
  400. goto out;
  401. }
  402. if (evict) {
  403. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
  404. ret = nouveau_bo_validate(nvbo, false, false);
  405. if (ret)
  406. goto out;
  407. }
  408. nvbo->pin_refcnt++;
  409. nouveau_bo_placement_set(nvbo, domain, 0);
  410. /* drop pin_refcnt temporarily, so we don't trip the assertion
  411. * in nouveau_bo_move() that makes sure we're not trying to
  412. * move a pinned buffer
  413. */
  414. nvbo->pin_refcnt--;
  415. ret = nouveau_bo_validate(nvbo, false, false);
  416. if (ret)
  417. goto out;
  418. nvbo->pin_refcnt++;
  419. switch (bo->mem.mem_type) {
  420. case TTM_PL_VRAM:
  421. drm->gem.vram_available -= bo->mem.size;
  422. break;
  423. case TTM_PL_TT:
  424. drm->gem.gart_available -= bo->mem.size;
  425. break;
  426. default:
  427. break;
  428. }
  429. out:
  430. if (force && ret)
  431. nvbo->contig = false;
  432. ttm_bo_unreserve(bo);
  433. return ret;
  434. }
  435. int
  436. nouveau_bo_unpin(struct nouveau_bo *nvbo)
  437. {
  438. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  439. struct ttm_buffer_object *bo = &nvbo->bo;
  440. int ret, ref;
  441. ret = ttm_bo_reserve(bo, false, false, NULL);
  442. if (ret)
  443. return ret;
  444. ref = --nvbo->pin_refcnt;
  445. WARN_ON_ONCE(ref < 0);
  446. if (ref)
  447. goto out;
  448. switch (bo->mem.mem_type) {
  449. case TTM_PL_VRAM:
  450. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0);
  451. break;
  452. case TTM_PL_TT:
  453. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART, 0);
  454. break;
  455. default:
  456. break;
  457. }
  458. ret = nouveau_bo_validate(nvbo, false, false);
  459. if (ret == 0) {
  460. switch (bo->mem.mem_type) {
  461. case TTM_PL_VRAM:
  462. drm->gem.vram_available += bo->mem.size;
  463. break;
  464. case TTM_PL_TT:
  465. drm->gem.gart_available += bo->mem.size;
  466. break;
  467. default:
  468. break;
  469. }
  470. }
  471. out:
  472. ttm_bo_unreserve(bo);
  473. return ret;
  474. }
  475. int
  476. nouveau_bo_map(struct nouveau_bo *nvbo)
  477. {
  478. int ret;
  479. ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
  480. if (ret)
  481. return ret;
  482. ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages, &nvbo->kmap);
  483. ttm_bo_unreserve(&nvbo->bo);
  484. return ret;
  485. }
  486. void
  487. nouveau_bo_unmap(struct nouveau_bo *nvbo)
  488. {
  489. if (!nvbo)
  490. return;
  491. ttm_bo_kunmap(&nvbo->kmap);
  492. }
  493. void
  494. nouveau_bo_sync_for_device(struct nouveau_bo *nvbo)
  495. {
  496. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  497. struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
  498. int i;
  499. if (!ttm_dma || !ttm_dma->dma_address)
  500. return;
  501. /* Don't waste time looping if the object is coherent */
  502. if (nvbo->force_coherent)
  503. return;
  504. for (i = 0; i < ttm_dma->ttm.num_pages; i++)
  505. dma_sync_single_for_device(drm->dev->dev,
  506. ttm_dma->dma_address[i],
  507. PAGE_SIZE, DMA_TO_DEVICE);
  508. }
  509. void
  510. nouveau_bo_sync_for_cpu(struct nouveau_bo *nvbo)
  511. {
  512. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  513. struct ttm_dma_tt *ttm_dma = (struct ttm_dma_tt *)nvbo->bo.ttm;
  514. int i;
  515. if (!ttm_dma || !ttm_dma->dma_address)
  516. return;
  517. /* Don't waste time looping if the object is coherent */
  518. if (nvbo->force_coherent)
  519. return;
  520. for (i = 0; i < ttm_dma->ttm.num_pages; i++)
  521. dma_sync_single_for_cpu(drm->dev->dev, ttm_dma->dma_address[i],
  522. PAGE_SIZE, DMA_FROM_DEVICE);
  523. }
  524. void nouveau_bo_add_io_reserve_lru(struct ttm_buffer_object *bo)
  525. {
  526. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  527. struct nouveau_bo *nvbo = nouveau_bo(bo);
  528. mutex_lock(&drm->ttm.io_reserve_mutex);
  529. list_move_tail(&nvbo->io_reserve_lru, &drm->ttm.io_reserve_lru);
  530. mutex_unlock(&drm->ttm.io_reserve_mutex);
  531. }
  532. void nouveau_bo_del_io_reserve_lru(struct ttm_buffer_object *bo)
  533. {
  534. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  535. struct nouveau_bo *nvbo = nouveau_bo(bo);
  536. mutex_lock(&drm->ttm.io_reserve_mutex);
  537. list_del_init(&nvbo->io_reserve_lru);
  538. mutex_unlock(&drm->ttm.io_reserve_mutex);
  539. }
  540. int
  541. nouveau_bo_validate(struct nouveau_bo *nvbo, bool interruptible,
  542. bool no_wait_gpu)
  543. {
  544. struct ttm_operation_ctx ctx = { interruptible, no_wait_gpu };
  545. int ret;
  546. ret = ttm_bo_validate(&nvbo->bo, &nvbo->placement, &ctx);
  547. if (ret)
  548. return ret;
  549. nouveau_bo_sync_for_device(nvbo);
  550. return 0;
  551. }
  552. void
  553. nouveau_bo_wr16(struct nouveau_bo *nvbo, unsigned index, u16 val)
  554. {
  555. bool is_iomem;
  556. u16 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
  557. mem += index;
  558. if (is_iomem)
  559. iowrite16_native(val, (void __force __iomem *)mem);
  560. else
  561. *mem = val;
  562. }
  563. u32
  564. nouveau_bo_rd32(struct nouveau_bo *nvbo, unsigned index)
  565. {
  566. bool is_iomem;
  567. u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
  568. mem += index;
  569. if (is_iomem)
  570. return ioread32_native((void __force __iomem *)mem);
  571. else
  572. return *mem;
  573. }
  574. void
  575. nouveau_bo_wr32(struct nouveau_bo *nvbo, unsigned index, u32 val)
  576. {
  577. bool is_iomem;
  578. u32 *mem = ttm_kmap_obj_virtual(&nvbo->kmap, &is_iomem);
  579. mem += index;
  580. if (is_iomem)
  581. iowrite32_native(val, (void __force __iomem *)mem);
  582. else
  583. *mem = val;
  584. }
  585. static struct ttm_tt *
  586. nouveau_ttm_tt_create(struct ttm_buffer_object *bo, uint32_t page_flags)
  587. {
  588. #if IS_ENABLED(CONFIG_AGP)
  589. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  590. if (drm->agp.bridge) {
  591. return ttm_agp_tt_create(bo, drm->agp.bridge, page_flags);
  592. }
  593. #endif
  594. return nouveau_sgdma_create_ttm(bo, page_flags);
  595. }
  596. static int
  597. nouveau_ttm_tt_bind(struct ttm_bo_device *bdev, struct ttm_tt *ttm,
  598. struct ttm_resource *reg)
  599. {
  600. #if IS_ENABLED(CONFIG_AGP)
  601. struct nouveau_drm *drm = nouveau_bdev(bdev);
  602. #endif
  603. if (!reg)
  604. return -EINVAL;
  605. #if IS_ENABLED(CONFIG_AGP)
  606. if (drm->agp.bridge)
  607. return ttm_agp_bind(ttm, reg);
  608. #endif
  609. return nouveau_sgdma_bind(bdev, ttm, reg);
  610. }
  611. static void
  612. nouveau_ttm_tt_unbind(struct ttm_bo_device *bdev, struct ttm_tt *ttm)
  613. {
  614. #if IS_ENABLED(CONFIG_AGP)
  615. struct nouveau_drm *drm = nouveau_bdev(bdev);
  616. if (drm->agp.bridge) {
  617. ttm_agp_unbind(ttm);
  618. return;
  619. }
  620. #endif
  621. nouveau_sgdma_unbind(bdev, ttm);
  622. }
  623. static void
  624. nouveau_bo_evict_flags(struct ttm_buffer_object *bo, struct ttm_placement *pl)
  625. {
  626. struct nouveau_bo *nvbo = nouveau_bo(bo);
  627. switch (bo->mem.mem_type) {
  628. case TTM_PL_VRAM:
  629. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART,
  630. NOUVEAU_GEM_DOMAIN_CPU);
  631. break;
  632. default:
  633. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_CPU, 0);
  634. break;
  635. }
  636. *pl = nvbo->placement;
  637. }
  638. static int
  639. nouveau_bo_move_prep(struct nouveau_drm *drm, struct ttm_buffer_object *bo,
  640. struct ttm_resource *reg)
  641. {
  642. struct nouveau_mem *old_mem = nouveau_mem(&bo->mem);
  643. struct nouveau_mem *new_mem = nouveau_mem(reg);
  644. struct nvif_vmm *vmm = &drm->client.vmm.vmm;
  645. int ret;
  646. ret = nvif_vmm_get(vmm, LAZY, false, old_mem->mem.page, 0,
  647. old_mem->mem.size, &old_mem->vma[0]);
  648. if (ret)
  649. return ret;
  650. ret = nvif_vmm_get(vmm, LAZY, false, new_mem->mem.page, 0,
  651. new_mem->mem.size, &old_mem->vma[1]);
  652. if (ret)
  653. goto done;
  654. ret = nouveau_mem_map(old_mem, vmm, &old_mem->vma[0]);
  655. if (ret)
  656. goto done;
  657. ret = nouveau_mem_map(new_mem, vmm, &old_mem->vma[1]);
  658. done:
  659. if (ret) {
  660. nvif_vmm_put(vmm, &old_mem->vma[1]);
  661. nvif_vmm_put(vmm, &old_mem->vma[0]);
  662. }
  663. return 0;
  664. }
  665. static int
  666. nouveau_bo_move_m2mf(struct ttm_buffer_object *bo, int evict, bool intr,
  667. bool no_wait_gpu, struct ttm_resource *new_reg)
  668. {
  669. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  670. struct nouveau_channel *chan = drm->ttm.chan;
  671. struct nouveau_cli *cli = (void *)chan->user.client;
  672. struct nouveau_fence *fence;
  673. int ret;
  674. /* create temporary vmas for the transfer and attach them to the
  675. * old nvkm_mem node, these will get cleaned up after ttm has
  676. * destroyed the ttm_resource
  677. */
  678. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA) {
  679. ret = nouveau_bo_move_prep(drm, bo, new_reg);
  680. if (ret)
  681. return ret;
  682. }
  683. mutex_lock_nested(&cli->mutex, SINGLE_DEPTH_NESTING);
  684. ret = nouveau_fence_sync(nouveau_bo(bo), chan, true, intr);
  685. if (ret == 0) {
  686. ret = drm->ttm.move(chan, bo, &bo->mem, new_reg);
  687. if (ret == 0) {
  688. ret = nouveau_fence_new(chan, false, &fence);
  689. if (ret == 0) {
  690. ret = ttm_bo_move_accel_cleanup(bo,
  691. &fence->base,
  692. evict, false,
  693. new_reg);
  694. nouveau_fence_unref(&fence);
  695. }
  696. }
  697. }
  698. mutex_unlock(&cli->mutex);
  699. return ret;
  700. }
  701. void
  702. nouveau_bo_move_init(struct nouveau_drm *drm)
  703. {
  704. static const struct _method_table {
  705. const char *name;
  706. int engine;
  707. s32 oclass;
  708. int (*exec)(struct nouveau_channel *,
  709. struct ttm_buffer_object *,
  710. struct ttm_resource *, struct ttm_resource *);
  711. int (*init)(struct nouveau_channel *, u32 handle);
  712. } _methods[] = {
  713. { "COPY", 4, 0xc5b5, nve0_bo_move_copy, nve0_bo_move_init },
  714. { "GRCE", 0, 0xc5b5, nve0_bo_move_copy, nvc0_bo_move_init },
  715. { "COPY", 4, 0xc3b5, nve0_bo_move_copy, nve0_bo_move_init },
  716. { "GRCE", 0, 0xc3b5, nve0_bo_move_copy, nvc0_bo_move_init },
  717. { "COPY", 4, 0xc1b5, nve0_bo_move_copy, nve0_bo_move_init },
  718. { "GRCE", 0, 0xc1b5, nve0_bo_move_copy, nvc0_bo_move_init },
  719. { "COPY", 4, 0xc0b5, nve0_bo_move_copy, nve0_bo_move_init },
  720. { "GRCE", 0, 0xc0b5, nve0_bo_move_copy, nvc0_bo_move_init },
  721. { "COPY", 4, 0xb0b5, nve0_bo_move_copy, nve0_bo_move_init },
  722. { "GRCE", 0, 0xb0b5, nve0_bo_move_copy, nvc0_bo_move_init },
  723. { "COPY", 4, 0xa0b5, nve0_bo_move_copy, nve0_bo_move_init },
  724. { "GRCE", 0, 0xa0b5, nve0_bo_move_copy, nvc0_bo_move_init },
  725. { "COPY1", 5, 0x90b8, nvc0_bo_move_copy, nvc0_bo_move_init },
  726. { "COPY0", 4, 0x90b5, nvc0_bo_move_copy, nvc0_bo_move_init },
  727. { "COPY", 0, 0x85b5, nva3_bo_move_copy, nv50_bo_move_init },
  728. { "CRYPT", 0, 0x74c1, nv84_bo_move_exec, nv50_bo_move_init },
  729. { "M2MF", 0, 0x9039, nvc0_bo_move_m2mf, nvc0_bo_move_init },
  730. { "M2MF", 0, 0x5039, nv50_bo_move_m2mf, nv50_bo_move_init },
  731. { "M2MF", 0, 0x0039, nv04_bo_move_m2mf, nv04_bo_move_init },
  732. {},
  733. };
  734. const struct _method_table *mthd = _methods;
  735. const char *name = "CPU";
  736. int ret;
  737. do {
  738. struct nouveau_channel *chan;
  739. if (mthd->engine)
  740. chan = drm->cechan;
  741. else
  742. chan = drm->channel;
  743. if (chan == NULL)
  744. continue;
  745. ret = nvif_object_ctor(&chan->user, "ttmBoMove",
  746. mthd->oclass | (mthd->engine << 16),
  747. mthd->oclass, NULL, 0,
  748. &drm->ttm.copy);
  749. if (ret == 0) {
  750. ret = mthd->init(chan, drm->ttm.copy.handle);
  751. if (ret) {
  752. nvif_object_dtor(&drm->ttm.copy);
  753. continue;
  754. }
  755. drm->ttm.move = mthd->exec;
  756. drm->ttm.chan = chan;
  757. name = mthd->name;
  758. break;
  759. }
  760. } while ((++mthd)->exec);
  761. NV_INFO(drm, "MM: using %s for buffer copies\n", name);
  762. }
  763. static int
  764. nouveau_bo_move_flipd(struct ttm_buffer_object *bo, bool evict, bool intr,
  765. bool no_wait_gpu, struct ttm_resource *new_reg)
  766. {
  767. struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
  768. struct ttm_place placement_memtype = {
  769. .fpfn = 0,
  770. .lpfn = 0,
  771. .mem_type = TTM_PL_TT,
  772. .flags = TTM_PL_MASK_CACHING
  773. };
  774. struct ttm_placement placement;
  775. struct ttm_resource tmp_reg;
  776. int ret;
  777. placement.num_placement = placement.num_busy_placement = 1;
  778. placement.placement = placement.busy_placement = &placement_memtype;
  779. tmp_reg = *new_reg;
  780. tmp_reg.mm_node = NULL;
  781. ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
  782. if (ret)
  783. return ret;
  784. ret = ttm_tt_populate(bo->bdev, bo->ttm, &ctx);
  785. if (ret)
  786. goto out;
  787. ret = nouveau_ttm_tt_bind(bo->bdev, bo->ttm, &tmp_reg);
  788. if (ret)
  789. goto out;
  790. ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, &tmp_reg);
  791. if (ret)
  792. goto out;
  793. ret = ttm_bo_move_ttm(bo, &ctx, new_reg);
  794. out:
  795. ttm_resource_free(bo, &tmp_reg);
  796. return ret;
  797. }
  798. static int
  799. nouveau_bo_move_flips(struct ttm_buffer_object *bo, bool evict, bool intr,
  800. bool no_wait_gpu, struct ttm_resource *new_reg)
  801. {
  802. struct ttm_operation_ctx ctx = { intr, no_wait_gpu };
  803. struct ttm_place placement_memtype = {
  804. .fpfn = 0,
  805. .lpfn = 0,
  806. .mem_type = TTM_PL_TT,
  807. .flags = TTM_PL_MASK_CACHING
  808. };
  809. struct ttm_placement placement;
  810. struct ttm_resource tmp_reg;
  811. int ret;
  812. placement.num_placement = placement.num_busy_placement = 1;
  813. placement.placement = placement.busy_placement = &placement_memtype;
  814. tmp_reg = *new_reg;
  815. tmp_reg.mm_node = NULL;
  816. ret = ttm_bo_mem_space(bo, &placement, &tmp_reg, &ctx);
  817. if (ret)
  818. return ret;
  819. ret = ttm_bo_move_ttm(bo, &ctx, &tmp_reg);
  820. if (ret)
  821. goto out;
  822. ret = nouveau_bo_move_m2mf(bo, true, intr, no_wait_gpu, new_reg);
  823. if (ret)
  824. goto out;
  825. out:
  826. ttm_resource_free(bo, &tmp_reg);
  827. return ret;
  828. }
  829. static void
  830. nouveau_bo_move_ntfy(struct ttm_buffer_object *bo, bool evict,
  831. struct ttm_resource *new_reg)
  832. {
  833. struct nouveau_mem *mem = new_reg ? nouveau_mem(new_reg) : NULL;
  834. struct nouveau_bo *nvbo = nouveau_bo(bo);
  835. struct nouveau_vma *vma;
  836. /* ttm can now (stupidly) pass the driver bos it didn't create... */
  837. if (bo->destroy != nouveau_bo_del_ttm)
  838. return;
  839. nouveau_bo_del_io_reserve_lru(bo);
  840. if (mem && new_reg->mem_type != TTM_PL_SYSTEM &&
  841. mem->mem.page == nvbo->page) {
  842. list_for_each_entry(vma, &nvbo->vma_list, head) {
  843. nouveau_vma_map(vma, mem);
  844. }
  845. } else {
  846. list_for_each_entry(vma, &nvbo->vma_list, head) {
  847. WARN_ON(ttm_bo_wait(bo, false, false));
  848. nouveau_vma_unmap(vma);
  849. }
  850. }
  851. if (new_reg) {
  852. if (new_reg->mm_node)
  853. nvbo->offset = (new_reg->start << PAGE_SHIFT);
  854. else
  855. nvbo->offset = 0;
  856. }
  857. }
  858. static int
  859. nouveau_bo_vm_bind(struct ttm_buffer_object *bo, struct ttm_resource *new_reg,
  860. struct nouveau_drm_tile **new_tile)
  861. {
  862. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  863. struct drm_device *dev = drm->dev;
  864. struct nouveau_bo *nvbo = nouveau_bo(bo);
  865. u64 offset = new_reg->start << PAGE_SHIFT;
  866. *new_tile = NULL;
  867. if (new_reg->mem_type != TTM_PL_VRAM)
  868. return 0;
  869. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_CELSIUS) {
  870. *new_tile = nv10_bo_set_tiling(dev, offset, new_reg->size,
  871. nvbo->mode, nvbo->zeta);
  872. }
  873. return 0;
  874. }
  875. static void
  876. nouveau_bo_vm_cleanup(struct ttm_buffer_object *bo,
  877. struct nouveau_drm_tile *new_tile,
  878. struct nouveau_drm_tile **old_tile)
  879. {
  880. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  881. struct drm_device *dev = drm->dev;
  882. struct dma_fence *fence = dma_resv_get_excl(bo->base.resv);
  883. nv10_bo_put_tile_region(dev, *old_tile, fence);
  884. *old_tile = new_tile;
  885. }
  886. static int
  887. nouveau_bo_move(struct ttm_buffer_object *bo, bool evict,
  888. struct ttm_operation_ctx *ctx,
  889. struct ttm_resource *new_reg)
  890. {
  891. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  892. struct nouveau_bo *nvbo = nouveau_bo(bo);
  893. struct ttm_resource *old_reg = &bo->mem;
  894. struct nouveau_drm_tile *new_tile = NULL;
  895. int ret = 0;
  896. ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
  897. if (ret)
  898. return ret;
  899. if (nvbo->pin_refcnt)
  900. NV_WARN(drm, "Moving pinned object %p!\n", nvbo);
  901. if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
  902. ret = nouveau_bo_vm_bind(bo, new_reg, &new_tile);
  903. if (ret)
  904. return ret;
  905. }
  906. /* Fake bo copy. */
  907. if (old_reg->mem_type == TTM_PL_SYSTEM && !bo->ttm) {
  908. ttm_bo_move_null(bo, new_reg);
  909. goto out;
  910. }
  911. /* Hardware assisted copy. */
  912. if (drm->ttm.move) {
  913. if (new_reg->mem_type == TTM_PL_SYSTEM)
  914. ret = nouveau_bo_move_flipd(bo, evict,
  915. ctx->interruptible,
  916. ctx->no_wait_gpu, new_reg);
  917. else if (old_reg->mem_type == TTM_PL_SYSTEM)
  918. ret = nouveau_bo_move_flips(bo, evict,
  919. ctx->interruptible,
  920. ctx->no_wait_gpu, new_reg);
  921. else
  922. ret = nouveau_bo_move_m2mf(bo, evict,
  923. ctx->interruptible,
  924. ctx->no_wait_gpu, new_reg);
  925. if (!ret)
  926. goto out;
  927. }
  928. /* Fallback to software copy. */
  929. ret = ttm_bo_wait(bo, ctx->interruptible, ctx->no_wait_gpu);
  930. if (ret == 0)
  931. ret = ttm_bo_move_memcpy(bo, ctx, new_reg);
  932. out:
  933. if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
  934. if (ret)
  935. nouveau_bo_vm_cleanup(bo, NULL, &new_tile);
  936. else
  937. nouveau_bo_vm_cleanup(bo, new_tile, &nvbo->tile);
  938. }
  939. return ret;
  940. }
  941. static int
  942. nouveau_bo_verify_access(struct ttm_buffer_object *bo, struct file *filp)
  943. {
  944. struct nouveau_bo *nvbo = nouveau_bo(bo);
  945. return drm_vma_node_verify_access(&nvbo->bo.base.vma_node,
  946. filp->private_data);
  947. }
  948. static void
  949. nouveau_ttm_io_mem_free_locked(struct nouveau_drm *drm,
  950. struct ttm_resource *reg)
  951. {
  952. struct nouveau_mem *mem = nouveau_mem(reg);
  953. if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
  954. switch (reg->mem_type) {
  955. case TTM_PL_TT:
  956. if (mem->kind)
  957. nvif_object_unmap_handle(&mem->mem.object);
  958. break;
  959. case TTM_PL_VRAM:
  960. nvif_object_unmap_handle(&mem->mem.object);
  961. break;
  962. default:
  963. break;
  964. }
  965. }
  966. }
  967. static int
  968. nouveau_ttm_io_mem_reserve(struct ttm_bo_device *bdev, struct ttm_resource *reg)
  969. {
  970. struct nouveau_drm *drm = nouveau_bdev(bdev);
  971. struct nvkm_device *device = nvxx_device(&drm->client.device);
  972. struct nouveau_mem *mem = nouveau_mem(reg);
  973. int ret;
  974. mutex_lock(&drm->ttm.io_reserve_mutex);
  975. retry:
  976. switch (reg->mem_type) {
  977. case TTM_PL_SYSTEM:
  978. /* System memory */
  979. ret = 0;
  980. goto out;
  981. case TTM_PL_TT:
  982. #if IS_ENABLED(CONFIG_AGP)
  983. if (drm->agp.bridge) {
  984. reg->bus.offset = (reg->start << PAGE_SHIFT) +
  985. drm->agp.base;
  986. reg->bus.is_iomem = !drm->agp.cma;
  987. }
  988. #endif
  989. if (drm->client.mem->oclass < NVIF_CLASS_MEM_NV50 ||
  990. !mem->kind) {
  991. /* untiled */
  992. ret = 0;
  993. break;
  994. }
  995. fallthrough; /* tiled memory */
  996. case TTM_PL_VRAM:
  997. reg->bus.offset = (reg->start << PAGE_SHIFT) +
  998. device->func->resource_addr(device, 1);
  999. reg->bus.is_iomem = true;
  1000. if (drm->client.mem->oclass >= NVIF_CLASS_MEM_NV50) {
  1001. union {
  1002. struct nv50_mem_map_v0 nv50;
  1003. struct gf100_mem_map_v0 gf100;
  1004. } args;
  1005. u64 handle, length;
  1006. u32 argc = 0;
  1007. switch (mem->mem.object.oclass) {
  1008. case NVIF_CLASS_MEM_NV50:
  1009. args.nv50.version = 0;
  1010. args.nv50.ro = 0;
  1011. args.nv50.kind = mem->kind;
  1012. args.nv50.comp = mem->comp;
  1013. argc = sizeof(args.nv50);
  1014. break;
  1015. case NVIF_CLASS_MEM_GF100:
  1016. args.gf100.version = 0;
  1017. args.gf100.ro = 0;
  1018. args.gf100.kind = mem->kind;
  1019. argc = sizeof(args.gf100);
  1020. break;
  1021. default:
  1022. WARN_ON(1);
  1023. break;
  1024. }
  1025. ret = nvif_object_map_handle(&mem->mem.object,
  1026. &args, argc,
  1027. &handle, &length);
  1028. if (ret != 1) {
  1029. if (WARN_ON(ret == 0))
  1030. ret = -EINVAL;
  1031. goto out;
  1032. }
  1033. reg->bus.offset = handle;
  1034. }
  1035. ret = 0;
  1036. break;
  1037. default:
  1038. ret = -EINVAL;
  1039. }
  1040. out:
  1041. if (ret == -ENOSPC) {
  1042. struct nouveau_bo *nvbo;
  1043. nvbo = list_first_entry_or_null(&drm->ttm.io_reserve_lru,
  1044. typeof(*nvbo),
  1045. io_reserve_lru);
  1046. if (nvbo) {
  1047. list_del_init(&nvbo->io_reserve_lru);
  1048. drm_vma_node_unmap(&nvbo->bo.base.vma_node,
  1049. bdev->dev_mapping);
  1050. nouveau_ttm_io_mem_free_locked(drm, &nvbo->bo.mem);
  1051. goto retry;
  1052. }
  1053. }
  1054. mutex_unlock(&drm->ttm.io_reserve_mutex);
  1055. return ret;
  1056. }
  1057. static void
  1058. nouveau_ttm_io_mem_free(struct ttm_bo_device *bdev, struct ttm_resource *reg)
  1059. {
  1060. struct nouveau_drm *drm = nouveau_bdev(bdev);
  1061. mutex_lock(&drm->ttm.io_reserve_mutex);
  1062. nouveau_ttm_io_mem_free_locked(drm, reg);
  1063. mutex_unlock(&drm->ttm.io_reserve_mutex);
  1064. }
  1065. static int
  1066. nouveau_ttm_fault_reserve_notify(struct ttm_buffer_object *bo)
  1067. {
  1068. struct nouveau_drm *drm = nouveau_bdev(bo->bdev);
  1069. struct nouveau_bo *nvbo = nouveau_bo(bo);
  1070. struct nvkm_device *device = nvxx_device(&drm->client.device);
  1071. u32 mappable = device->func->resource_size(device, 1) >> PAGE_SHIFT;
  1072. int i, ret;
  1073. /* as long as the bo isn't in vram, and isn't tiled, we've got
  1074. * nothing to do here.
  1075. */
  1076. if (bo->mem.mem_type != TTM_PL_VRAM) {
  1077. if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA ||
  1078. !nvbo->kind)
  1079. return 0;
  1080. if (bo->mem.mem_type == TTM_PL_SYSTEM) {
  1081. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_GART,
  1082. 0);
  1083. ret = nouveau_bo_validate(nvbo, false, false);
  1084. if (ret)
  1085. return ret;
  1086. }
  1087. return 0;
  1088. }
  1089. /* make sure bo is in mappable vram */
  1090. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA ||
  1091. bo->mem.start + bo->mem.num_pages < mappable)
  1092. return 0;
  1093. for (i = 0; i < nvbo->placement.num_placement; ++i) {
  1094. nvbo->placements[i].fpfn = 0;
  1095. nvbo->placements[i].lpfn = mappable;
  1096. }
  1097. for (i = 0; i < nvbo->placement.num_busy_placement; ++i) {
  1098. nvbo->busy_placements[i].fpfn = 0;
  1099. nvbo->busy_placements[i].lpfn = mappable;
  1100. }
  1101. nouveau_bo_placement_set(nvbo, NOUVEAU_GEM_DOMAIN_VRAM, 0);
  1102. return nouveau_bo_validate(nvbo, false, false);
  1103. }
  1104. static int
  1105. nouveau_ttm_tt_populate(struct ttm_bo_device *bdev,
  1106. struct ttm_tt *ttm, struct ttm_operation_ctx *ctx)
  1107. {
  1108. struct ttm_dma_tt *ttm_dma = (void *)ttm;
  1109. struct nouveau_drm *drm;
  1110. struct device *dev;
  1111. bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
  1112. if (ttm_tt_is_populated(ttm))
  1113. return 0;
  1114. if (slave && ttm->sg) {
  1115. /* make userspace faulting work */
  1116. drm_prime_sg_to_page_addr_arrays(ttm->sg, ttm->pages,
  1117. ttm_dma->dma_address, ttm->num_pages);
  1118. ttm_tt_set_populated(ttm);
  1119. return 0;
  1120. }
  1121. drm = nouveau_bdev(bdev);
  1122. dev = drm->dev->dev;
  1123. #if IS_ENABLED(CONFIG_AGP)
  1124. if (drm->agp.bridge) {
  1125. return ttm_pool_populate(ttm, ctx);
  1126. }
  1127. #endif
  1128. #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
  1129. if (swiotlb_nr_tbl()) {
  1130. return ttm_dma_populate((void *)ttm, dev, ctx);
  1131. }
  1132. #endif
  1133. return ttm_populate_and_map_pages(dev, ttm_dma, ctx);
  1134. }
  1135. static void
  1136. nouveau_ttm_tt_unpopulate(struct ttm_bo_device *bdev,
  1137. struct ttm_tt *ttm)
  1138. {
  1139. struct ttm_dma_tt *ttm_dma = (void *)ttm;
  1140. struct nouveau_drm *drm;
  1141. struct device *dev;
  1142. bool slave = !!(ttm->page_flags & TTM_PAGE_FLAG_SG);
  1143. if (slave)
  1144. return;
  1145. drm = nouveau_bdev(bdev);
  1146. dev = drm->dev->dev;
  1147. #if IS_ENABLED(CONFIG_AGP)
  1148. if (drm->agp.bridge) {
  1149. ttm_pool_unpopulate(ttm);
  1150. return;
  1151. }
  1152. #endif
  1153. #if IS_ENABLED(CONFIG_SWIOTLB) && IS_ENABLED(CONFIG_X86)
  1154. if (swiotlb_nr_tbl()) {
  1155. ttm_dma_unpopulate((void *)ttm, dev);
  1156. return;
  1157. }
  1158. #endif
  1159. ttm_unmap_and_unpopulate_pages(dev, ttm_dma);
  1160. }
  1161. static void
  1162. nouveau_ttm_tt_destroy(struct ttm_bo_device *bdev,
  1163. struct ttm_tt *ttm)
  1164. {
  1165. #if IS_ENABLED(CONFIG_AGP)
  1166. struct nouveau_drm *drm = nouveau_bdev(bdev);
  1167. if (drm->agp.bridge) {
  1168. ttm_agp_unbind(ttm);
  1169. ttm_tt_destroy_common(bdev, ttm);
  1170. ttm_agp_destroy(ttm);
  1171. return;
  1172. }
  1173. #endif
  1174. nouveau_sgdma_destroy(bdev, ttm);
  1175. }
  1176. void
  1177. nouveau_bo_fence(struct nouveau_bo *nvbo, struct nouveau_fence *fence, bool exclusive)
  1178. {
  1179. struct dma_resv *resv = nvbo->bo.base.resv;
  1180. if (exclusive)
  1181. dma_resv_add_excl_fence(resv, &fence->base);
  1182. else if (fence)
  1183. dma_resv_add_shared_fence(resv, &fence->base);
  1184. }
  1185. struct ttm_bo_driver nouveau_bo_driver = {
  1186. .ttm_tt_create = &nouveau_ttm_tt_create,
  1187. .ttm_tt_populate = &nouveau_ttm_tt_populate,
  1188. .ttm_tt_unpopulate = &nouveau_ttm_tt_unpopulate,
  1189. .ttm_tt_bind = &nouveau_ttm_tt_bind,
  1190. .ttm_tt_unbind = &nouveau_ttm_tt_unbind,
  1191. .ttm_tt_destroy = &nouveau_ttm_tt_destroy,
  1192. .eviction_valuable = ttm_bo_eviction_valuable,
  1193. .evict_flags = nouveau_bo_evict_flags,
  1194. .move_notify = nouveau_bo_move_ntfy,
  1195. .move = nouveau_bo_move,
  1196. .verify_access = nouveau_bo_verify_access,
  1197. .fault_reserve_notify = &nouveau_ttm_fault_reserve_notify,
  1198. .io_mem_reserve = &nouveau_ttm_io_mem_reserve,
  1199. .io_mem_free = &nouveau_ttm_io_mem_free,
  1200. };