nouveau_gem.c 24 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964
  1. /*
  2. * Copyright (C) 2008 Ben Skeggs.
  3. * All Rights Reserved.
  4. *
  5. * Permission is hereby granted, free of charge, to any person obtaining
  6. * a copy of this software and associated documentation files (the
  7. * "Software"), to deal in the Software without restriction, including
  8. * without limitation the rights to use, copy, modify, merge, publish,
  9. * distribute, sublicense, and/or sell copies of the Software, and to
  10. * permit persons to whom the Software is furnished to do so, subject to
  11. * the following conditions:
  12. *
  13. * The above copyright notice and this permission notice (including the
  14. * next paragraph) shall be included in all copies or substantial
  15. * portions of the Software.
  16. *
  17. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  18. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  19. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  20. * IN NO EVENT SHALL THE COPYRIGHT OWNER(S) AND/OR ITS SUPPLIERS BE
  21. * LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
  22. * OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
  23. * WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  24. *
  25. */
  26. #include "nouveau_drv.h"
  27. #include "nouveau_dma.h"
  28. #include "nouveau_fence.h"
  29. #include "nouveau_abi16.h"
  30. #include "nouveau_ttm.h"
  31. #include "nouveau_gem.h"
  32. #include "nouveau_mem.h"
  33. #include "nouveau_vmm.h"
  34. #include <nvif/class.h>
  35. #include <nvif/push206e.h>
  36. void
  37. nouveau_gem_object_del(struct drm_gem_object *gem)
  38. {
  39. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  40. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  41. struct device *dev = drm->dev->dev;
  42. int ret;
  43. ret = pm_runtime_get_sync(dev);
  44. if (WARN_ON(ret < 0 && ret != -EACCES)) {
  45. pm_runtime_put_autosuspend(dev);
  46. return;
  47. }
  48. if (gem->import_attach)
  49. drm_prime_gem_destroy(gem, nvbo->bo.sg);
  50. ttm_bo_put(&nvbo->bo);
  51. pm_runtime_mark_last_busy(dev);
  52. pm_runtime_put_autosuspend(dev);
  53. }
  54. int
  55. nouveau_gem_object_open(struct drm_gem_object *gem, struct drm_file *file_priv)
  56. {
  57. struct nouveau_cli *cli = nouveau_cli(file_priv);
  58. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  59. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  60. struct device *dev = drm->dev->dev;
  61. struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
  62. struct nouveau_vma *vma;
  63. int ret;
  64. if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
  65. return 0;
  66. ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
  67. if (ret)
  68. return ret;
  69. ret = pm_runtime_get_sync(dev);
  70. if (ret < 0 && ret != -EACCES) {
  71. pm_runtime_put_autosuspend(dev);
  72. goto out;
  73. }
  74. ret = nouveau_vma_new(nvbo, vmm, &vma);
  75. pm_runtime_mark_last_busy(dev);
  76. pm_runtime_put_autosuspend(dev);
  77. out:
  78. ttm_bo_unreserve(&nvbo->bo);
  79. return ret;
  80. }
  81. struct nouveau_gem_object_unmap {
  82. struct nouveau_cli_work work;
  83. struct nouveau_vma *vma;
  84. };
  85. static void
  86. nouveau_gem_object_delete(struct nouveau_vma *vma)
  87. {
  88. nouveau_fence_unref(&vma->fence);
  89. nouveau_vma_del(&vma);
  90. }
  91. static void
  92. nouveau_gem_object_delete_work(struct nouveau_cli_work *w)
  93. {
  94. struct nouveau_gem_object_unmap *work =
  95. container_of(w, typeof(*work), work);
  96. nouveau_gem_object_delete(work->vma);
  97. kfree(work);
  98. }
  99. static void
  100. nouveau_gem_object_unmap(struct nouveau_bo *nvbo, struct nouveau_vma *vma)
  101. {
  102. struct dma_fence *fence = vma->fence ? &vma->fence->base : NULL;
  103. struct nouveau_gem_object_unmap *work;
  104. list_del_init(&vma->head);
  105. if (!fence) {
  106. nouveau_gem_object_delete(vma);
  107. return;
  108. }
  109. if (!(work = kmalloc(sizeof(*work), GFP_KERNEL))) {
  110. WARN_ON(dma_fence_wait_timeout(fence, false, 2 * HZ) <= 0);
  111. nouveau_gem_object_delete(vma);
  112. return;
  113. }
  114. work->work.func = nouveau_gem_object_delete_work;
  115. work->vma = vma;
  116. nouveau_cli_work_queue(vma->vmm->cli, fence, &work->work);
  117. }
  118. void
  119. nouveau_gem_object_close(struct drm_gem_object *gem, struct drm_file *file_priv)
  120. {
  121. struct nouveau_cli *cli = nouveau_cli(file_priv);
  122. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  123. struct nouveau_drm *drm = nouveau_bdev(nvbo->bo.bdev);
  124. struct device *dev = drm->dev->dev;
  125. struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : & cli->vmm;
  126. struct nouveau_vma *vma;
  127. int ret;
  128. if (vmm->vmm.object.oclass < NVIF_CLASS_VMM_NV50)
  129. return;
  130. ret = ttm_bo_reserve(&nvbo->bo, false, false, NULL);
  131. if (ret)
  132. return;
  133. vma = nouveau_vma_find(nvbo, vmm);
  134. if (vma) {
  135. if (--vma->refs == 0) {
  136. ret = pm_runtime_get_sync(dev);
  137. if (!WARN_ON(ret < 0 && ret != -EACCES)) {
  138. nouveau_gem_object_unmap(nvbo, vma);
  139. pm_runtime_mark_last_busy(dev);
  140. }
  141. pm_runtime_put_autosuspend(dev);
  142. }
  143. }
  144. ttm_bo_unreserve(&nvbo->bo);
  145. }
  146. int
  147. nouveau_gem_new(struct nouveau_cli *cli, u64 size, int align, uint32_t domain,
  148. uint32_t tile_mode, uint32_t tile_flags,
  149. struct nouveau_bo **pnvbo)
  150. {
  151. struct nouveau_drm *drm = cli->drm;
  152. struct nouveau_bo *nvbo;
  153. int ret;
  154. if (!(domain & (NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART)))
  155. domain |= NOUVEAU_GEM_DOMAIN_CPU;
  156. nvbo = nouveau_bo_alloc(cli, &size, &align, domain, tile_mode,
  157. tile_flags);
  158. if (IS_ERR(nvbo))
  159. return PTR_ERR(nvbo);
  160. /* Initialize the embedded gem-object. We return a single gem-reference
  161. * to the caller, instead of a normal nouveau_bo ttm reference. */
  162. ret = drm_gem_object_init(drm->dev, &nvbo->bo.base, size);
  163. if (ret) {
  164. drm_gem_object_release(&nvbo->bo.base);
  165. kfree(nvbo);
  166. return ret;
  167. }
  168. ret = nouveau_bo_init(nvbo, size, align, domain, NULL, NULL);
  169. if (ret)
  170. return ret;
  171. /* we restrict allowed domains on nv50+ to only the types
  172. * that were requested at creation time. not possibly on
  173. * earlier chips without busting the ABI.
  174. */
  175. nvbo->valid_domains = NOUVEAU_GEM_DOMAIN_VRAM |
  176. NOUVEAU_GEM_DOMAIN_GART;
  177. if (drm->client.device.info.family >= NV_DEVICE_INFO_V0_TESLA)
  178. nvbo->valid_domains &= domain;
  179. nvbo->bo.persistent_swap_storage = nvbo->bo.base.filp;
  180. *pnvbo = nvbo;
  181. return 0;
  182. }
  183. static int
  184. nouveau_gem_info(struct drm_file *file_priv, struct drm_gem_object *gem,
  185. struct drm_nouveau_gem_info *rep)
  186. {
  187. struct nouveau_cli *cli = nouveau_cli(file_priv);
  188. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  189. struct nouveau_vmm *vmm = cli->svm.cli ? &cli->svm : &cli->vmm;
  190. struct nouveau_vma *vma;
  191. if (is_power_of_2(nvbo->valid_domains))
  192. rep->domain = nvbo->valid_domains;
  193. else if (nvbo->bo.mem.mem_type == TTM_PL_TT)
  194. rep->domain = NOUVEAU_GEM_DOMAIN_GART;
  195. else
  196. rep->domain = NOUVEAU_GEM_DOMAIN_VRAM;
  197. rep->offset = nvbo->offset;
  198. if (vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
  199. vma = nouveau_vma_find(nvbo, vmm);
  200. if (!vma)
  201. return -EINVAL;
  202. rep->offset = vma->addr;
  203. }
  204. rep->size = nvbo->bo.mem.num_pages << PAGE_SHIFT;
  205. rep->map_handle = drm_vma_node_offset_addr(&nvbo->bo.base.vma_node);
  206. rep->tile_mode = nvbo->mode;
  207. rep->tile_flags = nvbo->contig ? 0 : NOUVEAU_GEM_TILE_NONCONTIG;
  208. if (cli->device.info.family >= NV_DEVICE_INFO_V0_FERMI)
  209. rep->tile_flags |= nvbo->kind << 8;
  210. else
  211. if (cli->device.info.family >= NV_DEVICE_INFO_V0_TESLA)
  212. rep->tile_flags |= nvbo->kind << 8 | nvbo->comp << 16;
  213. else
  214. rep->tile_flags |= nvbo->zeta;
  215. return 0;
  216. }
  217. int
  218. nouveau_gem_ioctl_new(struct drm_device *dev, void *data,
  219. struct drm_file *file_priv)
  220. {
  221. struct nouveau_cli *cli = nouveau_cli(file_priv);
  222. struct drm_nouveau_gem_new *req = data;
  223. struct nouveau_bo *nvbo = NULL;
  224. int ret = 0;
  225. ret = nouveau_gem_new(cli, req->info.size, req->align,
  226. req->info.domain, req->info.tile_mode,
  227. req->info.tile_flags, &nvbo);
  228. if (ret)
  229. return ret;
  230. ret = drm_gem_handle_create(file_priv, &nvbo->bo.base,
  231. &req->info.handle);
  232. if (ret == 0) {
  233. ret = nouveau_gem_info(file_priv, &nvbo->bo.base, &req->info);
  234. if (ret)
  235. drm_gem_handle_delete(file_priv, req->info.handle);
  236. }
  237. /* drop reference from allocate - handle holds it now */
  238. drm_gem_object_put(&nvbo->bo.base);
  239. return ret;
  240. }
  241. static int
  242. nouveau_gem_set_domain(struct drm_gem_object *gem, uint32_t read_domains,
  243. uint32_t write_domains, uint32_t valid_domains)
  244. {
  245. struct nouveau_bo *nvbo = nouveau_gem_object(gem);
  246. struct ttm_buffer_object *bo = &nvbo->bo;
  247. uint32_t domains = valid_domains & nvbo->valid_domains &
  248. (write_domains ? write_domains : read_domains);
  249. uint32_t pref_domains = 0;;
  250. if (!domains)
  251. return -EINVAL;
  252. valid_domains &= ~(NOUVEAU_GEM_DOMAIN_VRAM | NOUVEAU_GEM_DOMAIN_GART);
  253. if ((domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
  254. bo->mem.mem_type == TTM_PL_VRAM)
  255. pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
  256. else if ((domains & NOUVEAU_GEM_DOMAIN_GART) &&
  257. bo->mem.mem_type == TTM_PL_TT)
  258. pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
  259. else if (domains & NOUVEAU_GEM_DOMAIN_VRAM)
  260. pref_domains |= NOUVEAU_GEM_DOMAIN_VRAM;
  261. else
  262. pref_domains |= NOUVEAU_GEM_DOMAIN_GART;
  263. nouveau_bo_placement_set(nvbo, pref_domains, valid_domains);
  264. return 0;
  265. }
  266. struct validate_op {
  267. struct list_head list;
  268. struct ww_acquire_ctx ticket;
  269. };
  270. static void
  271. validate_fini_no_ticket(struct validate_op *op, struct nouveau_channel *chan,
  272. struct nouveau_fence *fence,
  273. struct drm_nouveau_gem_pushbuf_bo *pbbo)
  274. {
  275. struct nouveau_bo *nvbo;
  276. struct drm_nouveau_gem_pushbuf_bo *b;
  277. while (!list_empty(&op->list)) {
  278. nvbo = list_entry(op->list.next, struct nouveau_bo, entry);
  279. b = &pbbo[nvbo->pbbo_index];
  280. if (likely(fence)) {
  281. nouveau_bo_fence(nvbo, fence, !!b->write_domains);
  282. if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
  283. struct nouveau_vma *vma =
  284. (void *)(unsigned long)b->user_priv;
  285. nouveau_fence_unref(&vma->fence);
  286. dma_fence_get(&fence->base);
  287. vma->fence = fence;
  288. }
  289. }
  290. if (unlikely(nvbo->validate_mapped)) {
  291. ttm_bo_kunmap(&nvbo->kmap);
  292. nvbo->validate_mapped = false;
  293. }
  294. list_del(&nvbo->entry);
  295. nvbo->reserved_by = NULL;
  296. ttm_bo_unreserve(&nvbo->bo);
  297. drm_gem_object_put(&nvbo->bo.base);
  298. }
  299. }
  300. static void
  301. validate_fini(struct validate_op *op, struct nouveau_channel *chan,
  302. struct nouveau_fence *fence,
  303. struct drm_nouveau_gem_pushbuf_bo *pbbo)
  304. {
  305. validate_fini_no_ticket(op, chan, fence, pbbo);
  306. ww_acquire_fini(&op->ticket);
  307. }
  308. static int
  309. validate_init(struct nouveau_channel *chan, struct drm_file *file_priv,
  310. struct drm_nouveau_gem_pushbuf_bo *pbbo,
  311. int nr_buffers, struct validate_op *op)
  312. {
  313. struct nouveau_cli *cli = nouveau_cli(file_priv);
  314. int trycnt = 0;
  315. int ret = -EINVAL, i;
  316. struct nouveau_bo *res_bo = NULL;
  317. LIST_HEAD(gart_list);
  318. LIST_HEAD(vram_list);
  319. LIST_HEAD(both_list);
  320. ww_acquire_init(&op->ticket, &reservation_ww_class);
  321. retry:
  322. if (++trycnt > 100000) {
  323. NV_PRINTK(err, cli, "%s failed and gave up.\n", __func__);
  324. return -EINVAL;
  325. }
  326. for (i = 0; i < nr_buffers; i++) {
  327. struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[i];
  328. struct drm_gem_object *gem;
  329. struct nouveau_bo *nvbo;
  330. gem = drm_gem_object_lookup(file_priv, b->handle);
  331. if (!gem) {
  332. NV_PRINTK(err, cli, "Unknown handle 0x%08x\n", b->handle);
  333. ret = -ENOENT;
  334. break;
  335. }
  336. nvbo = nouveau_gem_object(gem);
  337. if (nvbo == res_bo) {
  338. res_bo = NULL;
  339. drm_gem_object_put(gem);
  340. continue;
  341. }
  342. if (nvbo->reserved_by && nvbo->reserved_by == file_priv) {
  343. NV_PRINTK(err, cli, "multiple instances of buffer %d on "
  344. "validation list\n", b->handle);
  345. drm_gem_object_put(gem);
  346. ret = -EINVAL;
  347. break;
  348. }
  349. ret = ttm_bo_reserve(&nvbo->bo, true, false, &op->ticket);
  350. if (ret) {
  351. list_splice_tail_init(&vram_list, &op->list);
  352. list_splice_tail_init(&gart_list, &op->list);
  353. list_splice_tail_init(&both_list, &op->list);
  354. validate_fini_no_ticket(op, chan, NULL, NULL);
  355. if (unlikely(ret == -EDEADLK)) {
  356. ret = ttm_bo_reserve_slowpath(&nvbo->bo, true,
  357. &op->ticket);
  358. if (!ret)
  359. res_bo = nvbo;
  360. }
  361. if (unlikely(ret)) {
  362. if (ret != -ERESTARTSYS)
  363. NV_PRINTK(err, cli, "fail reserve\n");
  364. break;
  365. }
  366. }
  367. if (chan->vmm->vmm.object.oclass >= NVIF_CLASS_VMM_NV50) {
  368. struct nouveau_vmm *vmm = chan->vmm;
  369. struct nouveau_vma *vma = nouveau_vma_find(nvbo, vmm);
  370. if (!vma) {
  371. NV_PRINTK(err, cli, "vma not found!\n");
  372. ret = -EINVAL;
  373. break;
  374. }
  375. b->user_priv = (uint64_t)(unsigned long)vma;
  376. } else {
  377. b->user_priv = (uint64_t)(unsigned long)nvbo;
  378. }
  379. nvbo->reserved_by = file_priv;
  380. nvbo->pbbo_index = i;
  381. if ((b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM) &&
  382. (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART))
  383. list_add_tail(&nvbo->entry, &both_list);
  384. else
  385. if (b->valid_domains & NOUVEAU_GEM_DOMAIN_VRAM)
  386. list_add_tail(&nvbo->entry, &vram_list);
  387. else
  388. if (b->valid_domains & NOUVEAU_GEM_DOMAIN_GART)
  389. list_add_tail(&nvbo->entry, &gart_list);
  390. else {
  391. NV_PRINTK(err, cli, "invalid valid domains: 0x%08x\n",
  392. b->valid_domains);
  393. list_add_tail(&nvbo->entry, &both_list);
  394. ret = -EINVAL;
  395. break;
  396. }
  397. if (nvbo == res_bo)
  398. goto retry;
  399. }
  400. ww_acquire_done(&op->ticket);
  401. list_splice_tail(&vram_list, &op->list);
  402. list_splice_tail(&gart_list, &op->list);
  403. list_splice_tail(&both_list, &op->list);
  404. if (ret)
  405. validate_fini(op, chan, NULL, NULL);
  406. return ret;
  407. }
  408. static int
  409. validate_list(struct nouveau_channel *chan, struct nouveau_cli *cli,
  410. struct list_head *list, struct drm_nouveau_gem_pushbuf_bo *pbbo)
  411. {
  412. struct nouveau_drm *drm = chan->drm;
  413. struct nouveau_bo *nvbo;
  414. int ret, relocs = 0;
  415. list_for_each_entry(nvbo, list, entry) {
  416. struct drm_nouveau_gem_pushbuf_bo *b = &pbbo[nvbo->pbbo_index];
  417. ret = nouveau_gem_set_domain(&nvbo->bo.base, b->read_domains,
  418. b->write_domains,
  419. b->valid_domains);
  420. if (unlikely(ret)) {
  421. NV_PRINTK(err, cli, "fail set_domain\n");
  422. return ret;
  423. }
  424. ret = nouveau_bo_validate(nvbo, true, false);
  425. if (unlikely(ret)) {
  426. if (ret != -ERESTARTSYS)
  427. NV_PRINTK(err, cli, "fail ttm_validate\n");
  428. return ret;
  429. }
  430. ret = nouveau_fence_sync(nvbo, chan, !!b->write_domains, true);
  431. if (unlikely(ret)) {
  432. if (ret != -ERESTARTSYS)
  433. NV_PRINTK(err, cli, "fail post-validate sync\n");
  434. return ret;
  435. }
  436. if (drm->client.device.info.family < NV_DEVICE_INFO_V0_TESLA) {
  437. if (nvbo->offset == b->presumed.offset &&
  438. ((nvbo->bo.mem.mem_type == TTM_PL_VRAM &&
  439. b->presumed.domain & NOUVEAU_GEM_DOMAIN_VRAM) ||
  440. (nvbo->bo.mem.mem_type == TTM_PL_TT &&
  441. b->presumed.domain & NOUVEAU_GEM_DOMAIN_GART)))
  442. continue;
  443. if (nvbo->bo.mem.mem_type == TTM_PL_TT)
  444. b->presumed.domain = NOUVEAU_GEM_DOMAIN_GART;
  445. else
  446. b->presumed.domain = NOUVEAU_GEM_DOMAIN_VRAM;
  447. b->presumed.offset = nvbo->offset;
  448. b->presumed.valid = 0;
  449. relocs++;
  450. }
  451. }
  452. return relocs;
  453. }
  454. static int
  455. nouveau_gem_pushbuf_validate(struct nouveau_channel *chan,
  456. struct drm_file *file_priv,
  457. struct drm_nouveau_gem_pushbuf_bo *pbbo,
  458. int nr_buffers,
  459. struct validate_op *op, bool *apply_relocs)
  460. {
  461. struct nouveau_cli *cli = nouveau_cli(file_priv);
  462. int ret;
  463. INIT_LIST_HEAD(&op->list);
  464. if (nr_buffers == 0)
  465. return 0;
  466. ret = validate_init(chan, file_priv, pbbo, nr_buffers, op);
  467. if (unlikely(ret)) {
  468. if (ret != -ERESTARTSYS)
  469. NV_PRINTK(err, cli, "validate_init\n");
  470. return ret;
  471. }
  472. ret = validate_list(chan, cli, &op->list, pbbo);
  473. if (unlikely(ret < 0)) {
  474. if (ret != -ERESTARTSYS)
  475. NV_PRINTK(err, cli, "validating bo list\n");
  476. validate_fini(op, chan, NULL, NULL);
  477. return ret;
  478. } else if (ret > 0) {
  479. *apply_relocs = true;
  480. }
  481. return 0;
  482. }
  483. static inline void
  484. u_free(void *addr)
  485. {
  486. kvfree(addr);
  487. }
  488. static inline void *
  489. u_memcpya(uint64_t user, unsigned nmemb, unsigned size)
  490. {
  491. void *mem;
  492. void __user *userptr = (void __force __user *)(uintptr_t)user;
  493. size *= nmemb;
  494. mem = kvmalloc(size, GFP_KERNEL);
  495. if (!mem)
  496. return ERR_PTR(-ENOMEM);
  497. if (copy_from_user(mem, userptr, size)) {
  498. u_free(mem);
  499. return ERR_PTR(-EFAULT);
  500. }
  501. return mem;
  502. }
  503. static int
  504. nouveau_gem_pushbuf_reloc_apply(struct nouveau_cli *cli,
  505. struct drm_nouveau_gem_pushbuf *req,
  506. struct drm_nouveau_gem_pushbuf_reloc *reloc,
  507. struct drm_nouveau_gem_pushbuf_bo *bo)
  508. {
  509. int ret = 0;
  510. unsigned i;
  511. for (i = 0; i < req->nr_relocs; i++) {
  512. struct drm_nouveau_gem_pushbuf_reloc *r = &reloc[i];
  513. struct drm_nouveau_gem_pushbuf_bo *b;
  514. struct nouveau_bo *nvbo;
  515. uint32_t data;
  516. if (unlikely(r->bo_index >= req->nr_buffers)) {
  517. NV_PRINTK(err, cli, "reloc bo index invalid\n");
  518. ret = -EINVAL;
  519. break;
  520. }
  521. b = &bo[r->bo_index];
  522. if (b->presumed.valid)
  523. continue;
  524. if (unlikely(r->reloc_bo_index >= req->nr_buffers)) {
  525. NV_PRINTK(err, cli, "reloc container bo index invalid\n");
  526. ret = -EINVAL;
  527. break;
  528. }
  529. nvbo = (void *)(unsigned long)bo[r->reloc_bo_index].user_priv;
  530. if (unlikely(r->reloc_bo_offset + 4 >
  531. nvbo->bo.mem.num_pages << PAGE_SHIFT)) {
  532. NV_PRINTK(err, cli, "reloc outside of bo\n");
  533. ret = -EINVAL;
  534. break;
  535. }
  536. if (!nvbo->kmap.virtual) {
  537. ret = ttm_bo_kmap(&nvbo->bo, 0, nvbo->bo.mem.num_pages,
  538. &nvbo->kmap);
  539. if (ret) {
  540. NV_PRINTK(err, cli, "failed kmap for reloc\n");
  541. break;
  542. }
  543. nvbo->validate_mapped = true;
  544. }
  545. if (r->flags & NOUVEAU_GEM_RELOC_LOW)
  546. data = b->presumed.offset + r->data;
  547. else
  548. if (r->flags & NOUVEAU_GEM_RELOC_HIGH)
  549. data = (b->presumed.offset + r->data) >> 32;
  550. else
  551. data = r->data;
  552. if (r->flags & NOUVEAU_GEM_RELOC_OR) {
  553. if (b->presumed.domain == NOUVEAU_GEM_DOMAIN_GART)
  554. data |= r->tor;
  555. else
  556. data |= r->vor;
  557. }
  558. ret = ttm_bo_wait(&nvbo->bo, false, false);
  559. if (ret) {
  560. NV_PRINTK(err, cli, "reloc wait_idle failed: %d\n", ret);
  561. break;
  562. }
  563. nouveau_bo_wr32(nvbo, r->reloc_bo_offset >> 2, data);
  564. }
  565. return ret;
  566. }
  567. int
  568. nouveau_gem_ioctl_pushbuf(struct drm_device *dev, void *data,
  569. struct drm_file *file_priv)
  570. {
  571. struct nouveau_abi16 *abi16 = nouveau_abi16_get(file_priv);
  572. struct nouveau_cli *cli = nouveau_cli(file_priv);
  573. struct nouveau_abi16_chan *temp;
  574. struct nouveau_drm *drm = nouveau_drm(dev);
  575. struct drm_nouveau_gem_pushbuf *req = data;
  576. struct drm_nouveau_gem_pushbuf_push *push;
  577. struct drm_nouveau_gem_pushbuf_reloc *reloc = NULL;
  578. struct drm_nouveau_gem_pushbuf_bo *bo;
  579. struct nouveau_channel *chan = NULL;
  580. struct validate_op op;
  581. struct nouveau_fence *fence = NULL;
  582. int i, j, ret = 0;
  583. bool do_reloc = false, sync = false;
  584. if (unlikely(!abi16))
  585. return -ENOMEM;
  586. list_for_each_entry(temp, &abi16->channels, head) {
  587. if (temp->chan->chid == req->channel) {
  588. chan = temp->chan;
  589. break;
  590. }
  591. }
  592. if (!chan)
  593. return nouveau_abi16_put(abi16, -ENOENT);
  594. if (unlikely(atomic_read(&chan->killed)))
  595. return nouveau_abi16_put(abi16, -ENODEV);
  596. sync = req->vram_available & NOUVEAU_GEM_PUSHBUF_SYNC;
  597. req->vram_available = drm->gem.vram_available;
  598. req->gart_available = drm->gem.gart_available;
  599. if (unlikely(req->nr_push == 0))
  600. goto out_next;
  601. if (unlikely(req->nr_push > NOUVEAU_GEM_MAX_PUSH)) {
  602. NV_PRINTK(err, cli, "pushbuf push count exceeds limit: %d max %d\n",
  603. req->nr_push, NOUVEAU_GEM_MAX_PUSH);
  604. return nouveau_abi16_put(abi16, -EINVAL);
  605. }
  606. if (unlikely(req->nr_buffers > NOUVEAU_GEM_MAX_BUFFERS)) {
  607. NV_PRINTK(err, cli, "pushbuf bo count exceeds limit: %d max %d\n",
  608. req->nr_buffers, NOUVEAU_GEM_MAX_BUFFERS);
  609. return nouveau_abi16_put(abi16, -EINVAL);
  610. }
  611. if (unlikely(req->nr_relocs > NOUVEAU_GEM_MAX_RELOCS)) {
  612. NV_PRINTK(err, cli, "pushbuf reloc count exceeds limit: %d max %d\n",
  613. req->nr_relocs, NOUVEAU_GEM_MAX_RELOCS);
  614. return nouveau_abi16_put(abi16, -EINVAL);
  615. }
  616. push = u_memcpya(req->push, req->nr_push, sizeof(*push));
  617. if (IS_ERR(push))
  618. return nouveau_abi16_put(abi16, PTR_ERR(push));
  619. bo = u_memcpya(req->buffers, req->nr_buffers, sizeof(*bo));
  620. if (IS_ERR(bo)) {
  621. u_free(push);
  622. return nouveau_abi16_put(abi16, PTR_ERR(bo));
  623. }
  624. /* Ensure all push buffers are on validate list */
  625. for (i = 0; i < req->nr_push; i++) {
  626. if (push[i].bo_index >= req->nr_buffers) {
  627. NV_PRINTK(err, cli, "push %d buffer not in list\n", i);
  628. ret = -EINVAL;
  629. goto out_prevalid;
  630. }
  631. }
  632. /* Validate buffer list */
  633. revalidate:
  634. ret = nouveau_gem_pushbuf_validate(chan, file_priv, bo,
  635. req->nr_buffers, &op, &do_reloc);
  636. if (ret) {
  637. if (ret != -ERESTARTSYS)
  638. NV_PRINTK(err, cli, "validate: %d\n", ret);
  639. goto out_prevalid;
  640. }
  641. /* Apply any relocations that are required */
  642. if (do_reloc) {
  643. if (!reloc) {
  644. validate_fini(&op, chan, NULL, bo);
  645. reloc = u_memcpya(req->relocs, req->nr_relocs, sizeof(*reloc));
  646. if (IS_ERR(reloc)) {
  647. ret = PTR_ERR(reloc);
  648. goto out_prevalid;
  649. }
  650. goto revalidate;
  651. }
  652. ret = nouveau_gem_pushbuf_reloc_apply(cli, req, reloc, bo);
  653. if (ret) {
  654. NV_PRINTK(err, cli, "reloc apply: %d\n", ret);
  655. goto out;
  656. }
  657. }
  658. if (chan->dma.ib_max) {
  659. ret = nouveau_dma_wait(chan, req->nr_push + 1, 16);
  660. if (ret) {
  661. NV_PRINTK(err, cli, "nv50cal_space: %d\n", ret);
  662. goto out;
  663. }
  664. for (i = 0; i < req->nr_push; i++) {
  665. struct nouveau_vma *vma = (void *)(unsigned long)
  666. bo[push[i].bo_index].user_priv;
  667. nv50_dma_push(chan, vma->addr + push[i].offset,
  668. push[i].length);
  669. }
  670. } else
  671. if (drm->client.device.info.chipset >= 0x25) {
  672. ret = PUSH_WAIT(chan->chan.push, req->nr_push * 2);
  673. if (ret) {
  674. NV_PRINTK(err, cli, "cal_space: %d\n", ret);
  675. goto out;
  676. }
  677. for (i = 0; i < req->nr_push; i++) {
  678. struct nouveau_bo *nvbo = (void *)(unsigned long)
  679. bo[push[i].bo_index].user_priv;
  680. PUSH_CALL(chan->chan.push, nvbo->offset + push[i].offset);
  681. PUSH_DATA(chan->chan.push, 0);
  682. }
  683. } else {
  684. ret = PUSH_WAIT(chan->chan.push, req->nr_push * (2 + NOUVEAU_DMA_SKIPS));
  685. if (ret) {
  686. NV_PRINTK(err, cli, "jmp_space: %d\n", ret);
  687. goto out;
  688. }
  689. for (i = 0; i < req->nr_push; i++) {
  690. struct nouveau_bo *nvbo = (void *)(unsigned long)
  691. bo[push[i].bo_index].user_priv;
  692. uint32_t cmd;
  693. cmd = chan->push.addr + ((chan->dma.cur + 2) << 2);
  694. cmd |= 0x20000000;
  695. if (unlikely(cmd != req->suffix0)) {
  696. if (!nvbo->kmap.virtual) {
  697. ret = ttm_bo_kmap(&nvbo->bo, 0,
  698. nvbo->bo.mem.
  699. num_pages,
  700. &nvbo->kmap);
  701. if (ret) {
  702. WIND_RING(chan);
  703. goto out;
  704. }
  705. nvbo->validate_mapped = true;
  706. }
  707. nouveau_bo_wr32(nvbo, (push[i].offset +
  708. push[i].length - 8) / 4, cmd);
  709. }
  710. PUSH_JUMP(chan->chan.push, nvbo->offset + push[i].offset);
  711. PUSH_DATA(chan->chan.push, 0);
  712. for (j = 0; j < NOUVEAU_DMA_SKIPS; j++)
  713. PUSH_DATA(chan->chan.push, 0);
  714. }
  715. }
  716. ret = nouveau_fence_new(chan, false, &fence);
  717. if (ret) {
  718. NV_PRINTK(err, cli, "error fencing pushbuf: %d\n", ret);
  719. WIND_RING(chan);
  720. goto out;
  721. }
  722. if (sync) {
  723. if (!(ret = nouveau_fence_wait(fence, false, false))) {
  724. if ((ret = dma_fence_get_status(&fence->base)) == 1)
  725. ret = 0;
  726. }
  727. }
  728. out:
  729. validate_fini(&op, chan, fence, bo);
  730. nouveau_fence_unref(&fence);
  731. if (do_reloc) {
  732. struct drm_nouveau_gem_pushbuf_bo __user *upbbo =
  733. u64_to_user_ptr(req->buffers);
  734. for (i = 0; i < req->nr_buffers; i++) {
  735. if (bo[i].presumed.valid)
  736. continue;
  737. if (copy_to_user(&upbbo[i].presumed, &bo[i].presumed,
  738. sizeof(bo[i].presumed))) {
  739. ret = -EFAULT;
  740. break;
  741. }
  742. }
  743. }
  744. out_prevalid:
  745. if (!IS_ERR(reloc))
  746. u_free(reloc);
  747. u_free(bo);
  748. u_free(push);
  749. out_next:
  750. if (chan->dma.ib_max) {
  751. req->suffix0 = 0x00000000;
  752. req->suffix1 = 0x00000000;
  753. } else
  754. if (drm->client.device.info.chipset >= 0x25) {
  755. req->suffix0 = 0x00020000;
  756. req->suffix1 = 0x00000000;
  757. } else {
  758. req->suffix0 = 0x20000000 |
  759. (chan->push.addr + ((chan->dma.cur + 2) << 2));
  760. req->suffix1 = 0x00000000;
  761. }
  762. return nouveau_abi16_put(abi16, ret);
  763. }
  764. int
  765. nouveau_gem_ioctl_cpu_prep(struct drm_device *dev, void *data,
  766. struct drm_file *file_priv)
  767. {
  768. struct drm_nouveau_gem_cpu_prep *req = data;
  769. struct drm_gem_object *gem;
  770. struct nouveau_bo *nvbo;
  771. bool no_wait = !!(req->flags & NOUVEAU_GEM_CPU_PREP_NOWAIT);
  772. bool write = !!(req->flags & NOUVEAU_GEM_CPU_PREP_WRITE);
  773. long lret;
  774. int ret;
  775. gem = drm_gem_object_lookup(file_priv, req->handle);
  776. if (!gem)
  777. return -ENOENT;
  778. nvbo = nouveau_gem_object(gem);
  779. lret = dma_resv_wait_timeout_rcu(nvbo->bo.base.resv, write, true,
  780. no_wait ? 0 : 30 * HZ);
  781. if (!lret)
  782. ret = -EBUSY;
  783. else if (lret > 0)
  784. ret = 0;
  785. else
  786. ret = lret;
  787. nouveau_bo_sync_for_cpu(nvbo);
  788. drm_gem_object_put(gem);
  789. return ret;
  790. }
  791. int
  792. nouveau_gem_ioctl_cpu_fini(struct drm_device *dev, void *data,
  793. struct drm_file *file_priv)
  794. {
  795. struct drm_nouveau_gem_cpu_fini *req = data;
  796. struct drm_gem_object *gem;
  797. struct nouveau_bo *nvbo;
  798. gem = drm_gem_object_lookup(file_priv, req->handle);
  799. if (!gem)
  800. return -ENOENT;
  801. nvbo = nouveau_gem_object(gem);
  802. nouveau_bo_sync_for_device(nvbo);
  803. drm_gem_object_put(gem);
  804. return 0;
  805. }
  806. int
  807. nouveau_gem_ioctl_info(struct drm_device *dev, void *data,
  808. struct drm_file *file_priv)
  809. {
  810. struct drm_nouveau_gem_info *req = data;
  811. struct drm_gem_object *gem;
  812. int ret;
  813. gem = drm_gem_object_lookup(file_priv, req->handle);
  814. if (!gem)
  815. return -ENOENT;
  816. ret = nouveau_gem_info(file_priv, gem, req);
  817. drm_gem_object_put(gem);
  818. return ret;
  819. }