vmwgfx_resource.c 31 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176
  1. // SPDX-License-Identifier: GPL-2.0 OR MIT
  2. /**************************************************************************
  3. *
  4. * Copyright 2009-2015 VMware, Inc., Palo Alto, CA., USA
  5. *
  6. * Permission is hereby granted, free of charge, to any person obtaining a
  7. * copy of this software and associated documentation files (the
  8. * "Software"), to deal in the Software without restriction, including
  9. * without limitation the rights to use, copy, modify, merge, publish,
  10. * distribute, sub license, and/or sell copies of the Software, and to
  11. * permit persons to whom the Software is furnished to do so, subject to
  12. * the following conditions:
  13. *
  14. * The above copyright notice and this permission notice (including the
  15. * next paragraph) shall be included in all copies or substantial portions
  16. * of the Software.
  17. *
  18. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
  19. * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
  20. * FITNESS FOR A PARTICULAR PURPOSE AND NON-INFRINGEMENT. IN NO EVENT SHALL
  21. * THE COPYRIGHT HOLDERS, AUTHORS AND/OR ITS SUPPLIERS BE LIABLE FOR ANY CLAIM,
  22. * DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
  23. * OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE
  24. * USE OR OTHER DEALINGS IN THE SOFTWARE.
  25. *
  26. **************************************************************************/
  27. #include <drm/ttm/ttm_placement.h>
  28. #include "vmwgfx_resource_priv.h"
  29. #include "vmwgfx_binding.h"
  30. #include "vmwgfx_drv.h"
  31. #define VMW_RES_EVICT_ERR_COUNT 10
  32. /**
  33. * vmw_resource_mob_attach - Mark a resource as attached to its backing mob
  34. * @res: The resource
  35. */
  36. void vmw_resource_mob_attach(struct vmw_resource *res)
  37. {
  38. struct vmw_buffer_object *backup = res->backup;
  39. struct rb_node **new = &backup->res_tree.rb_node, *parent = NULL;
  40. dma_resv_assert_held(res->backup->base.base.resv);
  41. res->used_prio = (res->res_dirty) ? res->func->dirty_prio :
  42. res->func->prio;
  43. while (*new) {
  44. struct vmw_resource *this =
  45. container_of(*new, struct vmw_resource, mob_node);
  46. parent = *new;
  47. new = (res->backup_offset < this->backup_offset) ?
  48. &((*new)->rb_left) : &((*new)->rb_right);
  49. }
  50. rb_link_node(&res->mob_node, parent, new);
  51. rb_insert_color(&res->mob_node, &backup->res_tree);
  52. vmw_bo_prio_add(backup, res->used_prio);
  53. }
  54. /**
  55. * vmw_resource_mob_detach - Mark a resource as detached from its backing mob
  56. * @res: The resource
  57. */
  58. void vmw_resource_mob_detach(struct vmw_resource *res)
  59. {
  60. struct vmw_buffer_object *backup = res->backup;
  61. dma_resv_assert_held(backup->base.base.resv);
  62. if (vmw_resource_mob_attached(res)) {
  63. rb_erase(&res->mob_node, &backup->res_tree);
  64. RB_CLEAR_NODE(&res->mob_node);
  65. vmw_bo_prio_del(backup, res->used_prio);
  66. }
  67. }
  68. struct vmw_resource *vmw_resource_reference(struct vmw_resource *res)
  69. {
  70. kref_get(&res->kref);
  71. return res;
  72. }
  73. struct vmw_resource *
  74. vmw_resource_reference_unless_doomed(struct vmw_resource *res)
  75. {
  76. return kref_get_unless_zero(&res->kref) ? res : NULL;
  77. }
  78. /**
  79. * vmw_resource_release_id - release a resource id to the id manager.
  80. *
  81. * @res: Pointer to the resource.
  82. *
  83. * Release the resource id to the resource id manager and set it to -1
  84. */
  85. void vmw_resource_release_id(struct vmw_resource *res)
  86. {
  87. struct vmw_private *dev_priv = res->dev_priv;
  88. struct idr *idr = &dev_priv->res_idr[res->func->res_type];
  89. spin_lock(&dev_priv->resource_lock);
  90. if (res->id != -1)
  91. idr_remove(idr, res->id);
  92. res->id = -1;
  93. spin_unlock(&dev_priv->resource_lock);
  94. }
  95. static void vmw_resource_release(struct kref *kref)
  96. {
  97. struct vmw_resource *res =
  98. container_of(kref, struct vmw_resource, kref);
  99. struct vmw_private *dev_priv = res->dev_priv;
  100. int id;
  101. struct idr *idr = &dev_priv->res_idr[res->func->res_type];
  102. spin_lock(&dev_priv->resource_lock);
  103. list_del_init(&res->lru_head);
  104. spin_unlock(&dev_priv->resource_lock);
  105. if (res->backup) {
  106. struct ttm_buffer_object *bo = &res->backup->base;
  107. ttm_bo_reserve(bo, false, false, NULL);
  108. if (vmw_resource_mob_attached(res) &&
  109. res->func->unbind != NULL) {
  110. struct ttm_validate_buffer val_buf;
  111. val_buf.bo = bo;
  112. val_buf.num_shared = 0;
  113. res->func->unbind(res, false, &val_buf);
  114. }
  115. res->backup_dirty = false;
  116. vmw_resource_mob_detach(res);
  117. if (res->dirty)
  118. res->func->dirty_free(res);
  119. if (res->coherent)
  120. vmw_bo_dirty_release(res->backup);
  121. ttm_bo_unreserve(bo);
  122. vmw_bo_unreference(&res->backup);
  123. }
  124. if (likely(res->hw_destroy != NULL)) {
  125. mutex_lock(&dev_priv->binding_mutex);
  126. vmw_binding_res_list_kill(&res->binding_head);
  127. mutex_unlock(&dev_priv->binding_mutex);
  128. res->hw_destroy(res);
  129. }
  130. id = res->id;
  131. if (res->res_free != NULL)
  132. res->res_free(res);
  133. else
  134. kfree(res);
  135. spin_lock(&dev_priv->resource_lock);
  136. if (id != -1)
  137. idr_remove(idr, id);
  138. spin_unlock(&dev_priv->resource_lock);
  139. }
  140. void vmw_resource_unreference(struct vmw_resource **p_res)
  141. {
  142. struct vmw_resource *res = *p_res;
  143. *p_res = NULL;
  144. kref_put(&res->kref, vmw_resource_release);
  145. }
  146. /**
  147. * vmw_resource_alloc_id - release a resource id to the id manager.
  148. *
  149. * @res: Pointer to the resource.
  150. *
  151. * Allocate the lowest free resource from the resource manager, and set
  152. * @res->id to that id. Returns 0 on success and -ENOMEM on failure.
  153. */
  154. int vmw_resource_alloc_id(struct vmw_resource *res)
  155. {
  156. struct vmw_private *dev_priv = res->dev_priv;
  157. int ret;
  158. struct idr *idr = &dev_priv->res_idr[res->func->res_type];
  159. BUG_ON(res->id != -1);
  160. idr_preload(GFP_KERNEL);
  161. spin_lock(&dev_priv->resource_lock);
  162. ret = idr_alloc(idr, res, 1, 0, GFP_NOWAIT);
  163. if (ret >= 0)
  164. res->id = ret;
  165. spin_unlock(&dev_priv->resource_lock);
  166. idr_preload_end();
  167. return ret < 0 ? ret : 0;
  168. }
  169. /**
  170. * vmw_resource_init - initialize a struct vmw_resource
  171. *
  172. * @dev_priv: Pointer to a device private struct.
  173. * @res: The struct vmw_resource to initialize.
  174. * @obj_type: Resource object type.
  175. * @delay_id: Boolean whether to defer device id allocation until
  176. * the first validation.
  177. * @res_free: Resource destructor.
  178. * @func: Resource function table.
  179. */
  180. int vmw_resource_init(struct vmw_private *dev_priv, struct vmw_resource *res,
  181. bool delay_id,
  182. void (*res_free) (struct vmw_resource *res),
  183. const struct vmw_res_func *func)
  184. {
  185. kref_init(&res->kref);
  186. res->hw_destroy = NULL;
  187. res->res_free = res_free;
  188. res->dev_priv = dev_priv;
  189. res->func = func;
  190. RB_CLEAR_NODE(&res->mob_node);
  191. INIT_LIST_HEAD(&res->lru_head);
  192. INIT_LIST_HEAD(&res->binding_head);
  193. res->id = -1;
  194. res->backup = NULL;
  195. res->backup_offset = 0;
  196. res->backup_dirty = false;
  197. res->res_dirty = false;
  198. res->coherent = false;
  199. res->used_prio = 3;
  200. res->dirty = NULL;
  201. if (delay_id)
  202. return 0;
  203. else
  204. return vmw_resource_alloc_id(res);
  205. }
  206. /**
  207. * vmw_user_resource_lookup_handle - lookup a struct resource from a
  208. * TTM user-space handle and perform basic type checks
  209. *
  210. * @dev_priv: Pointer to a device private struct
  211. * @tfile: Pointer to a struct ttm_object_file identifying the caller
  212. * @handle: The TTM user-space handle
  213. * @converter: Pointer to an object describing the resource type
  214. * @p_res: On successful return the location pointed to will contain
  215. * a pointer to a refcounted struct vmw_resource.
  216. *
  217. * If the handle can't be found or is associated with an incorrect resource
  218. * type, -EINVAL will be returned.
  219. */
  220. int vmw_user_resource_lookup_handle(struct vmw_private *dev_priv,
  221. struct ttm_object_file *tfile,
  222. uint32_t handle,
  223. const struct vmw_user_resource_conv
  224. *converter,
  225. struct vmw_resource **p_res)
  226. {
  227. struct ttm_base_object *base;
  228. struct vmw_resource *res;
  229. int ret = -EINVAL;
  230. base = ttm_base_object_lookup(tfile, handle);
  231. if (unlikely(base == NULL))
  232. return -EINVAL;
  233. if (unlikely(ttm_base_object_type(base) != converter->object_type))
  234. goto out_bad_resource;
  235. res = converter->base_obj_to_res(base);
  236. kref_get(&res->kref);
  237. *p_res = res;
  238. ret = 0;
  239. out_bad_resource:
  240. ttm_base_object_unref(&base);
  241. return ret;
  242. }
  243. /**
  244. * vmw_user_resource_lookup_handle - lookup a struct resource from a
  245. * TTM user-space handle and perform basic type checks
  246. *
  247. * @dev_priv: Pointer to a device private struct
  248. * @tfile: Pointer to a struct ttm_object_file identifying the caller
  249. * @handle: The TTM user-space handle
  250. * @converter: Pointer to an object describing the resource type
  251. * @p_res: On successful return the location pointed to will contain
  252. * a pointer to a refcounted struct vmw_resource.
  253. *
  254. * If the handle can't be found or is associated with an incorrect resource
  255. * type, -EINVAL will be returned.
  256. */
  257. struct vmw_resource *
  258. vmw_user_resource_noref_lookup_handle(struct vmw_private *dev_priv,
  259. struct ttm_object_file *tfile,
  260. uint32_t handle,
  261. const struct vmw_user_resource_conv
  262. *converter)
  263. {
  264. struct ttm_base_object *base;
  265. base = ttm_base_object_noref_lookup(tfile, handle);
  266. if (!base)
  267. return ERR_PTR(-ESRCH);
  268. if (unlikely(ttm_base_object_type(base) != converter->object_type)) {
  269. ttm_base_object_noref_release();
  270. return ERR_PTR(-EINVAL);
  271. }
  272. return converter->base_obj_to_res(base);
  273. }
  274. /**
  275. * Helper function that looks either a surface or bo.
  276. *
  277. * The pointer this pointed at by out_surf and out_buf needs to be null.
  278. */
  279. int vmw_user_lookup_handle(struct vmw_private *dev_priv,
  280. struct ttm_object_file *tfile,
  281. uint32_t handle,
  282. struct vmw_surface **out_surf,
  283. struct vmw_buffer_object **out_buf)
  284. {
  285. struct vmw_resource *res;
  286. int ret;
  287. BUG_ON(*out_surf || *out_buf);
  288. ret = vmw_user_resource_lookup_handle(dev_priv, tfile, handle,
  289. user_surface_converter,
  290. &res);
  291. if (!ret) {
  292. *out_surf = vmw_res_to_srf(res);
  293. return 0;
  294. }
  295. *out_surf = NULL;
  296. ret = vmw_user_bo_lookup(tfile, handle, out_buf, NULL);
  297. return ret;
  298. }
  299. /**
  300. * vmw_resource_buf_alloc - Allocate a backup buffer for a resource.
  301. *
  302. * @res: The resource for which to allocate a backup buffer.
  303. * @interruptible: Whether any sleeps during allocation should be
  304. * performed while interruptible.
  305. */
  306. static int vmw_resource_buf_alloc(struct vmw_resource *res,
  307. bool interruptible)
  308. {
  309. unsigned long size =
  310. (res->backup_size + PAGE_SIZE - 1) & PAGE_MASK;
  311. struct vmw_buffer_object *backup;
  312. int ret;
  313. if (likely(res->backup)) {
  314. BUG_ON(res->backup->base.num_pages * PAGE_SIZE < size);
  315. return 0;
  316. }
  317. backup = kzalloc(sizeof(*backup), GFP_KERNEL);
  318. if (unlikely(!backup))
  319. return -ENOMEM;
  320. ret = vmw_bo_init(res->dev_priv, backup, res->backup_size,
  321. res->func->backup_placement,
  322. interruptible,
  323. &vmw_bo_bo_free);
  324. if (unlikely(ret != 0))
  325. goto out_no_bo;
  326. res->backup = backup;
  327. out_no_bo:
  328. return ret;
  329. }
  330. /**
  331. * vmw_resource_do_validate - Make a resource up-to-date and visible
  332. * to the device.
  333. *
  334. * @res: The resource to make visible to the device.
  335. * @val_buf: Information about a buffer possibly
  336. * containing backup data if a bind operation is needed.
  337. *
  338. * On hardware resource shortage, this function returns -EBUSY and
  339. * should be retried once resources have been freed up.
  340. */
  341. static int vmw_resource_do_validate(struct vmw_resource *res,
  342. struct ttm_validate_buffer *val_buf,
  343. bool dirtying)
  344. {
  345. int ret = 0;
  346. const struct vmw_res_func *func = res->func;
  347. if (unlikely(res->id == -1)) {
  348. ret = func->create(res);
  349. if (unlikely(ret != 0))
  350. return ret;
  351. }
  352. if (func->bind &&
  353. ((func->needs_backup && !vmw_resource_mob_attached(res) &&
  354. val_buf->bo != NULL) ||
  355. (!func->needs_backup && val_buf->bo != NULL))) {
  356. ret = func->bind(res, val_buf);
  357. if (unlikely(ret != 0))
  358. goto out_bind_failed;
  359. if (func->needs_backup)
  360. vmw_resource_mob_attach(res);
  361. }
  362. /*
  363. * Handle the case where the backup mob is marked coherent but
  364. * the resource isn't.
  365. */
  366. if (func->dirty_alloc && vmw_resource_mob_attached(res) &&
  367. !res->coherent) {
  368. if (res->backup->dirty && !res->dirty) {
  369. ret = func->dirty_alloc(res);
  370. if (ret)
  371. return ret;
  372. } else if (!res->backup->dirty && res->dirty) {
  373. func->dirty_free(res);
  374. }
  375. }
  376. /*
  377. * Transfer the dirty regions to the resource and update
  378. * the resource.
  379. */
  380. if (res->dirty) {
  381. if (dirtying && !res->res_dirty) {
  382. pgoff_t start = res->backup_offset >> PAGE_SHIFT;
  383. pgoff_t end = __KERNEL_DIV_ROUND_UP
  384. (res->backup_offset + res->backup_size,
  385. PAGE_SIZE);
  386. vmw_bo_dirty_unmap(res->backup, start, end);
  387. }
  388. vmw_bo_dirty_transfer_to_res(res);
  389. return func->dirty_sync(res);
  390. }
  391. return 0;
  392. out_bind_failed:
  393. func->destroy(res);
  394. return ret;
  395. }
  396. /**
  397. * vmw_resource_unreserve - Unreserve a resource previously reserved for
  398. * command submission.
  399. *
  400. * @res: Pointer to the struct vmw_resource to unreserve.
  401. * @dirty_set: Change dirty status of the resource.
  402. * @dirty: When changing dirty status indicates the new status.
  403. * @switch_backup: Backup buffer has been switched.
  404. * @new_backup: Pointer to new backup buffer if command submission
  405. * switched. May be NULL.
  406. * @new_backup_offset: New backup offset if @switch_backup is true.
  407. *
  408. * Currently unreserving a resource means putting it back on the device's
  409. * resource lru list, so that it can be evicted if necessary.
  410. */
  411. void vmw_resource_unreserve(struct vmw_resource *res,
  412. bool dirty_set,
  413. bool dirty,
  414. bool switch_backup,
  415. struct vmw_buffer_object *new_backup,
  416. unsigned long new_backup_offset)
  417. {
  418. struct vmw_private *dev_priv = res->dev_priv;
  419. if (!list_empty(&res->lru_head))
  420. return;
  421. if (switch_backup && new_backup != res->backup) {
  422. if (res->backup) {
  423. vmw_resource_mob_detach(res);
  424. if (res->coherent)
  425. vmw_bo_dirty_release(res->backup);
  426. vmw_bo_unreference(&res->backup);
  427. }
  428. if (new_backup) {
  429. res->backup = vmw_bo_reference(new_backup);
  430. /*
  431. * The validation code should already have added a
  432. * dirty tracker here.
  433. */
  434. WARN_ON(res->coherent && !new_backup->dirty);
  435. vmw_resource_mob_attach(res);
  436. } else {
  437. res->backup = NULL;
  438. }
  439. } else if (switch_backup && res->coherent) {
  440. vmw_bo_dirty_release(res->backup);
  441. }
  442. if (switch_backup)
  443. res->backup_offset = new_backup_offset;
  444. if (dirty_set)
  445. res->res_dirty = dirty;
  446. if (!res->func->may_evict || res->id == -1 || res->pin_count)
  447. return;
  448. spin_lock(&dev_priv->resource_lock);
  449. list_add_tail(&res->lru_head,
  450. &res->dev_priv->res_lru[res->func->res_type]);
  451. spin_unlock(&dev_priv->resource_lock);
  452. }
  453. /**
  454. * vmw_resource_check_buffer - Check whether a backup buffer is needed
  455. * for a resource and in that case, allocate
  456. * one, reserve and validate it.
  457. *
  458. * @ticket: The ww aqcquire context to use, or NULL if trylocking.
  459. * @res: The resource for which to allocate a backup buffer.
  460. * @interruptible: Whether any sleeps during allocation should be
  461. * performed while interruptible.
  462. * @val_buf: On successful return contains data about the
  463. * reserved and validated backup buffer.
  464. */
  465. static int
  466. vmw_resource_check_buffer(struct ww_acquire_ctx *ticket,
  467. struct vmw_resource *res,
  468. bool interruptible,
  469. struct ttm_validate_buffer *val_buf)
  470. {
  471. struct ttm_operation_ctx ctx = { true, false };
  472. struct list_head val_list;
  473. bool backup_dirty = false;
  474. int ret;
  475. if (unlikely(res->backup == NULL)) {
  476. ret = vmw_resource_buf_alloc(res, interruptible);
  477. if (unlikely(ret != 0))
  478. return ret;
  479. }
  480. INIT_LIST_HEAD(&val_list);
  481. ttm_bo_get(&res->backup->base);
  482. val_buf->bo = &res->backup->base;
  483. val_buf->num_shared = 0;
  484. list_add_tail(&val_buf->head, &val_list);
  485. ret = ttm_eu_reserve_buffers(ticket, &val_list, interruptible, NULL);
  486. if (unlikely(ret != 0))
  487. goto out_no_reserve;
  488. if (res->func->needs_backup && !vmw_resource_mob_attached(res))
  489. return 0;
  490. backup_dirty = res->backup_dirty;
  491. ret = ttm_bo_validate(&res->backup->base,
  492. res->func->backup_placement,
  493. &ctx);
  494. if (unlikely(ret != 0))
  495. goto out_no_validate;
  496. return 0;
  497. out_no_validate:
  498. ttm_eu_backoff_reservation(ticket, &val_list);
  499. out_no_reserve:
  500. ttm_bo_put(val_buf->bo);
  501. val_buf->bo = NULL;
  502. if (backup_dirty)
  503. vmw_bo_unreference(&res->backup);
  504. return ret;
  505. }
  506. /**
  507. * vmw_resource_reserve - Reserve a resource for command submission
  508. *
  509. * @res: The resource to reserve.
  510. *
  511. * This function takes the resource off the LRU list and make sure
  512. * a backup buffer is present for guest-backed resources. However,
  513. * the buffer may not be bound to the resource at this point.
  514. *
  515. */
  516. int vmw_resource_reserve(struct vmw_resource *res, bool interruptible,
  517. bool no_backup)
  518. {
  519. struct vmw_private *dev_priv = res->dev_priv;
  520. int ret;
  521. spin_lock(&dev_priv->resource_lock);
  522. list_del_init(&res->lru_head);
  523. spin_unlock(&dev_priv->resource_lock);
  524. if (res->func->needs_backup && res->backup == NULL &&
  525. !no_backup) {
  526. ret = vmw_resource_buf_alloc(res, interruptible);
  527. if (unlikely(ret != 0)) {
  528. DRM_ERROR("Failed to allocate a backup buffer "
  529. "of size %lu. bytes\n",
  530. (unsigned long) res->backup_size);
  531. return ret;
  532. }
  533. }
  534. return 0;
  535. }
  536. /**
  537. * vmw_resource_backoff_reservation - Unreserve and unreference a
  538. * backup buffer
  539. *.
  540. * @ticket: The ww acquire ctx used for reservation.
  541. * @val_buf: Backup buffer information.
  542. */
  543. static void
  544. vmw_resource_backoff_reservation(struct ww_acquire_ctx *ticket,
  545. struct ttm_validate_buffer *val_buf)
  546. {
  547. struct list_head val_list;
  548. if (likely(val_buf->bo == NULL))
  549. return;
  550. INIT_LIST_HEAD(&val_list);
  551. list_add_tail(&val_buf->head, &val_list);
  552. ttm_eu_backoff_reservation(ticket, &val_list);
  553. ttm_bo_put(val_buf->bo);
  554. val_buf->bo = NULL;
  555. }
  556. /**
  557. * vmw_resource_do_evict - Evict a resource, and transfer its data
  558. * to a backup buffer.
  559. *
  560. * @ticket: The ww acquire ticket to use, or NULL if trylocking.
  561. * @res: The resource to evict.
  562. * @interruptible: Whether to wait interruptible.
  563. */
  564. static int vmw_resource_do_evict(struct ww_acquire_ctx *ticket,
  565. struct vmw_resource *res, bool interruptible)
  566. {
  567. struct ttm_validate_buffer val_buf;
  568. const struct vmw_res_func *func = res->func;
  569. int ret;
  570. BUG_ON(!func->may_evict);
  571. val_buf.bo = NULL;
  572. val_buf.num_shared = 0;
  573. ret = vmw_resource_check_buffer(ticket, res, interruptible, &val_buf);
  574. if (unlikely(ret != 0))
  575. return ret;
  576. if (unlikely(func->unbind != NULL &&
  577. (!func->needs_backup || vmw_resource_mob_attached(res)))) {
  578. ret = func->unbind(res, res->res_dirty, &val_buf);
  579. if (unlikely(ret != 0))
  580. goto out_no_unbind;
  581. vmw_resource_mob_detach(res);
  582. }
  583. ret = func->destroy(res);
  584. res->backup_dirty = true;
  585. res->res_dirty = false;
  586. out_no_unbind:
  587. vmw_resource_backoff_reservation(ticket, &val_buf);
  588. return ret;
  589. }
  590. /**
  591. * vmw_resource_validate - Make a resource up-to-date and visible
  592. * to the device.
  593. * @res: The resource to make visible to the device.
  594. * @intr: Perform waits interruptible if possible.
  595. * @dirtying: Pending GPU operation will dirty the resource
  596. *
  597. * On succesful return, any backup DMA buffer pointed to by @res->backup will
  598. * be reserved and validated.
  599. * On hardware resource shortage, this function will repeatedly evict
  600. * resources of the same type until the validation succeeds.
  601. *
  602. * Return: Zero on success, -ERESTARTSYS if interrupted, negative error code
  603. * on failure.
  604. */
  605. int vmw_resource_validate(struct vmw_resource *res, bool intr,
  606. bool dirtying)
  607. {
  608. int ret;
  609. struct vmw_resource *evict_res;
  610. struct vmw_private *dev_priv = res->dev_priv;
  611. struct list_head *lru_list = &dev_priv->res_lru[res->func->res_type];
  612. struct ttm_validate_buffer val_buf;
  613. unsigned err_count = 0;
  614. if (!res->func->create)
  615. return 0;
  616. val_buf.bo = NULL;
  617. val_buf.num_shared = 0;
  618. if (res->backup)
  619. val_buf.bo = &res->backup->base;
  620. do {
  621. ret = vmw_resource_do_validate(res, &val_buf, dirtying);
  622. if (likely(ret != -EBUSY))
  623. break;
  624. spin_lock(&dev_priv->resource_lock);
  625. if (list_empty(lru_list) || !res->func->may_evict) {
  626. DRM_ERROR("Out of device device resources "
  627. "for %s.\n", res->func->type_name);
  628. ret = -EBUSY;
  629. spin_unlock(&dev_priv->resource_lock);
  630. break;
  631. }
  632. evict_res = vmw_resource_reference
  633. (list_first_entry(lru_list, struct vmw_resource,
  634. lru_head));
  635. list_del_init(&evict_res->lru_head);
  636. spin_unlock(&dev_priv->resource_lock);
  637. /* Trylock backup buffers with a NULL ticket. */
  638. ret = vmw_resource_do_evict(NULL, evict_res, intr);
  639. if (unlikely(ret != 0)) {
  640. spin_lock(&dev_priv->resource_lock);
  641. list_add_tail(&evict_res->lru_head, lru_list);
  642. spin_unlock(&dev_priv->resource_lock);
  643. if (ret == -ERESTARTSYS ||
  644. ++err_count > VMW_RES_EVICT_ERR_COUNT) {
  645. vmw_resource_unreference(&evict_res);
  646. goto out_no_validate;
  647. }
  648. }
  649. vmw_resource_unreference(&evict_res);
  650. } while (1);
  651. if (unlikely(ret != 0))
  652. goto out_no_validate;
  653. else if (!res->func->needs_backup && res->backup) {
  654. WARN_ON_ONCE(vmw_resource_mob_attached(res));
  655. vmw_bo_unreference(&res->backup);
  656. }
  657. return 0;
  658. out_no_validate:
  659. return ret;
  660. }
  661. /**
  662. * vmw_resource_unbind_list
  663. *
  664. * @vbo: Pointer to the current backing MOB.
  665. *
  666. * Evicts the Guest Backed hardware resource if the backup
  667. * buffer is being moved out of MOB memory.
  668. * Note that this function will not race with the resource
  669. * validation code, since resource validation and eviction
  670. * both require the backup buffer to be reserved.
  671. */
  672. void vmw_resource_unbind_list(struct vmw_buffer_object *vbo)
  673. {
  674. struct ttm_validate_buffer val_buf = {
  675. .bo = &vbo->base,
  676. .num_shared = 0
  677. };
  678. dma_resv_assert_held(vbo->base.base.resv);
  679. while (!RB_EMPTY_ROOT(&vbo->res_tree)) {
  680. struct rb_node *node = vbo->res_tree.rb_node;
  681. struct vmw_resource *res =
  682. container_of(node, struct vmw_resource, mob_node);
  683. if (!WARN_ON_ONCE(!res->func->unbind))
  684. (void) res->func->unbind(res, res->res_dirty, &val_buf);
  685. res->backup_dirty = true;
  686. res->res_dirty = false;
  687. vmw_resource_mob_detach(res);
  688. }
  689. (void) ttm_bo_wait(&vbo->base, false, false);
  690. }
  691. /**
  692. * vmw_query_readback_all - Read back cached query states
  693. *
  694. * @dx_query_mob: Buffer containing the DX query MOB
  695. *
  696. * Read back cached states from the device if they exist. This function
  697. * assumings binding_mutex is held.
  698. */
  699. int vmw_query_readback_all(struct vmw_buffer_object *dx_query_mob)
  700. {
  701. struct vmw_resource *dx_query_ctx;
  702. struct vmw_private *dev_priv;
  703. struct {
  704. SVGA3dCmdHeader header;
  705. SVGA3dCmdDXReadbackAllQuery body;
  706. } *cmd;
  707. /* No query bound, so do nothing */
  708. if (!dx_query_mob || !dx_query_mob->dx_query_ctx)
  709. return 0;
  710. dx_query_ctx = dx_query_mob->dx_query_ctx;
  711. dev_priv = dx_query_ctx->dev_priv;
  712. cmd = VMW_FIFO_RESERVE_DX(dev_priv, sizeof(*cmd), dx_query_ctx->id);
  713. if (unlikely(cmd == NULL))
  714. return -ENOMEM;
  715. cmd->header.id = SVGA_3D_CMD_DX_READBACK_ALL_QUERY;
  716. cmd->header.size = sizeof(cmd->body);
  717. cmd->body.cid = dx_query_ctx->id;
  718. vmw_fifo_commit(dev_priv, sizeof(*cmd));
  719. /* Triggers a rebind the next time affected context is bound */
  720. dx_query_mob->dx_query_ctx = NULL;
  721. return 0;
  722. }
  723. /**
  724. * vmw_query_move_notify - Read back cached query states
  725. *
  726. * @bo: The TTM buffer object about to move.
  727. * @mem: The memory region @bo is moving to.
  728. *
  729. * Called before the query MOB is swapped out to read back cached query
  730. * states from the device.
  731. */
  732. void vmw_query_move_notify(struct ttm_buffer_object *bo,
  733. struct ttm_resource *mem)
  734. {
  735. struct vmw_buffer_object *dx_query_mob;
  736. struct ttm_bo_device *bdev = bo->bdev;
  737. struct vmw_private *dev_priv;
  738. dev_priv = container_of(bdev, struct vmw_private, bdev);
  739. mutex_lock(&dev_priv->binding_mutex);
  740. dx_query_mob = container_of(bo, struct vmw_buffer_object, base);
  741. if (mem == NULL || !dx_query_mob || !dx_query_mob->dx_query_ctx) {
  742. mutex_unlock(&dev_priv->binding_mutex);
  743. return;
  744. }
  745. /* If BO is being moved from MOB to system memory */
  746. if (mem->mem_type == TTM_PL_SYSTEM && bo->mem.mem_type == VMW_PL_MOB) {
  747. struct vmw_fence_obj *fence;
  748. (void) vmw_query_readback_all(dx_query_mob);
  749. mutex_unlock(&dev_priv->binding_mutex);
  750. /* Create a fence and attach the BO to it */
  751. (void) vmw_execbuf_fence_commands(NULL, dev_priv, &fence, NULL);
  752. vmw_bo_fence_single(bo, fence);
  753. if (fence != NULL)
  754. vmw_fence_obj_unreference(&fence);
  755. (void) ttm_bo_wait(bo, false, false);
  756. } else
  757. mutex_unlock(&dev_priv->binding_mutex);
  758. }
  759. /**
  760. * vmw_resource_needs_backup - Return whether a resource needs a backup buffer.
  761. *
  762. * @res: The resource being queried.
  763. */
  764. bool vmw_resource_needs_backup(const struct vmw_resource *res)
  765. {
  766. return res->func->needs_backup;
  767. }
  768. /**
  769. * vmw_resource_evict_type - Evict all resources of a specific type
  770. *
  771. * @dev_priv: Pointer to a device private struct
  772. * @type: The resource type to evict
  773. *
  774. * To avoid thrashing starvation or as part of the hibernation sequence,
  775. * try to evict all evictable resources of a specific type.
  776. */
  777. static void vmw_resource_evict_type(struct vmw_private *dev_priv,
  778. enum vmw_res_type type)
  779. {
  780. struct list_head *lru_list = &dev_priv->res_lru[type];
  781. struct vmw_resource *evict_res;
  782. unsigned err_count = 0;
  783. int ret;
  784. struct ww_acquire_ctx ticket;
  785. do {
  786. spin_lock(&dev_priv->resource_lock);
  787. if (list_empty(lru_list))
  788. goto out_unlock;
  789. evict_res = vmw_resource_reference(
  790. list_first_entry(lru_list, struct vmw_resource,
  791. lru_head));
  792. list_del_init(&evict_res->lru_head);
  793. spin_unlock(&dev_priv->resource_lock);
  794. /* Wait lock backup buffers with a ticket. */
  795. ret = vmw_resource_do_evict(&ticket, evict_res, false);
  796. if (unlikely(ret != 0)) {
  797. spin_lock(&dev_priv->resource_lock);
  798. list_add_tail(&evict_res->lru_head, lru_list);
  799. spin_unlock(&dev_priv->resource_lock);
  800. if (++err_count > VMW_RES_EVICT_ERR_COUNT) {
  801. vmw_resource_unreference(&evict_res);
  802. return;
  803. }
  804. }
  805. vmw_resource_unreference(&evict_res);
  806. } while (1);
  807. out_unlock:
  808. spin_unlock(&dev_priv->resource_lock);
  809. }
  810. /**
  811. * vmw_resource_evict_all - Evict all evictable resources
  812. *
  813. * @dev_priv: Pointer to a device private struct
  814. *
  815. * To avoid thrashing starvation or as part of the hibernation sequence,
  816. * evict all evictable resources. In particular this means that all
  817. * guest-backed resources that are registered with the device are
  818. * evicted and the OTable becomes clean.
  819. */
  820. void vmw_resource_evict_all(struct vmw_private *dev_priv)
  821. {
  822. enum vmw_res_type type;
  823. mutex_lock(&dev_priv->cmdbuf_mutex);
  824. for (type = 0; type < vmw_res_max; ++type)
  825. vmw_resource_evict_type(dev_priv, type);
  826. mutex_unlock(&dev_priv->cmdbuf_mutex);
  827. }
  828. /**
  829. * vmw_resource_pin - Add a pin reference on a resource
  830. *
  831. * @res: The resource to add a pin reference on
  832. *
  833. * This function adds a pin reference, and if needed validates the resource.
  834. * Having a pin reference means that the resource can never be evicted, and
  835. * its id will never change as long as there is a pin reference.
  836. * This function returns 0 on success and a negative error code on failure.
  837. */
  838. int vmw_resource_pin(struct vmw_resource *res, bool interruptible)
  839. {
  840. struct ttm_operation_ctx ctx = { interruptible, false };
  841. struct vmw_private *dev_priv = res->dev_priv;
  842. int ret;
  843. ttm_write_lock(&dev_priv->reservation_sem, interruptible);
  844. mutex_lock(&dev_priv->cmdbuf_mutex);
  845. ret = vmw_resource_reserve(res, interruptible, false);
  846. if (ret)
  847. goto out_no_reserve;
  848. if (res->pin_count == 0) {
  849. struct vmw_buffer_object *vbo = NULL;
  850. if (res->backup) {
  851. vbo = res->backup;
  852. ttm_bo_reserve(&vbo->base, interruptible, false, NULL);
  853. if (!vbo->pin_count) {
  854. ret = ttm_bo_validate
  855. (&vbo->base,
  856. res->func->backup_placement,
  857. &ctx);
  858. if (ret) {
  859. ttm_bo_unreserve(&vbo->base);
  860. goto out_no_validate;
  861. }
  862. }
  863. /* Do we really need to pin the MOB as well? */
  864. vmw_bo_pin_reserved(vbo, true);
  865. }
  866. ret = vmw_resource_validate(res, interruptible, true);
  867. if (vbo)
  868. ttm_bo_unreserve(&vbo->base);
  869. if (ret)
  870. goto out_no_validate;
  871. }
  872. res->pin_count++;
  873. out_no_validate:
  874. vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
  875. out_no_reserve:
  876. mutex_unlock(&dev_priv->cmdbuf_mutex);
  877. ttm_write_unlock(&dev_priv->reservation_sem);
  878. return ret;
  879. }
  880. /**
  881. * vmw_resource_unpin - Remove a pin reference from a resource
  882. *
  883. * @res: The resource to remove a pin reference from
  884. *
  885. * Having a pin reference means that the resource can never be evicted, and
  886. * its id will never change as long as there is a pin reference.
  887. */
  888. void vmw_resource_unpin(struct vmw_resource *res)
  889. {
  890. struct vmw_private *dev_priv = res->dev_priv;
  891. int ret;
  892. (void) ttm_read_lock(&dev_priv->reservation_sem, false);
  893. mutex_lock(&dev_priv->cmdbuf_mutex);
  894. ret = vmw_resource_reserve(res, false, true);
  895. WARN_ON(ret);
  896. WARN_ON(res->pin_count == 0);
  897. if (--res->pin_count == 0 && res->backup) {
  898. struct vmw_buffer_object *vbo = res->backup;
  899. (void) ttm_bo_reserve(&vbo->base, false, false, NULL);
  900. vmw_bo_pin_reserved(vbo, false);
  901. ttm_bo_unreserve(&vbo->base);
  902. }
  903. vmw_resource_unreserve(res, false, false, false, NULL, 0UL);
  904. mutex_unlock(&dev_priv->cmdbuf_mutex);
  905. ttm_read_unlock(&dev_priv->reservation_sem);
  906. }
  907. /**
  908. * vmw_res_type - Return the resource type
  909. *
  910. * @res: Pointer to the resource
  911. */
  912. enum vmw_res_type vmw_res_type(const struct vmw_resource *res)
  913. {
  914. return res->func->res_type;
  915. }
  916. /**
  917. * vmw_resource_update_dirty - Update a resource's dirty tracker with a
  918. * sequential range of touched backing store memory.
  919. * @res: The resource.
  920. * @start: The first page touched.
  921. * @end: The last page touched + 1.
  922. */
  923. void vmw_resource_dirty_update(struct vmw_resource *res, pgoff_t start,
  924. pgoff_t end)
  925. {
  926. if (res->dirty)
  927. res->func->dirty_range_add(res, start << PAGE_SHIFT,
  928. end << PAGE_SHIFT);
  929. }
  930. /**
  931. * vmw_resources_clean - Clean resources intersecting a mob range
  932. * @vbo: The mob buffer object
  933. * @start: The mob page offset starting the range
  934. * @end: The mob page offset ending the range
  935. * @num_prefault: Returns how many pages including the first have been
  936. * cleaned and are ok to prefault
  937. */
  938. int vmw_resources_clean(struct vmw_buffer_object *vbo, pgoff_t start,
  939. pgoff_t end, pgoff_t *num_prefault)
  940. {
  941. struct rb_node *cur = vbo->res_tree.rb_node;
  942. struct vmw_resource *found = NULL;
  943. unsigned long res_start = start << PAGE_SHIFT;
  944. unsigned long res_end = end << PAGE_SHIFT;
  945. unsigned long last_cleaned = 0;
  946. /*
  947. * Find the resource with lowest backup_offset that intersects the
  948. * range.
  949. */
  950. while (cur) {
  951. struct vmw_resource *cur_res =
  952. container_of(cur, struct vmw_resource, mob_node);
  953. if (cur_res->backup_offset >= res_end) {
  954. cur = cur->rb_left;
  955. } else if (cur_res->backup_offset + cur_res->backup_size <=
  956. res_start) {
  957. cur = cur->rb_right;
  958. } else {
  959. found = cur_res;
  960. cur = cur->rb_left;
  961. /* Continue to look for resources with lower offsets */
  962. }
  963. }
  964. /*
  965. * In order of increasing backup_offset, clean dirty resorces
  966. * intersecting the range.
  967. */
  968. while (found) {
  969. if (found->res_dirty) {
  970. int ret;
  971. if (!found->func->clean)
  972. return -EINVAL;
  973. ret = found->func->clean(found);
  974. if (ret)
  975. return ret;
  976. found->res_dirty = false;
  977. }
  978. last_cleaned = found->backup_offset + found->backup_size;
  979. cur = rb_next(&found->mob_node);
  980. if (!cur)
  981. break;
  982. found = container_of(cur, struct vmw_resource, mob_node);
  983. if (found->backup_offset >= res_end)
  984. break;
  985. }
  986. /*
  987. * Set number of pages allowed prefaulting and fence the buffer object
  988. */
  989. *num_prefault = 1;
  990. if (last_cleaned > res_start) {
  991. struct ttm_buffer_object *bo = &vbo->base;
  992. *num_prefault = __KERNEL_DIV_ROUND_UP(last_cleaned - res_start,
  993. PAGE_SIZE);
  994. vmw_bo_fence_single(bo, NULL);
  995. if (bo->moving)
  996. dma_fence_put(bo->moving);
  997. bo->moving = dma_fence_get
  998. (dma_resv_get_excl(bo->base.resv));
  999. }
  1000. return 0;
  1001. }