rdma_core.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980
  1. /*
  2. * Copyright (c) 2016, Mellanox Technologies inc. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. */
  32. #include <linux/file.h>
  33. #include <linux/anon_inodes.h>
  34. #include <linux/sched/mm.h>
  35. #include <rdma/ib_verbs.h>
  36. #include <rdma/uverbs_types.h>
  37. #include <linux/rcupdate.h>
  38. #include <rdma/uverbs_ioctl.h>
  39. #include <rdma/rdma_user_ioctl.h>
  40. #include "uverbs.h"
  41. #include "core_priv.h"
  42. #include "rdma_core.h"
  43. static void uverbs_uobject_free(struct kref *ref)
  44. {
  45. kfree_rcu(container_of(ref, struct ib_uobject, ref), rcu);
  46. }
  47. /*
  48. * In order to indicate we no longer needs this uobject, uverbs_uobject_put
  49. * is called. When the reference count is decreased, the uobject is freed.
  50. * For example, this is used when attaching a completion channel to a CQ.
  51. */
  52. void uverbs_uobject_put(struct ib_uobject *uobject)
  53. {
  54. kref_put(&uobject->ref, uverbs_uobject_free);
  55. }
  56. EXPORT_SYMBOL(uverbs_uobject_put);
  57. static int uverbs_try_lock_object(struct ib_uobject *uobj,
  58. enum rdma_lookup_mode mode)
  59. {
  60. /*
  61. * When a shared access is required, we use a positive counter. Each
  62. * shared access request checks that the value != -1 and increment it.
  63. * Exclusive access is required for operations like write or destroy.
  64. * In exclusive access mode, we check that the counter is zero (nobody
  65. * claimed this object) and we set it to -1. Releasing a shared access
  66. * lock is done simply by decreasing the counter. As for exclusive
  67. * access locks, since only a single one of them is is allowed
  68. * concurrently, setting the counter to zero is enough for releasing
  69. * this lock.
  70. */
  71. switch (mode) {
  72. case UVERBS_LOOKUP_READ:
  73. return atomic_fetch_add_unless(&uobj->usecnt, 1, -1) == -1 ?
  74. -EBUSY : 0;
  75. case UVERBS_LOOKUP_WRITE:
  76. /* lock is exclusive */
  77. return atomic_cmpxchg(&uobj->usecnt, 0, -1) == 0 ? 0 : -EBUSY;
  78. case UVERBS_LOOKUP_DESTROY:
  79. return 0;
  80. }
  81. return 0;
  82. }
  83. static void assert_uverbs_usecnt(struct ib_uobject *uobj,
  84. enum rdma_lookup_mode mode)
  85. {
  86. #ifdef CONFIG_LOCKDEP
  87. switch (mode) {
  88. case UVERBS_LOOKUP_READ:
  89. WARN_ON(atomic_read(&uobj->usecnt) <= 0);
  90. break;
  91. case UVERBS_LOOKUP_WRITE:
  92. WARN_ON(atomic_read(&uobj->usecnt) != -1);
  93. break;
  94. case UVERBS_LOOKUP_DESTROY:
  95. break;
  96. }
  97. #endif
  98. }
  99. /*
  100. * This must be called with the hw_destroy_rwsem locked for read or write,
  101. * also the uobject itself must be locked for write.
  102. *
  103. * Upon return the HW object is guaranteed to be destroyed.
  104. *
  105. * For RDMA_REMOVE_ABORT, the hw_destroy_rwsem is not required to be held,
  106. * however the type's allocat_commit function cannot have been called and the
  107. * uobject cannot be on the uobjects_lists
  108. *
  109. * For RDMA_REMOVE_DESTROY the caller shold be holding a kref (eg via
  110. * rdma_lookup_get_uobject) and the object is left in a state where the caller
  111. * needs to call rdma_lookup_put_uobject.
  112. *
  113. * For all other destroy modes this function internally unlocks the uobject
  114. * and consumes the kref on the uobj.
  115. */
  116. static int uverbs_destroy_uobject(struct ib_uobject *uobj,
  117. enum rdma_remove_reason reason,
  118. struct uverbs_attr_bundle *attrs)
  119. {
  120. struct ib_uverbs_file *ufile = attrs->ufile;
  121. unsigned long flags;
  122. int ret;
  123. lockdep_assert_held(&ufile->hw_destroy_rwsem);
  124. assert_uverbs_usecnt(uobj, UVERBS_LOOKUP_WRITE);
  125. if (reason == RDMA_REMOVE_ABORT) {
  126. WARN_ON(!list_empty(&uobj->list));
  127. WARN_ON(!uobj->context);
  128. uobj->uapi_object->type_class->alloc_abort(uobj);
  129. } else if (uobj->object) {
  130. ret = uobj->uapi_object->type_class->destroy_hw(uobj, reason,
  131. attrs);
  132. if (ret) {
  133. if (ib_is_destroy_retryable(ret, reason, uobj))
  134. return ret;
  135. /* Nothing to be done, dangle the memory and move on */
  136. WARN(true,
  137. "ib_uverbs: failed to remove uobject id %d, driver err=%d",
  138. uobj->id, ret);
  139. }
  140. uobj->object = NULL;
  141. }
  142. uobj->context = NULL;
  143. /*
  144. * For DESTROY the usecnt is not changed, the caller is expected to
  145. * manage it via uobj_put_destroy(). Only DESTROY can remove the IDR
  146. * handle.
  147. */
  148. if (reason != RDMA_REMOVE_DESTROY)
  149. atomic_set(&uobj->usecnt, 0);
  150. else
  151. uobj->uapi_object->type_class->remove_handle(uobj);
  152. if (!list_empty(&uobj->list)) {
  153. spin_lock_irqsave(&ufile->uobjects_lock, flags);
  154. list_del_init(&uobj->list);
  155. spin_unlock_irqrestore(&ufile->uobjects_lock, flags);
  156. /*
  157. * Pairs with the get in rdma_alloc_commit_uobject(), could
  158. * destroy uobj.
  159. */
  160. uverbs_uobject_put(uobj);
  161. }
  162. /*
  163. * When aborting the stack kref remains owned by the core code, and is
  164. * not transferred into the type. Pairs with the get in alloc_uobj
  165. */
  166. if (reason == RDMA_REMOVE_ABORT)
  167. uverbs_uobject_put(uobj);
  168. return 0;
  169. }
  170. /*
  171. * This calls uverbs_destroy_uobject() using the RDMA_REMOVE_DESTROY
  172. * sequence. It should only be used from command callbacks. On success the
  173. * caller must pair this with uobj_put_destroy(). This
  174. * version requires the caller to have already obtained an
  175. * LOOKUP_DESTROY uobject kref.
  176. */
  177. int uobj_destroy(struct ib_uobject *uobj, struct uverbs_attr_bundle *attrs)
  178. {
  179. struct ib_uverbs_file *ufile = attrs->ufile;
  180. int ret;
  181. down_read(&ufile->hw_destroy_rwsem);
  182. /*
  183. * Once the uobject is destroyed by RDMA_REMOVE_DESTROY then it is left
  184. * write locked as the callers put it back with UVERBS_LOOKUP_DESTROY.
  185. * This is because any other concurrent thread can still see the object
  186. * in the xarray due to RCU. Leaving it locked ensures nothing else will
  187. * touch it.
  188. */
  189. ret = uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE);
  190. if (ret)
  191. goto out_unlock;
  192. ret = uverbs_destroy_uobject(uobj, RDMA_REMOVE_DESTROY, attrs);
  193. if (ret) {
  194. atomic_set(&uobj->usecnt, 0);
  195. goto out_unlock;
  196. }
  197. out_unlock:
  198. up_read(&ufile->hw_destroy_rwsem);
  199. return ret;
  200. }
  201. /*
  202. * uobj_get_destroy destroys the HW object and returns a handle to the uobj
  203. * with a NULL object pointer. The caller must pair this with
  204. * uobj_put_destroy().
  205. */
  206. struct ib_uobject *__uobj_get_destroy(const struct uverbs_api_object *obj,
  207. u32 id, struct uverbs_attr_bundle *attrs)
  208. {
  209. struct ib_uobject *uobj;
  210. int ret;
  211. uobj = rdma_lookup_get_uobject(obj, attrs->ufile, id,
  212. UVERBS_LOOKUP_DESTROY, attrs);
  213. if (IS_ERR(uobj))
  214. return uobj;
  215. ret = uobj_destroy(uobj, attrs);
  216. if (ret) {
  217. rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
  218. return ERR_PTR(ret);
  219. }
  220. return uobj;
  221. }
  222. /*
  223. * Does both uobj_get_destroy() and uobj_put_destroy(). Returns 0 on success
  224. * (negative errno on failure). For use by callers that do not need the uobj.
  225. */
  226. int __uobj_perform_destroy(const struct uverbs_api_object *obj, u32 id,
  227. struct uverbs_attr_bundle *attrs)
  228. {
  229. struct ib_uobject *uobj;
  230. uobj = __uobj_get_destroy(obj, id, attrs);
  231. if (IS_ERR(uobj))
  232. return PTR_ERR(uobj);
  233. uobj_put_destroy(uobj);
  234. return 0;
  235. }
  236. /* alloc_uobj must be undone by uverbs_destroy_uobject() */
  237. static struct ib_uobject *alloc_uobj(struct uverbs_attr_bundle *attrs,
  238. const struct uverbs_api_object *obj)
  239. {
  240. struct ib_uverbs_file *ufile = attrs->ufile;
  241. struct ib_uobject *uobj;
  242. if (!attrs->context) {
  243. struct ib_ucontext *ucontext =
  244. ib_uverbs_get_ucontext_file(ufile);
  245. if (IS_ERR(ucontext))
  246. return ERR_CAST(ucontext);
  247. attrs->context = ucontext;
  248. }
  249. uobj = kzalloc(obj->type_attrs->obj_size, GFP_KERNEL);
  250. if (!uobj)
  251. return ERR_PTR(-ENOMEM);
  252. /*
  253. * user_handle should be filled by the handler,
  254. * The object is added to the list in the commit stage.
  255. */
  256. uobj->ufile = ufile;
  257. uobj->context = attrs->context;
  258. INIT_LIST_HEAD(&uobj->list);
  259. uobj->uapi_object = obj;
  260. /*
  261. * Allocated objects start out as write locked to deny any other
  262. * syscalls from accessing them until they are committed. See
  263. * rdma_alloc_commit_uobject
  264. */
  265. atomic_set(&uobj->usecnt, -1);
  266. kref_init(&uobj->ref);
  267. return uobj;
  268. }
  269. static int idr_add_uobj(struct ib_uobject *uobj)
  270. {
  271. /*
  272. * We start with allocating an idr pointing to NULL. This represents an
  273. * object which isn't initialized yet. We'll replace it later on with
  274. * the real object once we commit.
  275. */
  276. return xa_alloc(&uobj->ufile->idr, &uobj->id, NULL, xa_limit_32b,
  277. GFP_KERNEL);
  278. }
  279. /* Returns the ib_uobject or an error. The caller should check for IS_ERR. */
  280. static struct ib_uobject *
  281. lookup_get_idr_uobject(const struct uverbs_api_object *obj,
  282. struct ib_uverbs_file *ufile, s64 id,
  283. enum rdma_lookup_mode mode)
  284. {
  285. struct ib_uobject *uobj;
  286. if (id < 0 || id > ULONG_MAX)
  287. return ERR_PTR(-EINVAL);
  288. rcu_read_lock();
  289. /*
  290. * The idr_find is guaranteed to return a pointer to something that
  291. * isn't freed yet, or NULL, as the free after idr_remove goes through
  292. * kfree_rcu(). However the object may still have been released and
  293. * kfree() could be called at any time.
  294. */
  295. uobj = xa_load(&ufile->idr, id);
  296. if (!uobj || !kref_get_unless_zero(&uobj->ref))
  297. uobj = ERR_PTR(-ENOENT);
  298. rcu_read_unlock();
  299. return uobj;
  300. }
  301. static struct ib_uobject *
  302. lookup_get_fd_uobject(const struct uverbs_api_object *obj,
  303. struct ib_uverbs_file *ufile, s64 id,
  304. enum rdma_lookup_mode mode)
  305. {
  306. const struct uverbs_obj_fd_type *fd_type;
  307. struct file *f;
  308. struct ib_uobject *uobject;
  309. int fdno = id;
  310. if (fdno != id)
  311. return ERR_PTR(-EINVAL);
  312. if (mode != UVERBS_LOOKUP_READ)
  313. return ERR_PTR(-EOPNOTSUPP);
  314. if (!obj->type_attrs)
  315. return ERR_PTR(-EIO);
  316. fd_type =
  317. container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
  318. f = fget(fdno);
  319. if (!f)
  320. return ERR_PTR(-EBADF);
  321. uobject = f->private_data;
  322. /*
  323. * fget(id) ensures we are not currently running
  324. * uverbs_uobject_fd_release(), and the caller is expected to ensure
  325. * that release is never done while a call to lookup is possible.
  326. */
  327. if (f->f_op != fd_type->fops || uobject->ufile != ufile) {
  328. fput(f);
  329. return ERR_PTR(-EBADF);
  330. }
  331. uverbs_uobject_get(uobject);
  332. return uobject;
  333. }
  334. struct ib_uobject *rdma_lookup_get_uobject(const struct uverbs_api_object *obj,
  335. struct ib_uverbs_file *ufile, s64 id,
  336. enum rdma_lookup_mode mode,
  337. struct uverbs_attr_bundle *attrs)
  338. {
  339. struct ib_uobject *uobj;
  340. int ret;
  341. if (obj == ERR_PTR(-ENOMSG)) {
  342. /* must be UVERBS_IDR_ANY_OBJECT, see uapi_get_object() */
  343. uobj = lookup_get_idr_uobject(NULL, ufile, id, mode);
  344. if (IS_ERR(uobj))
  345. return uobj;
  346. } else {
  347. if (IS_ERR(obj))
  348. return ERR_PTR(-EINVAL);
  349. uobj = obj->type_class->lookup_get(obj, ufile, id, mode);
  350. if (IS_ERR(uobj))
  351. return uobj;
  352. if (uobj->uapi_object != obj) {
  353. ret = -EINVAL;
  354. goto free;
  355. }
  356. }
  357. /*
  358. * If we have been disassociated block every command except for
  359. * DESTROY based commands.
  360. */
  361. if (mode != UVERBS_LOOKUP_DESTROY &&
  362. !srcu_dereference(ufile->device->ib_dev,
  363. &ufile->device->disassociate_srcu)) {
  364. ret = -EIO;
  365. goto free;
  366. }
  367. ret = uverbs_try_lock_object(uobj, mode);
  368. if (ret)
  369. goto free;
  370. if (attrs)
  371. attrs->context = uobj->context;
  372. return uobj;
  373. free:
  374. uobj->uapi_object->type_class->lookup_put(uobj, mode);
  375. uverbs_uobject_put(uobj);
  376. return ERR_PTR(ret);
  377. }
  378. static struct ib_uobject *
  379. alloc_begin_idr_uobject(const struct uverbs_api_object *obj,
  380. struct uverbs_attr_bundle *attrs)
  381. {
  382. int ret;
  383. struct ib_uobject *uobj;
  384. uobj = alloc_uobj(attrs, obj);
  385. if (IS_ERR(uobj))
  386. return uobj;
  387. ret = idr_add_uobj(uobj);
  388. if (ret)
  389. goto uobj_put;
  390. ret = ib_rdmacg_try_charge(&uobj->cg_obj, uobj->context->device,
  391. RDMACG_RESOURCE_HCA_OBJECT);
  392. if (ret)
  393. goto remove;
  394. return uobj;
  395. remove:
  396. xa_erase(&attrs->ufile->idr, uobj->id);
  397. uobj_put:
  398. uverbs_uobject_put(uobj);
  399. return ERR_PTR(ret);
  400. }
  401. static struct ib_uobject *
  402. alloc_begin_fd_uobject(const struct uverbs_api_object *obj,
  403. struct uverbs_attr_bundle *attrs)
  404. {
  405. const struct uverbs_obj_fd_type *fd_type;
  406. int new_fd;
  407. struct ib_uobject *uobj, *ret;
  408. struct file *filp;
  409. uobj = alloc_uobj(attrs, obj);
  410. if (IS_ERR(uobj))
  411. return uobj;
  412. fd_type =
  413. container_of(obj->type_attrs, struct uverbs_obj_fd_type, type);
  414. if (WARN_ON(fd_type->fops->release != &uverbs_uobject_fd_release &&
  415. fd_type->fops->release != &uverbs_async_event_release)) {
  416. ret = ERR_PTR(-EINVAL);
  417. goto err_fd;
  418. }
  419. new_fd = get_unused_fd_flags(O_CLOEXEC);
  420. if (new_fd < 0) {
  421. ret = ERR_PTR(new_fd);
  422. goto err_fd;
  423. }
  424. /* Note that uverbs_uobject_fd_release() is called during abort */
  425. filp = anon_inode_getfile(fd_type->name, fd_type->fops, NULL,
  426. fd_type->flags);
  427. if (IS_ERR(filp)) {
  428. ret = ERR_CAST(filp);
  429. goto err_getfile;
  430. }
  431. uobj->object = filp;
  432. uobj->id = new_fd;
  433. return uobj;
  434. err_getfile:
  435. put_unused_fd(new_fd);
  436. err_fd:
  437. uverbs_uobject_put(uobj);
  438. return ret;
  439. }
  440. struct ib_uobject *rdma_alloc_begin_uobject(const struct uverbs_api_object *obj,
  441. struct uverbs_attr_bundle *attrs)
  442. {
  443. struct ib_uverbs_file *ufile = attrs->ufile;
  444. struct ib_uobject *ret;
  445. if (IS_ERR(obj))
  446. return ERR_PTR(-EINVAL);
  447. /*
  448. * The hw_destroy_rwsem is held across the entire object creation and
  449. * released during rdma_alloc_commit_uobject or
  450. * rdma_alloc_abort_uobject
  451. */
  452. if (!down_read_trylock(&ufile->hw_destroy_rwsem))
  453. return ERR_PTR(-EIO);
  454. ret = obj->type_class->alloc_begin(obj, attrs);
  455. if (IS_ERR(ret)) {
  456. up_read(&ufile->hw_destroy_rwsem);
  457. return ret;
  458. }
  459. return ret;
  460. }
  461. static void alloc_abort_idr_uobject(struct ib_uobject *uobj)
  462. {
  463. ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device,
  464. RDMACG_RESOURCE_HCA_OBJECT);
  465. xa_erase(&uobj->ufile->idr, uobj->id);
  466. }
  467. static int __must_check destroy_hw_idr_uobject(struct ib_uobject *uobj,
  468. enum rdma_remove_reason why,
  469. struct uverbs_attr_bundle *attrs)
  470. {
  471. const struct uverbs_obj_idr_type *idr_type =
  472. container_of(uobj->uapi_object->type_attrs,
  473. struct uverbs_obj_idr_type, type);
  474. int ret = idr_type->destroy_object(uobj, why, attrs);
  475. /*
  476. * We can only fail gracefully if the user requested to destroy the
  477. * object or when a retry may be called upon an error.
  478. * In the rest of the cases, just remove whatever you can.
  479. */
  480. if (ib_is_destroy_retryable(ret, why, uobj))
  481. return ret;
  482. if (why == RDMA_REMOVE_ABORT)
  483. return 0;
  484. ib_rdmacg_uncharge(&uobj->cg_obj, uobj->context->device,
  485. RDMACG_RESOURCE_HCA_OBJECT);
  486. return 0;
  487. }
  488. static void remove_handle_idr_uobject(struct ib_uobject *uobj)
  489. {
  490. xa_erase(&uobj->ufile->idr, uobj->id);
  491. /* Matches the kref in alloc_commit_idr_uobject */
  492. uverbs_uobject_put(uobj);
  493. }
  494. static void alloc_abort_fd_uobject(struct ib_uobject *uobj)
  495. {
  496. struct file *filp = uobj->object;
  497. fput(filp);
  498. put_unused_fd(uobj->id);
  499. }
  500. static int __must_check destroy_hw_fd_uobject(struct ib_uobject *uobj,
  501. enum rdma_remove_reason why,
  502. struct uverbs_attr_bundle *attrs)
  503. {
  504. const struct uverbs_obj_fd_type *fd_type = container_of(
  505. uobj->uapi_object->type_attrs, struct uverbs_obj_fd_type, type);
  506. int ret = fd_type->destroy_object(uobj, why);
  507. if (ib_is_destroy_retryable(ret, why, uobj))
  508. return ret;
  509. return 0;
  510. }
  511. static void remove_handle_fd_uobject(struct ib_uobject *uobj)
  512. {
  513. }
  514. static void alloc_commit_idr_uobject(struct ib_uobject *uobj)
  515. {
  516. struct ib_uverbs_file *ufile = uobj->ufile;
  517. void *old;
  518. /*
  519. * We already allocated this IDR with a NULL object, so
  520. * this shouldn't fail.
  521. *
  522. * NOTE: Storing the uobj transfers our kref on uobj to the XArray.
  523. * It will be put by remove_commit_idr_uobject()
  524. */
  525. old = xa_store(&ufile->idr, uobj->id, uobj, GFP_KERNEL);
  526. WARN_ON(old != NULL);
  527. }
  528. static void alloc_commit_fd_uobject(struct ib_uobject *uobj)
  529. {
  530. int fd = uobj->id;
  531. struct file *filp = uobj->object;
  532. /* Matching put will be done in uverbs_uobject_fd_release() */
  533. kref_get(&uobj->ufile->ref);
  534. /* This shouldn't be used anymore. Use the file object instead */
  535. uobj->id = 0;
  536. /*
  537. * NOTE: Once we install the file we loose ownership of our kref on
  538. * uobj. It will be put by uverbs_uobject_fd_release()
  539. */
  540. filp->private_data = uobj;
  541. fd_install(fd, filp);
  542. }
  543. /*
  544. * In all cases rdma_alloc_commit_uobject() consumes the kref to uobj and the
  545. * caller can no longer assume uobj is valid. If this function fails it
  546. * destroys the uboject, including the attached HW object.
  547. */
  548. void rdma_alloc_commit_uobject(struct ib_uobject *uobj,
  549. struct uverbs_attr_bundle *attrs)
  550. {
  551. struct ib_uverbs_file *ufile = attrs->ufile;
  552. /* kref is held so long as the uobj is on the uobj list. */
  553. uverbs_uobject_get(uobj);
  554. spin_lock_irq(&ufile->uobjects_lock);
  555. list_add(&uobj->list, &ufile->uobjects);
  556. spin_unlock_irq(&ufile->uobjects_lock);
  557. /* matches atomic_set(-1) in alloc_uobj */
  558. atomic_set(&uobj->usecnt, 0);
  559. /* alloc_commit consumes the uobj kref */
  560. uobj->uapi_object->type_class->alloc_commit(uobj);
  561. /* Matches the down_read in rdma_alloc_begin_uobject */
  562. up_read(&ufile->hw_destroy_rwsem);
  563. }
  564. /*
  565. * This consumes the kref for uobj. It is up to the caller to unwind the HW
  566. * object and anything else connected to uobj before calling this.
  567. */
  568. void rdma_alloc_abort_uobject(struct ib_uobject *uobj,
  569. struct uverbs_attr_bundle *attrs,
  570. bool hw_obj_valid)
  571. {
  572. struct ib_uverbs_file *ufile = uobj->ufile;
  573. int ret;
  574. if (hw_obj_valid) {
  575. ret = uobj->uapi_object->type_class->destroy_hw(
  576. uobj, RDMA_REMOVE_ABORT, attrs);
  577. /*
  578. * If the driver couldn't destroy the object then go ahead and
  579. * commit it. Leaking objects that can't be destroyed is only
  580. * done during FD close after the driver has a few more tries to
  581. * destroy it.
  582. */
  583. if (WARN_ON(ret))
  584. return rdma_alloc_commit_uobject(uobj, attrs);
  585. }
  586. uverbs_destroy_uobject(uobj, RDMA_REMOVE_ABORT, attrs);
  587. /* Matches the down_read in rdma_alloc_begin_uobject */
  588. up_read(&ufile->hw_destroy_rwsem);
  589. }
  590. static void lookup_put_idr_uobject(struct ib_uobject *uobj,
  591. enum rdma_lookup_mode mode)
  592. {
  593. }
  594. static void lookup_put_fd_uobject(struct ib_uobject *uobj,
  595. enum rdma_lookup_mode mode)
  596. {
  597. struct file *filp = uobj->object;
  598. WARN_ON(mode != UVERBS_LOOKUP_READ);
  599. /*
  600. * This indirectly calls uverbs_uobject_fd_release() and free the
  601. * object
  602. */
  603. fput(filp);
  604. }
  605. void rdma_lookup_put_uobject(struct ib_uobject *uobj,
  606. enum rdma_lookup_mode mode)
  607. {
  608. assert_uverbs_usecnt(uobj, mode);
  609. /*
  610. * In order to unlock an object, either decrease its usecnt for
  611. * read access or zero it in case of exclusive access. See
  612. * uverbs_try_lock_object for locking schema information.
  613. */
  614. switch (mode) {
  615. case UVERBS_LOOKUP_READ:
  616. atomic_dec(&uobj->usecnt);
  617. break;
  618. case UVERBS_LOOKUP_WRITE:
  619. atomic_set(&uobj->usecnt, 0);
  620. break;
  621. case UVERBS_LOOKUP_DESTROY:
  622. break;
  623. }
  624. uobj->uapi_object->type_class->lookup_put(uobj, mode);
  625. /* Pairs with the kref obtained by type->lookup_get */
  626. uverbs_uobject_put(uobj);
  627. }
  628. void setup_ufile_idr_uobject(struct ib_uverbs_file *ufile)
  629. {
  630. xa_init_flags(&ufile->idr, XA_FLAGS_ALLOC);
  631. }
  632. void release_ufile_idr_uobject(struct ib_uverbs_file *ufile)
  633. {
  634. struct ib_uobject *entry;
  635. unsigned long id;
  636. /*
  637. * At this point uverbs_cleanup_ufile() is guaranteed to have run, and
  638. * there are no HW objects left, however the xarray is still populated
  639. * with anything that has not been cleaned up by userspace. Since the
  640. * kref on ufile is 0, nothing is allowed to call lookup_get.
  641. *
  642. * This is an optimized equivalent to remove_handle_idr_uobject
  643. */
  644. xa_for_each(&ufile->idr, id, entry) {
  645. WARN_ON(entry->object);
  646. uverbs_uobject_put(entry);
  647. }
  648. xa_destroy(&ufile->idr);
  649. }
  650. const struct uverbs_obj_type_class uverbs_idr_class = {
  651. .alloc_begin = alloc_begin_idr_uobject,
  652. .lookup_get = lookup_get_idr_uobject,
  653. .alloc_commit = alloc_commit_idr_uobject,
  654. .alloc_abort = alloc_abort_idr_uobject,
  655. .lookup_put = lookup_put_idr_uobject,
  656. .destroy_hw = destroy_hw_idr_uobject,
  657. .remove_handle = remove_handle_idr_uobject,
  658. };
  659. EXPORT_SYMBOL(uverbs_idr_class);
  660. /*
  661. * Users of UVERBS_TYPE_ALLOC_FD should set this function as the struct
  662. * file_operations release method.
  663. */
  664. int uverbs_uobject_fd_release(struct inode *inode, struct file *filp)
  665. {
  666. struct ib_uverbs_file *ufile;
  667. struct ib_uobject *uobj;
  668. /*
  669. * This can only happen if the fput came from alloc_abort_fd_uobject()
  670. */
  671. if (!filp->private_data)
  672. return 0;
  673. uobj = filp->private_data;
  674. ufile = uobj->ufile;
  675. if (down_read_trylock(&ufile->hw_destroy_rwsem)) {
  676. struct uverbs_attr_bundle attrs = {
  677. .context = uobj->context,
  678. .ufile = ufile,
  679. };
  680. /*
  681. * lookup_get_fd_uobject holds the kref on the struct file any
  682. * time a FD uobj is locked, which prevents this release
  683. * method from being invoked. Meaning we can always get the
  684. * write lock here, or we have a kernel bug.
  685. */
  686. WARN_ON(uverbs_try_lock_object(uobj, UVERBS_LOOKUP_WRITE));
  687. uverbs_destroy_uobject(uobj, RDMA_REMOVE_CLOSE, &attrs);
  688. up_read(&ufile->hw_destroy_rwsem);
  689. }
  690. /* Matches the get in alloc_commit_fd_uobject() */
  691. kref_put(&ufile->ref, ib_uverbs_release_file);
  692. /* Pairs with filp->private_data in alloc_begin_fd_uobject */
  693. uverbs_uobject_put(uobj);
  694. return 0;
  695. }
  696. EXPORT_SYMBOL(uverbs_uobject_fd_release);
  697. /*
  698. * Drop the ucontext off the ufile and completely disconnect it from the
  699. * ib_device
  700. */
  701. static void ufile_destroy_ucontext(struct ib_uverbs_file *ufile,
  702. enum rdma_remove_reason reason)
  703. {
  704. struct ib_ucontext *ucontext = ufile->ucontext;
  705. struct ib_device *ib_dev = ucontext->device;
  706. /*
  707. * If we are closing the FD then the user mmap VMAs must have
  708. * already been destroyed as they hold on to the filep, otherwise
  709. * they need to be zap'd.
  710. */
  711. if (reason == RDMA_REMOVE_DRIVER_REMOVE) {
  712. uverbs_user_mmap_disassociate(ufile);
  713. if (ib_dev->ops.disassociate_ucontext)
  714. ib_dev->ops.disassociate_ucontext(ucontext);
  715. }
  716. ib_rdmacg_uncharge(&ucontext->cg_obj, ib_dev,
  717. RDMACG_RESOURCE_HCA_HANDLE);
  718. rdma_restrack_del(&ucontext->res);
  719. ib_dev->ops.dealloc_ucontext(ucontext);
  720. WARN_ON(!xa_empty(&ucontext->mmap_xa));
  721. kfree(ucontext);
  722. ufile->ucontext = NULL;
  723. }
  724. static int __uverbs_cleanup_ufile(struct ib_uverbs_file *ufile,
  725. enum rdma_remove_reason reason)
  726. {
  727. struct ib_uobject *obj, *next_obj;
  728. int ret = -EINVAL;
  729. struct uverbs_attr_bundle attrs = { .ufile = ufile };
  730. /*
  731. * This shouldn't run while executing other commands on this
  732. * context. Thus, the only thing we should take care of is
  733. * releasing a FD while traversing this list. The FD could be
  734. * closed and released from the _release fop of this FD.
  735. * In order to mitigate this, we add a lock.
  736. * We take and release the lock per traversal in order to let
  737. * other threads (which might still use the FDs) chance to run.
  738. */
  739. list_for_each_entry_safe(obj, next_obj, &ufile->uobjects, list) {
  740. attrs.context = obj->context;
  741. /*
  742. * if we hit this WARN_ON, that means we are
  743. * racing with a lookup_get.
  744. */
  745. WARN_ON(uverbs_try_lock_object(obj, UVERBS_LOOKUP_WRITE));
  746. if (!uverbs_destroy_uobject(obj, reason, &attrs))
  747. ret = 0;
  748. else
  749. atomic_set(&obj->usecnt, 0);
  750. }
  751. return ret;
  752. }
  753. /*
  754. * Destroy the uncontext and every uobject associated with it.
  755. *
  756. * This is internally locked and can be called in parallel from multiple
  757. * contexts.
  758. */
  759. void uverbs_destroy_ufile_hw(struct ib_uverbs_file *ufile,
  760. enum rdma_remove_reason reason)
  761. {
  762. down_write(&ufile->hw_destroy_rwsem);
  763. /*
  764. * If a ucontext was never created then we can't have any uobjects to
  765. * cleanup, nothing to do.
  766. */
  767. if (!ufile->ucontext)
  768. goto done;
  769. ufile->ucontext->cleanup_retryable = true;
  770. while (!list_empty(&ufile->uobjects))
  771. if (__uverbs_cleanup_ufile(ufile, reason)) {
  772. /*
  773. * No entry was cleaned-up successfully during this
  774. * iteration. It is a driver bug to fail destruction.
  775. */
  776. WARN_ON(!list_empty(&ufile->uobjects));
  777. break;
  778. }
  779. ufile->ucontext->cleanup_retryable = false;
  780. if (!list_empty(&ufile->uobjects))
  781. __uverbs_cleanup_ufile(ufile, reason);
  782. ufile_destroy_ucontext(ufile, reason);
  783. done:
  784. up_write(&ufile->hw_destroy_rwsem);
  785. }
  786. const struct uverbs_obj_type_class uverbs_fd_class = {
  787. .alloc_begin = alloc_begin_fd_uobject,
  788. .lookup_get = lookup_get_fd_uobject,
  789. .alloc_commit = alloc_commit_fd_uobject,
  790. .alloc_abort = alloc_abort_fd_uobject,
  791. .lookup_put = lookup_put_fd_uobject,
  792. .destroy_hw = destroy_hw_fd_uobject,
  793. .remove_handle = remove_handle_fd_uobject,
  794. };
  795. EXPORT_SYMBOL(uverbs_fd_class);
  796. struct ib_uobject *
  797. uverbs_get_uobject_from_file(u16 object_id, enum uverbs_obj_access access,
  798. s64 id, struct uverbs_attr_bundle *attrs)
  799. {
  800. const struct uverbs_api_object *obj =
  801. uapi_get_object(attrs->ufile->device->uapi, object_id);
  802. switch (access) {
  803. case UVERBS_ACCESS_READ:
  804. return rdma_lookup_get_uobject(obj, attrs->ufile, id,
  805. UVERBS_LOOKUP_READ, attrs);
  806. case UVERBS_ACCESS_DESTROY:
  807. /* Actual destruction is done inside uverbs_handle_method */
  808. return rdma_lookup_get_uobject(obj, attrs->ufile, id,
  809. UVERBS_LOOKUP_DESTROY, attrs);
  810. case UVERBS_ACCESS_WRITE:
  811. return rdma_lookup_get_uobject(obj, attrs->ufile, id,
  812. UVERBS_LOOKUP_WRITE, attrs);
  813. case UVERBS_ACCESS_NEW:
  814. return rdma_alloc_begin_uobject(obj, attrs);
  815. default:
  816. WARN_ON(true);
  817. return ERR_PTR(-EOPNOTSUPP);
  818. }
  819. }
  820. void uverbs_finalize_object(struct ib_uobject *uobj,
  821. enum uverbs_obj_access access, bool hw_obj_valid,
  822. bool commit, struct uverbs_attr_bundle *attrs)
  823. {
  824. /*
  825. * refcounts should be handled at the object level and not at the
  826. * uobject level. Refcounts of the objects themselves are done in
  827. * handlers.
  828. */
  829. switch (access) {
  830. case UVERBS_ACCESS_READ:
  831. rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_READ);
  832. break;
  833. case UVERBS_ACCESS_WRITE:
  834. rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_WRITE);
  835. break;
  836. case UVERBS_ACCESS_DESTROY:
  837. if (uobj)
  838. rdma_lookup_put_uobject(uobj, UVERBS_LOOKUP_DESTROY);
  839. break;
  840. case UVERBS_ACCESS_NEW:
  841. if (commit)
  842. rdma_alloc_commit_uobject(uobj, attrs);
  843. else
  844. rdma_alloc_abort_uobject(uobj, attrs, hw_obj_valid);
  845. break;
  846. default:
  847. WARN_ON(true);
  848. }
  849. }