blk-ioc.c 10 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Functions related to io context handling
  4. */
  5. #include <linux/kernel.h>
  6. #include <linux/module.h>
  7. #include <linux/init.h>
  8. #include <linux/bio.h>
  9. #include <linux/blkdev.h>
  10. #include <linux/slab.h>
  11. #include <linux/sched/task.h>
  12. #include "blk.h"
  13. /*
  14. * For io context allocations
  15. */
  16. static struct kmem_cache *iocontext_cachep;
  17. /**
  18. * get_io_context - increment reference count to io_context
  19. * @ioc: io_context to get
  20. *
  21. * Increment reference count to @ioc.
  22. */
  23. void get_io_context(struct io_context *ioc)
  24. {
  25. BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
  26. atomic_long_inc(&ioc->refcount);
  27. }
  28. static void icq_free_icq_rcu(struct rcu_head *head)
  29. {
  30. struct io_cq *icq = container_of(head, struct io_cq, __rcu_head);
  31. kmem_cache_free(icq->__rcu_icq_cache, icq);
  32. }
  33. /*
  34. * Exit an icq. Called with ioc locked for blk-mq, and with both ioc
  35. * and queue locked for legacy.
  36. */
  37. static void ioc_exit_icq(struct io_cq *icq)
  38. {
  39. struct elevator_type *et = icq->q->elevator->type;
  40. if (icq->flags & ICQ_EXITED)
  41. return;
  42. if (et->ops.exit_icq)
  43. et->ops.exit_icq(icq);
  44. icq->flags |= ICQ_EXITED;
  45. }
  46. /*
  47. * Release an icq. Called with ioc locked for blk-mq, and with both ioc
  48. * and queue locked for legacy.
  49. */
  50. static void ioc_destroy_icq(struct io_cq *icq)
  51. {
  52. struct io_context *ioc = icq->ioc;
  53. struct request_queue *q = icq->q;
  54. struct elevator_type *et = q->elevator->type;
  55. lockdep_assert_held(&ioc->lock);
  56. radix_tree_delete(&ioc->icq_tree, icq->q->id);
  57. hlist_del_init(&icq->ioc_node);
  58. list_del_init(&icq->q_node);
  59. /*
  60. * Both setting lookup hint to and clearing it from @icq are done
  61. * under queue_lock. If it's not pointing to @icq now, it never
  62. * will. Hint assignment itself can race safely.
  63. */
  64. if (rcu_access_pointer(ioc->icq_hint) == icq)
  65. rcu_assign_pointer(ioc->icq_hint, NULL);
  66. ioc_exit_icq(icq);
  67. /*
  68. * @icq->q might have gone away by the time RCU callback runs
  69. * making it impossible to determine icq_cache. Record it in @icq.
  70. */
  71. icq->__rcu_icq_cache = et->icq_cache;
  72. icq->flags |= ICQ_DESTROYED;
  73. call_rcu(&icq->__rcu_head, icq_free_icq_rcu);
  74. }
  75. /*
  76. * Slow path for ioc release in put_io_context(). Performs double-lock
  77. * dancing to unlink all icq's and then frees ioc.
  78. */
  79. static void ioc_release_fn(struct work_struct *work)
  80. {
  81. struct io_context *ioc = container_of(work, struct io_context,
  82. release_work);
  83. spin_lock_irq(&ioc->lock);
  84. while (!hlist_empty(&ioc->icq_list)) {
  85. struct io_cq *icq = hlist_entry(ioc->icq_list.first,
  86. struct io_cq, ioc_node);
  87. struct request_queue *q = icq->q;
  88. if (spin_trylock(&q->queue_lock)) {
  89. ioc_destroy_icq(icq);
  90. spin_unlock(&q->queue_lock);
  91. } else {
  92. /* Make sure q and icq cannot be freed. */
  93. rcu_read_lock();
  94. /* Re-acquire the locks in the correct order. */
  95. spin_unlock(&ioc->lock);
  96. spin_lock(&q->queue_lock);
  97. spin_lock(&ioc->lock);
  98. /*
  99. * The icq may have been destroyed when the ioc lock
  100. * was released.
  101. */
  102. if (!(icq->flags & ICQ_DESTROYED))
  103. ioc_destroy_icq(icq);
  104. spin_unlock(&q->queue_lock);
  105. rcu_read_unlock();
  106. }
  107. }
  108. spin_unlock_irq(&ioc->lock);
  109. kmem_cache_free(iocontext_cachep, ioc);
  110. }
  111. /**
  112. * put_io_context - put a reference of io_context
  113. * @ioc: io_context to put
  114. *
  115. * Decrement reference count of @ioc and release it if the count reaches
  116. * zero.
  117. */
  118. void put_io_context(struct io_context *ioc)
  119. {
  120. unsigned long flags;
  121. bool free_ioc = false;
  122. if (ioc == NULL)
  123. return;
  124. BUG_ON(atomic_long_read(&ioc->refcount) <= 0);
  125. /*
  126. * Releasing ioc requires reverse order double locking and we may
  127. * already be holding a queue_lock. Do it asynchronously from wq.
  128. */
  129. if (atomic_long_dec_and_test(&ioc->refcount)) {
  130. spin_lock_irqsave(&ioc->lock, flags);
  131. if (!hlist_empty(&ioc->icq_list))
  132. queue_work(system_power_efficient_wq,
  133. &ioc->release_work);
  134. else
  135. free_ioc = true;
  136. spin_unlock_irqrestore(&ioc->lock, flags);
  137. }
  138. if (free_ioc)
  139. kmem_cache_free(iocontext_cachep, ioc);
  140. }
  141. /**
  142. * put_io_context_active - put active reference on ioc
  143. * @ioc: ioc of interest
  144. *
  145. * Undo get_io_context_active(). If active reference reaches zero after
  146. * put, @ioc can never issue further IOs and ioscheds are notified.
  147. */
  148. void put_io_context_active(struct io_context *ioc)
  149. {
  150. struct io_cq *icq;
  151. if (!atomic_dec_and_test(&ioc->active_ref)) {
  152. put_io_context(ioc);
  153. return;
  154. }
  155. spin_lock_irq(&ioc->lock);
  156. hlist_for_each_entry(icq, &ioc->icq_list, ioc_node) {
  157. if (icq->flags & ICQ_EXITED)
  158. continue;
  159. ioc_exit_icq(icq);
  160. }
  161. spin_unlock_irq(&ioc->lock);
  162. put_io_context(ioc);
  163. }
  164. /* Called by the exiting task */
  165. void exit_io_context(struct task_struct *task)
  166. {
  167. struct io_context *ioc;
  168. task_lock(task);
  169. ioc = task->io_context;
  170. task->io_context = NULL;
  171. task_unlock(task);
  172. atomic_dec(&ioc->nr_tasks);
  173. put_io_context_active(ioc);
  174. }
  175. static void __ioc_clear_queue(struct list_head *icq_list)
  176. {
  177. unsigned long flags;
  178. rcu_read_lock();
  179. while (!list_empty(icq_list)) {
  180. struct io_cq *icq = list_entry(icq_list->next,
  181. struct io_cq, q_node);
  182. struct io_context *ioc = icq->ioc;
  183. spin_lock_irqsave(&ioc->lock, flags);
  184. if (icq->flags & ICQ_DESTROYED) {
  185. spin_unlock_irqrestore(&ioc->lock, flags);
  186. continue;
  187. }
  188. ioc_destroy_icq(icq);
  189. spin_unlock_irqrestore(&ioc->lock, flags);
  190. }
  191. rcu_read_unlock();
  192. }
  193. /**
  194. * ioc_clear_queue - break any ioc association with the specified queue
  195. * @q: request_queue being cleared
  196. *
  197. * Walk @q->icq_list and exit all io_cq's.
  198. */
  199. void ioc_clear_queue(struct request_queue *q)
  200. {
  201. LIST_HEAD(icq_list);
  202. spin_lock_irq(&q->queue_lock);
  203. list_splice_init(&q->icq_list, &icq_list);
  204. spin_unlock_irq(&q->queue_lock);
  205. __ioc_clear_queue(&icq_list);
  206. }
  207. int create_task_io_context(struct task_struct *task, gfp_t gfp_flags, int node)
  208. {
  209. struct io_context *ioc;
  210. int ret;
  211. ioc = kmem_cache_alloc_node(iocontext_cachep, gfp_flags | __GFP_ZERO,
  212. node);
  213. if (unlikely(!ioc))
  214. return -ENOMEM;
  215. /* initialize */
  216. atomic_long_set(&ioc->refcount, 1);
  217. atomic_set(&ioc->nr_tasks, 1);
  218. atomic_set(&ioc->active_ref, 1);
  219. spin_lock_init(&ioc->lock);
  220. INIT_RADIX_TREE(&ioc->icq_tree, GFP_ATOMIC);
  221. INIT_HLIST_HEAD(&ioc->icq_list);
  222. INIT_WORK(&ioc->release_work, ioc_release_fn);
  223. /*
  224. * Try to install. ioc shouldn't be installed if someone else
  225. * already did or @task, which isn't %current, is exiting. Note
  226. * that we need to allow ioc creation on exiting %current as exit
  227. * path may issue IOs from e.g. exit_files(). The exit path is
  228. * responsible for not issuing IO after exit_io_context().
  229. */
  230. task_lock(task);
  231. if (!task->io_context &&
  232. (task == current || !(task->flags & PF_EXITING)))
  233. task->io_context = ioc;
  234. else
  235. kmem_cache_free(iocontext_cachep, ioc);
  236. ret = task->io_context ? 0 : -EBUSY;
  237. task_unlock(task);
  238. return ret;
  239. }
  240. /**
  241. * get_task_io_context - get io_context of a task
  242. * @task: task of interest
  243. * @gfp_flags: allocation flags, used if allocation is necessary
  244. * @node: allocation node, used if allocation is necessary
  245. *
  246. * Return io_context of @task. If it doesn't exist, it is created with
  247. * @gfp_flags and @node. The returned io_context has its reference count
  248. * incremented.
  249. *
  250. * This function always goes through task_lock() and it's better to use
  251. * %current->io_context + get_io_context() for %current.
  252. */
  253. struct io_context *get_task_io_context(struct task_struct *task,
  254. gfp_t gfp_flags, int node)
  255. {
  256. struct io_context *ioc;
  257. might_sleep_if(gfpflags_allow_blocking(gfp_flags));
  258. do {
  259. task_lock(task);
  260. ioc = task->io_context;
  261. if (likely(ioc)) {
  262. get_io_context(ioc);
  263. task_unlock(task);
  264. return ioc;
  265. }
  266. task_unlock(task);
  267. } while (!create_task_io_context(task, gfp_flags, node));
  268. return NULL;
  269. }
  270. /**
  271. * ioc_lookup_icq - lookup io_cq from ioc
  272. * @ioc: the associated io_context
  273. * @q: the associated request_queue
  274. *
  275. * Look up io_cq associated with @ioc - @q pair from @ioc. Must be called
  276. * with @q->queue_lock held.
  277. */
  278. struct io_cq *ioc_lookup_icq(struct io_context *ioc, struct request_queue *q)
  279. {
  280. struct io_cq *icq;
  281. lockdep_assert_held(&q->queue_lock);
  282. /*
  283. * icq's are indexed from @ioc using radix tree and hint pointer,
  284. * both of which are protected with RCU. All removals are done
  285. * holding both q and ioc locks, and we're holding q lock - if we
  286. * find a icq which points to us, it's guaranteed to be valid.
  287. */
  288. rcu_read_lock();
  289. icq = rcu_dereference(ioc->icq_hint);
  290. if (icq && icq->q == q)
  291. goto out;
  292. icq = radix_tree_lookup(&ioc->icq_tree, q->id);
  293. if (icq && icq->q == q)
  294. rcu_assign_pointer(ioc->icq_hint, icq); /* allowed to race */
  295. else
  296. icq = NULL;
  297. out:
  298. rcu_read_unlock();
  299. return icq;
  300. }
  301. EXPORT_SYMBOL(ioc_lookup_icq);
  302. /**
  303. * ioc_create_icq - create and link io_cq
  304. * @ioc: io_context of interest
  305. * @q: request_queue of interest
  306. * @gfp_mask: allocation mask
  307. *
  308. * Make sure io_cq linking @ioc and @q exists. If icq doesn't exist, they
  309. * will be created using @gfp_mask.
  310. *
  311. * The caller is responsible for ensuring @ioc won't go away and @q is
  312. * alive and will stay alive until this function returns.
  313. */
  314. struct io_cq *ioc_create_icq(struct io_context *ioc, struct request_queue *q,
  315. gfp_t gfp_mask)
  316. {
  317. struct elevator_type *et = q->elevator->type;
  318. struct io_cq *icq;
  319. /* allocate stuff */
  320. icq = kmem_cache_alloc_node(et->icq_cache, gfp_mask | __GFP_ZERO,
  321. q->node);
  322. if (!icq)
  323. return NULL;
  324. if (radix_tree_maybe_preload(gfp_mask) < 0) {
  325. kmem_cache_free(et->icq_cache, icq);
  326. return NULL;
  327. }
  328. icq->ioc = ioc;
  329. icq->q = q;
  330. INIT_LIST_HEAD(&icq->q_node);
  331. INIT_HLIST_NODE(&icq->ioc_node);
  332. /* lock both q and ioc and try to link @icq */
  333. spin_lock_irq(&q->queue_lock);
  334. spin_lock(&ioc->lock);
  335. if (likely(!radix_tree_insert(&ioc->icq_tree, q->id, icq))) {
  336. hlist_add_head(&icq->ioc_node, &ioc->icq_list);
  337. list_add(&icq->q_node, &q->icq_list);
  338. if (et->ops.init_icq)
  339. et->ops.init_icq(icq);
  340. } else {
  341. kmem_cache_free(et->icq_cache, icq);
  342. icq = ioc_lookup_icq(ioc, q);
  343. if (!icq)
  344. printk(KERN_ERR "cfq: icq link failed!\n");
  345. }
  346. spin_unlock(&ioc->lock);
  347. spin_unlock_irq(&q->queue_lock);
  348. radix_tree_preload_end();
  349. return icq;
  350. }
  351. static int __init blk_ioc_init(void)
  352. {
  353. iocontext_cachep = kmem_cache_create("blkdev_ioc",
  354. sizeof(struct io_context), 0, SLAB_PANIC, NULL);
  355. return 0;
  356. }
  357. subsys_initcall(blk_ioc_init);