blk-cgroup.c 50 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Common Block IO controller cgroup interface
  4. *
  5. * Based on ideas and code from CFQ, CFS and BFQ:
  6. * Copyright (C) 2003 Jens Axboe <axboe@kernel.dk>
  7. *
  8. * Copyright (C) 2008 Fabio Checconi <fabio@gandalf.sssup.it>
  9. * Paolo Valente <paolo.valente@unimore.it>
  10. *
  11. * Copyright (C) 2009 Vivek Goyal <vgoyal@redhat.com>
  12. * Nauman Rafique <nauman@google.com>
  13. *
  14. * For policy-specific per-blkcg data:
  15. * Copyright (C) 2015 Paolo Valente <paolo.valente@unimore.it>
  16. * Arianna Avanzini <avanzini.arianna@gmail.com>
  17. */
  18. #include <linux/ioprio.h>
  19. #include <linux/kdev_t.h>
  20. #include <linux/module.h>
  21. #include <linux/sched/signal.h>
  22. #include <linux/err.h>
  23. #include <linux/blkdev.h>
  24. #include <linux/backing-dev.h>
  25. #include <linux/slab.h>
  26. #include <linux/genhd.h>
  27. #include <linux/delay.h>
  28. #include <linux/atomic.h>
  29. #include <linux/ctype.h>
  30. #include <linux/blk-cgroup.h>
  31. #include <linux/tracehook.h>
  32. #include <linux/psi.h>
  33. #include "blk.h"
  34. #include "blk-ioprio.h"
  35. #define MAX_KEY_LEN 100
  36. /*
  37. * blkcg_pol_mutex protects blkcg_policy[] and policy [de]activation.
  38. * blkcg_pol_register_mutex nests outside of it and synchronizes entire
  39. * policy [un]register operations including cgroup file additions /
  40. * removals. Putting cgroup file registration outside blkcg_pol_mutex
  41. * allows grabbing it from cgroup callbacks.
  42. */
  43. static DEFINE_MUTEX(blkcg_pol_register_mutex);
  44. static DEFINE_MUTEX(blkcg_pol_mutex);
  45. struct blkcg blkcg_root;
  46. EXPORT_SYMBOL_GPL(blkcg_root);
  47. struct cgroup_subsys_state * const blkcg_root_css = &blkcg_root.css;
  48. EXPORT_SYMBOL_GPL(blkcg_root_css);
  49. static struct blkcg_policy *blkcg_policy[BLKCG_MAX_POLS];
  50. static LIST_HEAD(all_blkcgs); /* protected by blkcg_pol_mutex */
  51. bool blkcg_debug_stats = false;
  52. static struct workqueue_struct *blkcg_punt_bio_wq;
  53. static bool blkcg_policy_enabled(struct request_queue *q,
  54. const struct blkcg_policy *pol)
  55. {
  56. return pol && test_bit(pol->plid, q->blkcg_pols);
  57. }
  58. /**
  59. * blkg_free - free a blkg
  60. * @blkg: blkg to free
  61. *
  62. * Free @blkg which may be partially allocated.
  63. */
  64. static void blkg_free(struct blkcg_gq *blkg)
  65. {
  66. int i;
  67. if (!blkg)
  68. return;
  69. for (i = 0; i < BLKCG_MAX_POLS; i++)
  70. if (blkg->pd[i])
  71. blkcg_policy[i]->pd_free_fn(blkg->pd[i]);
  72. free_percpu(blkg->iostat_cpu);
  73. percpu_ref_exit(&blkg->refcnt);
  74. kfree(blkg);
  75. }
  76. static void __blkg_release(struct rcu_head *rcu)
  77. {
  78. struct blkcg_gq *blkg = container_of(rcu, struct blkcg_gq, rcu_head);
  79. WARN_ON(!bio_list_empty(&blkg->async_bios));
  80. /* release the blkcg and parent blkg refs this blkg has been holding */
  81. css_put(&blkg->blkcg->css);
  82. if (blkg->parent)
  83. blkg_put(blkg->parent);
  84. blkg_free(blkg);
  85. }
  86. /*
  87. * A group is RCU protected, but having an rcu lock does not mean that one
  88. * can access all the fields of blkg and assume these are valid. For
  89. * example, don't try to follow throtl_data and request queue links.
  90. *
  91. * Having a reference to blkg under an rcu allows accesses to only values
  92. * local to groups like group stats and group rate limits.
  93. */
  94. static void blkg_release(struct percpu_ref *ref)
  95. {
  96. struct blkcg_gq *blkg = container_of(ref, struct blkcg_gq, refcnt);
  97. call_rcu(&blkg->rcu_head, __blkg_release);
  98. }
  99. static void blkg_async_bio_workfn(struct work_struct *work)
  100. {
  101. struct blkcg_gq *blkg = container_of(work, struct blkcg_gq,
  102. async_bio_work);
  103. struct bio_list bios = BIO_EMPTY_LIST;
  104. struct bio *bio;
  105. struct blk_plug plug;
  106. bool need_plug = false;
  107. /* as long as there are pending bios, @blkg can't go away */
  108. spin_lock_bh(&blkg->async_bio_lock);
  109. bio_list_merge(&bios, &blkg->async_bios);
  110. bio_list_init(&blkg->async_bios);
  111. spin_unlock_bh(&blkg->async_bio_lock);
  112. /* start plug only when bio_list contains at least 2 bios */
  113. if (bios.head && bios.head->bi_next) {
  114. need_plug = true;
  115. blk_start_plug(&plug);
  116. }
  117. while ((bio = bio_list_pop(&bios)))
  118. submit_bio(bio);
  119. if (need_plug)
  120. blk_finish_plug(&plug);
  121. }
  122. /**
  123. * blkg_alloc - allocate a blkg
  124. * @blkcg: block cgroup the new blkg is associated with
  125. * @q: request_queue the new blkg is associated with
  126. * @gfp_mask: allocation mask to use
  127. *
  128. * Allocate a new blkg assocating @blkcg and @q.
  129. */
  130. static struct blkcg_gq *blkg_alloc(struct blkcg *blkcg, struct request_queue *q,
  131. gfp_t gfp_mask)
  132. {
  133. struct blkcg_gq *blkg;
  134. int i, cpu;
  135. /* alloc and init base part */
  136. blkg = kzalloc_node(sizeof(*blkg), gfp_mask, q->node);
  137. if (!blkg)
  138. return NULL;
  139. if (percpu_ref_init(&blkg->refcnt, blkg_release, 0, gfp_mask))
  140. goto err_free;
  141. blkg->iostat_cpu = alloc_percpu_gfp(struct blkg_iostat_set, gfp_mask);
  142. if (!blkg->iostat_cpu)
  143. goto err_free;
  144. blkg->q = q;
  145. INIT_LIST_HEAD(&blkg->q_node);
  146. spin_lock_init(&blkg->async_bio_lock);
  147. bio_list_init(&blkg->async_bios);
  148. INIT_WORK(&blkg->async_bio_work, blkg_async_bio_workfn);
  149. blkg->blkcg = blkcg;
  150. u64_stats_init(&blkg->iostat.sync);
  151. for_each_possible_cpu(cpu)
  152. u64_stats_init(&per_cpu_ptr(blkg->iostat_cpu, cpu)->sync);
  153. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  154. struct blkcg_policy *pol = blkcg_policy[i];
  155. struct blkg_policy_data *pd;
  156. if (!blkcg_policy_enabled(q, pol))
  157. continue;
  158. /* alloc per-policy data and attach it to blkg */
  159. pd = pol->pd_alloc_fn(gfp_mask, q, blkcg);
  160. if (!pd)
  161. goto err_free;
  162. blkg->pd[i] = pd;
  163. pd->blkg = blkg;
  164. pd->plid = i;
  165. }
  166. return blkg;
  167. err_free:
  168. blkg_free(blkg);
  169. return NULL;
  170. }
  171. struct blkcg_gq *blkg_lookup_slowpath(struct blkcg *blkcg,
  172. struct request_queue *q, bool update_hint)
  173. {
  174. struct blkcg_gq *blkg;
  175. /*
  176. * Hint didn't match. Look up from the radix tree. Note that the
  177. * hint can only be updated under queue_lock as otherwise @blkg
  178. * could have already been removed from blkg_tree. The caller is
  179. * responsible for grabbing queue_lock if @update_hint.
  180. */
  181. blkg = radix_tree_lookup(&blkcg->blkg_tree, q->id);
  182. if (blkg && blkg->q == q) {
  183. if (update_hint) {
  184. lockdep_assert_held(&q->queue_lock);
  185. rcu_assign_pointer(blkcg->blkg_hint, blkg);
  186. }
  187. return blkg;
  188. }
  189. return NULL;
  190. }
  191. EXPORT_SYMBOL_GPL(blkg_lookup_slowpath);
  192. /*
  193. * If @new_blkg is %NULL, this function tries to allocate a new one as
  194. * necessary using %GFP_NOWAIT. @new_blkg is always consumed on return.
  195. */
  196. static struct blkcg_gq *blkg_create(struct blkcg *blkcg,
  197. struct request_queue *q,
  198. struct blkcg_gq *new_blkg)
  199. {
  200. struct blkcg_gq *blkg;
  201. int i, ret;
  202. WARN_ON_ONCE(!rcu_read_lock_held());
  203. lockdep_assert_held(&q->queue_lock);
  204. /* request_queue is dying, do not create/recreate a blkg */
  205. if (blk_queue_dying(q)) {
  206. ret = -ENODEV;
  207. goto err_free_blkg;
  208. }
  209. /* blkg holds a reference to blkcg */
  210. if (!css_tryget_online(&blkcg->css)) {
  211. ret = -ENODEV;
  212. goto err_free_blkg;
  213. }
  214. /* allocate */
  215. if (!new_blkg) {
  216. new_blkg = blkg_alloc(blkcg, q, GFP_NOWAIT | __GFP_NOWARN);
  217. if (unlikely(!new_blkg)) {
  218. ret = -ENOMEM;
  219. goto err_put_css;
  220. }
  221. }
  222. blkg = new_blkg;
  223. /* link parent */
  224. if (blkcg_parent(blkcg)) {
  225. blkg->parent = __blkg_lookup(blkcg_parent(blkcg), q, false);
  226. if (WARN_ON_ONCE(!blkg->parent)) {
  227. ret = -ENODEV;
  228. goto err_put_css;
  229. }
  230. blkg_get(blkg->parent);
  231. }
  232. /* invoke per-policy init */
  233. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  234. struct blkcg_policy *pol = blkcg_policy[i];
  235. if (blkg->pd[i] && pol->pd_init_fn)
  236. pol->pd_init_fn(blkg->pd[i]);
  237. }
  238. /* insert */
  239. spin_lock(&blkcg->lock);
  240. ret = radix_tree_insert(&blkcg->blkg_tree, q->id, blkg);
  241. if (likely(!ret)) {
  242. hlist_add_head_rcu(&blkg->blkcg_node, &blkcg->blkg_list);
  243. list_add(&blkg->q_node, &q->blkg_list);
  244. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  245. struct blkcg_policy *pol = blkcg_policy[i];
  246. if (blkg->pd[i] && pol->pd_online_fn)
  247. pol->pd_online_fn(blkg->pd[i]);
  248. }
  249. }
  250. blkg->online = true;
  251. spin_unlock(&blkcg->lock);
  252. if (!ret)
  253. return blkg;
  254. /* @blkg failed fully initialized, use the usual release path */
  255. blkg_put(blkg);
  256. return ERR_PTR(ret);
  257. err_put_css:
  258. css_put(&blkcg->css);
  259. err_free_blkg:
  260. blkg_free(new_blkg);
  261. return ERR_PTR(ret);
  262. }
  263. /**
  264. * blkg_lookup_create - lookup blkg, try to create one if not there
  265. * @blkcg: blkcg of interest
  266. * @q: request_queue of interest
  267. *
  268. * Lookup blkg for the @blkcg - @q pair. If it doesn't exist, try to
  269. * create one. blkg creation is performed recursively from blkcg_root such
  270. * that all non-root blkg's have access to the parent blkg. This function
  271. * should be called under RCU read lock and takes @q->queue_lock.
  272. *
  273. * Returns the blkg or the closest blkg if blkg_create() fails as it walks
  274. * down from root.
  275. */
  276. static struct blkcg_gq *blkg_lookup_create(struct blkcg *blkcg,
  277. struct request_queue *q)
  278. {
  279. struct blkcg_gq *blkg;
  280. unsigned long flags;
  281. WARN_ON_ONCE(!rcu_read_lock_held());
  282. blkg = blkg_lookup(blkcg, q);
  283. if (blkg)
  284. return blkg;
  285. spin_lock_irqsave(&q->queue_lock, flags);
  286. blkg = __blkg_lookup(blkcg, q, true);
  287. if (blkg)
  288. goto found;
  289. /*
  290. * Create blkgs walking down from blkcg_root to @blkcg, so that all
  291. * non-root blkgs have access to their parents. Returns the closest
  292. * blkg to the intended blkg should blkg_create() fail.
  293. */
  294. while (true) {
  295. struct blkcg *pos = blkcg;
  296. struct blkcg *parent = blkcg_parent(blkcg);
  297. struct blkcg_gq *ret_blkg = q->root_blkg;
  298. while (parent) {
  299. blkg = __blkg_lookup(parent, q, false);
  300. if (blkg) {
  301. /* remember closest blkg */
  302. ret_blkg = blkg;
  303. break;
  304. }
  305. pos = parent;
  306. parent = blkcg_parent(parent);
  307. }
  308. blkg = blkg_create(pos, q, NULL);
  309. if (IS_ERR(blkg)) {
  310. blkg = ret_blkg;
  311. break;
  312. }
  313. if (pos == blkcg)
  314. break;
  315. }
  316. found:
  317. spin_unlock_irqrestore(&q->queue_lock, flags);
  318. return blkg;
  319. }
  320. static void blkg_destroy(struct blkcg_gq *blkg)
  321. {
  322. struct blkcg *blkcg = blkg->blkcg;
  323. int i;
  324. lockdep_assert_held(&blkg->q->queue_lock);
  325. lockdep_assert_held(&blkcg->lock);
  326. /* Something wrong if we are trying to remove same group twice */
  327. WARN_ON_ONCE(list_empty(&blkg->q_node));
  328. WARN_ON_ONCE(hlist_unhashed(&blkg->blkcg_node));
  329. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  330. struct blkcg_policy *pol = blkcg_policy[i];
  331. if (blkg->pd[i] && pol->pd_offline_fn)
  332. pol->pd_offline_fn(blkg->pd[i]);
  333. }
  334. blkg->online = false;
  335. radix_tree_delete(&blkcg->blkg_tree, blkg->q->id);
  336. list_del_init(&blkg->q_node);
  337. hlist_del_init_rcu(&blkg->blkcg_node);
  338. /*
  339. * Both setting lookup hint to and clearing it from @blkg are done
  340. * under queue_lock. If it's not pointing to @blkg now, it never
  341. * will. Hint assignment itself can race safely.
  342. */
  343. if (rcu_access_pointer(blkcg->blkg_hint) == blkg)
  344. rcu_assign_pointer(blkcg->blkg_hint, NULL);
  345. /*
  346. * Put the reference taken at the time of creation so that when all
  347. * queues are gone, group can be destroyed.
  348. */
  349. percpu_ref_kill(&blkg->refcnt);
  350. }
  351. /**
  352. * blkg_destroy_all - destroy all blkgs associated with a request_queue
  353. * @q: request_queue of interest
  354. *
  355. * Destroy all blkgs associated with @q.
  356. */
  357. static void blkg_destroy_all(struct request_queue *q)
  358. {
  359. struct blkcg_gq *blkg, *n;
  360. spin_lock_irq(&q->queue_lock);
  361. list_for_each_entry_safe(blkg, n, &q->blkg_list, q_node) {
  362. struct blkcg *blkcg = blkg->blkcg;
  363. spin_lock(&blkcg->lock);
  364. blkg_destroy(blkg);
  365. spin_unlock(&blkcg->lock);
  366. }
  367. q->root_blkg = NULL;
  368. spin_unlock_irq(&q->queue_lock);
  369. }
  370. static int blkcg_reset_stats(struct cgroup_subsys_state *css,
  371. struct cftype *cftype, u64 val)
  372. {
  373. struct blkcg *blkcg = css_to_blkcg(css);
  374. struct blkcg_gq *blkg;
  375. int i, cpu;
  376. mutex_lock(&blkcg_pol_mutex);
  377. spin_lock_irq(&blkcg->lock);
  378. /*
  379. * Note that stat reset is racy - it doesn't synchronize against
  380. * stat updates. This is a debug feature which shouldn't exist
  381. * anyway. If you get hit by a race, retry.
  382. */
  383. hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
  384. for_each_possible_cpu(cpu) {
  385. struct blkg_iostat_set *bis =
  386. per_cpu_ptr(blkg->iostat_cpu, cpu);
  387. memset(bis, 0, sizeof(*bis));
  388. }
  389. memset(&blkg->iostat, 0, sizeof(blkg->iostat));
  390. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  391. struct blkcg_policy *pol = blkcg_policy[i];
  392. if (blkg->pd[i] && pol->pd_reset_stats_fn)
  393. pol->pd_reset_stats_fn(blkg->pd[i]);
  394. }
  395. }
  396. spin_unlock_irq(&blkcg->lock);
  397. mutex_unlock(&blkcg_pol_mutex);
  398. return 0;
  399. }
  400. const char *blkg_dev_name(struct blkcg_gq *blkg)
  401. {
  402. /* some drivers (floppy) instantiate a queue w/o disk registered */
  403. if (blkg->q->backing_dev_info->dev)
  404. return bdi_dev_name(blkg->q->backing_dev_info);
  405. return NULL;
  406. }
  407. /**
  408. * blkcg_print_blkgs - helper for printing per-blkg data
  409. * @sf: seq_file to print to
  410. * @blkcg: blkcg of interest
  411. * @prfill: fill function to print out a blkg
  412. * @pol: policy in question
  413. * @data: data to be passed to @prfill
  414. * @show_total: to print out sum of prfill return values or not
  415. *
  416. * This function invokes @prfill on each blkg of @blkcg if pd for the
  417. * policy specified by @pol exists. @prfill is invoked with @sf, the
  418. * policy data and @data and the matching queue lock held. If @show_total
  419. * is %true, the sum of the return values from @prfill is printed with
  420. * "Total" label at the end.
  421. *
  422. * This is to be used to construct print functions for
  423. * cftype->read_seq_string method.
  424. */
  425. void blkcg_print_blkgs(struct seq_file *sf, struct blkcg *blkcg,
  426. u64 (*prfill)(struct seq_file *,
  427. struct blkg_policy_data *, int),
  428. const struct blkcg_policy *pol, int data,
  429. bool show_total)
  430. {
  431. struct blkcg_gq *blkg;
  432. u64 total = 0;
  433. rcu_read_lock();
  434. hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
  435. spin_lock_irq(&blkg->q->queue_lock);
  436. if (blkcg_policy_enabled(blkg->q, pol))
  437. total += prfill(sf, blkg->pd[pol->plid], data);
  438. spin_unlock_irq(&blkg->q->queue_lock);
  439. }
  440. rcu_read_unlock();
  441. if (show_total)
  442. seq_printf(sf, "Total %llu\n", (unsigned long long)total);
  443. }
  444. EXPORT_SYMBOL_GPL(blkcg_print_blkgs);
  445. /**
  446. * __blkg_prfill_u64 - prfill helper for a single u64 value
  447. * @sf: seq_file to print to
  448. * @pd: policy private data of interest
  449. * @v: value to print
  450. *
  451. * Print @v to @sf for the device assocaited with @pd.
  452. */
  453. u64 __blkg_prfill_u64(struct seq_file *sf, struct blkg_policy_data *pd, u64 v)
  454. {
  455. const char *dname = blkg_dev_name(pd->blkg);
  456. if (!dname)
  457. return 0;
  458. seq_printf(sf, "%s %llu\n", dname, (unsigned long long)v);
  459. return v;
  460. }
  461. EXPORT_SYMBOL_GPL(__blkg_prfill_u64);
  462. /* Performs queue bypass and policy enabled checks then looks up blkg. */
  463. static struct blkcg_gq *blkg_lookup_check(struct blkcg *blkcg,
  464. const struct blkcg_policy *pol,
  465. struct request_queue *q)
  466. {
  467. WARN_ON_ONCE(!rcu_read_lock_held());
  468. lockdep_assert_held(&q->queue_lock);
  469. if (!blkcg_policy_enabled(q, pol))
  470. return ERR_PTR(-EOPNOTSUPP);
  471. return __blkg_lookup(blkcg, q, true /* update_hint */);
  472. }
  473. /**
  474. * blkg_conf_prep - parse and prepare for per-blkg config update
  475. * @inputp: input string pointer
  476. *
  477. * Parse the device node prefix part, MAJ:MIN, of per-blkg config update
  478. * from @input and get and return the matching gendisk. *@inputp is
  479. * updated to point past the device node prefix. Returns an ERR_PTR()
  480. * value on error.
  481. *
  482. * Use this function iff blkg_conf_prep() can't be used for some reason.
  483. */
  484. struct gendisk *blkcg_conf_get_disk(char **inputp)
  485. {
  486. char *input = *inputp;
  487. unsigned int major, minor;
  488. struct gendisk *disk;
  489. int key_len, part;
  490. if (sscanf(input, "%u:%u%n", &major, &minor, &key_len) != 2)
  491. return ERR_PTR(-EINVAL);
  492. input += key_len;
  493. if (!isspace(*input))
  494. return ERR_PTR(-EINVAL);
  495. input = skip_spaces(input);
  496. disk = get_gendisk(MKDEV(major, minor), &part);
  497. if (!disk)
  498. return ERR_PTR(-ENODEV);
  499. if (part) {
  500. put_disk_and_module(disk);
  501. return ERR_PTR(-ENODEV);
  502. }
  503. *inputp = input;
  504. return disk;
  505. }
  506. /**
  507. * blkg_conf_prep - parse and prepare for per-blkg config update
  508. * @blkcg: target block cgroup
  509. * @pol: target policy
  510. * @input: input string
  511. * @ctx: blkg_conf_ctx to be filled
  512. *
  513. * Parse per-blkg config update from @input and initialize @ctx with the
  514. * result. @ctx->blkg points to the blkg to be updated and @ctx->body the
  515. * part of @input following MAJ:MIN. This function returns with RCU read
  516. * lock and queue lock held and must be paired with blkg_conf_finish().
  517. */
  518. int blkg_conf_prep(struct blkcg *blkcg, const struct blkcg_policy *pol,
  519. char *input, struct blkg_conf_ctx *ctx)
  520. __acquires(rcu) __acquires(&disk->queue->queue_lock)
  521. {
  522. struct gendisk *disk;
  523. struct request_queue *q;
  524. struct blkcg_gq *blkg;
  525. int ret;
  526. disk = blkcg_conf_get_disk(&input);
  527. if (IS_ERR(disk))
  528. return PTR_ERR(disk);
  529. q = disk->queue;
  530. rcu_read_lock();
  531. spin_lock_irq(&q->queue_lock);
  532. blkg = blkg_lookup_check(blkcg, pol, q);
  533. if (IS_ERR(blkg)) {
  534. ret = PTR_ERR(blkg);
  535. goto fail_unlock;
  536. }
  537. if (blkg)
  538. goto success;
  539. /*
  540. * Create blkgs walking down from blkcg_root to @blkcg, so that all
  541. * non-root blkgs have access to their parents.
  542. */
  543. while (true) {
  544. struct blkcg *pos = blkcg;
  545. struct blkcg *parent;
  546. struct blkcg_gq *new_blkg;
  547. parent = blkcg_parent(blkcg);
  548. while (parent && !__blkg_lookup(parent, q, false)) {
  549. pos = parent;
  550. parent = blkcg_parent(parent);
  551. }
  552. /* Drop locks to do new blkg allocation with GFP_KERNEL. */
  553. spin_unlock_irq(&q->queue_lock);
  554. rcu_read_unlock();
  555. new_blkg = blkg_alloc(pos, q, GFP_KERNEL);
  556. if (unlikely(!new_blkg)) {
  557. ret = -ENOMEM;
  558. goto fail;
  559. }
  560. if (radix_tree_preload(GFP_KERNEL)) {
  561. blkg_free(new_blkg);
  562. ret = -ENOMEM;
  563. goto fail;
  564. }
  565. rcu_read_lock();
  566. spin_lock_irq(&q->queue_lock);
  567. blkg = blkg_lookup_check(pos, pol, q);
  568. if (IS_ERR(blkg)) {
  569. ret = PTR_ERR(blkg);
  570. blkg_free(new_blkg);
  571. goto fail_preloaded;
  572. }
  573. if (blkg) {
  574. blkg_free(new_blkg);
  575. } else {
  576. blkg = blkg_create(pos, q, new_blkg);
  577. if (IS_ERR(blkg)) {
  578. ret = PTR_ERR(blkg);
  579. goto fail_preloaded;
  580. }
  581. }
  582. radix_tree_preload_end();
  583. if (pos == blkcg)
  584. goto success;
  585. }
  586. success:
  587. ctx->disk = disk;
  588. ctx->blkg = blkg;
  589. ctx->body = input;
  590. return 0;
  591. fail_preloaded:
  592. radix_tree_preload_end();
  593. fail_unlock:
  594. spin_unlock_irq(&q->queue_lock);
  595. rcu_read_unlock();
  596. fail:
  597. put_disk_and_module(disk);
  598. /*
  599. * If queue was bypassing, we should retry. Do so after a
  600. * short msleep(). It isn't strictly necessary but queue
  601. * can be bypassing for some time and it's always nice to
  602. * avoid busy looping.
  603. */
  604. if (ret == -EBUSY) {
  605. msleep(10);
  606. ret = restart_syscall();
  607. }
  608. return ret;
  609. }
  610. EXPORT_SYMBOL_GPL(blkg_conf_prep);
  611. /**
  612. * blkg_conf_finish - finish up per-blkg config update
  613. * @ctx: blkg_conf_ctx intiailized by blkg_conf_prep()
  614. *
  615. * Finish up after per-blkg config update. This function must be paired
  616. * with blkg_conf_prep().
  617. */
  618. void blkg_conf_finish(struct blkg_conf_ctx *ctx)
  619. __releases(&ctx->disk->queue->queue_lock) __releases(rcu)
  620. {
  621. spin_unlock_irq(&ctx->disk->queue->queue_lock);
  622. rcu_read_unlock();
  623. put_disk_and_module(ctx->disk);
  624. }
  625. EXPORT_SYMBOL_GPL(blkg_conf_finish);
  626. static void blkg_iostat_set(struct blkg_iostat *dst, struct blkg_iostat *src)
  627. {
  628. int i;
  629. for (i = 0; i < BLKG_IOSTAT_NR; i++) {
  630. dst->bytes[i] = src->bytes[i];
  631. dst->ios[i] = src->ios[i];
  632. }
  633. }
  634. static void blkg_iostat_add(struct blkg_iostat *dst, struct blkg_iostat *src)
  635. {
  636. int i;
  637. for (i = 0; i < BLKG_IOSTAT_NR; i++) {
  638. dst->bytes[i] += src->bytes[i];
  639. dst->ios[i] += src->ios[i];
  640. }
  641. }
  642. static void blkg_iostat_sub(struct blkg_iostat *dst, struct blkg_iostat *src)
  643. {
  644. int i;
  645. for (i = 0; i < BLKG_IOSTAT_NR; i++) {
  646. dst->bytes[i] -= src->bytes[i];
  647. dst->ios[i] -= src->ios[i];
  648. }
  649. }
  650. static void blkcg_rstat_flush(struct cgroup_subsys_state *css, int cpu)
  651. {
  652. struct blkcg *blkcg = css_to_blkcg(css);
  653. struct blkcg_gq *blkg;
  654. rcu_read_lock();
  655. hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
  656. struct blkcg_gq *parent = blkg->parent;
  657. struct blkg_iostat_set *bisc = per_cpu_ptr(blkg->iostat_cpu, cpu);
  658. struct blkg_iostat cur, delta;
  659. unsigned int seq;
  660. /* fetch the current per-cpu values */
  661. do {
  662. seq = u64_stats_fetch_begin(&bisc->sync);
  663. blkg_iostat_set(&cur, &bisc->cur);
  664. } while (u64_stats_fetch_retry(&bisc->sync, seq));
  665. /* propagate percpu delta to global */
  666. u64_stats_update_begin(&blkg->iostat.sync);
  667. blkg_iostat_set(&delta, &cur);
  668. blkg_iostat_sub(&delta, &bisc->last);
  669. blkg_iostat_add(&blkg->iostat.cur, &delta);
  670. blkg_iostat_add(&bisc->last, &delta);
  671. u64_stats_update_end(&blkg->iostat.sync);
  672. /* propagate global delta to parent */
  673. if (parent) {
  674. u64_stats_update_begin(&parent->iostat.sync);
  675. blkg_iostat_set(&delta, &blkg->iostat.cur);
  676. blkg_iostat_sub(&delta, &blkg->iostat.last);
  677. blkg_iostat_add(&parent->iostat.cur, &delta);
  678. blkg_iostat_add(&blkg->iostat.last, &delta);
  679. u64_stats_update_end(&parent->iostat.sync);
  680. }
  681. }
  682. rcu_read_unlock();
  683. }
  684. /*
  685. * The rstat algorithms intentionally don't handle the root cgroup to avoid
  686. * incurring overhead when no cgroups are defined. For that reason,
  687. * cgroup_rstat_flush in blkcg_print_stat does not actually fill out the
  688. * iostat in the root cgroup's blkcg_gq.
  689. *
  690. * However, we would like to re-use the printing code between the root and
  691. * non-root cgroups to the extent possible. For that reason, we simulate
  692. * flushing the root cgroup's stats by explicitly filling in the iostat
  693. * with disk level statistics.
  694. */
  695. static void blkcg_fill_root_iostats(void)
  696. {
  697. struct class_dev_iter iter;
  698. struct device *dev;
  699. class_dev_iter_init(&iter, &block_class, NULL, &disk_type);
  700. while ((dev = class_dev_iter_next(&iter))) {
  701. struct gendisk *disk = dev_to_disk(dev);
  702. struct hd_struct *part = disk_get_part(disk, 0);
  703. struct blkcg_gq *blkg = blk_queue_root_blkg(disk->queue);
  704. struct blkg_iostat tmp;
  705. int cpu;
  706. memset(&tmp, 0, sizeof(tmp));
  707. for_each_possible_cpu(cpu) {
  708. struct disk_stats *cpu_dkstats;
  709. cpu_dkstats = per_cpu_ptr(part->dkstats, cpu);
  710. tmp.ios[BLKG_IOSTAT_READ] +=
  711. cpu_dkstats->ios[STAT_READ];
  712. tmp.ios[BLKG_IOSTAT_WRITE] +=
  713. cpu_dkstats->ios[STAT_WRITE];
  714. tmp.ios[BLKG_IOSTAT_DISCARD] +=
  715. cpu_dkstats->ios[STAT_DISCARD];
  716. // convert sectors to bytes
  717. tmp.bytes[BLKG_IOSTAT_READ] +=
  718. cpu_dkstats->sectors[STAT_READ] << 9;
  719. tmp.bytes[BLKG_IOSTAT_WRITE] +=
  720. cpu_dkstats->sectors[STAT_WRITE] << 9;
  721. tmp.bytes[BLKG_IOSTAT_DISCARD] +=
  722. cpu_dkstats->sectors[STAT_DISCARD] << 9;
  723. u64_stats_update_begin(&blkg->iostat.sync);
  724. blkg_iostat_set(&blkg->iostat.cur, &tmp);
  725. u64_stats_update_end(&blkg->iostat.sync);
  726. }
  727. disk_put_part(part);
  728. }
  729. }
  730. static int blkcg_print_stat(struct seq_file *sf, void *v)
  731. {
  732. struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
  733. struct blkcg_gq *blkg;
  734. if (!seq_css(sf)->parent)
  735. blkcg_fill_root_iostats();
  736. else
  737. cgroup_rstat_flush(blkcg->css.cgroup);
  738. rcu_read_lock();
  739. hlist_for_each_entry_rcu(blkg, &blkcg->blkg_list, blkcg_node) {
  740. struct blkg_iostat_set *bis = &blkg->iostat;
  741. const char *dname;
  742. char *buf;
  743. u64 rbytes, wbytes, rios, wios, dbytes, dios;
  744. size_t size = seq_get_buf(sf, &buf), off = 0;
  745. int i;
  746. bool has_stats = false;
  747. unsigned seq;
  748. spin_lock_irq(&blkg->q->queue_lock);
  749. if (!blkg->online)
  750. goto skip;
  751. dname = blkg_dev_name(blkg);
  752. if (!dname)
  753. goto skip;
  754. /*
  755. * Hooray string manipulation, count is the size written NOT
  756. * INCLUDING THE \0, so size is now count+1 less than what we
  757. * had before, but we want to start writing the next bit from
  758. * the \0 so we only add count to buf.
  759. */
  760. off += scnprintf(buf+off, size-off, "%s ", dname);
  761. do {
  762. seq = u64_stats_fetch_begin(&bis->sync);
  763. rbytes = bis->cur.bytes[BLKG_IOSTAT_READ];
  764. wbytes = bis->cur.bytes[BLKG_IOSTAT_WRITE];
  765. dbytes = bis->cur.bytes[BLKG_IOSTAT_DISCARD];
  766. rios = bis->cur.ios[BLKG_IOSTAT_READ];
  767. wios = bis->cur.ios[BLKG_IOSTAT_WRITE];
  768. dios = bis->cur.ios[BLKG_IOSTAT_DISCARD];
  769. } while (u64_stats_fetch_retry(&bis->sync, seq));
  770. if (rbytes || wbytes || rios || wios) {
  771. has_stats = true;
  772. off += scnprintf(buf+off, size-off,
  773. "rbytes=%llu wbytes=%llu rios=%llu wios=%llu dbytes=%llu dios=%llu",
  774. rbytes, wbytes, rios, wios,
  775. dbytes, dios);
  776. }
  777. if (blkcg_debug_stats && atomic_read(&blkg->use_delay)) {
  778. has_stats = true;
  779. off += scnprintf(buf+off, size-off,
  780. " use_delay=%d delay_nsec=%llu",
  781. atomic_read(&blkg->use_delay),
  782. (unsigned long long)atomic64_read(&blkg->delay_nsec));
  783. }
  784. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  785. struct blkcg_policy *pol = blkcg_policy[i];
  786. size_t written;
  787. if (!blkg->pd[i] || !pol->pd_stat_fn)
  788. continue;
  789. written = pol->pd_stat_fn(blkg->pd[i], buf+off, size-off);
  790. if (written)
  791. has_stats = true;
  792. off += written;
  793. }
  794. if (has_stats) {
  795. if (off < size - 1) {
  796. off += scnprintf(buf+off, size-off, "\n");
  797. seq_commit(sf, off);
  798. } else {
  799. seq_commit(sf, -1);
  800. }
  801. }
  802. skip:
  803. spin_unlock_irq(&blkg->q->queue_lock);
  804. }
  805. rcu_read_unlock();
  806. return 0;
  807. }
  808. static struct cftype blkcg_files[] = {
  809. {
  810. .name = "stat",
  811. .seq_show = blkcg_print_stat,
  812. },
  813. { } /* terminate */
  814. };
  815. static struct cftype blkcg_legacy_files[] = {
  816. {
  817. .name = "reset_stats",
  818. .write_u64 = blkcg_reset_stats,
  819. },
  820. { } /* terminate */
  821. };
  822. /*
  823. * blkcg destruction is a three-stage process.
  824. *
  825. * 1. Destruction starts. The blkcg_css_offline() callback is invoked
  826. * which offlines writeback. Here we tie the next stage of blkg destruction
  827. * to the completion of writeback associated with the blkcg. This lets us
  828. * avoid punting potentially large amounts of outstanding writeback to root
  829. * while maintaining any ongoing policies. The next stage is triggered when
  830. * the nr_cgwbs count goes to zero.
  831. *
  832. * 2. When the nr_cgwbs count goes to zero, blkcg_destroy_blkgs() is called
  833. * and handles the destruction of blkgs. Here the css reference held by
  834. * the blkg is put back eventually allowing blkcg_css_free() to be called.
  835. * This work may occur in cgwb_release_workfn() on the cgwb_release
  836. * workqueue. Any submitted ios that fail to get the blkg ref will be
  837. * punted to the root_blkg.
  838. *
  839. * 3. Once the blkcg ref count goes to zero, blkcg_css_free() is called.
  840. * This finally frees the blkcg.
  841. */
  842. /**
  843. * blkcg_css_offline - cgroup css_offline callback
  844. * @css: css of interest
  845. *
  846. * This function is called when @css is about to go away. Here the cgwbs are
  847. * offlined first and only once writeback associated with the blkcg has
  848. * finished do we start step 2 (see above).
  849. */
  850. static void blkcg_css_offline(struct cgroup_subsys_state *css)
  851. {
  852. struct blkcg *blkcg = css_to_blkcg(css);
  853. /* this prevents anyone from attaching or migrating to this blkcg */
  854. wb_blkcg_offline(blkcg);
  855. /* put the base online pin allowing step 2 to be triggered */
  856. blkcg_unpin_online(blkcg);
  857. }
  858. /**
  859. * blkcg_destroy_blkgs - responsible for shooting down blkgs
  860. * @blkcg: blkcg of interest
  861. *
  862. * blkgs should be removed while holding both q and blkcg locks. As blkcg lock
  863. * is nested inside q lock, this function performs reverse double lock dancing.
  864. * Destroying the blkgs releases the reference held on the blkcg's css allowing
  865. * blkcg_css_free to eventually be called.
  866. *
  867. * This is the blkcg counterpart of ioc_release_fn().
  868. */
  869. void blkcg_destroy_blkgs(struct blkcg *blkcg)
  870. {
  871. might_sleep();
  872. spin_lock_irq(&blkcg->lock);
  873. while (!hlist_empty(&blkcg->blkg_list)) {
  874. struct blkcg_gq *blkg = hlist_entry(blkcg->blkg_list.first,
  875. struct blkcg_gq, blkcg_node);
  876. struct request_queue *q = blkg->q;
  877. if (need_resched() || !spin_trylock(&q->queue_lock)) {
  878. /*
  879. * Given that the system can accumulate a huge number
  880. * of blkgs in pathological cases, check to see if we
  881. * need to rescheduling to avoid softlockup.
  882. */
  883. spin_unlock_irq(&blkcg->lock);
  884. cond_resched();
  885. spin_lock_irq(&blkcg->lock);
  886. continue;
  887. }
  888. blkg_destroy(blkg);
  889. spin_unlock(&q->queue_lock);
  890. }
  891. spin_unlock_irq(&blkcg->lock);
  892. }
  893. static void blkcg_css_free(struct cgroup_subsys_state *css)
  894. {
  895. struct blkcg *blkcg = css_to_blkcg(css);
  896. int i;
  897. mutex_lock(&blkcg_pol_mutex);
  898. list_del(&blkcg->all_blkcgs_node);
  899. for (i = 0; i < BLKCG_MAX_POLS; i++)
  900. if (blkcg->cpd[i])
  901. blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
  902. mutex_unlock(&blkcg_pol_mutex);
  903. kfree(blkcg);
  904. }
  905. static struct cgroup_subsys_state *
  906. blkcg_css_alloc(struct cgroup_subsys_state *parent_css)
  907. {
  908. struct blkcg *blkcg;
  909. struct cgroup_subsys_state *ret;
  910. int i;
  911. mutex_lock(&blkcg_pol_mutex);
  912. if (!parent_css) {
  913. blkcg = &blkcg_root;
  914. } else {
  915. blkcg = kzalloc(sizeof(*blkcg), GFP_KERNEL);
  916. if (!blkcg) {
  917. ret = ERR_PTR(-ENOMEM);
  918. goto unlock;
  919. }
  920. }
  921. for (i = 0; i < BLKCG_MAX_POLS ; i++) {
  922. struct blkcg_policy *pol = blkcg_policy[i];
  923. struct blkcg_policy_data *cpd;
  924. /*
  925. * If the policy hasn't been attached yet, wait for it
  926. * to be attached before doing anything else. Otherwise,
  927. * check if the policy requires any specific per-cgroup
  928. * data: if it does, allocate and initialize it.
  929. */
  930. if (!pol || !pol->cpd_alloc_fn)
  931. continue;
  932. cpd = pol->cpd_alloc_fn(GFP_KERNEL);
  933. if (!cpd) {
  934. ret = ERR_PTR(-ENOMEM);
  935. goto free_pd_blkcg;
  936. }
  937. blkcg->cpd[i] = cpd;
  938. cpd->blkcg = blkcg;
  939. cpd->plid = i;
  940. if (pol->cpd_init_fn)
  941. pol->cpd_init_fn(cpd);
  942. }
  943. spin_lock_init(&blkcg->lock);
  944. refcount_set(&blkcg->online_pin, 1);
  945. INIT_RADIX_TREE(&blkcg->blkg_tree, GFP_NOWAIT | __GFP_NOWARN);
  946. INIT_HLIST_HEAD(&blkcg->blkg_list);
  947. #ifdef CONFIG_CGROUP_WRITEBACK
  948. INIT_LIST_HEAD(&blkcg->cgwb_list);
  949. #endif
  950. list_add_tail(&blkcg->all_blkcgs_node, &all_blkcgs);
  951. mutex_unlock(&blkcg_pol_mutex);
  952. return &blkcg->css;
  953. free_pd_blkcg:
  954. for (i--; i >= 0; i--)
  955. if (blkcg->cpd[i])
  956. blkcg_policy[i]->cpd_free_fn(blkcg->cpd[i]);
  957. if (blkcg != &blkcg_root)
  958. kfree(blkcg);
  959. unlock:
  960. mutex_unlock(&blkcg_pol_mutex);
  961. return ret;
  962. }
  963. static int blkcg_css_online(struct cgroup_subsys_state *css)
  964. {
  965. struct blkcg *blkcg = css_to_blkcg(css);
  966. struct blkcg *parent = blkcg_parent(blkcg);
  967. /*
  968. * blkcg_pin_online() is used to delay blkcg offline so that blkgs
  969. * don't go offline while cgwbs are still active on them. Pin the
  970. * parent so that offline always happens towards the root.
  971. */
  972. if (parent)
  973. blkcg_pin_online(parent);
  974. return 0;
  975. }
  976. /**
  977. * blkcg_init_queue - initialize blkcg part of request queue
  978. * @q: request_queue to initialize
  979. *
  980. * Called from blk_alloc_queue(). Responsible for initializing blkcg
  981. * part of new request_queue @q.
  982. *
  983. * RETURNS:
  984. * 0 on success, -errno on failure.
  985. */
  986. int blkcg_init_queue(struct request_queue *q)
  987. {
  988. struct blkcg_gq *new_blkg, *blkg;
  989. bool preloaded;
  990. int ret;
  991. new_blkg = blkg_alloc(&blkcg_root, q, GFP_KERNEL);
  992. if (!new_blkg)
  993. return -ENOMEM;
  994. preloaded = !radix_tree_preload(GFP_KERNEL);
  995. /* Make sure the root blkg exists. */
  996. rcu_read_lock();
  997. spin_lock_irq(&q->queue_lock);
  998. blkg = blkg_create(&blkcg_root, q, new_blkg);
  999. if (IS_ERR(blkg))
  1000. goto err_unlock;
  1001. q->root_blkg = blkg;
  1002. spin_unlock_irq(&q->queue_lock);
  1003. rcu_read_unlock();
  1004. if (preloaded)
  1005. radix_tree_preload_end();
  1006. ret = blk_ioprio_init(q);
  1007. if (ret)
  1008. goto err_destroy_all;
  1009. ret = blk_throtl_init(q);
  1010. if (ret)
  1011. goto err_destroy_all;
  1012. ret = blk_iolatency_init(q);
  1013. if (ret) {
  1014. blk_throtl_exit(q);
  1015. goto err_destroy_all;
  1016. }
  1017. return 0;
  1018. err_destroy_all:
  1019. blkg_destroy_all(q);
  1020. return ret;
  1021. err_unlock:
  1022. spin_unlock_irq(&q->queue_lock);
  1023. rcu_read_unlock();
  1024. if (preloaded)
  1025. radix_tree_preload_end();
  1026. return PTR_ERR(blkg);
  1027. }
  1028. /**
  1029. * blkcg_exit_queue - exit and release blkcg part of request_queue
  1030. * @q: request_queue being released
  1031. *
  1032. * Called from blk_exit_queue(). Responsible for exiting blkcg part.
  1033. */
  1034. void blkcg_exit_queue(struct request_queue *q)
  1035. {
  1036. blkg_destroy_all(q);
  1037. blk_throtl_exit(q);
  1038. }
  1039. /*
  1040. * We cannot support shared io contexts, as we have no mean to support
  1041. * two tasks with the same ioc in two different groups without major rework
  1042. * of the main cic data structures. For now we allow a task to change
  1043. * its cgroup only if it's the only owner of its ioc.
  1044. */
  1045. static int blkcg_can_attach(struct cgroup_taskset *tset)
  1046. {
  1047. struct task_struct *task;
  1048. struct cgroup_subsys_state *dst_css;
  1049. struct io_context *ioc;
  1050. int ret = 0;
  1051. /* task_lock() is needed to avoid races with exit_io_context() */
  1052. cgroup_taskset_for_each(task, dst_css, tset) {
  1053. task_lock(task);
  1054. ioc = task->io_context;
  1055. if (ioc && atomic_read(&ioc->nr_tasks) > 1)
  1056. ret = -EINVAL;
  1057. task_unlock(task);
  1058. if (ret)
  1059. break;
  1060. }
  1061. return ret;
  1062. }
  1063. static void blkcg_bind(struct cgroup_subsys_state *root_css)
  1064. {
  1065. int i;
  1066. mutex_lock(&blkcg_pol_mutex);
  1067. for (i = 0; i < BLKCG_MAX_POLS; i++) {
  1068. struct blkcg_policy *pol = blkcg_policy[i];
  1069. struct blkcg *blkcg;
  1070. if (!pol || !pol->cpd_bind_fn)
  1071. continue;
  1072. list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node)
  1073. if (blkcg->cpd[pol->plid])
  1074. pol->cpd_bind_fn(blkcg->cpd[pol->plid]);
  1075. }
  1076. mutex_unlock(&blkcg_pol_mutex);
  1077. }
  1078. static void blkcg_exit(struct task_struct *tsk)
  1079. {
  1080. if (tsk->throttle_queue)
  1081. blk_put_queue(tsk->throttle_queue);
  1082. tsk->throttle_queue = NULL;
  1083. }
  1084. struct cgroup_subsys io_cgrp_subsys = {
  1085. .css_alloc = blkcg_css_alloc,
  1086. .css_online = blkcg_css_online,
  1087. .css_offline = blkcg_css_offline,
  1088. .css_free = blkcg_css_free,
  1089. .can_attach = blkcg_can_attach,
  1090. .css_rstat_flush = blkcg_rstat_flush,
  1091. .bind = blkcg_bind,
  1092. .dfl_cftypes = blkcg_files,
  1093. .legacy_cftypes = blkcg_legacy_files,
  1094. .legacy_name = "blkio",
  1095. .exit = blkcg_exit,
  1096. #ifdef CONFIG_MEMCG
  1097. /*
  1098. * This ensures that, if available, memcg is automatically enabled
  1099. * together on the default hierarchy so that the owner cgroup can
  1100. * be retrieved from writeback pages.
  1101. */
  1102. .depends_on = 1 << memory_cgrp_id,
  1103. #endif
  1104. };
  1105. EXPORT_SYMBOL_GPL(io_cgrp_subsys);
  1106. /**
  1107. * blkcg_activate_policy - activate a blkcg policy on a request_queue
  1108. * @q: request_queue of interest
  1109. * @pol: blkcg policy to activate
  1110. *
  1111. * Activate @pol on @q. Requires %GFP_KERNEL context. @q goes through
  1112. * bypass mode to populate its blkgs with policy_data for @pol.
  1113. *
  1114. * Activation happens with @q bypassed, so nobody would be accessing blkgs
  1115. * from IO path. Update of each blkg is protected by both queue and blkcg
  1116. * locks so that holding either lock and testing blkcg_policy_enabled() is
  1117. * always enough for dereferencing policy data.
  1118. *
  1119. * The caller is responsible for synchronizing [de]activations and policy
  1120. * [un]registerations. Returns 0 on success, -errno on failure.
  1121. */
  1122. int blkcg_activate_policy(struct request_queue *q,
  1123. const struct blkcg_policy *pol)
  1124. {
  1125. struct blkg_policy_data *pd_prealloc = NULL;
  1126. struct blkcg_gq *blkg, *pinned_blkg = NULL;
  1127. int ret;
  1128. if (blkcg_policy_enabled(q, pol))
  1129. return 0;
  1130. if (queue_is_mq(q))
  1131. blk_mq_freeze_queue(q);
  1132. retry:
  1133. spin_lock_irq(&q->queue_lock);
  1134. /* blkg_list is pushed at the head, reverse walk to allocate parents first */
  1135. list_for_each_entry_reverse(blkg, &q->blkg_list, q_node) {
  1136. struct blkg_policy_data *pd;
  1137. if (blkg->pd[pol->plid])
  1138. continue;
  1139. /* If prealloc matches, use it; otherwise try GFP_NOWAIT */
  1140. if (blkg == pinned_blkg) {
  1141. pd = pd_prealloc;
  1142. pd_prealloc = NULL;
  1143. } else {
  1144. pd = pol->pd_alloc_fn(GFP_NOWAIT | __GFP_NOWARN, q,
  1145. blkg->blkcg);
  1146. }
  1147. if (!pd) {
  1148. /*
  1149. * GFP_NOWAIT failed. Free the existing one and
  1150. * prealloc for @blkg w/ GFP_KERNEL.
  1151. */
  1152. if (pinned_blkg)
  1153. blkg_put(pinned_blkg);
  1154. blkg_get(blkg);
  1155. pinned_blkg = blkg;
  1156. spin_unlock_irq(&q->queue_lock);
  1157. if (pd_prealloc)
  1158. pol->pd_free_fn(pd_prealloc);
  1159. pd_prealloc = pol->pd_alloc_fn(GFP_KERNEL, q,
  1160. blkg->blkcg);
  1161. if (pd_prealloc)
  1162. goto retry;
  1163. else
  1164. goto enomem;
  1165. }
  1166. blkg->pd[pol->plid] = pd;
  1167. pd->blkg = blkg;
  1168. pd->plid = pol->plid;
  1169. }
  1170. /* all allocated, init in the same order */
  1171. if (pol->pd_init_fn)
  1172. list_for_each_entry_reverse(blkg, &q->blkg_list, q_node)
  1173. pol->pd_init_fn(blkg->pd[pol->plid]);
  1174. __set_bit(pol->plid, q->blkcg_pols);
  1175. ret = 0;
  1176. spin_unlock_irq(&q->queue_lock);
  1177. out:
  1178. if (queue_is_mq(q))
  1179. blk_mq_unfreeze_queue(q);
  1180. if (pinned_blkg)
  1181. blkg_put(pinned_blkg);
  1182. if (pd_prealloc)
  1183. pol->pd_free_fn(pd_prealloc);
  1184. return ret;
  1185. enomem:
  1186. /* alloc failed, nothing's initialized yet, free everything */
  1187. spin_lock_irq(&q->queue_lock);
  1188. list_for_each_entry(blkg, &q->blkg_list, q_node) {
  1189. struct blkcg *blkcg = blkg->blkcg;
  1190. spin_lock(&blkcg->lock);
  1191. if (blkg->pd[pol->plid]) {
  1192. pol->pd_free_fn(blkg->pd[pol->plid]);
  1193. blkg->pd[pol->plid] = NULL;
  1194. }
  1195. spin_unlock(&blkcg->lock);
  1196. }
  1197. spin_unlock_irq(&q->queue_lock);
  1198. ret = -ENOMEM;
  1199. goto out;
  1200. }
  1201. EXPORT_SYMBOL_GPL(blkcg_activate_policy);
  1202. /**
  1203. * blkcg_deactivate_policy - deactivate a blkcg policy on a request_queue
  1204. * @q: request_queue of interest
  1205. * @pol: blkcg policy to deactivate
  1206. *
  1207. * Deactivate @pol on @q. Follows the same synchronization rules as
  1208. * blkcg_activate_policy().
  1209. */
  1210. void blkcg_deactivate_policy(struct request_queue *q,
  1211. const struct blkcg_policy *pol)
  1212. {
  1213. struct blkcg_gq *blkg;
  1214. if (!blkcg_policy_enabled(q, pol))
  1215. return;
  1216. if (queue_is_mq(q))
  1217. blk_mq_freeze_queue(q);
  1218. spin_lock_irq(&q->queue_lock);
  1219. __clear_bit(pol->plid, q->blkcg_pols);
  1220. list_for_each_entry(blkg, &q->blkg_list, q_node) {
  1221. struct blkcg *blkcg = blkg->blkcg;
  1222. spin_lock(&blkcg->lock);
  1223. if (blkg->pd[pol->plid]) {
  1224. if (pol->pd_offline_fn)
  1225. pol->pd_offline_fn(blkg->pd[pol->plid]);
  1226. pol->pd_free_fn(blkg->pd[pol->plid]);
  1227. blkg->pd[pol->plid] = NULL;
  1228. }
  1229. spin_unlock(&blkcg->lock);
  1230. }
  1231. spin_unlock_irq(&q->queue_lock);
  1232. if (queue_is_mq(q))
  1233. blk_mq_unfreeze_queue(q);
  1234. }
  1235. EXPORT_SYMBOL_GPL(blkcg_deactivate_policy);
  1236. /**
  1237. * blkcg_policy_register - register a blkcg policy
  1238. * @pol: blkcg policy to register
  1239. *
  1240. * Register @pol with blkcg core. Might sleep and @pol may be modified on
  1241. * successful registration. Returns 0 on success and -errno on failure.
  1242. */
  1243. int blkcg_policy_register(struct blkcg_policy *pol)
  1244. {
  1245. struct blkcg *blkcg;
  1246. int i, ret;
  1247. mutex_lock(&blkcg_pol_register_mutex);
  1248. mutex_lock(&blkcg_pol_mutex);
  1249. /* find an empty slot */
  1250. ret = -ENOSPC;
  1251. for (i = 0; i < BLKCG_MAX_POLS; i++)
  1252. if (!blkcg_policy[i])
  1253. break;
  1254. if (i >= BLKCG_MAX_POLS) {
  1255. pr_warn("blkcg_policy_register: BLKCG_MAX_POLS too small\n");
  1256. goto err_unlock;
  1257. }
  1258. /* Make sure cpd/pd_alloc_fn and cpd/pd_free_fn in pairs */
  1259. if ((!pol->cpd_alloc_fn ^ !pol->cpd_free_fn) ||
  1260. (!pol->pd_alloc_fn ^ !pol->pd_free_fn))
  1261. goto err_unlock;
  1262. /* register @pol */
  1263. pol->plid = i;
  1264. blkcg_policy[pol->plid] = pol;
  1265. /* allocate and install cpd's */
  1266. if (pol->cpd_alloc_fn) {
  1267. list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
  1268. struct blkcg_policy_data *cpd;
  1269. cpd = pol->cpd_alloc_fn(GFP_KERNEL);
  1270. if (!cpd)
  1271. goto err_free_cpds;
  1272. blkcg->cpd[pol->plid] = cpd;
  1273. cpd->blkcg = blkcg;
  1274. cpd->plid = pol->plid;
  1275. if (pol->cpd_init_fn)
  1276. pol->cpd_init_fn(cpd);
  1277. }
  1278. }
  1279. mutex_unlock(&blkcg_pol_mutex);
  1280. /* everything is in place, add intf files for the new policy */
  1281. if (pol->dfl_cftypes)
  1282. WARN_ON(cgroup_add_dfl_cftypes(&io_cgrp_subsys,
  1283. pol->dfl_cftypes));
  1284. if (pol->legacy_cftypes)
  1285. WARN_ON(cgroup_add_legacy_cftypes(&io_cgrp_subsys,
  1286. pol->legacy_cftypes));
  1287. mutex_unlock(&blkcg_pol_register_mutex);
  1288. return 0;
  1289. err_free_cpds:
  1290. if (pol->cpd_free_fn) {
  1291. list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
  1292. if (blkcg->cpd[pol->plid]) {
  1293. pol->cpd_free_fn(blkcg->cpd[pol->plid]);
  1294. blkcg->cpd[pol->plid] = NULL;
  1295. }
  1296. }
  1297. }
  1298. blkcg_policy[pol->plid] = NULL;
  1299. err_unlock:
  1300. mutex_unlock(&blkcg_pol_mutex);
  1301. mutex_unlock(&blkcg_pol_register_mutex);
  1302. return ret;
  1303. }
  1304. EXPORT_SYMBOL_GPL(blkcg_policy_register);
  1305. /**
  1306. * blkcg_policy_unregister - unregister a blkcg policy
  1307. * @pol: blkcg policy to unregister
  1308. *
  1309. * Undo blkcg_policy_register(@pol). Might sleep.
  1310. */
  1311. void blkcg_policy_unregister(struct blkcg_policy *pol)
  1312. {
  1313. struct blkcg *blkcg;
  1314. mutex_lock(&blkcg_pol_register_mutex);
  1315. if (WARN_ON(blkcg_policy[pol->plid] != pol))
  1316. goto out_unlock;
  1317. /* kill the intf files first */
  1318. if (pol->dfl_cftypes)
  1319. cgroup_rm_cftypes(pol->dfl_cftypes);
  1320. if (pol->legacy_cftypes)
  1321. cgroup_rm_cftypes(pol->legacy_cftypes);
  1322. /* remove cpds and unregister */
  1323. mutex_lock(&blkcg_pol_mutex);
  1324. if (pol->cpd_free_fn) {
  1325. list_for_each_entry(blkcg, &all_blkcgs, all_blkcgs_node) {
  1326. if (blkcg->cpd[pol->plid]) {
  1327. pol->cpd_free_fn(blkcg->cpd[pol->plid]);
  1328. blkcg->cpd[pol->plid] = NULL;
  1329. }
  1330. }
  1331. }
  1332. blkcg_policy[pol->plid] = NULL;
  1333. mutex_unlock(&blkcg_pol_mutex);
  1334. out_unlock:
  1335. mutex_unlock(&blkcg_pol_register_mutex);
  1336. }
  1337. EXPORT_SYMBOL_GPL(blkcg_policy_unregister);
  1338. bool __blkcg_punt_bio_submit(struct bio *bio)
  1339. {
  1340. struct blkcg_gq *blkg = bio->bi_blkg;
  1341. /* consume the flag first */
  1342. bio->bi_opf &= ~REQ_CGROUP_PUNT;
  1343. /* never bounce for the root cgroup */
  1344. if (!blkg->parent)
  1345. return false;
  1346. spin_lock_bh(&blkg->async_bio_lock);
  1347. bio_list_add(&blkg->async_bios, bio);
  1348. spin_unlock_bh(&blkg->async_bio_lock);
  1349. queue_work(blkcg_punt_bio_wq, &blkg->async_bio_work);
  1350. return true;
  1351. }
  1352. /*
  1353. * Scale the accumulated delay based on how long it has been since we updated
  1354. * the delay. We only call this when we are adding delay, in case it's been a
  1355. * while since we added delay, and when we are checking to see if we need to
  1356. * delay a task, to account for any delays that may have occurred.
  1357. */
  1358. static void blkcg_scale_delay(struct blkcg_gq *blkg, u64 now)
  1359. {
  1360. u64 old = atomic64_read(&blkg->delay_start);
  1361. /* negative use_delay means no scaling, see blkcg_set_delay() */
  1362. if (atomic_read(&blkg->use_delay) < 0)
  1363. return;
  1364. /*
  1365. * We only want to scale down every second. The idea here is that we
  1366. * want to delay people for min(delay_nsec, NSEC_PER_SEC) in a certain
  1367. * time window. We only want to throttle tasks for recent delay that
  1368. * has occurred, in 1 second time windows since that's the maximum
  1369. * things can be throttled. We save the current delay window in
  1370. * blkg->last_delay so we know what amount is still left to be charged
  1371. * to the blkg from this point onward. blkg->last_use keeps track of
  1372. * the use_delay counter. The idea is if we're unthrottling the blkg we
  1373. * are ok with whatever is happening now, and we can take away more of
  1374. * the accumulated delay as we've already throttled enough that
  1375. * everybody is happy with their IO latencies.
  1376. */
  1377. if (time_before64(old + NSEC_PER_SEC, now) &&
  1378. atomic64_cmpxchg(&blkg->delay_start, old, now) == old) {
  1379. u64 cur = atomic64_read(&blkg->delay_nsec);
  1380. u64 sub = min_t(u64, blkg->last_delay, now - old);
  1381. int cur_use = atomic_read(&blkg->use_delay);
  1382. /*
  1383. * We've been unthrottled, subtract a larger chunk of our
  1384. * accumulated delay.
  1385. */
  1386. if (cur_use < blkg->last_use)
  1387. sub = max_t(u64, sub, blkg->last_delay >> 1);
  1388. /*
  1389. * This shouldn't happen, but handle it anyway. Our delay_nsec
  1390. * should only ever be growing except here where we subtract out
  1391. * min(last_delay, 1 second), but lord knows bugs happen and I'd
  1392. * rather not end up with negative numbers.
  1393. */
  1394. if (unlikely(cur < sub)) {
  1395. atomic64_set(&blkg->delay_nsec, 0);
  1396. blkg->last_delay = 0;
  1397. } else {
  1398. atomic64_sub(sub, &blkg->delay_nsec);
  1399. blkg->last_delay = cur - sub;
  1400. }
  1401. blkg->last_use = cur_use;
  1402. }
  1403. }
  1404. /*
  1405. * This is called when we want to actually walk up the hierarchy and check to
  1406. * see if we need to throttle, and then actually throttle if there is some
  1407. * accumulated delay. This should only be called upon return to user space so
  1408. * we're not holding some lock that would induce a priority inversion.
  1409. */
  1410. static void blkcg_maybe_throttle_blkg(struct blkcg_gq *blkg, bool use_memdelay)
  1411. {
  1412. unsigned long pflags;
  1413. bool clamp;
  1414. u64 now = ktime_to_ns(ktime_get());
  1415. u64 exp;
  1416. u64 delay_nsec = 0;
  1417. int tok;
  1418. while (blkg->parent) {
  1419. int use_delay = atomic_read(&blkg->use_delay);
  1420. if (use_delay) {
  1421. u64 this_delay;
  1422. blkcg_scale_delay(blkg, now);
  1423. this_delay = atomic64_read(&blkg->delay_nsec);
  1424. if (this_delay > delay_nsec) {
  1425. delay_nsec = this_delay;
  1426. clamp = use_delay > 0;
  1427. }
  1428. }
  1429. blkg = blkg->parent;
  1430. }
  1431. if (!delay_nsec)
  1432. return;
  1433. /*
  1434. * Let's not sleep for all eternity if we've amassed a huge delay.
  1435. * Swapping or metadata IO can accumulate 10's of seconds worth of
  1436. * delay, and we want userspace to be able to do _something_ so cap the
  1437. * delays at 0.25s. If there's 10's of seconds worth of delay then the
  1438. * tasks will be delayed for 0.25 second for every syscall. If
  1439. * blkcg_set_delay() was used as indicated by negative use_delay, the
  1440. * caller is responsible for regulating the range.
  1441. */
  1442. if (clamp)
  1443. delay_nsec = min_t(u64, delay_nsec, 250 * NSEC_PER_MSEC);
  1444. if (use_memdelay)
  1445. psi_memstall_enter(&pflags);
  1446. exp = ktime_add_ns(now, delay_nsec);
  1447. tok = io_schedule_prepare();
  1448. do {
  1449. __set_current_state(TASK_KILLABLE);
  1450. if (!schedule_hrtimeout(&exp, HRTIMER_MODE_ABS))
  1451. break;
  1452. } while (!fatal_signal_pending(current));
  1453. io_schedule_finish(tok);
  1454. if (use_memdelay)
  1455. psi_memstall_leave(&pflags);
  1456. }
  1457. /**
  1458. * blkcg_maybe_throttle_current - throttle the current task if it has been marked
  1459. *
  1460. * This is only called if we've been marked with set_notify_resume(). Obviously
  1461. * we can be set_notify_resume() for reasons other than blkcg throttling, so we
  1462. * check to see if current->throttle_queue is set and if not this doesn't do
  1463. * anything. This should only ever be called by the resume code, it's not meant
  1464. * to be called by people willy-nilly as it will actually do the work to
  1465. * throttle the task if it is setup for throttling.
  1466. */
  1467. void blkcg_maybe_throttle_current(void)
  1468. {
  1469. struct request_queue *q = current->throttle_queue;
  1470. struct cgroup_subsys_state *css;
  1471. struct blkcg *blkcg;
  1472. struct blkcg_gq *blkg;
  1473. bool use_memdelay = current->use_memdelay;
  1474. if (!q)
  1475. return;
  1476. current->throttle_queue = NULL;
  1477. current->use_memdelay = false;
  1478. rcu_read_lock();
  1479. css = kthread_blkcg();
  1480. if (css)
  1481. blkcg = css_to_blkcg(css);
  1482. else
  1483. blkcg = css_to_blkcg(task_css(current, io_cgrp_id));
  1484. if (!blkcg)
  1485. goto out;
  1486. blkg = blkg_lookup(blkcg, q);
  1487. if (!blkg)
  1488. goto out;
  1489. if (!blkg_tryget(blkg))
  1490. goto out;
  1491. rcu_read_unlock();
  1492. blkcg_maybe_throttle_blkg(blkg, use_memdelay);
  1493. blkg_put(blkg);
  1494. blk_put_queue(q);
  1495. return;
  1496. out:
  1497. rcu_read_unlock();
  1498. blk_put_queue(q);
  1499. }
  1500. /**
  1501. * blkcg_schedule_throttle - this task needs to check for throttling
  1502. * @q: the request queue IO was submitted on
  1503. * @use_memdelay: do we charge this to memory delay for PSI
  1504. *
  1505. * This is called by the IO controller when we know there's delay accumulated
  1506. * for the blkg for this task. We do not pass the blkg because there are places
  1507. * we call this that may not have that information, the swapping code for
  1508. * instance will only have a request_queue at that point. This set's the
  1509. * notify_resume for the task to check and see if it requires throttling before
  1510. * returning to user space.
  1511. *
  1512. * We will only schedule once per syscall. You can call this over and over
  1513. * again and it will only do the check once upon return to user space, and only
  1514. * throttle once. If the task needs to be throttled again it'll need to be
  1515. * re-set at the next time we see the task.
  1516. */
  1517. void blkcg_schedule_throttle(struct request_queue *q, bool use_memdelay)
  1518. {
  1519. if (unlikely(current->flags & PF_KTHREAD))
  1520. return;
  1521. if (!blk_get_queue(q))
  1522. return;
  1523. if (current->throttle_queue)
  1524. blk_put_queue(current->throttle_queue);
  1525. current->throttle_queue = q;
  1526. if (use_memdelay)
  1527. current->use_memdelay = use_memdelay;
  1528. set_notify_resume(current);
  1529. }
  1530. /**
  1531. * blkcg_add_delay - add delay to this blkg
  1532. * @blkg: blkg of interest
  1533. * @now: the current time in nanoseconds
  1534. * @delta: how many nanoseconds of delay to add
  1535. *
  1536. * Charge @delta to the blkg's current delay accumulation. This is used to
  1537. * throttle tasks if an IO controller thinks we need more throttling.
  1538. */
  1539. void blkcg_add_delay(struct blkcg_gq *blkg, u64 now, u64 delta)
  1540. {
  1541. if (WARN_ON_ONCE(atomic_read(&blkg->use_delay) < 0))
  1542. return;
  1543. blkcg_scale_delay(blkg, now);
  1544. atomic64_add(delta, &blkg->delay_nsec);
  1545. }
  1546. /**
  1547. * blkg_tryget_closest - try and get a blkg ref on the closet blkg
  1548. * @bio: target bio
  1549. * @css: target css
  1550. *
  1551. * As the failure mode here is to walk up the blkg tree, this ensure that the
  1552. * blkg->parent pointers are always valid. This returns the blkg that it ended
  1553. * up taking a reference on or %NULL if no reference was taken.
  1554. */
  1555. static inline struct blkcg_gq *blkg_tryget_closest(struct bio *bio,
  1556. struct cgroup_subsys_state *css)
  1557. {
  1558. struct blkcg_gq *blkg, *ret_blkg = NULL;
  1559. rcu_read_lock();
  1560. blkg = blkg_lookup_create(css_to_blkcg(css), bio->bi_disk->queue);
  1561. while (blkg) {
  1562. if (blkg_tryget(blkg)) {
  1563. ret_blkg = blkg;
  1564. break;
  1565. }
  1566. blkg = blkg->parent;
  1567. }
  1568. rcu_read_unlock();
  1569. return ret_blkg;
  1570. }
  1571. /**
  1572. * bio_associate_blkg_from_css - associate a bio with a specified css
  1573. * @bio: target bio
  1574. * @css: target css
  1575. *
  1576. * Associate @bio with the blkg found by combining the css's blkg and the
  1577. * request_queue of the @bio. An association failure is handled by walking up
  1578. * the blkg tree. Therefore, the blkg associated can be anything between @blkg
  1579. * and q->root_blkg. This situation only happens when a cgroup is dying and
  1580. * then the remaining bios will spill to the closest alive blkg.
  1581. *
  1582. * A reference will be taken on the blkg and will be released when @bio is
  1583. * freed.
  1584. */
  1585. void bio_associate_blkg_from_css(struct bio *bio,
  1586. struct cgroup_subsys_state *css)
  1587. {
  1588. if (bio->bi_blkg)
  1589. blkg_put(bio->bi_blkg);
  1590. if (css && css->parent) {
  1591. bio->bi_blkg = blkg_tryget_closest(bio, css);
  1592. } else {
  1593. blkg_get(bio->bi_disk->queue->root_blkg);
  1594. bio->bi_blkg = bio->bi_disk->queue->root_blkg;
  1595. }
  1596. }
  1597. EXPORT_SYMBOL_GPL(bio_associate_blkg_from_css);
  1598. /**
  1599. * bio_associate_blkg - associate a bio with a blkg
  1600. * @bio: target bio
  1601. *
  1602. * Associate @bio with the blkg found from the bio's css and request_queue.
  1603. * If one is not found, bio_lookup_blkg() creates the blkg. If a blkg is
  1604. * already associated, the css is reused and association redone as the
  1605. * request_queue may have changed.
  1606. */
  1607. void bio_associate_blkg(struct bio *bio)
  1608. {
  1609. struct cgroup_subsys_state *css;
  1610. rcu_read_lock();
  1611. if (bio->bi_blkg)
  1612. css = &bio_blkcg(bio)->css;
  1613. else
  1614. css = blkcg_css();
  1615. bio_associate_blkg_from_css(bio, css);
  1616. rcu_read_unlock();
  1617. }
  1618. EXPORT_SYMBOL_GPL(bio_associate_blkg);
  1619. /**
  1620. * bio_clone_blkg_association - clone blkg association from src to dst bio
  1621. * @dst: destination bio
  1622. * @src: source bio
  1623. */
  1624. void bio_clone_blkg_association(struct bio *dst, struct bio *src)
  1625. {
  1626. if (src->bi_blkg) {
  1627. if (dst->bi_blkg)
  1628. blkg_put(dst->bi_blkg);
  1629. blkg_get(src->bi_blkg);
  1630. dst->bi_blkg = src->bi_blkg;
  1631. }
  1632. }
  1633. EXPORT_SYMBOL_GPL(bio_clone_blkg_association);
  1634. static int blk_cgroup_io_type(struct bio *bio)
  1635. {
  1636. if (op_is_discard(bio->bi_opf))
  1637. return BLKG_IOSTAT_DISCARD;
  1638. if (op_is_write(bio->bi_opf))
  1639. return BLKG_IOSTAT_WRITE;
  1640. return BLKG_IOSTAT_READ;
  1641. }
  1642. void blk_cgroup_bio_start(struct bio *bio)
  1643. {
  1644. int rwd = blk_cgroup_io_type(bio), cpu;
  1645. struct blkg_iostat_set *bis;
  1646. cpu = get_cpu();
  1647. bis = per_cpu_ptr(bio->bi_blkg->iostat_cpu, cpu);
  1648. u64_stats_update_begin(&bis->sync);
  1649. /*
  1650. * If the bio is flagged with BIO_CGROUP_ACCT it means this is a split
  1651. * bio and we would have already accounted for the size of the bio.
  1652. */
  1653. if (!bio_flagged(bio, BIO_CGROUP_ACCT)) {
  1654. bio_set_flag(bio, BIO_CGROUP_ACCT);
  1655. bis->cur.bytes[rwd] += bio->bi_iter.bi_size;
  1656. }
  1657. bis->cur.ios[rwd]++;
  1658. u64_stats_update_end(&bis->sync);
  1659. if (cgroup_subsys_on_dfl(io_cgrp_subsys))
  1660. cgroup_rstat_updated(bio->bi_blkg->blkcg->css.cgroup, cpu);
  1661. put_cpu();
  1662. }
  1663. static int __init blkcg_init(void)
  1664. {
  1665. blkcg_punt_bio_wq = alloc_workqueue("blkcg_punt_bio",
  1666. WQ_MEM_RECLAIM | WQ_FREEZABLE |
  1667. WQ_UNBOUND | WQ_SYSFS, 0);
  1668. if (!blkcg_punt_bio_wq)
  1669. return -ENOMEM;
  1670. return 0;
  1671. }
  1672. subsys_initcall(blkcg_init);
  1673. module_param(blkcg_debug_stats, bool, 0644);
  1674. MODULE_PARM_DESC(blkcg_debug_stats, "True if you want debug stats, false if not");