blk-throttle.c 68 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Interface for controlling IO bandwidth on a request queue
  4. *
  5. * Copyright (C) 2010 Vivek Goyal <vgoyal@redhat.com>
  6. */
  7. #include <linux/module.h>
  8. #include <linux/slab.h>
  9. #include <linux/blkdev.h>
  10. #include <linux/bio.h>
  11. #include <linux/blktrace_api.h>
  12. #include <linux/blk-cgroup.h>
  13. #include "blk.h"
  14. #include "blk-cgroup-rwstat.h"
  15. /* Max dispatch from a group in 1 round */
  16. #define THROTL_GRP_QUANTUM 8
  17. /* Total max dispatch from all groups in one round */
  18. #define THROTL_QUANTUM 32
  19. /* Throttling is performed over a slice and after that slice is renewed */
  20. #define DFL_THROTL_SLICE_HD (HZ / 10)
  21. #define DFL_THROTL_SLICE_SSD (HZ / 50)
  22. #define MAX_THROTL_SLICE (HZ)
  23. #define MAX_IDLE_TIME (5L * 1000 * 1000) /* 5 s */
  24. #define MIN_THROTL_BPS (320 * 1024)
  25. #define MIN_THROTL_IOPS (10)
  26. #define DFL_LATENCY_TARGET (-1L)
  27. #define DFL_IDLE_THRESHOLD (0)
  28. #define DFL_HD_BASELINE_LATENCY (4000L) /* 4ms */
  29. #define LATENCY_FILTERED_SSD (0)
  30. /*
  31. * For HD, very small latency comes from sequential IO. Such IO is helpless to
  32. * help determine if its IO is impacted by others, hence we ignore the IO
  33. */
  34. #define LATENCY_FILTERED_HD (1000L) /* 1ms */
  35. static struct blkcg_policy blkcg_policy_throtl;
  36. /* A workqueue to queue throttle related work */
  37. static struct workqueue_struct *kthrotld_workqueue;
  38. /*
  39. * To implement hierarchical throttling, throtl_grps form a tree and bios
  40. * are dispatched upwards level by level until they reach the top and get
  41. * issued. When dispatching bios from the children and local group at each
  42. * level, if the bios are dispatched into a single bio_list, there's a risk
  43. * of a local or child group which can queue many bios at once filling up
  44. * the list starving others.
  45. *
  46. * To avoid such starvation, dispatched bios are queued separately
  47. * according to where they came from. When they are again dispatched to
  48. * the parent, they're popped in round-robin order so that no single source
  49. * hogs the dispatch window.
  50. *
  51. * throtl_qnode is used to keep the queued bios separated by their sources.
  52. * Bios are queued to throtl_qnode which in turn is queued to
  53. * throtl_service_queue and then dispatched in round-robin order.
  54. *
  55. * It's also used to track the reference counts on blkg's. A qnode always
  56. * belongs to a throtl_grp and gets queued on itself or the parent, so
  57. * incrementing the reference of the associated throtl_grp when a qnode is
  58. * queued and decrementing when dequeued is enough to keep the whole blkg
  59. * tree pinned while bios are in flight.
  60. */
  61. struct throtl_qnode {
  62. struct list_head node; /* service_queue->queued[] */
  63. struct bio_list bios; /* queued bios */
  64. struct throtl_grp *tg; /* tg this qnode belongs to */
  65. };
  66. struct throtl_service_queue {
  67. struct throtl_service_queue *parent_sq; /* the parent service_queue */
  68. /*
  69. * Bios queued directly to this service_queue or dispatched from
  70. * children throtl_grp's.
  71. */
  72. struct list_head queued[2]; /* throtl_qnode [READ/WRITE] */
  73. unsigned int nr_queued[2]; /* number of queued bios */
  74. /*
  75. * RB tree of active children throtl_grp's, which are sorted by
  76. * their ->disptime.
  77. */
  78. struct rb_root_cached pending_tree; /* RB tree of active tgs */
  79. unsigned int nr_pending; /* # queued in the tree */
  80. unsigned long first_pending_disptime; /* disptime of the first tg */
  81. struct timer_list pending_timer; /* fires on first_pending_disptime */
  82. };
  83. enum tg_state_flags {
  84. THROTL_TG_PENDING = 1 << 0, /* on parent's pending tree */
  85. THROTL_TG_WAS_EMPTY = 1 << 1, /* bio_lists[] became non-empty */
  86. };
  87. #define rb_entry_tg(node) rb_entry((node), struct throtl_grp, rb_node)
  88. enum {
  89. LIMIT_LOW,
  90. LIMIT_MAX,
  91. LIMIT_CNT,
  92. };
  93. struct throtl_grp {
  94. /* must be the first member */
  95. struct blkg_policy_data pd;
  96. /* active throtl group service_queue member */
  97. struct rb_node rb_node;
  98. /* throtl_data this group belongs to */
  99. struct throtl_data *td;
  100. /* this group's service queue */
  101. struct throtl_service_queue service_queue;
  102. /*
  103. * qnode_on_self is used when bios are directly queued to this
  104. * throtl_grp so that local bios compete fairly with bios
  105. * dispatched from children. qnode_on_parent is used when bios are
  106. * dispatched from this throtl_grp into its parent and will compete
  107. * with the sibling qnode_on_parents and the parent's
  108. * qnode_on_self.
  109. */
  110. struct throtl_qnode qnode_on_self[2];
  111. struct throtl_qnode qnode_on_parent[2];
  112. /*
  113. * Dispatch time in jiffies. This is the estimated time when group
  114. * will unthrottle and is ready to dispatch more bio. It is used as
  115. * key to sort active groups in service tree.
  116. */
  117. unsigned long disptime;
  118. unsigned int flags;
  119. /* are there any throtl rules between this group and td? */
  120. bool has_rules[2];
  121. /* internally used bytes per second rate limits */
  122. uint64_t bps[2][LIMIT_CNT];
  123. /* user configured bps limits */
  124. uint64_t bps_conf[2][LIMIT_CNT];
  125. /* internally used IOPS limits */
  126. unsigned int iops[2][LIMIT_CNT];
  127. /* user configured IOPS limits */
  128. unsigned int iops_conf[2][LIMIT_CNT];
  129. /* Number of bytes dispatched in current slice */
  130. uint64_t bytes_disp[2];
  131. /* Number of bio's dispatched in current slice */
  132. unsigned int io_disp[2];
  133. unsigned long last_low_overflow_time[2];
  134. uint64_t last_bytes_disp[2];
  135. unsigned int last_io_disp[2];
  136. unsigned long last_check_time;
  137. unsigned long latency_target; /* us */
  138. unsigned long latency_target_conf; /* us */
  139. /* When did we start a new slice */
  140. unsigned long slice_start[2];
  141. unsigned long slice_end[2];
  142. unsigned long last_finish_time; /* ns / 1024 */
  143. unsigned long checked_last_finish_time; /* ns / 1024 */
  144. unsigned long avg_idletime; /* ns / 1024 */
  145. unsigned long idletime_threshold; /* us */
  146. unsigned long idletime_threshold_conf; /* us */
  147. unsigned int bio_cnt; /* total bios */
  148. unsigned int bad_bio_cnt; /* bios exceeding latency threshold */
  149. unsigned long bio_cnt_reset_time;
  150. atomic_t io_split_cnt[2];
  151. atomic_t last_io_split_cnt[2];
  152. struct blkg_rwstat stat_bytes;
  153. struct blkg_rwstat stat_ios;
  154. };
  155. /* We measure latency for request size from <= 4k to >= 1M */
  156. #define LATENCY_BUCKET_SIZE 9
  157. struct latency_bucket {
  158. unsigned long total_latency; /* ns / 1024 */
  159. int samples;
  160. };
  161. struct avg_latency_bucket {
  162. unsigned long latency; /* ns / 1024 */
  163. bool valid;
  164. };
  165. struct throtl_data
  166. {
  167. /* service tree for active throtl groups */
  168. struct throtl_service_queue service_queue;
  169. struct request_queue *queue;
  170. /* Total Number of queued bios on READ and WRITE lists */
  171. unsigned int nr_queued[2];
  172. unsigned int throtl_slice;
  173. /* Work for dispatching throttled bios */
  174. struct work_struct dispatch_work;
  175. unsigned int limit_index;
  176. bool limit_valid[LIMIT_CNT];
  177. unsigned long low_upgrade_time;
  178. unsigned long low_downgrade_time;
  179. unsigned int scale;
  180. struct latency_bucket tmp_buckets[2][LATENCY_BUCKET_SIZE];
  181. struct avg_latency_bucket avg_buckets[2][LATENCY_BUCKET_SIZE];
  182. struct latency_bucket __percpu *latency_buckets[2];
  183. unsigned long last_calculate_time;
  184. unsigned long filtered_latency;
  185. bool track_bio_latency;
  186. };
  187. static void throtl_pending_timer_fn(struct timer_list *t);
  188. static inline struct throtl_grp *pd_to_tg(struct blkg_policy_data *pd)
  189. {
  190. return pd ? container_of(pd, struct throtl_grp, pd) : NULL;
  191. }
  192. static inline struct throtl_grp *blkg_to_tg(struct blkcg_gq *blkg)
  193. {
  194. return pd_to_tg(blkg_to_pd(blkg, &blkcg_policy_throtl));
  195. }
  196. static inline struct blkcg_gq *tg_to_blkg(struct throtl_grp *tg)
  197. {
  198. return pd_to_blkg(&tg->pd);
  199. }
  200. /**
  201. * sq_to_tg - return the throl_grp the specified service queue belongs to
  202. * @sq: the throtl_service_queue of interest
  203. *
  204. * Return the throtl_grp @sq belongs to. If @sq is the top-level one
  205. * embedded in throtl_data, %NULL is returned.
  206. */
  207. static struct throtl_grp *sq_to_tg(struct throtl_service_queue *sq)
  208. {
  209. if (sq && sq->parent_sq)
  210. return container_of(sq, struct throtl_grp, service_queue);
  211. else
  212. return NULL;
  213. }
  214. /**
  215. * sq_to_td - return throtl_data the specified service queue belongs to
  216. * @sq: the throtl_service_queue of interest
  217. *
  218. * A service_queue can be embedded in either a throtl_grp or throtl_data.
  219. * Determine the associated throtl_data accordingly and return it.
  220. */
  221. static struct throtl_data *sq_to_td(struct throtl_service_queue *sq)
  222. {
  223. struct throtl_grp *tg = sq_to_tg(sq);
  224. if (tg)
  225. return tg->td;
  226. else
  227. return container_of(sq, struct throtl_data, service_queue);
  228. }
  229. /*
  230. * cgroup's limit in LIMIT_MAX is scaled if low limit is set. This scale is to
  231. * make the IO dispatch more smooth.
  232. * Scale up: linearly scale up according to lapsed time since upgrade. For
  233. * every throtl_slice, the limit scales up 1/2 .low limit till the
  234. * limit hits .max limit
  235. * Scale down: exponentially scale down if a cgroup doesn't hit its .low limit
  236. */
  237. static uint64_t throtl_adjusted_limit(uint64_t low, struct throtl_data *td)
  238. {
  239. /* arbitrary value to avoid too big scale */
  240. if (td->scale < 4096 && time_after_eq(jiffies,
  241. td->low_upgrade_time + td->scale * td->throtl_slice))
  242. td->scale = (jiffies - td->low_upgrade_time) / td->throtl_slice;
  243. return low + (low >> 1) * td->scale;
  244. }
  245. static uint64_t tg_bps_limit(struct throtl_grp *tg, int rw)
  246. {
  247. struct blkcg_gq *blkg = tg_to_blkg(tg);
  248. struct throtl_data *td;
  249. uint64_t ret;
  250. if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
  251. return U64_MAX;
  252. td = tg->td;
  253. ret = tg->bps[rw][td->limit_index];
  254. if (ret == 0 && td->limit_index == LIMIT_LOW) {
  255. /* intermediate node or iops isn't 0 */
  256. if (!list_empty(&blkg->blkcg->css.children) ||
  257. tg->iops[rw][td->limit_index])
  258. return U64_MAX;
  259. else
  260. return MIN_THROTL_BPS;
  261. }
  262. if (td->limit_index == LIMIT_MAX && tg->bps[rw][LIMIT_LOW] &&
  263. tg->bps[rw][LIMIT_LOW] != tg->bps[rw][LIMIT_MAX]) {
  264. uint64_t adjusted;
  265. adjusted = throtl_adjusted_limit(tg->bps[rw][LIMIT_LOW], td);
  266. ret = min(tg->bps[rw][LIMIT_MAX], adjusted);
  267. }
  268. return ret;
  269. }
  270. static unsigned int tg_iops_limit(struct throtl_grp *tg, int rw)
  271. {
  272. struct blkcg_gq *blkg = tg_to_blkg(tg);
  273. struct throtl_data *td;
  274. unsigned int ret;
  275. if (cgroup_subsys_on_dfl(io_cgrp_subsys) && !blkg->parent)
  276. return UINT_MAX;
  277. td = tg->td;
  278. ret = tg->iops[rw][td->limit_index];
  279. if (ret == 0 && tg->td->limit_index == LIMIT_LOW) {
  280. /* intermediate node or bps isn't 0 */
  281. if (!list_empty(&blkg->blkcg->css.children) ||
  282. tg->bps[rw][td->limit_index])
  283. return UINT_MAX;
  284. else
  285. return MIN_THROTL_IOPS;
  286. }
  287. if (td->limit_index == LIMIT_MAX && tg->iops[rw][LIMIT_LOW] &&
  288. tg->iops[rw][LIMIT_LOW] != tg->iops[rw][LIMIT_MAX]) {
  289. uint64_t adjusted;
  290. adjusted = throtl_adjusted_limit(tg->iops[rw][LIMIT_LOW], td);
  291. if (adjusted > UINT_MAX)
  292. adjusted = UINT_MAX;
  293. ret = min_t(unsigned int, tg->iops[rw][LIMIT_MAX], adjusted);
  294. }
  295. return ret;
  296. }
  297. #define request_bucket_index(sectors) \
  298. clamp_t(int, order_base_2(sectors) - 3, 0, LATENCY_BUCKET_SIZE - 1)
  299. /**
  300. * throtl_log - log debug message via blktrace
  301. * @sq: the service_queue being reported
  302. * @fmt: printf format string
  303. * @args: printf args
  304. *
  305. * The messages are prefixed with "throtl BLKG_NAME" if @sq belongs to a
  306. * throtl_grp; otherwise, just "throtl".
  307. */
  308. #define throtl_log(sq, fmt, args...) do { \
  309. struct throtl_grp *__tg = sq_to_tg((sq)); \
  310. struct throtl_data *__td = sq_to_td((sq)); \
  311. \
  312. (void)__td; \
  313. if (likely(!blk_trace_note_message_enabled(__td->queue))) \
  314. break; \
  315. if ((__tg)) { \
  316. blk_add_cgroup_trace_msg(__td->queue, \
  317. tg_to_blkg(__tg)->blkcg, "throtl " fmt, ##args);\
  318. } else { \
  319. blk_add_trace_msg(__td->queue, "throtl " fmt, ##args); \
  320. } \
  321. } while (0)
  322. static inline unsigned int throtl_bio_data_size(struct bio *bio)
  323. {
  324. /* assume it's one sector */
  325. if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
  326. return 512;
  327. return bio->bi_iter.bi_size;
  328. }
  329. static void throtl_qnode_init(struct throtl_qnode *qn, struct throtl_grp *tg)
  330. {
  331. INIT_LIST_HEAD(&qn->node);
  332. bio_list_init(&qn->bios);
  333. qn->tg = tg;
  334. }
  335. /**
  336. * throtl_qnode_add_bio - add a bio to a throtl_qnode and activate it
  337. * @bio: bio being added
  338. * @qn: qnode to add bio to
  339. * @queued: the service_queue->queued[] list @qn belongs to
  340. *
  341. * Add @bio to @qn and put @qn on @queued if it's not already on.
  342. * @qn->tg's reference count is bumped when @qn is activated. See the
  343. * comment on top of throtl_qnode definition for details.
  344. */
  345. static void throtl_qnode_add_bio(struct bio *bio, struct throtl_qnode *qn,
  346. struct list_head *queued)
  347. {
  348. bio_list_add(&qn->bios, bio);
  349. if (list_empty(&qn->node)) {
  350. list_add_tail(&qn->node, queued);
  351. blkg_get(tg_to_blkg(qn->tg));
  352. }
  353. }
  354. /**
  355. * throtl_peek_queued - peek the first bio on a qnode list
  356. * @queued: the qnode list to peek
  357. */
  358. static struct bio *throtl_peek_queued(struct list_head *queued)
  359. {
  360. struct throtl_qnode *qn;
  361. struct bio *bio;
  362. if (list_empty(queued))
  363. return NULL;
  364. qn = list_first_entry(queued, struct throtl_qnode, node);
  365. bio = bio_list_peek(&qn->bios);
  366. WARN_ON_ONCE(!bio);
  367. return bio;
  368. }
  369. /**
  370. * throtl_pop_queued - pop the first bio form a qnode list
  371. * @queued: the qnode list to pop a bio from
  372. * @tg_to_put: optional out argument for throtl_grp to put
  373. *
  374. * Pop the first bio from the qnode list @queued. After popping, the first
  375. * qnode is removed from @queued if empty or moved to the end of @queued so
  376. * that the popping order is round-robin.
  377. *
  378. * When the first qnode is removed, its associated throtl_grp should be put
  379. * too. If @tg_to_put is NULL, this function automatically puts it;
  380. * otherwise, *@tg_to_put is set to the throtl_grp to put and the caller is
  381. * responsible for putting it.
  382. */
  383. static struct bio *throtl_pop_queued(struct list_head *queued,
  384. struct throtl_grp **tg_to_put)
  385. {
  386. struct throtl_qnode *qn;
  387. struct bio *bio;
  388. if (list_empty(queued))
  389. return NULL;
  390. qn = list_first_entry(queued, struct throtl_qnode, node);
  391. bio = bio_list_pop(&qn->bios);
  392. WARN_ON_ONCE(!bio);
  393. if (bio_list_empty(&qn->bios)) {
  394. list_del_init(&qn->node);
  395. if (tg_to_put)
  396. *tg_to_put = qn->tg;
  397. else
  398. blkg_put(tg_to_blkg(qn->tg));
  399. } else {
  400. list_move_tail(&qn->node, queued);
  401. }
  402. return bio;
  403. }
  404. /* init a service_queue, assumes the caller zeroed it */
  405. static void throtl_service_queue_init(struct throtl_service_queue *sq)
  406. {
  407. INIT_LIST_HEAD(&sq->queued[0]);
  408. INIT_LIST_HEAD(&sq->queued[1]);
  409. sq->pending_tree = RB_ROOT_CACHED;
  410. timer_setup(&sq->pending_timer, throtl_pending_timer_fn, 0);
  411. }
  412. static struct blkg_policy_data *throtl_pd_alloc(gfp_t gfp,
  413. struct request_queue *q,
  414. struct blkcg *blkcg)
  415. {
  416. struct throtl_grp *tg;
  417. int rw;
  418. tg = kzalloc_node(sizeof(*tg), gfp, q->node);
  419. if (!tg)
  420. return NULL;
  421. if (blkg_rwstat_init(&tg->stat_bytes, gfp))
  422. goto err_free_tg;
  423. if (blkg_rwstat_init(&tg->stat_ios, gfp))
  424. goto err_exit_stat_bytes;
  425. throtl_service_queue_init(&tg->service_queue);
  426. for (rw = READ; rw <= WRITE; rw++) {
  427. throtl_qnode_init(&tg->qnode_on_self[rw], tg);
  428. throtl_qnode_init(&tg->qnode_on_parent[rw], tg);
  429. }
  430. RB_CLEAR_NODE(&tg->rb_node);
  431. tg->bps[READ][LIMIT_MAX] = U64_MAX;
  432. tg->bps[WRITE][LIMIT_MAX] = U64_MAX;
  433. tg->iops[READ][LIMIT_MAX] = UINT_MAX;
  434. tg->iops[WRITE][LIMIT_MAX] = UINT_MAX;
  435. tg->bps_conf[READ][LIMIT_MAX] = U64_MAX;
  436. tg->bps_conf[WRITE][LIMIT_MAX] = U64_MAX;
  437. tg->iops_conf[READ][LIMIT_MAX] = UINT_MAX;
  438. tg->iops_conf[WRITE][LIMIT_MAX] = UINT_MAX;
  439. /* LIMIT_LOW will have default value 0 */
  440. tg->latency_target = DFL_LATENCY_TARGET;
  441. tg->latency_target_conf = DFL_LATENCY_TARGET;
  442. tg->idletime_threshold = DFL_IDLE_THRESHOLD;
  443. tg->idletime_threshold_conf = DFL_IDLE_THRESHOLD;
  444. return &tg->pd;
  445. err_exit_stat_bytes:
  446. blkg_rwstat_exit(&tg->stat_bytes);
  447. err_free_tg:
  448. kfree(tg);
  449. return NULL;
  450. }
  451. static void throtl_pd_init(struct blkg_policy_data *pd)
  452. {
  453. struct throtl_grp *tg = pd_to_tg(pd);
  454. struct blkcg_gq *blkg = tg_to_blkg(tg);
  455. struct throtl_data *td = blkg->q->td;
  456. struct throtl_service_queue *sq = &tg->service_queue;
  457. /*
  458. * If on the default hierarchy, we switch to properly hierarchical
  459. * behavior where limits on a given throtl_grp are applied to the
  460. * whole subtree rather than just the group itself. e.g. If 16M
  461. * read_bps limit is set on the root group, the whole system can't
  462. * exceed 16M for the device.
  463. *
  464. * If not on the default hierarchy, the broken flat hierarchy
  465. * behavior is retained where all throtl_grps are treated as if
  466. * they're all separate root groups right below throtl_data.
  467. * Limits of a group don't interact with limits of other groups
  468. * regardless of the position of the group in the hierarchy.
  469. */
  470. sq->parent_sq = &td->service_queue;
  471. if (cgroup_subsys_on_dfl(io_cgrp_subsys) && blkg->parent)
  472. sq->parent_sq = &blkg_to_tg(blkg->parent)->service_queue;
  473. tg->td = td;
  474. }
  475. /*
  476. * Set has_rules[] if @tg or any of its parents have limits configured.
  477. * This doesn't require walking up to the top of the hierarchy as the
  478. * parent's has_rules[] is guaranteed to be correct.
  479. */
  480. static void tg_update_has_rules(struct throtl_grp *tg)
  481. {
  482. struct throtl_grp *parent_tg = sq_to_tg(tg->service_queue.parent_sq);
  483. struct throtl_data *td = tg->td;
  484. int rw;
  485. for (rw = READ; rw <= WRITE; rw++)
  486. tg->has_rules[rw] = (parent_tg && parent_tg->has_rules[rw]) ||
  487. (td->limit_valid[td->limit_index] &&
  488. (tg_bps_limit(tg, rw) != U64_MAX ||
  489. tg_iops_limit(tg, rw) != UINT_MAX));
  490. }
  491. static void throtl_pd_online(struct blkg_policy_data *pd)
  492. {
  493. struct throtl_grp *tg = pd_to_tg(pd);
  494. /*
  495. * We don't want new groups to escape the limits of its ancestors.
  496. * Update has_rules[] after a new group is brought online.
  497. */
  498. tg_update_has_rules(tg);
  499. }
  500. static void blk_throtl_update_limit_valid(struct throtl_data *td)
  501. {
  502. struct cgroup_subsys_state *pos_css;
  503. struct blkcg_gq *blkg;
  504. bool low_valid = false;
  505. rcu_read_lock();
  506. blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
  507. struct throtl_grp *tg = blkg_to_tg(blkg);
  508. if (tg->bps[READ][LIMIT_LOW] || tg->bps[WRITE][LIMIT_LOW] ||
  509. tg->iops[READ][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) {
  510. low_valid = true;
  511. break;
  512. }
  513. }
  514. rcu_read_unlock();
  515. td->limit_valid[LIMIT_LOW] = low_valid;
  516. }
  517. static void throtl_upgrade_state(struct throtl_data *td);
  518. static void throtl_pd_offline(struct blkg_policy_data *pd)
  519. {
  520. struct throtl_grp *tg = pd_to_tg(pd);
  521. tg->bps[READ][LIMIT_LOW] = 0;
  522. tg->bps[WRITE][LIMIT_LOW] = 0;
  523. tg->iops[READ][LIMIT_LOW] = 0;
  524. tg->iops[WRITE][LIMIT_LOW] = 0;
  525. blk_throtl_update_limit_valid(tg->td);
  526. if (!tg->td->limit_valid[tg->td->limit_index])
  527. throtl_upgrade_state(tg->td);
  528. }
  529. static void throtl_pd_free(struct blkg_policy_data *pd)
  530. {
  531. struct throtl_grp *tg = pd_to_tg(pd);
  532. del_timer_sync(&tg->service_queue.pending_timer);
  533. blkg_rwstat_exit(&tg->stat_bytes);
  534. blkg_rwstat_exit(&tg->stat_ios);
  535. kfree(tg);
  536. }
  537. static struct throtl_grp *
  538. throtl_rb_first(struct throtl_service_queue *parent_sq)
  539. {
  540. struct rb_node *n;
  541. n = rb_first_cached(&parent_sq->pending_tree);
  542. WARN_ON_ONCE(!n);
  543. if (!n)
  544. return NULL;
  545. return rb_entry_tg(n);
  546. }
  547. static void throtl_rb_erase(struct rb_node *n,
  548. struct throtl_service_queue *parent_sq)
  549. {
  550. rb_erase_cached(n, &parent_sq->pending_tree);
  551. RB_CLEAR_NODE(n);
  552. --parent_sq->nr_pending;
  553. }
  554. static void update_min_dispatch_time(struct throtl_service_queue *parent_sq)
  555. {
  556. struct throtl_grp *tg;
  557. tg = throtl_rb_first(parent_sq);
  558. if (!tg)
  559. return;
  560. parent_sq->first_pending_disptime = tg->disptime;
  561. }
  562. static void tg_service_queue_add(struct throtl_grp *tg)
  563. {
  564. struct throtl_service_queue *parent_sq = tg->service_queue.parent_sq;
  565. struct rb_node **node = &parent_sq->pending_tree.rb_root.rb_node;
  566. struct rb_node *parent = NULL;
  567. struct throtl_grp *__tg;
  568. unsigned long key = tg->disptime;
  569. bool leftmost = true;
  570. while (*node != NULL) {
  571. parent = *node;
  572. __tg = rb_entry_tg(parent);
  573. if (time_before(key, __tg->disptime))
  574. node = &parent->rb_left;
  575. else {
  576. node = &parent->rb_right;
  577. leftmost = false;
  578. }
  579. }
  580. rb_link_node(&tg->rb_node, parent, node);
  581. rb_insert_color_cached(&tg->rb_node, &parent_sq->pending_tree,
  582. leftmost);
  583. }
  584. static void throtl_enqueue_tg(struct throtl_grp *tg)
  585. {
  586. if (!(tg->flags & THROTL_TG_PENDING)) {
  587. tg_service_queue_add(tg);
  588. tg->flags |= THROTL_TG_PENDING;
  589. tg->service_queue.parent_sq->nr_pending++;
  590. }
  591. }
  592. static void throtl_dequeue_tg(struct throtl_grp *tg)
  593. {
  594. if (tg->flags & THROTL_TG_PENDING) {
  595. throtl_rb_erase(&tg->rb_node, tg->service_queue.parent_sq);
  596. tg->flags &= ~THROTL_TG_PENDING;
  597. }
  598. }
  599. /* Call with queue lock held */
  600. static void throtl_schedule_pending_timer(struct throtl_service_queue *sq,
  601. unsigned long expires)
  602. {
  603. unsigned long max_expire = jiffies + 8 * sq_to_td(sq)->throtl_slice;
  604. /*
  605. * Since we are adjusting the throttle limit dynamically, the sleep
  606. * time calculated according to previous limit might be invalid. It's
  607. * possible the cgroup sleep time is very long and no other cgroups
  608. * have IO running so notify the limit changes. Make sure the cgroup
  609. * doesn't sleep too long to avoid the missed notification.
  610. */
  611. if (time_after(expires, max_expire))
  612. expires = max_expire;
  613. mod_timer(&sq->pending_timer, expires);
  614. throtl_log(sq, "schedule timer. delay=%lu jiffies=%lu",
  615. expires - jiffies, jiffies);
  616. }
  617. /**
  618. * throtl_schedule_next_dispatch - schedule the next dispatch cycle
  619. * @sq: the service_queue to schedule dispatch for
  620. * @force: force scheduling
  621. *
  622. * Arm @sq->pending_timer so that the next dispatch cycle starts on the
  623. * dispatch time of the first pending child. Returns %true if either timer
  624. * is armed or there's no pending child left. %false if the current
  625. * dispatch window is still open and the caller should continue
  626. * dispatching.
  627. *
  628. * If @force is %true, the dispatch timer is always scheduled and this
  629. * function is guaranteed to return %true. This is to be used when the
  630. * caller can't dispatch itself and needs to invoke pending_timer
  631. * unconditionally. Note that forced scheduling is likely to induce short
  632. * delay before dispatch starts even if @sq->first_pending_disptime is not
  633. * in the future and thus shouldn't be used in hot paths.
  634. */
  635. static bool throtl_schedule_next_dispatch(struct throtl_service_queue *sq,
  636. bool force)
  637. {
  638. /* any pending children left? */
  639. if (!sq->nr_pending)
  640. return true;
  641. update_min_dispatch_time(sq);
  642. /* is the next dispatch time in the future? */
  643. if (force || time_after(sq->first_pending_disptime, jiffies)) {
  644. throtl_schedule_pending_timer(sq, sq->first_pending_disptime);
  645. return true;
  646. }
  647. /* tell the caller to continue dispatching */
  648. return false;
  649. }
  650. static inline void throtl_start_new_slice_with_credit(struct throtl_grp *tg,
  651. bool rw, unsigned long start)
  652. {
  653. tg->bytes_disp[rw] = 0;
  654. tg->io_disp[rw] = 0;
  655. atomic_set(&tg->io_split_cnt[rw], 0);
  656. /*
  657. * Previous slice has expired. We must have trimmed it after last
  658. * bio dispatch. That means since start of last slice, we never used
  659. * that bandwidth. Do try to make use of that bandwidth while giving
  660. * credit.
  661. */
  662. if (time_after_eq(start, tg->slice_start[rw]))
  663. tg->slice_start[rw] = start;
  664. tg->slice_end[rw] = jiffies + tg->td->throtl_slice;
  665. throtl_log(&tg->service_queue,
  666. "[%c] new slice with credit start=%lu end=%lu jiffies=%lu",
  667. rw == READ ? 'R' : 'W', tg->slice_start[rw],
  668. tg->slice_end[rw], jiffies);
  669. }
  670. static inline void throtl_start_new_slice(struct throtl_grp *tg, bool rw)
  671. {
  672. tg->bytes_disp[rw] = 0;
  673. tg->io_disp[rw] = 0;
  674. tg->slice_start[rw] = jiffies;
  675. tg->slice_end[rw] = jiffies + tg->td->throtl_slice;
  676. atomic_set(&tg->io_split_cnt[rw], 0);
  677. throtl_log(&tg->service_queue,
  678. "[%c] new slice start=%lu end=%lu jiffies=%lu",
  679. rw == READ ? 'R' : 'W', tg->slice_start[rw],
  680. tg->slice_end[rw], jiffies);
  681. }
  682. static inline void throtl_set_slice_end(struct throtl_grp *tg, bool rw,
  683. unsigned long jiffy_end)
  684. {
  685. tg->slice_end[rw] = roundup(jiffy_end, tg->td->throtl_slice);
  686. }
  687. static inline void throtl_extend_slice(struct throtl_grp *tg, bool rw,
  688. unsigned long jiffy_end)
  689. {
  690. throtl_set_slice_end(tg, rw, jiffy_end);
  691. throtl_log(&tg->service_queue,
  692. "[%c] extend slice start=%lu end=%lu jiffies=%lu",
  693. rw == READ ? 'R' : 'W', tg->slice_start[rw],
  694. tg->slice_end[rw], jiffies);
  695. }
  696. /* Determine if previously allocated or extended slice is complete or not */
  697. static bool throtl_slice_used(struct throtl_grp *tg, bool rw)
  698. {
  699. if (time_in_range(jiffies, tg->slice_start[rw], tg->slice_end[rw]))
  700. return false;
  701. return true;
  702. }
  703. /* Trim the used slices and adjust slice start accordingly */
  704. static inline void throtl_trim_slice(struct throtl_grp *tg, bool rw)
  705. {
  706. unsigned long nr_slices, time_elapsed, io_trim;
  707. u64 bytes_trim, tmp;
  708. BUG_ON(time_before(tg->slice_end[rw], tg->slice_start[rw]));
  709. /*
  710. * If bps are unlimited (-1), then time slice don't get
  711. * renewed. Don't try to trim the slice if slice is used. A new
  712. * slice will start when appropriate.
  713. */
  714. if (throtl_slice_used(tg, rw))
  715. return;
  716. /*
  717. * A bio has been dispatched. Also adjust slice_end. It might happen
  718. * that initially cgroup limit was very low resulting in high
  719. * slice_end, but later limit was bumped up and bio was dispatched
  720. * sooner, then we need to reduce slice_end. A high bogus slice_end
  721. * is bad because it does not allow new slice to start.
  722. */
  723. throtl_set_slice_end(tg, rw, jiffies + tg->td->throtl_slice);
  724. time_elapsed = jiffies - tg->slice_start[rw];
  725. nr_slices = time_elapsed / tg->td->throtl_slice;
  726. if (!nr_slices)
  727. return;
  728. tmp = tg_bps_limit(tg, rw) * tg->td->throtl_slice * nr_slices;
  729. do_div(tmp, HZ);
  730. bytes_trim = tmp;
  731. io_trim = (tg_iops_limit(tg, rw) * tg->td->throtl_slice * nr_slices) /
  732. HZ;
  733. if (!bytes_trim && !io_trim)
  734. return;
  735. if (tg->bytes_disp[rw] >= bytes_trim)
  736. tg->bytes_disp[rw] -= bytes_trim;
  737. else
  738. tg->bytes_disp[rw] = 0;
  739. if (tg->io_disp[rw] >= io_trim)
  740. tg->io_disp[rw] -= io_trim;
  741. else
  742. tg->io_disp[rw] = 0;
  743. tg->slice_start[rw] += nr_slices * tg->td->throtl_slice;
  744. throtl_log(&tg->service_queue,
  745. "[%c] trim slice nr=%lu bytes=%llu io=%lu start=%lu end=%lu jiffies=%lu",
  746. rw == READ ? 'R' : 'W', nr_slices, bytes_trim, io_trim,
  747. tg->slice_start[rw], tg->slice_end[rw], jiffies);
  748. }
  749. static bool tg_with_in_iops_limit(struct throtl_grp *tg, struct bio *bio,
  750. u32 iops_limit, unsigned long *wait)
  751. {
  752. bool rw = bio_data_dir(bio);
  753. unsigned int io_allowed;
  754. unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
  755. u64 tmp;
  756. if (iops_limit == UINT_MAX) {
  757. if (wait)
  758. *wait = 0;
  759. return true;
  760. }
  761. jiffy_elapsed = jiffies - tg->slice_start[rw];
  762. /* Round up to the next throttle slice, wait time must be nonzero */
  763. jiffy_elapsed_rnd = roundup(jiffy_elapsed + 1, tg->td->throtl_slice);
  764. /*
  765. * jiffy_elapsed_rnd should not be a big value as minimum iops can be
  766. * 1 then at max jiffy elapsed should be equivalent of 1 second as we
  767. * will allow dispatch after 1 second and after that slice should
  768. * have been trimmed.
  769. */
  770. tmp = (u64)iops_limit * jiffy_elapsed_rnd;
  771. do_div(tmp, HZ);
  772. if (tmp > UINT_MAX)
  773. io_allowed = UINT_MAX;
  774. else
  775. io_allowed = tmp;
  776. if (tg->io_disp[rw] + 1 <= io_allowed) {
  777. if (wait)
  778. *wait = 0;
  779. return true;
  780. }
  781. /* Calc approx time to dispatch */
  782. jiffy_wait = jiffy_elapsed_rnd - jiffy_elapsed;
  783. if (wait)
  784. *wait = jiffy_wait;
  785. return false;
  786. }
  787. static bool tg_with_in_bps_limit(struct throtl_grp *tg, struct bio *bio,
  788. u64 bps_limit, unsigned long *wait)
  789. {
  790. bool rw = bio_data_dir(bio);
  791. u64 bytes_allowed, extra_bytes, tmp;
  792. unsigned long jiffy_elapsed, jiffy_wait, jiffy_elapsed_rnd;
  793. unsigned int bio_size = throtl_bio_data_size(bio);
  794. if (bps_limit == U64_MAX) {
  795. if (wait)
  796. *wait = 0;
  797. return true;
  798. }
  799. jiffy_elapsed = jiffy_elapsed_rnd = jiffies - tg->slice_start[rw];
  800. /* Slice has just started. Consider one slice interval */
  801. if (!jiffy_elapsed)
  802. jiffy_elapsed_rnd = tg->td->throtl_slice;
  803. jiffy_elapsed_rnd = roundup(jiffy_elapsed_rnd, tg->td->throtl_slice);
  804. tmp = bps_limit * jiffy_elapsed_rnd;
  805. do_div(tmp, HZ);
  806. bytes_allowed = tmp;
  807. if (tg->bytes_disp[rw] + bio_size <= bytes_allowed) {
  808. if (wait)
  809. *wait = 0;
  810. return true;
  811. }
  812. /* Calc approx time to dispatch */
  813. extra_bytes = tg->bytes_disp[rw] + bio_size - bytes_allowed;
  814. jiffy_wait = div64_u64(extra_bytes * HZ, bps_limit);
  815. if (!jiffy_wait)
  816. jiffy_wait = 1;
  817. /*
  818. * This wait time is without taking into consideration the rounding
  819. * up we did. Add that time also.
  820. */
  821. jiffy_wait = jiffy_wait + (jiffy_elapsed_rnd - jiffy_elapsed);
  822. if (wait)
  823. *wait = jiffy_wait;
  824. return false;
  825. }
  826. /*
  827. * Returns whether one can dispatch a bio or not. Also returns approx number
  828. * of jiffies to wait before this bio is with-in IO rate and can be dispatched
  829. */
  830. static bool tg_may_dispatch(struct throtl_grp *tg, struct bio *bio,
  831. unsigned long *wait)
  832. {
  833. bool rw = bio_data_dir(bio);
  834. unsigned long bps_wait = 0, iops_wait = 0, max_wait = 0;
  835. u64 bps_limit = tg_bps_limit(tg, rw);
  836. u32 iops_limit = tg_iops_limit(tg, rw);
  837. /*
  838. * Currently whole state machine of group depends on first bio
  839. * queued in the group bio list. So one should not be calling
  840. * this function with a different bio if there are other bios
  841. * queued.
  842. */
  843. BUG_ON(tg->service_queue.nr_queued[rw] &&
  844. bio != throtl_peek_queued(&tg->service_queue.queued[rw]));
  845. /* If tg->bps = -1, then BW is unlimited */
  846. if (bps_limit == U64_MAX && iops_limit == UINT_MAX) {
  847. if (wait)
  848. *wait = 0;
  849. return true;
  850. }
  851. /*
  852. * If previous slice expired, start a new one otherwise renew/extend
  853. * existing slice to make sure it is at least throtl_slice interval
  854. * long since now. New slice is started only for empty throttle group.
  855. * If there is queued bio, that means there should be an active
  856. * slice and it should be extended instead.
  857. */
  858. if (throtl_slice_used(tg, rw) && !(tg->service_queue.nr_queued[rw]))
  859. throtl_start_new_slice(tg, rw);
  860. else {
  861. if (time_before(tg->slice_end[rw],
  862. jiffies + tg->td->throtl_slice))
  863. throtl_extend_slice(tg, rw,
  864. jiffies + tg->td->throtl_slice);
  865. }
  866. if (iops_limit != UINT_MAX)
  867. tg->io_disp[rw] += atomic_xchg(&tg->io_split_cnt[rw], 0);
  868. if (tg_with_in_bps_limit(tg, bio, bps_limit, &bps_wait) &&
  869. tg_with_in_iops_limit(tg, bio, iops_limit, &iops_wait)) {
  870. if (wait)
  871. *wait = 0;
  872. return true;
  873. }
  874. max_wait = max(bps_wait, iops_wait);
  875. if (wait)
  876. *wait = max_wait;
  877. if (time_before(tg->slice_end[rw], jiffies + max_wait))
  878. throtl_extend_slice(tg, rw, jiffies + max_wait);
  879. return false;
  880. }
  881. static void throtl_charge_bio(struct throtl_grp *tg, struct bio *bio)
  882. {
  883. bool rw = bio_data_dir(bio);
  884. unsigned int bio_size = throtl_bio_data_size(bio);
  885. /* Charge the bio to the group */
  886. tg->bytes_disp[rw] += bio_size;
  887. tg->io_disp[rw]++;
  888. tg->last_bytes_disp[rw] += bio_size;
  889. tg->last_io_disp[rw]++;
  890. /*
  891. * BIO_THROTTLED is used to prevent the same bio to be throttled
  892. * more than once as a throttled bio will go through blk-throtl the
  893. * second time when it eventually gets issued. Set it when a bio
  894. * is being charged to a tg.
  895. */
  896. if (!bio_flagged(bio, BIO_THROTTLED))
  897. bio_set_flag(bio, BIO_THROTTLED);
  898. }
  899. /**
  900. * throtl_add_bio_tg - add a bio to the specified throtl_grp
  901. * @bio: bio to add
  902. * @qn: qnode to use
  903. * @tg: the target throtl_grp
  904. *
  905. * Add @bio to @tg's service_queue using @qn. If @qn is not specified,
  906. * tg->qnode_on_self[] is used.
  907. */
  908. static void throtl_add_bio_tg(struct bio *bio, struct throtl_qnode *qn,
  909. struct throtl_grp *tg)
  910. {
  911. struct throtl_service_queue *sq = &tg->service_queue;
  912. bool rw = bio_data_dir(bio);
  913. if (!qn)
  914. qn = &tg->qnode_on_self[rw];
  915. /*
  916. * If @tg doesn't currently have any bios queued in the same
  917. * direction, queueing @bio can change when @tg should be
  918. * dispatched. Mark that @tg was empty. This is automatically
  919. * cleared on the next tg_update_disptime().
  920. */
  921. if (!sq->nr_queued[rw])
  922. tg->flags |= THROTL_TG_WAS_EMPTY;
  923. throtl_qnode_add_bio(bio, qn, &sq->queued[rw]);
  924. sq->nr_queued[rw]++;
  925. throtl_enqueue_tg(tg);
  926. }
  927. static void tg_update_disptime(struct throtl_grp *tg)
  928. {
  929. struct throtl_service_queue *sq = &tg->service_queue;
  930. unsigned long read_wait = -1, write_wait = -1, min_wait = -1, disptime;
  931. struct bio *bio;
  932. bio = throtl_peek_queued(&sq->queued[READ]);
  933. if (bio)
  934. tg_may_dispatch(tg, bio, &read_wait);
  935. bio = throtl_peek_queued(&sq->queued[WRITE]);
  936. if (bio)
  937. tg_may_dispatch(tg, bio, &write_wait);
  938. min_wait = min(read_wait, write_wait);
  939. disptime = jiffies + min_wait;
  940. /* Update dispatch time */
  941. throtl_dequeue_tg(tg);
  942. tg->disptime = disptime;
  943. throtl_enqueue_tg(tg);
  944. /* see throtl_add_bio_tg() */
  945. tg->flags &= ~THROTL_TG_WAS_EMPTY;
  946. }
  947. static void start_parent_slice_with_credit(struct throtl_grp *child_tg,
  948. struct throtl_grp *parent_tg, bool rw)
  949. {
  950. if (throtl_slice_used(parent_tg, rw)) {
  951. throtl_start_new_slice_with_credit(parent_tg, rw,
  952. child_tg->slice_start[rw]);
  953. }
  954. }
  955. static void tg_dispatch_one_bio(struct throtl_grp *tg, bool rw)
  956. {
  957. struct throtl_service_queue *sq = &tg->service_queue;
  958. struct throtl_service_queue *parent_sq = sq->parent_sq;
  959. struct throtl_grp *parent_tg = sq_to_tg(parent_sq);
  960. struct throtl_grp *tg_to_put = NULL;
  961. struct bio *bio;
  962. /*
  963. * @bio is being transferred from @tg to @parent_sq. Popping a bio
  964. * from @tg may put its reference and @parent_sq might end up
  965. * getting released prematurely. Remember the tg to put and put it
  966. * after @bio is transferred to @parent_sq.
  967. */
  968. bio = throtl_pop_queued(&sq->queued[rw], &tg_to_put);
  969. sq->nr_queued[rw]--;
  970. throtl_charge_bio(tg, bio);
  971. /*
  972. * If our parent is another tg, we just need to transfer @bio to
  973. * the parent using throtl_add_bio_tg(). If our parent is
  974. * @td->service_queue, @bio is ready to be issued. Put it on its
  975. * bio_lists[] and decrease total number queued. The caller is
  976. * responsible for issuing these bios.
  977. */
  978. if (parent_tg) {
  979. throtl_add_bio_tg(bio, &tg->qnode_on_parent[rw], parent_tg);
  980. start_parent_slice_with_credit(tg, parent_tg, rw);
  981. } else {
  982. throtl_qnode_add_bio(bio, &tg->qnode_on_parent[rw],
  983. &parent_sq->queued[rw]);
  984. BUG_ON(tg->td->nr_queued[rw] <= 0);
  985. tg->td->nr_queued[rw]--;
  986. }
  987. throtl_trim_slice(tg, rw);
  988. if (tg_to_put)
  989. blkg_put(tg_to_blkg(tg_to_put));
  990. }
  991. static int throtl_dispatch_tg(struct throtl_grp *tg)
  992. {
  993. struct throtl_service_queue *sq = &tg->service_queue;
  994. unsigned int nr_reads = 0, nr_writes = 0;
  995. unsigned int max_nr_reads = THROTL_GRP_QUANTUM * 3 / 4;
  996. unsigned int max_nr_writes = THROTL_GRP_QUANTUM - max_nr_reads;
  997. struct bio *bio;
  998. /* Try to dispatch 75% READS and 25% WRITES */
  999. while ((bio = throtl_peek_queued(&sq->queued[READ])) &&
  1000. tg_may_dispatch(tg, bio, NULL)) {
  1001. tg_dispatch_one_bio(tg, bio_data_dir(bio));
  1002. nr_reads++;
  1003. if (nr_reads >= max_nr_reads)
  1004. break;
  1005. }
  1006. while ((bio = throtl_peek_queued(&sq->queued[WRITE])) &&
  1007. tg_may_dispatch(tg, bio, NULL)) {
  1008. tg_dispatch_one_bio(tg, bio_data_dir(bio));
  1009. nr_writes++;
  1010. if (nr_writes >= max_nr_writes)
  1011. break;
  1012. }
  1013. return nr_reads + nr_writes;
  1014. }
  1015. static int throtl_select_dispatch(struct throtl_service_queue *parent_sq)
  1016. {
  1017. unsigned int nr_disp = 0;
  1018. while (1) {
  1019. struct throtl_grp *tg;
  1020. struct throtl_service_queue *sq;
  1021. if (!parent_sq->nr_pending)
  1022. break;
  1023. tg = throtl_rb_first(parent_sq);
  1024. if (!tg)
  1025. break;
  1026. if (time_before(jiffies, tg->disptime))
  1027. break;
  1028. throtl_dequeue_tg(tg);
  1029. nr_disp += throtl_dispatch_tg(tg);
  1030. sq = &tg->service_queue;
  1031. if (sq->nr_queued[0] || sq->nr_queued[1])
  1032. tg_update_disptime(tg);
  1033. if (nr_disp >= THROTL_QUANTUM)
  1034. break;
  1035. }
  1036. return nr_disp;
  1037. }
  1038. static bool throtl_can_upgrade(struct throtl_data *td,
  1039. struct throtl_grp *this_tg);
  1040. /**
  1041. * throtl_pending_timer_fn - timer function for service_queue->pending_timer
  1042. * @t: the pending_timer member of the throtl_service_queue being serviced
  1043. *
  1044. * This timer is armed when a child throtl_grp with active bio's become
  1045. * pending and queued on the service_queue's pending_tree and expires when
  1046. * the first child throtl_grp should be dispatched. This function
  1047. * dispatches bio's from the children throtl_grps to the parent
  1048. * service_queue.
  1049. *
  1050. * If the parent's parent is another throtl_grp, dispatching is propagated
  1051. * by either arming its pending_timer or repeating dispatch directly. If
  1052. * the top-level service_tree is reached, throtl_data->dispatch_work is
  1053. * kicked so that the ready bio's are issued.
  1054. */
  1055. static void throtl_pending_timer_fn(struct timer_list *t)
  1056. {
  1057. struct throtl_service_queue *sq = from_timer(sq, t, pending_timer);
  1058. struct throtl_grp *tg = sq_to_tg(sq);
  1059. struct throtl_data *td = sq_to_td(sq);
  1060. struct request_queue *q = td->queue;
  1061. struct throtl_service_queue *parent_sq;
  1062. bool dispatched;
  1063. int ret;
  1064. spin_lock_irq(&q->queue_lock);
  1065. if (throtl_can_upgrade(td, NULL))
  1066. throtl_upgrade_state(td);
  1067. again:
  1068. parent_sq = sq->parent_sq;
  1069. dispatched = false;
  1070. while (true) {
  1071. throtl_log(sq, "dispatch nr_queued=%u read=%u write=%u",
  1072. sq->nr_queued[READ] + sq->nr_queued[WRITE],
  1073. sq->nr_queued[READ], sq->nr_queued[WRITE]);
  1074. ret = throtl_select_dispatch(sq);
  1075. if (ret) {
  1076. throtl_log(sq, "bios disp=%u", ret);
  1077. dispatched = true;
  1078. }
  1079. if (throtl_schedule_next_dispatch(sq, false))
  1080. break;
  1081. /* this dispatch windows is still open, relax and repeat */
  1082. spin_unlock_irq(&q->queue_lock);
  1083. cpu_relax();
  1084. spin_lock_irq(&q->queue_lock);
  1085. }
  1086. if (!dispatched)
  1087. goto out_unlock;
  1088. if (parent_sq) {
  1089. /* @parent_sq is another throl_grp, propagate dispatch */
  1090. if (tg->flags & THROTL_TG_WAS_EMPTY) {
  1091. tg_update_disptime(tg);
  1092. if (!throtl_schedule_next_dispatch(parent_sq, false)) {
  1093. /* window is already open, repeat dispatching */
  1094. sq = parent_sq;
  1095. tg = sq_to_tg(sq);
  1096. goto again;
  1097. }
  1098. }
  1099. } else {
  1100. /* reached the top-level, queue issuing */
  1101. queue_work(kthrotld_workqueue, &td->dispatch_work);
  1102. }
  1103. out_unlock:
  1104. spin_unlock_irq(&q->queue_lock);
  1105. }
  1106. /**
  1107. * blk_throtl_dispatch_work_fn - work function for throtl_data->dispatch_work
  1108. * @work: work item being executed
  1109. *
  1110. * This function is queued for execution when bios reach the bio_lists[]
  1111. * of throtl_data->service_queue. Those bios are ready and issued by this
  1112. * function.
  1113. */
  1114. static void blk_throtl_dispatch_work_fn(struct work_struct *work)
  1115. {
  1116. struct throtl_data *td = container_of(work, struct throtl_data,
  1117. dispatch_work);
  1118. struct throtl_service_queue *td_sq = &td->service_queue;
  1119. struct request_queue *q = td->queue;
  1120. struct bio_list bio_list_on_stack;
  1121. struct bio *bio;
  1122. struct blk_plug plug;
  1123. int rw;
  1124. bio_list_init(&bio_list_on_stack);
  1125. spin_lock_irq(&q->queue_lock);
  1126. for (rw = READ; rw <= WRITE; rw++)
  1127. while ((bio = throtl_pop_queued(&td_sq->queued[rw], NULL)))
  1128. bio_list_add(&bio_list_on_stack, bio);
  1129. spin_unlock_irq(&q->queue_lock);
  1130. if (!bio_list_empty(&bio_list_on_stack)) {
  1131. blk_start_plug(&plug);
  1132. while ((bio = bio_list_pop(&bio_list_on_stack)))
  1133. submit_bio_noacct(bio);
  1134. blk_finish_plug(&plug);
  1135. }
  1136. }
  1137. static u64 tg_prfill_conf_u64(struct seq_file *sf, struct blkg_policy_data *pd,
  1138. int off)
  1139. {
  1140. struct throtl_grp *tg = pd_to_tg(pd);
  1141. u64 v = *(u64 *)((void *)tg + off);
  1142. if (v == U64_MAX)
  1143. return 0;
  1144. return __blkg_prfill_u64(sf, pd, v);
  1145. }
  1146. static u64 tg_prfill_conf_uint(struct seq_file *sf, struct blkg_policy_data *pd,
  1147. int off)
  1148. {
  1149. struct throtl_grp *tg = pd_to_tg(pd);
  1150. unsigned int v = *(unsigned int *)((void *)tg + off);
  1151. if (v == UINT_MAX)
  1152. return 0;
  1153. return __blkg_prfill_u64(sf, pd, v);
  1154. }
  1155. static int tg_print_conf_u64(struct seq_file *sf, void *v)
  1156. {
  1157. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_conf_u64,
  1158. &blkcg_policy_throtl, seq_cft(sf)->private, false);
  1159. return 0;
  1160. }
  1161. static int tg_print_conf_uint(struct seq_file *sf, void *v)
  1162. {
  1163. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_conf_uint,
  1164. &blkcg_policy_throtl, seq_cft(sf)->private, false);
  1165. return 0;
  1166. }
  1167. static void tg_conf_updated(struct throtl_grp *tg, bool global)
  1168. {
  1169. struct throtl_service_queue *sq = &tg->service_queue;
  1170. struct cgroup_subsys_state *pos_css;
  1171. struct blkcg_gq *blkg;
  1172. throtl_log(&tg->service_queue,
  1173. "limit change rbps=%llu wbps=%llu riops=%u wiops=%u",
  1174. tg_bps_limit(tg, READ), tg_bps_limit(tg, WRITE),
  1175. tg_iops_limit(tg, READ), tg_iops_limit(tg, WRITE));
  1176. /*
  1177. * Update has_rules[] flags for the updated tg's subtree. A tg is
  1178. * considered to have rules if either the tg itself or any of its
  1179. * ancestors has rules. This identifies groups without any
  1180. * restrictions in the whole hierarchy and allows them to bypass
  1181. * blk-throttle.
  1182. */
  1183. blkg_for_each_descendant_pre(blkg, pos_css,
  1184. global ? tg->td->queue->root_blkg : tg_to_blkg(tg)) {
  1185. struct throtl_grp *this_tg = blkg_to_tg(blkg);
  1186. struct throtl_grp *parent_tg;
  1187. tg_update_has_rules(this_tg);
  1188. /* ignore root/second level */
  1189. if (!cgroup_subsys_on_dfl(io_cgrp_subsys) || !blkg->parent ||
  1190. !blkg->parent->parent)
  1191. continue;
  1192. parent_tg = blkg_to_tg(blkg->parent);
  1193. /*
  1194. * make sure all children has lower idle time threshold and
  1195. * higher latency target
  1196. */
  1197. this_tg->idletime_threshold = min(this_tg->idletime_threshold,
  1198. parent_tg->idletime_threshold);
  1199. this_tg->latency_target = max(this_tg->latency_target,
  1200. parent_tg->latency_target);
  1201. }
  1202. /*
  1203. * We're already holding queue_lock and know @tg is valid. Let's
  1204. * apply the new config directly.
  1205. *
  1206. * Restart the slices for both READ and WRITES. It might happen
  1207. * that a group's limit are dropped suddenly and we don't want to
  1208. * account recently dispatched IO with new low rate.
  1209. */
  1210. throtl_start_new_slice(tg, READ);
  1211. throtl_start_new_slice(tg, WRITE);
  1212. if (tg->flags & THROTL_TG_PENDING) {
  1213. tg_update_disptime(tg);
  1214. throtl_schedule_next_dispatch(sq->parent_sq, true);
  1215. }
  1216. }
  1217. static ssize_t tg_set_conf(struct kernfs_open_file *of,
  1218. char *buf, size_t nbytes, loff_t off, bool is_u64)
  1219. {
  1220. struct blkcg *blkcg = css_to_blkcg(of_css(of));
  1221. struct blkg_conf_ctx ctx;
  1222. struct throtl_grp *tg;
  1223. int ret;
  1224. u64 v;
  1225. ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
  1226. if (ret)
  1227. return ret;
  1228. ret = -EINVAL;
  1229. if (sscanf(ctx.body, "%llu", &v) != 1)
  1230. goto out_finish;
  1231. if (!v)
  1232. v = U64_MAX;
  1233. tg = blkg_to_tg(ctx.blkg);
  1234. if (is_u64)
  1235. *(u64 *)((void *)tg + of_cft(of)->private) = v;
  1236. else
  1237. *(unsigned int *)((void *)tg + of_cft(of)->private) = v;
  1238. tg_conf_updated(tg, false);
  1239. ret = 0;
  1240. out_finish:
  1241. blkg_conf_finish(&ctx);
  1242. return ret ?: nbytes;
  1243. }
  1244. static ssize_t tg_set_conf_u64(struct kernfs_open_file *of,
  1245. char *buf, size_t nbytes, loff_t off)
  1246. {
  1247. return tg_set_conf(of, buf, nbytes, off, true);
  1248. }
  1249. static ssize_t tg_set_conf_uint(struct kernfs_open_file *of,
  1250. char *buf, size_t nbytes, loff_t off)
  1251. {
  1252. return tg_set_conf(of, buf, nbytes, off, false);
  1253. }
  1254. static int tg_print_rwstat(struct seq_file *sf, void *v)
  1255. {
  1256. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  1257. blkg_prfill_rwstat, &blkcg_policy_throtl,
  1258. seq_cft(sf)->private, true);
  1259. return 0;
  1260. }
  1261. static u64 tg_prfill_rwstat_recursive(struct seq_file *sf,
  1262. struct blkg_policy_data *pd, int off)
  1263. {
  1264. struct blkg_rwstat_sample sum;
  1265. blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_throtl, off,
  1266. &sum);
  1267. return __blkg_prfill_rwstat(sf, pd, &sum);
  1268. }
  1269. static int tg_print_rwstat_recursive(struct seq_file *sf, void *v)
  1270. {
  1271. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  1272. tg_prfill_rwstat_recursive, &blkcg_policy_throtl,
  1273. seq_cft(sf)->private, true);
  1274. return 0;
  1275. }
  1276. static struct cftype throtl_legacy_files[] = {
  1277. {
  1278. .name = "throttle.read_bps_device",
  1279. .private = offsetof(struct throtl_grp, bps[READ][LIMIT_MAX]),
  1280. .seq_show = tg_print_conf_u64,
  1281. .write = tg_set_conf_u64,
  1282. },
  1283. {
  1284. .name = "throttle.write_bps_device",
  1285. .private = offsetof(struct throtl_grp, bps[WRITE][LIMIT_MAX]),
  1286. .seq_show = tg_print_conf_u64,
  1287. .write = tg_set_conf_u64,
  1288. },
  1289. {
  1290. .name = "throttle.read_iops_device",
  1291. .private = offsetof(struct throtl_grp, iops[READ][LIMIT_MAX]),
  1292. .seq_show = tg_print_conf_uint,
  1293. .write = tg_set_conf_uint,
  1294. },
  1295. {
  1296. .name = "throttle.write_iops_device",
  1297. .private = offsetof(struct throtl_grp, iops[WRITE][LIMIT_MAX]),
  1298. .seq_show = tg_print_conf_uint,
  1299. .write = tg_set_conf_uint,
  1300. },
  1301. {
  1302. .name = "throttle.io_service_bytes",
  1303. .private = offsetof(struct throtl_grp, stat_bytes),
  1304. .seq_show = tg_print_rwstat,
  1305. },
  1306. {
  1307. .name = "throttle.io_service_bytes_recursive",
  1308. .private = offsetof(struct throtl_grp, stat_bytes),
  1309. .seq_show = tg_print_rwstat_recursive,
  1310. },
  1311. {
  1312. .name = "throttle.io_serviced",
  1313. .private = offsetof(struct throtl_grp, stat_ios),
  1314. .seq_show = tg_print_rwstat,
  1315. },
  1316. {
  1317. .name = "throttle.io_serviced_recursive",
  1318. .private = offsetof(struct throtl_grp, stat_ios),
  1319. .seq_show = tg_print_rwstat_recursive,
  1320. },
  1321. { } /* terminate */
  1322. };
  1323. static u64 tg_prfill_limit(struct seq_file *sf, struct blkg_policy_data *pd,
  1324. int off)
  1325. {
  1326. struct throtl_grp *tg = pd_to_tg(pd);
  1327. const char *dname = blkg_dev_name(pd->blkg);
  1328. char bufs[4][21] = { "max", "max", "max", "max" };
  1329. u64 bps_dft;
  1330. unsigned int iops_dft;
  1331. char idle_time[26] = "";
  1332. char latency_time[26] = "";
  1333. if (!dname)
  1334. return 0;
  1335. if (off == LIMIT_LOW) {
  1336. bps_dft = 0;
  1337. iops_dft = 0;
  1338. } else {
  1339. bps_dft = U64_MAX;
  1340. iops_dft = UINT_MAX;
  1341. }
  1342. if (tg->bps_conf[READ][off] == bps_dft &&
  1343. tg->bps_conf[WRITE][off] == bps_dft &&
  1344. tg->iops_conf[READ][off] == iops_dft &&
  1345. tg->iops_conf[WRITE][off] == iops_dft &&
  1346. (off != LIMIT_LOW ||
  1347. (tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD &&
  1348. tg->latency_target_conf == DFL_LATENCY_TARGET)))
  1349. return 0;
  1350. if (tg->bps_conf[READ][off] != U64_MAX)
  1351. snprintf(bufs[0], sizeof(bufs[0]), "%llu",
  1352. tg->bps_conf[READ][off]);
  1353. if (tg->bps_conf[WRITE][off] != U64_MAX)
  1354. snprintf(bufs[1], sizeof(bufs[1]), "%llu",
  1355. tg->bps_conf[WRITE][off]);
  1356. if (tg->iops_conf[READ][off] != UINT_MAX)
  1357. snprintf(bufs[2], sizeof(bufs[2]), "%u",
  1358. tg->iops_conf[READ][off]);
  1359. if (tg->iops_conf[WRITE][off] != UINT_MAX)
  1360. snprintf(bufs[3], sizeof(bufs[3]), "%u",
  1361. tg->iops_conf[WRITE][off]);
  1362. if (off == LIMIT_LOW) {
  1363. if (tg->idletime_threshold_conf == ULONG_MAX)
  1364. strcpy(idle_time, " idle=max");
  1365. else
  1366. snprintf(idle_time, sizeof(idle_time), " idle=%lu",
  1367. tg->idletime_threshold_conf);
  1368. if (tg->latency_target_conf == ULONG_MAX)
  1369. strcpy(latency_time, " latency=max");
  1370. else
  1371. snprintf(latency_time, sizeof(latency_time),
  1372. " latency=%lu", tg->latency_target_conf);
  1373. }
  1374. seq_printf(sf, "%s rbps=%s wbps=%s riops=%s wiops=%s%s%s\n",
  1375. dname, bufs[0], bufs[1], bufs[2], bufs[3], idle_time,
  1376. latency_time);
  1377. return 0;
  1378. }
  1379. static int tg_print_limit(struct seq_file *sf, void *v)
  1380. {
  1381. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), tg_prfill_limit,
  1382. &blkcg_policy_throtl, seq_cft(sf)->private, false);
  1383. return 0;
  1384. }
  1385. static ssize_t tg_set_limit(struct kernfs_open_file *of,
  1386. char *buf, size_t nbytes, loff_t off)
  1387. {
  1388. struct blkcg *blkcg = css_to_blkcg(of_css(of));
  1389. struct blkg_conf_ctx ctx;
  1390. struct throtl_grp *tg;
  1391. u64 v[4];
  1392. unsigned long idle_time;
  1393. unsigned long latency_time;
  1394. int ret;
  1395. int index = of_cft(of)->private;
  1396. ret = blkg_conf_prep(blkcg, &blkcg_policy_throtl, buf, &ctx);
  1397. if (ret)
  1398. return ret;
  1399. tg = blkg_to_tg(ctx.blkg);
  1400. v[0] = tg->bps_conf[READ][index];
  1401. v[1] = tg->bps_conf[WRITE][index];
  1402. v[2] = tg->iops_conf[READ][index];
  1403. v[3] = tg->iops_conf[WRITE][index];
  1404. idle_time = tg->idletime_threshold_conf;
  1405. latency_time = tg->latency_target_conf;
  1406. while (true) {
  1407. char tok[27]; /* wiops=18446744073709551616 */
  1408. char *p;
  1409. u64 val = U64_MAX;
  1410. int len;
  1411. if (sscanf(ctx.body, "%26s%n", tok, &len) != 1)
  1412. break;
  1413. if (tok[0] == '\0')
  1414. break;
  1415. ctx.body += len;
  1416. ret = -EINVAL;
  1417. p = tok;
  1418. strsep(&p, "=");
  1419. if (!p || (sscanf(p, "%llu", &val) != 1 && strcmp(p, "max")))
  1420. goto out_finish;
  1421. ret = -ERANGE;
  1422. if (!val)
  1423. goto out_finish;
  1424. ret = -EINVAL;
  1425. if (!strcmp(tok, "rbps") && val > 1)
  1426. v[0] = val;
  1427. else if (!strcmp(tok, "wbps") && val > 1)
  1428. v[1] = val;
  1429. else if (!strcmp(tok, "riops") && val > 1)
  1430. v[2] = min_t(u64, val, UINT_MAX);
  1431. else if (!strcmp(tok, "wiops") && val > 1)
  1432. v[3] = min_t(u64, val, UINT_MAX);
  1433. else if (off == LIMIT_LOW && !strcmp(tok, "idle"))
  1434. idle_time = val;
  1435. else if (off == LIMIT_LOW && !strcmp(tok, "latency"))
  1436. latency_time = val;
  1437. else
  1438. goto out_finish;
  1439. }
  1440. tg->bps_conf[READ][index] = v[0];
  1441. tg->bps_conf[WRITE][index] = v[1];
  1442. tg->iops_conf[READ][index] = v[2];
  1443. tg->iops_conf[WRITE][index] = v[3];
  1444. if (index == LIMIT_MAX) {
  1445. tg->bps[READ][index] = v[0];
  1446. tg->bps[WRITE][index] = v[1];
  1447. tg->iops[READ][index] = v[2];
  1448. tg->iops[WRITE][index] = v[3];
  1449. }
  1450. tg->bps[READ][LIMIT_LOW] = min(tg->bps_conf[READ][LIMIT_LOW],
  1451. tg->bps_conf[READ][LIMIT_MAX]);
  1452. tg->bps[WRITE][LIMIT_LOW] = min(tg->bps_conf[WRITE][LIMIT_LOW],
  1453. tg->bps_conf[WRITE][LIMIT_MAX]);
  1454. tg->iops[READ][LIMIT_LOW] = min(tg->iops_conf[READ][LIMIT_LOW],
  1455. tg->iops_conf[READ][LIMIT_MAX]);
  1456. tg->iops[WRITE][LIMIT_LOW] = min(tg->iops_conf[WRITE][LIMIT_LOW],
  1457. tg->iops_conf[WRITE][LIMIT_MAX]);
  1458. tg->idletime_threshold_conf = idle_time;
  1459. tg->latency_target_conf = latency_time;
  1460. /* force user to configure all settings for low limit */
  1461. if (!(tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW] ||
  1462. tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW]) ||
  1463. tg->idletime_threshold_conf == DFL_IDLE_THRESHOLD ||
  1464. tg->latency_target_conf == DFL_LATENCY_TARGET) {
  1465. tg->bps[READ][LIMIT_LOW] = 0;
  1466. tg->bps[WRITE][LIMIT_LOW] = 0;
  1467. tg->iops[READ][LIMIT_LOW] = 0;
  1468. tg->iops[WRITE][LIMIT_LOW] = 0;
  1469. tg->idletime_threshold = DFL_IDLE_THRESHOLD;
  1470. tg->latency_target = DFL_LATENCY_TARGET;
  1471. } else if (index == LIMIT_LOW) {
  1472. tg->idletime_threshold = tg->idletime_threshold_conf;
  1473. tg->latency_target = tg->latency_target_conf;
  1474. }
  1475. blk_throtl_update_limit_valid(tg->td);
  1476. if (tg->td->limit_valid[LIMIT_LOW]) {
  1477. if (index == LIMIT_LOW)
  1478. tg->td->limit_index = LIMIT_LOW;
  1479. } else
  1480. tg->td->limit_index = LIMIT_MAX;
  1481. tg_conf_updated(tg, index == LIMIT_LOW &&
  1482. tg->td->limit_valid[LIMIT_LOW]);
  1483. ret = 0;
  1484. out_finish:
  1485. blkg_conf_finish(&ctx);
  1486. return ret ?: nbytes;
  1487. }
  1488. static struct cftype throtl_files[] = {
  1489. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  1490. {
  1491. .name = "low",
  1492. .flags = CFTYPE_NOT_ON_ROOT,
  1493. .seq_show = tg_print_limit,
  1494. .write = tg_set_limit,
  1495. .private = LIMIT_LOW,
  1496. },
  1497. #endif
  1498. {
  1499. .name = "max",
  1500. .flags = CFTYPE_NOT_ON_ROOT,
  1501. .seq_show = tg_print_limit,
  1502. .write = tg_set_limit,
  1503. .private = LIMIT_MAX,
  1504. },
  1505. { } /* terminate */
  1506. };
  1507. static void throtl_shutdown_wq(struct request_queue *q)
  1508. {
  1509. struct throtl_data *td = q->td;
  1510. cancel_work_sync(&td->dispatch_work);
  1511. }
  1512. static struct blkcg_policy blkcg_policy_throtl = {
  1513. .dfl_cftypes = throtl_files,
  1514. .legacy_cftypes = throtl_legacy_files,
  1515. .pd_alloc_fn = throtl_pd_alloc,
  1516. .pd_init_fn = throtl_pd_init,
  1517. .pd_online_fn = throtl_pd_online,
  1518. .pd_offline_fn = throtl_pd_offline,
  1519. .pd_free_fn = throtl_pd_free,
  1520. };
  1521. static unsigned long __tg_last_low_overflow_time(struct throtl_grp *tg)
  1522. {
  1523. unsigned long rtime = jiffies, wtime = jiffies;
  1524. if (tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW])
  1525. rtime = tg->last_low_overflow_time[READ];
  1526. if (tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW])
  1527. wtime = tg->last_low_overflow_time[WRITE];
  1528. return min(rtime, wtime);
  1529. }
  1530. /* tg should not be an intermediate node */
  1531. static unsigned long tg_last_low_overflow_time(struct throtl_grp *tg)
  1532. {
  1533. struct throtl_service_queue *parent_sq;
  1534. struct throtl_grp *parent = tg;
  1535. unsigned long ret = __tg_last_low_overflow_time(tg);
  1536. while (true) {
  1537. parent_sq = parent->service_queue.parent_sq;
  1538. parent = sq_to_tg(parent_sq);
  1539. if (!parent)
  1540. break;
  1541. /*
  1542. * The parent doesn't have low limit, it always reaches low
  1543. * limit. Its overflow time is useless for children
  1544. */
  1545. if (!parent->bps[READ][LIMIT_LOW] &&
  1546. !parent->iops[READ][LIMIT_LOW] &&
  1547. !parent->bps[WRITE][LIMIT_LOW] &&
  1548. !parent->iops[WRITE][LIMIT_LOW])
  1549. continue;
  1550. if (time_after(__tg_last_low_overflow_time(parent), ret))
  1551. ret = __tg_last_low_overflow_time(parent);
  1552. }
  1553. return ret;
  1554. }
  1555. static bool throtl_tg_is_idle(struct throtl_grp *tg)
  1556. {
  1557. /*
  1558. * cgroup is idle if:
  1559. * - single idle is too long, longer than a fixed value (in case user
  1560. * configure a too big threshold) or 4 times of idletime threshold
  1561. * - average think time is more than threshold
  1562. * - IO latency is largely below threshold
  1563. */
  1564. unsigned long time;
  1565. bool ret;
  1566. time = min_t(unsigned long, MAX_IDLE_TIME, 4 * tg->idletime_threshold);
  1567. ret = tg->latency_target == DFL_LATENCY_TARGET ||
  1568. tg->idletime_threshold == DFL_IDLE_THRESHOLD ||
  1569. (ktime_get_ns() >> 10) - tg->last_finish_time > time ||
  1570. tg->avg_idletime > tg->idletime_threshold ||
  1571. (tg->latency_target && tg->bio_cnt &&
  1572. tg->bad_bio_cnt * 5 < tg->bio_cnt);
  1573. throtl_log(&tg->service_queue,
  1574. "avg_idle=%ld, idle_threshold=%ld, bad_bio=%d, total_bio=%d, is_idle=%d, scale=%d",
  1575. tg->avg_idletime, tg->idletime_threshold, tg->bad_bio_cnt,
  1576. tg->bio_cnt, ret, tg->td->scale);
  1577. return ret;
  1578. }
  1579. static bool throtl_tg_can_upgrade(struct throtl_grp *tg)
  1580. {
  1581. struct throtl_service_queue *sq = &tg->service_queue;
  1582. bool read_limit, write_limit;
  1583. /*
  1584. * if cgroup reaches low limit (if low limit is 0, the cgroup always
  1585. * reaches), it's ok to upgrade to next limit
  1586. */
  1587. read_limit = tg->bps[READ][LIMIT_LOW] || tg->iops[READ][LIMIT_LOW];
  1588. write_limit = tg->bps[WRITE][LIMIT_LOW] || tg->iops[WRITE][LIMIT_LOW];
  1589. if (!read_limit && !write_limit)
  1590. return true;
  1591. if (read_limit && sq->nr_queued[READ] &&
  1592. (!write_limit || sq->nr_queued[WRITE]))
  1593. return true;
  1594. if (write_limit && sq->nr_queued[WRITE] &&
  1595. (!read_limit || sq->nr_queued[READ]))
  1596. return true;
  1597. if (time_after_eq(jiffies,
  1598. tg_last_low_overflow_time(tg) + tg->td->throtl_slice) &&
  1599. throtl_tg_is_idle(tg))
  1600. return true;
  1601. return false;
  1602. }
  1603. static bool throtl_hierarchy_can_upgrade(struct throtl_grp *tg)
  1604. {
  1605. while (true) {
  1606. if (throtl_tg_can_upgrade(tg))
  1607. return true;
  1608. tg = sq_to_tg(tg->service_queue.parent_sq);
  1609. if (!tg || !tg_to_blkg(tg)->parent)
  1610. return false;
  1611. }
  1612. return false;
  1613. }
  1614. static bool throtl_can_upgrade(struct throtl_data *td,
  1615. struct throtl_grp *this_tg)
  1616. {
  1617. struct cgroup_subsys_state *pos_css;
  1618. struct blkcg_gq *blkg;
  1619. if (td->limit_index != LIMIT_LOW)
  1620. return false;
  1621. if (time_before(jiffies, td->low_downgrade_time + td->throtl_slice))
  1622. return false;
  1623. rcu_read_lock();
  1624. blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
  1625. struct throtl_grp *tg = blkg_to_tg(blkg);
  1626. if (tg == this_tg)
  1627. continue;
  1628. if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
  1629. continue;
  1630. if (!throtl_hierarchy_can_upgrade(tg)) {
  1631. rcu_read_unlock();
  1632. return false;
  1633. }
  1634. }
  1635. rcu_read_unlock();
  1636. return true;
  1637. }
  1638. static void throtl_upgrade_check(struct throtl_grp *tg)
  1639. {
  1640. unsigned long now = jiffies;
  1641. if (tg->td->limit_index != LIMIT_LOW)
  1642. return;
  1643. if (time_after(tg->last_check_time + tg->td->throtl_slice, now))
  1644. return;
  1645. tg->last_check_time = now;
  1646. if (!time_after_eq(now,
  1647. __tg_last_low_overflow_time(tg) + tg->td->throtl_slice))
  1648. return;
  1649. if (throtl_can_upgrade(tg->td, NULL))
  1650. throtl_upgrade_state(tg->td);
  1651. }
  1652. static void throtl_upgrade_state(struct throtl_data *td)
  1653. {
  1654. struct cgroup_subsys_state *pos_css;
  1655. struct blkcg_gq *blkg;
  1656. throtl_log(&td->service_queue, "upgrade to max");
  1657. td->limit_index = LIMIT_MAX;
  1658. td->low_upgrade_time = jiffies;
  1659. td->scale = 0;
  1660. rcu_read_lock();
  1661. blkg_for_each_descendant_post(blkg, pos_css, td->queue->root_blkg) {
  1662. struct throtl_grp *tg = blkg_to_tg(blkg);
  1663. struct throtl_service_queue *sq = &tg->service_queue;
  1664. tg->disptime = jiffies - 1;
  1665. throtl_select_dispatch(sq);
  1666. throtl_schedule_next_dispatch(sq, true);
  1667. }
  1668. rcu_read_unlock();
  1669. throtl_select_dispatch(&td->service_queue);
  1670. throtl_schedule_next_dispatch(&td->service_queue, true);
  1671. queue_work(kthrotld_workqueue, &td->dispatch_work);
  1672. }
  1673. static void throtl_downgrade_state(struct throtl_data *td)
  1674. {
  1675. td->scale /= 2;
  1676. throtl_log(&td->service_queue, "downgrade, scale %d", td->scale);
  1677. if (td->scale) {
  1678. td->low_upgrade_time = jiffies - td->scale * td->throtl_slice;
  1679. return;
  1680. }
  1681. td->limit_index = LIMIT_LOW;
  1682. td->low_downgrade_time = jiffies;
  1683. }
  1684. static bool throtl_tg_can_downgrade(struct throtl_grp *tg)
  1685. {
  1686. struct throtl_data *td = tg->td;
  1687. unsigned long now = jiffies;
  1688. /*
  1689. * If cgroup is below low limit, consider downgrade and throttle other
  1690. * cgroups
  1691. */
  1692. if (time_after_eq(now, td->low_upgrade_time + td->throtl_slice) &&
  1693. time_after_eq(now, tg_last_low_overflow_time(tg) +
  1694. td->throtl_slice) &&
  1695. (!throtl_tg_is_idle(tg) ||
  1696. !list_empty(&tg_to_blkg(tg)->blkcg->css.children)))
  1697. return true;
  1698. return false;
  1699. }
  1700. static bool throtl_hierarchy_can_downgrade(struct throtl_grp *tg)
  1701. {
  1702. while (true) {
  1703. if (!throtl_tg_can_downgrade(tg))
  1704. return false;
  1705. tg = sq_to_tg(tg->service_queue.parent_sq);
  1706. if (!tg || !tg_to_blkg(tg)->parent)
  1707. break;
  1708. }
  1709. return true;
  1710. }
  1711. static void throtl_downgrade_check(struct throtl_grp *tg)
  1712. {
  1713. uint64_t bps;
  1714. unsigned int iops;
  1715. unsigned long elapsed_time;
  1716. unsigned long now = jiffies;
  1717. if (tg->td->limit_index != LIMIT_MAX ||
  1718. !tg->td->limit_valid[LIMIT_LOW])
  1719. return;
  1720. if (!list_empty(&tg_to_blkg(tg)->blkcg->css.children))
  1721. return;
  1722. if (time_after(tg->last_check_time + tg->td->throtl_slice, now))
  1723. return;
  1724. elapsed_time = now - tg->last_check_time;
  1725. tg->last_check_time = now;
  1726. if (time_before(now, tg_last_low_overflow_time(tg) +
  1727. tg->td->throtl_slice))
  1728. return;
  1729. if (tg->bps[READ][LIMIT_LOW]) {
  1730. bps = tg->last_bytes_disp[READ] * HZ;
  1731. do_div(bps, elapsed_time);
  1732. if (bps >= tg->bps[READ][LIMIT_LOW])
  1733. tg->last_low_overflow_time[READ] = now;
  1734. }
  1735. if (tg->bps[WRITE][LIMIT_LOW]) {
  1736. bps = tg->last_bytes_disp[WRITE] * HZ;
  1737. do_div(bps, elapsed_time);
  1738. if (bps >= tg->bps[WRITE][LIMIT_LOW])
  1739. tg->last_low_overflow_time[WRITE] = now;
  1740. }
  1741. if (tg->iops[READ][LIMIT_LOW]) {
  1742. tg->last_io_disp[READ] += atomic_xchg(&tg->last_io_split_cnt[READ], 0);
  1743. iops = tg->last_io_disp[READ] * HZ / elapsed_time;
  1744. if (iops >= tg->iops[READ][LIMIT_LOW])
  1745. tg->last_low_overflow_time[READ] = now;
  1746. }
  1747. if (tg->iops[WRITE][LIMIT_LOW]) {
  1748. tg->last_io_disp[WRITE] += atomic_xchg(&tg->last_io_split_cnt[WRITE], 0);
  1749. iops = tg->last_io_disp[WRITE] * HZ / elapsed_time;
  1750. if (iops >= tg->iops[WRITE][LIMIT_LOW])
  1751. tg->last_low_overflow_time[WRITE] = now;
  1752. }
  1753. /*
  1754. * If cgroup is below low limit, consider downgrade and throttle other
  1755. * cgroups
  1756. */
  1757. if (throtl_hierarchy_can_downgrade(tg))
  1758. throtl_downgrade_state(tg->td);
  1759. tg->last_bytes_disp[READ] = 0;
  1760. tg->last_bytes_disp[WRITE] = 0;
  1761. tg->last_io_disp[READ] = 0;
  1762. tg->last_io_disp[WRITE] = 0;
  1763. }
  1764. static void blk_throtl_update_idletime(struct throtl_grp *tg)
  1765. {
  1766. unsigned long now;
  1767. unsigned long last_finish_time = tg->last_finish_time;
  1768. if (last_finish_time == 0)
  1769. return;
  1770. now = ktime_get_ns() >> 10;
  1771. if (now <= last_finish_time ||
  1772. last_finish_time == tg->checked_last_finish_time)
  1773. return;
  1774. tg->avg_idletime = (tg->avg_idletime * 7 + now - last_finish_time) >> 3;
  1775. tg->checked_last_finish_time = last_finish_time;
  1776. }
  1777. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  1778. static void throtl_update_latency_buckets(struct throtl_data *td)
  1779. {
  1780. struct avg_latency_bucket avg_latency[2][LATENCY_BUCKET_SIZE];
  1781. int i, cpu, rw;
  1782. unsigned long last_latency[2] = { 0 };
  1783. unsigned long latency[2];
  1784. if (!blk_queue_nonrot(td->queue) || !td->limit_valid[LIMIT_LOW])
  1785. return;
  1786. if (time_before(jiffies, td->last_calculate_time + HZ))
  1787. return;
  1788. td->last_calculate_time = jiffies;
  1789. memset(avg_latency, 0, sizeof(avg_latency));
  1790. for (rw = READ; rw <= WRITE; rw++) {
  1791. for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
  1792. struct latency_bucket *tmp = &td->tmp_buckets[rw][i];
  1793. for_each_possible_cpu(cpu) {
  1794. struct latency_bucket *bucket;
  1795. /* this isn't race free, but ok in practice */
  1796. bucket = per_cpu_ptr(td->latency_buckets[rw],
  1797. cpu);
  1798. tmp->total_latency += bucket[i].total_latency;
  1799. tmp->samples += bucket[i].samples;
  1800. bucket[i].total_latency = 0;
  1801. bucket[i].samples = 0;
  1802. }
  1803. if (tmp->samples >= 32) {
  1804. int samples = tmp->samples;
  1805. latency[rw] = tmp->total_latency;
  1806. tmp->total_latency = 0;
  1807. tmp->samples = 0;
  1808. latency[rw] /= samples;
  1809. if (latency[rw] == 0)
  1810. continue;
  1811. avg_latency[rw][i].latency = latency[rw];
  1812. }
  1813. }
  1814. }
  1815. for (rw = READ; rw <= WRITE; rw++) {
  1816. for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
  1817. if (!avg_latency[rw][i].latency) {
  1818. if (td->avg_buckets[rw][i].latency < last_latency[rw])
  1819. td->avg_buckets[rw][i].latency =
  1820. last_latency[rw];
  1821. continue;
  1822. }
  1823. if (!td->avg_buckets[rw][i].valid)
  1824. latency[rw] = avg_latency[rw][i].latency;
  1825. else
  1826. latency[rw] = (td->avg_buckets[rw][i].latency * 7 +
  1827. avg_latency[rw][i].latency) >> 3;
  1828. td->avg_buckets[rw][i].latency = max(latency[rw],
  1829. last_latency[rw]);
  1830. td->avg_buckets[rw][i].valid = true;
  1831. last_latency[rw] = td->avg_buckets[rw][i].latency;
  1832. }
  1833. }
  1834. for (i = 0; i < LATENCY_BUCKET_SIZE; i++)
  1835. throtl_log(&td->service_queue,
  1836. "Latency bucket %d: read latency=%ld, read valid=%d, "
  1837. "write latency=%ld, write valid=%d", i,
  1838. td->avg_buckets[READ][i].latency,
  1839. td->avg_buckets[READ][i].valid,
  1840. td->avg_buckets[WRITE][i].latency,
  1841. td->avg_buckets[WRITE][i].valid);
  1842. }
  1843. #else
  1844. static inline void throtl_update_latency_buckets(struct throtl_data *td)
  1845. {
  1846. }
  1847. #endif
  1848. void blk_throtl_charge_bio_split(struct bio *bio)
  1849. {
  1850. struct blkcg_gq *blkg = bio->bi_blkg;
  1851. struct throtl_grp *parent = blkg_to_tg(blkg);
  1852. struct throtl_service_queue *parent_sq;
  1853. bool rw = bio_data_dir(bio);
  1854. do {
  1855. if (!parent->has_rules[rw])
  1856. break;
  1857. atomic_inc(&parent->io_split_cnt[rw]);
  1858. atomic_inc(&parent->last_io_split_cnt[rw]);
  1859. parent_sq = parent->service_queue.parent_sq;
  1860. parent = sq_to_tg(parent_sq);
  1861. } while (parent);
  1862. }
  1863. bool blk_throtl_bio(struct bio *bio)
  1864. {
  1865. struct request_queue *q = bio->bi_disk->queue;
  1866. struct blkcg_gq *blkg = bio->bi_blkg;
  1867. struct throtl_qnode *qn = NULL;
  1868. struct throtl_grp *tg = blkg_to_tg(blkg);
  1869. struct throtl_service_queue *sq;
  1870. bool rw = bio_data_dir(bio);
  1871. bool throttled = false;
  1872. struct throtl_data *td = tg->td;
  1873. rcu_read_lock();
  1874. /* see throtl_charge_bio() */
  1875. if (bio_flagged(bio, BIO_THROTTLED))
  1876. goto out;
  1877. if (!cgroup_subsys_on_dfl(io_cgrp_subsys)) {
  1878. blkg_rwstat_add(&tg->stat_bytes, bio->bi_opf,
  1879. bio->bi_iter.bi_size);
  1880. blkg_rwstat_add(&tg->stat_ios, bio->bi_opf, 1);
  1881. }
  1882. if (!tg->has_rules[rw])
  1883. goto out;
  1884. spin_lock_irq(&q->queue_lock);
  1885. throtl_update_latency_buckets(td);
  1886. blk_throtl_update_idletime(tg);
  1887. sq = &tg->service_queue;
  1888. again:
  1889. while (true) {
  1890. if (tg->last_low_overflow_time[rw] == 0)
  1891. tg->last_low_overflow_time[rw] = jiffies;
  1892. throtl_downgrade_check(tg);
  1893. throtl_upgrade_check(tg);
  1894. /* throtl is FIFO - if bios are already queued, should queue */
  1895. if (sq->nr_queued[rw])
  1896. break;
  1897. /* if above limits, break to queue */
  1898. if (!tg_may_dispatch(tg, bio, NULL)) {
  1899. tg->last_low_overflow_time[rw] = jiffies;
  1900. if (throtl_can_upgrade(td, tg)) {
  1901. throtl_upgrade_state(td);
  1902. goto again;
  1903. }
  1904. break;
  1905. }
  1906. /* within limits, let's charge and dispatch directly */
  1907. throtl_charge_bio(tg, bio);
  1908. /*
  1909. * We need to trim slice even when bios are not being queued
  1910. * otherwise it might happen that a bio is not queued for
  1911. * a long time and slice keeps on extending and trim is not
  1912. * called for a long time. Now if limits are reduced suddenly
  1913. * we take into account all the IO dispatched so far at new
  1914. * low rate and * newly queued IO gets a really long dispatch
  1915. * time.
  1916. *
  1917. * So keep on trimming slice even if bio is not queued.
  1918. */
  1919. throtl_trim_slice(tg, rw);
  1920. /*
  1921. * @bio passed through this layer without being throttled.
  1922. * Climb up the ladder. If we're already at the top, it
  1923. * can be executed directly.
  1924. */
  1925. qn = &tg->qnode_on_parent[rw];
  1926. sq = sq->parent_sq;
  1927. tg = sq_to_tg(sq);
  1928. if (!tg)
  1929. goto out_unlock;
  1930. }
  1931. /* out-of-limit, queue to @tg */
  1932. throtl_log(sq, "[%c] bio. bdisp=%llu sz=%u bps=%llu iodisp=%u iops=%u queued=%d/%d",
  1933. rw == READ ? 'R' : 'W',
  1934. tg->bytes_disp[rw], bio->bi_iter.bi_size,
  1935. tg_bps_limit(tg, rw),
  1936. tg->io_disp[rw], tg_iops_limit(tg, rw),
  1937. sq->nr_queued[READ], sq->nr_queued[WRITE]);
  1938. tg->last_low_overflow_time[rw] = jiffies;
  1939. td->nr_queued[rw]++;
  1940. throtl_add_bio_tg(bio, qn, tg);
  1941. throttled = true;
  1942. /*
  1943. * Update @tg's dispatch time and force schedule dispatch if @tg
  1944. * was empty before @bio. The forced scheduling isn't likely to
  1945. * cause undue delay as @bio is likely to be dispatched directly if
  1946. * its @tg's disptime is not in the future.
  1947. */
  1948. if (tg->flags & THROTL_TG_WAS_EMPTY) {
  1949. tg_update_disptime(tg);
  1950. throtl_schedule_next_dispatch(tg->service_queue.parent_sq, true);
  1951. }
  1952. out_unlock:
  1953. spin_unlock_irq(&q->queue_lock);
  1954. out:
  1955. bio_set_flag(bio, BIO_THROTTLED);
  1956. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  1957. if (throttled || !td->track_bio_latency)
  1958. bio->bi_issue.value |= BIO_ISSUE_THROTL_SKIP_LATENCY;
  1959. #endif
  1960. rcu_read_unlock();
  1961. return throttled;
  1962. }
  1963. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  1964. static void throtl_track_latency(struct throtl_data *td, sector_t size,
  1965. int op, unsigned long time)
  1966. {
  1967. struct latency_bucket *latency;
  1968. int index;
  1969. if (!td || td->limit_index != LIMIT_LOW ||
  1970. !(op == REQ_OP_READ || op == REQ_OP_WRITE) ||
  1971. !blk_queue_nonrot(td->queue))
  1972. return;
  1973. index = request_bucket_index(size);
  1974. latency = get_cpu_ptr(td->latency_buckets[op]);
  1975. latency[index].total_latency += time;
  1976. latency[index].samples++;
  1977. put_cpu_ptr(td->latency_buckets[op]);
  1978. }
  1979. void blk_throtl_stat_add(struct request *rq, u64 time_ns)
  1980. {
  1981. struct request_queue *q = rq->q;
  1982. struct throtl_data *td = q->td;
  1983. throtl_track_latency(td, blk_rq_stats_sectors(rq), req_op(rq),
  1984. time_ns >> 10);
  1985. }
  1986. void blk_throtl_bio_endio(struct bio *bio)
  1987. {
  1988. struct blkcg_gq *blkg;
  1989. struct throtl_grp *tg;
  1990. u64 finish_time_ns;
  1991. unsigned long finish_time;
  1992. unsigned long start_time;
  1993. unsigned long lat;
  1994. int rw = bio_data_dir(bio);
  1995. blkg = bio->bi_blkg;
  1996. if (!blkg)
  1997. return;
  1998. tg = blkg_to_tg(blkg);
  1999. if (!tg->td->limit_valid[LIMIT_LOW])
  2000. return;
  2001. finish_time_ns = ktime_get_ns();
  2002. tg->last_finish_time = finish_time_ns >> 10;
  2003. start_time = bio_issue_time(&bio->bi_issue) >> 10;
  2004. finish_time = __bio_issue_time(finish_time_ns) >> 10;
  2005. if (!start_time || finish_time <= start_time)
  2006. return;
  2007. lat = finish_time - start_time;
  2008. /* this is only for bio based driver */
  2009. if (!(bio->bi_issue.value & BIO_ISSUE_THROTL_SKIP_LATENCY))
  2010. throtl_track_latency(tg->td, bio_issue_size(&bio->bi_issue),
  2011. bio_op(bio), lat);
  2012. if (tg->latency_target && lat >= tg->td->filtered_latency) {
  2013. int bucket;
  2014. unsigned int threshold;
  2015. bucket = request_bucket_index(bio_issue_size(&bio->bi_issue));
  2016. threshold = tg->td->avg_buckets[rw][bucket].latency +
  2017. tg->latency_target;
  2018. if (lat > threshold)
  2019. tg->bad_bio_cnt++;
  2020. /*
  2021. * Not race free, could get wrong count, which means cgroups
  2022. * will be throttled
  2023. */
  2024. tg->bio_cnt++;
  2025. }
  2026. if (time_after(jiffies, tg->bio_cnt_reset_time) || tg->bio_cnt > 1024) {
  2027. tg->bio_cnt_reset_time = tg->td->throtl_slice + jiffies;
  2028. tg->bio_cnt /= 2;
  2029. tg->bad_bio_cnt /= 2;
  2030. }
  2031. }
  2032. #endif
  2033. int blk_throtl_init(struct request_queue *q)
  2034. {
  2035. struct throtl_data *td;
  2036. int ret;
  2037. td = kzalloc_node(sizeof(*td), GFP_KERNEL, q->node);
  2038. if (!td)
  2039. return -ENOMEM;
  2040. td->latency_buckets[READ] = __alloc_percpu(sizeof(struct latency_bucket) *
  2041. LATENCY_BUCKET_SIZE, __alignof__(u64));
  2042. if (!td->latency_buckets[READ]) {
  2043. kfree(td);
  2044. return -ENOMEM;
  2045. }
  2046. td->latency_buckets[WRITE] = __alloc_percpu(sizeof(struct latency_bucket) *
  2047. LATENCY_BUCKET_SIZE, __alignof__(u64));
  2048. if (!td->latency_buckets[WRITE]) {
  2049. free_percpu(td->latency_buckets[READ]);
  2050. kfree(td);
  2051. return -ENOMEM;
  2052. }
  2053. INIT_WORK(&td->dispatch_work, blk_throtl_dispatch_work_fn);
  2054. throtl_service_queue_init(&td->service_queue);
  2055. q->td = td;
  2056. td->queue = q;
  2057. td->limit_valid[LIMIT_MAX] = true;
  2058. td->limit_index = LIMIT_MAX;
  2059. td->low_upgrade_time = jiffies;
  2060. td->low_downgrade_time = jiffies;
  2061. /* activate policy */
  2062. ret = blkcg_activate_policy(q, &blkcg_policy_throtl);
  2063. if (ret) {
  2064. free_percpu(td->latency_buckets[READ]);
  2065. free_percpu(td->latency_buckets[WRITE]);
  2066. kfree(td);
  2067. }
  2068. return ret;
  2069. }
  2070. void blk_throtl_exit(struct request_queue *q)
  2071. {
  2072. BUG_ON(!q->td);
  2073. del_timer_sync(&q->td->service_queue.pending_timer);
  2074. throtl_shutdown_wq(q);
  2075. blkcg_deactivate_policy(q, &blkcg_policy_throtl);
  2076. free_percpu(q->td->latency_buckets[READ]);
  2077. free_percpu(q->td->latency_buckets[WRITE]);
  2078. kfree(q->td);
  2079. }
  2080. void blk_throtl_register_queue(struct request_queue *q)
  2081. {
  2082. struct throtl_data *td;
  2083. int i;
  2084. td = q->td;
  2085. BUG_ON(!td);
  2086. if (blk_queue_nonrot(q)) {
  2087. td->throtl_slice = DFL_THROTL_SLICE_SSD;
  2088. td->filtered_latency = LATENCY_FILTERED_SSD;
  2089. } else {
  2090. td->throtl_slice = DFL_THROTL_SLICE_HD;
  2091. td->filtered_latency = LATENCY_FILTERED_HD;
  2092. for (i = 0; i < LATENCY_BUCKET_SIZE; i++) {
  2093. td->avg_buckets[READ][i].latency = DFL_HD_BASELINE_LATENCY;
  2094. td->avg_buckets[WRITE][i].latency = DFL_HD_BASELINE_LATENCY;
  2095. }
  2096. }
  2097. #ifndef CONFIG_BLK_DEV_THROTTLING_LOW
  2098. /* if no low limit, use previous default */
  2099. td->throtl_slice = DFL_THROTL_SLICE_HD;
  2100. #endif
  2101. td->track_bio_latency = !queue_is_mq(q);
  2102. if (!td->track_bio_latency)
  2103. blk_stat_enable_accounting(q);
  2104. }
  2105. #ifdef CONFIG_BLK_DEV_THROTTLING_LOW
  2106. ssize_t blk_throtl_sample_time_show(struct request_queue *q, char *page)
  2107. {
  2108. if (!q->td)
  2109. return -EINVAL;
  2110. return sprintf(page, "%u\n", jiffies_to_msecs(q->td->throtl_slice));
  2111. }
  2112. ssize_t blk_throtl_sample_time_store(struct request_queue *q,
  2113. const char *page, size_t count)
  2114. {
  2115. unsigned long v;
  2116. unsigned long t;
  2117. if (!q->td)
  2118. return -EINVAL;
  2119. if (kstrtoul(page, 10, &v))
  2120. return -EINVAL;
  2121. t = msecs_to_jiffies(v);
  2122. if (t == 0 || t > MAX_THROTL_SLICE)
  2123. return -EINVAL;
  2124. q->td->throtl_slice = t;
  2125. return count;
  2126. }
  2127. #endif
  2128. static int __init throtl_init(void)
  2129. {
  2130. kthrotld_workqueue = alloc_workqueue("kthrotld", WQ_MEM_RECLAIM, 0);
  2131. if (!kthrotld_workqueue)
  2132. panic("Failed to create kthrotld\n");
  2133. return blkcg_policy_register(&blkcg_policy_throtl);
  2134. }
  2135. module_init(throtl_init);