bfq-cgroup.c 39 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * cgroups support for the BFQ I/O scheduler.
  4. */
  5. #include <linux/module.h>
  6. #include <linux/slab.h>
  7. #include <linux/blkdev.h>
  8. #include <linux/cgroup.h>
  9. #include <linux/elevator.h>
  10. #include <linux/ktime.h>
  11. #include <linux/rbtree.h>
  12. #include <linux/ioprio.h>
  13. #include <linux/sbitmap.h>
  14. #include <linux/delay.h>
  15. #include "bfq-iosched.h"
  16. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  17. static int bfq_stat_init(struct bfq_stat *stat, gfp_t gfp)
  18. {
  19. int ret;
  20. ret = percpu_counter_init(&stat->cpu_cnt, 0, gfp);
  21. if (ret)
  22. return ret;
  23. atomic64_set(&stat->aux_cnt, 0);
  24. return 0;
  25. }
  26. static void bfq_stat_exit(struct bfq_stat *stat)
  27. {
  28. percpu_counter_destroy(&stat->cpu_cnt);
  29. }
  30. /**
  31. * bfq_stat_add - add a value to a bfq_stat
  32. * @stat: target bfq_stat
  33. * @val: value to add
  34. *
  35. * Add @val to @stat. The caller must ensure that IRQ on the same CPU
  36. * don't re-enter this function for the same counter.
  37. */
  38. static inline void bfq_stat_add(struct bfq_stat *stat, uint64_t val)
  39. {
  40. percpu_counter_add_batch(&stat->cpu_cnt, val, BLKG_STAT_CPU_BATCH);
  41. }
  42. /**
  43. * bfq_stat_read - read the current value of a bfq_stat
  44. * @stat: bfq_stat to read
  45. */
  46. static inline uint64_t bfq_stat_read(struct bfq_stat *stat)
  47. {
  48. return percpu_counter_sum_positive(&stat->cpu_cnt);
  49. }
  50. /**
  51. * bfq_stat_reset - reset a bfq_stat
  52. * @stat: bfq_stat to reset
  53. */
  54. static inline void bfq_stat_reset(struct bfq_stat *stat)
  55. {
  56. percpu_counter_set(&stat->cpu_cnt, 0);
  57. atomic64_set(&stat->aux_cnt, 0);
  58. }
  59. /**
  60. * bfq_stat_add_aux - add a bfq_stat into another's aux count
  61. * @to: the destination bfq_stat
  62. * @from: the source
  63. *
  64. * Add @from's count including the aux one to @to's aux count.
  65. */
  66. static inline void bfq_stat_add_aux(struct bfq_stat *to,
  67. struct bfq_stat *from)
  68. {
  69. atomic64_add(bfq_stat_read(from) + atomic64_read(&from->aux_cnt),
  70. &to->aux_cnt);
  71. }
  72. /**
  73. * blkg_prfill_stat - prfill callback for bfq_stat
  74. * @sf: seq_file to print to
  75. * @pd: policy private data of interest
  76. * @off: offset to the bfq_stat in @pd
  77. *
  78. * prfill callback for printing a bfq_stat.
  79. */
  80. static u64 blkg_prfill_stat(struct seq_file *sf, struct blkg_policy_data *pd,
  81. int off)
  82. {
  83. return __blkg_prfill_u64(sf, pd, bfq_stat_read((void *)pd + off));
  84. }
  85. /* bfqg stats flags */
  86. enum bfqg_stats_flags {
  87. BFQG_stats_waiting = 0,
  88. BFQG_stats_idling,
  89. BFQG_stats_empty,
  90. };
  91. #define BFQG_FLAG_FNS(name) \
  92. static void bfqg_stats_mark_##name(struct bfqg_stats *stats) \
  93. { \
  94. stats->flags |= (1 << BFQG_stats_##name); \
  95. } \
  96. static void bfqg_stats_clear_##name(struct bfqg_stats *stats) \
  97. { \
  98. stats->flags &= ~(1 << BFQG_stats_##name); \
  99. } \
  100. static int bfqg_stats_##name(struct bfqg_stats *stats) \
  101. { \
  102. return (stats->flags & (1 << BFQG_stats_##name)) != 0; \
  103. } \
  104. BFQG_FLAG_FNS(waiting)
  105. BFQG_FLAG_FNS(idling)
  106. BFQG_FLAG_FNS(empty)
  107. #undef BFQG_FLAG_FNS
  108. /* This should be called with the scheduler lock held. */
  109. static void bfqg_stats_update_group_wait_time(struct bfqg_stats *stats)
  110. {
  111. u64 now;
  112. if (!bfqg_stats_waiting(stats))
  113. return;
  114. now = ktime_get_ns();
  115. if (now > stats->start_group_wait_time)
  116. bfq_stat_add(&stats->group_wait_time,
  117. now - stats->start_group_wait_time);
  118. bfqg_stats_clear_waiting(stats);
  119. }
  120. /* This should be called with the scheduler lock held. */
  121. static void bfqg_stats_set_start_group_wait_time(struct bfq_group *bfqg,
  122. struct bfq_group *curr_bfqg)
  123. {
  124. struct bfqg_stats *stats = &bfqg->stats;
  125. if (bfqg_stats_waiting(stats))
  126. return;
  127. if (bfqg == curr_bfqg)
  128. return;
  129. stats->start_group_wait_time = ktime_get_ns();
  130. bfqg_stats_mark_waiting(stats);
  131. }
  132. /* This should be called with the scheduler lock held. */
  133. static void bfqg_stats_end_empty_time(struct bfqg_stats *stats)
  134. {
  135. u64 now;
  136. if (!bfqg_stats_empty(stats))
  137. return;
  138. now = ktime_get_ns();
  139. if (now > stats->start_empty_time)
  140. bfq_stat_add(&stats->empty_time,
  141. now - stats->start_empty_time);
  142. bfqg_stats_clear_empty(stats);
  143. }
  144. void bfqg_stats_update_dequeue(struct bfq_group *bfqg)
  145. {
  146. bfq_stat_add(&bfqg->stats.dequeue, 1);
  147. }
  148. void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg)
  149. {
  150. struct bfqg_stats *stats = &bfqg->stats;
  151. if (blkg_rwstat_total(&stats->queued))
  152. return;
  153. /*
  154. * group is already marked empty. This can happen if bfqq got new
  155. * request in parent group and moved to this group while being added
  156. * to service tree. Just ignore the event and move on.
  157. */
  158. if (bfqg_stats_empty(stats))
  159. return;
  160. stats->start_empty_time = ktime_get_ns();
  161. bfqg_stats_mark_empty(stats);
  162. }
  163. void bfqg_stats_update_idle_time(struct bfq_group *bfqg)
  164. {
  165. struct bfqg_stats *stats = &bfqg->stats;
  166. if (bfqg_stats_idling(stats)) {
  167. u64 now = ktime_get_ns();
  168. if (now > stats->start_idle_time)
  169. bfq_stat_add(&stats->idle_time,
  170. now - stats->start_idle_time);
  171. bfqg_stats_clear_idling(stats);
  172. }
  173. }
  174. void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg)
  175. {
  176. struct bfqg_stats *stats = &bfqg->stats;
  177. stats->start_idle_time = ktime_get_ns();
  178. bfqg_stats_mark_idling(stats);
  179. }
  180. void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg)
  181. {
  182. struct bfqg_stats *stats = &bfqg->stats;
  183. bfq_stat_add(&stats->avg_queue_size_sum,
  184. blkg_rwstat_total(&stats->queued));
  185. bfq_stat_add(&stats->avg_queue_size_samples, 1);
  186. bfqg_stats_update_group_wait_time(stats);
  187. }
  188. void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
  189. unsigned int op)
  190. {
  191. blkg_rwstat_add(&bfqg->stats.queued, op, 1);
  192. bfqg_stats_end_empty_time(&bfqg->stats);
  193. if (!(bfqq == ((struct bfq_data *)bfqg->bfqd)->in_service_queue))
  194. bfqg_stats_set_start_group_wait_time(bfqg, bfqq_group(bfqq));
  195. }
  196. void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op)
  197. {
  198. blkg_rwstat_add(&bfqg->stats.queued, op, -1);
  199. }
  200. void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op)
  201. {
  202. blkg_rwstat_add(&bfqg->stats.merged, op, 1);
  203. }
  204. void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
  205. u64 io_start_time_ns, unsigned int op)
  206. {
  207. struct bfqg_stats *stats = &bfqg->stats;
  208. u64 now = ktime_get_ns();
  209. if (now > io_start_time_ns)
  210. blkg_rwstat_add(&stats->service_time, op,
  211. now - io_start_time_ns);
  212. if (io_start_time_ns > start_time_ns)
  213. blkg_rwstat_add(&stats->wait_time, op,
  214. io_start_time_ns - start_time_ns);
  215. }
  216. #else /* CONFIG_BFQ_CGROUP_DEBUG */
  217. void bfqg_stats_update_io_add(struct bfq_group *bfqg, struct bfq_queue *bfqq,
  218. unsigned int op) { }
  219. void bfqg_stats_update_io_remove(struct bfq_group *bfqg, unsigned int op) { }
  220. void bfqg_stats_update_io_merged(struct bfq_group *bfqg, unsigned int op) { }
  221. void bfqg_stats_update_completion(struct bfq_group *bfqg, u64 start_time_ns,
  222. u64 io_start_time_ns, unsigned int op) { }
  223. void bfqg_stats_update_dequeue(struct bfq_group *bfqg) { }
  224. void bfqg_stats_set_start_empty_time(struct bfq_group *bfqg) { }
  225. void bfqg_stats_update_idle_time(struct bfq_group *bfqg) { }
  226. void bfqg_stats_set_start_idle_time(struct bfq_group *bfqg) { }
  227. void bfqg_stats_update_avg_queue_size(struct bfq_group *bfqg) { }
  228. #endif /* CONFIG_BFQ_CGROUP_DEBUG */
  229. #ifdef CONFIG_BFQ_GROUP_IOSCHED
  230. /*
  231. * blk-cgroup policy-related handlers
  232. * The following functions help in converting between blk-cgroup
  233. * internal structures and BFQ-specific structures.
  234. */
  235. static struct bfq_group *pd_to_bfqg(struct blkg_policy_data *pd)
  236. {
  237. return pd ? container_of(pd, struct bfq_group, pd) : NULL;
  238. }
  239. struct blkcg_gq *bfqg_to_blkg(struct bfq_group *bfqg)
  240. {
  241. return pd_to_blkg(&bfqg->pd);
  242. }
  243. static struct bfq_group *blkg_to_bfqg(struct blkcg_gq *blkg)
  244. {
  245. return pd_to_bfqg(blkg_to_pd(blkg, &blkcg_policy_bfq));
  246. }
  247. /*
  248. * bfq_group handlers
  249. * The following functions help in navigating the bfq_group hierarchy
  250. * by allowing to find the parent of a bfq_group or the bfq_group
  251. * associated to a bfq_queue.
  252. */
  253. static struct bfq_group *bfqg_parent(struct bfq_group *bfqg)
  254. {
  255. struct blkcg_gq *pblkg = bfqg_to_blkg(bfqg)->parent;
  256. return pblkg ? blkg_to_bfqg(pblkg) : NULL;
  257. }
  258. struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
  259. {
  260. struct bfq_entity *group_entity = bfqq->entity.parent;
  261. return group_entity ? container_of(group_entity, struct bfq_group,
  262. entity) :
  263. bfqq->bfqd->root_group;
  264. }
  265. /*
  266. * The following two functions handle get and put of a bfq_group by
  267. * wrapping the related blk-cgroup hooks.
  268. */
  269. static void bfqg_get(struct bfq_group *bfqg)
  270. {
  271. bfqg->ref++;
  272. }
  273. static void bfqg_put(struct bfq_group *bfqg)
  274. {
  275. bfqg->ref--;
  276. if (bfqg->ref == 0)
  277. kfree(bfqg);
  278. }
  279. static void bfqg_and_blkg_get(struct bfq_group *bfqg)
  280. {
  281. /* see comments in bfq_bic_update_cgroup for why refcounting bfqg */
  282. bfqg_get(bfqg);
  283. blkg_get(bfqg_to_blkg(bfqg));
  284. }
  285. void bfqg_and_blkg_put(struct bfq_group *bfqg)
  286. {
  287. blkg_put(bfqg_to_blkg(bfqg));
  288. bfqg_put(bfqg);
  289. }
  290. void bfqg_stats_update_legacy_io(struct request_queue *q, struct request *rq)
  291. {
  292. struct bfq_group *bfqg = blkg_to_bfqg(rq->bio->bi_blkg);
  293. if (!bfqg)
  294. return;
  295. blkg_rwstat_add(&bfqg->stats.bytes, rq->cmd_flags, blk_rq_bytes(rq));
  296. blkg_rwstat_add(&bfqg->stats.ios, rq->cmd_flags, 1);
  297. }
  298. /* @stats = 0 */
  299. static void bfqg_stats_reset(struct bfqg_stats *stats)
  300. {
  301. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  302. /* queued stats shouldn't be cleared */
  303. blkg_rwstat_reset(&stats->merged);
  304. blkg_rwstat_reset(&stats->service_time);
  305. blkg_rwstat_reset(&stats->wait_time);
  306. bfq_stat_reset(&stats->time);
  307. bfq_stat_reset(&stats->avg_queue_size_sum);
  308. bfq_stat_reset(&stats->avg_queue_size_samples);
  309. bfq_stat_reset(&stats->dequeue);
  310. bfq_stat_reset(&stats->group_wait_time);
  311. bfq_stat_reset(&stats->idle_time);
  312. bfq_stat_reset(&stats->empty_time);
  313. #endif
  314. }
  315. /* @to += @from */
  316. static void bfqg_stats_add_aux(struct bfqg_stats *to, struct bfqg_stats *from)
  317. {
  318. if (!to || !from)
  319. return;
  320. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  321. /* queued stats shouldn't be cleared */
  322. blkg_rwstat_add_aux(&to->merged, &from->merged);
  323. blkg_rwstat_add_aux(&to->service_time, &from->service_time);
  324. blkg_rwstat_add_aux(&to->wait_time, &from->wait_time);
  325. bfq_stat_add_aux(&from->time, &from->time);
  326. bfq_stat_add_aux(&to->avg_queue_size_sum, &from->avg_queue_size_sum);
  327. bfq_stat_add_aux(&to->avg_queue_size_samples,
  328. &from->avg_queue_size_samples);
  329. bfq_stat_add_aux(&to->dequeue, &from->dequeue);
  330. bfq_stat_add_aux(&to->group_wait_time, &from->group_wait_time);
  331. bfq_stat_add_aux(&to->idle_time, &from->idle_time);
  332. bfq_stat_add_aux(&to->empty_time, &from->empty_time);
  333. #endif
  334. }
  335. /*
  336. * Transfer @bfqg's stats to its parent's aux counts so that the ancestors'
  337. * recursive stats can still account for the amount used by this bfqg after
  338. * it's gone.
  339. */
  340. static void bfqg_stats_xfer_dead(struct bfq_group *bfqg)
  341. {
  342. struct bfq_group *parent;
  343. if (!bfqg) /* root_group */
  344. return;
  345. parent = bfqg_parent(bfqg);
  346. lockdep_assert_held(&bfqg_to_blkg(bfqg)->q->queue_lock);
  347. if (unlikely(!parent))
  348. return;
  349. bfqg_stats_add_aux(&parent->stats, &bfqg->stats);
  350. bfqg_stats_reset(&bfqg->stats);
  351. }
  352. void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
  353. {
  354. struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
  355. entity->weight = entity->new_weight;
  356. entity->orig_weight = entity->new_weight;
  357. if (bfqq) {
  358. bfqq->ioprio = bfqq->new_ioprio;
  359. bfqq->ioprio_class = bfqq->new_ioprio_class;
  360. /*
  361. * Make sure that bfqg and its associated blkg do not
  362. * disappear before entity.
  363. */
  364. bfqg_and_blkg_get(bfqg);
  365. }
  366. entity->parent = bfqg->my_entity; /* NULL for root group */
  367. entity->sched_data = &bfqg->sched_data;
  368. }
  369. static void bfqg_stats_exit(struct bfqg_stats *stats)
  370. {
  371. blkg_rwstat_exit(&stats->bytes);
  372. blkg_rwstat_exit(&stats->ios);
  373. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  374. blkg_rwstat_exit(&stats->merged);
  375. blkg_rwstat_exit(&stats->service_time);
  376. blkg_rwstat_exit(&stats->wait_time);
  377. blkg_rwstat_exit(&stats->queued);
  378. bfq_stat_exit(&stats->time);
  379. bfq_stat_exit(&stats->avg_queue_size_sum);
  380. bfq_stat_exit(&stats->avg_queue_size_samples);
  381. bfq_stat_exit(&stats->dequeue);
  382. bfq_stat_exit(&stats->group_wait_time);
  383. bfq_stat_exit(&stats->idle_time);
  384. bfq_stat_exit(&stats->empty_time);
  385. #endif
  386. }
  387. static int bfqg_stats_init(struct bfqg_stats *stats, gfp_t gfp)
  388. {
  389. if (blkg_rwstat_init(&stats->bytes, gfp) ||
  390. blkg_rwstat_init(&stats->ios, gfp))
  391. return -ENOMEM;
  392. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  393. if (blkg_rwstat_init(&stats->merged, gfp) ||
  394. blkg_rwstat_init(&stats->service_time, gfp) ||
  395. blkg_rwstat_init(&stats->wait_time, gfp) ||
  396. blkg_rwstat_init(&stats->queued, gfp) ||
  397. bfq_stat_init(&stats->time, gfp) ||
  398. bfq_stat_init(&stats->avg_queue_size_sum, gfp) ||
  399. bfq_stat_init(&stats->avg_queue_size_samples, gfp) ||
  400. bfq_stat_init(&stats->dequeue, gfp) ||
  401. bfq_stat_init(&stats->group_wait_time, gfp) ||
  402. bfq_stat_init(&stats->idle_time, gfp) ||
  403. bfq_stat_init(&stats->empty_time, gfp)) {
  404. bfqg_stats_exit(stats);
  405. return -ENOMEM;
  406. }
  407. #endif
  408. return 0;
  409. }
  410. static struct bfq_group_data *cpd_to_bfqgd(struct blkcg_policy_data *cpd)
  411. {
  412. return cpd ? container_of(cpd, struct bfq_group_data, pd) : NULL;
  413. }
  414. static struct bfq_group_data *blkcg_to_bfqgd(struct blkcg *blkcg)
  415. {
  416. return cpd_to_bfqgd(blkcg_to_cpd(blkcg, &blkcg_policy_bfq));
  417. }
  418. static struct blkcg_policy_data *bfq_cpd_alloc(gfp_t gfp)
  419. {
  420. struct bfq_group_data *bgd;
  421. bgd = kzalloc(sizeof(*bgd), gfp);
  422. if (!bgd)
  423. return NULL;
  424. return &bgd->pd;
  425. }
  426. static void bfq_cpd_init(struct blkcg_policy_data *cpd)
  427. {
  428. struct bfq_group_data *d = cpd_to_bfqgd(cpd);
  429. d->weight = cgroup_subsys_on_dfl(io_cgrp_subsys) ?
  430. CGROUP_WEIGHT_DFL : BFQ_WEIGHT_LEGACY_DFL;
  431. }
  432. static void bfq_cpd_free(struct blkcg_policy_data *cpd)
  433. {
  434. kfree(cpd_to_bfqgd(cpd));
  435. }
  436. static struct blkg_policy_data *bfq_pd_alloc(gfp_t gfp, struct request_queue *q,
  437. struct blkcg *blkcg)
  438. {
  439. struct bfq_group *bfqg;
  440. bfqg = kzalloc_node(sizeof(*bfqg), gfp, q->node);
  441. if (!bfqg)
  442. return NULL;
  443. if (bfqg_stats_init(&bfqg->stats, gfp)) {
  444. kfree(bfqg);
  445. return NULL;
  446. }
  447. /* see comments in bfq_bic_update_cgroup for why refcounting */
  448. bfqg_get(bfqg);
  449. return &bfqg->pd;
  450. }
  451. static void bfq_pd_init(struct blkg_policy_data *pd)
  452. {
  453. struct blkcg_gq *blkg = pd_to_blkg(pd);
  454. struct bfq_group *bfqg = blkg_to_bfqg(blkg);
  455. struct bfq_data *bfqd = blkg->q->elevator->elevator_data;
  456. struct bfq_entity *entity = &bfqg->entity;
  457. struct bfq_group_data *d = blkcg_to_bfqgd(blkg->blkcg);
  458. entity->orig_weight = entity->weight = entity->new_weight = d->weight;
  459. entity->my_sched_data = &bfqg->sched_data;
  460. bfqg->my_entity = entity; /*
  461. * the root_group's will be set to NULL
  462. * in bfq_init_queue()
  463. */
  464. bfqg->bfqd = bfqd;
  465. bfqg->active_entities = 0;
  466. bfqg->rq_pos_tree = RB_ROOT;
  467. }
  468. static void bfq_pd_free(struct blkg_policy_data *pd)
  469. {
  470. struct bfq_group *bfqg = pd_to_bfqg(pd);
  471. bfqg_stats_exit(&bfqg->stats);
  472. bfqg_put(bfqg);
  473. }
  474. static void bfq_pd_reset_stats(struct blkg_policy_data *pd)
  475. {
  476. struct bfq_group *bfqg = pd_to_bfqg(pd);
  477. bfqg_stats_reset(&bfqg->stats);
  478. }
  479. static void bfq_group_set_parent(struct bfq_group *bfqg,
  480. struct bfq_group *parent)
  481. {
  482. struct bfq_entity *entity;
  483. entity = &bfqg->entity;
  484. entity->parent = parent->my_entity;
  485. entity->sched_data = &parent->sched_data;
  486. }
  487. static struct bfq_group *bfq_lookup_bfqg(struct bfq_data *bfqd,
  488. struct blkcg *blkcg)
  489. {
  490. struct blkcg_gq *blkg;
  491. blkg = blkg_lookup(blkcg, bfqd->queue);
  492. if (likely(blkg))
  493. return blkg_to_bfqg(blkg);
  494. return NULL;
  495. }
  496. struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd,
  497. struct blkcg *blkcg)
  498. {
  499. struct bfq_group *bfqg, *parent;
  500. struct bfq_entity *entity;
  501. bfqg = bfq_lookup_bfqg(bfqd, blkcg);
  502. if (unlikely(!bfqg))
  503. return NULL;
  504. /*
  505. * Update chain of bfq_groups as we might be handling a leaf group
  506. * which, along with some of its relatives, has not been hooked yet
  507. * to the private hierarchy of BFQ.
  508. */
  509. entity = &bfqg->entity;
  510. for_each_entity(entity) {
  511. struct bfq_group *curr_bfqg = container_of(entity,
  512. struct bfq_group, entity);
  513. if (curr_bfqg != bfqd->root_group) {
  514. parent = bfqg_parent(curr_bfqg);
  515. if (!parent)
  516. parent = bfqd->root_group;
  517. bfq_group_set_parent(curr_bfqg, parent);
  518. }
  519. }
  520. return bfqg;
  521. }
  522. /**
  523. * bfq_bfqq_move - migrate @bfqq to @bfqg.
  524. * @bfqd: queue descriptor.
  525. * @bfqq: the queue to move.
  526. * @bfqg: the group to move to.
  527. *
  528. * Move @bfqq to @bfqg, deactivating it from its old group and reactivating
  529. * it on the new one. Avoid putting the entity on the old group idle tree.
  530. *
  531. * Must be called under the scheduler lock, to make sure that the blkg
  532. * owning @bfqg does not disappear (see comments in
  533. * bfq_bic_update_cgroup on guaranteeing the consistency of blkg
  534. * objects).
  535. */
  536. void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  537. struct bfq_group *bfqg)
  538. {
  539. struct bfq_entity *entity = &bfqq->entity;
  540. /*
  541. * oom_bfqq is not allowed to move, oom_bfqq will hold ref to root_group
  542. * until elevator exit.
  543. */
  544. if (bfqq == &bfqd->oom_bfqq)
  545. return;
  546. /*
  547. * Get extra reference to prevent bfqq from being freed in
  548. * next possible expire or deactivate.
  549. */
  550. bfqq->ref++;
  551. /* If bfqq is empty, then bfq_bfqq_expire also invokes
  552. * bfq_del_bfqq_busy, thereby removing bfqq and its entity
  553. * from data structures related to current group. Otherwise we
  554. * need to remove bfqq explicitly with bfq_deactivate_bfqq, as
  555. * we do below.
  556. */
  557. if (bfqq == bfqd->in_service_queue)
  558. bfq_bfqq_expire(bfqd, bfqd->in_service_queue,
  559. false, BFQQE_PREEMPTED);
  560. if (bfq_bfqq_busy(bfqq))
  561. bfq_deactivate_bfqq(bfqd, bfqq, false, false);
  562. else if (entity->on_st_or_in_serv)
  563. bfq_put_idle_entity(bfq_entity_service_tree(entity), entity);
  564. bfqg_and_blkg_put(bfqq_group(bfqq));
  565. entity->parent = bfqg->my_entity;
  566. entity->sched_data = &bfqg->sched_data;
  567. /* pin down bfqg and its associated blkg */
  568. bfqg_and_blkg_get(bfqg);
  569. if (bfq_bfqq_busy(bfqq)) {
  570. if (unlikely(!bfqd->nonrot_with_queueing))
  571. bfq_pos_tree_add_move(bfqd, bfqq);
  572. bfq_activate_bfqq(bfqd, bfqq);
  573. }
  574. if (!bfqd->in_service_queue && !bfqd->rq_in_driver)
  575. bfq_schedule_dispatch(bfqd);
  576. /* release extra ref taken above, bfqq may happen to be freed now */
  577. bfq_put_queue(bfqq);
  578. }
  579. /**
  580. * __bfq_bic_change_cgroup - move @bic to @cgroup.
  581. * @bfqd: the queue descriptor.
  582. * @bic: the bic to move.
  583. * @blkcg: the blk-cgroup to move to.
  584. *
  585. * Move bic to blkcg, assuming that bfqd->lock is held; which makes
  586. * sure that the reference to cgroup is valid across the call (see
  587. * comments in bfq_bic_update_cgroup on this issue)
  588. *
  589. * NOTE: an alternative approach might have been to store the current
  590. * cgroup in bfqq and getting a reference to it, reducing the lookup
  591. * time here, at the price of slightly more complex code.
  592. */
  593. static struct bfq_group *__bfq_bic_change_cgroup(struct bfq_data *bfqd,
  594. struct bfq_io_cq *bic,
  595. struct blkcg *blkcg)
  596. {
  597. struct bfq_queue *async_bfqq = bic_to_bfqq(bic, 0);
  598. struct bfq_queue *sync_bfqq = bic_to_bfqq(bic, 1);
  599. struct bfq_group *bfqg;
  600. struct bfq_entity *entity;
  601. bfqg = bfq_find_set_group(bfqd, blkcg);
  602. if (unlikely(!bfqg))
  603. bfqg = bfqd->root_group;
  604. if (async_bfqq) {
  605. entity = &async_bfqq->entity;
  606. if (entity->sched_data != &bfqg->sched_data) {
  607. bic_set_bfqq(bic, NULL, 0);
  608. bfq_release_process_ref(bfqd, async_bfqq);
  609. }
  610. }
  611. if (sync_bfqq) {
  612. entity = &sync_bfqq->entity;
  613. if (entity->sched_data != &bfqg->sched_data)
  614. bfq_bfqq_move(bfqd, sync_bfqq, bfqg);
  615. }
  616. return bfqg;
  617. }
  618. void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio)
  619. {
  620. struct bfq_data *bfqd = bic_to_bfqd(bic);
  621. struct bfq_group *bfqg = NULL;
  622. uint64_t serial_nr;
  623. rcu_read_lock();
  624. serial_nr = __bio_blkcg(bio)->css.serial_nr;
  625. /*
  626. * Check whether blkcg has changed. The condition may trigger
  627. * spuriously on a newly created cic but there's no harm.
  628. */
  629. if (unlikely(!bfqd) || likely(bic->blkcg_serial_nr == serial_nr))
  630. goto out;
  631. bfqg = __bfq_bic_change_cgroup(bfqd, bic, __bio_blkcg(bio));
  632. /*
  633. * Update blkg_path for bfq_log_* functions. We cache this
  634. * path, and update it here, for the following
  635. * reasons. Operations on blkg objects in blk-cgroup are
  636. * protected with the request_queue lock, and not with the
  637. * lock that protects the instances of this scheduler
  638. * (bfqd->lock). This exposes BFQ to the following sort of
  639. * race.
  640. *
  641. * The blkg_lookup performed in bfq_get_queue, protected
  642. * through rcu, may happen to return the address of a copy of
  643. * the original blkg. If this is the case, then the
  644. * bfqg_and_blkg_get performed in bfq_get_queue, to pin down
  645. * the blkg, is useless: it does not prevent blk-cgroup code
  646. * from destroying both the original blkg and all objects
  647. * directly or indirectly referred by the copy of the
  648. * blkg.
  649. *
  650. * On the bright side, destroy operations on a blkg invoke, as
  651. * a first step, hooks of the scheduler associated with the
  652. * blkg. And these hooks are executed with bfqd->lock held for
  653. * BFQ. As a consequence, for any blkg associated with the
  654. * request queue this instance of the scheduler is attached
  655. * to, we are guaranteed that such a blkg is not destroyed, and
  656. * that all the pointers it contains are consistent, while we
  657. * are holding bfqd->lock. A blkg_lookup performed with
  658. * bfqd->lock held then returns a fully consistent blkg, which
  659. * remains consistent until this lock is held.
  660. *
  661. * Thanks to the last fact, and to the fact that: (1) bfqg has
  662. * been obtained through a blkg_lookup in the above
  663. * assignment, and (2) bfqd->lock is being held, here we can
  664. * safely use the policy data for the involved blkg (i.e., the
  665. * field bfqg->pd) to get to the blkg associated with bfqg,
  666. * and then we can safely use any field of blkg. After we
  667. * release bfqd->lock, even just getting blkg through this
  668. * bfqg may cause dangling references to be traversed, as
  669. * bfqg->pd may not exist any more.
  670. *
  671. * In view of the above facts, here we cache, in the bfqg, any
  672. * blkg data we may need for this bic, and for its associated
  673. * bfq_queue. As of now, we need to cache only the path of the
  674. * blkg, which is used in the bfq_log_* functions.
  675. *
  676. * Finally, note that bfqg itself needs to be protected from
  677. * destruction on the blkg_free of the original blkg (which
  678. * invokes bfq_pd_free). We use an additional private
  679. * refcounter for bfqg, to let it disappear only after no
  680. * bfq_queue refers to it any longer.
  681. */
  682. blkg_path(bfqg_to_blkg(bfqg), bfqg->blkg_path, sizeof(bfqg->blkg_path));
  683. bic->blkcg_serial_nr = serial_nr;
  684. out:
  685. rcu_read_unlock();
  686. }
  687. /**
  688. * bfq_flush_idle_tree - deactivate any entity on the idle tree of @st.
  689. * @st: the service tree being flushed.
  690. */
  691. static void bfq_flush_idle_tree(struct bfq_service_tree *st)
  692. {
  693. struct bfq_entity *entity = st->first_idle;
  694. for (; entity ; entity = st->first_idle)
  695. __bfq_deactivate_entity(entity, false);
  696. }
  697. /**
  698. * bfq_reparent_leaf_entity - move leaf entity to the root_group.
  699. * @bfqd: the device data structure with the root group.
  700. * @entity: the entity to move, if entity is a leaf; or the parent entity
  701. * of an active leaf entity to move, if entity is not a leaf.
  702. */
  703. static void bfq_reparent_leaf_entity(struct bfq_data *bfqd,
  704. struct bfq_entity *entity,
  705. int ioprio_class)
  706. {
  707. struct bfq_queue *bfqq;
  708. struct bfq_entity *child_entity = entity;
  709. while (child_entity->my_sched_data) { /* leaf not reached yet */
  710. struct bfq_sched_data *child_sd = child_entity->my_sched_data;
  711. struct bfq_service_tree *child_st = child_sd->service_tree +
  712. ioprio_class;
  713. struct rb_root *child_active = &child_st->active;
  714. child_entity = bfq_entity_of(rb_first(child_active));
  715. if (!child_entity)
  716. child_entity = child_sd->in_service_entity;
  717. }
  718. bfqq = bfq_entity_to_bfqq(child_entity);
  719. bfq_bfqq_move(bfqd, bfqq, bfqd->root_group);
  720. }
  721. /**
  722. * bfq_reparent_active_queues - move to the root group all active queues.
  723. * @bfqd: the device data structure with the root group.
  724. * @bfqg: the group to move from.
  725. * @st: the service tree to start the search from.
  726. */
  727. static void bfq_reparent_active_queues(struct bfq_data *bfqd,
  728. struct bfq_group *bfqg,
  729. struct bfq_service_tree *st,
  730. int ioprio_class)
  731. {
  732. struct rb_root *active = &st->active;
  733. struct bfq_entity *entity;
  734. while ((entity = bfq_entity_of(rb_first(active))))
  735. bfq_reparent_leaf_entity(bfqd, entity, ioprio_class);
  736. if (bfqg->sched_data.in_service_entity)
  737. bfq_reparent_leaf_entity(bfqd,
  738. bfqg->sched_data.in_service_entity,
  739. ioprio_class);
  740. }
  741. /**
  742. * bfq_pd_offline - deactivate the entity associated with @pd,
  743. * and reparent its children entities.
  744. * @pd: descriptor of the policy going offline.
  745. *
  746. * blkio already grabs the queue_lock for us, so no need to use
  747. * RCU-based magic
  748. */
  749. static void bfq_pd_offline(struct blkg_policy_data *pd)
  750. {
  751. struct bfq_service_tree *st;
  752. struct bfq_group *bfqg = pd_to_bfqg(pd);
  753. struct bfq_data *bfqd = bfqg->bfqd;
  754. struct bfq_entity *entity = bfqg->my_entity;
  755. unsigned long flags;
  756. int i;
  757. spin_lock_irqsave(&bfqd->lock, flags);
  758. if (!entity) /* root group */
  759. goto put_async_queues;
  760. /*
  761. * Empty all service_trees belonging to this group before
  762. * deactivating the group itself.
  763. */
  764. for (i = 0; i < BFQ_IOPRIO_CLASSES; i++) {
  765. st = bfqg->sched_data.service_tree + i;
  766. /*
  767. * It may happen that some queues are still active
  768. * (busy) upon group destruction (if the corresponding
  769. * processes have been forced to terminate). We move
  770. * all the leaf entities corresponding to these queues
  771. * to the root_group.
  772. * Also, it may happen that the group has an entity
  773. * in service, which is disconnected from the active
  774. * tree: it must be moved, too.
  775. * There is no need to put the sync queues, as the
  776. * scheduler has taken no reference.
  777. */
  778. bfq_reparent_active_queues(bfqd, bfqg, st, i);
  779. /*
  780. * The idle tree may still contain bfq_queues
  781. * belonging to exited task because they never
  782. * migrated to a different cgroup from the one being
  783. * destroyed now. In addition, even
  784. * bfq_reparent_active_queues() may happen to add some
  785. * entities to the idle tree. It happens if, in some
  786. * of the calls to bfq_bfqq_move() performed by
  787. * bfq_reparent_active_queues(), the queue to move is
  788. * empty and gets expired.
  789. */
  790. bfq_flush_idle_tree(st);
  791. }
  792. __bfq_deactivate_entity(entity, false);
  793. put_async_queues:
  794. bfq_put_async_queues(bfqd, bfqg);
  795. spin_unlock_irqrestore(&bfqd->lock, flags);
  796. /*
  797. * @blkg is going offline and will be ignored by
  798. * blkg_[rw]stat_recursive_sum(). Transfer stats to the parent so
  799. * that they don't get lost. If IOs complete after this point, the
  800. * stats for them will be lost. Oh well...
  801. */
  802. bfqg_stats_xfer_dead(bfqg);
  803. }
  804. void bfq_end_wr_async(struct bfq_data *bfqd)
  805. {
  806. struct blkcg_gq *blkg;
  807. list_for_each_entry(blkg, &bfqd->queue->blkg_list, q_node) {
  808. struct bfq_group *bfqg = blkg_to_bfqg(blkg);
  809. bfq_end_wr_async_queues(bfqd, bfqg);
  810. }
  811. bfq_end_wr_async_queues(bfqd, bfqd->root_group);
  812. }
  813. static int bfq_io_show_weight_legacy(struct seq_file *sf, void *v)
  814. {
  815. struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
  816. struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
  817. unsigned int val = 0;
  818. if (bfqgd)
  819. val = bfqgd->weight;
  820. seq_printf(sf, "%u\n", val);
  821. return 0;
  822. }
  823. static u64 bfqg_prfill_weight_device(struct seq_file *sf,
  824. struct blkg_policy_data *pd, int off)
  825. {
  826. struct bfq_group *bfqg = pd_to_bfqg(pd);
  827. if (!bfqg->entity.dev_weight)
  828. return 0;
  829. return __blkg_prfill_u64(sf, pd, bfqg->entity.dev_weight);
  830. }
  831. static int bfq_io_show_weight(struct seq_file *sf, void *v)
  832. {
  833. struct blkcg *blkcg = css_to_blkcg(seq_css(sf));
  834. struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
  835. seq_printf(sf, "default %u\n", bfqgd->weight);
  836. blkcg_print_blkgs(sf, blkcg, bfqg_prfill_weight_device,
  837. &blkcg_policy_bfq, 0, false);
  838. return 0;
  839. }
  840. static void bfq_group_set_weight(struct bfq_group *bfqg, u64 weight, u64 dev_weight)
  841. {
  842. weight = dev_weight ?: weight;
  843. bfqg->entity.dev_weight = dev_weight;
  844. /*
  845. * Setting the prio_changed flag of the entity
  846. * to 1 with new_weight == weight would re-set
  847. * the value of the weight to its ioprio mapping.
  848. * Set the flag only if necessary.
  849. */
  850. if ((unsigned short)weight != bfqg->entity.new_weight) {
  851. bfqg->entity.new_weight = (unsigned short)weight;
  852. /*
  853. * Make sure that the above new value has been
  854. * stored in bfqg->entity.new_weight before
  855. * setting the prio_changed flag. In fact,
  856. * this flag may be read asynchronously (in
  857. * critical sections protected by a different
  858. * lock than that held here), and finding this
  859. * flag set may cause the execution of the code
  860. * for updating parameters whose value may
  861. * depend also on bfqg->entity.new_weight (in
  862. * __bfq_entity_update_weight_prio).
  863. * This barrier makes sure that the new value
  864. * of bfqg->entity.new_weight is correctly
  865. * seen in that code.
  866. */
  867. smp_wmb();
  868. bfqg->entity.prio_changed = 1;
  869. }
  870. }
  871. static int bfq_io_set_weight_legacy(struct cgroup_subsys_state *css,
  872. struct cftype *cftype,
  873. u64 val)
  874. {
  875. struct blkcg *blkcg = css_to_blkcg(css);
  876. struct bfq_group_data *bfqgd = blkcg_to_bfqgd(blkcg);
  877. struct blkcg_gq *blkg;
  878. int ret = -ERANGE;
  879. if (val < BFQ_MIN_WEIGHT || val > BFQ_MAX_WEIGHT)
  880. return ret;
  881. ret = 0;
  882. spin_lock_irq(&blkcg->lock);
  883. bfqgd->weight = (unsigned short)val;
  884. hlist_for_each_entry(blkg, &blkcg->blkg_list, blkcg_node) {
  885. struct bfq_group *bfqg = blkg_to_bfqg(blkg);
  886. if (bfqg)
  887. bfq_group_set_weight(bfqg, val, 0);
  888. }
  889. spin_unlock_irq(&blkcg->lock);
  890. return ret;
  891. }
  892. static ssize_t bfq_io_set_device_weight(struct kernfs_open_file *of,
  893. char *buf, size_t nbytes,
  894. loff_t off)
  895. {
  896. int ret;
  897. struct blkg_conf_ctx ctx;
  898. struct blkcg *blkcg = css_to_blkcg(of_css(of));
  899. struct bfq_group *bfqg;
  900. u64 v;
  901. ret = blkg_conf_prep(blkcg, &blkcg_policy_bfq, buf, &ctx);
  902. if (ret)
  903. return ret;
  904. if (sscanf(ctx.body, "%llu", &v) == 1) {
  905. /* require "default" on dfl */
  906. ret = -ERANGE;
  907. if (!v)
  908. goto out;
  909. } else if (!strcmp(strim(ctx.body), "default")) {
  910. v = 0;
  911. } else {
  912. ret = -EINVAL;
  913. goto out;
  914. }
  915. bfqg = blkg_to_bfqg(ctx.blkg);
  916. ret = -ERANGE;
  917. if (!v || (v >= BFQ_MIN_WEIGHT && v <= BFQ_MAX_WEIGHT)) {
  918. bfq_group_set_weight(bfqg, bfqg->entity.weight, v);
  919. ret = 0;
  920. }
  921. out:
  922. blkg_conf_finish(&ctx);
  923. return ret ?: nbytes;
  924. }
  925. static ssize_t bfq_io_set_weight(struct kernfs_open_file *of,
  926. char *buf, size_t nbytes,
  927. loff_t off)
  928. {
  929. char *endp;
  930. int ret;
  931. u64 v;
  932. buf = strim(buf);
  933. /* "WEIGHT" or "default WEIGHT" sets the default weight */
  934. v = simple_strtoull(buf, &endp, 0);
  935. if (*endp == '\0' || sscanf(buf, "default %llu", &v) == 1) {
  936. ret = bfq_io_set_weight_legacy(of_css(of), NULL, v);
  937. return ret ?: nbytes;
  938. }
  939. return bfq_io_set_device_weight(of, buf, nbytes, off);
  940. }
  941. static int bfqg_print_rwstat(struct seq_file *sf, void *v)
  942. {
  943. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_rwstat,
  944. &blkcg_policy_bfq, seq_cft(sf)->private, true);
  945. return 0;
  946. }
  947. static u64 bfqg_prfill_rwstat_recursive(struct seq_file *sf,
  948. struct blkg_policy_data *pd, int off)
  949. {
  950. struct blkg_rwstat_sample sum;
  951. blkg_rwstat_recursive_sum(pd_to_blkg(pd), &blkcg_policy_bfq, off, &sum);
  952. return __blkg_prfill_rwstat(sf, pd, &sum);
  953. }
  954. static int bfqg_print_rwstat_recursive(struct seq_file *sf, void *v)
  955. {
  956. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  957. bfqg_prfill_rwstat_recursive, &blkcg_policy_bfq,
  958. seq_cft(sf)->private, true);
  959. return 0;
  960. }
  961. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  962. static int bfqg_print_stat(struct seq_file *sf, void *v)
  963. {
  964. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)), blkg_prfill_stat,
  965. &blkcg_policy_bfq, seq_cft(sf)->private, false);
  966. return 0;
  967. }
  968. static u64 bfqg_prfill_stat_recursive(struct seq_file *sf,
  969. struct blkg_policy_data *pd, int off)
  970. {
  971. struct blkcg_gq *blkg = pd_to_blkg(pd);
  972. struct blkcg_gq *pos_blkg;
  973. struct cgroup_subsys_state *pos_css;
  974. u64 sum = 0;
  975. lockdep_assert_held(&blkg->q->queue_lock);
  976. rcu_read_lock();
  977. blkg_for_each_descendant_pre(pos_blkg, pos_css, blkg) {
  978. struct bfq_stat *stat;
  979. if (!pos_blkg->online)
  980. continue;
  981. stat = (void *)blkg_to_pd(pos_blkg, &blkcg_policy_bfq) + off;
  982. sum += bfq_stat_read(stat) + atomic64_read(&stat->aux_cnt);
  983. }
  984. rcu_read_unlock();
  985. return __blkg_prfill_u64(sf, pd, sum);
  986. }
  987. static int bfqg_print_stat_recursive(struct seq_file *sf, void *v)
  988. {
  989. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  990. bfqg_prfill_stat_recursive, &blkcg_policy_bfq,
  991. seq_cft(sf)->private, false);
  992. return 0;
  993. }
  994. static u64 bfqg_prfill_sectors(struct seq_file *sf, struct blkg_policy_data *pd,
  995. int off)
  996. {
  997. struct bfq_group *bfqg = blkg_to_bfqg(pd->blkg);
  998. u64 sum = blkg_rwstat_total(&bfqg->stats.bytes);
  999. return __blkg_prfill_u64(sf, pd, sum >> 9);
  1000. }
  1001. static int bfqg_print_stat_sectors(struct seq_file *sf, void *v)
  1002. {
  1003. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  1004. bfqg_prfill_sectors, &blkcg_policy_bfq, 0, false);
  1005. return 0;
  1006. }
  1007. static u64 bfqg_prfill_sectors_recursive(struct seq_file *sf,
  1008. struct blkg_policy_data *pd, int off)
  1009. {
  1010. struct blkg_rwstat_sample tmp;
  1011. blkg_rwstat_recursive_sum(pd->blkg, &blkcg_policy_bfq,
  1012. offsetof(struct bfq_group, stats.bytes), &tmp);
  1013. return __blkg_prfill_u64(sf, pd,
  1014. (tmp.cnt[BLKG_RWSTAT_READ] + tmp.cnt[BLKG_RWSTAT_WRITE]) >> 9);
  1015. }
  1016. static int bfqg_print_stat_sectors_recursive(struct seq_file *sf, void *v)
  1017. {
  1018. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  1019. bfqg_prfill_sectors_recursive, &blkcg_policy_bfq, 0,
  1020. false);
  1021. return 0;
  1022. }
  1023. static u64 bfqg_prfill_avg_queue_size(struct seq_file *sf,
  1024. struct blkg_policy_data *pd, int off)
  1025. {
  1026. struct bfq_group *bfqg = pd_to_bfqg(pd);
  1027. u64 samples = bfq_stat_read(&bfqg->stats.avg_queue_size_samples);
  1028. u64 v = 0;
  1029. if (samples) {
  1030. v = bfq_stat_read(&bfqg->stats.avg_queue_size_sum);
  1031. v = div64_u64(v, samples);
  1032. }
  1033. __blkg_prfill_u64(sf, pd, v);
  1034. return 0;
  1035. }
  1036. /* print avg_queue_size */
  1037. static int bfqg_print_avg_queue_size(struct seq_file *sf, void *v)
  1038. {
  1039. blkcg_print_blkgs(sf, css_to_blkcg(seq_css(sf)),
  1040. bfqg_prfill_avg_queue_size, &blkcg_policy_bfq,
  1041. 0, false);
  1042. return 0;
  1043. }
  1044. #endif /* CONFIG_BFQ_CGROUP_DEBUG */
  1045. struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
  1046. {
  1047. int ret;
  1048. ret = blkcg_activate_policy(bfqd->queue, &blkcg_policy_bfq);
  1049. if (ret)
  1050. return NULL;
  1051. return blkg_to_bfqg(bfqd->queue->root_blkg);
  1052. }
  1053. struct blkcg_policy blkcg_policy_bfq = {
  1054. .dfl_cftypes = bfq_blkg_files,
  1055. .legacy_cftypes = bfq_blkcg_legacy_files,
  1056. .cpd_alloc_fn = bfq_cpd_alloc,
  1057. .cpd_init_fn = bfq_cpd_init,
  1058. .cpd_bind_fn = bfq_cpd_init,
  1059. .cpd_free_fn = bfq_cpd_free,
  1060. .pd_alloc_fn = bfq_pd_alloc,
  1061. .pd_init_fn = bfq_pd_init,
  1062. .pd_offline_fn = bfq_pd_offline,
  1063. .pd_free_fn = bfq_pd_free,
  1064. .pd_reset_stats_fn = bfq_pd_reset_stats,
  1065. };
  1066. struct cftype bfq_blkcg_legacy_files[] = {
  1067. {
  1068. .name = "bfq.weight",
  1069. .flags = CFTYPE_NOT_ON_ROOT,
  1070. .seq_show = bfq_io_show_weight_legacy,
  1071. .write_u64 = bfq_io_set_weight_legacy,
  1072. },
  1073. {
  1074. .name = "bfq.weight_device",
  1075. .flags = CFTYPE_NOT_ON_ROOT,
  1076. .seq_show = bfq_io_show_weight,
  1077. .write = bfq_io_set_weight,
  1078. },
  1079. /* statistics, covers only the tasks in the bfqg */
  1080. {
  1081. .name = "bfq.io_service_bytes",
  1082. .private = offsetof(struct bfq_group, stats.bytes),
  1083. .seq_show = bfqg_print_rwstat,
  1084. },
  1085. {
  1086. .name = "bfq.io_serviced",
  1087. .private = offsetof(struct bfq_group, stats.ios),
  1088. .seq_show = bfqg_print_rwstat,
  1089. },
  1090. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  1091. {
  1092. .name = "bfq.time",
  1093. .private = offsetof(struct bfq_group, stats.time),
  1094. .seq_show = bfqg_print_stat,
  1095. },
  1096. {
  1097. .name = "bfq.sectors",
  1098. .seq_show = bfqg_print_stat_sectors,
  1099. },
  1100. {
  1101. .name = "bfq.io_service_time",
  1102. .private = offsetof(struct bfq_group, stats.service_time),
  1103. .seq_show = bfqg_print_rwstat,
  1104. },
  1105. {
  1106. .name = "bfq.io_wait_time",
  1107. .private = offsetof(struct bfq_group, stats.wait_time),
  1108. .seq_show = bfqg_print_rwstat,
  1109. },
  1110. {
  1111. .name = "bfq.io_merged",
  1112. .private = offsetof(struct bfq_group, stats.merged),
  1113. .seq_show = bfqg_print_rwstat,
  1114. },
  1115. {
  1116. .name = "bfq.io_queued",
  1117. .private = offsetof(struct bfq_group, stats.queued),
  1118. .seq_show = bfqg_print_rwstat,
  1119. },
  1120. #endif /* CONFIG_BFQ_CGROUP_DEBUG */
  1121. /* the same statistics which cover the bfqg and its descendants */
  1122. {
  1123. .name = "bfq.io_service_bytes_recursive",
  1124. .private = offsetof(struct bfq_group, stats.bytes),
  1125. .seq_show = bfqg_print_rwstat_recursive,
  1126. },
  1127. {
  1128. .name = "bfq.io_serviced_recursive",
  1129. .private = offsetof(struct bfq_group, stats.ios),
  1130. .seq_show = bfqg_print_rwstat_recursive,
  1131. },
  1132. #ifdef CONFIG_BFQ_CGROUP_DEBUG
  1133. {
  1134. .name = "bfq.time_recursive",
  1135. .private = offsetof(struct bfq_group, stats.time),
  1136. .seq_show = bfqg_print_stat_recursive,
  1137. },
  1138. {
  1139. .name = "bfq.sectors_recursive",
  1140. .seq_show = bfqg_print_stat_sectors_recursive,
  1141. },
  1142. {
  1143. .name = "bfq.io_service_time_recursive",
  1144. .private = offsetof(struct bfq_group, stats.service_time),
  1145. .seq_show = bfqg_print_rwstat_recursive,
  1146. },
  1147. {
  1148. .name = "bfq.io_wait_time_recursive",
  1149. .private = offsetof(struct bfq_group, stats.wait_time),
  1150. .seq_show = bfqg_print_rwstat_recursive,
  1151. },
  1152. {
  1153. .name = "bfq.io_merged_recursive",
  1154. .private = offsetof(struct bfq_group, stats.merged),
  1155. .seq_show = bfqg_print_rwstat_recursive,
  1156. },
  1157. {
  1158. .name = "bfq.io_queued_recursive",
  1159. .private = offsetof(struct bfq_group, stats.queued),
  1160. .seq_show = bfqg_print_rwstat_recursive,
  1161. },
  1162. {
  1163. .name = "bfq.avg_queue_size",
  1164. .seq_show = bfqg_print_avg_queue_size,
  1165. },
  1166. {
  1167. .name = "bfq.group_wait_time",
  1168. .private = offsetof(struct bfq_group, stats.group_wait_time),
  1169. .seq_show = bfqg_print_stat,
  1170. },
  1171. {
  1172. .name = "bfq.idle_time",
  1173. .private = offsetof(struct bfq_group, stats.idle_time),
  1174. .seq_show = bfqg_print_stat,
  1175. },
  1176. {
  1177. .name = "bfq.empty_time",
  1178. .private = offsetof(struct bfq_group, stats.empty_time),
  1179. .seq_show = bfqg_print_stat,
  1180. },
  1181. {
  1182. .name = "bfq.dequeue",
  1183. .private = offsetof(struct bfq_group, stats.dequeue),
  1184. .seq_show = bfqg_print_stat,
  1185. },
  1186. #endif /* CONFIG_BFQ_CGROUP_DEBUG */
  1187. { } /* terminate */
  1188. };
  1189. struct cftype bfq_blkg_files[] = {
  1190. {
  1191. .name = "bfq.weight",
  1192. .flags = CFTYPE_NOT_ON_ROOT,
  1193. .seq_show = bfq_io_show_weight,
  1194. .write = bfq_io_set_weight,
  1195. },
  1196. {} /* terminate */
  1197. };
  1198. #else /* CONFIG_BFQ_GROUP_IOSCHED */
  1199. void bfq_bfqq_move(struct bfq_data *bfqd, struct bfq_queue *bfqq,
  1200. struct bfq_group *bfqg) {}
  1201. void bfq_init_entity(struct bfq_entity *entity, struct bfq_group *bfqg)
  1202. {
  1203. struct bfq_queue *bfqq = bfq_entity_to_bfqq(entity);
  1204. entity->weight = entity->new_weight;
  1205. entity->orig_weight = entity->new_weight;
  1206. if (bfqq) {
  1207. bfqq->ioprio = bfqq->new_ioprio;
  1208. bfqq->ioprio_class = bfqq->new_ioprio_class;
  1209. }
  1210. entity->sched_data = &bfqg->sched_data;
  1211. }
  1212. void bfq_bic_update_cgroup(struct bfq_io_cq *bic, struct bio *bio) {}
  1213. void bfq_end_wr_async(struct bfq_data *bfqd)
  1214. {
  1215. bfq_end_wr_async_queues(bfqd, bfqd->root_group);
  1216. }
  1217. struct bfq_group *bfq_find_set_group(struct bfq_data *bfqd, struct blkcg *blkcg)
  1218. {
  1219. return bfqd->root_group;
  1220. }
  1221. struct bfq_group *bfqq_group(struct bfq_queue *bfqq)
  1222. {
  1223. return bfqq->bfqd->root_group;
  1224. }
  1225. void bfqg_and_blkg_get(struct bfq_group *bfqg) {}
  1226. void bfqg_and_blkg_put(struct bfq_group *bfqg) {}
  1227. struct bfq_group *bfq_create_group_hierarchy(struct bfq_data *bfqd, int node)
  1228. {
  1229. struct bfq_group *bfqg;
  1230. int i;
  1231. bfqg = kmalloc_node(sizeof(*bfqg), GFP_KERNEL | __GFP_ZERO, node);
  1232. if (!bfqg)
  1233. return NULL;
  1234. for (i = 0; i < BFQ_IOPRIO_CLASSES; i++)
  1235. bfqg->sched_data.service_tree[i] = BFQ_SERVICE_TREE_INIT;
  1236. return bfqg;
  1237. }
  1238. #endif /* CONFIG_BFQ_GROUP_IOSCHED */