blk-ioprio.c 6.4 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Block rq-qos policy for assigning an I/O priority class to requests.
  4. *
  5. * Using an rq-qos policy for assigning I/O priority class has two advantages
  6. * over using the ioprio_set() system call:
  7. *
  8. * - This policy is cgroup based so it has all the advantages of cgroups.
  9. * - While ioprio_set() does not affect page cache writeback I/O, this rq-qos
  10. * controller affects page cache writeback I/O for filesystems that support
  11. * assiociating a cgroup with writeback I/O. See also
  12. * Documentation/admin-guide/cgroup-v2.rst.
  13. */
  14. #include <linux/blk-cgroup.h>
  15. #include <linux/blk-mq.h>
  16. #include <linux/blk_types.h>
  17. #include <linux/kernel.h>
  18. #include <linux/module.h>
  19. #include "blk-ioprio.h"
  20. #include "blk-rq-qos.h"
  21. /**
  22. * enum prio_policy - I/O priority class policy.
  23. * @POLICY_NO_CHANGE: (default) do not modify the I/O priority class.
  24. * @POLICY_NONE_TO_RT: modify IOPRIO_CLASS_NONE into IOPRIO_CLASS_RT.
  25. * @POLICY_RESTRICT_TO_BE: modify IOPRIO_CLASS_NONE and IOPRIO_CLASS_RT into
  26. * IOPRIO_CLASS_BE.
  27. * @POLICY_ALL_TO_IDLE: change the I/O priority class into IOPRIO_CLASS_IDLE.
  28. *
  29. * See also <linux/ioprio.h>.
  30. */
  31. enum prio_policy {
  32. POLICY_NO_CHANGE = 0,
  33. POLICY_NONE_TO_RT = 1,
  34. POLICY_RESTRICT_TO_BE = 2,
  35. POLICY_ALL_TO_IDLE = 3,
  36. };
  37. static const char *policy_name[] = {
  38. [POLICY_NO_CHANGE] = "no-change",
  39. [POLICY_NONE_TO_RT] = "none-to-rt",
  40. [POLICY_RESTRICT_TO_BE] = "restrict-to-be",
  41. [POLICY_ALL_TO_IDLE] = "idle",
  42. };
  43. static struct blkcg_policy ioprio_policy;
  44. /**
  45. * struct ioprio_blkg - Per (cgroup, request queue) data.
  46. * @pd: blkg_policy_data structure.
  47. */
  48. struct ioprio_blkg {
  49. struct blkg_policy_data pd;
  50. };
  51. /**
  52. * struct ioprio_blkcg - Per cgroup data.
  53. * @cpd: blkcg_policy_data structure.
  54. * @prio_policy: One of the IOPRIO_CLASS_* values. See also <linux/ioprio.h>.
  55. */
  56. struct ioprio_blkcg {
  57. struct blkcg_policy_data cpd;
  58. enum prio_policy prio_policy;
  59. };
  60. static inline struct ioprio_blkg *pd_to_ioprio(struct blkg_policy_data *pd)
  61. {
  62. return pd ? container_of(pd, struct ioprio_blkg, pd) : NULL;
  63. }
  64. static struct ioprio_blkcg *blkcg_to_ioprio_blkcg(struct blkcg *blkcg)
  65. {
  66. return container_of(blkcg_to_cpd(blkcg, &ioprio_policy),
  67. struct ioprio_blkcg, cpd);
  68. }
  69. static struct ioprio_blkcg *
  70. ioprio_blkcg_from_css(struct cgroup_subsys_state *css)
  71. {
  72. return blkcg_to_ioprio_blkcg(css_to_blkcg(css));
  73. }
  74. static struct ioprio_blkcg *ioprio_blkcg_from_bio(struct bio *bio)
  75. {
  76. struct blkg_policy_data *pd = blkg_to_pd(bio->bi_blkg, &ioprio_policy);
  77. if (!pd)
  78. return NULL;
  79. return blkcg_to_ioprio_blkcg(pd->blkg->blkcg);
  80. }
  81. static int ioprio_show_prio_policy(struct seq_file *sf, void *v)
  82. {
  83. struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(seq_css(sf));
  84. seq_printf(sf, "%s\n", policy_name[blkcg->prio_policy]);
  85. return 0;
  86. }
  87. static ssize_t ioprio_set_prio_policy(struct kernfs_open_file *of, char *buf,
  88. size_t nbytes, loff_t off)
  89. {
  90. struct ioprio_blkcg *blkcg = ioprio_blkcg_from_css(of_css(of));
  91. int ret;
  92. if (off != 0)
  93. return -EIO;
  94. /* kernfs_fop_write_iter() terminates 'buf' with '\0'. */
  95. ret = sysfs_match_string(policy_name, buf);
  96. if (ret < 0)
  97. return ret;
  98. blkcg->prio_policy = ret;
  99. return nbytes;
  100. }
  101. static struct blkg_policy_data *
  102. ioprio_alloc_pd(gfp_t gfp, struct request_queue *q, struct blkcg *blkcg)
  103. {
  104. struct ioprio_blkg *ioprio_blkg;
  105. ioprio_blkg = kzalloc(sizeof(*ioprio_blkg), gfp);
  106. if (!ioprio_blkg)
  107. return NULL;
  108. return &ioprio_blkg->pd;
  109. }
  110. static void ioprio_free_pd(struct blkg_policy_data *pd)
  111. {
  112. struct ioprio_blkg *ioprio_blkg = pd_to_ioprio(pd);
  113. kfree(ioprio_blkg);
  114. }
  115. static struct blkcg_policy_data *ioprio_alloc_cpd(gfp_t gfp)
  116. {
  117. struct ioprio_blkcg *blkcg;
  118. blkcg = kzalloc(sizeof(*blkcg), gfp);
  119. if (!blkcg)
  120. return NULL;
  121. blkcg->prio_policy = POLICY_NO_CHANGE;
  122. return &blkcg->cpd;
  123. }
  124. static void ioprio_free_cpd(struct blkcg_policy_data *cpd)
  125. {
  126. struct ioprio_blkcg *blkcg = container_of(cpd, typeof(*blkcg), cpd);
  127. kfree(blkcg);
  128. }
  129. #define IOPRIO_ATTRS \
  130. { \
  131. .name = "prio.class", \
  132. .seq_show = ioprio_show_prio_policy, \
  133. .write = ioprio_set_prio_policy, \
  134. }, \
  135. { } /* sentinel */
  136. /* cgroup v2 attributes */
  137. static struct cftype ioprio_files[] = {
  138. IOPRIO_ATTRS
  139. };
  140. /* cgroup v1 attributes */
  141. static struct cftype ioprio_legacy_files[] = {
  142. IOPRIO_ATTRS
  143. };
  144. static struct blkcg_policy ioprio_policy = {
  145. .dfl_cftypes = ioprio_files,
  146. .legacy_cftypes = ioprio_legacy_files,
  147. .cpd_alloc_fn = ioprio_alloc_cpd,
  148. .cpd_free_fn = ioprio_free_cpd,
  149. .pd_alloc_fn = ioprio_alloc_pd,
  150. .pd_free_fn = ioprio_free_pd,
  151. };
  152. struct blk_ioprio {
  153. struct rq_qos rqos;
  154. };
  155. static void blkcg_ioprio_track(struct rq_qos *rqos, struct request *rq,
  156. struct bio *bio)
  157. {
  158. struct ioprio_blkcg *blkcg = ioprio_blkcg_from_bio(bio);
  159. /*
  160. * Except for IOPRIO_CLASS_NONE, higher I/O priority numbers
  161. * correspond to a lower priority. Hence, the max_t() below selects
  162. * the lower priority of bi_ioprio and the cgroup I/O priority class.
  163. * If the cgroup policy has been set to POLICY_NO_CHANGE == 0, the
  164. * bio I/O priority is not modified. If the bio I/O priority equals
  165. * IOPRIO_CLASS_NONE, the cgroup I/O priority is assigned to the bio.
  166. */
  167. bio->bi_ioprio = max_t(u16, bio->bi_ioprio,
  168. IOPRIO_PRIO_VALUE(blkcg->prio_policy, 0));
  169. }
  170. static void blkcg_ioprio_exit(struct rq_qos *rqos)
  171. {
  172. struct blk_ioprio *blkioprio_blkg =
  173. container_of(rqos, typeof(*blkioprio_blkg), rqos);
  174. blkcg_deactivate_policy(rqos->q, &ioprio_policy);
  175. kfree(blkioprio_blkg);
  176. }
  177. static struct rq_qos_ops blkcg_ioprio_ops = {
  178. .track = blkcg_ioprio_track,
  179. .exit = blkcg_ioprio_exit,
  180. };
  181. int blk_ioprio_init(struct request_queue *q)
  182. {
  183. struct blk_ioprio *blkioprio_blkg;
  184. struct rq_qos *rqos;
  185. int ret;
  186. blkioprio_blkg = kzalloc(sizeof(*blkioprio_blkg), GFP_KERNEL);
  187. if (!blkioprio_blkg)
  188. return -ENOMEM;
  189. ret = blkcg_activate_policy(q, &ioprio_policy);
  190. if (ret) {
  191. kfree(blkioprio_blkg);
  192. return ret;
  193. }
  194. rqos = &blkioprio_blkg->rqos;
  195. rqos->id = RQ_QOS_IOPRIO;
  196. rqos->ops = &blkcg_ioprio_ops;
  197. rqos->q = q;
  198. /*
  199. * Registering the rq-qos policy after activating the blk-cgroup
  200. * policy guarantees that ioprio_blkcg_from_bio(bio) != NULL in the
  201. * rq-qos callbacks.
  202. */
  203. rq_qos_add(q, rqos);
  204. return 0;
  205. }
  206. static int __init ioprio_init(void)
  207. {
  208. return blkcg_policy_register(&ioprio_policy);
  209. }
  210. static void __exit ioprio_exit(void)
  211. {
  212. blkcg_policy_unregister(&ioprio_policy);
  213. }
  214. module_init(ioprio_init);
  215. module_exit(ioprio_exit);