blk-mq-sysfs.c 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/kernel.h>
  3. #include <linux/module.h>
  4. #include <linux/backing-dev.h>
  5. #include <linux/bio.h>
  6. #include <linux/blkdev.h>
  7. #include <linux/mm.h>
  8. #include <linux/init.h>
  9. #include <linux/slab.h>
  10. #include <linux/workqueue.h>
  11. #include <linux/smp.h>
  12. #include <linux/blk-mq.h>
  13. #include "blk.h"
  14. #include "blk-mq.h"
  15. #include "blk-mq-tag.h"
  16. static void blk_mq_sysfs_release(struct kobject *kobj)
  17. {
  18. struct blk_mq_ctxs *ctxs = container_of(kobj, struct blk_mq_ctxs, kobj);
  19. free_percpu(ctxs->queue_ctx);
  20. kfree(ctxs);
  21. }
  22. static void blk_mq_ctx_sysfs_release(struct kobject *kobj)
  23. {
  24. struct blk_mq_ctx *ctx = container_of(kobj, struct blk_mq_ctx, kobj);
  25. /* ctx->ctxs won't be released until all ctx are freed */
  26. kobject_put(&ctx->ctxs->kobj);
  27. }
  28. static void blk_mq_hw_sysfs_release(struct kobject *kobj)
  29. {
  30. struct blk_mq_hw_ctx *hctx = container_of(kobj, struct blk_mq_hw_ctx,
  31. kobj);
  32. if (hctx->flags & BLK_MQ_F_BLOCKING)
  33. cleanup_srcu_struct(hctx->srcu);
  34. blk_free_flush_queue(hctx->fq);
  35. sbitmap_free(&hctx->ctx_map);
  36. free_cpumask_var(hctx->cpumask);
  37. kfree(hctx->ctxs);
  38. kfree(hctx);
  39. }
  40. struct blk_mq_ctx_sysfs_entry {
  41. struct attribute attr;
  42. ssize_t (*show)(struct blk_mq_ctx *, char *);
  43. ssize_t (*store)(struct blk_mq_ctx *, const char *, size_t);
  44. };
  45. struct blk_mq_hw_ctx_sysfs_entry {
  46. struct attribute attr;
  47. ssize_t (*show)(struct blk_mq_hw_ctx *, char *);
  48. ssize_t (*store)(struct blk_mq_hw_ctx *, const char *, size_t);
  49. };
  50. static ssize_t blk_mq_sysfs_show(struct kobject *kobj, struct attribute *attr,
  51. char *page)
  52. {
  53. struct blk_mq_ctx_sysfs_entry *entry;
  54. struct blk_mq_ctx *ctx;
  55. struct request_queue *q;
  56. ssize_t res;
  57. entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
  58. ctx = container_of(kobj, struct blk_mq_ctx, kobj);
  59. q = ctx->queue;
  60. if (!entry->show)
  61. return -EIO;
  62. mutex_lock(&q->sysfs_lock);
  63. res = entry->show(ctx, page);
  64. mutex_unlock(&q->sysfs_lock);
  65. return res;
  66. }
  67. static ssize_t blk_mq_sysfs_store(struct kobject *kobj, struct attribute *attr,
  68. const char *page, size_t length)
  69. {
  70. struct blk_mq_ctx_sysfs_entry *entry;
  71. struct blk_mq_ctx *ctx;
  72. struct request_queue *q;
  73. ssize_t res;
  74. entry = container_of(attr, struct blk_mq_ctx_sysfs_entry, attr);
  75. ctx = container_of(kobj, struct blk_mq_ctx, kobj);
  76. q = ctx->queue;
  77. if (!entry->store)
  78. return -EIO;
  79. mutex_lock(&q->sysfs_lock);
  80. res = entry->store(ctx, page, length);
  81. mutex_unlock(&q->sysfs_lock);
  82. return res;
  83. }
  84. static ssize_t blk_mq_hw_sysfs_show(struct kobject *kobj,
  85. struct attribute *attr, char *page)
  86. {
  87. struct blk_mq_hw_ctx_sysfs_entry *entry;
  88. struct blk_mq_hw_ctx *hctx;
  89. struct request_queue *q;
  90. ssize_t res;
  91. entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
  92. hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
  93. q = hctx->queue;
  94. if (!entry->show)
  95. return -EIO;
  96. mutex_lock(&q->sysfs_lock);
  97. res = entry->show(hctx, page);
  98. mutex_unlock(&q->sysfs_lock);
  99. return res;
  100. }
  101. static ssize_t blk_mq_hw_sysfs_store(struct kobject *kobj,
  102. struct attribute *attr, const char *page,
  103. size_t length)
  104. {
  105. struct blk_mq_hw_ctx_sysfs_entry *entry;
  106. struct blk_mq_hw_ctx *hctx;
  107. struct request_queue *q;
  108. ssize_t res;
  109. entry = container_of(attr, struct blk_mq_hw_ctx_sysfs_entry, attr);
  110. hctx = container_of(kobj, struct blk_mq_hw_ctx, kobj);
  111. q = hctx->queue;
  112. if (!entry->store)
  113. return -EIO;
  114. mutex_lock(&q->sysfs_lock);
  115. res = entry->store(hctx, page, length);
  116. mutex_unlock(&q->sysfs_lock);
  117. return res;
  118. }
  119. static ssize_t blk_mq_hw_sysfs_nr_tags_show(struct blk_mq_hw_ctx *hctx,
  120. char *page)
  121. {
  122. return sprintf(page, "%u\n", hctx->tags->nr_tags);
  123. }
  124. static ssize_t blk_mq_hw_sysfs_nr_reserved_tags_show(struct blk_mq_hw_ctx *hctx,
  125. char *page)
  126. {
  127. return sprintf(page, "%u\n", hctx->tags->nr_reserved_tags);
  128. }
  129. static ssize_t blk_mq_hw_sysfs_cpus_show(struct blk_mq_hw_ctx *hctx, char *page)
  130. {
  131. const size_t size = PAGE_SIZE - 1;
  132. unsigned int i, first = 1;
  133. int ret = 0, pos = 0;
  134. for_each_cpu(i, hctx->cpumask) {
  135. if (first)
  136. ret = snprintf(pos + page, size - pos, "%u", i);
  137. else
  138. ret = snprintf(pos + page, size - pos, ", %u", i);
  139. if (ret >= size - pos)
  140. break;
  141. first = 0;
  142. pos += ret;
  143. }
  144. ret = snprintf(pos + page, size + 1 - pos, "\n");
  145. return pos + ret;
  146. }
  147. static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_tags = {
  148. .attr = {.name = "nr_tags", .mode = 0444 },
  149. .show = blk_mq_hw_sysfs_nr_tags_show,
  150. };
  151. static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_nr_reserved_tags = {
  152. .attr = {.name = "nr_reserved_tags", .mode = 0444 },
  153. .show = blk_mq_hw_sysfs_nr_reserved_tags_show,
  154. };
  155. static struct blk_mq_hw_ctx_sysfs_entry blk_mq_hw_sysfs_cpus = {
  156. .attr = {.name = "cpu_list", .mode = 0444 },
  157. .show = blk_mq_hw_sysfs_cpus_show,
  158. };
  159. static struct attribute *default_hw_ctx_attrs[] = {
  160. &blk_mq_hw_sysfs_nr_tags.attr,
  161. &blk_mq_hw_sysfs_nr_reserved_tags.attr,
  162. &blk_mq_hw_sysfs_cpus.attr,
  163. NULL,
  164. };
  165. ATTRIBUTE_GROUPS(default_hw_ctx);
  166. static const struct sysfs_ops blk_mq_sysfs_ops = {
  167. .show = blk_mq_sysfs_show,
  168. .store = blk_mq_sysfs_store,
  169. };
  170. static const struct sysfs_ops blk_mq_hw_sysfs_ops = {
  171. .show = blk_mq_hw_sysfs_show,
  172. .store = blk_mq_hw_sysfs_store,
  173. };
  174. static struct kobj_type blk_mq_ktype = {
  175. .sysfs_ops = &blk_mq_sysfs_ops,
  176. .release = blk_mq_sysfs_release,
  177. };
  178. static struct kobj_type blk_mq_ctx_ktype = {
  179. .sysfs_ops = &blk_mq_sysfs_ops,
  180. .release = blk_mq_ctx_sysfs_release,
  181. };
  182. static struct kobj_type blk_mq_hw_ktype = {
  183. .sysfs_ops = &blk_mq_hw_sysfs_ops,
  184. .default_groups = default_hw_ctx_groups,
  185. .release = blk_mq_hw_sysfs_release,
  186. };
  187. static void blk_mq_unregister_hctx(struct blk_mq_hw_ctx *hctx)
  188. {
  189. struct blk_mq_ctx *ctx;
  190. int i;
  191. if (!hctx->nr_ctx)
  192. return;
  193. hctx_for_each_ctx(hctx, ctx, i)
  194. kobject_del(&ctx->kobj);
  195. kobject_del(&hctx->kobj);
  196. }
  197. static int blk_mq_register_hctx(struct blk_mq_hw_ctx *hctx)
  198. {
  199. struct request_queue *q = hctx->queue;
  200. struct blk_mq_ctx *ctx;
  201. int i, ret;
  202. if (!hctx->nr_ctx)
  203. return 0;
  204. ret = kobject_add(&hctx->kobj, q->mq_kobj, "%u", hctx->queue_num);
  205. if (ret)
  206. return ret;
  207. hctx_for_each_ctx(hctx, ctx, i) {
  208. ret = kobject_add(&ctx->kobj, &hctx->kobj, "cpu%u", ctx->cpu);
  209. if (ret)
  210. break;
  211. }
  212. return ret;
  213. }
  214. void blk_mq_unregister_dev(struct device *dev, struct request_queue *q)
  215. {
  216. struct blk_mq_hw_ctx *hctx;
  217. int i;
  218. lockdep_assert_held(&q->sysfs_dir_lock);
  219. queue_for_each_hw_ctx(q, hctx, i)
  220. blk_mq_unregister_hctx(hctx);
  221. kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
  222. kobject_del(q->mq_kobj);
  223. kobject_put(&dev->kobj);
  224. q->mq_sysfs_init_done = false;
  225. }
  226. void blk_mq_hctx_kobj_init(struct blk_mq_hw_ctx *hctx)
  227. {
  228. kobject_init(&hctx->kobj, &blk_mq_hw_ktype);
  229. }
  230. void blk_mq_sysfs_deinit(struct request_queue *q)
  231. {
  232. struct blk_mq_ctx *ctx;
  233. int cpu;
  234. for_each_possible_cpu(cpu) {
  235. ctx = per_cpu_ptr(q->queue_ctx, cpu);
  236. kobject_put(&ctx->kobj);
  237. }
  238. kobject_put(q->mq_kobj);
  239. }
  240. void blk_mq_sysfs_init(struct request_queue *q)
  241. {
  242. struct blk_mq_ctx *ctx;
  243. int cpu;
  244. kobject_init(q->mq_kobj, &blk_mq_ktype);
  245. for_each_possible_cpu(cpu) {
  246. ctx = per_cpu_ptr(q->queue_ctx, cpu);
  247. kobject_get(q->mq_kobj);
  248. kobject_init(&ctx->kobj, &blk_mq_ctx_ktype);
  249. }
  250. }
  251. int __blk_mq_register_dev(struct device *dev, struct request_queue *q)
  252. {
  253. struct blk_mq_hw_ctx *hctx;
  254. int ret, i;
  255. WARN_ON_ONCE(!q->kobj.parent);
  256. lockdep_assert_held(&q->sysfs_dir_lock);
  257. ret = kobject_add(q->mq_kobj, kobject_get(&dev->kobj), "%s", "mq");
  258. if (ret < 0)
  259. goto out;
  260. kobject_uevent(q->mq_kobj, KOBJ_ADD);
  261. queue_for_each_hw_ctx(q, hctx, i) {
  262. ret = blk_mq_register_hctx(hctx);
  263. if (ret)
  264. goto unreg;
  265. }
  266. q->mq_sysfs_init_done = true;
  267. out:
  268. return ret;
  269. unreg:
  270. while (--i >= 0)
  271. blk_mq_unregister_hctx(q->queue_hw_ctx[i]);
  272. kobject_uevent(q->mq_kobj, KOBJ_REMOVE);
  273. kobject_del(q->mq_kobj);
  274. kobject_put(&dev->kobj);
  275. return ret;
  276. }
  277. void blk_mq_sysfs_unregister(struct request_queue *q)
  278. {
  279. struct blk_mq_hw_ctx *hctx;
  280. int i;
  281. mutex_lock(&q->sysfs_dir_lock);
  282. if (!q->mq_sysfs_init_done)
  283. goto unlock;
  284. queue_for_each_hw_ctx(q, hctx, i)
  285. blk_mq_unregister_hctx(hctx);
  286. unlock:
  287. mutex_unlock(&q->sysfs_dir_lock);
  288. }
  289. int blk_mq_sysfs_register(struct request_queue *q)
  290. {
  291. struct blk_mq_hw_ctx *hctx;
  292. int i, ret = 0;
  293. mutex_lock(&q->sysfs_dir_lock);
  294. if (!q->mq_sysfs_init_done)
  295. goto unlock;
  296. queue_for_each_hw_ctx(q, hctx, i) {
  297. ret = blk_mq_register_hctx(hctx);
  298. if (ret)
  299. break;
  300. }
  301. unlock:
  302. mutex_unlock(&q->sysfs_dir_lock);
  303. return ret;
  304. }