elevator.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Block device elevator/IO-scheduler.
  4. *
  5. * Copyright (C) 2000 Andrea Arcangeli <andrea@suse.de> SuSE
  6. *
  7. * 30042000 Jens Axboe <axboe@kernel.dk> :
  8. *
  9. * Split the elevator a bit so that it is possible to choose a different
  10. * one or even write a new "plug in". There are three pieces:
  11. * - elevator_fn, inserts a new request in the queue list
  12. * - elevator_merge_fn, decides whether a new buffer can be merged with
  13. * an existing request
  14. * - elevator_dequeue_fn, called when a request is taken off the active list
  15. *
  16. * 20082000 Dave Jones <davej@suse.de> :
  17. * Removed tests for max-bomb-segments, which was breaking elvtune
  18. * when run without -bN
  19. *
  20. * Jens:
  21. * - Rework again to work with bio instead of buffer_heads
  22. * - loose bi_dev comparisons, partition handling is right now
  23. * - completely modularize elevator setup and teardown
  24. *
  25. */
  26. #include <linux/kernel.h>
  27. #include <linux/fs.h>
  28. #include <linux/blkdev.h>
  29. #include <linux/elevator.h>
  30. #include <linux/bio.h>
  31. #include <linux/module.h>
  32. #include <linux/slab.h>
  33. #include <linux/init.h>
  34. #include <linux/compiler.h>
  35. #include <linux/blktrace_api.h>
  36. #include <linux/hash.h>
  37. #include <linux/uaccess.h>
  38. #include <linux/pm_runtime.h>
  39. #include <linux/blk-cgroup.h>
  40. #include <trace/events/block.h>
  41. #include "blk.h"
  42. #include "blk-mq-sched.h"
  43. #include "blk-pm.h"
  44. #include "blk-wbt.h"
  45. static DEFINE_SPINLOCK(elv_list_lock);
  46. static LIST_HEAD(elv_list);
  47. /*
  48. * Merge hash stuff.
  49. */
  50. #define rq_hash_key(rq) (blk_rq_pos(rq) + blk_rq_sectors(rq))
  51. /*
  52. * Query io scheduler to see if the current process issuing bio may be
  53. * merged with rq.
  54. */
  55. static int elv_iosched_allow_bio_merge(struct request *rq, struct bio *bio)
  56. {
  57. struct request_queue *q = rq->q;
  58. struct elevator_queue *e = q->elevator;
  59. if (e->type->ops.allow_merge)
  60. return e->type->ops.allow_merge(q, rq, bio);
  61. return 1;
  62. }
  63. /*
  64. * can we safely merge with this request?
  65. */
  66. bool elv_bio_merge_ok(struct request *rq, struct bio *bio)
  67. {
  68. if (!blk_rq_merge_ok(rq, bio))
  69. return false;
  70. if (!elv_iosched_allow_bio_merge(rq, bio))
  71. return false;
  72. return true;
  73. }
  74. EXPORT_SYMBOL(elv_bio_merge_ok);
  75. static inline bool elv_support_features(unsigned int elv_features,
  76. unsigned int required_features)
  77. {
  78. return (required_features & elv_features) == required_features;
  79. }
  80. /**
  81. * elevator_match - Test an elevator name and features
  82. * @e: Scheduler to test
  83. * @name: Elevator name to test
  84. * @required_features: Features that the elevator must provide
  85. *
  86. * Return true if the elevator @e name matches @name and if @e provides all
  87. * the features specified by @required_features.
  88. */
  89. static bool elevator_match(const struct elevator_type *e, const char *name,
  90. unsigned int required_features)
  91. {
  92. if (!elv_support_features(e->elevator_features, required_features))
  93. return false;
  94. if (!strcmp(e->elevator_name, name))
  95. return true;
  96. if (e->elevator_alias && !strcmp(e->elevator_alias, name))
  97. return true;
  98. return false;
  99. }
  100. /**
  101. * elevator_find - Find an elevator
  102. * @name: Name of the elevator to find
  103. * @required_features: Features that the elevator must provide
  104. *
  105. * Return the first registered scheduler with name @name and supporting the
  106. * features @required_features and NULL otherwise.
  107. */
  108. static struct elevator_type *elevator_find(const char *name,
  109. unsigned int required_features)
  110. {
  111. struct elevator_type *e;
  112. list_for_each_entry(e, &elv_list, list) {
  113. if (elevator_match(e, name, required_features))
  114. return e;
  115. }
  116. return NULL;
  117. }
  118. static void elevator_put(struct elevator_type *e)
  119. {
  120. module_put(e->elevator_owner);
  121. }
  122. static struct elevator_type *elevator_get(struct request_queue *q,
  123. const char *name, bool try_loading)
  124. {
  125. struct elevator_type *e;
  126. spin_lock(&elv_list_lock);
  127. e = elevator_find(name, q->required_elevator_features);
  128. if (!e && try_loading) {
  129. spin_unlock(&elv_list_lock);
  130. request_module("%s-iosched", name);
  131. spin_lock(&elv_list_lock);
  132. e = elevator_find(name, q->required_elevator_features);
  133. }
  134. if (e && !try_module_get(e->elevator_owner))
  135. e = NULL;
  136. spin_unlock(&elv_list_lock);
  137. return e;
  138. }
  139. static struct kobj_type elv_ktype;
  140. struct elevator_queue *elevator_alloc(struct request_queue *q,
  141. struct elevator_type *e)
  142. {
  143. struct elevator_queue *eq;
  144. eq = kzalloc_node(sizeof(*eq), GFP_KERNEL, q->node);
  145. if (unlikely(!eq))
  146. return NULL;
  147. eq->type = e;
  148. kobject_init(&eq->kobj, &elv_ktype);
  149. mutex_init(&eq->sysfs_lock);
  150. hash_init(eq->hash);
  151. return eq;
  152. }
  153. EXPORT_SYMBOL(elevator_alloc);
  154. static void elevator_release(struct kobject *kobj)
  155. {
  156. struct elevator_queue *e;
  157. e = container_of(kobj, struct elevator_queue, kobj);
  158. elevator_put(e->type);
  159. kfree(e);
  160. }
  161. void __elevator_exit(struct request_queue *q, struct elevator_queue *e)
  162. {
  163. mutex_lock(&e->sysfs_lock);
  164. blk_mq_exit_sched(q, e);
  165. mutex_unlock(&e->sysfs_lock);
  166. kobject_put(&e->kobj);
  167. }
  168. static inline void __elv_rqhash_del(struct request *rq)
  169. {
  170. hash_del(&rq->hash);
  171. rq->rq_flags &= ~RQF_HASHED;
  172. }
  173. void elv_rqhash_del(struct request_queue *q, struct request *rq)
  174. {
  175. if (ELV_ON_HASH(rq))
  176. __elv_rqhash_del(rq);
  177. }
  178. EXPORT_SYMBOL_GPL(elv_rqhash_del);
  179. void elv_rqhash_add(struct request_queue *q, struct request *rq)
  180. {
  181. struct elevator_queue *e = q->elevator;
  182. BUG_ON(ELV_ON_HASH(rq));
  183. hash_add(e->hash, &rq->hash, rq_hash_key(rq));
  184. rq->rq_flags |= RQF_HASHED;
  185. }
  186. EXPORT_SYMBOL_GPL(elv_rqhash_add);
  187. void elv_rqhash_reposition(struct request_queue *q, struct request *rq)
  188. {
  189. __elv_rqhash_del(rq);
  190. elv_rqhash_add(q, rq);
  191. }
  192. struct request *elv_rqhash_find(struct request_queue *q, sector_t offset)
  193. {
  194. struct elevator_queue *e = q->elevator;
  195. struct hlist_node *next;
  196. struct request *rq;
  197. hash_for_each_possible_safe(e->hash, rq, next, hash, offset) {
  198. BUG_ON(!ELV_ON_HASH(rq));
  199. if (unlikely(!rq_mergeable(rq))) {
  200. __elv_rqhash_del(rq);
  201. continue;
  202. }
  203. if (rq_hash_key(rq) == offset)
  204. return rq;
  205. }
  206. return NULL;
  207. }
  208. /*
  209. * RB-tree support functions for inserting/lookup/removal of requests
  210. * in a sorted RB tree.
  211. */
  212. void elv_rb_add(struct rb_root *root, struct request *rq)
  213. {
  214. struct rb_node **p = &root->rb_node;
  215. struct rb_node *parent = NULL;
  216. struct request *__rq;
  217. while (*p) {
  218. parent = *p;
  219. __rq = rb_entry(parent, struct request, rb_node);
  220. if (blk_rq_pos(rq) < blk_rq_pos(__rq))
  221. p = &(*p)->rb_left;
  222. else if (blk_rq_pos(rq) >= blk_rq_pos(__rq))
  223. p = &(*p)->rb_right;
  224. }
  225. rb_link_node(&rq->rb_node, parent, p);
  226. rb_insert_color(&rq->rb_node, root);
  227. }
  228. EXPORT_SYMBOL(elv_rb_add);
  229. void elv_rb_del(struct rb_root *root, struct request *rq)
  230. {
  231. BUG_ON(RB_EMPTY_NODE(&rq->rb_node));
  232. rb_erase(&rq->rb_node, root);
  233. RB_CLEAR_NODE(&rq->rb_node);
  234. }
  235. EXPORT_SYMBOL(elv_rb_del);
  236. struct request *elv_rb_find(struct rb_root *root, sector_t sector)
  237. {
  238. struct rb_node *n = root->rb_node;
  239. struct request *rq;
  240. while (n) {
  241. rq = rb_entry(n, struct request, rb_node);
  242. if (sector < blk_rq_pos(rq))
  243. n = n->rb_left;
  244. else if (sector > blk_rq_pos(rq))
  245. n = n->rb_right;
  246. else
  247. return rq;
  248. }
  249. return NULL;
  250. }
  251. EXPORT_SYMBOL(elv_rb_find);
  252. enum elv_merge elv_merge(struct request_queue *q, struct request **req,
  253. struct bio *bio)
  254. {
  255. struct elevator_queue *e = q->elevator;
  256. struct request *__rq;
  257. /*
  258. * Levels of merges:
  259. * nomerges: No merges at all attempted
  260. * noxmerges: Only simple one-hit cache try
  261. * merges: All merge tries attempted
  262. */
  263. if (blk_queue_nomerges(q) || !bio_mergeable(bio))
  264. return ELEVATOR_NO_MERGE;
  265. /*
  266. * First try one-hit cache.
  267. */
  268. if (q->last_merge && elv_bio_merge_ok(q->last_merge, bio)) {
  269. enum elv_merge ret = blk_try_merge(q->last_merge, bio);
  270. if (ret != ELEVATOR_NO_MERGE) {
  271. *req = q->last_merge;
  272. return ret;
  273. }
  274. }
  275. if (blk_queue_noxmerges(q))
  276. return ELEVATOR_NO_MERGE;
  277. /*
  278. * See if our hash lookup can find a potential backmerge.
  279. */
  280. __rq = elv_rqhash_find(q, bio->bi_iter.bi_sector);
  281. if (__rq && elv_bio_merge_ok(__rq, bio)) {
  282. *req = __rq;
  283. if (blk_discard_mergable(__rq))
  284. return ELEVATOR_DISCARD_MERGE;
  285. return ELEVATOR_BACK_MERGE;
  286. }
  287. if (e->type->ops.request_merge)
  288. return e->type->ops.request_merge(q, req, bio);
  289. return ELEVATOR_NO_MERGE;
  290. }
  291. /*
  292. * Attempt to do an insertion back merge. Only check for the case where
  293. * we can append 'rq' to an existing request, so we can throw 'rq' away
  294. * afterwards.
  295. *
  296. * Returns true if we merged, false otherwise
  297. */
  298. bool elv_attempt_insert_merge(struct request_queue *q, struct request *rq)
  299. {
  300. struct request *__rq;
  301. bool ret;
  302. if (blk_queue_nomerges(q))
  303. return false;
  304. /*
  305. * First try one-hit cache.
  306. */
  307. if (q->last_merge && blk_attempt_req_merge(q, q->last_merge, rq))
  308. return true;
  309. if (blk_queue_noxmerges(q))
  310. return false;
  311. ret = false;
  312. /*
  313. * See if our hash lookup can find a potential backmerge.
  314. */
  315. while (1) {
  316. __rq = elv_rqhash_find(q, blk_rq_pos(rq));
  317. if (!__rq || !blk_attempt_req_merge(q, __rq, rq))
  318. break;
  319. /* The merged request could be merged with others, try again */
  320. ret = true;
  321. rq = __rq;
  322. }
  323. return ret;
  324. }
  325. void elv_merged_request(struct request_queue *q, struct request *rq,
  326. enum elv_merge type)
  327. {
  328. struct elevator_queue *e = q->elevator;
  329. if (e->type->ops.request_merged)
  330. e->type->ops.request_merged(q, rq, type);
  331. if (type == ELEVATOR_BACK_MERGE)
  332. elv_rqhash_reposition(q, rq);
  333. q->last_merge = rq;
  334. }
  335. void elv_merge_requests(struct request_queue *q, struct request *rq,
  336. struct request *next)
  337. {
  338. struct elevator_queue *e = q->elevator;
  339. if (e->type->ops.requests_merged)
  340. e->type->ops.requests_merged(q, rq, next);
  341. elv_rqhash_reposition(q, rq);
  342. q->last_merge = rq;
  343. }
  344. struct request *elv_latter_request(struct request_queue *q, struct request *rq)
  345. {
  346. struct elevator_queue *e = q->elevator;
  347. if (e->type->ops.next_request)
  348. return e->type->ops.next_request(q, rq);
  349. return NULL;
  350. }
  351. struct request *elv_former_request(struct request_queue *q, struct request *rq)
  352. {
  353. struct elevator_queue *e = q->elevator;
  354. if (e->type->ops.former_request)
  355. return e->type->ops.former_request(q, rq);
  356. return NULL;
  357. }
  358. #define to_elv(atr) container_of((atr), struct elv_fs_entry, attr)
  359. static ssize_t
  360. elv_attr_show(struct kobject *kobj, struct attribute *attr, char *page)
  361. {
  362. struct elv_fs_entry *entry = to_elv(attr);
  363. struct elevator_queue *e;
  364. ssize_t error;
  365. if (!entry->show)
  366. return -EIO;
  367. e = container_of(kobj, struct elevator_queue, kobj);
  368. mutex_lock(&e->sysfs_lock);
  369. error = e->type ? entry->show(e, page) : -ENOENT;
  370. mutex_unlock(&e->sysfs_lock);
  371. return error;
  372. }
  373. static ssize_t
  374. elv_attr_store(struct kobject *kobj, struct attribute *attr,
  375. const char *page, size_t length)
  376. {
  377. struct elv_fs_entry *entry = to_elv(attr);
  378. struct elevator_queue *e;
  379. ssize_t error;
  380. if (!entry->store)
  381. return -EIO;
  382. e = container_of(kobj, struct elevator_queue, kobj);
  383. mutex_lock(&e->sysfs_lock);
  384. error = e->type ? entry->store(e, page, length) : -ENOENT;
  385. mutex_unlock(&e->sysfs_lock);
  386. return error;
  387. }
  388. static const struct sysfs_ops elv_sysfs_ops = {
  389. .show = elv_attr_show,
  390. .store = elv_attr_store,
  391. };
  392. static struct kobj_type elv_ktype = {
  393. .sysfs_ops = &elv_sysfs_ops,
  394. .release = elevator_release,
  395. };
  396. int elv_register_queue(struct request_queue *q, bool uevent)
  397. {
  398. struct elevator_queue *e = q->elevator;
  399. int error;
  400. lockdep_assert_held(&q->sysfs_lock);
  401. error = kobject_add(&e->kobj, &q->kobj, "%s", "iosched");
  402. if (!error) {
  403. struct elv_fs_entry *attr = e->type->elevator_attrs;
  404. if (attr) {
  405. while (attr->attr.name) {
  406. if (sysfs_create_file(&e->kobj, &attr->attr))
  407. break;
  408. attr++;
  409. }
  410. }
  411. if (uevent)
  412. kobject_uevent(&e->kobj, KOBJ_ADD);
  413. e->registered = 1;
  414. }
  415. return error;
  416. }
  417. void elv_unregister_queue(struct request_queue *q)
  418. {
  419. lockdep_assert_held(&q->sysfs_lock);
  420. if (q) {
  421. struct elevator_queue *e = q->elevator;
  422. kobject_uevent(&e->kobj, KOBJ_REMOVE);
  423. kobject_del(&e->kobj);
  424. e->registered = 0;
  425. }
  426. }
  427. int elv_register(struct elevator_type *e)
  428. {
  429. /* create icq_cache if requested */
  430. if (e->icq_size) {
  431. if (WARN_ON(e->icq_size < sizeof(struct io_cq)) ||
  432. WARN_ON(e->icq_align < __alignof__(struct io_cq)))
  433. return -EINVAL;
  434. snprintf(e->icq_cache_name, sizeof(e->icq_cache_name),
  435. "%s_io_cq", e->elevator_name);
  436. e->icq_cache = kmem_cache_create(e->icq_cache_name, e->icq_size,
  437. e->icq_align, 0, NULL);
  438. if (!e->icq_cache)
  439. return -ENOMEM;
  440. }
  441. /* register, don't allow duplicate names */
  442. spin_lock(&elv_list_lock);
  443. if (elevator_find(e->elevator_name, 0)) {
  444. spin_unlock(&elv_list_lock);
  445. kmem_cache_destroy(e->icq_cache);
  446. return -EBUSY;
  447. }
  448. list_add_tail(&e->list, &elv_list);
  449. spin_unlock(&elv_list_lock);
  450. printk(KERN_INFO "io scheduler %s registered\n", e->elevator_name);
  451. return 0;
  452. }
  453. EXPORT_SYMBOL_GPL(elv_register);
  454. void elv_unregister(struct elevator_type *e)
  455. {
  456. /* unregister */
  457. spin_lock(&elv_list_lock);
  458. list_del_init(&e->list);
  459. spin_unlock(&elv_list_lock);
  460. /*
  461. * Destroy icq_cache if it exists. icq's are RCU managed. Make
  462. * sure all RCU operations are complete before proceeding.
  463. */
  464. if (e->icq_cache) {
  465. rcu_barrier();
  466. kmem_cache_destroy(e->icq_cache);
  467. e->icq_cache = NULL;
  468. }
  469. }
  470. EXPORT_SYMBOL_GPL(elv_unregister);
  471. int elevator_switch_mq(struct request_queue *q,
  472. struct elevator_type *new_e)
  473. {
  474. int ret;
  475. lockdep_assert_held(&q->sysfs_lock);
  476. if (q->elevator) {
  477. if (q->elevator->registered)
  478. elv_unregister_queue(q);
  479. ioc_clear_queue(q);
  480. elevator_exit(q, q->elevator);
  481. }
  482. ret = blk_mq_init_sched(q, new_e);
  483. if (ret)
  484. goto out;
  485. if (new_e) {
  486. ret = elv_register_queue(q, true);
  487. if (ret) {
  488. elevator_exit(q, q->elevator);
  489. goto out;
  490. }
  491. }
  492. if (new_e)
  493. blk_add_trace_msg(q, "elv switch: %s", new_e->elevator_name);
  494. else
  495. blk_add_trace_msg(q, "elv switch: none");
  496. out:
  497. return ret;
  498. }
  499. static inline bool elv_support_iosched(struct request_queue *q)
  500. {
  501. if (!queue_is_mq(q) ||
  502. (q->tag_set && (q->tag_set->flags & BLK_MQ_F_NO_SCHED)))
  503. return false;
  504. return true;
  505. }
  506. /*
  507. * For single queue devices, default to using mq-deadline. If we have multiple
  508. * queues or mq-deadline is not available, default to "none".
  509. */
  510. static struct elevator_type *elevator_get_default(struct request_queue *q)
  511. {
  512. if (q->tag_set && q->tag_set->flags & BLK_MQ_F_NO_SCHED_BY_DEFAULT)
  513. return NULL;
  514. if (q->nr_hw_queues != 1)
  515. return NULL;
  516. return elevator_get(q, "mq-deadline", false);
  517. }
  518. /*
  519. * Get the first elevator providing the features required by the request queue.
  520. * Default to "none" if no matching elevator is found.
  521. */
  522. static struct elevator_type *elevator_get_by_features(struct request_queue *q)
  523. {
  524. struct elevator_type *e, *found = NULL;
  525. spin_lock(&elv_list_lock);
  526. list_for_each_entry(e, &elv_list, list) {
  527. if (elv_support_features(e->elevator_features,
  528. q->required_elevator_features)) {
  529. found = e;
  530. break;
  531. }
  532. }
  533. if (found && !try_module_get(found->elevator_owner))
  534. found = NULL;
  535. spin_unlock(&elv_list_lock);
  536. return found;
  537. }
  538. /*
  539. * For a device queue that has no required features, use the default elevator
  540. * settings. Otherwise, use the first elevator available matching the required
  541. * features. If no suitable elevator is find or if the chosen elevator
  542. * initialization fails, fall back to the "none" elevator (no elevator).
  543. */
  544. void elevator_init_mq(struct request_queue *q)
  545. {
  546. struct elevator_type *e;
  547. int err;
  548. if (!elv_support_iosched(q))
  549. return;
  550. WARN_ON_ONCE(blk_queue_registered(q));
  551. if (unlikely(q->elevator))
  552. return;
  553. if (!q->required_elevator_features)
  554. e = elevator_get_default(q);
  555. else
  556. e = elevator_get_by_features(q);
  557. if (!e)
  558. return;
  559. blk_mq_freeze_queue(q);
  560. blk_mq_quiesce_queue(q);
  561. err = blk_mq_init_sched(q, e);
  562. blk_mq_unquiesce_queue(q);
  563. blk_mq_unfreeze_queue(q);
  564. if (err) {
  565. pr_warn("\"%s\" elevator initialization failed, "
  566. "falling back to \"none\"\n", e->elevator_name);
  567. elevator_put(e);
  568. }
  569. }
  570. /*
  571. * switch to new_e io scheduler. be careful not to introduce deadlocks -
  572. * we don't free the old io scheduler, before we have allocated what we
  573. * need for the new one. this way we have a chance of going back to the old
  574. * one, if the new one fails init for some reason.
  575. */
  576. static int elevator_switch(struct request_queue *q, struct elevator_type *new_e)
  577. {
  578. int err;
  579. lockdep_assert_held(&q->sysfs_lock);
  580. blk_mq_freeze_queue(q);
  581. blk_mq_quiesce_queue(q);
  582. err = elevator_switch_mq(q, new_e);
  583. blk_mq_unquiesce_queue(q);
  584. blk_mq_unfreeze_queue(q);
  585. return err;
  586. }
  587. /*
  588. * Switch this queue to the given IO scheduler.
  589. */
  590. static int __elevator_change(struct request_queue *q, const char *name)
  591. {
  592. char elevator_name[ELV_NAME_MAX];
  593. struct elevator_type *e;
  594. /* Make sure queue is not in the middle of being removed */
  595. if (!blk_queue_registered(q))
  596. return -ENOENT;
  597. /*
  598. * Special case for mq, turn off scheduling
  599. */
  600. if (!strncmp(name, "none", 4)) {
  601. if (!q->elevator)
  602. return 0;
  603. return elevator_switch(q, NULL);
  604. }
  605. strlcpy(elevator_name, name, sizeof(elevator_name));
  606. e = elevator_get(q, strstrip(elevator_name), true);
  607. if (!e)
  608. return -EINVAL;
  609. if (q->elevator &&
  610. elevator_match(q->elevator->type, elevator_name, 0)) {
  611. elevator_put(e);
  612. return 0;
  613. }
  614. return elevator_switch(q, e);
  615. }
  616. ssize_t elv_iosched_store(struct request_queue *q, const char *name,
  617. size_t count)
  618. {
  619. int ret;
  620. if (!elv_support_iosched(q))
  621. return count;
  622. ret = __elevator_change(q, name);
  623. if (!ret)
  624. return count;
  625. return ret;
  626. }
  627. ssize_t elv_iosched_show(struct request_queue *q, char *name)
  628. {
  629. struct elevator_queue *e = q->elevator;
  630. struct elevator_type *elv = NULL;
  631. struct elevator_type *__e;
  632. int len = 0;
  633. if (!queue_is_mq(q))
  634. return sprintf(name, "none\n");
  635. if (!q->elevator)
  636. len += sprintf(name+len, "[none] ");
  637. else
  638. elv = e->type;
  639. spin_lock(&elv_list_lock);
  640. list_for_each_entry(__e, &elv_list, list) {
  641. if (elv && elevator_match(elv, __e->elevator_name, 0)) {
  642. len += sprintf(name+len, "[%s] ", elv->elevator_name);
  643. continue;
  644. }
  645. if (elv_support_iosched(q) &&
  646. elevator_match(__e, __e->elevator_name,
  647. q->required_elevator_features))
  648. len += sprintf(name+len, "%s ", __e->elevator_name);
  649. }
  650. spin_unlock(&elv_list_lock);
  651. if (q->elevator)
  652. len += sprintf(name+len, "none");
  653. len += sprintf(len+name, "\n");
  654. return len;
  655. }
  656. struct request *elv_rb_former_request(struct request_queue *q,
  657. struct request *rq)
  658. {
  659. struct rb_node *rbprev = rb_prev(&rq->rb_node);
  660. if (rbprev)
  661. return rb_entry_rq(rbprev);
  662. return NULL;
  663. }
  664. EXPORT_SYMBOL(elv_rb_former_request);
  665. struct request *elv_rb_latter_request(struct request_queue *q,
  666. struct request *rq)
  667. {
  668. struct rb_node *rbnext = rb_next(&rq->rb_node);
  669. if (rbnext)
  670. return rb_entry_rq(rbnext);
  671. return NULL;
  672. }
  673. EXPORT_SYMBOL(elv_rb_latter_request);
  674. static int __init elevator_setup(char *str)
  675. {
  676. pr_warn("Kernel parameter elevator= does not have any effect anymore.\n"
  677. "Please use sysfs to set IO scheduler for individual devices.\n");
  678. return 1;
  679. }
  680. __setup("elevator=", elevator_setup);