md-multipath.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * multipath.c : Multiple Devices driver for Linux
  4. *
  5. * Copyright (C) 1999, 2000, 2001 Ingo Molnar, Red Hat
  6. *
  7. * Copyright (C) 1996, 1997, 1998 Ingo Molnar, Miguel de Icaza, Gadi Oxman
  8. *
  9. * MULTIPATH management functions.
  10. *
  11. * derived from raid1.c.
  12. */
  13. #include <linux/blkdev.h>
  14. #include <linux/module.h>
  15. #include <linux/raid/md_u.h>
  16. #include <linux/seq_file.h>
  17. #include <linux/slab.h>
  18. #include "md.h"
  19. #include "md-multipath.h"
  20. #define MAX_WORK_PER_DISK 128
  21. #define NR_RESERVED_BUFS 32
  22. static int multipath_map (struct mpconf *conf)
  23. {
  24. int i, disks = conf->raid_disks;
  25. /*
  26. * Later we do read balancing on the read side
  27. * now we use the first available disk.
  28. */
  29. rcu_read_lock();
  30. for (i = 0; i < disks; i++) {
  31. struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev);
  32. if (rdev && test_bit(In_sync, &rdev->flags) &&
  33. !test_bit(Faulty, &rdev->flags)) {
  34. atomic_inc(&rdev->nr_pending);
  35. rcu_read_unlock();
  36. return i;
  37. }
  38. }
  39. rcu_read_unlock();
  40. pr_crit_ratelimited("multipath_map(): no more operational IO paths?\n");
  41. return (-1);
  42. }
  43. static void multipath_reschedule_retry (struct multipath_bh *mp_bh)
  44. {
  45. unsigned long flags;
  46. struct mddev *mddev = mp_bh->mddev;
  47. struct mpconf *conf = mddev->private;
  48. spin_lock_irqsave(&conf->device_lock, flags);
  49. list_add(&mp_bh->retry_list, &conf->retry_list);
  50. spin_unlock_irqrestore(&conf->device_lock, flags);
  51. md_wakeup_thread(mddev->thread);
  52. }
  53. /*
  54. * multipath_end_bh_io() is called when we have finished servicing a multipathed
  55. * operation and are ready to return a success/failure code to the buffer
  56. * cache layer.
  57. */
  58. static void multipath_end_bh_io(struct multipath_bh *mp_bh, blk_status_t status)
  59. {
  60. struct bio *bio = mp_bh->master_bio;
  61. struct mpconf *conf = mp_bh->mddev->private;
  62. bio->bi_status = status;
  63. bio_endio(bio);
  64. mempool_free(mp_bh, &conf->pool);
  65. }
  66. static void multipath_end_request(struct bio *bio)
  67. {
  68. struct multipath_bh *mp_bh = bio->bi_private;
  69. struct mpconf *conf = mp_bh->mddev->private;
  70. struct md_rdev *rdev = conf->multipaths[mp_bh->path].rdev;
  71. if (!bio->bi_status)
  72. multipath_end_bh_io(mp_bh, 0);
  73. else if (!(bio->bi_opf & REQ_RAHEAD)) {
  74. /*
  75. * oops, IO error:
  76. */
  77. char b[BDEVNAME_SIZE];
  78. md_error (mp_bh->mddev, rdev);
  79. pr_info("multipath: %s: rescheduling sector %llu\n",
  80. bdevname(rdev->bdev,b),
  81. (unsigned long long)bio->bi_iter.bi_sector);
  82. multipath_reschedule_retry(mp_bh);
  83. } else
  84. multipath_end_bh_io(mp_bh, bio->bi_status);
  85. rdev_dec_pending(rdev, conf->mddev);
  86. }
  87. static bool multipath_make_request(struct mddev *mddev, struct bio * bio)
  88. {
  89. struct mpconf *conf = mddev->private;
  90. struct multipath_bh * mp_bh;
  91. struct multipath_info *multipath;
  92. if (unlikely(bio->bi_opf & REQ_PREFLUSH)
  93. && md_flush_request(mddev, bio))
  94. return true;
  95. mp_bh = mempool_alloc(&conf->pool, GFP_NOIO);
  96. mp_bh->master_bio = bio;
  97. mp_bh->mddev = mddev;
  98. mp_bh->path = multipath_map(conf);
  99. if (mp_bh->path < 0) {
  100. bio_io_error(bio);
  101. mempool_free(mp_bh, &conf->pool);
  102. return true;
  103. }
  104. multipath = conf->multipaths + mp_bh->path;
  105. bio_init(&mp_bh->bio, NULL, 0);
  106. __bio_clone_fast(&mp_bh->bio, bio);
  107. mp_bh->bio.bi_iter.bi_sector += multipath->rdev->data_offset;
  108. bio_set_dev(&mp_bh->bio, multipath->rdev->bdev);
  109. mp_bh->bio.bi_opf |= REQ_FAILFAST_TRANSPORT;
  110. mp_bh->bio.bi_end_io = multipath_end_request;
  111. mp_bh->bio.bi_private = mp_bh;
  112. mddev_check_writesame(mddev, &mp_bh->bio);
  113. mddev_check_write_zeroes(mddev, &mp_bh->bio);
  114. submit_bio_noacct(&mp_bh->bio);
  115. return true;
  116. }
  117. static void multipath_status(struct seq_file *seq, struct mddev *mddev)
  118. {
  119. struct mpconf *conf = mddev->private;
  120. int i;
  121. seq_printf (seq, " [%d/%d] [", conf->raid_disks,
  122. conf->raid_disks - mddev->degraded);
  123. rcu_read_lock();
  124. for (i = 0; i < conf->raid_disks; i++) {
  125. struct md_rdev *rdev = rcu_dereference(conf->multipaths[i].rdev);
  126. seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
  127. }
  128. rcu_read_unlock();
  129. seq_putc(seq, ']');
  130. }
  131. /*
  132. * Careful, this can execute in IRQ contexts as well!
  133. */
  134. static void multipath_error (struct mddev *mddev, struct md_rdev *rdev)
  135. {
  136. struct mpconf *conf = mddev->private;
  137. char b[BDEVNAME_SIZE];
  138. if (conf->raid_disks - mddev->degraded <= 1) {
  139. /*
  140. * Uh oh, we can do nothing if this is our last path, but
  141. * first check if this is a queued request for a device
  142. * which has just failed.
  143. */
  144. pr_warn("multipath: only one IO path left and IO error.\n");
  145. /* leave it active... it's all we have */
  146. return;
  147. }
  148. /*
  149. * Mark disk as unusable
  150. */
  151. if (test_and_clear_bit(In_sync, &rdev->flags)) {
  152. unsigned long flags;
  153. spin_lock_irqsave(&conf->device_lock, flags);
  154. mddev->degraded++;
  155. spin_unlock_irqrestore(&conf->device_lock, flags);
  156. }
  157. set_bit(Faulty, &rdev->flags);
  158. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  159. pr_err("multipath: IO failure on %s, disabling IO path.\n"
  160. "multipath: Operation continuing on %d IO paths.\n",
  161. bdevname(rdev->bdev, b),
  162. conf->raid_disks - mddev->degraded);
  163. }
  164. static void print_multipath_conf (struct mpconf *conf)
  165. {
  166. int i;
  167. struct multipath_info *tmp;
  168. pr_debug("MULTIPATH conf printout:\n");
  169. if (!conf) {
  170. pr_debug("(conf==NULL)\n");
  171. return;
  172. }
  173. pr_debug(" --- wd:%d rd:%d\n", conf->raid_disks - conf->mddev->degraded,
  174. conf->raid_disks);
  175. for (i = 0; i < conf->raid_disks; i++) {
  176. char b[BDEVNAME_SIZE];
  177. tmp = conf->multipaths + i;
  178. if (tmp->rdev)
  179. pr_debug(" disk%d, o:%d, dev:%s\n",
  180. i,!test_bit(Faulty, &tmp->rdev->flags),
  181. bdevname(tmp->rdev->bdev,b));
  182. }
  183. }
  184. static int multipath_add_disk(struct mddev *mddev, struct md_rdev *rdev)
  185. {
  186. struct mpconf *conf = mddev->private;
  187. int err = -EEXIST;
  188. int path;
  189. struct multipath_info *p;
  190. int first = 0;
  191. int last = mddev->raid_disks - 1;
  192. if (rdev->raid_disk >= 0)
  193. first = last = rdev->raid_disk;
  194. print_multipath_conf(conf);
  195. for (path = first; path <= last; path++)
  196. if ((p=conf->multipaths+path)->rdev == NULL) {
  197. disk_stack_limits(mddev->gendisk, rdev->bdev,
  198. rdev->data_offset << 9);
  199. err = md_integrity_add_rdev(rdev, mddev);
  200. if (err)
  201. break;
  202. spin_lock_irq(&conf->device_lock);
  203. mddev->degraded--;
  204. rdev->raid_disk = path;
  205. set_bit(In_sync, &rdev->flags);
  206. spin_unlock_irq(&conf->device_lock);
  207. rcu_assign_pointer(p->rdev, rdev);
  208. err = 0;
  209. break;
  210. }
  211. print_multipath_conf(conf);
  212. return err;
  213. }
  214. static int multipath_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
  215. {
  216. struct mpconf *conf = mddev->private;
  217. int err = 0;
  218. int number = rdev->raid_disk;
  219. struct multipath_info *p = conf->multipaths + number;
  220. print_multipath_conf(conf);
  221. if (rdev == p->rdev) {
  222. if (test_bit(In_sync, &rdev->flags) ||
  223. atomic_read(&rdev->nr_pending)) {
  224. pr_warn("hot-remove-disk, slot %d is identified but is still operational!\n", number);
  225. err = -EBUSY;
  226. goto abort;
  227. }
  228. p->rdev = NULL;
  229. if (!test_bit(RemoveSynchronized, &rdev->flags)) {
  230. synchronize_rcu();
  231. if (atomic_read(&rdev->nr_pending)) {
  232. /* lost the race, try later */
  233. err = -EBUSY;
  234. p->rdev = rdev;
  235. goto abort;
  236. }
  237. }
  238. err = md_integrity_register(mddev);
  239. }
  240. abort:
  241. print_multipath_conf(conf);
  242. return err;
  243. }
  244. /*
  245. * This is a kernel thread which:
  246. *
  247. * 1. Retries failed read operations on working multipaths.
  248. * 2. Updates the raid superblock when problems encounter.
  249. * 3. Performs writes following reads for array syncronising.
  250. */
  251. static void multipathd(struct md_thread *thread)
  252. {
  253. struct mddev *mddev = thread->mddev;
  254. struct multipath_bh *mp_bh;
  255. struct bio *bio;
  256. unsigned long flags;
  257. struct mpconf *conf = mddev->private;
  258. struct list_head *head = &conf->retry_list;
  259. md_check_recovery(mddev);
  260. for (;;) {
  261. char b[BDEVNAME_SIZE];
  262. spin_lock_irqsave(&conf->device_lock, flags);
  263. if (list_empty(head))
  264. break;
  265. mp_bh = list_entry(head->prev, struct multipath_bh, retry_list);
  266. list_del(head->prev);
  267. spin_unlock_irqrestore(&conf->device_lock, flags);
  268. bio = &mp_bh->bio;
  269. bio->bi_iter.bi_sector = mp_bh->master_bio->bi_iter.bi_sector;
  270. if ((mp_bh->path = multipath_map (conf))<0) {
  271. pr_err("multipath: %s: unrecoverable IO read error for block %llu\n",
  272. bio_devname(bio, b),
  273. (unsigned long long)bio->bi_iter.bi_sector);
  274. multipath_end_bh_io(mp_bh, BLK_STS_IOERR);
  275. } else {
  276. pr_err("multipath: %s: redirecting sector %llu to another IO path\n",
  277. bio_devname(bio, b),
  278. (unsigned long long)bio->bi_iter.bi_sector);
  279. *bio = *(mp_bh->master_bio);
  280. bio->bi_iter.bi_sector +=
  281. conf->multipaths[mp_bh->path].rdev->data_offset;
  282. bio_set_dev(bio, conf->multipaths[mp_bh->path].rdev->bdev);
  283. bio->bi_opf |= REQ_FAILFAST_TRANSPORT;
  284. bio->bi_end_io = multipath_end_request;
  285. bio->bi_private = mp_bh;
  286. submit_bio_noacct(bio);
  287. }
  288. }
  289. spin_unlock_irqrestore(&conf->device_lock, flags);
  290. }
  291. static sector_t multipath_size(struct mddev *mddev, sector_t sectors, int raid_disks)
  292. {
  293. WARN_ONCE(sectors || raid_disks,
  294. "%s does not support generic reshape\n", __func__);
  295. return mddev->dev_sectors;
  296. }
  297. static int multipath_run (struct mddev *mddev)
  298. {
  299. struct mpconf *conf;
  300. int disk_idx;
  301. struct multipath_info *disk;
  302. struct md_rdev *rdev;
  303. int working_disks;
  304. int ret;
  305. if (md_check_no_bitmap(mddev))
  306. return -EINVAL;
  307. if (mddev->level != LEVEL_MULTIPATH) {
  308. pr_warn("multipath: %s: raid level not set to multipath IO (%d)\n",
  309. mdname(mddev), mddev->level);
  310. goto out;
  311. }
  312. /*
  313. * copy the already verified devices into our private MULTIPATH
  314. * bookkeeping area. [whatever we allocate in multipath_run(),
  315. * should be freed in multipath_free()]
  316. */
  317. conf = kzalloc(sizeof(struct mpconf), GFP_KERNEL);
  318. mddev->private = conf;
  319. if (!conf)
  320. goto out;
  321. conf->multipaths = kcalloc(mddev->raid_disks,
  322. sizeof(struct multipath_info),
  323. GFP_KERNEL);
  324. if (!conf->multipaths)
  325. goto out_free_conf;
  326. working_disks = 0;
  327. rdev_for_each(rdev, mddev) {
  328. disk_idx = rdev->raid_disk;
  329. if (disk_idx < 0 ||
  330. disk_idx >= mddev->raid_disks)
  331. continue;
  332. disk = conf->multipaths + disk_idx;
  333. disk->rdev = rdev;
  334. disk_stack_limits(mddev->gendisk, rdev->bdev,
  335. rdev->data_offset << 9);
  336. if (!test_bit(Faulty, &rdev->flags))
  337. working_disks++;
  338. }
  339. conf->raid_disks = mddev->raid_disks;
  340. conf->mddev = mddev;
  341. spin_lock_init(&conf->device_lock);
  342. INIT_LIST_HEAD(&conf->retry_list);
  343. if (!working_disks) {
  344. pr_warn("multipath: no operational IO paths for %s\n",
  345. mdname(mddev));
  346. goto out_free_conf;
  347. }
  348. mddev->degraded = conf->raid_disks - working_disks;
  349. ret = mempool_init_kmalloc_pool(&conf->pool, NR_RESERVED_BUFS,
  350. sizeof(struct multipath_bh));
  351. if (ret)
  352. goto out_free_conf;
  353. mddev->thread = md_register_thread(multipathd, mddev,
  354. "multipath");
  355. if (!mddev->thread)
  356. goto out_free_conf;
  357. pr_info("multipath: array %s active with %d out of %d IO paths\n",
  358. mdname(mddev), conf->raid_disks - mddev->degraded,
  359. mddev->raid_disks);
  360. /*
  361. * Ok, everything is just fine now
  362. */
  363. md_set_array_sectors(mddev, multipath_size(mddev, 0, 0));
  364. if (md_integrity_register(mddev))
  365. goto out_free_conf;
  366. return 0;
  367. out_free_conf:
  368. mempool_exit(&conf->pool);
  369. kfree(conf->multipaths);
  370. kfree(conf);
  371. mddev->private = NULL;
  372. out:
  373. return -EIO;
  374. }
  375. static void multipath_free(struct mddev *mddev, void *priv)
  376. {
  377. struct mpconf *conf = priv;
  378. mempool_exit(&conf->pool);
  379. kfree(conf->multipaths);
  380. kfree(conf);
  381. }
  382. static struct md_personality multipath_personality =
  383. {
  384. .name = "multipath",
  385. .level = LEVEL_MULTIPATH,
  386. .owner = THIS_MODULE,
  387. .make_request = multipath_make_request,
  388. .run = multipath_run,
  389. .free = multipath_free,
  390. .status = multipath_status,
  391. .error_handler = multipath_error,
  392. .hot_add_disk = multipath_add_disk,
  393. .hot_remove_disk= multipath_remove_disk,
  394. .size = multipath_size,
  395. };
  396. static int __init multipath_init (void)
  397. {
  398. return register_md_personality (&multipath_personality);
  399. }
  400. static void __exit multipath_exit (void)
  401. {
  402. unregister_md_personality (&multipath_personality);
  403. }
  404. module_init(multipath_init);
  405. module_exit(multipath_exit);
  406. MODULE_LICENSE("GPL");
  407. MODULE_DESCRIPTION("simple multi-path personality for MD");
  408. MODULE_ALIAS("md-personality-7"); /* MULTIPATH */
  409. MODULE_ALIAS("md-multipath");
  410. MODULE_ALIAS("md-level--4");