blk-flush.c 14 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Functions to sequence PREFLUSH and FUA writes.
  4. *
  5. * Copyright (C) 2011 Max Planck Institute for Gravitational Physics
  6. * Copyright (C) 2011 Tejun Heo <tj@kernel.org>
  7. *
  8. * REQ_{PREFLUSH|FUA} requests are decomposed to sequences consisted of three
  9. * optional steps - PREFLUSH, DATA and POSTFLUSH - according to the request
  10. * properties and hardware capability.
  11. *
  12. * If a request doesn't have data, only REQ_PREFLUSH makes sense, which
  13. * indicates a simple flush request. If there is data, REQ_PREFLUSH indicates
  14. * that the device cache should be flushed before the data is executed, and
  15. * REQ_FUA means that the data must be on non-volatile media on request
  16. * completion.
  17. *
  18. * If the device doesn't have writeback cache, PREFLUSH and FUA don't make any
  19. * difference. The requests are either completed immediately if there's no data
  20. * or executed as normal requests otherwise.
  21. *
  22. * If the device has writeback cache and supports FUA, REQ_PREFLUSH is
  23. * translated to PREFLUSH but REQ_FUA is passed down directly with DATA.
  24. *
  25. * If the device has writeback cache and doesn't support FUA, REQ_PREFLUSH
  26. * is translated to PREFLUSH and REQ_FUA to POSTFLUSH.
  27. *
  28. * The actual execution of flush is double buffered. Whenever a request
  29. * needs to execute PRE or POSTFLUSH, it queues at
  30. * fq->flush_queue[fq->flush_pending_idx]. Once certain criteria are met, a
  31. * REQ_OP_FLUSH is issued and the pending_idx is toggled. When the flush
  32. * completes, all the requests which were pending are proceeded to the next
  33. * step. This allows arbitrary merging of different types of PREFLUSH/FUA
  34. * requests.
  35. *
  36. * Currently, the following conditions are used to determine when to issue
  37. * flush.
  38. *
  39. * C1. At any given time, only one flush shall be in progress. This makes
  40. * double buffering sufficient.
  41. *
  42. * C2. Flush is deferred if any request is executing DATA of its sequence.
  43. * This avoids issuing separate POSTFLUSHes for requests which shared
  44. * PREFLUSH.
  45. *
  46. * C3. The second condition is ignored if there is a request which has
  47. * waited longer than FLUSH_PENDING_TIMEOUT. This is to avoid
  48. * starvation in the unlikely case where there are continuous stream of
  49. * FUA (without PREFLUSH) requests.
  50. *
  51. * For devices which support FUA, it isn't clear whether C2 (and thus C3)
  52. * is beneficial.
  53. *
  54. * Note that a sequenced PREFLUSH/FUA request with DATA is completed twice.
  55. * Once while executing DATA and again after the whole sequence is
  56. * complete. The first completion updates the contained bio but doesn't
  57. * finish it so that the bio submitter is notified only after the whole
  58. * sequence is complete. This is implemented by testing RQF_FLUSH_SEQ in
  59. * req_bio_endio().
  60. *
  61. * The above peculiarity requires that each PREFLUSH/FUA request has only one
  62. * bio attached to it, which is guaranteed as they aren't allowed to be
  63. * merged in the usual way.
  64. */
  65. #include <linux/kernel.h>
  66. #include <linux/module.h>
  67. #include <linux/bio.h>
  68. #include <linux/blkdev.h>
  69. #include <linux/gfp.h>
  70. #include <linux/blk-mq.h>
  71. #include <linux/lockdep.h>
  72. #include "blk.h"
  73. #include "blk-mq.h"
  74. #include "blk-mq-tag.h"
  75. #include "blk-mq-sched.h"
  76. /* PREFLUSH/FUA sequences */
  77. enum {
  78. REQ_FSEQ_PREFLUSH = (1 << 0), /* pre-flushing in progress */
  79. REQ_FSEQ_DATA = (1 << 1), /* data write in progress */
  80. REQ_FSEQ_POSTFLUSH = (1 << 2), /* post-flushing in progress */
  81. REQ_FSEQ_DONE = (1 << 3),
  82. REQ_FSEQ_ACTIONS = REQ_FSEQ_PREFLUSH | REQ_FSEQ_DATA |
  83. REQ_FSEQ_POSTFLUSH,
  84. /*
  85. * If flush has been pending longer than the following timeout,
  86. * it's issued even if flush_data requests are still in flight.
  87. */
  88. FLUSH_PENDING_TIMEOUT = 5 * HZ,
  89. };
  90. static void blk_kick_flush(struct request_queue *q,
  91. struct blk_flush_queue *fq, unsigned int flags);
  92. static unsigned int blk_flush_policy(unsigned long fflags, struct request *rq)
  93. {
  94. unsigned int policy = 0;
  95. if (blk_rq_sectors(rq))
  96. policy |= REQ_FSEQ_DATA;
  97. if (fflags & (1UL << QUEUE_FLAG_WC)) {
  98. if (rq->cmd_flags & REQ_PREFLUSH)
  99. policy |= REQ_FSEQ_PREFLUSH;
  100. if (!(fflags & (1UL << QUEUE_FLAG_FUA)) &&
  101. (rq->cmd_flags & REQ_FUA))
  102. policy |= REQ_FSEQ_POSTFLUSH;
  103. }
  104. return policy;
  105. }
  106. static unsigned int blk_flush_cur_seq(struct request *rq)
  107. {
  108. return 1 << ffz(rq->flush.seq);
  109. }
  110. static void blk_flush_restore_request(struct request *rq)
  111. {
  112. /*
  113. * After flush data completion, @rq->bio is %NULL but we need to
  114. * complete the bio again. @rq->biotail is guaranteed to equal the
  115. * original @rq->bio. Restore it.
  116. */
  117. rq->bio = rq->biotail;
  118. /* make @rq a normal request */
  119. rq->rq_flags &= ~RQF_FLUSH_SEQ;
  120. rq->end_io = rq->flush.saved_end_io;
  121. }
  122. static void blk_flush_queue_rq(struct request *rq, bool add_front)
  123. {
  124. blk_mq_add_to_requeue_list(rq, add_front, true);
  125. }
  126. static void blk_account_io_flush(struct request *rq)
  127. {
  128. struct hd_struct *part = &rq->rq_disk->part0;
  129. part_stat_lock();
  130. part_stat_inc(part, ios[STAT_FLUSH]);
  131. part_stat_add(part, nsecs[STAT_FLUSH],
  132. ktime_get_ns() - rq->start_time_ns);
  133. part_stat_unlock();
  134. }
  135. /**
  136. * blk_flush_complete_seq - complete flush sequence
  137. * @rq: PREFLUSH/FUA request being sequenced
  138. * @fq: flush queue
  139. * @seq: sequences to complete (mask of %REQ_FSEQ_*, can be zero)
  140. * @error: whether an error occurred
  141. *
  142. * @rq just completed @seq part of its flush sequence, record the
  143. * completion and trigger the next step.
  144. *
  145. * CONTEXT:
  146. * spin_lock_irq(fq->mq_flush_lock)
  147. */
  148. static void blk_flush_complete_seq(struct request *rq,
  149. struct blk_flush_queue *fq,
  150. unsigned int seq, blk_status_t error)
  151. {
  152. struct request_queue *q = rq->q;
  153. struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
  154. unsigned int cmd_flags;
  155. BUG_ON(rq->flush.seq & seq);
  156. rq->flush.seq |= seq;
  157. cmd_flags = rq->cmd_flags;
  158. if (likely(!error))
  159. seq = blk_flush_cur_seq(rq);
  160. else
  161. seq = REQ_FSEQ_DONE;
  162. switch (seq) {
  163. case REQ_FSEQ_PREFLUSH:
  164. case REQ_FSEQ_POSTFLUSH:
  165. /* queue for flush */
  166. if (list_empty(pending))
  167. fq->flush_pending_since = jiffies;
  168. list_move_tail(&rq->flush.list, pending);
  169. break;
  170. case REQ_FSEQ_DATA:
  171. list_move_tail(&rq->flush.list, &fq->flush_data_in_flight);
  172. blk_flush_queue_rq(rq, true);
  173. break;
  174. case REQ_FSEQ_DONE:
  175. /*
  176. * @rq was previously adjusted by blk_insert_flush() for
  177. * flush sequencing and may already have gone through the
  178. * flush data request completion path. Restore @rq for
  179. * normal completion and end it.
  180. */
  181. BUG_ON(!list_empty(&rq->queuelist));
  182. list_del_init(&rq->flush.list);
  183. blk_flush_restore_request(rq);
  184. blk_mq_end_request(rq, error);
  185. break;
  186. default:
  187. BUG();
  188. }
  189. blk_kick_flush(q, fq, cmd_flags);
  190. }
  191. static void flush_end_io(struct request *flush_rq, blk_status_t error)
  192. {
  193. struct request_queue *q = flush_rq->q;
  194. struct list_head *running;
  195. struct request *rq, *n;
  196. unsigned long flags = 0;
  197. struct blk_flush_queue *fq = blk_get_flush_queue(q, flush_rq->mq_ctx);
  198. /* release the tag's ownership to the req cloned from */
  199. spin_lock_irqsave(&fq->mq_flush_lock, flags);
  200. if (!refcount_dec_and_test(&flush_rq->ref)) {
  201. fq->rq_status = error;
  202. spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
  203. return;
  204. }
  205. blk_account_io_flush(flush_rq);
  206. /*
  207. * Flush request has to be marked as IDLE when it is really ended
  208. * because its .end_io() is called from timeout code path too for
  209. * avoiding use-after-free.
  210. */
  211. WRITE_ONCE(flush_rq->state, MQ_RQ_IDLE);
  212. if (fq->rq_status != BLK_STS_OK) {
  213. error = fq->rq_status;
  214. fq->rq_status = BLK_STS_OK;
  215. }
  216. if (!q->elevator) {
  217. flush_rq->tag = BLK_MQ_NO_TAG;
  218. } else {
  219. blk_mq_put_driver_tag(flush_rq);
  220. flush_rq->internal_tag = BLK_MQ_NO_TAG;
  221. }
  222. running = &fq->flush_queue[fq->flush_running_idx];
  223. BUG_ON(fq->flush_pending_idx == fq->flush_running_idx);
  224. /* account completion of the flush request */
  225. fq->flush_running_idx ^= 1;
  226. /* and push the waiting requests to the next stage */
  227. list_for_each_entry_safe(rq, n, running, flush.list) {
  228. unsigned int seq = blk_flush_cur_seq(rq);
  229. BUG_ON(seq != REQ_FSEQ_PREFLUSH && seq != REQ_FSEQ_POSTFLUSH);
  230. blk_flush_complete_seq(rq, fq, seq, error);
  231. }
  232. spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
  233. }
  234. bool is_flush_rq(struct request *rq)
  235. {
  236. return rq->end_io == flush_end_io;
  237. }
  238. /**
  239. * blk_kick_flush - consider issuing flush request
  240. * @q: request_queue being kicked
  241. * @fq: flush queue
  242. * @flags: cmd_flags of the original request
  243. *
  244. * Flush related states of @q have changed, consider issuing flush request.
  245. * Please read the comment at the top of this file for more info.
  246. *
  247. * CONTEXT:
  248. * spin_lock_irq(fq->mq_flush_lock)
  249. *
  250. */
  251. static void blk_kick_flush(struct request_queue *q, struct blk_flush_queue *fq,
  252. unsigned int flags)
  253. {
  254. struct list_head *pending = &fq->flush_queue[fq->flush_pending_idx];
  255. struct request *first_rq =
  256. list_first_entry(pending, struct request, flush.list);
  257. struct request *flush_rq = fq->flush_rq;
  258. /* C1 described at the top of this file */
  259. if (fq->flush_pending_idx != fq->flush_running_idx || list_empty(pending))
  260. return;
  261. /* C2 and C3 */
  262. if (!list_empty(&fq->flush_data_in_flight) &&
  263. time_before(jiffies,
  264. fq->flush_pending_since + FLUSH_PENDING_TIMEOUT))
  265. return;
  266. /*
  267. * Issue flush and toggle pending_idx. This makes pending_idx
  268. * different from running_idx, which means flush is in flight.
  269. */
  270. fq->flush_pending_idx ^= 1;
  271. blk_rq_init(q, flush_rq);
  272. /*
  273. * In case of none scheduler, borrow tag from the first request
  274. * since they can't be in flight at the same time. And acquire
  275. * the tag's ownership for flush req.
  276. *
  277. * In case of IO scheduler, flush rq need to borrow scheduler tag
  278. * just for cheating put/get driver tag.
  279. */
  280. flush_rq->mq_ctx = first_rq->mq_ctx;
  281. flush_rq->mq_hctx = first_rq->mq_hctx;
  282. if (!q->elevator) {
  283. flush_rq->tag = first_rq->tag;
  284. /*
  285. * We borrow data request's driver tag, so have to mark
  286. * this flush request as INFLIGHT for avoiding double
  287. * account of this driver tag
  288. */
  289. flush_rq->rq_flags |= RQF_MQ_INFLIGHT;
  290. } else
  291. flush_rq->internal_tag = first_rq->internal_tag;
  292. flush_rq->cmd_flags = REQ_OP_FLUSH | REQ_PREFLUSH;
  293. flush_rq->cmd_flags |= (flags & REQ_DRV) | (flags & REQ_FAILFAST_MASK);
  294. flush_rq->rq_flags |= RQF_FLUSH_SEQ;
  295. flush_rq->rq_disk = first_rq->rq_disk;
  296. flush_rq->end_io = flush_end_io;
  297. /*
  298. * Order WRITE ->end_io and WRITE rq->ref, and its pair is the one
  299. * implied in refcount_inc_not_zero() called from
  300. * blk_mq_find_and_get_req(), which orders WRITE/READ flush_rq->ref
  301. * and READ flush_rq->end_io
  302. */
  303. smp_wmb();
  304. refcount_set(&flush_rq->ref, 1);
  305. blk_flush_queue_rq(flush_rq, false);
  306. }
  307. static void mq_flush_data_end_io(struct request *rq, blk_status_t error)
  308. {
  309. struct request_queue *q = rq->q;
  310. struct blk_mq_hw_ctx *hctx = rq->mq_hctx;
  311. struct blk_mq_ctx *ctx = rq->mq_ctx;
  312. unsigned long flags;
  313. struct blk_flush_queue *fq = blk_get_flush_queue(q, ctx);
  314. if (q->elevator) {
  315. WARN_ON(rq->tag < 0);
  316. blk_mq_put_driver_tag(rq);
  317. }
  318. /*
  319. * After populating an empty queue, kick it to avoid stall. Read
  320. * the comment in flush_end_io().
  321. */
  322. spin_lock_irqsave(&fq->mq_flush_lock, flags);
  323. blk_flush_complete_seq(rq, fq, REQ_FSEQ_DATA, error);
  324. spin_unlock_irqrestore(&fq->mq_flush_lock, flags);
  325. blk_mq_sched_restart(hctx);
  326. }
  327. /**
  328. * blk_insert_flush - insert a new PREFLUSH/FUA request
  329. * @rq: request to insert
  330. *
  331. * To be called from __elv_add_request() for %ELEVATOR_INSERT_FLUSH insertions.
  332. * or __blk_mq_run_hw_queue() to dispatch request.
  333. * @rq is being submitted. Analyze what needs to be done and put it on the
  334. * right queue.
  335. */
  336. void blk_insert_flush(struct request *rq)
  337. {
  338. struct request_queue *q = rq->q;
  339. unsigned long fflags = q->queue_flags; /* may change, cache */
  340. unsigned int policy = blk_flush_policy(fflags, rq);
  341. struct blk_flush_queue *fq = blk_get_flush_queue(q, rq->mq_ctx);
  342. /*
  343. * @policy now records what operations need to be done. Adjust
  344. * REQ_PREFLUSH and FUA for the driver.
  345. */
  346. rq->cmd_flags &= ~REQ_PREFLUSH;
  347. if (!(fflags & (1UL << QUEUE_FLAG_FUA)))
  348. rq->cmd_flags &= ~REQ_FUA;
  349. /*
  350. * REQ_PREFLUSH|REQ_FUA implies REQ_SYNC, so if we clear any
  351. * of those flags, we have to set REQ_SYNC to avoid skewing
  352. * the request accounting.
  353. */
  354. rq->cmd_flags |= REQ_SYNC;
  355. /*
  356. * An empty flush handed down from a stacking driver may
  357. * translate into nothing if the underlying device does not
  358. * advertise a write-back cache. In this case, simply
  359. * complete the request.
  360. */
  361. if (!policy) {
  362. blk_mq_end_request(rq, 0);
  363. return;
  364. }
  365. BUG_ON(rq->bio != rq->biotail); /*assumes zero or single bio rq */
  366. /*
  367. * If there's data but flush is not necessary, the request can be
  368. * processed directly without going through flush machinery. Queue
  369. * for normal execution.
  370. */
  371. if ((policy & REQ_FSEQ_DATA) &&
  372. !(policy & (REQ_FSEQ_PREFLUSH | REQ_FSEQ_POSTFLUSH))) {
  373. blk_mq_request_bypass_insert(rq, false, false);
  374. return;
  375. }
  376. /*
  377. * @rq should go through flush machinery. Mark it part of flush
  378. * sequence and submit for further processing.
  379. */
  380. memset(&rq->flush, 0, sizeof(rq->flush));
  381. INIT_LIST_HEAD(&rq->flush.list);
  382. rq->rq_flags |= RQF_FLUSH_SEQ;
  383. rq->flush.saved_end_io = rq->end_io; /* Usually NULL */
  384. rq->end_io = mq_flush_data_end_io;
  385. spin_lock_irq(&fq->mq_flush_lock);
  386. blk_flush_complete_seq(rq, fq, REQ_FSEQ_ACTIONS & ~policy, 0);
  387. spin_unlock_irq(&fq->mq_flush_lock);
  388. }
  389. /**
  390. * blkdev_issue_flush - queue a flush
  391. * @bdev: blockdev to issue flush for
  392. * @gfp_mask: memory allocation flags (for bio_alloc)
  393. *
  394. * Description:
  395. * Issue a flush for the block device in question.
  396. */
  397. int blkdev_issue_flush(struct block_device *bdev, gfp_t gfp_mask)
  398. {
  399. struct bio *bio;
  400. int ret = 0;
  401. bio = bio_alloc(gfp_mask, 0);
  402. bio_set_dev(bio, bdev);
  403. bio->bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
  404. ret = submit_bio_wait(bio);
  405. bio_put(bio);
  406. return ret;
  407. }
  408. EXPORT_SYMBOL(blkdev_issue_flush);
  409. struct blk_flush_queue *blk_alloc_flush_queue(int node, int cmd_size,
  410. gfp_t flags)
  411. {
  412. struct blk_flush_queue *fq;
  413. int rq_sz = sizeof(struct request);
  414. fq = kzalloc_node(sizeof(*fq), flags, node);
  415. if (!fq)
  416. goto fail;
  417. spin_lock_init(&fq->mq_flush_lock);
  418. rq_sz = round_up(rq_sz + cmd_size, cache_line_size());
  419. fq->flush_rq = kzalloc_node(rq_sz, flags, node);
  420. if (!fq->flush_rq)
  421. goto fail_rq;
  422. INIT_LIST_HEAD(&fq->flush_queue[0]);
  423. INIT_LIST_HEAD(&fq->flush_queue[1]);
  424. INIT_LIST_HEAD(&fq->flush_data_in_flight);
  425. lockdep_register_key(&fq->key);
  426. lockdep_set_class(&fq->mq_flush_lock, &fq->key);
  427. return fq;
  428. fail_rq:
  429. kfree(fq);
  430. fail:
  431. return NULL;
  432. }
  433. void blk_free_flush_queue(struct blk_flush_queue *fq)
  434. {
  435. /* bio based request queue hasn't flush queue */
  436. if (!fq)
  437. return;
  438. lockdep_unregister_key(&fq->key);
  439. kfree(fq->flush_rq);
  440. kfree(fq);
  441. }