queue.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2003 Russell King, All Rights Reserved.
  4. * Copyright 2006-2007 Pierre Ossman
  5. */
  6. #include <linux/slab.h>
  7. #include <linux/module.h>
  8. #include <linux/blkdev.h>
  9. #include <linux/freezer.h>
  10. #include <linux/kthread.h>
  11. #include <linux/scatterlist.h>
  12. #include <linux/dma-mapping.h>
  13. #include <linux/backing-dev.h>
  14. #include <linux/mmc/card.h>
  15. #include <linux/mmc/host.h>
  16. #include "queue.h"
  17. #include "block.h"
  18. #include "core.h"
  19. #include "card.h"
  20. #include "crypto.h"
  21. #include "host.h"
  22. #define MMC_DMA_MAP_MERGE_SEGMENTS 512
  23. static inline bool mmc_cqe_dcmd_busy(struct mmc_queue *mq)
  24. {
  25. /* Allow only 1 DCMD at a time */
  26. return mq->in_flight[MMC_ISSUE_DCMD];
  27. }
  28. void mmc_cqe_check_busy(struct mmc_queue *mq)
  29. {
  30. if ((mq->cqe_busy & MMC_CQE_DCMD_BUSY) && !mmc_cqe_dcmd_busy(mq))
  31. mq->cqe_busy &= ~MMC_CQE_DCMD_BUSY;
  32. mq->cqe_busy &= ~MMC_CQE_QUEUE_FULL;
  33. }
  34. static inline bool mmc_cqe_can_dcmd(struct mmc_host *host)
  35. {
  36. return host->caps2 & MMC_CAP2_CQE_DCMD;
  37. }
  38. static enum mmc_issue_type mmc_cqe_issue_type(struct mmc_host *host,
  39. struct request *req)
  40. {
  41. switch (req_op(req)) {
  42. case REQ_OP_DRV_IN:
  43. case REQ_OP_DRV_OUT:
  44. case REQ_OP_DISCARD:
  45. case REQ_OP_SECURE_ERASE:
  46. return MMC_ISSUE_SYNC;
  47. case REQ_OP_FLUSH:
  48. return mmc_cqe_can_dcmd(host) ? MMC_ISSUE_DCMD : MMC_ISSUE_SYNC;
  49. default:
  50. return MMC_ISSUE_ASYNC;
  51. }
  52. }
  53. enum mmc_issue_type mmc_issue_type(struct mmc_queue *mq, struct request *req)
  54. {
  55. struct mmc_host *host = mq->card->host;
  56. if (mq->use_cqe && !host->hsq_enabled)
  57. return mmc_cqe_issue_type(host, req);
  58. if (req_op(req) == REQ_OP_READ || req_op(req) == REQ_OP_WRITE)
  59. return MMC_ISSUE_ASYNC;
  60. return MMC_ISSUE_SYNC;
  61. }
  62. EXPORT_SYMBOL_GPL(mmc_issue_type);
  63. static void __mmc_cqe_recovery_notifier(struct mmc_queue *mq)
  64. {
  65. if (!mq->recovery_needed) {
  66. mq->recovery_needed = true;
  67. schedule_work(&mq->recovery_work);
  68. }
  69. }
  70. void mmc_cqe_recovery_notifier(struct mmc_request *mrq)
  71. {
  72. struct mmc_queue_req *mqrq = container_of(mrq, struct mmc_queue_req,
  73. brq.mrq);
  74. struct request *req = mmc_queue_req_to_req(mqrq);
  75. struct request_queue *q = req->q;
  76. struct mmc_queue *mq = q->queuedata;
  77. unsigned long flags;
  78. spin_lock_irqsave(&mq->lock, flags);
  79. __mmc_cqe_recovery_notifier(mq);
  80. spin_unlock_irqrestore(&mq->lock, flags);
  81. }
  82. static enum blk_eh_timer_return mmc_cqe_timed_out(struct request *req)
  83. {
  84. struct mmc_queue_req *mqrq = req_to_mmc_queue_req(req);
  85. struct mmc_request *mrq = &mqrq->brq.mrq;
  86. struct mmc_queue *mq = req->q->queuedata;
  87. struct mmc_host *host = mq->card->host;
  88. enum mmc_issue_type issue_type = mmc_issue_type(mq, req);
  89. bool recovery_needed = false;
  90. switch (issue_type) {
  91. case MMC_ISSUE_ASYNC:
  92. case MMC_ISSUE_DCMD:
  93. if (host->cqe_ops->cqe_timeout(host, mrq, &recovery_needed)) {
  94. if (recovery_needed)
  95. mmc_cqe_recovery_notifier(mrq);
  96. return BLK_EH_RESET_TIMER;
  97. }
  98. /* The request has gone already */
  99. return BLK_EH_DONE;
  100. default:
  101. /* Timeout is handled by mmc core */
  102. return BLK_EH_RESET_TIMER;
  103. }
  104. }
  105. static enum blk_eh_timer_return mmc_mq_timed_out(struct request *req,
  106. bool reserved)
  107. {
  108. struct request_queue *q = req->q;
  109. struct mmc_queue *mq = q->queuedata;
  110. struct mmc_card *card = mq->card;
  111. struct mmc_host *host = card->host;
  112. unsigned long flags;
  113. bool ignore_tout;
  114. spin_lock_irqsave(&mq->lock, flags);
  115. ignore_tout = mq->recovery_needed || !mq->use_cqe || host->hsq_enabled;
  116. spin_unlock_irqrestore(&mq->lock, flags);
  117. return ignore_tout ? BLK_EH_RESET_TIMER : mmc_cqe_timed_out(req);
  118. }
  119. static void mmc_mq_recovery_handler(struct work_struct *work)
  120. {
  121. struct mmc_queue *mq = container_of(work, struct mmc_queue,
  122. recovery_work);
  123. struct request_queue *q = mq->queue;
  124. struct mmc_host *host = mq->card->host;
  125. mmc_get_card(mq->card, &mq->ctx);
  126. mq->in_recovery = true;
  127. if (mq->use_cqe && !host->hsq_enabled)
  128. mmc_blk_cqe_recovery(mq);
  129. else
  130. mmc_blk_mq_recovery(mq);
  131. mq->in_recovery = false;
  132. spin_lock_irq(&mq->lock);
  133. mq->recovery_needed = false;
  134. spin_unlock_irq(&mq->lock);
  135. if (host->hsq_enabled)
  136. host->cqe_ops->cqe_recovery_finish(host);
  137. mmc_put_card(mq->card, &mq->ctx);
  138. blk_mq_run_hw_queues(q, true);
  139. }
  140. static struct scatterlist *mmc_alloc_sg(int sg_len, gfp_t gfp)
  141. {
  142. struct scatterlist *sg;
  143. sg = kmalloc_array(sg_len, sizeof(*sg), gfp);
  144. if (sg)
  145. sg_init_table(sg, sg_len);
  146. return sg;
  147. }
  148. static void mmc_queue_setup_discard(struct request_queue *q,
  149. struct mmc_card *card)
  150. {
  151. unsigned max_discard;
  152. max_discard = mmc_calc_max_discard(card);
  153. if (!max_discard)
  154. return;
  155. blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
  156. blk_queue_max_discard_sectors(q, max_discard);
  157. q->limits.discard_granularity = card->pref_erase << 9;
  158. /* granularity must not be greater than max. discard */
  159. if (card->pref_erase > max_discard)
  160. q->limits.discard_granularity = SECTOR_SIZE;
  161. if (mmc_can_secure_erase_trim(card))
  162. blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
  163. }
  164. static unsigned int mmc_get_max_segments(struct mmc_host *host)
  165. {
  166. return host->can_dma_map_merge ? MMC_DMA_MAP_MERGE_SEGMENTS :
  167. host->max_segs;
  168. }
  169. /**
  170. * mmc_init_request() - initialize the MMC-specific per-request data
  171. * @mq: the request queue
  172. * @req: the request
  173. * @gfp: memory allocation policy
  174. */
  175. static int __mmc_init_request(struct mmc_queue *mq, struct request *req,
  176. gfp_t gfp)
  177. {
  178. struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req);
  179. struct mmc_card *card = mq->card;
  180. struct mmc_host *host = card->host;
  181. mq_rq->sg = mmc_alloc_sg(mmc_get_max_segments(host), gfp);
  182. if (!mq_rq->sg)
  183. return -ENOMEM;
  184. return 0;
  185. }
  186. static void mmc_exit_request(struct request_queue *q, struct request *req)
  187. {
  188. struct mmc_queue_req *mq_rq = req_to_mmc_queue_req(req);
  189. kfree(mq_rq->sg);
  190. mq_rq->sg = NULL;
  191. }
  192. static int mmc_mq_init_request(struct blk_mq_tag_set *set, struct request *req,
  193. unsigned int hctx_idx, unsigned int numa_node)
  194. {
  195. return __mmc_init_request(set->driver_data, req, GFP_KERNEL);
  196. }
  197. static void mmc_mq_exit_request(struct blk_mq_tag_set *set, struct request *req,
  198. unsigned int hctx_idx)
  199. {
  200. struct mmc_queue *mq = set->driver_data;
  201. mmc_exit_request(mq->queue, req);
  202. }
  203. static blk_status_t mmc_mq_queue_rq(struct blk_mq_hw_ctx *hctx,
  204. const struct blk_mq_queue_data *bd)
  205. {
  206. struct request *req = bd->rq;
  207. struct request_queue *q = req->q;
  208. struct mmc_queue *mq = q->queuedata;
  209. struct mmc_card *card = mq->card;
  210. struct mmc_host *host = card->host;
  211. enum mmc_issue_type issue_type;
  212. enum mmc_issued issued;
  213. bool get_card, cqe_retune_ok;
  214. int ret;
  215. if (mmc_card_removed(mq->card)) {
  216. req->rq_flags |= RQF_QUIET;
  217. return BLK_STS_IOERR;
  218. }
  219. issue_type = mmc_issue_type(mq, req);
  220. spin_lock_irq(&mq->lock);
  221. if (mq->recovery_needed || mq->busy) {
  222. spin_unlock_irq(&mq->lock);
  223. return BLK_STS_RESOURCE;
  224. }
  225. switch (issue_type) {
  226. case MMC_ISSUE_DCMD:
  227. if (mmc_cqe_dcmd_busy(mq)) {
  228. mq->cqe_busy |= MMC_CQE_DCMD_BUSY;
  229. spin_unlock_irq(&mq->lock);
  230. return BLK_STS_RESOURCE;
  231. }
  232. break;
  233. case MMC_ISSUE_ASYNC:
  234. /*
  235. * For MMC host software queue, we only allow 2 requests in
  236. * flight to avoid a long latency.
  237. */
  238. if (host->hsq_enabled && mq->in_flight[issue_type] > 2) {
  239. spin_unlock_irq(&mq->lock);
  240. return BLK_STS_RESOURCE;
  241. }
  242. break;
  243. default:
  244. /*
  245. * Timeouts are handled by mmc core, and we don't have a host
  246. * API to abort requests, so we can't handle the timeout anyway.
  247. * However, when the timeout happens, blk_mq_complete_request()
  248. * no longer works (to stop the request disappearing under us).
  249. * To avoid racing with that, set a large timeout.
  250. */
  251. req->timeout = 600 * HZ;
  252. break;
  253. }
  254. /* Parallel dispatch of requests is not supported at the moment */
  255. mq->busy = true;
  256. mq->in_flight[issue_type] += 1;
  257. get_card = (mmc_tot_in_flight(mq) == 1);
  258. cqe_retune_ok = (mmc_cqe_qcnt(mq) == 1);
  259. spin_unlock_irq(&mq->lock);
  260. if (!(req->rq_flags & RQF_DONTPREP)) {
  261. req_to_mmc_queue_req(req)->retries = 0;
  262. req->rq_flags |= RQF_DONTPREP;
  263. }
  264. if (get_card)
  265. mmc_get_card(card, &mq->ctx);
  266. if (mq->use_cqe) {
  267. host->retune_now = host->need_retune && cqe_retune_ok &&
  268. !host->hold_retune;
  269. }
  270. blk_mq_start_request(req);
  271. issued = mmc_blk_mq_issue_rq(mq, req);
  272. switch (issued) {
  273. case MMC_REQ_BUSY:
  274. ret = BLK_STS_RESOURCE;
  275. break;
  276. case MMC_REQ_FAILED_TO_START:
  277. ret = BLK_STS_IOERR;
  278. break;
  279. default:
  280. ret = BLK_STS_OK;
  281. break;
  282. }
  283. if (issued != MMC_REQ_STARTED) {
  284. bool put_card = false;
  285. spin_lock_irq(&mq->lock);
  286. mq->in_flight[issue_type] -= 1;
  287. if (mmc_tot_in_flight(mq) == 0)
  288. put_card = true;
  289. mq->busy = false;
  290. spin_unlock_irq(&mq->lock);
  291. if (put_card)
  292. mmc_put_card(card, &mq->ctx);
  293. } else {
  294. WRITE_ONCE(mq->busy, false);
  295. }
  296. return ret;
  297. }
  298. static const struct blk_mq_ops mmc_mq_ops = {
  299. .queue_rq = mmc_mq_queue_rq,
  300. .init_request = mmc_mq_init_request,
  301. .exit_request = mmc_mq_exit_request,
  302. .complete = mmc_blk_mq_complete,
  303. .timeout = mmc_mq_timed_out,
  304. };
  305. static void mmc_setup_queue(struct mmc_queue *mq, struct mmc_card *card)
  306. {
  307. struct mmc_host *host = card->host;
  308. unsigned block_size = 512;
  309. blk_queue_flag_set(QUEUE_FLAG_NONROT, mq->queue);
  310. blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, mq->queue);
  311. if (mmc_can_erase(card))
  312. mmc_queue_setup_discard(mq->queue, card);
  313. if (!mmc_dev(host)->dma_mask || !*mmc_dev(host)->dma_mask)
  314. blk_queue_bounce_limit(mq->queue, BLK_BOUNCE_HIGH);
  315. blk_queue_max_hw_sectors(mq->queue,
  316. min(host->max_blk_count, host->max_req_size / 512));
  317. if (host->can_dma_map_merge)
  318. WARN(!blk_queue_can_use_dma_map_merging(mq->queue,
  319. mmc_dev(host)),
  320. "merging was advertised but not possible");
  321. blk_queue_max_segments(mq->queue, mmc_get_max_segments(host));
  322. if (mmc_card_mmc(card) && card->ext_csd.data_sector_size) {
  323. block_size = card->ext_csd.data_sector_size;
  324. WARN_ON(block_size != 512 && block_size != 4096);
  325. }
  326. blk_queue_logical_block_size(mq->queue, block_size);
  327. /*
  328. * After blk_queue_can_use_dma_map_merging() was called with succeed,
  329. * since it calls blk_queue_virt_boundary(), the mmc should not call
  330. * both blk_queue_max_segment_size().
  331. */
  332. if (!host->can_dma_map_merge)
  333. blk_queue_max_segment_size(mq->queue,
  334. round_down(host->max_seg_size, block_size));
  335. dma_set_max_seg_size(mmc_dev(host), queue_max_segment_size(mq->queue));
  336. INIT_WORK(&mq->recovery_work, mmc_mq_recovery_handler);
  337. INIT_WORK(&mq->complete_work, mmc_blk_mq_complete_work);
  338. mutex_init(&mq->complete_lock);
  339. init_waitqueue_head(&mq->wait);
  340. mmc_crypto_setup_queue(mq->queue, host);
  341. }
  342. static inline bool mmc_merge_capable(struct mmc_host *host)
  343. {
  344. return host->caps2 & MMC_CAP2_MERGE_CAPABLE;
  345. }
  346. /* Set queue depth to get a reasonable value for q->nr_requests */
  347. #define MMC_QUEUE_DEPTH 64
  348. /**
  349. * mmc_init_queue - initialise a queue structure.
  350. * @mq: mmc queue
  351. * @card: mmc card to attach this queue
  352. *
  353. * Initialise a MMC card request queue.
  354. */
  355. int mmc_init_queue(struct mmc_queue *mq, struct mmc_card *card)
  356. {
  357. struct mmc_host *host = card->host;
  358. int ret;
  359. mq->card = card;
  360. mq->use_cqe = host->cqe_enabled;
  361. spin_lock_init(&mq->lock);
  362. memset(&mq->tag_set, 0, sizeof(mq->tag_set));
  363. mq->tag_set.ops = &mmc_mq_ops;
  364. /*
  365. * The queue depth for CQE must match the hardware because the request
  366. * tag is used to index the hardware queue.
  367. */
  368. if (mq->use_cqe && !host->hsq_enabled)
  369. mq->tag_set.queue_depth =
  370. min_t(int, card->ext_csd.cmdq_depth, host->cqe_qdepth);
  371. else
  372. mq->tag_set.queue_depth = MMC_QUEUE_DEPTH;
  373. mq->tag_set.numa_node = NUMA_NO_NODE;
  374. mq->tag_set.flags = BLK_MQ_F_SHOULD_MERGE | BLK_MQ_F_BLOCKING;
  375. mq->tag_set.nr_hw_queues = 1;
  376. mq->tag_set.cmd_size = sizeof(struct mmc_queue_req);
  377. mq->tag_set.driver_data = mq;
  378. /*
  379. * Since blk_mq_alloc_tag_set() calls .init_request() of mmc_mq_ops,
  380. * the host->can_dma_map_merge should be set before to get max_segs
  381. * from mmc_get_max_segments().
  382. */
  383. if (mmc_merge_capable(host) &&
  384. host->max_segs < MMC_DMA_MAP_MERGE_SEGMENTS &&
  385. dma_get_merge_boundary(mmc_dev(host)))
  386. host->can_dma_map_merge = 1;
  387. else
  388. host->can_dma_map_merge = 0;
  389. ret = blk_mq_alloc_tag_set(&mq->tag_set);
  390. if (ret)
  391. return ret;
  392. mq->queue = blk_mq_init_queue(&mq->tag_set);
  393. if (IS_ERR(mq->queue)) {
  394. ret = PTR_ERR(mq->queue);
  395. goto free_tag_set;
  396. }
  397. if (mmc_host_is_spi(host) && host->use_spi_crc)
  398. blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, mq->queue);
  399. mq->queue->queuedata = mq;
  400. blk_queue_rq_timeout(mq->queue, 60 * HZ);
  401. mmc_setup_queue(mq, card);
  402. return 0;
  403. free_tag_set:
  404. blk_mq_free_tag_set(&mq->tag_set);
  405. return ret;
  406. }
  407. void mmc_queue_suspend(struct mmc_queue *mq)
  408. {
  409. blk_mq_quiesce_queue(mq->queue);
  410. /*
  411. * The host remains claimed while there are outstanding requests, so
  412. * simply claiming and releasing here ensures there are none.
  413. */
  414. mmc_claim_host(mq->card->host);
  415. mmc_release_host(mq->card->host);
  416. }
  417. void mmc_queue_resume(struct mmc_queue *mq)
  418. {
  419. blk_mq_unquiesce_queue(mq->queue);
  420. }
  421. void mmc_cleanup_queue(struct mmc_queue *mq)
  422. {
  423. struct request_queue *q = mq->queue;
  424. /*
  425. * The legacy code handled the possibility of being suspended,
  426. * so do that here too.
  427. */
  428. if (blk_queue_quiesced(q))
  429. blk_mq_unquiesce_queue(q);
  430. blk_cleanup_queue(q);
  431. blk_mq_free_tag_set(&mq->tag_set);
  432. /*
  433. * A request can be completed before the next request, potentially
  434. * leaving a complete_work with nothing to do. Such a work item might
  435. * still be queued at this point. Flush it.
  436. */
  437. flush_work(&mq->complete_work);
  438. mq->card = NULL;
  439. }
  440. /*
  441. * Prepare the sg list(s) to be handed of to the host driver
  442. */
  443. unsigned int mmc_queue_map_sg(struct mmc_queue *mq, struct mmc_queue_req *mqrq)
  444. {
  445. struct request *req = mmc_queue_req_to_req(mqrq);
  446. return blk_rq_map_sg(mq->queue, req, mqrq->sg);
  447. }