pblk-read.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2016 CNEX Labs
  4. * Initial release: Javier Gonzalez <javier@cnexlabs.com>
  5. * Matias Bjorling <matias@cnexlabs.com>
  6. *
  7. * This program is free software; you can redistribute it and/or
  8. * modify it under the terms of the GNU General Public License version
  9. * 2 as published by the Free Software Foundation.
  10. *
  11. * This program is distributed in the hope that it will be useful, but
  12. * WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * General Public License for more details.
  15. *
  16. * pblk-read.c - pblk's read path
  17. */
  18. #include "pblk.h"
  19. /*
  20. * There is no guarantee that the value read from cache has not been updated and
  21. * resides at another location in the cache. We guarantee though that if the
  22. * value is read from the cache, it belongs to the mapped lba. In order to
  23. * guarantee and order between writes and reads are ordered, a flush must be
  24. * issued.
  25. */
  26. static int pblk_read_from_cache(struct pblk *pblk, struct bio *bio,
  27. sector_t lba, struct ppa_addr ppa)
  28. {
  29. #ifdef CONFIG_NVM_PBLK_DEBUG
  30. /* Callers must ensure that the ppa points to a cache address */
  31. BUG_ON(pblk_ppa_empty(ppa));
  32. BUG_ON(!pblk_addr_in_cache(ppa));
  33. #endif
  34. return pblk_rb_copy_to_bio(&pblk->rwb, bio, lba, ppa);
  35. }
  36. static int pblk_read_ppalist_rq(struct pblk *pblk, struct nvm_rq *rqd,
  37. struct bio *bio, sector_t blba,
  38. bool *from_cache)
  39. {
  40. void *meta_list = rqd->meta_list;
  41. int nr_secs, i;
  42. retry:
  43. nr_secs = pblk_lookup_l2p_seq(pblk, rqd->ppa_list, blba, rqd->nr_ppas,
  44. from_cache);
  45. if (!*from_cache)
  46. goto end;
  47. for (i = 0; i < nr_secs; i++) {
  48. struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
  49. sector_t lba = blba + i;
  50. if (pblk_ppa_empty(rqd->ppa_list[i])) {
  51. __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
  52. meta->lba = addr_empty;
  53. } else if (pblk_addr_in_cache(rqd->ppa_list[i])) {
  54. /*
  55. * Try to read from write buffer. The address is later
  56. * checked on the write buffer to prevent retrieving
  57. * overwritten data.
  58. */
  59. if (!pblk_read_from_cache(pblk, bio, lba,
  60. rqd->ppa_list[i])) {
  61. if (i == 0) {
  62. /*
  63. * We didn't call with bio_advance()
  64. * yet, so we can just retry.
  65. */
  66. goto retry;
  67. } else {
  68. /*
  69. * We already call bio_advance()
  70. * so we cannot retry and we need
  71. * to quit that function in order
  72. * to allow caller to handle the bio
  73. * splitting in the current sector
  74. * position.
  75. */
  76. nr_secs = i;
  77. goto end;
  78. }
  79. }
  80. meta->lba = cpu_to_le64(lba);
  81. #ifdef CONFIG_NVM_PBLK_DEBUG
  82. atomic_long_inc(&pblk->cache_reads);
  83. #endif
  84. }
  85. bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
  86. }
  87. end:
  88. if (pblk_io_aligned(pblk, nr_secs))
  89. rqd->is_seq = 1;
  90. #ifdef CONFIG_NVM_PBLK_DEBUG
  91. atomic_long_add(nr_secs, &pblk->inflight_reads);
  92. #endif
  93. return nr_secs;
  94. }
  95. static void pblk_read_check_seq(struct pblk *pblk, struct nvm_rq *rqd,
  96. sector_t blba)
  97. {
  98. void *meta_list = rqd->meta_list;
  99. int nr_lbas = rqd->nr_ppas;
  100. int i;
  101. if (!pblk_is_oob_meta_supported(pblk))
  102. return;
  103. for (i = 0; i < nr_lbas; i++) {
  104. struct pblk_sec_meta *meta = pblk_get_meta(pblk, meta_list, i);
  105. u64 lba = le64_to_cpu(meta->lba);
  106. if (lba == ADDR_EMPTY)
  107. continue;
  108. if (lba != blba + i) {
  109. #ifdef CONFIG_NVM_PBLK_DEBUG
  110. struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
  111. print_ppa(pblk, &ppa_list[i], "seq", i);
  112. #endif
  113. pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
  114. lba, (u64)blba + i);
  115. WARN_ON(1);
  116. }
  117. }
  118. }
  119. /*
  120. * There can be holes in the lba list.
  121. */
  122. static void pblk_read_check_rand(struct pblk *pblk, struct nvm_rq *rqd,
  123. u64 *lba_list, int nr_lbas)
  124. {
  125. void *meta_lba_list = rqd->meta_list;
  126. int i, j;
  127. if (!pblk_is_oob_meta_supported(pblk))
  128. return;
  129. for (i = 0, j = 0; i < nr_lbas; i++) {
  130. struct pblk_sec_meta *meta = pblk_get_meta(pblk,
  131. meta_lba_list, j);
  132. u64 lba = lba_list[i];
  133. u64 meta_lba;
  134. if (lba == ADDR_EMPTY)
  135. continue;
  136. meta_lba = le64_to_cpu(meta->lba);
  137. if (lba != meta_lba) {
  138. #ifdef CONFIG_NVM_PBLK_DEBUG
  139. struct ppa_addr *ppa_list = nvm_rq_to_ppa_list(rqd);
  140. print_ppa(pblk, &ppa_list[j], "rnd", j);
  141. #endif
  142. pblk_err(pblk, "corrupted read LBA (%llu/%llu)\n",
  143. meta_lba, lba);
  144. WARN_ON(1);
  145. }
  146. j++;
  147. }
  148. WARN_ONCE(j != rqd->nr_ppas, "pblk: corrupted random request\n");
  149. }
  150. static void pblk_end_user_read(struct bio *bio, int error)
  151. {
  152. if (error && error != NVM_RSP_WARN_HIGHECC)
  153. bio_io_error(bio);
  154. else
  155. bio_endio(bio);
  156. }
  157. static void __pblk_end_io_read(struct pblk *pblk, struct nvm_rq *rqd,
  158. bool put_line)
  159. {
  160. struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
  161. struct bio *int_bio = rqd->bio;
  162. unsigned long start_time = r_ctx->start_time;
  163. bio_end_io_acct(int_bio, start_time);
  164. if (rqd->error)
  165. pblk_log_read_err(pblk, rqd);
  166. pblk_read_check_seq(pblk, rqd, r_ctx->lba);
  167. bio_put(int_bio);
  168. if (put_line)
  169. pblk_rq_to_line_put(pblk, rqd);
  170. #ifdef CONFIG_NVM_PBLK_DEBUG
  171. atomic_long_add(rqd->nr_ppas, &pblk->sync_reads);
  172. atomic_long_sub(rqd->nr_ppas, &pblk->inflight_reads);
  173. #endif
  174. pblk_free_rqd(pblk, rqd, PBLK_READ);
  175. atomic_dec(&pblk->inflight_io);
  176. }
  177. static void pblk_end_io_read(struct nvm_rq *rqd)
  178. {
  179. struct pblk *pblk = rqd->private;
  180. struct pblk_g_ctx *r_ctx = nvm_rq_to_pdu(rqd);
  181. struct bio *bio = (struct bio *)r_ctx->private;
  182. pblk_end_user_read(bio, rqd->error);
  183. __pblk_end_io_read(pblk, rqd, true);
  184. }
  185. static void pblk_read_rq(struct pblk *pblk, struct nvm_rq *rqd, struct bio *bio,
  186. sector_t lba, bool *from_cache)
  187. {
  188. struct pblk_sec_meta *meta = pblk_get_meta(pblk, rqd->meta_list, 0);
  189. struct ppa_addr ppa;
  190. pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);
  191. #ifdef CONFIG_NVM_PBLK_DEBUG
  192. atomic_long_inc(&pblk->inflight_reads);
  193. #endif
  194. retry:
  195. if (pblk_ppa_empty(ppa)) {
  196. __le64 addr_empty = cpu_to_le64(ADDR_EMPTY);
  197. meta->lba = addr_empty;
  198. return;
  199. }
  200. /* Try to read from write buffer. The address is later checked on the
  201. * write buffer to prevent retrieving overwritten data.
  202. */
  203. if (pblk_addr_in_cache(ppa)) {
  204. if (!pblk_read_from_cache(pblk, bio, lba, ppa)) {
  205. pblk_lookup_l2p_seq(pblk, &ppa, lba, 1, from_cache);
  206. goto retry;
  207. }
  208. meta->lba = cpu_to_le64(lba);
  209. #ifdef CONFIG_NVM_PBLK_DEBUG
  210. atomic_long_inc(&pblk->cache_reads);
  211. #endif
  212. } else {
  213. rqd->ppa_addr = ppa;
  214. }
  215. }
  216. void pblk_submit_read(struct pblk *pblk, struct bio *bio)
  217. {
  218. sector_t blba = pblk_get_lba(bio);
  219. unsigned int nr_secs = pblk_get_secs(bio);
  220. bool from_cache;
  221. struct pblk_g_ctx *r_ctx;
  222. struct nvm_rq *rqd;
  223. struct bio *int_bio, *split_bio;
  224. unsigned long start_time;
  225. start_time = bio_start_io_acct(bio);
  226. rqd = pblk_alloc_rqd(pblk, PBLK_READ);
  227. rqd->opcode = NVM_OP_PREAD;
  228. rqd->nr_ppas = nr_secs;
  229. rqd->private = pblk;
  230. rqd->end_io = pblk_end_io_read;
  231. r_ctx = nvm_rq_to_pdu(rqd);
  232. r_ctx->start_time = start_time;
  233. r_ctx->lba = blba;
  234. if (pblk_alloc_rqd_meta(pblk, rqd)) {
  235. bio_io_error(bio);
  236. pblk_free_rqd(pblk, rqd, PBLK_READ);
  237. return;
  238. }
  239. /* Clone read bio to deal internally with:
  240. * -read errors when reading from drive
  241. * -bio_advance() calls during cache reads
  242. */
  243. int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
  244. if (nr_secs > 1)
  245. nr_secs = pblk_read_ppalist_rq(pblk, rqd, int_bio, blba,
  246. &from_cache);
  247. else
  248. pblk_read_rq(pblk, rqd, int_bio, blba, &from_cache);
  249. split_retry:
  250. r_ctx->private = bio; /* original bio */
  251. rqd->bio = int_bio; /* internal bio */
  252. if (from_cache && nr_secs == rqd->nr_ppas) {
  253. /* All data was read from cache, we can complete the IO. */
  254. pblk_end_user_read(bio, 0);
  255. atomic_inc(&pblk->inflight_io);
  256. __pblk_end_io_read(pblk, rqd, false);
  257. } else if (nr_secs != rqd->nr_ppas) {
  258. /* The read bio request could be partially filled by the write
  259. * buffer, but there are some holes that need to be read from
  260. * the drive. In order to handle this, we will use block layer
  261. * mechanism to split this request in to smaller ones and make
  262. * a chain of it.
  263. */
  264. split_bio = bio_split(bio, nr_secs * NR_PHY_IN_LOG, GFP_KERNEL,
  265. &pblk_bio_set);
  266. bio_chain(split_bio, bio);
  267. submit_bio_noacct(bio);
  268. /* New bio contains first N sectors of the previous one, so
  269. * we can continue to use existing rqd, but we need to shrink
  270. * the number of PPAs in it. New bio is also guaranteed that
  271. * it contains only either data from cache or from drive, newer
  272. * mix of them.
  273. */
  274. bio = split_bio;
  275. rqd->nr_ppas = nr_secs;
  276. if (rqd->nr_ppas == 1)
  277. rqd->ppa_addr = rqd->ppa_list[0];
  278. /* Recreate int_bio - existing might have some needed internal
  279. * fields modified already.
  280. */
  281. bio_put(int_bio);
  282. int_bio = bio_clone_fast(bio, GFP_KERNEL, &pblk_bio_set);
  283. goto split_retry;
  284. } else if (pblk_submit_io(pblk, rqd, NULL)) {
  285. /* Submitting IO to drive failed, let's report an error */
  286. rqd->error = -ENODEV;
  287. pblk_end_io_read(rqd);
  288. }
  289. }
  290. static int read_ppalist_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
  291. struct pblk_line *line, u64 *lba_list,
  292. u64 *paddr_list_gc, unsigned int nr_secs)
  293. {
  294. struct ppa_addr ppa_list_l2p[NVM_MAX_VLBA];
  295. struct ppa_addr ppa_gc;
  296. int valid_secs = 0;
  297. int i;
  298. pblk_lookup_l2p_rand(pblk, ppa_list_l2p, lba_list, nr_secs);
  299. for (i = 0; i < nr_secs; i++) {
  300. if (lba_list[i] == ADDR_EMPTY)
  301. continue;
  302. ppa_gc = addr_to_gen_ppa(pblk, paddr_list_gc[i], line->id);
  303. if (!pblk_ppa_comp(ppa_list_l2p[i], ppa_gc)) {
  304. paddr_list_gc[i] = lba_list[i] = ADDR_EMPTY;
  305. continue;
  306. }
  307. rqd->ppa_list[valid_secs++] = ppa_list_l2p[i];
  308. }
  309. #ifdef CONFIG_NVM_PBLK_DEBUG
  310. atomic_long_add(valid_secs, &pblk->inflight_reads);
  311. #endif
  312. return valid_secs;
  313. }
  314. static int read_rq_gc(struct pblk *pblk, struct nvm_rq *rqd,
  315. struct pblk_line *line, sector_t lba,
  316. u64 paddr_gc)
  317. {
  318. struct ppa_addr ppa_l2p, ppa_gc;
  319. int valid_secs = 0;
  320. if (lba == ADDR_EMPTY)
  321. goto out;
  322. /* logic error: lba out-of-bounds */
  323. if (lba >= pblk->capacity) {
  324. WARN(1, "pblk: read lba out of bounds\n");
  325. goto out;
  326. }
  327. spin_lock(&pblk->trans_lock);
  328. ppa_l2p = pblk_trans_map_get(pblk, lba);
  329. spin_unlock(&pblk->trans_lock);
  330. ppa_gc = addr_to_gen_ppa(pblk, paddr_gc, line->id);
  331. if (!pblk_ppa_comp(ppa_l2p, ppa_gc))
  332. goto out;
  333. rqd->ppa_addr = ppa_l2p;
  334. valid_secs = 1;
  335. #ifdef CONFIG_NVM_PBLK_DEBUG
  336. atomic_long_inc(&pblk->inflight_reads);
  337. #endif
  338. out:
  339. return valid_secs;
  340. }
  341. int pblk_submit_read_gc(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
  342. {
  343. struct nvm_rq rqd;
  344. int ret = NVM_IO_OK;
  345. memset(&rqd, 0, sizeof(struct nvm_rq));
  346. ret = pblk_alloc_rqd_meta(pblk, &rqd);
  347. if (ret)
  348. return ret;
  349. if (gc_rq->nr_secs > 1) {
  350. gc_rq->secs_to_gc = read_ppalist_rq_gc(pblk, &rqd, gc_rq->line,
  351. gc_rq->lba_list,
  352. gc_rq->paddr_list,
  353. gc_rq->nr_secs);
  354. if (gc_rq->secs_to_gc == 1)
  355. rqd.ppa_addr = rqd.ppa_list[0];
  356. } else {
  357. gc_rq->secs_to_gc = read_rq_gc(pblk, &rqd, gc_rq->line,
  358. gc_rq->lba_list[0],
  359. gc_rq->paddr_list[0]);
  360. }
  361. if (!(gc_rq->secs_to_gc))
  362. goto out;
  363. rqd.opcode = NVM_OP_PREAD;
  364. rqd.nr_ppas = gc_rq->secs_to_gc;
  365. if (pblk_submit_io_sync(pblk, &rqd, gc_rq->data)) {
  366. ret = -EIO;
  367. goto err_free_dma;
  368. }
  369. pblk_read_check_rand(pblk, &rqd, gc_rq->lba_list, gc_rq->nr_secs);
  370. atomic_dec(&pblk->inflight_io);
  371. if (rqd.error) {
  372. atomic_long_inc(&pblk->read_failed_gc);
  373. #ifdef CONFIG_NVM_PBLK_DEBUG
  374. pblk_print_failed_rqd(pblk, &rqd, rqd.error);
  375. #endif
  376. }
  377. #ifdef CONFIG_NVM_PBLK_DEBUG
  378. atomic_long_add(gc_rq->secs_to_gc, &pblk->sync_reads);
  379. atomic_long_add(gc_rq->secs_to_gc, &pblk->recov_gc_reads);
  380. atomic_long_sub(gc_rq->secs_to_gc, &pblk->inflight_reads);
  381. #endif
  382. out:
  383. pblk_free_rqd_meta(pblk, &rqd);
  384. return ret;
  385. err_free_dma:
  386. pblk_free_rqd_meta(pblk, &rqd);
  387. return ret;
  388. }