pblk-cache.c 3.5 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2016 CNEX Labs
  4. * Initial release: Javier Gonzalez <javier@cnexlabs.com>
  5. * Matias Bjorling <matias@cnexlabs.com>
  6. *
  7. * This program is free software; you can redistribute it and/or
  8. * modify it under the terms of the GNU General Public License version
  9. * 2 as published by the Free Software Foundation.
  10. *
  11. * This program is distributed in the hope that it will be useful, but
  12. * WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * General Public License for more details.
  15. *
  16. * pblk-cache.c - pblk's write cache
  17. */
  18. #include "pblk.h"
  19. void pblk_write_to_cache(struct pblk *pblk, struct bio *bio,
  20. unsigned long flags)
  21. {
  22. struct pblk_w_ctx w_ctx;
  23. sector_t lba = pblk_get_lba(bio);
  24. unsigned long start_time;
  25. unsigned int bpos, pos;
  26. int nr_entries = pblk_get_secs(bio);
  27. int i, ret;
  28. start_time = bio_start_io_acct(bio);
  29. /* Update the write buffer head (mem) with the entries that we can
  30. * write. The write in itself cannot fail, so there is no need to
  31. * rollback from here on.
  32. */
  33. retry:
  34. ret = pblk_rb_may_write_user(&pblk->rwb, bio, nr_entries, &bpos);
  35. switch (ret) {
  36. case NVM_IO_REQUEUE:
  37. io_schedule();
  38. goto retry;
  39. case NVM_IO_ERR:
  40. pblk_pipeline_stop(pblk);
  41. bio_io_error(bio);
  42. goto out;
  43. }
  44. pblk_ppa_set_empty(&w_ctx.ppa);
  45. w_ctx.flags = flags;
  46. if (bio->bi_opf & REQ_PREFLUSH) {
  47. w_ctx.flags |= PBLK_FLUSH_ENTRY;
  48. pblk_write_kick(pblk);
  49. }
  50. if (unlikely(!bio_has_data(bio)))
  51. goto out;
  52. for (i = 0; i < nr_entries; i++) {
  53. void *data = bio_data(bio);
  54. w_ctx.lba = lba + i;
  55. pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + i);
  56. pblk_rb_write_entry_user(&pblk->rwb, data, w_ctx, pos);
  57. bio_advance(bio, PBLK_EXPOSED_PAGE_SIZE);
  58. }
  59. atomic64_add(nr_entries, &pblk->user_wa);
  60. #ifdef CONFIG_NVM_PBLK_DEBUG
  61. atomic_long_add(nr_entries, &pblk->inflight_writes);
  62. atomic_long_add(nr_entries, &pblk->req_writes);
  63. #endif
  64. pblk_rl_inserted(&pblk->rl, nr_entries);
  65. out:
  66. bio_end_io_acct(bio, start_time);
  67. pblk_write_should_kick(pblk);
  68. if (ret == NVM_IO_DONE)
  69. bio_endio(bio);
  70. }
  71. /*
  72. * On GC the incoming lbas are not necessarily sequential. Also, some of the
  73. * lbas might not be valid entries, which are marked as empty by the GC thread
  74. */
  75. int pblk_write_gc_to_cache(struct pblk *pblk, struct pblk_gc_rq *gc_rq)
  76. {
  77. struct pblk_w_ctx w_ctx;
  78. unsigned int bpos, pos;
  79. void *data = gc_rq->data;
  80. int i, valid_entries;
  81. /* Update the write buffer head (mem) with the entries that we can
  82. * write. The write in itself cannot fail, so there is no need to
  83. * rollback from here on.
  84. */
  85. retry:
  86. if (!pblk_rb_may_write_gc(&pblk->rwb, gc_rq->secs_to_gc, &bpos)) {
  87. io_schedule();
  88. goto retry;
  89. }
  90. w_ctx.flags = PBLK_IOTYPE_GC;
  91. pblk_ppa_set_empty(&w_ctx.ppa);
  92. for (i = 0, valid_entries = 0; i < gc_rq->nr_secs; i++) {
  93. if (gc_rq->lba_list[i] == ADDR_EMPTY)
  94. continue;
  95. w_ctx.lba = gc_rq->lba_list[i];
  96. pos = pblk_rb_wrap_pos(&pblk->rwb, bpos + valid_entries);
  97. pblk_rb_write_entry_gc(&pblk->rwb, data, w_ctx, gc_rq->line,
  98. gc_rq->paddr_list[i], pos);
  99. data += PBLK_EXPOSED_PAGE_SIZE;
  100. valid_entries++;
  101. }
  102. WARN_ONCE(gc_rq->secs_to_gc != valid_entries,
  103. "pblk: inconsistent GC write\n");
  104. atomic64_add(valid_entries, &pblk->gc_wa);
  105. #ifdef CONFIG_NVM_PBLK_DEBUG
  106. atomic_long_add(valid_entries, &pblk->inflight_writes);
  107. atomic_long_add(valid_entries, &pblk->recov_gc_writes);
  108. #endif
  109. pblk_write_should_kick(pblk);
  110. return NVM_IO_OK;
  111. }