pblk-init.c 33 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2015 IT University of Copenhagen (rrpc.c)
  4. * Copyright (C) 2016 CNEX Labs
  5. * Initial release: Javier Gonzalez <javier@cnexlabs.com>
  6. * Matias Bjorling <matias@cnexlabs.com>
  7. *
  8. * This program is free software; you can redistribute it and/or
  9. * modify it under the terms of the GNU General Public License version
  10. * 2 as published by the Free Software Foundation.
  11. *
  12. * This program is distributed in the hope that it will be useful, but
  13. * WITHOUT ANY WARRANTY; without even the implied warranty of
  14. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  15. * General Public License for more details.
  16. *
  17. * Implementation of a physical block-device target for Open-channel SSDs.
  18. *
  19. * pblk-init.c - pblk's initialization.
  20. */
  21. #include "pblk.h"
  22. #include "pblk-trace.h"
  23. static unsigned int write_buffer_size;
  24. module_param(write_buffer_size, uint, 0644);
  25. MODULE_PARM_DESC(write_buffer_size, "number of entries in a write buffer");
  26. struct pblk_global_caches {
  27. struct kmem_cache *ws;
  28. struct kmem_cache *rec;
  29. struct kmem_cache *g_rq;
  30. struct kmem_cache *w_rq;
  31. struct kref kref;
  32. struct mutex mutex; /* Ensures consistency between
  33. * caches and kref
  34. */
  35. };
  36. static struct pblk_global_caches pblk_caches = {
  37. .mutex = __MUTEX_INITIALIZER(pblk_caches.mutex),
  38. .kref = KREF_INIT(0),
  39. };
  40. struct bio_set pblk_bio_set;
  41. static blk_qc_t pblk_submit_bio(struct bio *bio)
  42. {
  43. struct pblk *pblk = bio->bi_disk->queue->queuedata;
  44. if (bio_op(bio) == REQ_OP_DISCARD) {
  45. pblk_discard(pblk, bio);
  46. if (!(bio->bi_opf & REQ_PREFLUSH)) {
  47. bio_endio(bio);
  48. return BLK_QC_T_NONE;
  49. }
  50. }
  51. /* Read requests must be <= 256kb due to NVMe's 64 bit completion bitmap
  52. * constraint. Writes can be of arbitrary size.
  53. */
  54. if (bio_data_dir(bio) == READ) {
  55. blk_queue_split(&bio);
  56. pblk_submit_read(pblk, bio);
  57. } else {
  58. /* Prevent deadlock in the case of a modest LUN configuration
  59. * and large user I/Os. Unless stalled, the rate limiter
  60. * leaves at least 256KB available for user I/O.
  61. */
  62. if (pblk_get_secs(bio) > pblk_rl_max_io(&pblk->rl))
  63. blk_queue_split(&bio);
  64. pblk_write_to_cache(pblk, bio, PBLK_IOTYPE_USER);
  65. }
  66. return BLK_QC_T_NONE;
  67. }
  68. static const struct block_device_operations pblk_bops = {
  69. .owner = THIS_MODULE,
  70. .submit_bio = pblk_submit_bio,
  71. };
  72. static size_t pblk_trans_map_size(struct pblk *pblk)
  73. {
  74. int entry_size = 8;
  75. if (pblk->addrf_len < 32)
  76. entry_size = 4;
  77. return entry_size * pblk->capacity;
  78. }
  79. #ifdef CONFIG_NVM_PBLK_DEBUG
  80. static u32 pblk_l2p_crc(struct pblk *pblk)
  81. {
  82. size_t map_size;
  83. u32 crc = ~(u32)0;
  84. map_size = pblk_trans_map_size(pblk);
  85. crc = crc32_le(crc, pblk->trans_map, map_size);
  86. return crc;
  87. }
  88. #endif
  89. static void pblk_l2p_free(struct pblk *pblk)
  90. {
  91. vfree(pblk->trans_map);
  92. }
  93. static int pblk_l2p_recover(struct pblk *pblk, bool factory_init)
  94. {
  95. struct pblk_line *line = NULL;
  96. if (factory_init) {
  97. guid_gen(&pblk->instance_uuid);
  98. } else {
  99. line = pblk_recov_l2p(pblk);
  100. if (IS_ERR(line)) {
  101. pblk_err(pblk, "could not recover l2p table\n");
  102. return -EFAULT;
  103. }
  104. }
  105. #ifdef CONFIG_NVM_PBLK_DEBUG
  106. pblk_info(pblk, "init: L2P CRC: %x\n", pblk_l2p_crc(pblk));
  107. #endif
  108. /* Free full lines directly as GC has not been started yet */
  109. pblk_gc_free_full_lines(pblk);
  110. if (!line) {
  111. /* Configure next line for user data */
  112. line = pblk_line_get_first_data(pblk);
  113. if (!line)
  114. return -EFAULT;
  115. }
  116. return 0;
  117. }
  118. static int pblk_l2p_init(struct pblk *pblk, bool factory_init)
  119. {
  120. sector_t i;
  121. struct ppa_addr ppa;
  122. size_t map_size;
  123. int ret = 0;
  124. map_size = pblk_trans_map_size(pblk);
  125. pblk->trans_map = __vmalloc(map_size, GFP_KERNEL | __GFP_NOWARN |
  126. __GFP_RETRY_MAYFAIL | __GFP_HIGHMEM);
  127. if (!pblk->trans_map) {
  128. pblk_err(pblk, "failed to allocate L2P (need %zu of memory)\n",
  129. map_size);
  130. return -ENOMEM;
  131. }
  132. pblk_ppa_set_empty(&ppa);
  133. for (i = 0; i < pblk->capacity; i++)
  134. pblk_trans_map_set(pblk, i, ppa);
  135. ret = pblk_l2p_recover(pblk, factory_init);
  136. if (ret)
  137. vfree(pblk->trans_map);
  138. return ret;
  139. }
  140. static void pblk_rwb_free(struct pblk *pblk)
  141. {
  142. if (pblk_rb_tear_down_check(&pblk->rwb))
  143. pblk_err(pblk, "write buffer error on tear down\n");
  144. pblk_rb_free(&pblk->rwb);
  145. }
  146. static int pblk_rwb_init(struct pblk *pblk)
  147. {
  148. struct nvm_tgt_dev *dev = pblk->dev;
  149. struct nvm_geo *geo = &dev->geo;
  150. unsigned long buffer_size;
  151. int pgs_in_buffer, threshold;
  152. threshold = geo->mw_cunits * geo->all_luns;
  153. pgs_in_buffer = (max(geo->mw_cunits, geo->ws_opt) + geo->ws_opt)
  154. * geo->all_luns;
  155. if (write_buffer_size && (write_buffer_size > pgs_in_buffer))
  156. buffer_size = write_buffer_size;
  157. else
  158. buffer_size = pgs_in_buffer;
  159. return pblk_rb_init(&pblk->rwb, buffer_size, threshold, geo->csecs);
  160. }
  161. static int pblk_set_addrf_12(struct pblk *pblk, struct nvm_geo *geo,
  162. struct nvm_addrf_12 *dst)
  163. {
  164. struct nvm_addrf_12 *src = (struct nvm_addrf_12 *)&geo->addrf;
  165. int power_len;
  166. /* Re-calculate channel and lun format to adapt to configuration */
  167. power_len = get_count_order(geo->num_ch);
  168. if (1 << power_len != geo->num_ch) {
  169. pblk_err(pblk, "supports only power-of-two channel config.\n");
  170. return -EINVAL;
  171. }
  172. dst->ch_len = power_len;
  173. power_len = get_count_order(geo->num_lun);
  174. if (1 << power_len != geo->num_lun) {
  175. pblk_err(pblk, "supports only power-of-two LUN config.\n");
  176. return -EINVAL;
  177. }
  178. dst->lun_len = power_len;
  179. dst->blk_len = src->blk_len;
  180. dst->pg_len = src->pg_len;
  181. dst->pln_len = src->pln_len;
  182. dst->sec_len = src->sec_len;
  183. dst->sec_offset = 0;
  184. dst->pln_offset = dst->sec_len;
  185. dst->ch_offset = dst->pln_offset + dst->pln_len;
  186. dst->lun_offset = dst->ch_offset + dst->ch_len;
  187. dst->pg_offset = dst->lun_offset + dst->lun_len;
  188. dst->blk_offset = dst->pg_offset + dst->pg_len;
  189. dst->sec_mask = ((1ULL << dst->sec_len) - 1) << dst->sec_offset;
  190. dst->pln_mask = ((1ULL << dst->pln_len) - 1) << dst->pln_offset;
  191. dst->ch_mask = ((1ULL << dst->ch_len) - 1) << dst->ch_offset;
  192. dst->lun_mask = ((1ULL << dst->lun_len) - 1) << dst->lun_offset;
  193. dst->pg_mask = ((1ULL << dst->pg_len) - 1) << dst->pg_offset;
  194. dst->blk_mask = ((1ULL << dst->blk_len) - 1) << dst->blk_offset;
  195. return dst->blk_offset + src->blk_len;
  196. }
  197. static int pblk_set_addrf_20(struct nvm_geo *geo, struct nvm_addrf *adst,
  198. struct pblk_addrf *udst)
  199. {
  200. struct nvm_addrf *src = &geo->addrf;
  201. adst->ch_len = get_count_order(geo->num_ch);
  202. adst->lun_len = get_count_order(geo->num_lun);
  203. adst->chk_len = src->chk_len;
  204. adst->sec_len = src->sec_len;
  205. adst->sec_offset = 0;
  206. adst->ch_offset = adst->sec_len;
  207. adst->lun_offset = adst->ch_offset + adst->ch_len;
  208. adst->chk_offset = adst->lun_offset + adst->lun_len;
  209. adst->sec_mask = ((1ULL << adst->sec_len) - 1) << adst->sec_offset;
  210. adst->chk_mask = ((1ULL << adst->chk_len) - 1) << adst->chk_offset;
  211. adst->lun_mask = ((1ULL << adst->lun_len) - 1) << adst->lun_offset;
  212. adst->ch_mask = ((1ULL << adst->ch_len) - 1) << adst->ch_offset;
  213. udst->sec_stripe = geo->ws_opt;
  214. udst->ch_stripe = geo->num_ch;
  215. udst->lun_stripe = geo->num_lun;
  216. udst->sec_lun_stripe = udst->sec_stripe * udst->ch_stripe;
  217. udst->sec_ws_stripe = udst->sec_lun_stripe * udst->lun_stripe;
  218. return adst->chk_offset + adst->chk_len;
  219. }
  220. static int pblk_set_addrf(struct pblk *pblk)
  221. {
  222. struct nvm_tgt_dev *dev = pblk->dev;
  223. struct nvm_geo *geo = &dev->geo;
  224. int mod;
  225. switch (geo->version) {
  226. case NVM_OCSSD_SPEC_12:
  227. div_u64_rem(geo->clba, pblk->min_write_pgs, &mod);
  228. if (mod) {
  229. pblk_err(pblk, "bad configuration of sectors/pages\n");
  230. return -EINVAL;
  231. }
  232. pblk->addrf_len = pblk_set_addrf_12(pblk, geo,
  233. (void *)&pblk->addrf);
  234. break;
  235. case NVM_OCSSD_SPEC_20:
  236. pblk->addrf_len = pblk_set_addrf_20(geo, (void *)&pblk->addrf,
  237. &pblk->uaddrf);
  238. break;
  239. default:
  240. pblk_err(pblk, "OCSSD revision not supported (%d)\n",
  241. geo->version);
  242. return -EINVAL;
  243. }
  244. return 0;
  245. }
  246. static int pblk_create_global_caches(void)
  247. {
  248. pblk_caches.ws = kmem_cache_create("pblk_blk_ws",
  249. sizeof(struct pblk_line_ws), 0, 0, NULL);
  250. if (!pblk_caches.ws)
  251. return -ENOMEM;
  252. pblk_caches.rec = kmem_cache_create("pblk_rec",
  253. sizeof(struct pblk_rec_ctx), 0, 0, NULL);
  254. if (!pblk_caches.rec)
  255. goto fail_destroy_ws;
  256. pblk_caches.g_rq = kmem_cache_create("pblk_g_rq", pblk_g_rq_size,
  257. 0, 0, NULL);
  258. if (!pblk_caches.g_rq)
  259. goto fail_destroy_rec;
  260. pblk_caches.w_rq = kmem_cache_create("pblk_w_rq", pblk_w_rq_size,
  261. 0, 0, NULL);
  262. if (!pblk_caches.w_rq)
  263. goto fail_destroy_g_rq;
  264. return 0;
  265. fail_destroy_g_rq:
  266. kmem_cache_destroy(pblk_caches.g_rq);
  267. fail_destroy_rec:
  268. kmem_cache_destroy(pblk_caches.rec);
  269. fail_destroy_ws:
  270. kmem_cache_destroy(pblk_caches.ws);
  271. return -ENOMEM;
  272. }
  273. static int pblk_get_global_caches(void)
  274. {
  275. int ret = 0;
  276. mutex_lock(&pblk_caches.mutex);
  277. if (kref_get_unless_zero(&pblk_caches.kref))
  278. goto out;
  279. ret = pblk_create_global_caches();
  280. if (!ret)
  281. kref_init(&pblk_caches.kref);
  282. out:
  283. mutex_unlock(&pblk_caches.mutex);
  284. return ret;
  285. }
  286. static void pblk_destroy_global_caches(struct kref *ref)
  287. {
  288. struct pblk_global_caches *c;
  289. c = container_of(ref, struct pblk_global_caches, kref);
  290. kmem_cache_destroy(c->ws);
  291. kmem_cache_destroy(c->rec);
  292. kmem_cache_destroy(c->g_rq);
  293. kmem_cache_destroy(c->w_rq);
  294. }
  295. static void pblk_put_global_caches(void)
  296. {
  297. mutex_lock(&pblk_caches.mutex);
  298. kref_put(&pblk_caches.kref, pblk_destroy_global_caches);
  299. mutex_unlock(&pblk_caches.mutex);
  300. }
  301. static int pblk_core_init(struct pblk *pblk)
  302. {
  303. struct nvm_tgt_dev *dev = pblk->dev;
  304. struct nvm_geo *geo = &dev->geo;
  305. int ret, max_write_ppas;
  306. atomic64_set(&pblk->user_wa, 0);
  307. atomic64_set(&pblk->pad_wa, 0);
  308. atomic64_set(&pblk->gc_wa, 0);
  309. pblk->user_rst_wa = 0;
  310. pblk->pad_rst_wa = 0;
  311. pblk->gc_rst_wa = 0;
  312. atomic64_set(&pblk->nr_flush, 0);
  313. pblk->nr_flush_rst = 0;
  314. pblk->min_write_pgs = geo->ws_opt;
  315. pblk->min_write_pgs_data = pblk->min_write_pgs;
  316. max_write_ppas = pblk->min_write_pgs * geo->all_luns;
  317. pblk->max_write_pgs = min_t(int, max_write_ppas, NVM_MAX_VLBA);
  318. pblk->max_write_pgs = min_t(int, pblk->max_write_pgs,
  319. queue_max_hw_sectors(dev->q) / (geo->csecs >> SECTOR_SHIFT));
  320. pblk_set_sec_per_write(pblk, pblk->min_write_pgs);
  321. pblk->oob_meta_size = geo->sos;
  322. if (!pblk_is_oob_meta_supported(pblk)) {
  323. /* For drives which does not have OOB metadata feature
  324. * in order to support recovery feature we need to use
  325. * so called packed metadata. Packed metada will store
  326. * the same information as OOB metadata (l2p table mapping,
  327. * but in the form of the single page at the end of
  328. * every write request.
  329. */
  330. if (pblk->min_write_pgs
  331. * sizeof(struct pblk_sec_meta) > PAGE_SIZE) {
  332. /* We want to keep all the packed metadata on single
  333. * page per write requests. So we need to ensure that
  334. * it will fit.
  335. *
  336. * This is more like sanity check, since there is
  337. * no device with such a big minimal write size
  338. * (above 1 metabytes).
  339. */
  340. pblk_err(pblk, "Not supported min write size\n");
  341. return -EINVAL;
  342. }
  343. /* For packed meta approach we do some simplification.
  344. * On read path we always issue requests which size
  345. * equal to max_write_pgs, with all pages filled with
  346. * user payload except of last one page which will be
  347. * filled with packed metadata.
  348. */
  349. pblk->max_write_pgs = pblk->min_write_pgs;
  350. pblk->min_write_pgs_data = pblk->min_write_pgs - 1;
  351. }
  352. pblk->pad_dist = kcalloc(pblk->min_write_pgs - 1, sizeof(atomic64_t),
  353. GFP_KERNEL);
  354. if (!pblk->pad_dist)
  355. return -ENOMEM;
  356. if (pblk_get_global_caches())
  357. goto fail_free_pad_dist;
  358. /* Internal bios can be at most the sectors signaled by the device. */
  359. ret = mempool_init_page_pool(&pblk->page_bio_pool, NVM_MAX_VLBA, 0);
  360. if (ret)
  361. goto free_global_caches;
  362. ret = mempool_init_slab_pool(&pblk->gen_ws_pool, PBLK_GEN_WS_POOL_SIZE,
  363. pblk_caches.ws);
  364. if (ret)
  365. goto free_page_bio_pool;
  366. ret = mempool_init_slab_pool(&pblk->rec_pool, geo->all_luns,
  367. pblk_caches.rec);
  368. if (ret)
  369. goto free_gen_ws_pool;
  370. ret = mempool_init_slab_pool(&pblk->r_rq_pool, geo->all_luns,
  371. pblk_caches.g_rq);
  372. if (ret)
  373. goto free_rec_pool;
  374. ret = mempool_init_slab_pool(&pblk->e_rq_pool, geo->all_luns,
  375. pblk_caches.g_rq);
  376. if (ret)
  377. goto free_r_rq_pool;
  378. ret = mempool_init_slab_pool(&pblk->w_rq_pool, geo->all_luns,
  379. pblk_caches.w_rq);
  380. if (ret)
  381. goto free_e_rq_pool;
  382. pblk->close_wq = alloc_workqueue("pblk-close-wq",
  383. WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_NR_CLOSE_JOBS);
  384. if (!pblk->close_wq)
  385. goto free_w_rq_pool;
  386. pblk->bb_wq = alloc_workqueue("pblk-bb-wq",
  387. WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
  388. if (!pblk->bb_wq)
  389. goto free_close_wq;
  390. pblk->r_end_wq = alloc_workqueue("pblk-read-end-wq",
  391. WQ_MEM_RECLAIM | WQ_UNBOUND, 0);
  392. if (!pblk->r_end_wq)
  393. goto free_bb_wq;
  394. if (pblk_set_addrf(pblk))
  395. goto free_r_end_wq;
  396. INIT_LIST_HEAD(&pblk->compl_list);
  397. INIT_LIST_HEAD(&pblk->resubmit_list);
  398. return 0;
  399. free_r_end_wq:
  400. destroy_workqueue(pblk->r_end_wq);
  401. free_bb_wq:
  402. destroy_workqueue(pblk->bb_wq);
  403. free_close_wq:
  404. destroy_workqueue(pblk->close_wq);
  405. free_w_rq_pool:
  406. mempool_exit(&pblk->w_rq_pool);
  407. free_e_rq_pool:
  408. mempool_exit(&pblk->e_rq_pool);
  409. free_r_rq_pool:
  410. mempool_exit(&pblk->r_rq_pool);
  411. free_rec_pool:
  412. mempool_exit(&pblk->rec_pool);
  413. free_gen_ws_pool:
  414. mempool_exit(&pblk->gen_ws_pool);
  415. free_page_bio_pool:
  416. mempool_exit(&pblk->page_bio_pool);
  417. free_global_caches:
  418. pblk_put_global_caches();
  419. fail_free_pad_dist:
  420. kfree(pblk->pad_dist);
  421. return -ENOMEM;
  422. }
  423. static void pblk_core_free(struct pblk *pblk)
  424. {
  425. if (pblk->close_wq)
  426. destroy_workqueue(pblk->close_wq);
  427. if (pblk->r_end_wq)
  428. destroy_workqueue(pblk->r_end_wq);
  429. if (pblk->bb_wq)
  430. destroy_workqueue(pblk->bb_wq);
  431. mempool_exit(&pblk->page_bio_pool);
  432. mempool_exit(&pblk->gen_ws_pool);
  433. mempool_exit(&pblk->rec_pool);
  434. mempool_exit(&pblk->r_rq_pool);
  435. mempool_exit(&pblk->e_rq_pool);
  436. mempool_exit(&pblk->w_rq_pool);
  437. pblk_put_global_caches();
  438. kfree(pblk->pad_dist);
  439. }
  440. static void pblk_line_mg_free(struct pblk *pblk)
  441. {
  442. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  443. int i;
  444. kfree(l_mg->bb_template);
  445. kfree(l_mg->bb_aux);
  446. kfree(l_mg->vsc_list);
  447. for (i = 0; i < PBLK_DATA_LINES; i++) {
  448. kfree(l_mg->sline_meta[i]);
  449. kvfree(l_mg->eline_meta[i]->buf);
  450. kfree(l_mg->eline_meta[i]);
  451. }
  452. mempool_destroy(l_mg->bitmap_pool);
  453. kmem_cache_destroy(l_mg->bitmap_cache);
  454. }
  455. static void pblk_line_meta_free(struct pblk_line_mgmt *l_mg,
  456. struct pblk_line *line)
  457. {
  458. struct pblk_w_err_gc *w_err_gc = line->w_err_gc;
  459. kfree(line->blk_bitmap);
  460. kfree(line->erase_bitmap);
  461. kfree(line->chks);
  462. kvfree(w_err_gc->lba_list);
  463. kfree(w_err_gc);
  464. }
  465. static void pblk_lines_free(struct pblk *pblk)
  466. {
  467. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  468. struct pblk_line *line;
  469. int i;
  470. for (i = 0; i < l_mg->nr_lines; i++) {
  471. line = &pblk->lines[i];
  472. pblk_line_free(line);
  473. pblk_line_meta_free(l_mg, line);
  474. }
  475. pblk_line_mg_free(pblk);
  476. kfree(pblk->luns);
  477. kfree(pblk->lines);
  478. }
  479. static int pblk_luns_init(struct pblk *pblk)
  480. {
  481. struct nvm_tgt_dev *dev = pblk->dev;
  482. struct nvm_geo *geo = &dev->geo;
  483. struct pblk_lun *rlun;
  484. int i;
  485. /* TODO: Implement unbalanced LUN support */
  486. if (geo->num_lun < 0) {
  487. pblk_err(pblk, "unbalanced LUN config.\n");
  488. return -EINVAL;
  489. }
  490. pblk->luns = kcalloc(geo->all_luns, sizeof(struct pblk_lun),
  491. GFP_KERNEL);
  492. if (!pblk->luns)
  493. return -ENOMEM;
  494. for (i = 0; i < geo->all_luns; i++) {
  495. /* Stripe across channels */
  496. int ch = i % geo->num_ch;
  497. int lun_raw = i / geo->num_ch;
  498. int lunid = lun_raw + ch * geo->num_lun;
  499. rlun = &pblk->luns[i];
  500. rlun->bppa = dev->luns[lunid];
  501. sema_init(&rlun->wr_sem, 1);
  502. }
  503. return 0;
  504. }
  505. /* See comment over struct line_emeta definition */
  506. static unsigned int calc_emeta_len(struct pblk *pblk)
  507. {
  508. struct pblk_line_meta *lm = &pblk->lm;
  509. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  510. struct nvm_tgt_dev *dev = pblk->dev;
  511. struct nvm_geo *geo = &dev->geo;
  512. /* Round to sector size so that lba_list starts on its own sector */
  513. lm->emeta_sec[1] = DIV_ROUND_UP(
  514. sizeof(struct line_emeta) + lm->blk_bitmap_len +
  515. sizeof(struct wa_counters), geo->csecs);
  516. lm->emeta_len[1] = lm->emeta_sec[1] * geo->csecs;
  517. /* Round to sector size so that vsc_list starts on its own sector */
  518. lm->dsec_per_line = lm->sec_per_line - lm->emeta_sec[0];
  519. lm->emeta_sec[2] = DIV_ROUND_UP(lm->dsec_per_line * sizeof(u64),
  520. geo->csecs);
  521. lm->emeta_len[2] = lm->emeta_sec[2] * geo->csecs;
  522. lm->emeta_sec[3] = DIV_ROUND_UP(l_mg->nr_lines * sizeof(u32),
  523. geo->csecs);
  524. lm->emeta_len[3] = lm->emeta_sec[3] * geo->csecs;
  525. lm->vsc_list_len = l_mg->nr_lines * sizeof(u32);
  526. return (lm->emeta_len[1] + lm->emeta_len[2] + lm->emeta_len[3]);
  527. }
  528. static int pblk_set_provision(struct pblk *pblk, int nr_free_chks)
  529. {
  530. struct nvm_tgt_dev *dev = pblk->dev;
  531. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  532. struct pblk_line_meta *lm = &pblk->lm;
  533. struct nvm_geo *geo = &dev->geo;
  534. sector_t provisioned;
  535. int sec_meta, blk_meta, clba;
  536. int minimum;
  537. if (geo->op == NVM_TARGET_DEFAULT_OP)
  538. pblk->op = PBLK_DEFAULT_OP;
  539. else
  540. pblk->op = geo->op;
  541. minimum = pblk_get_min_chks(pblk);
  542. provisioned = nr_free_chks;
  543. provisioned *= (100 - pblk->op);
  544. sector_div(provisioned, 100);
  545. if ((nr_free_chks - provisioned) < minimum) {
  546. if (geo->op != NVM_TARGET_DEFAULT_OP) {
  547. pblk_err(pblk, "OP too small to create a sane instance\n");
  548. return -EINTR;
  549. }
  550. /* If the user did not specify an OP value, and PBLK_DEFAULT_OP
  551. * is not enough, calculate and set sane value
  552. */
  553. provisioned = nr_free_chks - minimum;
  554. pblk->op = (100 * minimum) / nr_free_chks;
  555. pblk_info(pblk, "Default OP insufficient, adjusting OP to %d\n",
  556. pblk->op);
  557. }
  558. pblk->op_blks = nr_free_chks - provisioned;
  559. /* Internally pblk manages all free blocks, but all calculations based
  560. * on user capacity consider only provisioned blocks
  561. */
  562. pblk->rl.total_blocks = nr_free_chks;
  563. /* Consider sectors used for metadata */
  564. sec_meta = (lm->smeta_sec + lm->emeta_sec[0]) * l_mg->nr_free_lines;
  565. blk_meta = DIV_ROUND_UP(sec_meta, geo->clba);
  566. clba = (geo->clba / pblk->min_write_pgs) * pblk->min_write_pgs_data;
  567. pblk->capacity = (provisioned - blk_meta) * clba;
  568. atomic_set(&pblk->rl.free_blocks, nr_free_chks);
  569. atomic_set(&pblk->rl.free_user_blocks, nr_free_chks);
  570. return 0;
  571. }
  572. static int pblk_setup_line_meta_chk(struct pblk *pblk, struct pblk_line *line,
  573. struct nvm_chk_meta *meta)
  574. {
  575. struct nvm_tgt_dev *dev = pblk->dev;
  576. struct nvm_geo *geo = &dev->geo;
  577. struct pblk_line_meta *lm = &pblk->lm;
  578. int i, nr_bad_chks = 0;
  579. for (i = 0; i < lm->blk_per_line; i++) {
  580. struct pblk_lun *rlun = &pblk->luns[i];
  581. struct nvm_chk_meta *chunk;
  582. struct nvm_chk_meta *chunk_meta;
  583. struct ppa_addr ppa;
  584. int pos;
  585. ppa = rlun->bppa;
  586. pos = pblk_ppa_to_pos(geo, ppa);
  587. chunk = &line->chks[pos];
  588. ppa.m.chk = line->id;
  589. chunk_meta = pblk_chunk_get_off(pblk, meta, ppa);
  590. chunk->state = chunk_meta->state;
  591. chunk->type = chunk_meta->type;
  592. chunk->wi = chunk_meta->wi;
  593. chunk->slba = chunk_meta->slba;
  594. chunk->cnlb = chunk_meta->cnlb;
  595. chunk->wp = chunk_meta->wp;
  596. trace_pblk_chunk_state(pblk_disk_name(pblk), &ppa,
  597. chunk->state);
  598. if (chunk->type & NVM_CHK_TP_SZ_SPEC) {
  599. WARN_ONCE(1, "pblk: custom-sized chunks unsupported\n");
  600. continue;
  601. }
  602. if (!(chunk->state & NVM_CHK_ST_OFFLINE))
  603. continue;
  604. set_bit(pos, line->blk_bitmap);
  605. nr_bad_chks++;
  606. }
  607. return nr_bad_chks;
  608. }
  609. static long pblk_setup_line_meta(struct pblk *pblk, struct pblk_line *line,
  610. void *chunk_meta, int line_id)
  611. {
  612. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  613. struct pblk_line_meta *lm = &pblk->lm;
  614. long nr_bad_chks, chk_in_line;
  615. line->pblk = pblk;
  616. line->id = line_id;
  617. line->type = PBLK_LINETYPE_FREE;
  618. line->state = PBLK_LINESTATE_NEW;
  619. line->gc_group = PBLK_LINEGC_NONE;
  620. line->vsc = &l_mg->vsc_list[line_id];
  621. spin_lock_init(&line->lock);
  622. nr_bad_chks = pblk_setup_line_meta_chk(pblk, line, chunk_meta);
  623. chk_in_line = lm->blk_per_line - nr_bad_chks;
  624. if (nr_bad_chks < 0 || nr_bad_chks > lm->blk_per_line ||
  625. chk_in_line < lm->min_blk_line) {
  626. line->state = PBLK_LINESTATE_BAD;
  627. list_add_tail(&line->list, &l_mg->bad_list);
  628. return 0;
  629. }
  630. atomic_set(&line->blk_in_line, chk_in_line);
  631. list_add_tail(&line->list, &l_mg->free_list);
  632. l_mg->nr_free_lines++;
  633. return chk_in_line;
  634. }
  635. static int pblk_alloc_line_meta(struct pblk *pblk, struct pblk_line *line)
  636. {
  637. struct pblk_line_meta *lm = &pblk->lm;
  638. line->blk_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
  639. if (!line->blk_bitmap)
  640. return -ENOMEM;
  641. line->erase_bitmap = kzalloc(lm->blk_bitmap_len, GFP_KERNEL);
  642. if (!line->erase_bitmap)
  643. goto free_blk_bitmap;
  644. line->chks = kmalloc_array(lm->blk_per_line,
  645. sizeof(struct nvm_chk_meta), GFP_KERNEL);
  646. if (!line->chks)
  647. goto free_erase_bitmap;
  648. line->w_err_gc = kzalloc(sizeof(struct pblk_w_err_gc), GFP_KERNEL);
  649. if (!line->w_err_gc)
  650. goto free_chks;
  651. return 0;
  652. free_chks:
  653. kfree(line->chks);
  654. free_erase_bitmap:
  655. kfree(line->erase_bitmap);
  656. free_blk_bitmap:
  657. kfree(line->blk_bitmap);
  658. return -ENOMEM;
  659. }
  660. static int pblk_line_mg_init(struct pblk *pblk)
  661. {
  662. struct nvm_tgt_dev *dev = pblk->dev;
  663. struct nvm_geo *geo = &dev->geo;
  664. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  665. struct pblk_line_meta *lm = &pblk->lm;
  666. int i, bb_distance;
  667. l_mg->nr_lines = geo->num_chk;
  668. l_mg->log_line = l_mg->data_line = NULL;
  669. l_mg->l_seq_nr = l_mg->d_seq_nr = 0;
  670. l_mg->nr_free_lines = 0;
  671. bitmap_zero(&l_mg->meta_bitmap, PBLK_DATA_LINES);
  672. INIT_LIST_HEAD(&l_mg->free_list);
  673. INIT_LIST_HEAD(&l_mg->corrupt_list);
  674. INIT_LIST_HEAD(&l_mg->bad_list);
  675. INIT_LIST_HEAD(&l_mg->gc_full_list);
  676. INIT_LIST_HEAD(&l_mg->gc_high_list);
  677. INIT_LIST_HEAD(&l_mg->gc_mid_list);
  678. INIT_LIST_HEAD(&l_mg->gc_low_list);
  679. INIT_LIST_HEAD(&l_mg->gc_empty_list);
  680. INIT_LIST_HEAD(&l_mg->gc_werr_list);
  681. INIT_LIST_HEAD(&l_mg->emeta_list);
  682. l_mg->gc_lists[0] = &l_mg->gc_werr_list;
  683. l_mg->gc_lists[1] = &l_mg->gc_high_list;
  684. l_mg->gc_lists[2] = &l_mg->gc_mid_list;
  685. l_mg->gc_lists[3] = &l_mg->gc_low_list;
  686. spin_lock_init(&l_mg->free_lock);
  687. spin_lock_init(&l_mg->close_lock);
  688. spin_lock_init(&l_mg->gc_lock);
  689. l_mg->vsc_list = kcalloc(l_mg->nr_lines, sizeof(__le32), GFP_KERNEL);
  690. if (!l_mg->vsc_list)
  691. goto fail;
  692. l_mg->bb_template = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
  693. if (!l_mg->bb_template)
  694. goto fail_free_vsc_list;
  695. l_mg->bb_aux = kzalloc(lm->sec_bitmap_len, GFP_KERNEL);
  696. if (!l_mg->bb_aux)
  697. goto fail_free_bb_template;
  698. /* smeta is always small enough to fit on a kmalloc memory allocation,
  699. * emeta depends on the number of LUNs allocated to the pblk instance
  700. */
  701. for (i = 0; i < PBLK_DATA_LINES; i++) {
  702. l_mg->sline_meta[i] = kmalloc(lm->smeta_len, GFP_KERNEL);
  703. if (!l_mg->sline_meta[i])
  704. goto fail_free_smeta;
  705. }
  706. l_mg->bitmap_cache = kmem_cache_create("pblk_lm_bitmap",
  707. lm->sec_bitmap_len, 0, 0, NULL);
  708. if (!l_mg->bitmap_cache)
  709. goto fail_free_smeta;
  710. /* the bitmap pool is used for both valid and map bitmaps */
  711. l_mg->bitmap_pool = mempool_create_slab_pool(PBLK_DATA_LINES * 2,
  712. l_mg->bitmap_cache);
  713. if (!l_mg->bitmap_pool)
  714. goto fail_destroy_bitmap_cache;
  715. /* emeta allocates three different buffers for managing metadata with
  716. * in-memory and in-media layouts
  717. */
  718. for (i = 0; i < PBLK_DATA_LINES; i++) {
  719. struct pblk_emeta *emeta;
  720. emeta = kmalloc(sizeof(struct pblk_emeta), GFP_KERNEL);
  721. if (!emeta)
  722. goto fail_free_emeta;
  723. emeta->buf = kvmalloc(lm->emeta_len[0], GFP_KERNEL);
  724. if (!emeta->buf) {
  725. kfree(emeta);
  726. goto fail_free_emeta;
  727. }
  728. emeta->nr_entries = lm->emeta_sec[0];
  729. l_mg->eline_meta[i] = emeta;
  730. }
  731. for (i = 0; i < l_mg->nr_lines; i++)
  732. l_mg->vsc_list[i] = cpu_to_le32(EMPTY_ENTRY);
  733. bb_distance = (geo->all_luns) * geo->ws_opt;
  734. for (i = 0; i < lm->sec_per_line; i += bb_distance)
  735. bitmap_set(l_mg->bb_template, i, geo->ws_opt);
  736. return 0;
  737. fail_free_emeta:
  738. while (--i >= 0) {
  739. kvfree(l_mg->eline_meta[i]->buf);
  740. kfree(l_mg->eline_meta[i]);
  741. }
  742. mempool_destroy(l_mg->bitmap_pool);
  743. fail_destroy_bitmap_cache:
  744. kmem_cache_destroy(l_mg->bitmap_cache);
  745. fail_free_smeta:
  746. for (i = 0; i < PBLK_DATA_LINES; i++)
  747. kfree(l_mg->sline_meta[i]);
  748. kfree(l_mg->bb_aux);
  749. fail_free_bb_template:
  750. kfree(l_mg->bb_template);
  751. fail_free_vsc_list:
  752. kfree(l_mg->vsc_list);
  753. fail:
  754. return -ENOMEM;
  755. }
  756. static int pblk_line_meta_init(struct pblk *pblk)
  757. {
  758. struct nvm_tgt_dev *dev = pblk->dev;
  759. struct nvm_geo *geo = &dev->geo;
  760. struct pblk_line_meta *lm = &pblk->lm;
  761. unsigned int smeta_len, emeta_len;
  762. int i;
  763. lm->sec_per_line = geo->clba * geo->all_luns;
  764. lm->blk_per_line = geo->all_luns;
  765. lm->blk_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
  766. lm->sec_bitmap_len = BITS_TO_LONGS(lm->sec_per_line) * sizeof(long);
  767. lm->lun_bitmap_len = BITS_TO_LONGS(geo->all_luns) * sizeof(long);
  768. lm->mid_thrs = lm->sec_per_line / 2;
  769. lm->high_thrs = lm->sec_per_line / 4;
  770. lm->meta_distance = (geo->all_luns / 2) * pblk->min_write_pgs;
  771. /* Calculate necessary pages for smeta. See comment over struct
  772. * line_smeta definition
  773. */
  774. i = 1;
  775. add_smeta_page:
  776. lm->smeta_sec = i * geo->ws_opt;
  777. lm->smeta_len = lm->smeta_sec * geo->csecs;
  778. smeta_len = sizeof(struct line_smeta) + lm->lun_bitmap_len;
  779. if (smeta_len > lm->smeta_len) {
  780. i++;
  781. goto add_smeta_page;
  782. }
  783. /* Calculate necessary pages for emeta. See comment over struct
  784. * line_emeta definition
  785. */
  786. i = 1;
  787. add_emeta_page:
  788. lm->emeta_sec[0] = i * geo->ws_opt;
  789. lm->emeta_len[0] = lm->emeta_sec[0] * geo->csecs;
  790. emeta_len = calc_emeta_len(pblk);
  791. if (emeta_len > lm->emeta_len[0]) {
  792. i++;
  793. goto add_emeta_page;
  794. }
  795. lm->emeta_bb = geo->all_luns > i ? geo->all_luns - i : 0;
  796. lm->min_blk_line = 1;
  797. if (geo->all_luns > 1)
  798. lm->min_blk_line += DIV_ROUND_UP(lm->smeta_sec +
  799. lm->emeta_sec[0], geo->clba);
  800. if (lm->min_blk_line > lm->blk_per_line) {
  801. pblk_err(pblk, "config. not supported. Min. LUN in line:%d\n",
  802. lm->blk_per_line);
  803. return -EINVAL;
  804. }
  805. return 0;
  806. }
  807. static int pblk_lines_init(struct pblk *pblk)
  808. {
  809. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  810. struct pblk_line *line;
  811. void *chunk_meta;
  812. int nr_free_chks = 0;
  813. int i, ret;
  814. ret = pblk_line_meta_init(pblk);
  815. if (ret)
  816. return ret;
  817. ret = pblk_line_mg_init(pblk);
  818. if (ret)
  819. return ret;
  820. ret = pblk_luns_init(pblk);
  821. if (ret)
  822. goto fail_free_meta;
  823. chunk_meta = pblk_get_chunk_meta(pblk);
  824. if (IS_ERR(chunk_meta)) {
  825. ret = PTR_ERR(chunk_meta);
  826. goto fail_free_luns;
  827. }
  828. pblk->lines = kcalloc(l_mg->nr_lines, sizeof(struct pblk_line),
  829. GFP_KERNEL);
  830. if (!pblk->lines) {
  831. ret = -ENOMEM;
  832. goto fail_free_chunk_meta;
  833. }
  834. for (i = 0; i < l_mg->nr_lines; i++) {
  835. line = &pblk->lines[i];
  836. ret = pblk_alloc_line_meta(pblk, line);
  837. if (ret)
  838. goto fail_free_lines;
  839. nr_free_chks += pblk_setup_line_meta(pblk, line, chunk_meta, i);
  840. trace_pblk_line_state(pblk_disk_name(pblk), line->id,
  841. line->state);
  842. }
  843. if (!nr_free_chks) {
  844. pblk_err(pblk, "too many bad blocks prevent for sane instance\n");
  845. ret = -EINTR;
  846. goto fail_free_lines;
  847. }
  848. ret = pblk_set_provision(pblk, nr_free_chks);
  849. if (ret)
  850. goto fail_free_lines;
  851. vfree(chunk_meta);
  852. return 0;
  853. fail_free_lines:
  854. while (--i >= 0)
  855. pblk_line_meta_free(l_mg, &pblk->lines[i]);
  856. kfree(pblk->lines);
  857. fail_free_chunk_meta:
  858. vfree(chunk_meta);
  859. fail_free_luns:
  860. kfree(pblk->luns);
  861. fail_free_meta:
  862. pblk_line_mg_free(pblk);
  863. return ret;
  864. }
  865. static int pblk_writer_init(struct pblk *pblk)
  866. {
  867. pblk->writer_ts = kthread_create(pblk_write_ts, pblk, "pblk-writer-t");
  868. if (IS_ERR(pblk->writer_ts)) {
  869. int err = PTR_ERR(pblk->writer_ts);
  870. if (err != -EINTR)
  871. pblk_err(pblk, "could not allocate writer kthread (%d)\n",
  872. err);
  873. return err;
  874. }
  875. timer_setup(&pblk->wtimer, pblk_write_timer_fn, 0);
  876. mod_timer(&pblk->wtimer, jiffies + msecs_to_jiffies(100));
  877. return 0;
  878. }
  879. static void pblk_writer_stop(struct pblk *pblk)
  880. {
  881. /* The pipeline must be stopped and the write buffer emptied before the
  882. * write thread is stopped
  883. */
  884. WARN(pblk_rb_read_count(&pblk->rwb),
  885. "Stopping not fully persisted write buffer\n");
  886. WARN(pblk_rb_sync_count(&pblk->rwb),
  887. "Stopping not fully synced write buffer\n");
  888. del_timer_sync(&pblk->wtimer);
  889. if (pblk->writer_ts)
  890. kthread_stop(pblk->writer_ts);
  891. }
  892. static void pblk_free(struct pblk *pblk)
  893. {
  894. pblk_lines_free(pblk);
  895. pblk_l2p_free(pblk);
  896. pblk_rwb_free(pblk);
  897. pblk_core_free(pblk);
  898. kfree(pblk);
  899. }
  900. static void pblk_tear_down(struct pblk *pblk, bool graceful)
  901. {
  902. if (graceful)
  903. __pblk_pipeline_flush(pblk);
  904. __pblk_pipeline_stop(pblk);
  905. pblk_writer_stop(pblk);
  906. pblk_rb_sync_l2p(&pblk->rwb);
  907. pblk_rl_free(&pblk->rl);
  908. pblk_debug(pblk, "consistent tear down (graceful:%d)\n", graceful);
  909. }
  910. static void pblk_exit(void *private, bool graceful)
  911. {
  912. struct pblk *pblk = private;
  913. pblk_gc_exit(pblk, graceful);
  914. pblk_tear_down(pblk, graceful);
  915. #ifdef CONFIG_NVM_PBLK_DEBUG
  916. pblk_info(pblk, "exit: L2P CRC: %x\n", pblk_l2p_crc(pblk));
  917. #endif
  918. pblk_free(pblk);
  919. }
  920. static sector_t pblk_capacity(void *private)
  921. {
  922. struct pblk *pblk = private;
  923. return pblk->capacity * NR_PHY_IN_LOG;
  924. }
  925. static void *pblk_init(struct nvm_tgt_dev *dev, struct gendisk *tdisk,
  926. int flags)
  927. {
  928. struct nvm_geo *geo = &dev->geo;
  929. struct request_queue *bqueue = dev->q;
  930. struct request_queue *tqueue = tdisk->queue;
  931. struct pblk *pblk;
  932. int ret;
  933. pblk = kzalloc(sizeof(struct pblk), GFP_KERNEL);
  934. if (!pblk)
  935. return ERR_PTR(-ENOMEM);
  936. pblk->dev = dev;
  937. pblk->disk = tdisk;
  938. pblk->state = PBLK_STATE_RUNNING;
  939. trace_pblk_state(pblk_disk_name(pblk), pblk->state);
  940. pblk->gc.gc_enabled = 0;
  941. if (!(geo->version == NVM_OCSSD_SPEC_12 ||
  942. geo->version == NVM_OCSSD_SPEC_20)) {
  943. pblk_err(pblk, "OCSSD version not supported (%u)\n",
  944. geo->version);
  945. kfree(pblk);
  946. return ERR_PTR(-EINVAL);
  947. }
  948. if (geo->ext) {
  949. pblk_err(pblk, "extended metadata not supported\n");
  950. kfree(pblk);
  951. return ERR_PTR(-EINVAL);
  952. }
  953. spin_lock_init(&pblk->resubmit_lock);
  954. spin_lock_init(&pblk->trans_lock);
  955. spin_lock_init(&pblk->lock);
  956. #ifdef CONFIG_NVM_PBLK_DEBUG
  957. atomic_long_set(&pblk->inflight_writes, 0);
  958. atomic_long_set(&pblk->padded_writes, 0);
  959. atomic_long_set(&pblk->padded_wb, 0);
  960. atomic_long_set(&pblk->req_writes, 0);
  961. atomic_long_set(&pblk->sub_writes, 0);
  962. atomic_long_set(&pblk->sync_writes, 0);
  963. atomic_long_set(&pblk->inflight_reads, 0);
  964. atomic_long_set(&pblk->cache_reads, 0);
  965. atomic_long_set(&pblk->sync_reads, 0);
  966. atomic_long_set(&pblk->recov_writes, 0);
  967. atomic_long_set(&pblk->recov_writes, 0);
  968. atomic_long_set(&pblk->recov_gc_writes, 0);
  969. atomic_long_set(&pblk->recov_gc_reads, 0);
  970. #endif
  971. atomic_long_set(&pblk->read_failed, 0);
  972. atomic_long_set(&pblk->read_empty, 0);
  973. atomic_long_set(&pblk->read_high_ecc, 0);
  974. atomic_long_set(&pblk->read_failed_gc, 0);
  975. atomic_long_set(&pblk->write_failed, 0);
  976. atomic_long_set(&pblk->erase_failed, 0);
  977. ret = pblk_core_init(pblk);
  978. if (ret) {
  979. pblk_err(pblk, "could not initialize core\n");
  980. goto fail;
  981. }
  982. ret = pblk_lines_init(pblk);
  983. if (ret) {
  984. pblk_err(pblk, "could not initialize lines\n");
  985. goto fail_free_core;
  986. }
  987. ret = pblk_rwb_init(pblk);
  988. if (ret) {
  989. pblk_err(pblk, "could not initialize write buffer\n");
  990. goto fail_free_lines;
  991. }
  992. ret = pblk_l2p_init(pblk, flags & NVM_TARGET_FACTORY);
  993. if (ret) {
  994. pblk_err(pblk, "could not initialize maps\n");
  995. goto fail_free_rwb;
  996. }
  997. ret = pblk_writer_init(pblk);
  998. if (ret) {
  999. if (ret != -EINTR)
  1000. pblk_err(pblk, "could not initialize write thread\n");
  1001. goto fail_free_l2p;
  1002. }
  1003. ret = pblk_gc_init(pblk);
  1004. if (ret) {
  1005. pblk_err(pblk, "could not initialize gc\n");
  1006. goto fail_stop_writer;
  1007. }
  1008. /* inherit the size from the underlying device */
  1009. blk_queue_logical_block_size(tqueue, queue_physical_block_size(bqueue));
  1010. blk_queue_max_hw_sectors(tqueue, queue_max_hw_sectors(bqueue));
  1011. blk_queue_write_cache(tqueue, true, false);
  1012. tqueue->limits.discard_granularity = geo->clba * geo->csecs;
  1013. tqueue->limits.discard_alignment = 0;
  1014. blk_queue_max_discard_sectors(tqueue, UINT_MAX >> 9);
  1015. blk_queue_flag_set(QUEUE_FLAG_DISCARD, tqueue);
  1016. pblk_info(pblk, "luns:%u, lines:%d, secs:%llu, buf entries:%u\n",
  1017. geo->all_luns, pblk->l_mg.nr_lines,
  1018. (unsigned long long)pblk->capacity,
  1019. pblk->rwb.nr_entries);
  1020. wake_up_process(pblk->writer_ts);
  1021. /* Check if we need to start GC */
  1022. pblk_gc_should_kick(pblk);
  1023. return pblk;
  1024. fail_stop_writer:
  1025. pblk_writer_stop(pblk);
  1026. fail_free_l2p:
  1027. pblk_l2p_free(pblk);
  1028. fail_free_rwb:
  1029. pblk_rwb_free(pblk);
  1030. fail_free_lines:
  1031. pblk_lines_free(pblk);
  1032. fail_free_core:
  1033. pblk_core_free(pblk);
  1034. fail:
  1035. kfree(pblk);
  1036. return ERR_PTR(ret);
  1037. }
  1038. /* physical block device target */
  1039. static struct nvm_tgt_type tt_pblk = {
  1040. .name = "pblk",
  1041. .version = {1, 0, 0},
  1042. .bops = &pblk_bops,
  1043. .capacity = pblk_capacity,
  1044. .init = pblk_init,
  1045. .exit = pblk_exit,
  1046. .sysfs_init = pblk_sysfs_init,
  1047. .sysfs_exit = pblk_sysfs_exit,
  1048. .owner = THIS_MODULE,
  1049. };
  1050. static int __init pblk_module_init(void)
  1051. {
  1052. int ret;
  1053. ret = bioset_init(&pblk_bio_set, BIO_POOL_SIZE, 0, 0);
  1054. if (ret)
  1055. return ret;
  1056. ret = nvm_register_tgt_type(&tt_pblk);
  1057. if (ret)
  1058. bioset_exit(&pblk_bio_set);
  1059. return ret;
  1060. }
  1061. static void pblk_module_exit(void)
  1062. {
  1063. bioset_exit(&pblk_bio_set);
  1064. nvm_unregister_tgt_type(&tt_pblk);
  1065. }
  1066. module_init(pblk_module_init);
  1067. module_exit(pblk_module_exit);
  1068. MODULE_AUTHOR("Javier Gonzalez <javier@cnexlabs.com>");
  1069. MODULE_AUTHOR("Matias Bjorling <matias@cnexlabs.com>");
  1070. MODULE_LICENSE("GPL v2");
  1071. MODULE_DESCRIPTION("Physical Block-Device for Open-Channel SSDs");