pblk-gc.c 16 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2016 CNEX Labs
  4. * Initial release: Javier Gonzalez <javier@cnexlabs.com>
  5. * Matias Bjorling <matias@cnexlabs.com>
  6. *
  7. * This program is free software; you can redistribute it and/or
  8. * modify it under the terms of the GNU General Public License version
  9. * 2 as published by the Free Software Foundation.
  10. *
  11. * This program is distributed in the hope that it will be useful, but
  12. * WITHOUT ANY WARRANTY; without even the implied warranty of
  13. * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
  14. * General Public License for more details.
  15. *
  16. * pblk-gc.c - pblk's garbage collector
  17. */
  18. #include "pblk.h"
  19. #include "pblk-trace.h"
  20. #include <linux/delay.h>
  21. static void pblk_gc_free_gc_rq(struct pblk_gc_rq *gc_rq)
  22. {
  23. if (gc_rq->data)
  24. vfree(gc_rq->data);
  25. kfree(gc_rq);
  26. }
  27. static int pblk_gc_write(struct pblk *pblk)
  28. {
  29. struct pblk_gc *gc = &pblk->gc;
  30. struct pblk_gc_rq *gc_rq, *tgc_rq;
  31. LIST_HEAD(w_list);
  32. spin_lock(&gc->w_lock);
  33. if (list_empty(&gc->w_list)) {
  34. spin_unlock(&gc->w_lock);
  35. return 1;
  36. }
  37. list_cut_position(&w_list, &gc->w_list, gc->w_list.prev);
  38. gc->w_entries = 0;
  39. spin_unlock(&gc->w_lock);
  40. list_for_each_entry_safe(gc_rq, tgc_rq, &w_list, list) {
  41. pblk_write_gc_to_cache(pblk, gc_rq);
  42. list_del(&gc_rq->list);
  43. kref_put(&gc_rq->line->ref, pblk_line_put);
  44. pblk_gc_free_gc_rq(gc_rq);
  45. }
  46. return 0;
  47. }
  48. static void pblk_gc_writer_kick(struct pblk_gc *gc)
  49. {
  50. wake_up_process(gc->gc_writer_ts);
  51. }
  52. void pblk_put_line_back(struct pblk *pblk, struct pblk_line *line)
  53. {
  54. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  55. struct list_head *move_list;
  56. spin_lock(&l_mg->gc_lock);
  57. spin_lock(&line->lock);
  58. WARN_ON(line->state != PBLK_LINESTATE_GC);
  59. line->state = PBLK_LINESTATE_CLOSED;
  60. trace_pblk_line_state(pblk_disk_name(pblk), line->id,
  61. line->state);
  62. /* We need to reset gc_group in order to ensure that
  63. * pblk_line_gc_list will return proper move_list
  64. * since right now current line is not on any of the
  65. * gc lists.
  66. */
  67. line->gc_group = PBLK_LINEGC_NONE;
  68. move_list = pblk_line_gc_list(pblk, line);
  69. spin_unlock(&line->lock);
  70. list_add_tail(&line->list, move_list);
  71. spin_unlock(&l_mg->gc_lock);
  72. }
  73. static void pblk_gc_line_ws(struct work_struct *work)
  74. {
  75. struct pblk_line_ws *gc_rq_ws = container_of(work,
  76. struct pblk_line_ws, ws);
  77. struct pblk *pblk = gc_rq_ws->pblk;
  78. struct pblk_gc *gc = &pblk->gc;
  79. struct pblk_line *line = gc_rq_ws->line;
  80. struct pblk_gc_rq *gc_rq = gc_rq_ws->priv;
  81. int ret;
  82. up(&gc->gc_sem);
  83. /* Read from GC victim block */
  84. ret = pblk_submit_read_gc(pblk, gc_rq);
  85. if (ret) {
  86. line->w_err_gc->has_gc_err = 1;
  87. goto out;
  88. }
  89. if (!gc_rq->secs_to_gc)
  90. goto out;
  91. retry:
  92. spin_lock(&gc->w_lock);
  93. if (gc->w_entries >= PBLK_GC_RQ_QD) {
  94. spin_unlock(&gc->w_lock);
  95. pblk_gc_writer_kick(&pblk->gc);
  96. usleep_range(128, 256);
  97. goto retry;
  98. }
  99. gc->w_entries++;
  100. list_add_tail(&gc_rq->list, &gc->w_list);
  101. spin_unlock(&gc->w_lock);
  102. pblk_gc_writer_kick(&pblk->gc);
  103. kfree(gc_rq_ws);
  104. return;
  105. out:
  106. pblk_gc_free_gc_rq(gc_rq);
  107. kref_put(&line->ref, pblk_line_put);
  108. kfree(gc_rq_ws);
  109. }
  110. static __le64 *get_lba_list_from_emeta(struct pblk *pblk,
  111. struct pblk_line *line)
  112. {
  113. struct line_emeta *emeta_buf;
  114. struct pblk_line_meta *lm = &pblk->lm;
  115. unsigned int lba_list_size = lm->emeta_len[2];
  116. __le64 *lba_list;
  117. int ret;
  118. emeta_buf = kvmalloc(lm->emeta_len[0], GFP_KERNEL);
  119. if (!emeta_buf)
  120. return NULL;
  121. ret = pblk_line_emeta_read(pblk, line, emeta_buf);
  122. if (ret) {
  123. pblk_err(pblk, "line %d read emeta failed (%d)\n",
  124. line->id, ret);
  125. kvfree(emeta_buf);
  126. return NULL;
  127. }
  128. /* If this read fails, it means that emeta is corrupted.
  129. * For now, leave the line untouched.
  130. * TODO: Implement a recovery routine that scans and moves
  131. * all sectors on the line.
  132. */
  133. ret = pblk_recov_check_emeta(pblk, emeta_buf);
  134. if (ret) {
  135. pblk_err(pblk, "inconsistent emeta (line %d)\n",
  136. line->id);
  137. kvfree(emeta_buf);
  138. return NULL;
  139. }
  140. lba_list = kvmalloc(lba_list_size, GFP_KERNEL);
  141. if (lba_list)
  142. memcpy(lba_list, emeta_to_lbas(pblk, emeta_buf), lba_list_size);
  143. kvfree(emeta_buf);
  144. return lba_list;
  145. }
  146. static void pblk_gc_line_prepare_ws(struct work_struct *work)
  147. {
  148. struct pblk_line_ws *line_ws = container_of(work, struct pblk_line_ws,
  149. ws);
  150. struct pblk *pblk = line_ws->pblk;
  151. struct pblk_line *line = line_ws->line;
  152. struct pblk_line_meta *lm = &pblk->lm;
  153. struct nvm_tgt_dev *dev = pblk->dev;
  154. struct nvm_geo *geo = &dev->geo;
  155. struct pblk_gc *gc = &pblk->gc;
  156. struct pblk_line_ws *gc_rq_ws;
  157. struct pblk_gc_rq *gc_rq;
  158. __le64 *lba_list;
  159. unsigned long *invalid_bitmap;
  160. int sec_left, nr_secs, bit;
  161. invalid_bitmap = kmalloc(lm->sec_bitmap_len, GFP_KERNEL);
  162. if (!invalid_bitmap)
  163. goto fail_free_ws;
  164. if (line->w_err_gc->has_write_err) {
  165. lba_list = line->w_err_gc->lba_list;
  166. line->w_err_gc->lba_list = NULL;
  167. } else {
  168. lba_list = get_lba_list_from_emeta(pblk, line);
  169. if (!lba_list) {
  170. pblk_err(pblk, "could not interpret emeta (line %d)\n",
  171. line->id);
  172. goto fail_free_invalid_bitmap;
  173. }
  174. }
  175. spin_lock(&line->lock);
  176. bitmap_copy(invalid_bitmap, line->invalid_bitmap, lm->sec_per_line);
  177. sec_left = pblk_line_vsc(line);
  178. spin_unlock(&line->lock);
  179. if (sec_left < 0) {
  180. pblk_err(pblk, "corrupted GC line (%d)\n", line->id);
  181. goto fail_free_lba_list;
  182. }
  183. bit = -1;
  184. next_rq:
  185. gc_rq = kmalloc(sizeof(struct pblk_gc_rq), GFP_KERNEL);
  186. if (!gc_rq)
  187. goto fail_free_lba_list;
  188. nr_secs = 0;
  189. do {
  190. bit = find_next_zero_bit(invalid_bitmap, lm->sec_per_line,
  191. bit + 1);
  192. if (bit > line->emeta_ssec)
  193. break;
  194. gc_rq->paddr_list[nr_secs] = bit;
  195. gc_rq->lba_list[nr_secs++] = le64_to_cpu(lba_list[bit]);
  196. } while (nr_secs < pblk->max_write_pgs);
  197. if (unlikely(!nr_secs)) {
  198. kfree(gc_rq);
  199. goto out;
  200. }
  201. gc_rq->nr_secs = nr_secs;
  202. gc_rq->line = line;
  203. gc_rq->data = vmalloc(array_size(gc_rq->nr_secs, geo->csecs));
  204. if (!gc_rq->data)
  205. goto fail_free_gc_rq;
  206. gc_rq_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
  207. if (!gc_rq_ws)
  208. goto fail_free_gc_data;
  209. gc_rq_ws->pblk = pblk;
  210. gc_rq_ws->line = line;
  211. gc_rq_ws->priv = gc_rq;
  212. /* The write GC path can be much slower than the read GC one due to
  213. * the budget imposed by the rate-limiter. Balance in case that we get
  214. * back pressure from the write GC path.
  215. */
  216. while (down_timeout(&gc->gc_sem, msecs_to_jiffies(30000)))
  217. io_schedule();
  218. kref_get(&line->ref);
  219. INIT_WORK(&gc_rq_ws->ws, pblk_gc_line_ws);
  220. queue_work(gc->gc_line_reader_wq, &gc_rq_ws->ws);
  221. sec_left -= nr_secs;
  222. if (sec_left > 0)
  223. goto next_rq;
  224. out:
  225. kvfree(lba_list);
  226. kfree(line_ws);
  227. kfree(invalid_bitmap);
  228. kref_put(&line->ref, pblk_line_put);
  229. atomic_dec(&gc->read_inflight_gc);
  230. return;
  231. fail_free_gc_data:
  232. vfree(gc_rq->data);
  233. fail_free_gc_rq:
  234. kfree(gc_rq);
  235. fail_free_lba_list:
  236. kvfree(lba_list);
  237. fail_free_invalid_bitmap:
  238. kfree(invalid_bitmap);
  239. fail_free_ws:
  240. kfree(line_ws);
  241. /* Line goes back to closed state, so we cannot release additional
  242. * reference for line, since we do that only when we want to do
  243. * gc to free line state transition.
  244. */
  245. pblk_put_line_back(pblk, line);
  246. atomic_dec(&gc->read_inflight_gc);
  247. pblk_err(pblk, "failed to GC line %d\n", line->id);
  248. }
  249. static int pblk_gc_line(struct pblk *pblk, struct pblk_line *line)
  250. {
  251. struct pblk_gc *gc = &pblk->gc;
  252. struct pblk_line_ws *line_ws;
  253. pblk_debug(pblk, "line '%d' being reclaimed for GC\n", line->id);
  254. line_ws = kmalloc(sizeof(struct pblk_line_ws), GFP_KERNEL);
  255. if (!line_ws)
  256. return -ENOMEM;
  257. line_ws->pblk = pblk;
  258. line_ws->line = line;
  259. atomic_inc(&gc->pipeline_gc);
  260. INIT_WORK(&line_ws->ws, pblk_gc_line_prepare_ws);
  261. queue_work(gc->gc_reader_wq, &line_ws->ws);
  262. return 0;
  263. }
  264. static void pblk_gc_reader_kick(struct pblk_gc *gc)
  265. {
  266. wake_up_process(gc->gc_reader_ts);
  267. }
  268. static void pblk_gc_kick(struct pblk *pblk)
  269. {
  270. struct pblk_gc *gc = &pblk->gc;
  271. pblk_gc_writer_kick(gc);
  272. pblk_gc_reader_kick(gc);
  273. /* If we're shutting down GC, let's not start it up again */
  274. if (gc->gc_enabled) {
  275. wake_up_process(gc->gc_ts);
  276. mod_timer(&gc->gc_timer,
  277. jiffies + msecs_to_jiffies(GC_TIME_MSECS));
  278. }
  279. }
  280. static int pblk_gc_read(struct pblk *pblk)
  281. {
  282. struct pblk_gc *gc = &pblk->gc;
  283. struct pblk_line *line;
  284. spin_lock(&gc->r_lock);
  285. if (list_empty(&gc->r_list)) {
  286. spin_unlock(&gc->r_lock);
  287. return 1;
  288. }
  289. line = list_first_entry(&gc->r_list, struct pblk_line, list);
  290. list_del(&line->list);
  291. spin_unlock(&gc->r_lock);
  292. pblk_gc_kick(pblk);
  293. if (pblk_gc_line(pblk, line)) {
  294. pblk_err(pblk, "failed to GC line %d\n", line->id);
  295. /* rollback */
  296. spin_lock(&gc->r_lock);
  297. list_add_tail(&line->list, &gc->r_list);
  298. spin_unlock(&gc->r_lock);
  299. }
  300. return 0;
  301. }
  302. static struct pblk_line *pblk_gc_get_victim_line(struct pblk *pblk,
  303. struct list_head *group_list)
  304. {
  305. struct pblk_line *line, *victim;
  306. unsigned int line_vsc = ~0x0L, victim_vsc = ~0x0L;
  307. victim = list_first_entry(group_list, struct pblk_line, list);
  308. list_for_each_entry(line, group_list, list) {
  309. if (!atomic_read(&line->sec_to_update))
  310. line_vsc = le32_to_cpu(*line->vsc);
  311. if (line_vsc < victim_vsc) {
  312. victim = line;
  313. victim_vsc = le32_to_cpu(*victim->vsc);
  314. }
  315. }
  316. if (victim_vsc == ~0x0)
  317. return NULL;
  318. return victim;
  319. }
  320. static bool pblk_gc_should_run(struct pblk_gc *gc, struct pblk_rl *rl)
  321. {
  322. unsigned int nr_blocks_free, nr_blocks_need;
  323. unsigned int werr_lines = atomic_read(&rl->werr_lines);
  324. nr_blocks_need = pblk_rl_high_thrs(rl);
  325. nr_blocks_free = pblk_rl_nr_free_blks(rl);
  326. /* This is not critical, no need to take lock here */
  327. return ((werr_lines > 0) ||
  328. ((gc->gc_active) && (nr_blocks_need > nr_blocks_free)));
  329. }
  330. void pblk_gc_free_full_lines(struct pblk *pblk)
  331. {
  332. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  333. struct pblk_gc *gc = &pblk->gc;
  334. struct pblk_line *line;
  335. do {
  336. spin_lock(&l_mg->gc_lock);
  337. if (list_empty(&l_mg->gc_full_list)) {
  338. spin_unlock(&l_mg->gc_lock);
  339. return;
  340. }
  341. line = list_first_entry(&l_mg->gc_full_list,
  342. struct pblk_line, list);
  343. spin_lock(&line->lock);
  344. WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
  345. line->state = PBLK_LINESTATE_GC;
  346. trace_pblk_line_state(pblk_disk_name(pblk), line->id,
  347. line->state);
  348. spin_unlock(&line->lock);
  349. list_del(&line->list);
  350. spin_unlock(&l_mg->gc_lock);
  351. atomic_inc(&gc->pipeline_gc);
  352. kref_put(&line->ref, pblk_line_put);
  353. } while (1);
  354. }
  355. /*
  356. * Lines with no valid sectors will be returned to the free list immediately. If
  357. * GC is activated - either because the free block count is under the determined
  358. * threshold, or because it is being forced from user space - only lines with a
  359. * high count of invalid sectors will be recycled.
  360. */
  361. static void pblk_gc_run(struct pblk *pblk)
  362. {
  363. struct pblk_line_mgmt *l_mg = &pblk->l_mg;
  364. struct pblk_gc *gc = &pblk->gc;
  365. struct pblk_line *line;
  366. struct list_head *group_list;
  367. bool run_gc;
  368. int read_inflight_gc, gc_group = 0, prev_group = 0;
  369. pblk_gc_free_full_lines(pblk);
  370. run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
  371. if (!run_gc || (atomic_read(&gc->read_inflight_gc) >= PBLK_GC_L_QD))
  372. return;
  373. next_gc_group:
  374. group_list = l_mg->gc_lists[gc_group++];
  375. do {
  376. spin_lock(&l_mg->gc_lock);
  377. line = pblk_gc_get_victim_line(pblk, group_list);
  378. if (!line) {
  379. spin_unlock(&l_mg->gc_lock);
  380. break;
  381. }
  382. spin_lock(&line->lock);
  383. WARN_ON(line->state != PBLK_LINESTATE_CLOSED);
  384. line->state = PBLK_LINESTATE_GC;
  385. trace_pblk_line_state(pblk_disk_name(pblk), line->id,
  386. line->state);
  387. spin_unlock(&line->lock);
  388. list_del(&line->list);
  389. spin_unlock(&l_mg->gc_lock);
  390. spin_lock(&gc->r_lock);
  391. list_add_tail(&line->list, &gc->r_list);
  392. spin_unlock(&gc->r_lock);
  393. read_inflight_gc = atomic_inc_return(&gc->read_inflight_gc);
  394. pblk_gc_reader_kick(gc);
  395. prev_group = 1;
  396. /* No need to queue up more GC lines than we can handle */
  397. run_gc = pblk_gc_should_run(&pblk->gc, &pblk->rl);
  398. if (!run_gc || read_inflight_gc >= PBLK_GC_L_QD)
  399. break;
  400. } while (1);
  401. if (!prev_group && pblk->rl.rb_state > gc_group &&
  402. gc_group < PBLK_GC_NR_LISTS)
  403. goto next_gc_group;
  404. }
  405. static void pblk_gc_timer(struct timer_list *t)
  406. {
  407. struct pblk *pblk = from_timer(pblk, t, gc.gc_timer);
  408. pblk_gc_kick(pblk);
  409. }
  410. static int pblk_gc_ts(void *data)
  411. {
  412. struct pblk *pblk = data;
  413. while (!kthread_should_stop()) {
  414. pblk_gc_run(pblk);
  415. set_current_state(TASK_INTERRUPTIBLE);
  416. io_schedule();
  417. }
  418. return 0;
  419. }
  420. static int pblk_gc_writer_ts(void *data)
  421. {
  422. struct pblk *pblk = data;
  423. while (!kthread_should_stop()) {
  424. if (!pblk_gc_write(pblk))
  425. continue;
  426. set_current_state(TASK_INTERRUPTIBLE);
  427. io_schedule();
  428. }
  429. return 0;
  430. }
  431. static int pblk_gc_reader_ts(void *data)
  432. {
  433. struct pblk *pblk = data;
  434. struct pblk_gc *gc = &pblk->gc;
  435. while (!kthread_should_stop()) {
  436. if (!pblk_gc_read(pblk))
  437. continue;
  438. set_current_state(TASK_INTERRUPTIBLE);
  439. io_schedule();
  440. }
  441. #ifdef CONFIG_NVM_PBLK_DEBUG
  442. pblk_info(pblk, "flushing gc pipeline, %d lines left\n",
  443. atomic_read(&gc->pipeline_gc));
  444. #endif
  445. do {
  446. if (!atomic_read(&gc->pipeline_gc))
  447. break;
  448. schedule();
  449. } while (1);
  450. return 0;
  451. }
  452. static void pblk_gc_start(struct pblk *pblk)
  453. {
  454. pblk->gc.gc_active = 1;
  455. pblk_debug(pblk, "gc start\n");
  456. }
  457. void pblk_gc_should_start(struct pblk *pblk)
  458. {
  459. struct pblk_gc *gc = &pblk->gc;
  460. if (gc->gc_enabled && !gc->gc_active) {
  461. pblk_gc_start(pblk);
  462. pblk_gc_kick(pblk);
  463. }
  464. }
  465. void pblk_gc_should_stop(struct pblk *pblk)
  466. {
  467. struct pblk_gc *gc = &pblk->gc;
  468. if (gc->gc_active && !gc->gc_forced)
  469. gc->gc_active = 0;
  470. }
  471. void pblk_gc_should_kick(struct pblk *pblk)
  472. {
  473. pblk_rl_update_rates(&pblk->rl);
  474. }
  475. void pblk_gc_sysfs_state_show(struct pblk *pblk, int *gc_enabled,
  476. int *gc_active)
  477. {
  478. struct pblk_gc *gc = &pblk->gc;
  479. spin_lock(&gc->lock);
  480. *gc_enabled = gc->gc_enabled;
  481. *gc_active = gc->gc_active;
  482. spin_unlock(&gc->lock);
  483. }
  484. int pblk_gc_sysfs_force(struct pblk *pblk, int force)
  485. {
  486. struct pblk_gc *gc = &pblk->gc;
  487. if (force < 0 || force > 1)
  488. return -EINVAL;
  489. spin_lock(&gc->lock);
  490. gc->gc_forced = force;
  491. if (force)
  492. gc->gc_enabled = 1;
  493. else
  494. gc->gc_enabled = 0;
  495. spin_unlock(&gc->lock);
  496. pblk_gc_should_start(pblk);
  497. return 0;
  498. }
  499. int pblk_gc_init(struct pblk *pblk)
  500. {
  501. struct pblk_gc *gc = &pblk->gc;
  502. int ret;
  503. gc->gc_ts = kthread_create(pblk_gc_ts, pblk, "pblk-gc-ts");
  504. if (IS_ERR(gc->gc_ts)) {
  505. pblk_err(pblk, "could not allocate GC main kthread\n");
  506. return PTR_ERR(gc->gc_ts);
  507. }
  508. gc->gc_writer_ts = kthread_create(pblk_gc_writer_ts, pblk,
  509. "pblk-gc-writer-ts");
  510. if (IS_ERR(gc->gc_writer_ts)) {
  511. pblk_err(pblk, "could not allocate GC writer kthread\n");
  512. ret = PTR_ERR(gc->gc_writer_ts);
  513. goto fail_free_main_kthread;
  514. }
  515. gc->gc_reader_ts = kthread_create(pblk_gc_reader_ts, pblk,
  516. "pblk-gc-reader-ts");
  517. if (IS_ERR(gc->gc_reader_ts)) {
  518. pblk_err(pblk, "could not allocate GC reader kthread\n");
  519. ret = PTR_ERR(gc->gc_reader_ts);
  520. goto fail_free_writer_kthread;
  521. }
  522. timer_setup(&gc->gc_timer, pblk_gc_timer, 0);
  523. mod_timer(&gc->gc_timer, jiffies + msecs_to_jiffies(GC_TIME_MSECS));
  524. gc->gc_active = 0;
  525. gc->gc_forced = 0;
  526. gc->gc_enabled = 1;
  527. gc->w_entries = 0;
  528. atomic_set(&gc->read_inflight_gc, 0);
  529. atomic_set(&gc->pipeline_gc, 0);
  530. /* Workqueue that reads valid sectors from a line and submit them to the
  531. * GC writer to be recycled.
  532. */
  533. gc->gc_line_reader_wq = alloc_workqueue("pblk-gc-line-reader-wq",
  534. WQ_MEM_RECLAIM | WQ_UNBOUND, PBLK_GC_MAX_READERS);
  535. if (!gc->gc_line_reader_wq) {
  536. pblk_err(pblk, "could not allocate GC line reader workqueue\n");
  537. ret = -ENOMEM;
  538. goto fail_free_reader_kthread;
  539. }
  540. /* Workqueue that prepare lines for GC */
  541. gc->gc_reader_wq = alloc_workqueue("pblk-gc-line_wq",
  542. WQ_MEM_RECLAIM | WQ_UNBOUND, 1);
  543. if (!gc->gc_reader_wq) {
  544. pblk_err(pblk, "could not allocate GC reader workqueue\n");
  545. ret = -ENOMEM;
  546. goto fail_free_reader_line_wq;
  547. }
  548. spin_lock_init(&gc->lock);
  549. spin_lock_init(&gc->w_lock);
  550. spin_lock_init(&gc->r_lock);
  551. sema_init(&gc->gc_sem, PBLK_GC_RQ_QD);
  552. INIT_LIST_HEAD(&gc->w_list);
  553. INIT_LIST_HEAD(&gc->r_list);
  554. return 0;
  555. fail_free_reader_line_wq:
  556. destroy_workqueue(gc->gc_line_reader_wq);
  557. fail_free_reader_kthread:
  558. kthread_stop(gc->gc_reader_ts);
  559. fail_free_writer_kthread:
  560. kthread_stop(gc->gc_writer_ts);
  561. fail_free_main_kthread:
  562. kthread_stop(gc->gc_ts);
  563. return ret;
  564. }
  565. void pblk_gc_exit(struct pblk *pblk, bool graceful)
  566. {
  567. struct pblk_gc *gc = &pblk->gc;
  568. gc->gc_enabled = 0;
  569. del_timer_sync(&gc->gc_timer);
  570. gc->gc_active = 0;
  571. if (gc->gc_ts)
  572. kthread_stop(gc->gc_ts);
  573. if (gc->gc_reader_ts)
  574. kthread_stop(gc->gc_reader_ts);
  575. if (graceful) {
  576. flush_workqueue(gc->gc_reader_wq);
  577. flush_workqueue(gc->gc_line_reader_wq);
  578. }
  579. destroy_workqueue(gc->gc_reader_wq);
  580. destroy_workqueue(gc->gc_line_reader_wq);
  581. if (gc->gc_writer_ts)
  582. kthread_stop(gc->gc_writer_ts);
  583. }