dm-region-hash.c 18 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724
  1. /*
  2. * Copyright (C) 2003 Sistina Software Limited.
  3. * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include <linux/dm-dirty-log.h>
  8. #include <linux/dm-region-hash.h>
  9. #include <linux/ctype.h>
  10. #include <linux/init.h>
  11. #include <linux/module.h>
  12. #include <linux/slab.h>
  13. #include <linux/vmalloc.h>
  14. #include "dm.h"
  15. #define DM_MSG_PREFIX "region hash"
  16. /*-----------------------------------------------------------------
  17. * Region hash
  18. *
  19. * The mirror splits itself up into discrete regions. Each
  20. * region can be in one of three states: clean, dirty,
  21. * nosync. There is no need to put clean regions in the hash.
  22. *
  23. * In addition to being present in the hash table a region _may_
  24. * be present on one of three lists.
  25. *
  26. * clean_regions: Regions on this list have no io pending to
  27. * them, they are in sync, we are no longer interested in them,
  28. * they are dull. dm_rh_update_states() will remove them from the
  29. * hash table.
  30. *
  31. * quiesced_regions: These regions have been spun down, ready
  32. * for recovery. rh_recovery_start() will remove regions from
  33. * this list and hand them to kmirrord, which will schedule the
  34. * recovery io with kcopyd.
  35. *
  36. * recovered_regions: Regions that kcopyd has successfully
  37. * recovered. dm_rh_update_states() will now schedule any delayed
  38. * io, up the recovery_count, and remove the region from the
  39. * hash.
  40. *
  41. * There are 2 locks:
  42. * A rw spin lock 'hash_lock' protects just the hash table,
  43. * this is never held in write mode from interrupt context,
  44. * which I believe means that we only have to disable irqs when
  45. * doing a write lock.
  46. *
  47. * An ordinary spin lock 'region_lock' that protects the three
  48. * lists in the region_hash, with the 'state', 'list' and
  49. * 'delayed_bios' fields of the regions. This is used from irq
  50. * context, so all other uses will have to suspend local irqs.
  51. *---------------------------------------------------------------*/
  52. struct dm_region_hash {
  53. uint32_t region_size;
  54. unsigned region_shift;
  55. /* holds persistent region state */
  56. struct dm_dirty_log *log;
  57. /* hash table */
  58. rwlock_t hash_lock;
  59. unsigned mask;
  60. unsigned nr_buckets;
  61. unsigned prime;
  62. unsigned shift;
  63. struct list_head *buckets;
  64. /*
  65. * If there was a flush failure no regions can be marked clean.
  66. */
  67. int flush_failure;
  68. unsigned max_recovery; /* Max # of regions to recover in parallel */
  69. spinlock_t region_lock;
  70. atomic_t recovery_in_flight;
  71. struct list_head clean_regions;
  72. struct list_head quiesced_regions;
  73. struct list_head recovered_regions;
  74. struct list_head failed_recovered_regions;
  75. struct semaphore recovery_count;
  76. mempool_t region_pool;
  77. void *context;
  78. sector_t target_begin;
  79. /* Callback function to schedule bios writes */
  80. void (*dispatch_bios)(void *context, struct bio_list *bios);
  81. /* Callback function to wakeup callers worker thread. */
  82. void (*wakeup_workers)(void *context);
  83. /* Callback function to wakeup callers recovery waiters. */
  84. void (*wakeup_all_recovery_waiters)(void *context);
  85. };
  86. struct dm_region {
  87. struct dm_region_hash *rh; /* FIXME: can we get rid of this ? */
  88. region_t key;
  89. int state;
  90. struct list_head hash_list;
  91. struct list_head list;
  92. atomic_t pending;
  93. struct bio_list delayed_bios;
  94. };
  95. /*
  96. * Conversion fns
  97. */
  98. static region_t dm_rh_sector_to_region(struct dm_region_hash *rh, sector_t sector)
  99. {
  100. return sector >> rh->region_shift;
  101. }
  102. sector_t dm_rh_region_to_sector(struct dm_region_hash *rh, region_t region)
  103. {
  104. return region << rh->region_shift;
  105. }
  106. EXPORT_SYMBOL_GPL(dm_rh_region_to_sector);
  107. region_t dm_rh_bio_to_region(struct dm_region_hash *rh, struct bio *bio)
  108. {
  109. return dm_rh_sector_to_region(rh, bio->bi_iter.bi_sector -
  110. rh->target_begin);
  111. }
  112. EXPORT_SYMBOL_GPL(dm_rh_bio_to_region);
  113. void *dm_rh_region_context(struct dm_region *reg)
  114. {
  115. return reg->rh->context;
  116. }
  117. EXPORT_SYMBOL_GPL(dm_rh_region_context);
  118. region_t dm_rh_get_region_key(struct dm_region *reg)
  119. {
  120. return reg->key;
  121. }
  122. EXPORT_SYMBOL_GPL(dm_rh_get_region_key);
  123. sector_t dm_rh_get_region_size(struct dm_region_hash *rh)
  124. {
  125. return rh->region_size;
  126. }
  127. EXPORT_SYMBOL_GPL(dm_rh_get_region_size);
  128. /*
  129. * FIXME: shall we pass in a structure instead of all these args to
  130. * dm_region_hash_create()????
  131. */
  132. #define RH_HASH_MULT 2654435387U
  133. #define RH_HASH_SHIFT 12
  134. #define MIN_REGIONS 64
  135. struct dm_region_hash *dm_region_hash_create(
  136. void *context, void (*dispatch_bios)(void *context,
  137. struct bio_list *bios),
  138. void (*wakeup_workers)(void *context),
  139. void (*wakeup_all_recovery_waiters)(void *context),
  140. sector_t target_begin, unsigned max_recovery,
  141. struct dm_dirty_log *log, uint32_t region_size,
  142. region_t nr_regions)
  143. {
  144. struct dm_region_hash *rh;
  145. unsigned nr_buckets, max_buckets;
  146. size_t i;
  147. int ret;
  148. /*
  149. * Calculate a suitable number of buckets for our hash
  150. * table.
  151. */
  152. max_buckets = nr_regions >> 6;
  153. for (nr_buckets = 128u; nr_buckets < max_buckets; nr_buckets <<= 1)
  154. ;
  155. nr_buckets >>= 1;
  156. rh = kzalloc(sizeof(*rh), GFP_KERNEL);
  157. if (!rh) {
  158. DMERR("unable to allocate region hash memory");
  159. return ERR_PTR(-ENOMEM);
  160. }
  161. rh->context = context;
  162. rh->dispatch_bios = dispatch_bios;
  163. rh->wakeup_workers = wakeup_workers;
  164. rh->wakeup_all_recovery_waiters = wakeup_all_recovery_waiters;
  165. rh->target_begin = target_begin;
  166. rh->max_recovery = max_recovery;
  167. rh->log = log;
  168. rh->region_size = region_size;
  169. rh->region_shift = __ffs(region_size);
  170. rwlock_init(&rh->hash_lock);
  171. rh->mask = nr_buckets - 1;
  172. rh->nr_buckets = nr_buckets;
  173. rh->shift = RH_HASH_SHIFT;
  174. rh->prime = RH_HASH_MULT;
  175. rh->buckets = vmalloc(array_size(nr_buckets, sizeof(*rh->buckets)));
  176. if (!rh->buckets) {
  177. DMERR("unable to allocate region hash bucket memory");
  178. kfree(rh);
  179. return ERR_PTR(-ENOMEM);
  180. }
  181. for (i = 0; i < nr_buckets; i++)
  182. INIT_LIST_HEAD(rh->buckets + i);
  183. spin_lock_init(&rh->region_lock);
  184. sema_init(&rh->recovery_count, 0);
  185. atomic_set(&rh->recovery_in_flight, 0);
  186. INIT_LIST_HEAD(&rh->clean_regions);
  187. INIT_LIST_HEAD(&rh->quiesced_regions);
  188. INIT_LIST_HEAD(&rh->recovered_regions);
  189. INIT_LIST_HEAD(&rh->failed_recovered_regions);
  190. rh->flush_failure = 0;
  191. ret = mempool_init_kmalloc_pool(&rh->region_pool, MIN_REGIONS,
  192. sizeof(struct dm_region));
  193. if (ret) {
  194. vfree(rh->buckets);
  195. kfree(rh);
  196. rh = ERR_PTR(-ENOMEM);
  197. }
  198. return rh;
  199. }
  200. EXPORT_SYMBOL_GPL(dm_region_hash_create);
  201. void dm_region_hash_destroy(struct dm_region_hash *rh)
  202. {
  203. unsigned h;
  204. struct dm_region *reg, *nreg;
  205. BUG_ON(!list_empty(&rh->quiesced_regions));
  206. for (h = 0; h < rh->nr_buckets; h++) {
  207. list_for_each_entry_safe(reg, nreg, rh->buckets + h,
  208. hash_list) {
  209. BUG_ON(atomic_read(&reg->pending));
  210. mempool_free(reg, &rh->region_pool);
  211. }
  212. }
  213. if (rh->log)
  214. dm_dirty_log_destroy(rh->log);
  215. mempool_exit(&rh->region_pool);
  216. vfree(rh->buckets);
  217. kfree(rh);
  218. }
  219. EXPORT_SYMBOL_GPL(dm_region_hash_destroy);
  220. struct dm_dirty_log *dm_rh_dirty_log(struct dm_region_hash *rh)
  221. {
  222. return rh->log;
  223. }
  224. EXPORT_SYMBOL_GPL(dm_rh_dirty_log);
  225. static unsigned rh_hash(struct dm_region_hash *rh, region_t region)
  226. {
  227. return (unsigned) ((region * rh->prime) >> rh->shift) & rh->mask;
  228. }
  229. static struct dm_region *__rh_lookup(struct dm_region_hash *rh, region_t region)
  230. {
  231. struct dm_region *reg;
  232. struct list_head *bucket = rh->buckets + rh_hash(rh, region);
  233. list_for_each_entry(reg, bucket, hash_list)
  234. if (reg->key == region)
  235. return reg;
  236. return NULL;
  237. }
  238. static void __rh_insert(struct dm_region_hash *rh, struct dm_region *reg)
  239. {
  240. list_add(&reg->hash_list, rh->buckets + rh_hash(rh, reg->key));
  241. }
  242. static struct dm_region *__rh_alloc(struct dm_region_hash *rh, region_t region)
  243. {
  244. struct dm_region *reg, *nreg;
  245. nreg = mempool_alloc(&rh->region_pool, GFP_ATOMIC);
  246. if (unlikely(!nreg))
  247. nreg = kmalloc(sizeof(*nreg), GFP_NOIO | __GFP_NOFAIL);
  248. nreg->state = rh->log->type->in_sync(rh->log, region, 1) ?
  249. DM_RH_CLEAN : DM_RH_NOSYNC;
  250. nreg->rh = rh;
  251. nreg->key = region;
  252. INIT_LIST_HEAD(&nreg->list);
  253. atomic_set(&nreg->pending, 0);
  254. bio_list_init(&nreg->delayed_bios);
  255. write_lock_irq(&rh->hash_lock);
  256. reg = __rh_lookup(rh, region);
  257. if (reg)
  258. /* We lost the race. */
  259. mempool_free(nreg, &rh->region_pool);
  260. else {
  261. __rh_insert(rh, nreg);
  262. if (nreg->state == DM_RH_CLEAN) {
  263. spin_lock(&rh->region_lock);
  264. list_add(&nreg->list, &rh->clean_regions);
  265. spin_unlock(&rh->region_lock);
  266. }
  267. reg = nreg;
  268. }
  269. write_unlock_irq(&rh->hash_lock);
  270. return reg;
  271. }
  272. static struct dm_region *__rh_find(struct dm_region_hash *rh, region_t region)
  273. {
  274. struct dm_region *reg;
  275. reg = __rh_lookup(rh, region);
  276. if (!reg) {
  277. read_unlock(&rh->hash_lock);
  278. reg = __rh_alloc(rh, region);
  279. read_lock(&rh->hash_lock);
  280. }
  281. return reg;
  282. }
  283. int dm_rh_get_state(struct dm_region_hash *rh, region_t region, int may_block)
  284. {
  285. int r;
  286. struct dm_region *reg;
  287. read_lock(&rh->hash_lock);
  288. reg = __rh_lookup(rh, region);
  289. read_unlock(&rh->hash_lock);
  290. if (reg)
  291. return reg->state;
  292. /*
  293. * The region wasn't in the hash, so we fall back to the
  294. * dirty log.
  295. */
  296. r = rh->log->type->in_sync(rh->log, region, may_block);
  297. /*
  298. * Any error from the dirty log (eg. -EWOULDBLOCK) gets
  299. * taken as a DM_RH_NOSYNC
  300. */
  301. return r == 1 ? DM_RH_CLEAN : DM_RH_NOSYNC;
  302. }
  303. EXPORT_SYMBOL_GPL(dm_rh_get_state);
  304. static void complete_resync_work(struct dm_region *reg, int success)
  305. {
  306. struct dm_region_hash *rh = reg->rh;
  307. rh->log->type->set_region_sync(rh->log, reg->key, success);
  308. /*
  309. * Dispatch the bios before we call 'wake_up_all'.
  310. * This is important because if we are suspending,
  311. * we want to know that recovery is complete and
  312. * the work queue is flushed. If we wake_up_all
  313. * before we dispatch_bios (queue bios and call wake()),
  314. * then we risk suspending before the work queue
  315. * has been properly flushed.
  316. */
  317. rh->dispatch_bios(rh->context, &reg->delayed_bios);
  318. if (atomic_dec_and_test(&rh->recovery_in_flight))
  319. rh->wakeup_all_recovery_waiters(rh->context);
  320. up(&rh->recovery_count);
  321. }
  322. /* dm_rh_mark_nosync
  323. * @ms
  324. * @bio
  325. *
  326. * The bio was written on some mirror(s) but failed on other mirror(s).
  327. * We can successfully endio the bio but should avoid the region being
  328. * marked clean by setting the state DM_RH_NOSYNC.
  329. *
  330. * This function is _not_ safe in interrupt context!
  331. */
  332. void dm_rh_mark_nosync(struct dm_region_hash *rh, struct bio *bio)
  333. {
  334. unsigned long flags;
  335. struct dm_dirty_log *log = rh->log;
  336. struct dm_region *reg;
  337. region_t region = dm_rh_bio_to_region(rh, bio);
  338. int recovering = 0;
  339. if (bio->bi_opf & REQ_PREFLUSH) {
  340. rh->flush_failure = 1;
  341. return;
  342. }
  343. if (bio_op(bio) == REQ_OP_DISCARD)
  344. return;
  345. /* We must inform the log that the sync count has changed. */
  346. log->type->set_region_sync(log, region, 0);
  347. read_lock(&rh->hash_lock);
  348. reg = __rh_find(rh, region);
  349. read_unlock(&rh->hash_lock);
  350. /* region hash entry should exist because write was in-flight */
  351. BUG_ON(!reg);
  352. BUG_ON(!list_empty(&reg->list));
  353. spin_lock_irqsave(&rh->region_lock, flags);
  354. /*
  355. * Possible cases:
  356. * 1) DM_RH_DIRTY
  357. * 2) DM_RH_NOSYNC: was dirty, other preceding writes failed
  358. * 3) DM_RH_RECOVERING: flushing pending writes
  359. * Either case, the region should have not been connected to list.
  360. */
  361. recovering = (reg->state == DM_RH_RECOVERING);
  362. reg->state = DM_RH_NOSYNC;
  363. BUG_ON(!list_empty(&reg->list));
  364. spin_unlock_irqrestore(&rh->region_lock, flags);
  365. if (recovering)
  366. complete_resync_work(reg, 0);
  367. }
  368. EXPORT_SYMBOL_GPL(dm_rh_mark_nosync);
  369. void dm_rh_update_states(struct dm_region_hash *rh, int errors_handled)
  370. {
  371. struct dm_region *reg, *next;
  372. LIST_HEAD(clean);
  373. LIST_HEAD(recovered);
  374. LIST_HEAD(failed_recovered);
  375. /*
  376. * Quickly grab the lists.
  377. */
  378. write_lock_irq(&rh->hash_lock);
  379. spin_lock(&rh->region_lock);
  380. if (!list_empty(&rh->clean_regions)) {
  381. list_splice_init(&rh->clean_regions, &clean);
  382. list_for_each_entry(reg, &clean, list)
  383. list_del(&reg->hash_list);
  384. }
  385. if (!list_empty(&rh->recovered_regions)) {
  386. list_splice_init(&rh->recovered_regions, &recovered);
  387. list_for_each_entry(reg, &recovered, list)
  388. list_del(&reg->hash_list);
  389. }
  390. if (!list_empty(&rh->failed_recovered_regions)) {
  391. list_splice_init(&rh->failed_recovered_regions,
  392. &failed_recovered);
  393. list_for_each_entry(reg, &failed_recovered, list)
  394. list_del(&reg->hash_list);
  395. }
  396. spin_unlock(&rh->region_lock);
  397. write_unlock_irq(&rh->hash_lock);
  398. /*
  399. * All the regions on the recovered and clean lists have
  400. * now been pulled out of the system, so no need to do
  401. * any more locking.
  402. */
  403. list_for_each_entry_safe(reg, next, &recovered, list) {
  404. rh->log->type->clear_region(rh->log, reg->key);
  405. complete_resync_work(reg, 1);
  406. mempool_free(reg, &rh->region_pool);
  407. }
  408. list_for_each_entry_safe(reg, next, &failed_recovered, list) {
  409. complete_resync_work(reg, errors_handled ? 0 : 1);
  410. mempool_free(reg, &rh->region_pool);
  411. }
  412. list_for_each_entry_safe(reg, next, &clean, list) {
  413. rh->log->type->clear_region(rh->log, reg->key);
  414. mempool_free(reg, &rh->region_pool);
  415. }
  416. rh->log->type->flush(rh->log);
  417. }
  418. EXPORT_SYMBOL_GPL(dm_rh_update_states);
  419. static void rh_inc(struct dm_region_hash *rh, region_t region)
  420. {
  421. struct dm_region *reg;
  422. read_lock(&rh->hash_lock);
  423. reg = __rh_find(rh, region);
  424. spin_lock_irq(&rh->region_lock);
  425. atomic_inc(&reg->pending);
  426. if (reg->state == DM_RH_CLEAN) {
  427. reg->state = DM_RH_DIRTY;
  428. list_del_init(&reg->list); /* take off the clean list */
  429. spin_unlock_irq(&rh->region_lock);
  430. rh->log->type->mark_region(rh->log, reg->key);
  431. } else
  432. spin_unlock_irq(&rh->region_lock);
  433. read_unlock(&rh->hash_lock);
  434. }
  435. void dm_rh_inc_pending(struct dm_region_hash *rh, struct bio_list *bios)
  436. {
  437. struct bio *bio;
  438. for (bio = bios->head; bio; bio = bio->bi_next) {
  439. if (bio->bi_opf & REQ_PREFLUSH || bio_op(bio) == REQ_OP_DISCARD)
  440. continue;
  441. rh_inc(rh, dm_rh_bio_to_region(rh, bio));
  442. }
  443. }
  444. EXPORT_SYMBOL_GPL(dm_rh_inc_pending);
  445. void dm_rh_dec(struct dm_region_hash *rh, region_t region)
  446. {
  447. unsigned long flags;
  448. struct dm_region *reg;
  449. int should_wake = 0;
  450. read_lock(&rh->hash_lock);
  451. reg = __rh_lookup(rh, region);
  452. read_unlock(&rh->hash_lock);
  453. spin_lock_irqsave(&rh->region_lock, flags);
  454. if (atomic_dec_and_test(&reg->pending)) {
  455. /*
  456. * There is no pending I/O for this region.
  457. * We can move the region to corresponding list for next action.
  458. * At this point, the region is not yet connected to any list.
  459. *
  460. * If the state is DM_RH_NOSYNC, the region should be kept off
  461. * from clean list.
  462. * The hash entry for DM_RH_NOSYNC will remain in memory
  463. * until the region is recovered or the map is reloaded.
  464. */
  465. /* do nothing for DM_RH_NOSYNC */
  466. if (unlikely(rh->flush_failure)) {
  467. /*
  468. * If a write flush failed some time ago, we
  469. * don't know whether or not this write made it
  470. * to the disk, so we must resync the device.
  471. */
  472. reg->state = DM_RH_NOSYNC;
  473. } else if (reg->state == DM_RH_RECOVERING) {
  474. list_add_tail(&reg->list, &rh->quiesced_regions);
  475. } else if (reg->state == DM_RH_DIRTY) {
  476. reg->state = DM_RH_CLEAN;
  477. list_add(&reg->list, &rh->clean_regions);
  478. }
  479. should_wake = 1;
  480. }
  481. spin_unlock_irqrestore(&rh->region_lock, flags);
  482. if (should_wake)
  483. rh->wakeup_workers(rh->context);
  484. }
  485. EXPORT_SYMBOL_GPL(dm_rh_dec);
  486. /*
  487. * Starts quiescing a region in preparation for recovery.
  488. */
  489. static int __rh_recovery_prepare(struct dm_region_hash *rh)
  490. {
  491. int r;
  492. region_t region;
  493. struct dm_region *reg;
  494. /*
  495. * Ask the dirty log what's next.
  496. */
  497. r = rh->log->type->get_resync_work(rh->log, &region);
  498. if (r <= 0)
  499. return r;
  500. /*
  501. * Get this region, and start it quiescing by setting the
  502. * recovering flag.
  503. */
  504. read_lock(&rh->hash_lock);
  505. reg = __rh_find(rh, region);
  506. read_unlock(&rh->hash_lock);
  507. spin_lock_irq(&rh->region_lock);
  508. reg->state = DM_RH_RECOVERING;
  509. /* Already quiesced ? */
  510. if (atomic_read(&reg->pending))
  511. list_del_init(&reg->list);
  512. else
  513. list_move(&reg->list, &rh->quiesced_regions);
  514. spin_unlock_irq(&rh->region_lock);
  515. return 1;
  516. }
  517. void dm_rh_recovery_prepare(struct dm_region_hash *rh)
  518. {
  519. /* Extra reference to avoid race with dm_rh_stop_recovery */
  520. atomic_inc(&rh->recovery_in_flight);
  521. while (!down_trylock(&rh->recovery_count)) {
  522. atomic_inc(&rh->recovery_in_flight);
  523. if (__rh_recovery_prepare(rh) <= 0) {
  524. atomic_dec(&rh->recovery_in_flight);
  525. up(&rh->recovery_count);
  526. break;
  527. }
  528. }
  529. /* Drop the extra reference */
  530. if (atomic_dec_and_test(&rh->recovery_in_flight))
  531. rh->wakeup_all_recovery_waiters(rh->context);
  532. }
  533. EXPORT_SYMBOL_GPL(dm_rh_recovery_prepare);
  534. /*
  535. * Returns any quiesced regions.
  536. */
  537. struct dm_region *dm_rh_recovery_start(struct dm_region_hash *rh)
  538. {
  539. struct dm_region *reg = NULL;
  540. spin_lock_irq(&rh->region_lock);
  541. if (!list_empty(&rh->quiesced_regions)) {
  542. reg = list_entry(rh->quiesced_regions.next,
  543. struct dm_region, list);
  544. list_del_init(&reg->list); /* remove from the quiesced list */
  545. }
  546. spin_unlock_irq(&rh->region_lock);
  547. return reg;
  548. }
  549. EXPORT_SYMBOL_GPL(dm_rh_recovery_start);
  550. void dm_rh_recovery_end(struct dm_region *reg, int success)
  551. {
  552. struct dm_region_hash *rh = reg->rh;
  553. spin_lock_irq(&rh->region_lock);
  554. if (success)
  555. list_add(&reg->list, &reg->rh->recovered_regions);
  556. else
  557. list_add(&reg->list, &reg->rh->failed_recovered_regions);
  558. spin_unlock_irq(&rh->region_lock);
  559. rh->wakeup_workers(rh->context);
  560. }
  561. EXPORT_SYMBOL_GPL(dm_rh_recovery_end);
  562. /* Return recovery in flight count. */
  563. int dm_rh_recovery_in_flight(struct dm_region_hash *rh)
  564. {
  565. return atomic_read(&rh->recovery_in_flight);
  566. }
  567. EXPORT_SYMBOL_GPL(dm_rh_recovery_in_flight);
  568. int dm_rh_flush(struct dm_region_hash *rh)
  569. {
  570. return rh->log->type->flush(rh->log);
  571. }
  572. EXPORT_SYMBOL_GPL(dm_rh_flush);
  573. void dm_rh_delay(struct dm_region_hash *rh, struct bio *bio)
  574. {
  575. struct dm_region *reg;
  576. read_lock(&rh->hash_lock);
  577. reg = __rh_find(rh, dm_rh_bio_to_region(rh, bio));
  578. bio_list_add(&reg->delayed_bios, bio);
  579. read_unlock(&rh->hash_lock);
  580. }
  581. EXPORT_SYMBOL_GPL(dm_rh_delay);
  582. void dm_rh_stop_recovery(struct dm_region_hash *rh)
  583. {
  584. int i;
  585. /* wait for any recovering regions */
  586. for (i = 0; i < rh->max_recovery; i++)
  587. down(&rh->recovery_count);
  588. }
  589. EXPORT_SYMBOL_GPL(dm_rh_stop_recovery);
  590. void dm_rh_start_recovery(struct dm_region_hash *rh)
  591. {
  592. int i;
  593. for (i = 0; i < rh->max_recovery; i++)
  594. up(&rh->recovery_count);
  595. rh->wakeup_workers(rh->context);
  596. }
  597. EXPORT_SYMBOL_GPL(dm_rh_start_recovery);
  598. MODULE_DESCRIPTION(DM_NAME " region hash");
  599. MODULE_AUTHOR("Joe Thornber/Heinz Mauelshagen <dm-devel@redhat.com>");
  600. MODULE_LICENSE("GPL");