dm-snap-persistent.c 23 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971
  1. /*
  2. * Copyright (C) 2001-2002 Sistina Software (UK) Limited.
  3. * Copyright (C) 2006-2008 Red Hat GmbH
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include "dm-exception-store.h"
  8. #include <linux/ctype.h>
  9. #include <linux/mm.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/vmalloc.h>
  12. #include <linux/export.h>
  13. #include <linux/slab.h>
  14. #include <linux/dm-io.h>
  15. #include <linux/dm-bufio.h>
  16. #define DM_MSG_PREFIX "persistent snapshot"
  17. #define DM_CHUNK_SIZE_DEFAULT_SECTORS 32U /* 16KB */
  18. #define DM_PREFETCH_CHUNKS 12
  19. /*-----------------------------------------------------------------
  20. * Persistent snapshots, by persistent we mean that the snapshot
  21. * will survive a reboot.
  22. *---------------------------------------------------------------*/
  23. /*
  24. * We need to store a record of which parts of the origin have
  25. * been copied to the snapshot device. The snapshot code
  26. * requires that we copy exception chunks to chunk aligned areas
  27. * of the COW store. It makes sense therefore, to store the
  28. * metadata in chunk size blocks.
  29. *
  30. * There is no backward or forward compatibility implemented,
  31. * snapshots with different disk versions than the kernel will
  32. * not be usable. It is expected that "lvcreate" will blank out
  33. * the start of a fresh COW device before calling the snapshot
  34. * constructor.
  35. *
  36. * The first chunk of the COW device just contains the header.
  37. * After this there is a chunk filled with exception metadata,
  38. * followed by as many exception chunks as can fit in the
  39. * metadata areas.
  40. *
  41. * All on disk structures are in little-endian format. The end
  42. * of the exceptions info is indicated by an exception with a
  43. * new_chunk of 0, which is invalid since it would point to the
  44. * header chunk.
  45. */
  46. /*
  47. * Magic for persistent snapshots: "SnAp" - Feeble isn't it.
  48. */
  49. #define SNAP_MAGIC 0x70416e53
  50. /*
  51. * The on-disk version of the metadata.
  52. */
  53. #define SNAPSHOT_DISK_VERSION 1
  54. #define NUM_SNAPSHOT_HDR_CHUNKS 1
  55. struct disk_header {
  56. __le32 magic;
  57. /*
  58. * Is this snapshot valid. There is no way of recovering
  59. * an invalid snapshot.
  60. */
  61. __le32 valid;
  62. /*
  63. * Simple, incrementing version. no backward
  64. * compatibility.
  65. */
  66. __le32 version;
  67. /* In sectors */
  68. __le32 chunk_size;
  69. } __packed;
  70. struct disk_exception {
  71. __le64 old_chunk;
  72. __le64 new_chunk;
  73. } __packed;
  74. struct core_exception {
  75. uint64_t old_chunk;
  76. uint64_t new_chunk;
  77. };
  78. struct commit_callback {
  79. void (*callback)(void *, int success);
  80. void *context;
  81. };
  82. /*
  83. * The top level structure for a persistent exception store.
  84. */
  85. struct pstore {
  86. struct dm_exception_store *store;
  87. int version;
  88. int valid;
  89. uint32_t exceptions_per_area;
  90. /*
  91. * Now that we have an asynchronous kcopyd there is no
  92. * need for large chunk sizes, so it wont hurt to have a
  93. * whole chunks worth of metadata in memory at once.
  94. */
  95. void *area;
  96. /*
  97. * An area of zeros used to clear the next area.
  98. */
  99. void *zero_area;
  100. /*
  101. * An area used for header. The header can be written
  102. * concurrently with metadata (when invalidating the snapshot),
  103. * so it needs a separate buffer.
  104. */
  105. void *header_area;
  106. /*
  107. * Used to keep track of which metadata area the data in
  108. * 'chunk' refers to.
  109. */
  110. chunk_t current_area;
  111. /*
  112. * The next free chunk for an exception.
  113. *
  114. * When creating exceptions, all the chunks here and above are
  115. * free. It holds the next chunk to be allocated. On rare
  116. * occasions (e.g. after a system crash) holes can be left in
  117. * the exception store because chunks can be committed out of
  118. * order.
  119. *
  120. * When merging exceptions, it does not necessarily mean all the
  121. * chunks here and above are free. It holds the value it would
  122. * have held if all chunks had been committed in order of
  123. * allocation. Consequently the value may occasionally be
  124. * slightly too low, but since it's only used for 'status' and
  125. * it can never reach its minimum value too early this doesn't
  126. * matter.
  127. */
  128. chunk_t next_free;
  129. /*
  130. * The index of next free exception in the current
  131. * metadata area.
  132. */
  133. uint32_t current_committed;
  134. atomic_t pending_count;
  135. uint32_t callback_count;
  136. struct commit_callback *callbacks;
  137. struct dm_io_client *io_client;
  138. struct workqueue_struct *metadata_wq;
  139. };
  140. static int alloc_area(struct pstore *ps)
  141. {
  142. int r = -ENOMEM;
  143. size_t len;
  144. len = ps->store->chunk_size << SECTOR_SHIFT;
  145. /*
  146. * Allocate the chunk_size block of memory that will hold
  147. * a single metadata area.
  148. */
  149. ps->area = vmalloc(len);
  150. if (!ps->area)
  151. goto err_area;
  152. ps->zero_area = vzalloc(len);
  153. if (!ps->zero_area)
  154. goto err_zero_area;
  155. ps->header_area = vmalloc(len);
  156. if (!ps->header_area)
  157. goto err_header_area;
  158. return 0;
  159. err_header_area:
  160. vfree(ps->zero_area);
  161. err_zero_area:
  162. vfree(ps->area);
  163. err_area:
  164. return r;
  165. }
  166. static void free_area(struct pstore *ps)
  167. {
  168. vfree(ps->area);
  169. ps->area = NULL;
  170. vfree(ps->zero_area);
  171. ps->zero_area = NULL;
  172. vfree(ps->header_area);
  173. ps->header_area = NULL;
  174. }
  175. struct mdata_req {
  176. struct dm_io_region *where;
  177. struct dm_io_request *io_req;
  178. struct work_struct work;
  179. int result;
  180. };
  181. static void do_metadata(struct work_struct *work)
  182. {
  183. struct mdata_req *req = container_of(work, struct mdata_req, work);
  184. req->result = dm_io(req->io_req, 1, req->where, NULL);
  185. }
  186. /*
  187. * Read or write a chunk aligned and sized block of data from a device.
  188. */
  189. static int chunk_io(struct pstore *ps, void *area, chunk_t chunk, int op,
  190. int op_flags, int metadata)
  191. {
  192. struct dm_io_region where = {
  193. .bdev = dm_snap_cow(ps->store->snap)->bdev,
  194. .sector = ps->store->chunk_size * chunk,
  195. .count = ps->store->chunk_size,
  196. };
  197. struct dm_io_request io_req = {
  198. .bi_op = op,
  199. .bi_op_flags = op_flags,
  200. .mem.type = DM_IO_VMA,
  201. .mem.ptr.vma = area,
  202. .client = ps->io_client,
  203. .notify.fn = NULL,
  204. };
  205. struct mdata_req req;
  206. if (!metadata)
  207. return dm_io(&io_req, 1, &where, NULL);
  208. req.where = &where;
  209. req.io_req = &io_req;
  210. /*
  211. * Issue the synchronous I/O from a different thread
  212. * to avoid submit_bio_noacct recursion.
  213. */
  214. INIT_WORK_ONSTACK(&req.work, do_metadata);
  215. queue_work(ps->metadata_wq, &req.work);
  216. flush_workqueue(ps->metadata_wq);
  217. destroy_work_on_stack(&req.work);
  218. return req.result;
  219. }
  220. /*
  221. * Convert a metadata area index to a chunk index.
  222. */
  223. static chunk_t area_location(struct pstore *ps, chunk_t area)
  224. {
  225. return NUM_SNAPSHOT_HDR_CHUNKS + ((ps->exceptions_per_area + 1) * area);
  226. }
  227. static void skip_metadata(struct pstore *ps)
  228. {
  229. uint32_t stride = ps->exceptions_per_area + 1;
  230. chunk_t next_free = ps->next_free;
  231. if (sector_div(next_free, stride) == NUM_SNAPSHOT_HDR_CHUNKS)
  232. ps->next_free++;
  233. }
  234. /*
  235. * Read or write a metadata area. Remembering to skip the first
  236. * chunk which holds the header.
  237. */
  238. static int area_io(struct pstore *ps, int op, int op_flags)
  239. {
  240. chunk_t chunk = area_location(ps, ps->current_area);
  241. return chunk_io(ps, ps->area, chunk, op, op_flags, 0);
  242. }
  243. static void zero_memory_area(struct pstore *ps)
  244. {
  245. memset(ps->area, 0, ps->store->chunk_size << SECTOR_SHIFT);
  246. }
  247. static int zero_disk_area(struct pstore *ps, chunk_t area)
  248. {
  249. return chunk_io(ps, ps->zero_area, area_location(ps, area),
  250. REQ_OP_WRITE, 0, 0);
  251. }
  252. static int read_header(struct pstore *ps, int *new_snapshot)
  253. {
  254. int r;
  255. struct disk_header *dh;
  256. unsigned chunk_size;
  257. int chunk_size_supplied = 1;
  258. char *chunk_err;
  259. /*
  260. * Use default chunk size (or logical_block_size, if larger)
  261. * if none supplied
  262. */
  263. if (!ps->store->chunk_size) {
  264. ps->store->chunk_size = max(DM_CHUNK_SIZE_DEFAULT_SECTORS,
  265. bdev_logical_block_size(dm_snap_cow(ps->store->snap)->
  266. bdev) >> 9);
  267. ps->store->chunk_mask = ps->store->chunk_size - 1;
  268. ps->store->chunk_shift = __ffs(ps->store->chunk_size);
  269. chunk_size_supplied = 0;
  270. }
  271. ps->io_client = dm_io_client_create();
  272. if (IS_ERR(ps->io_client))
  273. return PTR_ERR(ps->io_client);
  274. r = alloc_area(ps);
  275. if (r)
  276. return r;
  277. r = chunk_io(ps, ps->header_area, 0, REQ_OP_READ, 0, 1);
  278. if (r)
  279. goto bad;
  280. dh = ps->header_area;
  281. if (le32_to_cpu(dh->magic) == 0) {
  282. *new_snapshot = 1;
  283. return 0;
  284. }
  285. if (le32_to_cpu(dh->magic) != SNAP_MAGIC) {
  286. DMWARN("Invalid or corrupt snapshot");
  287. r = -ENXIO;
  288. goto bad;
  289. }
  290. *new_snapshot = 0;
  291. ps->valid = le32_to_cpu(dh->valid);
  292. ps->version = le32_to_cpu(dh->version);
  293. chunk_size = le32_to_cpu(dh->chunk_size);
  294. if (ps->store->chunk_size == chunk_size)
  295. return 0;
  296. if (chunk_size_supplied)
  297. DMWARN("chunk size %u in device metadata overrides "
  298. "table chunk size of %u.",
  299. chunk_size, ps->store->chunk_size);
  300. /* We had a bogus chunk_size. Fix stuff up. */
  301. free_area(ps);
  302. r = dm_exception_store_set_chunk_size(ps->store, chunk_size,
  303. &chunk_err);
  304. if (r) {
  305. DMERR("invalid on-disk chunk size %u: %s.",
  306. chunk_size, chunk_err);
  307. return r;
  308. }
  309. r = alloc_area(ps);
  310. return r;
  311. bad:
  312. free_area(ps);
  313. return r;
  314. }
  315. static int write_header(struct pstore *ps)
  316. {
  317. struct disk_header *dh;
  318. memset(ps->header_area, 0, ps->store->chunk_size << SECTOR_SHIFT);
  319. dh = ps->header_area;
  320. dh->magic = cpu_to_le32(SNAP_MAGIC);
  321. dh->valid = cpu_to_le32(ps->valid);
  322. dh->version = cpu_to_le32(ps->version);
  323. dh->chunk_size = cpu_to_le32(ps->store->chunk_size);
  324. return chunk_io(ps, ps->header_area, 0, REQ_OP_WRITE, 0, 1);
  325. }
  326. /*
  327. * Access functions for the disk exceptions, these do the endian conversions.
  328. */
  329. static struct disk_exception *get_exception(struct pstore *ps, void *ps_area,
  330. uint32_t index)
  331. {
  332. BUG_ON(index >= ps->exceptions_per_area);
  333. return ((struct disk_exception *) ps_area) + index;
  334. }
  335. static void read_exception(struct pstore *ps, void *ps_area,
  336. uint32_t index, struct core_exception *result)
  337. {
  338. struct disk_exception *de = get_exception(ps, ps_area, index);
  339. /* copy it */
  340. result->old_chunk = le64_to_cpu(de->old_chunk);
  341. result->new_chunk = le64_to_cpu(de->new_chunk);
  342. }
  343. static void write_exception(struct pstore *ps,
  344. uint32_t index, struct core_exception *e)
  345. {
  346. struct disk_exception *de = get_exception(ps, ps->area, index);
  347. /* copy it */
  348. de->old_chunk = cpu_to_le64(e->old_chunk);
  349. de->new_chunk = cpu_to_le64(e->new_chunk);
  350. }
  351. static void clear_exception(struct pstore *ps, uint32_t index)
  352. {
  353. struct disk_exception *de = get_exception(ps, ps->area, index);
  354. /* clear it */
  355. de->old_chunk = 0;
  356. de->new_chunk = 0;
  357. }
  358. /*
  359. * Registers the exceptions that are present in the current area.
  360. * 'full' is filled in to indicate if the area has been
  361. * filled.
  362. */
  363. static int insert_exceptions(struct pstore *ps, void *ps_area,
  364. int (*callback)(void *callback_context,
  365. chunk_t old, chunk_t new),
  366. void *callback_context,
  367. int *full)
  368. {
  369. int r;
  370. unsigned int i;
  371. struct core_exception e;
  372. /* presume the area is full */
  373. *full = 1;
  374. for (i = 0; i < ps->exceptions_per_area; i++) {
  375. read_exception(ps, ps_area, i, &e);
  376. /*
  377. * If the new_chunk is pointing at the start of
  378. * the COW device, where the first metadata area
  379. * is we know that we've hit the end of the
  380. * exceptions. Therefore the area is not full.
  381. */
  382. if (e.new_chunk == 0LL) {
  383. ps->current_committed = i;
  384. *full = 0;
  385. break;
  386. }
  387. /*
  388. * Keep track of the start of the free chunks.
  389. */
  390. if (ps->next_free <= e.new_chunk)
  391. ps->next_free = e.new_chunk + 1;
  392. /*
  393. * Otherwise we add the exception to the snapshot.
  394. */
  395. r = callback(callback_context, e.old_chunk, e.new_chunk);
  396. if (r)
  397. return r;
  398. }
  399. return 0;
  400. }
  401. static int read_exceptions(struct pstore *ps,
  402. int (*callback)(void *callback_context, chunk_t old,
  403. chunk_t new),
  404. void *callback_context)
  405. {
  406. int r, full = 1;
  407. struct dm_bufio_client *client;
  408. chunk_t prefetch_area = 0;
  409. client = dm_bufio_client_create(dm_snap_cow(ps->store->snap)->bdev,
  410. ps->store->chunk_size << SECTOR_SHIFT,
  411. 1, 0, NULL, NULL);
  412. if (IS_ERR(client))
  413. return PTR_ERR(client);
  414. /*
  415. * Setup for one current buffer + desired readahead buffers.
  416. */
  417. dm_bufio_set_minimum_buffers(client, 1 + DM_PREFETCH_CHUNKS);
  418. /*
  419. * Keeping reading chunks and inserting exceptions until
  420. * we find a partially full area.
  421. */
  422. for (ps->current_area = 0; full; ps->current_area++) {
  423. struct dm_buffer *bp;
  424. void *area;
  425. chunk_t chunk;
  426. if (unlikely(prefetch_area < ps->current_area))
  427. prefetch_area = ps->current_area;
  428. if (DM_PREFETCH_CHUNKS) do {
  429. chunk_t pf_chunk = area_location(ps, prefetch_area);
  430. if (unlikely(pf_chunk >= dm_bufio_get_device_size(client)))
  431. break;
  432. dm_bufio_prefetch(client, pf_chunk, 1);
  433. prefetch_area++;
  434. if (unlikely(!prefetch_area))
  435. break;
  436. } while (prefetch_area <= ps->current_area + DM_PREFETCH_CHUNKS);
  437. chunk = area_location(ps, ps->current_area);
  438. area = dm_bufio_read(client, chunk, &bp);
  439. if (IS_ERR(area)) {
  440. r = PTR_ERR(area);
  441. goto ret_destroy_bufio;
  442. }
  443. r = insert_exceptions(ps, area, callback, callback_context,
  444. &full);
  445. if (!full)
  446. memcpy(ps->area, area, ps->store->chunk_size << SECTOR_SHIFT);
  447. dm_bufio_release(bp);
  448. dm_bufio_forget(client, chunk);
  449. if (unlikely(r))
  450. goto ret_destroy_bufio;
  451. }
  452. ps->current_area--;
  453. skip_metadata(ps);
  454. r = 0;
  455. ret_destroy_bufio:
  456. dm_bufio_client_destroy(client);
  457. return r;
  458. }
  459. static struct pstore *get_info(struct dm_exception_store *store)
  460. {
  461. return (struct pstore *) store->context;
  462. }
  463. static void persistent_usage(struct dm_exception_store *store,
  464. sector_t *total_sectors,
  465. sector_t *sectors_allocated,
  466. sector_t *metadata_sectors)
  467. {
  468. struct pstore *ps = get_info(store);
  469. *sectors_allocated = ps->next_free * store->chunk_size;
  470. *total_sectors = get_dev_size(dm_snap_cow(store->snap)->bdev);
  471. /*
  472. * First chunk is the fixed header.
  473. * Then there are (ps->current_area + 1) metadata chunks, each one
  474. * separated from the next by ps->exceptions_per_area data chunks.
  475. */
  476. *metadata_sectors = (ps->current_area + 1 + NUM_SNAPSHOT_HDR_CHUNKS) *
  477. store->chunk_size;
  478. }
  479. static void persistent_dtr(struct dm_exception_store *store)
  480. {
  481. struct pstore *ps = get_info(store);
  482. destroy_workqueue(ps->metadata_wq);
  483. /* Created in read_header */
  484. if (ps->io_client)
  485. dm_io_client_destroy(ps->io_client);
  486. free_area(ps);
  487. /* Allocated in persistent_read_metadata */
  488. vfree(ps->callbacks);
  489. kfree(ps);
  490. }
  491. static int persistent_read_metadata(struct dm_exception_store *store,
  492. int (*callback)(void *callback_context,
  493. chunk_t old, chunk_t new),
  494. void *callback_context)
  495. {
  496. int r, new_snapshot;
  497. struct pstore *ps = get_info(store);
  498. /*
  499. * Read the snapshot header.
  500. */
  501. r = read_header(ps, &new_snapshot);
  502. if (r)
  503. return r;
  504. /*
  505. * Now we know correct chunk_size, complete the initialisation.
  506. */
  507. ps->exceptions_per_area = (ps->store->chunk_size << SECTOR_SHIFT) /
  508. sizeof(struct disk_exception);
  509. ps->callbacks = dm_vcalloc(ps->exceptions_per_area,
  510. sizeof(*ps->callbacks));
  511. if (!ps->callbacks)
  512. return -ENOMEM;
  513. /*
  514. * Do we need to setup a new snapshot ?
  515. */
  516. if (new_snapshot) {
  517. r = write_header(ps);
  518. if (r) {
  519. DMWARN("write_header failed");
  520. return r;
  521. }
  522. ps->current_area = 0;
  523. zero_memory_area(ps);
  524. r = zero_disk_area(ps, 0);
  525. if (r)
  526. DMWARN("zero_disk_area(0) failed");
  527. return r;
  528. }
  529. /*
  530. * Sanity checks.
  531. */
  532. if (ps->version != SNAPSHOT_DISK_VERSION) {
  533. DMWARN("unable to handle snapshot disk version %d",
  534. ps->version);
  535. return -EINVAL;
  536. }
  537. /*
  538. * Metadata are valid, but snapshot is invalidated
  539. */
  540. if (!ps->valid)
  541. return 1;
  542. /*
  543. * Read the metadata.
  544. */
  545. r = read_exceptions(ps, callback, callback_context);
  546. return r;
  547. }
  548. static int persistent_prepare_exception(struct dm_exception_store *store,
  549. struct dm_exception *e)
  550. {
  551. struct pstore *ps = get_info(store);
  552. sector_t size = get_dev_size(dm_snap_cow(store->snap)->bdev);
  553. /* Is there enough room ? */
  554. if (size < ((ps->next_free + 1) * store->chunk_size))
  555. return -ENOSPC;
  556. e->new_chunk = ps->next_free;
  557. /*
  558. * Move onto the next free pending, making sure to take
  559. * into account the location of the metadata chunks.
  560. */
  561. ps->next_free++;
  562. skip_metadata(ps);
  563. atomic_inc(&ps->pending_count);
  564. return 0;
  565. }
  566. static void persistent_commit_exception(struct dm_exception_store *store,
  567. struct dm_exception *e, int valid,
  568. void (*callback) (void *, int success),
  569. void *callback_context)
  570. {
  571. unsigned int i;
  572. struct pstore *ps = get_info(store);
  573. struct core_exception ce;
  574. struct commit_callback *cb;
  575. if (!valid)
  576. ps->valid = 0;
  577. ce.old_chunk = e->old_chunk;
  578. ce.new_chunk = e->new_chunk;
  579. write_exception(ps, ps->current_committed++, &ce);
  580. /*
  581. * Add the callback to the back of the array. This code
  582. * is the only place where the callback array is
  583. * manipulated, and we know that it will never be called
  584. * multiple times concurrently.
  585. */
  586. cb = ps->callbacks + ps->callback_count++;
  587. cb->callback = callback;
  588. cb->context = callback_context;
  589. /*
  590. * If there are exceptions in flight and we have not yet
  591. * filled this metadata area there's nothing more to do.
  592. */
  593. if (!atomic_dec_and_test(&ps->pending_count) &&
  594. (ps->current_committed != ps->exceptions_per_area))
  595. return;
  596. /*
  597. * If we completely filled the current area, then wipe the next one.
  598. */
  599. if ((ps->current_committed == ps->exceptions_per_area) &&
  600. zero_disk_area(ps, ps->current_area + 1))
  601. ps->valid = 0;
  602. /*
  603. * Commit exceptions to disk.
  604. */
  605. if (ps->valid && area_io(ps, REQ_OP_WRITE,
  606. REQ_PREFLUSH | REQ_FUA | REQ_SYNC))
  607. ps->valid = 0;
  608. /*
  609. * Advance to the next area if this one is full.
  610. */
  611. if (ps->current_committed == ps->exceptions_per_area) {
  612. ps->current_committed = 0;
  613. ps->current_area++;
  614. zero_memory_area(ps);
  615. }
  616. for (i = 0; i < ps->callback_count; i++) {
  617. cb = ps->callbacks + i;
  618. cb->callback(cb->context, ps->valid);
  619. }
  620. ps->callback_count = 0;
  621. }
  622. static int persistent_prepare_merge(struct dm_exception_store *store,
  623. chunk_t *last_old_chunk,
  624. chunk_t *last_new_chunk)
  625. {
  626. struct pstore *ps = get_info(store);
  627. struct core_exception ce;
  628. int nr_consecutive;
  629. int r;
  630. /*
  631. * When current area is empty, move back to preceding area.
  632. */
  633. if (!ps->current_committed) {
  634. /*
  635. * Have we finished?
  636. */
  637. if (!ps->current_area)
  638. return 0;
  639. ps->current_area--;
  640. r = area_io(ps, REQ_OP_READ, 0);
  641. if (r < 0)
  642. return r;
  643. ps->current_committed = ps->exceptions_per_area;
  644. }
  645. read_exception(ps, ps->area, ps->current_committed - 1, &ce);
  646. *last_old_chunk = ce.old_chunk;
  647. *last_new_chunk = ce.new_chunk;
  648. /*
  649. * Find number of consecutive chunks within the current area,
  650. * working backwards.
  651. */
  652. for (nr_consecutive = 1; nr_consecutive < ps->current_committed;
  653. nr_consecutive++) {
  654. read_exception(ps, ps->area,
  655. ps->current_committed - 1 - nr_consecutive, &ce);
  656. if (ce.old_chunk != *last_old_chunk - nr_consecutive ||
  657. ce.new_chunk != *last_new_chunk - nr_consecutive)
  658. break;
  659. }
  660. return nr_consecutive;
  661. }
  662. static int persistent_commit_merge(struct dm_exception_store *store,
  663. int nr_merged)
  664. {
  665. int r, i;
  666. struct pstore *ps = get_info(store);
  667. BUG_ON(nr_merged > ps->current_committed);
  668. for (i = 0; i < nr_merged; i++)
  669. clear_exception(ps, ps->current_committed - 1 - i);
  670. r = area_io(ps, REQ_OP_WRITE, REQ_PREFLUSH | REQ_FUA);
  671. if (r < 0)
  672. return r;
  673. ps->current_committed -= nr_merged;
  674. /*
  675. * At this stage, only persistent_usage() uses ps->next_free, so
  676. * we make no attempt to keep ps->next_free strictly accurate
  677. * as exceptions may have been committed out-of-order originally.
  678. * Once a snapshot has become merging, we set it to the value it
  679. * would have held had all the exceptions been committed in order.
  680. *
  681. * ps->current_area does not get reduced by prepare_merge() until
  682. * after commit_merge() has removed the nr_merged previous exceptions.
  683. */
  684. ps->next_free = area_location(ps, ps->current_area) +
  685. ps->current_committed + 1;
  686. return 0;
  687. }
  688. static void persistent_drop_snapshot(struct dm_exception_store *store)
  689. {
  690. struct pstore *ps = get_info(store);
  691. ps->valid = 0;
  692. if (write_header(ps))
  693. DMWARN("write header failed");
  694. }
  695. static int persistent_ctr(struct dm_exception_store *store, char *options)
  696. {
  697. struct pstore *ps;
  698. int r;
  699. /* allocate the pstore */
  700. ps = kzalloc(sizeof(*ps), GFP_KERNEL);
  701. if (!ps)
  702. return -ENOMEM;
  703. ps->store = store;
  704. ps->valid = 1;
  705. ps->version = SNAPSHOT_DISK_VERSION;
  706. ps->area = NULL;
  707. ps->zero_area = NULL;
  708. ps->header_area = NULL;
  709. ps->next_free = NUM_SNAPSHOT_HDR_CHUNKS + 1; /* header and 1st area */
  710. ps->current_committed = 0;
  711. ps->callback_count = 0;
  712. atomic_set(&ps->pending_count, 0);
  713. ps->callbacks = NULL;
  714. ps->metadata_wq = alloc_workqueue("ksnaphd", WQ_MEM_RECLAIM, 0);
  715. if (!ps->metadata_wq) {
  716. DMERR("couldn't start header metadata update thread");
  717. r = -ENOMEM;
  718. goto err_workqueue;
  719. }
  720. if (options) {
  721. char overflow = toupper(options[0]);
  722. if (overflow == 'O')
  723. store->userspace_supports_overflow = true;
  724. else {
  725. DMERR("Unsupported persistent store option: %s", options);
  726. r = -EINVAL;
  727. goto err_options;
  728. }
  729. }
  730. store->context = ps;
  731. return 0;
  732. err_options:
  733. destroy_workqueue(ps->metadata_wq);
  734. err_workqueue:
  735. kfree(ps);
  736. return r;
  737. }
  738. static unsigned persistent_status(struct dm_exception_store *store,
  739. status_type_t status, char *result,
  740. unsigned maxlen)
  741. {
  742. unsigned sz = 0;
  743. switch (status) {
  744. case STATUSTYPE_INFO:
  745. break;
  746. case STATUSTYPE_TABLE:
  747. DMEMIT(" %s %llu", store->userspace_supports_overflow ? "PO" : "P",
  748. (unsigned long long)store->chunk_size);
  749. }
  750. return sz;
  751. }
  752. static struct dm_exception_store_type _persistent_type = {
  753. .name = "persistent",
  754. .module = THIS_MODULE,
  755. .ctr = persistent_ctr,
  756. .dtr = persistent_dtr,
  757. .read_metadata = persistent_read_metadata,
  758. .prepare_exception = persistent_prepare_exception,
  759. .commit_exception = persistent_commit_exception,
  760. .prepare_merge = persistent_prepare_merge,
  761. .commit_merge = persistent_commit_merge,
  762. .drop_snapshot = persistent_drop_snapshot,
  763. .usage = persistent_usage,
  764. .status = persistent_status,
  765. };
  766. static struct dm_exception_store_type _persistent_compat_type = {
  767. .name = "P",
  768. .module = THIS_MODULE,
  769. .ctr = persistent_ctr,
  770. .dtr = persistent_dtr,
  771. .read_metadata = persistent_read_metadata,
  772. .prepare_exception = persistent_prepare_exception,
  773. .commit_exception = persistent_commit_exception,
  774. .prepare_merge = persistent_prepare_merge,
  775. .commit_merge = persistent_commit_merge,
  776. .drop_snapshot = persistent_drop_snapshot,
  777. .usage = persistent_usage,
  778. .status = persistent_status,
  779. };
  780. int dm_persistent_snapshot_init(void)
  781. {
  782. int r;
  783. r = dm_exception_store_type_register(&_persistent_type);
  784. if (r) {
  785. DMERR("Unable to register persistent exception store type");
  786. return r;
  787. }
  788. r = dm_exception_store_type_register(&_persistent_compat_type);
  789. if (r) {
  790. DMERR("Unable to register old-style persistent exception "
  791. "store type");
  792. dm_exception_store_type_unregister(&_persistent_type);
  793. return r;
  794. }
  795. return r;
  796. }
  797. void dm_persistent_snapshot_exit(void)
  798. {
  799. dm_exception_store_type_unregister(&_persistent_type);
  800. dm_exception_store_type_unregister(&_persistent_compat_type);
  801. }