dm-era-target.c 38 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include "dm.h"
  3. #include "persistent-data/dm-transaction-manager.h"
  4. #include "persistent-data/dm-bitset.h"
  5. #include "persistent-data/dm-space-map.h"
  6. #include <linux/dm-io.h>
  7. #include <linux/dm-kcopyd.h>
  8. #include <linux/init.h>
  9. #include <linux/mempool.h>
  10. #include <linux/module.h>
  11. #include <linux/slab.h>
  12. #include <linux/vmalloc.h>
  13. #define DM_MSG_PREFIX "era"
  14. #define SUPERBLOCK_LOCATION 0
  15. #define SUPERBLOCK_MAGIC 2126579579
  16. #define SUPERBLOCK_CSUM_XOR 146538381
  17. #define MIN_ERA_VERSION 1
  18. #define MAX_ERA_VERSION 1
  19. #define INVALID_WRITESET_ROOT SUPERBLOCK_LOCATION
  20. #define MIN_BLOCK_SIZE 8
  21. /*----------------------------------------------------------------
  22. * Writeset
  23. *--------------------------------------------------------------*/
  24. struct writeset_metadata {
  25. uint32_t nr_bits;
  26. dm_block_t root;
  27. };
  28. struct writeset {
  29. struct writeset_metadata md;
  30. /*
  31. * An in core copy of the bits to save constantly doing look ups on
  32. * disk.
  33. */
  34. unsigned long *bits;
  35. };
  36. /*
  37. * This does not free off the on disk bitset as this will normally be done
  38. * after digesting into the era array.
  39. */
  40. static void writeset_free(struct writeset *ws)
  41. {
  42. vfree(ws->bits);
  43. ws->bits = NULL;
  44. }
  45. static int setup_on_disk_bitset(struct dm_disk_bitset *info,
  46. unsigned nr_bits, dm_block_t *root)
  47. {
  48. int r;
  49. r = dm_bitset_empty(info, root);
  50. if (r)
  51. return r;
  52. return dm_bitset_resize(info, *root, 0, nr_bits, false, root);
  53. }
  54. static size_t bitset_size(unsigned nr_bits)
  55. {
  56. return sizeof(unsigned long) * dm_div_up(nr_bits, BITS_PER_LONG);
  57. }
  58. /*
  59. * Allocates memory for the in core bitset.
  60. */
  61. static int writeset_alloc(struct writeset *ws, dm_block_t nr_blocks)
  62. {
  63. ws->bits = vzalloc(bitset_size(nr_blocks));
  64. if (!ws->bits) {
  65. DMERR("%s: couldn't allocate in memory bitset", __func__);
  66. return -ENOMEM;
  67. }
  68. return 0;
  69. }
  70. /*
  71. * Wipes the in-core bitset, and creates a new on disk bitset.
  72. */
  73. static int writeset_init(struct dm_disk_bitset *info, struct writeset *ws,
  74. dm_block_t nr_blocks)
  75. {
  76. int r;
  77. memset(ws->bits, 0, bitset_size(nr_blocks));
  78. ws->md.nr_bits = nr_blocks;
  79. r = setup_on_disk_bitset(info, ws->md.nr_bits, &ws->md.root);
  80. if (r) {
  81. DMERR("%s: setup_on_disk_bitset failed", __func__);
  82. return r;
  83. }
  84. return 0;
  85. }
  86. static bool writeset_marked(struct writeset *ws, dm_block_t block)
  87. {
  88. return test_bit(block, ws->bits);
  89. }
  90. static int writeset_marked_on_disk(struct dm_disk_bitset *info,
  91. struct writeset_metadata *m, dm_block_t block,
  92. bool *result)
  93. {
  94. dm_block_t old = m->root;
  95. /*
  96. * The bitset was flushed when it was archived, so we know there'll
  97. * be no change to the root.
  98. */
  99. int r = dm_bitset_test_bit(info, m->root, block, &m->root, result);
  100. if (r) {
  101. DMERR("%s: dm_bitset_test_bit failed", __func__);
  102. return r;
  103. }
  104. BUG_ON(m->root != old);
  105. return r;
  106. }
  107. /*
  108. * Returns < 0 on error, 0 if the bit wasn't previously set, 1 if it was.
  109. */
  110. static int writeset_test_and_set(struct dm_disk_bitset *info,
  111. struct writeset *ws, uint32_t block)
  112. {
  113. int r;
  114. if (!test_bit(block, ws->bits)) {
  115. r = dm_bitset_set_bit(info, ws->md.root, block, &ws->md.root);
  116. if (r) {
  117. /* FIXME: fail mode */
  118. return r;
  119. }
  120. return 0;
  121. }
  122. return 1;
  123. }
  124. /*----------------------------------------------------------------
  125. * On disk metadata layout
  126. *--------------------------------------------------------------*/
  127. #define SPACE_MAP_ROOT_SIZE 128
  128. #define UUID_LEN 16
  129. struct writeset_disk {
  130. __le32 nr_bits;
  131. __le64 root;
  132. } __packed;
  133. struct superblock_disk {
  134. __le32 csum;
  135. __le32 flags;
  136. __le64 blocknr;
  137. __u8 uuid[UUID_LEN];
  138. __le64 magic;
  139. __le32 version;
  140. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  141. __le32 data_block_size;
  142. __le32 metadata_block_size;
  143. __le32 nr_blocks;
  144. __le32 current_era;
  145. struct writeset_disk current_writeset;
  146. /*
  147. * Only these two fields are valid within the metadata snapshot.
  148. */
  149. __le64 writeset_tree_root;
  150. __le64 era_array_root;
  151. __le64 metadata_snap;
  152. } __packed;
  153. /*----------------------------------------------------------------
  154. * Superblock validation
  155. *--------------------------------------------------------------*/
  156. static void sb_prepare_for_write(struct dm_block_validator *v,
  157. struct dm_block *b,
  158. size_t sb_block_size)
  159. {
  160. struct superblock_disk *disk = dm_block_data(b);
  161. disk->blocknr = cpu_to_le64(dm_block_location(b));
  162. disk->csum = cpu_to_le32(dm_bm_checksum(&disk->flags,
  163. sb_block_size - sizeof(__le32),
  164. SUPERBLOCK_CSUM_XOR));
  165. }
  166. static int check_metadata_version(struct superblock_disk *disk)
  167. {
  168. uint32_t metadata_version = le32_to_cpu(disk->version);
  169. if (metadata_version < MIN_ERA_VERSION || metadata_version > MAX_ERA_VERSION) {
  170. DMERR("Era metadata version %u found, but only versions between %u and %u supported.",
  171. metadata_version, MIN_ERA_VERSION, MAX_ERA_VERSION);
  172. return -EINVAL;
  173. }
  174. return 0;
  175. }
  176. static int sb_check(struct dm_block_validator *v,
  177. struct dm_block *b,
  178. size_t sb_block_size)
  179. {
  180. struct superblock_disk *disk = dm_block_data(b);
  181. __le32 csum_le;
  182. if (dm_block_location(b) != le64_to_cpu(disk->blocknr)) {
  183. DMERR("sb_check failed: blocknr %llu: wanted %llu",
  184. le64_to_cpu(disk->blocknr),
  185. (unsigned long long)dm_block_location(b));
  186. return -ENOTBLK;
  187. }
  188. if (le64_to_cpu(disk->magic) != SUPERBLOCK_MAGIC) {
  189. DMERR("sb_check failed: magic %llu: wanted %llu",
  190. le64_to_cpu(disk->magic),
  191. (unsigned long long) SUPERBLOCK_MAGIC);
  192. return -EILSEQ;
  193. }
  194. csum_le = cpu_to_le32(dm_bm_checksum(&disk->flags,
  195. sb_block_size - sizeof(__le32),
  196. SUPERBLOCK_CSUM_XOR));
  197. if (csum_le != disk->csum) {
  198. DMERR("sb_check failed: csum %u: wanted %u",
  199. le32_to_cpu(csum_le), le32_to_cpu(disk->csum));
  200. return -EILSEQ;
  201. }
  202. return check_metadata_version(disk);
  203. }
  204. static struct dm_block_validator sb_validator = {
  205. .name = "superblock",
  206. .prepare_for_write = sb_prepare_for_write,
  207. .check = sb_check
  208. };
  209. /*----------------------------------------------------------------
  210. * Low level metadata handling
  211. *--------------------------------------------------------------*/
  212. #define DM_ERA_METADATA_BLOCK_SIZE 4096
  213. #define ERA_MAX_CONCURRENT_LOCKS 5
  214. struct era_metadata {
  215. struct block_device *bdev;
  216. struct dm_block_manager *bm;
  217. struct dm_space_map *sm;
  218. struct dm_transaction_manager *tm;
  219. dm_block_t block_size;
  220. uint32_t nr_blocks;
  221. uint32_t current_era;
  222. /*
  223. * We preallocate 2 writesets. When an era rolls over we
  224. * switch between them. This means the allocation is done at
  225. * preresume time, rather than on the io path.
  226. */
  227. struct writeset writesets[2];
  228. struct writeset *current_writeset;
  229. dm_block_t writeset_tree_root;
  230. dm_block_t era_array_root;
  231. struct dm_disk_bitset bitset_info;
  232. struct dm_btree_info writeset_tree_info;
  233. struct dm_array_info era_array_info;
  234. dm_block_t metadata_snap;
  235. /*
  236. * A flag that is set whenever a writeset has been archived.
  237. */
  238. bool archived_writesets;
  239. /*
  240. * Reading the space map root can fail, so we read it into this
  241. * buffer before the superblock is locked and updated.
  242. */
  243. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  244. };
  245. static int superblock_read_lock(struct era_metadata *md,
  246. struct dm_block **sblock)
  247. {
  248. return dm_bm_read_lock(md->bm, SUPERBLOCK_LOCATION,
  249. &sb_validator, sblock);
  250. }
  251. static int superblock_lock_zero(struct era_metadata *md,
  252. struct dm_block **sblock)
  253. {
  254. return dm_bm_write_lock_zero(md->bm, SUPERBLOCK_LOCATION,
  255. &sb_validator, sblock);
  256. }
  257. static int superblock_lock(struct era_metadata *md,
  258. struct dm_block **sblock)
  259. {
  260. return dm_bm_write_lock(md->bm, SUPERBLOCK_LOCATION,
  261. &sb_validator, sblock);
  262. }
  263. /* FIXME: duplication with cache and thin */
  264. static int superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
  265. {
  266. int r;
  267. unsigned i;
  268. struct dm_block *b;
  269. __le64 *data_le, zero = cpu_to_le64(0);
  270. unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
  271. /*
  272. * We can't use a validator here - it may be all zeroes.
  273. */
  274. r = dm_bm_read_lock(bm, SUPERBLOCK_LOCATION, NULL, &b);
  275. if (r)
  276. return r;
  277. data_le = dm_block_data(b);
  278. *result = true;
  279. for (i = 0; i < sb_block_size; i++) {
  280. if (data_le[i] != zero) {
  281. *result = false;
  282. break;
  283. }
  284. }
  285. dm_bm_unlock(b);
  286. return 0;
  287. }
  288. /*----------------------------------------------------------------*/
  289. static void ws_pack(const struct writeset_metadata *core, struct writeset_disk *disk)
  290. {
  291. disk->nr_bits = cpu_to_le32(core->nr_bits);
  292. disk->root = cpu_to_le64(core->root);
  293. }
  294. static void ws_unpack(const struct writeset_disk *disk, struct writeset_metadata *core)
  295. {
  296. core->nr_bits = le32_to_cpu(disk->nr_bits);
  297. core->root = le64_to_cpu(disk->root);
  298. }
  299. static void ws_inc(void *context, const void *value)
  300. {
  301. struct era_metadata *md = context;
  302. struct writeset_disk ws_d;
  303. dm_block_t b;
  304. memcpy(&ws_d, value, sizeof(ws_d));
  305. b = le64_to_cpu(ws_d.root);
  306. dm_tm_inc(md->tm, b);
  307. }
  308. static void ws_dec(void *context, const void *value)
  309. {
  310. struct era_metadata *md = context;
  311. struct writeset_disk ws_d;
  312. dm_block_t b;
  313. memcpy(&ws_d, value, sizeof(ws_d));
  314. b = le64_to_cpu(ws_d.root);
  315. dm_bitset_del(&md->bitset_info, b);
  316. }
  317. static int ws_eq(void *context, const void *value1, const void *value2)
  318. {
  319. return !memcmp(value1, value2, sizeof(struct writeset_disk));
  320. }
  321. /*----------------------------------------------------------------*/
  322. static void setup_writeset_tree_info(struct era_metadata *md)
  323. {
  324. struct dm_btree_value_type *vt = &md->writeset_tree_info.value_type;
  325. md->writeset_tree_info.tm = md->tm;
  326. md->writeset_tree_info.levels = 1;
  327. vt->context = md;
  328. vt->size = sizeof(struct writeset_disk);
  329. vt->inc = ws_inc;
  330. vt->dec = ws_dec;
  331. vt->equal = ws_eq;
  332. }
  333. static void setup_era_array_info(struct era_metadata *md)
  334. {
  335. struct dm_btree_value_type vt;
  336. vt.context = NULL;
  337. vt.size = sizeof(__le32);
  338. vt.inc = NULL;
  339. vt.dec = NULL;
  340. vt.equal = NULL;
  341. dm_array_info_init(&md->era_array_info, md->tm, &vt);
  342. }
  343. static void setup_infos(struct era_metadata *md)
  344. {
  345. dm_disk_bitset_init(md->tm, &md->bitset_info);
  346. setup_writeset_tree_info(md);
  347. setup_era_array_info(md);
  348. }
  349. /*----------------------------------------------------------------*/
  350. static int create_fresh_metadata(struct era_metadata *md)
  351. {
  352. int r;
  353. r = dm_tm_create_with_sm(md->bm, SUPERBLOCK_LOCATION,
  354. &md->tm, &md->sm);
  355. if (r < 0) {
  356. DMERR("dm_tm_create_with_sm failed");
  357. return r;
  358. }
  359. setup_infos(md);
  360. r = dm_btree_empty(&md->writeset_tree_info, &md->writeset_tree_root);
  361. if (r) {
  362. DMERR("couldn't create new writeset tree");
  363. goto bad;
  364. }
  365. r = dm_array_empty(&md->era_array_info, &md->era_array_root);
  366. if (r) {
  367. DMERR("couldn't create era array");
  368. goto bad;
  369. }
  370. return 0;
  371. bad:
  372. dm_sm_destroy(md->sm);
  373. dm_tm_destroy(md->tm);
  374. return r;
  375. }
  376. static int save_sm_root(struct era_metadata *md)
  377. {
  378. int r;
  379. size_t metadata_len;
  380. r = dm_sm_root_size(md->sm, &metadata_len);
  381. if (r < 0)
  382. return r;
  383. return dm_sm_copy_root(md->sm, &md->metadata_space_map_root,
  384. metadata_len);
  385. }
  386. static void copy_sm_root(struct era_metadata *md, struct superblock_disk *disk)
  387. {
  388. memcpy(&disk->metadata_space_map_root,
  389. &md->metadata_space_map_root,
  390. sizeof(md->metadata_space_map_root));
  391. }
  392. /*
  393. * Writes a superblock, including the static fields that don't get updated
  394. * with every commit (possible optimisation here). 'md' should be fully
  395. * constructed when this is called.
  396. */
  397. static void prepare_superblock(struct era_metadata *md, struct superblock_disk *disk)
  398. {
  399. disk->magic = cpu_to_le64(SUPERBLOCK_MAGIC);
  400. disk->flags = cpu_to_le32(0ul);
  401. /* FIXME: can't keep blanking the uuid (uuid is currently unused though) */
  402. memset(disk->uuid, 0, sizeof(disk->uuid));
  403. disk->version = cpu_to_le32(MAX_ERA_VERSION);
  404. copy_sm_root(md, disk);
  405. disk->data_block_size = cpu_to_le32(md->block_size);
  406. disk->metadata_block_size = cpu_to_le32(DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT);
  407. disk->nr_blocks = cpu_to_le32(md->nr_blocks);
  408. disk->current_era = cpu_to_le32(md->current_era);
  409. ws_pack(&md->current_writeset->md, &disk->current_writeset);
  410. disk->writeset_tree_root = cpu_to_le64(md->writeset_tree_root);
  411. disk->era_array_root = cpu_to_le64(md->era_array_root);
  412. disk->metadata_snap = cpu_to_le64(md->metadata_snap);
  413. }
  414. static int write_superblock(struct era_metadata *md)
  415. {
  416. int r;
  417. struct dm_block *sblock;
  418. struct superblock_disk *disk;
  419. r = save_sm_root(md);
  420. if (r) {
  421. DMERR("%s: save_sm_root failed", __func__);
  422. return r;
  423. }
  424. r = superblock_lock_zero(md, &sblock);
  425. if (r)
  426. return r;
  427. disk = dm_block_data(sblock);
  428. prepare_superblock(md, disk);
  429. return dm_tm_commit(md->tm, sblock);
  430. }
  431. /*
  432. * Assumes block_size and the infos are set.
  433. */
  434. static int format_metadata(struct era_metadata *md)
  435. {
  436. int r;
  437. r = create_fresh_metadata(md);
  438. if (r)
  439. return r;
  440. r = write_superblock(md);
  441. if (r) {
  442. dm_sm_destroy(md->sm);
  443. dm_tm_destroy(md->tm);
  444. return r;
  445. }
  446. return 0;
  447. }
  448. static int open_metadata(struct era_metadata *md)
  449. {
  450. int r;
  451. struct dm_block *sblock;
  452. struct superblock_disk *disk;
  453. r = superblock_read_lock(md, &sblock);
  454. if (r) {
  455. DMERR("couldn't read_lock superblock");
  456. return r;
  457. }
  458. disk = dm_block_data(sblock);
  459. /* Verify the data block size hasn't changed */
  460. if (le32_to_cpu(disk->data_block_size) != md->block_size) {
  461. DMERR("changing the data block size (from %u to %llu) is not supported",
  462. le32_to_cpu(disk->data_block_size), md->block_size);
  463. r = -EINVAL;
  464. goto bad;
  465. }
  466. r = dm_tm_open_with_sm(md->bm, SUPERBLOCK_LOCATION,
  467. disk->metadata_space_map_root,
  468. sizeof(disk->metadata_space_map_root),
  469. &md->tm, &md->sm);
  470. if (r) {
  471. DMERR("dm_tm_open_with_sm failed");
  472. goto bad;
  473. }
  474. setup_infos(md);
  475. md->nr_blocks = le32_to_cpu(disk->nr_blocks);
  476. md->current_era = le32_to_cpu(disk->current_era);
  477. ws_unpack(&disk->current_writeset, &md->current_writeset->md);
  478. md->writeset_tree_root = le64_to_cpu(disk->writeset_tree_root);
  479. md->era_array_root = le64_to_cpu(disk->era_array_root);
  480. md->metadata_snap = le64_to_cpu(disk->metadata_snap);
  481. md->archived_writesets = true;
  482. dm_bm_unlock(sblock);
  483. return 0;
  484. bad:
  485. dm_bm_unlock(sblock);
  486. return r;
  487. }
  488. static int open_or_format_metadata(struct era_metadata *md,
  489. bool may_format)
  490. {
  491. int r;
  492. bool unformatted = false;
  493. r = superblock_all_zeroes(md->bm, &unformatted);
  494. if (r)
  495. return r;
  496. if (unformatted)
  497. return may_format ? format_metadata(md) : -EPERM;
  498. return open_metadata(md);
  499. }
  500. static int create_persistent_data_objects(struct era_metadata *md,
  501. bool may_format)
  502. {
  503. int r;
  504. md->bm = dm_block_manager_create(md->bdev, DM_ERA_METADATA_BLOCK_SIZE,
  505. ERA_MAX_CONCURRENT_LOCKS);
  506. if (IS_ERR(md->bm)) {
  507. DMERR("could not create block manager");
  508. return PTR_ERR(md->bm);
  509. }
  510. r = open_or_format_metadata(md, may_format);
  511. if (r)
  512. dm_block_manager_destroy(md->bm);
  513. return r;
  514. }
  515. static void destroy_persistent_data_objects(struct era_metadata *md)
  516. {
  517. dm_sm_destroy(md->sm);
  518. dm_tm_destroy(md->tm);
  519. dm_block_manager_destroy(md->bm);
  520. }
  521. /*
  522. * This waits until all era_map threads have picked up the new filter.
  523. */
  524. static void swap_writeset(struct era_metadata *md, struct writeset *new_writeset)
  525. {
  526. rcu_assign_pointer(md->current_writeset, new_writeset);
  527. synchronize_rcu();
  528. }
  529. /*----------------------------------------------------------------
  530. * Writesets get 'digested' into the main era array.
  531. *
  532. * We're using a coroutine here so the worker thread can do the digestion,
  533. * thus avoiding synchronisation of the metadata. Digesting a whole
  534. * writeset in one go would cause too much latency.
  535. *--------------------------------------------------------------*/
  536. struct digest {
  537. uint32_t era;
  538. unsigned nr_bits, current_bit;
  539. struct writeset_metadata writeset;
  540. __le32 value;
  541. struct dm_disk_bitset info;
  542. int (*step)(struct era_metadata *, struct digest *);
  543. };
  544. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  545. struct digest *d);
  546. static int metadata_digest_remove_writeset(struct era_metadata *md,
  547. struct digest *d)
  548. {
  549. int r;
  550. uint64_t key = d->era;
  551. r = dm_btree_remove(&md->writeset_tree_info, md->writeset_tree_root,
  552. &key, &md->writeset_tree_root);
  553. if (r) {
  554. DMERR("%s: dm_btree_remove failed", __func__);
  555. return r;
  556. }
  557. d->step = metadata_digest_lookup_writeset;
  558. return 0;
  559. }
  560. #define INSERTS_PER_STEP 100
  561. static int metadata_digest_transcribe_writeset(struct era_metadata *md,
  562. struct digest *d)
  563. {
  564. int r;
  565. bool marked;
  566. unsigned b, e = min(d->current_bit + INSERTS_PER_STEP, d->nr_bits);
  567. for (b = d->current_bit; b < e; b++) {
  568. r = writeset_marked_on_disk(&d->info, &d->writeset, b, &marked);
  569. if (r) {
  570. DMERR("%s: writeset_marked_on_disk failed", __func__);
  571. return r;
  572. }
  573. if (!marked)
  574. continue;
  575. __dm_bless_for_disk(&d->value);
  576. r = dm_array_set_value(&md->era_array_info, md->era_array_root,
  577. b, &d->value, &md->era_array_root);
  578. if (r) {
  579. DMERR("%s: dm_array_set_value failed", __func__);
  580. return r;
  581. }
  582. }
  583. if (b == d->nr_bits)
  584. d->step = metadata_digest_remove_writeset;
  585. else
  586. d->current_bit = b;
  587. return 0;
  588. }
  589. static int metadata_digest_lookup_writeset(struct era_metadata *md,
  590. struct digest *d)
  591. {
  592. int r;
  593. uint64_t key;
  594. struct writeset_disk disk;
  595. r = dm_btree_find_lowest_key(&md->writeset_tree_info,
  596. md->writeset_tree_root, &key);
  597. if (r < 0)
  598. return r;
  599. d->era = key;
  600. r = dm_btree_lookup(&md->writeset_tree_info,
  601. md->writeset_tree_root, &key, &disk);
  602. if (r) {
  603. if (r == -ENODATA) {
  604. d->step = NULL;
  605. return 0;
  606. }
  607. DMERR("%s: dm_btree_lookup failed", __func__);
  608. return r;
  609. }
  610. ws_unpack(&disk, &d->writeset);
  611. d->value = cpu_to_le32(key);
  612. /*
  613. * We initialise another bitset info to avoid any caching side effects
  614. * with the previous one.
  615. */
  616. dm_disk_bitset_init(md->tm, &d->info);
  617. d->nr_bits = min(d->writeset.nr_bits, md->nr_blocks);
  618. d->current_bit = 0;
  619. d->step = metadata_digest_transcribe_writeset;
  620. return 0;
  621. }
  622. static int metadata_digest_start(struct era_metadata *md, struct digest *d)
  623. {
  624. if (d->step)
  625. return 0;
  626. memset(d, 0, sizeof(*d));
  627. d->step = metadata_digest_lookup_writeset;
  628. return 0;
  629. }
  630. /*----------------------------------------------------------------
  631. * High level metadata interface. Target methods should use these, and not
  632. * the lower level ones.
  633. *--------------------------------------------------------------*/
  634. static struct era_metadata *metadata_open(struct block_device *bdev,
  635. sector_t block_size,
  636. bool may_format)
  637. {
  638. int r;
  639. struct era_metadata *md = kzalloc(sizeof(*md), GFP_KERNEL);
  640. if (!md)
  641. return NULL;
  642. md->bdev = bdev;
  643. md->block_size = block_size;
  644. md->writesets[0].md.root = INVALID_WRITESET_ROOT;
  645. md->writesets[1].md.root = INVALID_WRITESET_ROOT;
  646. md->current_writeset = &md->writesets[0];
  647. r = create_persistent_data_objects(md, may_format);
  648. if (r) {
  649. kfree(md);
  650. return ERR_PTR(r);
  651. }
  652. return md;
  653. }
  654. static void metadata_close(struct era_metadata *md)
  655. {
  656. writeset_free(&md->writesets[0]);
  657. writeset_free(&md->writesets[1]);
  658. destroy_persistent_data_objects(md);
  659. kfree(md);
  660. }
  661. static bool valid_nr_blocks(dm_block_t n)
  662. {
  663. /*
  664. * dm_bitset restricts us to 2^32. test_bit & co. restrict us
  665. * further to 2^31 - 1
  666. */
  667. return n < (1ull << 31);
  668. }
  669. static int metadata_resize(struct era_metadata *md, void *arg)
  670. {
  671. int r;
  672. dm_block_t *new_size = arg;
  673. __le32 value;
  674. if (!valid_nr_blocks(*new_size)) {
  675. DMERR("Invalid number of origin blocks %llu",
  676. (unsigned long long) *new_size);
  677. return -EINVAL;
  678. }
  679. writeset_free(&md->writesets[0]);
  680. writeset_free(&md->writesets[1]);
  681. r = writeset_alloc(&md->writesets[0], *new_size);
  682. if (r) {
  683. DMERR("%s: writeset_alloc failed for writeset 0", __func__);
  684. return r;
  685. }
  686. r = writeset_alloc(&md->writesets[1], *new_size);
  687. if (r) {
  688. DMERR("%s: writeset_alloc failed for writeset 1", __func__);
  689. writeset_free(&md->writesets[0]);
  690. return r;
  691. }
  692. value = cpu_to_le32(0u);
  693. __dm_bless_for_disk(&value);
  694. r = dm_array_resize(&md->era_array_info, md->era_array_root,
  695. md->nr_blocks, *new_size,
  696. &value, &md->era_array_root);
  697. if (r) {
  698. DMERR("%s: dm_array_resize failed", __func__);
  699. writeset_free(&md->writesets[0]);
  700. writeset_free(&md->writesets[1]);
  701. return r;
  702. }
  703. md->nr_blocks = *new_size;
  704. return 0;
  705. }
  706. static int metadata_era_archive(struct era_metadata *md)
  707. {
  708. int r;
  709. uint64_t keys[1];
  710. struct writeset_disk value;
  711. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  712. &md->current_writeset->md.root);
  713. if (r) {
  714. DMERR("%s: dm_bitset_flush failed", __func__);
  715. return r;
  716. }
  717. ws_pack(&md->current_writeset->md, &value);
  718. keys[0] = md->current_era;
  719. __dm_bless_for_disk(&value);
  720. r = dm_btree_insert(&md->writeset_tree_info, md->writeset_tree_root,
  721. keys, &value, &md->writeset_tree_root);
  722. if (r) {
  723. DMERR("%s: couldn't insert writeset into btree", __func__);
  724. /* FIXME: fail mode */
  725. return r;
  726. }
  727. md->current_writeset->md.root = INVALID_WRITESET_ROOT;
  728. md->archived_writesets = true;
  729. return 0;
  730. }
  731. static struct writeset *next_writeset(struct era_metadata *md)
  732. {
  733. return (md->current_writeset == &md->writesets[0]) ?
  734. &md->writesets[1] : &md->writesets[0];
  735. }
  736. static int metadata_new_era(struct era_metadata *md)
  737. {
  738. int r;
  739. struct writeset *new_writeset = next_writeset(md);
  740. r = writeset_init(&md->bitset_info, new_writeset, md->nr_blocks);
  741. if (r) {
  742. DMERR("%s: writeset_init failed", __func__);
  743. return r;
  744. }
  745. swap_writeset(md, new_writeset);
  746. md->current_era++;
  747. return 0;
  748. }
  749. static int metadata_era_rollover(struct era_metadata *md)
  750. {
  751. int r;
  752. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  753. r = metadata_era_archive(md);
  754. if (r) {
  755. DMERR("%s: metadata_archive_era failed", __func__);
  756. /* FIXME: fail mode? */
  757. return r;
  758. }
  759. }
  760. r = metadata_new_era(md);
  761. if (r) {
  762. DMERR("%s: new era failed", __func__);
  763. /* FIXME: fail mode */
  764. return r;
  765. }
  766. return 0;
  767. }
  768. static bool metadata_current_marked(struct era_metadata *md, dm_block_t block)
  769. {
  770. bool r;
  771. struct writeset *ws;
  772. rcu_read_lock();
  773. ws = rcu_dereference(md->current_writeset);
  774. r = writeset_marked(ws, block);
  775. rcu_read_unlock();
  776. return r;
  777. }
  778. static int metadata_commit(struct era_metadata *md)
  779. {
  780. int r;
  781. struct dm_block *sblock;
  782. if (md->current_writeset->md.root != INVALID_WRITESET_ROOT) {
  783. r = dm_bitset_flush(&md->bitset_info, md->current_writeset->md.root,
  784. &md->current_writeset->md.root);
  785. if (r) {
  786. DMERR("%s: bitset flush failed", __func__);
  787. return r;
  788. }
  789. }
  790. r = dm_tm_pre_commit(md->tm);
  791. if (r) {
  792. DMERR("%s: pre commit failed", __func__);
  793. return r;
  794. }
  795. r = save_sm_root(md);
  796. if (r) {
  797. DMERR("%s: save_sm_root failed", __func__);
  798. return r;
  799. }
  800. r = superblock_lock(md, &sblock);
  801. if (r) {
  802. DMERR("%s: superblock lock failed", __func__);
  803. return r;
  804. }
  805. prepare_superblock(md, dm_block_data(sblock));
  806. return dm_tm_commit(md->tm, sblock);
  807. }
  808. static int metadata_checkpoint(struct era_metadata *md)
  809. {
  810. /*
  811. * For now we just rollover, but later I want to put a check in to
  812. * avoid this if the filter is still pretty fresh.
  813. */
  814. return metadata_era_rollover(md);
  815. }
  816. /*
  817. * Metadata snapshots allow userland to access era data.
  818. */
  819. static int metadata_take_snap(struct era_metadata *md)
  820. {
  821. int r, inc;
  822. struct dm_block *clone;
  823. if (md->metadata_snap != SUPERBLOCK_LOCATION) {
  824. DMERR("%s: metadata snapshot already exists", __func__);
  825. return -EINVAL;
  826. }
  827. r = metadata_era_rollover(md);
  828. if (r) {
  829. DMERR("%s: era rollover failed", __func__);
  830. return r;
  831. }
  832. r = metadata_commit(md);
  833. if (r) {
  834. DMERR("%s: pre commit failed", __func__);
  835. return r;
  836. }
  837. r = dm_sm_inc_block(md->sm, SUPERBLOCK_LOCATION);
  838. if (r) {
  839. DMERR("%s: couldn't increment superblock", __func__);
  840. return r;
  841. }
  842. r = dm_tm_shadow_block(md->tm, SUPERBLOCK_LOCATION,
  843. &sb_validator, &clone, &inc);
  844. if (r) {
  845. DMERR("%s: couldn't shadow superblock", __func__);
  846. dm_sm_dec_block(md->sm, SUPERBLOCK_LOCATION);
  847. return r;
  848. }
  849. BUG_ON(!inc);
  850. r = dm_sm_inc_block(md->sm, md->writeset_tree_root);
  851. if (r) {
  852. DMERR("%s: couldn't inc writeset tree root", __func__);
  853. dm_tm_unlock(md->tm, clone);
  854. return r;
  855. }
  856. r = dm_sm_inc_block(md->sm, md->era_array_root);
  857. if (r) {
  858. DMERR("%s: couldn't inc era tree root", __func__);
  859. dm_sm_dec_block(md->sm, md->writeset_tree_root);
  860. dm_tm_unlock(md->tm, clone);
  861. return r;
  862. }
  863. md->metadata_snap = dm_block_location(clone);
  864. dm_tm_unlock(md->tm, clone);
  865. return 0;
  866. }
  867. static int metadata_drop_snap(struct era_metadata *md)
  868. {
  869. int r;
  870. dm_block_t location;
  871. struct dm_block *clone;
  872. struct superblock_disk *disk;
  873. if (md->metadata_snap == SUPERBLOCK_LOCATION) {
  874. DMERR("%s: no snap to drop", __func__);
  875. return -EINVAL;
  876. }
  877. r = dm_tm_read_lock(md->tm, md->metadata_snap, &sb_validator, &clone);
  878. if (r) {
  879. DMERR("%s: couldn't read lock superblock clone", __func__);
  880. return r;
  881. }
  882. /*
  883. * Whatever happens now we'll commit with no record of the metadata
  884. * snap.
  885. */
  886. md->metadata_snap = SUPERBLOCK_LOCATION;
  887. disk = dm_block_data(clone);
  888. r = dm_btree_del(&md->writeset_tree_info,
  889. le64_to_cpu(disk->writeset_tree_root));
  890. if (r) {
  891. DMERR("%s: error deleting writeset tree clone", __func__);
  892. dm_tm_unlock(md->tm, clone);
  893. return r;
  894. }
  895. r = dm_array_del(&md->era_array_info, le64_to_cpu(disk->era_array_root));
  896. if (r) {
  897. DMERR("%s: error deleting era array clone", __func__);
  898. dm_tm_unlock(md->tm, clone);
  899. return r;
  900. }
  901. location = dm_block_location(clone);
  902. dm_tm_unlock(md->tm, clone);
  903. return dm_sm_dec_block(md->sm, location);
  904. }
  905. struct metadata_stats {
  906. dm_block_t used;
  907. dm_block_t total;
  908. dm_block_t snap;
  909. uint32_t era;
  910. };
  911. static int metadata_get_stats(struct era_metadata *md, void *ptr)
  912. {
  913. int r;
  914. struct metadata_stats *s = ptr;
  915. dm_block_t nr_free, nr_total;
  916. r = dm_sm_get_nr_free(md->sm, &nr_free);
  917. if (r) {
  918. DMERR("dm_sm_get_nr_free returned %d", r);
  919. return r;
  920. }
  921. r = dm_sm_get_nr_blocks(md->sm, &nr_total);
  922. if (r) {
  923. DMERR("dm_pool_get_metadata_dev_size returned %d", r);
  924. return r;
  925. }
  926. s->used = nr_total - nr_free;
  927. s->total = nr_total;
  928. s->snap = md->metadata_snap;
  929. s->era = md->current_era;
  930. return 0;
  931. }
  932. /*----------------------------------------------------------------*/
  933. struct era {
  934. struct dm_target *ti;
  935. struct dm_dev *metadata_dev;
  936. struct dm_dev *origin_dev;
  937. dm_block_t nr_blocks;
  938. uint32_t sectors_per_block;
  939. int sectors_per_block_shift;
  940. struct era_metadata *md;
  941. struct workqueue_struct *wq;
  942. struct work_struct worker;
  943. spinlock_t deferred_lock;
  944. struct bio_list deferred_bios;
  945. spinlock_t rpc_lock;
  946. struct list_head rpc_calls;
  947. struct digest digest;
  948. atomic_t suspended;
  949. };
  950. struct rpc {
  951. struct list_head list;
  952. int (*fn0)(struct era_metadata *);
  953. int (*fn1)(struct era_metadata *, void *);
  954. void *arg;
  955. int result;
  956. struct completion complete;
  957. };
  958. /*----------------------------------------------------------------
  959. * Remapping.
  960. *---------------------------------------------------------------*/
  961. static bool block_size_is_power_of_two(struct era *era)
  962. {
  963. return era->sectors_per_block_shift >= 0;
  964. }
  965. static dm_block_t get_block(struct era *era, struct bio *bio)
  966. {
  967. sector_t block_nr = bio->bi_iter.bi_sector;
  968. if (!block_size_is_power_of_two(era))
  969. (void) sector_div(block_nr, era->sectors_per_block);
  970. else
  971. block_nr >>= era->sectors_per_block_shift;
  972. return block_nr;
  973. }
  974. static void remap_to_origin(struct era *era, struct bio *bio)
  975. {
  976. bio_set_dev(bio, era->origin_dev->bdev);
  977. }
  978. /*----------------------------------------------------------------
  979. * Worker thread
  980. *--------------------------------------------------------------*/
  981. static void wake_worker(struct era *era)
  982. {
  983. if (!atomic_read(&era->suspended))
  984. queue_work(era->wq, &era->worker);
  985. }
  986. static void process_old_eras(struct era *era)
  987. {
  988. int r;
  989. if (!era->digest.step)
  990. return;
  991. r = era->digest.step(era->md, &era->digest);
  992. if (r < 0) {
  993. DMERR("%s: digest step failed, stopping digestion", __func__);
  994. era->digest.step = NULL;
  995. } else if (era->digest.step)
  996. wake_worker(era);
  997. }
  998. static void process_deferred_bios(struct era *era)
  999. {
  1000. int r;
  1001. struct bio_list deferred_bios, marked_bios;
  1002. struct bio *bio;
  1003. struct blk_plug plug;
  1004. bool commit_needed = false;
  1005. bool failed = false;
  1006. struct writeset *ws = era->md->current_writeset;
  1007. bio_list_init(&deferred_bios);
  1008. bio_list_init(&marked_bios);
  1009. spin_lock(&era->deferred_lock);
  1010. bio_list_merge(&deferred_bios, &era->deferred_bios);
  1011. bio_list_init(&era->deferred_bios);
  1012. spin_unlock(&era->deferred_lock);
  1013. if (bio_list_empty(&deferred_bios))
  1014. return;
  1015. while ((bio = bio_list_pop(&deferred_bios))) {
  1016. r = writeset_test_and_set(&era->md->bitset_info, ws,
  1017. get_block(era, bio));
  1018. if (r < 0) {
  1019. /*
  1020. * This is bad news, we need to rollback.
  1021. * FIXME: finish.
  1022. */
  1023. failed = true;
  1024. } else if (r == 0)
  1025. commit_needed = true;
  1026. bio_list_add(&marked_bios, bio);
  1027. }
  1028. if (commit_needed) {
  1029. r = metadata_commit(era->md);
  1030. if (r)
  1031. failed = true;
  1032. }
  1033. if (failed)
  1034. while ((bio = bio_list_pop(&marked_bios)))
  1035. bio_io_error(bio);
  1036. else {
  1037. blk_start_plug(&plug);
  1038. while ((bio = bio_list_pop(&marked_bios))) {
  1039. /*
  1040. * Only update the in-core writeset if the on-disk one
  1041. * was updated too.
  1042. */
  1043. if (commit_needed)
  1044. set_bit(get_block(era, bio), ws->bits);
  1045. submit_bio_noacct(bio);
  1046. }
  1047. blk_finish_plug(&plug);
  1048. }
  1049. }
  1050. static void process_rpc_calls(struct era *era)
  1051. {
  1052. int r;
  1053. bool need_commit = false;
  1054. struct list_head calls;
  1055. struct rpc *rpc, *tmp;
  1056. INIT_LIST_HEAD(&calls);
  1057. spin_lock(&era->rpc_lock);
  1058. list_splice_init(&era->rpc_calls, &calls);
  1059. spin_unlock(&era->rpc_lock);
  1060. list_for_each_entry_safe(rpc, tmp, &calls, list) {
  1061. rpc->result = rpc->fn0 ? rpc->fn0(era->md) : rpc->fn1(era->md, rpc->arg);
  1062. need_commit = true;
  1063. }
  1064. if (need_commit) {
  1065. r = metadata_commit(era->md);
  1066. if (r)
  1067. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1068. rpc->result = r;
  1069. }
  1070. list_for_each_entry_safe(rpc, tmp, &calls, list)
  1071. complete(&rpc->complete);
  1072. }
  1073. static void kick_off_digest(struct era *era)
  1074. {
  1075. if (era->md->archived_writesets) {
  1076. era->md->archived_writesets = false;
  1077. metadata_digest_start(era->md, &era->digest);
  1078. }
  1079. }
  1080. static void do_work(struct work_struct *ws)
  1081. {
  1082. struct era *era = container_of(ws, struct era, worker);
  1083. kick_off_digest(era);
  1084. process_old_eras(era);
  1085. process_deferred_bios(era);
  1086. process_rpc_calls(era);
  1087. }
  1088. static void defer_bio(struct era *era, struct bio *bio)
  1089. {
  1090. spin_lock(&era->deferred_lock);
  1091. bio_list_add(&era->deferred_bios, bio);
  1092. spin_unlock(&era->deferred_lock);
  1093. wake_worker(era);
  1094. }
  1095. /*
  1096. * Make an rpc call to the worker to change the metadata.
  1097. */
  1098. static int perform_rpc(struct era *era, struct rpc *rpc)
  1099. {
  1100. rpc->result = 0;
  1101. init_completion(&rpc->complete);
  1102. spin_lock(&era->rpc_lock);
  1103. list_add(&rpc->list, &era->rpc_calls);
  1104. spin_unlock(&era->rpc_lock);
  1105. wake_worker(era);
  1106. wait_for_completion(&rpc->complete);
  1107. return rpc->result;
  1108. }
  1109. static int in_worker0(struct era *era, int (*fn)(struct era_metadata *))
  1110. {
  1111. struct rpc rpc;
  1112. rpc.fn0 = fn;
  1113. rpc.fn1 = NULL;
  1114. return perform_rpc(era, &rpc);
  1115. }
  1116. static int in_worker1(struct era *era,
  1117. int (*fn)(struct era_metadata *, void *), void *arg)
  1118. {
  1119. struct rpc rpc;
  1120. rpc.fn0 = NULL;
  1121. rpc.fn1 = fn;
  1122. rpc.arg = arg;
  1123. return perform_rpc(era, &rpc);
  1124. }
  1125. static void start_worker(struct era *era)
  1126. {
  1127. atomic_set(&era->suspended, 0);
  1128. }
  1129. static void stop_worker(struct era *era)
  1130. {
  1131. atomic_set(&era->suspended, 1);
  1132. flush_workqueue(era->wq);
  1133. }
  1134. /*----------------------------------------------------------------
  1135. * Target methods
  1136. *--------------------------------------------------------------*/
  1137. static void era_destroy(struct era *era)
  1138. {
  1139. if (era->md)
  1140. metadata_close(era->md);
  1141. if (era->wq)
  1142. destroy_workqueue(era->wq);
  1143. if (era->origin_dev)
  1144. dm_put_device(era->ti, era->origin_dev);
  1145. if (era->metadata_dev)
  1146. dm_put_device(era->ti, era->metadata_dev);
  1147. kfree(era);
  1148. }
  1149. static dm_block_t calc_nr_blocks(struct era *era)
  1150. {
  1151. return dm_sector_div_up(era->ti->len, era->sectors_per_block);
  1152. }
  1153. static bool valid_block_size(dm_block_t block_size)
  1154. {
  1155. bool greater_than_zero = block_size > 0;
  1156. bool multiple_of_min_block_size = (block_size & (MIN_BLOCK_SIZE - 1)) == 0;
  1157. return greater_than_zero && multiple_of_min_block_size;
  1158. }
  1159. /*
  1160. * <metadata dev> <data dev> <data block size (sectors)>
  1161. */
  1162. static int era_ctr(struct dm_target *ti, unsigned argc, char **argv)
  1163. {
  1164. int r;
  1165. char dummy;
  1166. struct era *era;
  1167. struct era_metadata *md;
  1168. if (argc != 3) {
  1169. ti->error = "Invalid argument count";
  1170. return -EINVAL;
  1171. }
  1172. era = kzalloc(sizeof(*era), GFP_KERNEL);
  1173. if (!era) {
  1174. ti->error = "Error allocating era structure";
  1175. return -ENOMEM;
  1176. }
  1177. era->ti = ti;
  1178. r = dm_get_device(ti, argv[0], FMODE_READ | FMODE_WRITE, &era->metadata_dev);
  1179. if (r) {
  1180. ti->error = "Error opening metadata device";
  1181. era_destroy(era);
  1182. return -EINVAL;
  1183. }
  1184. r = dm_get_device(ti, argv[1], FMODE_READ | FMODE_WRITE, &era->origin_dev);
  1185. if (r) {
  1186. ti->error = "Error opening data device";
  1187. era_destroy(era);
  1188. return -EINVAL;
  1189. }
  1190. r = sscanf(argv[2], "%u%c", &era->sectors_per_block, &dummy);
  1191. if (r != 1) {
  1192. ti->error = "Error parsing block size";
  1193. era_destroy(era);
  1194. return -EINVAL;
  1195. }
  1196. r = dm_set_target_max_io_len(ti, era->sectors_per_block);
  1197. if (r) {
  1198. ti->error = "could not set max io len";
  1199. era_destroy(era);
  1200. return -EINVAL;
  1201. }
  1202. if (!valid_block_size(era->sectors_per_block)) {
  1203. ti->error = "Invalid block size";
  1204. era_destroy(era);
  1205. return -EINVAL;
  1206. }
  1207. if (era->sectors_per_block & (era->sectors_per_block - 1))
  1208. era->sectors_per_block_shift = -1;
  1209. else
  1210. era->sectors_per_block_shift = __ffs(era->sectors_per_block);
  1211. md = metadata_open(era->metadata_dev->bdev, era->sectors_per_block, true);
  1212. if (IS_ERR(md)) {
  1213. ti->error = "Error reading metadata";
  1214. era_destroy(era);
  1215. return PTR_ERR(md);
  1216. }
  1217. era->md = md;
  1218. era->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
  1219. if (!era->wq) {
  1220. ti->error = "could not create workqueue for metadata object";
  1221. era_destroy(era);
  1222. return -ENOMEM;
  1223. }
  1224. INIT_WORK(&era->worker, do_work);
  1225. spin_lock_init(&era->deferred_lock);
  1226. bio_list_init(&era->deferred_bios);
  1227. spin_lock_init(&era->rpc_lock);
  1228. INIT_LIST_HEAD(&era->rpc_calls);
  1229. ti->private = era;
  1230. ti->num_flush_bios = 1;
  1231. ti->flush_supported = true;
  1232. ti->num_discard_bios = 1;
  1233. return 0;
  1234. }
  1235. static void era_dtr(struct dm_target *ti)
  1236. {
  1237. era_destroy(ti->private);
  1238. }
  1239. static int era_map(struct dm_target *ti, struct bio *bio)
  1240. {
  1241. struct era *era = ti->private;
  1242. dm_block_t block = get_block(era, bio);
  1243. /*
  1244. * All bios get remapped to the origin device. We do this now, but
  1245. * it may not get issued until later. Depending on whether the
  1246. * block is marked in this era.
  1247. */
  1248. remap_to_origin(era, bio);
  1249. /*
  1250. * REQ_PREFLUSH bios carry no data, so we're not interested in them.
  1251. */
  1252. if (!(bio->bi_opf & REQ_PREFLUSH) &&
  1253. (bio_data_dir(bio) == WRITE) &&
  1254. !metadata_current_marked(era->md, block)) {
  1255. defer_bio(era, bio);
  1256. return DM_MAPIO_SUBMITTED;
  1257. }
  1258. return DM_MAPIO_REMAPPED;
  1259. }
  1260. static void era_postsuspend(struct dm_target *ti)
  1261. {
  1262. int r;
  1263. struct era *era = ti->private;
  1264. r = in_worker0(era, metadata_era_archive);
  1265. if (r) {
  1266. DMERR("%s: couldn't archive current era", __func__);
  1267. /* FIXME: fail mode */
  1268. }
  1269. stop_worker(era);
  1270. }
  1271. static int era_preresume(struct dm_target *ti)
  1272. {
  1273. int r;
  1274. struct era *era = ti->private;
  1275. dm_block_t new_size = calc_nr_blocks(era);
  1276. if (era->nr_blocks != new_size) {
  1277. r = metadata_resize(era->md, &new_size);
  1278. if (r) {
  1279. DMERR("%s: metadata_resize failed", __func__);
  1280. return r;
  1281. }
  1282. r = metadata_commit(era->md);
  1283. if (r) {
  1284. DMERR("%s: metadata_commit failed", __func__);
  1285. return r;
  1286. }
  1287. era->nr_blocks = new_size;
  1288. }
  1289. start_worker(era);
  1290. r = in_worker0(era, metadata_era_rollover);
  1291. if (r) {
  1292. DMERR("%s: metadata_era_rollover failed", __func__);
  1293. return r;
  1294. }
  1295. return 0;
  1296. }
  1297. /*
  1298. * Status format:
  1299. *
  1300. * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
  1301. * <current era> <held metadata root | '-'>
  1302. */
  1303. static void era_status(struct dm_target *ti, status_type_t type,
  1304. unsigned status_flags, char *result, unsigned maxlen)
  1305. {
  1306. int r;
  1307. struct era *era = ti->private;
  1308. ssize_t sz = 0;
  1309. struct metadata_stats stats;
  1310. char buf[BDEVNAME_SIZE];
  1311. switch (type) {
  1312. case STATUSTYPE_INFO:
  1313. r = in_worker1(era, metadata_get_stats, &stats);
  1314. if (r)
  1315. goto err;
  1316. DMEMIT("%u %llu/%llu %u",
  1317. (unsigned) (DM_ERA_METADATA_BLOCK_SIZE >> SECTOR_SHIFT),
  1318. (unsigned long long) stats.used,
  1319. (unsigned long long) stats.total,
  1320. (unsigned) stats.era);
  1321. if (stats.snap != SUPERBLOCK_LOCATION)
  1322. DMEMIT(" %llu", stats.snap);
  1323. else
  1324. DMEMIT(" -");
  1325. break;
  1326. case STATUSTYPE_TABLE:
  1327. format_dev_t(buf, era->metadata_dev->bdev->bd_dev);
  1328. DMEMIT("%s ", buf);
  1329. format_dev_t(buf, era->origin_dev->bdev->bd_dev);
  1330. DMEMIT("%s %u", buf, era->sectors_per_block);
  1331. break;
  1332. }
  1333. return;
  1334. err:
  1335. DMEMIT("Error");
  1336. }
  1337. static int era_message(struct dm_target *ti, unsigned argc, char **argv,
  1338. char *result, unsigned maxlen)
  1339. {
  1340. struct era *era = ti->private;
  1341. if (argc != 1) {
  1342. DMERR("incorrect number of message arguments");
  1343. return -EINVAL;
  1344. }
  1345. if (!strcasecmp(argv[0], "checkpoint"))
  1346. return in_worker0(era, metadata_checkpoint);
  1347. if (!strcasecmp(argv[0], "take_metadata_snap"))
  1348. return in_worker0(era, metadata_take_snap);
  1349. if (!strcasecmp(argv[0], "drop_metadata_snap"))
  1350. return in_worker0(era, metadata_drop_snap);
  1351. DMERR("unsupported message '%s'", argv[0]);
  1352. return -EINVAL;
  1353. }
  1354. static sector_t get_dev_size(struct dm_dev *dev)
  1355. {
  1356. return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
  1357. }
  1358. static int era_iterate_devices(struct dm_target *ti,
  1359. iterate_devices_callout_fn fn, void *data)
  1360. {
  1361. struct era *era = ti->private;
  1362. return fn(ti, era->origin_dev, 0, get_dev_size(era->origin_dev), data);
  1363. }
  1364. static void era_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1365. {
  1366. struct era *era = ti->private;
  1367. uint64_t io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
  1368. /*
  1369. * If the system-determined stacked limits are compatible with the
  1370. * era device's blocksize (io_opt is a factor) do not override them.
  1371. */
  1372. if (io_opt_sectors < era->sectors_per_block ||
  1373. do_div(io_opt_sectors, era->sectors_per_block)) {
  1374. blk_limits_io_min(limits, 0);
  1375. blk_limits_io_opt(limits, era->sectors_per_block << SECTOR_SHIFT);
  1376. }
  1377. }
  1378. /*----------------------------------------------------------------*/
  1379. static struct target_type era_target = {
  1380. .name = "era",
  1381. .version = {1, 0, 0},
  1382. .module = THIS_MODULE,
  1383. .ctr = era_ctr,
  1384. .dtr = era_dtr,
  1385. .map = era_map,
  1386. .postsuspend = era_postsuspend,
  1387. .preresume = era_preresume,
  1388. .status = era_status,
  1389. .message = era_message,
  1390. .iterate_devices = era_iterate_devices,
  1391. .io_hints = era_io_hints
  1392. };
  1393. static int __init dm_era_init(void)
  1394. {
  1395. int r;
  1396. r = dm_register_target(&era_target);
  1397. if (r) {
  1398. DMERR("era target registration failed: %d", r);
  1399. return r;
  1400. }
  1401. return 0;
  1402. }
  1403. static void __exit dm_era_exit(void)
  1404. {
  1405. dm_unregister_target(&era_target);
  1406. }
  1407. module_init(dm_era_init);
  1408. module_exit(dm_era_exit);
  1409. MODULE_DESCRIPTION(DM_NAME " era target");
  1410. MODULE_AUTHOR("Joe Thornber <ejt@redhat.com>");
  1411. MODULE_LICENSE("GPL");