dm-cache-metadata.c 42 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821
  1. /*
  2. * Copyright (C) 2012 Red Hat, Inc.
  3. *
  4. * This file is released under the GPL.
  5. */
  6. #include "dm-cache-metadata.h"
  7. #include "persistent-data/dm-array.h"
  8. #include "persistent-data/dm-bitset.h"
  9. #include "persistent-data/dm-space-map.h"
  10. #include "persistent-data/dm-space-map-disk.h"
  11. #include "persistent-data/dm-transaction-manager.h"
  12. #include <linux/device-mapper.h>
  13. #include <linux/refcount.h>
  14. /*----------------------------------------------------------------*/
  15. #define DM_MSG_PREFIX "cache metadata"
  16. #define CACHE_SUPERBLOCK_MAGIC 06142003
  17. #define CACHE_SUPERBLOCK_LOCATION 0
  18. /*
  19. * defines a range of metadata versions that this module can handle.
  20. */
  21. #define MIN_CACHE_VERSION 1
  22. #define MAX_CACHE_VERSION 2
  23. /*
  24. * 3 for btree insert +
  25. * 2 for btree lookup used within space map
  26. */
  27. #define CACHE_MAX_CONCURRENT_LOCKS 5
  28. #define SPACE_MAP_ROOT_SIZE 128
  29. enum superblock_flag_bits {
  30. /* for spotting crashes that would invalidate the dirty bitset */
  31. CLEAN_SHUTDOWN,
  32. /* metadata must be checked using the tools */
  33. NEEDS_CHECK,
  34. };
  35. /*
  36. * Each mapping from cache block -> origin block carries a set of flags.
  37. */
  38. enum mapping_bits {
  39. /*
  40. * A valid mapping. Because we're using an array we clear this
  41. * flag for an non existant mapping.
  42. */
  43. M_VALID = 1,
  44. /*
  45. * The data on the cache is different from that on the origin.
  46. * This flag is only used by metadata format 1.
  47. */
  48. M_DIRTY = 2
  49. };
  50. struct cache_disk_superblock {
  51. __le32 csum;
  52. __le32 flags;
  53. __le64 blocknr;
  54. __u8 uuid[16];
  55. __le64 magic;
  56. __le32 version;
  57. __u8 policy_name[CACHE_POLICY_NAME_SIZE];
  58. __le32 policy_hint_size;
  59. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  60. __le64 mapping_root;
  61. __le64 hint_root;
  62. __le64 discard_root;
  63. __le64 discard_block_size;
  64. __le64 discard_nr_blocks;
  65. __le32 data_block_size;
  66. __le32 metadata_block_size;
  67. __le32 cache_blocks;
  68. __le32 compat_flags;
  69. __le32 compat_ro_flags;
  70. __le32 incompat_flags;
  71. __le32 read_hits;
  72. __le32 read_misses;
  73. __le32 write_hits;
  74. __le32 write_misses;
  75. __le32 policy_version[CACHE_POLICY_VERSION_SIZE];
  76. /*
  77. * Metadata format 2 fields.
  78. */
  79. __le64 dirty_root;
  80. } __packed;
  81. struct dm_cache_metadata {
  82. refcount_t ref_count;
  83. struct list_head list;
  84. unsigned version;
  85. struct block_device *bdev;
  86. struct dm_block_manager *bm;
  87. struct dm_space_map *metadata_sm;
  88. struct dm_transaction_manager *tm;
  89. struct dm_array_info info;
  90. struct dm_array_info hint_info;
  91. struct dm_disk_bitset discard_info;
  92. struct rw_semaphore root_lock;
  93. unsigned long flags;
  94. dm_block_t root;
  95. dm_block_t hint_root;
  96. dm_block_t discard_root;
  97. sector_t discard_block_size;
  98. dm_dblock_t discard_nr_blocks;
  99. sector_t data_block_size;
  100. dm_cblock_t cache_blocks;
  101. bool changed:1;
  102. bool clean_when_opened:1;
  103. char policy_name[CACHE_POLICY_NAME_SIZE];
  104. unsigned policy_version[CACHE_POLICY_VERSION_SIZE];
  105. size_t policy_hint_size;
  106. struct dm_cache_statistics stats;
  107. /*
  108. * Reading the space map root can fail, so we read it into this
  109. * buffer before the superblock is locked and updated.
  110. */
  111. __u8 metadata_space_map_root[SPACE_MAP_ROOT_SIZE];
  112. /*
  113. * Set if a transaction has to be aborted but the attempt to roll
  114. * back to the previous (good) transaction failed. The only
  115. * metadata operation permissible in this state is the closing of
  116. * the device.
  117. */
  118. bool fail_io:1;
  119. /*
  120. * Metadata format 2 fields.
  121. */
  122. dm_block_t dirty_root;
  123. struct dm_disk_bitset dirty_info;
  124. /*
  125. * These structures are used when loading metadata. They're too
  126. * big to put on the stack.
  127. */
  128. struct dm_array_cursor mapping_cursor;
  129. struct dm_array_cursor hint_cursor;
  130. struct dm_bitset_cursor dirty_cursor;
  131. };
  132. /*-------------------------------------------------------------------
  133. * superblock validator
  134. *-----------------------------------------------------------------*/
  135. #define SUPERBLOCK_CSUM_XOR 9031977
  136. static void sb_prepare_for_write(struct dm_block_validator *v,
  137. struct dm_block *b,
  138. size_t sb_block_size)
  139. {
  140. struct cache_disk_superblock *disk_super = dm_block_data(b);
  141. disk_super->blocknr = cpu_to_le64(dm_block_location(b));
  142. disk_super->csum = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
  143. sb_block_size - sizeof(__le32),
  144. SUPERBLOCK_CSUM_XOR));
  145. }
  146. static int check_metadata_version(struct cache_disk_superblock *disk_super)
  147. {
  148. uint32_t metadata_version = le32_to_cpu(disk_super->version);
  149. if (metadata_version < MIN_CACHE_VERSION || metadata_version > MAX_CACHE_VERSION) {
  150. DMERR("Cache metadata version %u found, but only versions between %u and %u supported.",
  151. metadata_version, MIN_CACHE_VERSION, MAX_CACHE_VERSION);
  152. return -EINVAL;
  153. }
  154. return 0;
  155. }
  156. static int sb_check(struct dm_block_validator *v,
  157. struct dm_block *b,
  158. size_t sb_block_size)
  159. {
  160. struct cache_disk_superblock *disk_super = dm_block_data(b);
  161. __le32 csum_le;
  162. if (dm_block_location(b) != le64_to_cpu(disk_super->blocknr)) {
  163. DMERR("sb_check failed: blocknr %llu: wanted %llu",
  164. le64_to_cpu(disk_super->blocknr),
  165. (unsigned long long)dm_block_location(b));
  166. return -ENOTBLK;
  167. }
  168. if (le64_to_cpu(disk_super->magic) != CACHE_SUPERBLOCK_MAGIC) {
  169. DMERR("sb_check failed: magic %llu: wanted %llu",
  170. le64_to_cpu(disk_super->magic),
  171. (unsigned long long)CACHE_SUPERBLOCK_MAGIC);
  172. return -EILSEQ;
  173. }
  174. csum_le = cpu_to_le32(dm_bm_checksum(&disk_super->flags,
  175. sb_block_size - sizeof(__le32),
  176. SUPERBLOCK_CSUM_XOR));
  177. if (csum_le != disk_super->csum) {
  178. DMERR("sb_check failed: csum %u: wanted %u",
  179. le32_to_cpu(csum_le), le32_to_cpu(disk_super->csum));
  180. return -EILSEQ;
  181. }
  182. return check_metadata_version(disk_super);
  183. }
  184. static struct dm_block_validator sb_validator = {
  185. .name = "superblock",
  186. .prepare_for_write = sb_prepare_for_write,
  187. .check = sb_check
  188. };
  189. /*----------------------------------------------------------------*/
  190. static int superblock_read_lock(struct dm_cache_metadata *cmd,
  191. struct dm_block **sblock)
  192. {
  193. return dm_bm_read_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
  194. &sb_validator, sblock);
  195. }
  196. static int superblock_lock_zero(struct dm_cache_metadata *cmd,
  197. struct dm_block **sblock)
  198. {
  199. return dm_bm_write_lock_zero(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
  200. &sb_validator, sblock);
  201. }
  202. static int superblock_lock(struct dm_cache_metadata *cmd,
  203. struct dm_block **sblock)
  204. {
  205. return dm_bm_write_lock(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
  206. &sb_validator, sblock);
  207. }
  208. /*----------------------------------------------------------------*/
  209. static int __superblock_all_zeroes(struct dm_block_manager *bm, bool *result)
  210. {
  211. int r;
  212. unsigned i;
  213. struct dm_block *b;
  214. __le64 *data_le, zero = cpu_to_le64(0);
  215. unsigned sb_block_size = dm_bm_block_size(bm) / sizeof(__le64);
  216. /*
  217. * We can't use a validator here - it may be all zeroes.
  218. */
  219. r = dm_bm_read_lock(bm, CACHE_SUPERBLOCK_LOCATION, NULL, &b);
  220. if (r)
  221. return r;
  222. data_le = dm_block_data(b);
  223. *result = true;
  224. for (i = 0; i < sb_block_size; i++) {
  225. if (data_le[i] != zero) {
  226. *result = false;
  227. break;
  228. }
  229. }
  230. dm_bm_unlock(b);
  231. return 0;
  232. }
  233. static void __setup_mapping_info(struct dm_cache_metadata *cmd)
  234. {
  235. struct dm_btree_value_type vt;
  236. vt.context = NULL;
  237. vt.size = sizeof(__le64);
  238. vt.inc = NULL;
  239. vt.dec = NULL;
  240. vt.equal = NULL;
  241. dm_array_info_init(&cmd->info, cmd->tm, &vt);
  242. if (cmd->policy_hint_size) {
  243. vt.size = sizeof(__le32);
  244. dm_array_info_init(&cmd->hint_info, cmd->tm, &vt);
  245. }
  246. }
  247. static int __save_sm_root(struct dm_cache_metadata *cmd)
  248. {
  249. int r;
  250. size_t metadata_len;
  251. r = dm_sm_root_size(cmd->metadata_sm, &metadata_len);
  252. if (r < 0)
  253. return r;
  254. return dm_sm_copy_root(cmd->metadata_sm, &cmd->metadata_space_map_root,
  255. metadata_len);
  256. }
  257. static void __copy_sm_root(struct dm_cache_metadata *cmd,
  258. struct cache_disk_superblock *disk_super)
  259. {
  260. memcpy(&disk_super->metadata_space_map_root,
  261. &cmd->metadata_space_map_root,
  262. sizeof(cmd->metadata_space_map_root));
  263. }
  264. static bool separate_dirty_bits(struct dm_cache_metadata *cmd)
  265. {
  266. return cmd->version >= 2;
  267. }
  268. static int __write_initial_superblock(struct dm_cache_metadata *cmd)
  269. {
  270. int r;
  271. struct dm_block *sblock;
  272. struct cache_disk_superblock *disk_super;
  273. sector_t bdev_size = i_size_read(cmd->bdev->bd_inode) >> SECTOR_SHIFT;
  274. /* FIXME: see if we can lose the max sectors limit */
  275. if (bdev_size > DM_CACHE_METADATA_MAX_SECTORS)
  276. bdev_size = DM_CACHE_METADATA_MAX_SECTORS;
  277. r = dm_tm_pre_commit(cmd->tm);
  278. if (r < 0)
  279. return r;
  280. /*
  281. * dm_sm_copy_root() can fail. So we need to do it before we start
  282. * updating the superblock.
  283. */
  284. r = __save_sm_root(cmd);
  285. if (r)
  286. return r;
  287. r = superblock_lock_zero(cmd, &sblock);
  288. if (r)
  289. return r;
  290. disk_super = dm_block_data(sblock);
  291. disk_super->flags = 0;
  292. memset(disk_super->uuid, 0, sizeof(disk_super->uuid));
  293. disk_super->magic = cpu_to_le64(CACHE_SUPERBLOCK_MAGIC);
  294. disk_super->version = cpu_to_le32(cmd->version);
  295. memset(disk_super->policy_name, 0, sizeof(disk_super->policy_name));
  296. memset(disk_super->policy_version, 0, sizeof(disk_super->policy_version));
  297. disk_super->policy_hint_size = cpu_to_le32(0);
  298. __copy_sm_root(cmd, disk_super);
  299. disk_super->mapping_root = cpu_to_le64(cmd->root);
  300. disk_super->hint_root = cpu_to_le64(cmd->hint_root);
  301. disk_super->discard_root = cpu_to_le64(cmd->discard_root);
  302. disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
  303. disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
  304. disk_super->metadata_block_size = cpu_to_le32(DM_CACHE_METADATA_BLOCK_SIZE);
  305. disk_super->data_block_size = cpu_to_le32(cmd->data_block_size);
  306. disk_super->cache_blocks = cpu_to_le32(0);
  307. disk_super->read_hits = cpu_to_le32(0);
  308. disk_super->read_misses = cpu_to_le32(0);
  309. disk_super->write_hits = cpu_to_le32(0);
  310. disk_super->write_misses = cpu_to_le32(0);
  311. if (separate_dirty_bits(cmd))
  312. disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
  313. return dm_tm_commit(cmd->tm, sblock);
  314. }
  315. static int __format_metadata(struct dm_cache_metadata *cmd)
  316. {
  317. int r;
  318. r = dm_tm_create_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
  319. &cmd->tm, &cmd->metadata_sm);
  320. if (r < 0) {
  321. DMERR("tm_create_with_sm failed");
  322. return r;
  323. }
  324. __setup_mapping_info(cmd);
  325. r = dm_array_empty(&cmd->info, &cmd->root);
  326. if (r < 0)
  327. goto bad;
  328. if (separate_dirty_bits(cmd)) {
  329. dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
  330. r = dm_bitset_empty(&cmd->dirty_info, &cmd->dirty_root);
  331. if (r < 0)
  332. goto bad;
  333. }
  334. dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
  335. r = dm_bitset_empty(&cmd->discard_info, &cmd->discard_root);
  336. if (r < 0)
  337. goto bad;
  338. cmd->discard_block_size = 0;
  339. cmd->discard_nr_blocks = 0;
  340. r = __write_initial_superblock(cmd);
  341. if (r)
  342. goto bad;
  343. cmd->clean_when_opened = true;
  344. return 0;
  345. bad:
  346. dm_tm_destroy(cmd->tm);
  347. dm_sm_destroy(cmd->metadata_sm);
  348. return r;
  349. }
  350. static int __check_incompat_features(struct cache_disk_superblock *disk_super,
  351. struct dm_cache_metadata *cmd)
  352. {
  353. uint32_t incompat_flags, features;
  354. incompat_flags = le32_to_cpu(disk_super->incompat_flags);
  355. features = incompat_flags & ~DM_CACHE_FEATURE_INCOMPAT_SUPP;
  356. if (features) {
  357. DMERR("could not access metadata due to unsupported optional features (%lx).",
  358. (unsigned long)features);
  359. return -EINVAL;
  360. }
  361. /*
  362. * Check for read-only metadata to skip the following RDWR checks.
  363. */
  364. if (get_disk_ro(cmd->bdev->bd_disk))
  365. return 0;
  366. features = le32_to_cpu(disk_super->compat_ro_flags) & ~DM_CACHE_FEATURE_COMPAT_RO_SUPP;
  367. if (features) {
  368. DMERR("could not access metadata RDWR due to unsupported optional features (%lx).",
  369. (unsigned long)features);
  370. return -EINVAL;
  371. }
  372. return 0;
  373. }
  374. static int __open_metadata(struct dm_cache_metadata *cmd)
  375. {
  376. int r;
  377. struct dm_block *sblock;
  378. struct cache_disk_superblock *disk_super;
  379. unsigned long sb_flags;
  380. r = superblock_read_lock(cmd, &sblock);
  381. if (r < 0) {
  382. DMERR("couldn't read lock superblock");
  383. return r;
  384. }
  385. disk_super = dm_block_data(sblock);
  386. /* Verify the data block size hasn't changed */
  387. if (le32_to_cpu(disk_super->data_block_size) != cmd->data_block_size) {
  388. DMERR("changing the data block size (from %u to %llu) is not supported",
  389. le32_to_cpu(disk_super->data_block_size),
  390. (unsigned long long)cmd->data_block_size);
  391. r = -EINVAL;
  392. goto bad;
  393. }
  394. r = __check_incompat_features(disk_super, cmd);
  395. if (r < 0)
  396. goto bad;
  397. r = dm_tm_open_with_sm(cmd->bm, CACHE_SUPERBLOCK_LOCATION,
  398. disk_super->metadata_space_map_root,
  399. sizeof(disk_super->metadata_space_map_root),
  400. &cmd->tm, &cmd->metadata_sm);
  401. if (r < 0) {
  402. DMERR("tm_open_with_sm failed");
  403. goto bad;
  404. }
  405. __setup_mapping_info(cmd);
  406. dm_disk_bitset_init(cmd->tm, &cmd->dirty_info);
  407. dm_disk_bitset_init(cmd->tm, &cmd->discard_info);
  408. sb_flags = le32_to_cpu(disk_super->flags);
  409. cmd->clean_when_opened = test_bit(CLEAN_SHUTDOWN, &sb_flags);
  410. dm_bm_unlock(sblock);
  411. return 0;
  412. bad:
  413. dm_bm_unlock(sblock);
  414. return r;
  415. }
  416. static int __open_or_format_metadata(struct dm_cache_metadata *cmd,
  417. bool format_device)
  418. {
  419. int r;
  420. bool unformatted = false;
  421. r = __superblock_all_zeroes(cmd->bm, &unformatted);
  422. if (r)
  423. return r;
  424. if (unformatted)
  425. return format_device ? __format_metadata(cmd) : -EPERM;
  426. return __open_metadata(cmd);
  427. }
  428. static int __create_persistent_data_objects(struct dm_cache_metadata *cmd,
  429. bool may_format_device)
  430. {
  431. int r;
  432. cmd->bm = dm_block_manager_create(cmd->bdev, DM_CACHE_METADATA_BLOCK_SIZE << SECTOR_SHIFT,
  433. CACHE_MAX_CONCURRENT_LOCKS);
  434. if (IS_ERR(cmd->bm)) {
  435. DMERR("could not create block manager");
  436. r = PTR_ERR(cmd->bm);
  437. cmd->bm = NULL;
  438. return r;
  439. }
  440. r = __open_or_format_metadata(cmd, may_format_device);
  441. if (r) {
  442. dm_block_manager_destroy(cmd->bm);
  443. cmd->bm = NULL;
  444. }
  445. return r;
  446. }
  447. static void __destroy_persistent_data_objects(struct dm_cache_metadata *cmd)
  448. {
  449. dm_sm_destroy(cmd->metadata_sm);
  450. dm_tm_destroy(cmd->tm);
  451. dm_block_manager_destroy(cmd->bm);
  452. }
  453. typedef unsigned long (*flags_mutator)(unsigned long);
  454. static void update_flags(struct cache_disk_superblock *disk_super,
  455. flags_mutator mutator)
  456. {
  457. uint32_t sb_flags = mutator(le32_to_cpu(disk_super->flags));
  458. disk_super->flags = cpu_to_le32(sb_flags);
  459. }
  460. static unsigned long set_clean_shutdown(unsigned long flags)
  461. {
  462. set_bit(CLEAN_SHUTDOWN, &flags);
  463. return flags;
  464. }
  465. static unsigned long clear_clean_shutdown(unsigned long flags)
  466. {
  467. clear_bit(CLEAN_SHUTDOWN, &flags);
  468. return flags;
  469. }
  470. static void read_superblock_fields(struct dm_cache_metadata *cmd,
  471. struct cache_disk_superblock *disk_super)
  472. {
  473. cmd->version = le32_to_cpu(disk_super->version);
  474. cmd->flags = le32_to_cpu(disk_super->flags);
  475. cmd->root = le64_to_cpu(disk_super->mapping_root);
  476. cmd->hint_root = le64_to_cpu(disk_super->hint_root);
  477. cmd->discard_root = le64_to_cpu(disk_super->discard_root);
  478. cmd->discard_block_size = le64_to_cpu(disk_super->discard_block_size);
  479. cmd->discard_nr_blocks = to_dblock(le64_to_cpu(disk_super->discard_nr_blocks));
  480. cmd->data_block_size = le32_to_cpu(disk_super->data_block_size);
  481. cmd->cache_blocks = to_cblock(le32_to_cpu(disk_super->cache_blocks));
  482. strncpy(cmd->policy_name, disk_super->policy_name, sizeof(cmd->policy_name));
  483. cmd->policy_version[0] = le32_to_cpu(disk_super->policy_version[0]);
  484. cmd->policy_version[1] = le32_to_cpu(disk_super->policy_version[1]);
  485. cmd->policy_version[2] = le32_to_cpu(disk_super->policy_version[2]);
  486. cmd->policy_hint_size = le32_to_cpu(disk_super->policy_hint_size);
  487. cmd->stats.read_hits = le32_to_cpu(disk_super->read_hits);
  488. cmd->stats.read_misses = le32_to_cpu(disk_super->read_misses);
  489. cmd->stats.write_hits = le32_to_cpu(disk_super->write_hits);
  490. cmd->stats.write_misses = le32_to_cpu(disk_super->write_misses);
  491. if (separate_dirty_bits(cmd))
  492. cmd->dirty_root = le64_to_cpu(disk_super->dirty_root);
  493. cmd->changed = false;
  494. }
  495. /*
  496. * The mutator updates the superblock flags.
  497. */
  498. static int __begin_transaction_flags(struct dm_cache_metadata *cmd,
  499. flags_mutator mutator)
  500. {
  501. int r;
  502. struct cache_disk_superblock *disk_super;
  503. struct dm_block *sblock;
  504. r = superblock_lock(cmd, &sblock);
  505. if (r)
  506. return r;
  507. disk_super = dm_block_data(sblock);
  508. update_flags(disk_super, mutator);
  509. read_superblock_fields(cmd, disk_super);
  510. dm_bm_unlock(sblock);
  511. return dm_bm_flush(cmd->bm);
  512. }
  513. static int __begin_transaction(struct dm_cache_metadata *cmd)
  514. {
  515. int r;
  516. struct cache_disk_superblock *disk_super;
  517. struct dm_block *sblock;
  518. /*
  519. * We re-read the superblock every time. Shouldn't need to do this
  520. * really.
  521. */
  522. r = superblock_read_lock(cmd, &sblock);
  523. if (r)
  524. return r;
  525. disk_super = dm_block_data(sblock);
  526. read_superblock_fields(cmd, disk_super);
  527. dm_bm_unlock(sblock);
  528. return 0;
  529. }
  530. static int __commit_transaction(struct dm_cache_metadata *cmd,
  531. flags_mutator mutator)
  532. {
  533. int r;
  534. struct cache_disk_superblock *disk_super;
  535. struct dm_block *sblock;
  536. /*
  537. * We need to know if the cache_disk_superblock exceeds a 512-byte sector.
  538. */
  539. BUILD_BUG_ON(sizeof(struct cache_disk_superblock) > 512);
  540. if (separate_dirty_bits(cmd)) {
  541. r = dm_bitset_flush(&cmd->dirty_info, cmd->dirty_root,
  542. &cmd->dirty_root);
  543. if (r)
  544. return r;
  545. }
  546. r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root,
  547. &cmd->discard_root);
  548. if (r)
  549. return r;
  550. r = dm_tm_pre_commit(cmd->tm);
  551. if (r < 0)
  552. return r;
  553. r = __save_sm_root(cmd);
  554. if (r)
  555. return r;
  556. r = superblock_lock(cmd, &sblock);
  557. if (r)
  558. return r;
  559. disk_super = dm_block_data(sblock);
  560. disk_super->flags = cpu_to_le32(cmd->flags);
  561. if (mutator)
  562. update_flags(disk_super, mutator);
  563. disk_super->mapping_root = cpu_to_le64(cmd->root);
  564. if (separate_dirty_bits(cmd))
  565. disk_super->dirty_root = cpu_to_le64(cmd->dirty_root);
  566. disk_super->hint_root = cpu_to_le64(cmd->hint_root);
  567. disk_super->discard_root = cpu_to_le64(cmd->discard_root);
  568. disk_super->discard_block_size = cpu_to_le64(cmd->discard_block_size);
  569. disk_super->discard_nr_blocks = cpu_to_le64(from_dblock(cmd->discard_nr_blocks));
  570. disk_super->cache_blocks = cpu_to_le32(from_cblock(cmd->cache_blocks));
  571. strncpy(disk_super->policy_name, cmd->policy_name, sizeof(disk_super->policy_name));
  572. disk_super->policy_version[0] = cpu_to_le32(cmd->policy_version[0]);
  573. disk_super->policy_version[1] = cpu_to_le32(cmd->policy_version[1]);
  574. disk_super->policy_version[2] = cpu_to_le32(cmd->policy_version[2]);
  575. disk_super->policy_hint_size = cpu_to_le32(cmd->policy_hint_size);
  576. disk_super->read_hits = cpu_to_le32(cmd->stats.read_hits);
  577. disk_super->read_misses = cpu_to_le32(cmd->stats.read_misses);
  578. disk_super->write_hits = cpu_to_le32(cmd->stats.write_hits);
  579. disk_super->write_misses = cpu_to_le32(cmd->stats.write_misses);
  580. __copy_sm_root(cmd, disk_super);
  581. return dm_tm_commit(cmd->tm, sblock);
  582. }
  583. /*----------------------------------------------------------------*/
  584. /*
  585. * The mappings are held in a dm-array that has 64-bit values stored in
  586. * little-endian format. The index is the cblock, the high 48bits of the
  587. * value are the oblock and the low 16 bit the flags.
  588. */
  589. #define FLAGS_MASK ((1 << 16) - 1)
  590. static __le64 pack_value(dm_oblock_t block, unsigned flags)
  591. {
  592. uint64_t value = from_oblock(block);
  593. value <<= 16;
  594. value = value | (flags & FLAGS_MASK);
  595. return cpu_to_le64(value);
  596. }
  597. static void unpack_value(__le64 value_le, dm_oblock_t *block, unsigned *flags)
  598. {
  599. uint64_t value = le64_to_cpu(value_le);
  600. uint64_t b = value >> 16;
  601. *block = to_oblock(b);
  602. *flags = value & FLAGS_MASK;
  603. }
  604. /*----------------------------------------------------------------*/
  605. static struct dm_cache_metadata *metadata_open(struct block_device *bdev,
  606. sector_t data_block_size,
  607. bool may_format_device,
  608. size_t policy_hint_size,
  609. unsigned metadata_version)
  610. {
  611. int r;
  612. struct dm_cache_metadata *cmd;
  613. cmd = kzalloc(sizeof(*cmd), GFP_KERNEL);
  614. if (!cmd) {
  615. DMERR("could not allocate metadata struct");
  616. return ERR_PTR(-ENOMEM);
  617. }
  618. cmd->version = metadata_version;
  619. refcount_set(&cmd->ref_count, 1);
  620. init_rwsem(&cmd->root_lock);
  621. cmd->bdev = bdev;
  622. cmd->data_block_size = data_block_size;
  623. cmd->cache_blocks = 0;
  624. cmd->policy_hint_size = policy_hint_size;
  625. cmd->changed = true;
  626. cmd->fail_io = false;
  627. r = __create_persistent_data_objects(cmd, may_format_device);
  628. if (r) {
  629. kfree(cmd);
  630. return ERR_PTR(r);
  631. }
  632. r = __begin_transaction_flags(cmd, clear_clean_shutdown);
  633. if (r < 0) {
  634. dm_cache_metadata_close(cmd);
  635. return ERR_PTR(r);
  636. }
  637. return cmd;
  638. }
  639. /*
  640. * We keep a little list of ref counted metadata objects to prevent two
  641. * different target instances creating separate bufio instances. This is
  642. * an issue if a table is reloaded before the suspend.
  643. */
  644. static DEFINE_MUTEX(table_lock);
  645. static LIST_HEAD(table);
  646. static struct dm_cache_metadata *lookup(struct block_device *bdev)
  647. {
  648. struct dm_cache_metadata *cmd;
  649. list_for_each_entry(cmd, &table, list)
  650. if (cmd->bdev == bdev) {
  651. refcount_inc(&cmd->ref_count);
  652. return cmd;
  653. }
  654. return NULL;
  655. }
  656. static struct dm_cache_metadata *lookup_or_open(struct block_device *bdev,
  657. sector_t data_block_size,
  658. bool may_format_device,
  659. size_t policy_hint_size,
  660. unsigned metadata_version)
  661. {
  662. struct dm_cache_metadata *cmd, *cmd2;
  663. mutex_lock(&table_lock);
  664. cmd = lookup(bdev);
  665. mutex_unlock(&table_lock);
  666. if (cmd)
  667. return cmd;
  668. cmd = metadata_open(bdev, data_block_size, may_format_device,
  669. policy_hint_size, metadata_version);
  670. if (!IS_ERR(cmd)) {
  671. mutex_lock(&table_lock);
  672. cmd2 = lookup(bdev);
  673. if (cmd2) {
  674. mutex_unlock(&table_lock);
  675. __destroy_persistent_data_objects(cmd);
  676. kfree(cmd);
  677. return cmd2;
  678. }
  679. list_add(&cmd->list, &table);
  680. mutex_unlock(&table_lock);
  681. }
  682. return cmd;
  683. }
  684. static bool same_params(struct dm_cache_metadata *cmd, sector_t data_block_size)
  685. {
  686. if (cmd->data_block_size != data_block_size) {
  687. DMERR("data_block_size (%llu) different from that in metadata (%llu)",
  688. (unsigned long long) data_block_size,
  689. (unsigned long long) cmd->data_block_size);
  690. return false;
  691. }
  692. return true;
  693. }
  694. struct dm_cache_metadata *dm_cache_metadata_open(struct block_device *bdev,
  695. sector_t data_block_size,
  696. bool may_format_device,
  697. size_t policy_hint_size,
  698. unsigned metadata_version)
  699. {
  700. struct dm_cache_metadata *cmd = lookup_or_open(bdev, data_block_size, may_format_device,
  701. policy_hint_size, metadata_version);
  702. if (!IS_ERR(cmd) && !same_params(cmd, data_block_size)) {
  703. dm_cache_metadata_close(cmd);
  704. return ERR_PTR(-EINVAL);
  705. }
  706. return cmd;
  707. }
  708. void dm_cache_metadata_close(struct dm_cache_metadata *cmd)
  709. {
  710. if (refcount_dec_and_test(&cmd->ref_count)) {
  711. mutex_lock(&table_lock);
  712. list_del(&cmd->list);
  713. mutex_unlock(&table_lock);
  714. if (!cmd->fail_io)
  715. __destroy_persistent_data_objects(cmd);
  716. kfree(cmd);
  717. }
  718. }
  719. /*
  720. * Checks that the given cache block is either unmapped or clean.
  721. */
  722. static int block_clean_combined_dirty(struct dm_cache_metadata *cmd, dm_cblock_t b,
  723. bool *result)
  724. {
  725. int r;
  726. __le64 value;
  727. dm_oblock_t ob;
  728. unsigned flags;
  729. r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(b), &value);
  730. if (r)
  731. return r;
  732. unpack_value(value, &ob, &flags);
  733. *result = !((flags & M_VALID) && (flags & M_DIRTY));
  734. return 0;
  735. }
  736. static int blocks_are_clean_combined_dirty(struct dm_cache_metadata *cmd,
  737. dm_cblock_t begin, dm_cblock_t end,
  738. bool *result)
  739. {
  740. int r;
  741. *result = true;
  742. while (begin != end) {
  743. r = block_clean_combined_dirty(cmd, begin, result);
  744. if (r) {
  745. DMERR("block_clean_combined_dirty failed");
  746. return r;
  747. }
  748. if (!*result) {
  749. DMERR("cache block %llu is dirty",
  750. (unsigned long long) from_cblock(begin));
  751. return 0;
  752. }
  753. begin = to_cblock(from_cblock(begin) + 1);
  754. }
  755. return 0;
  756. }
  757. static int blocks_are_clean_separate_dirty(struct dm_cache_metadata *cmd,
  758. dm_cblock_t begin, dm_cblock_t end,
  759. bool *result)
  760. {
  761. int r;
  762. bool dirty_flag;
  763. *result = true;
  764. if (from_cblock(cmd->cache_blocks) == 0)
  765. /* Nothing to do */
  766. return 0;
  767. r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
  768. from_cblock(cmd->cache_blocks), &cmd->dirty_cursor);
  769. if (r) {
  770. DMERR("%s: dm_bitset_cursor_begin for dirty failed", __func__);
  771. return r;
  772. }
  773. r = dm_bitset_cursor_skip(&cmd->dirty_cursor, from_cblock(begin));
  774. if (r) {
  775. DMERR("%s: dm_bitset_cursor_skip for dirty failed", __func__);
  776. dm_bitset_cursor_end(&cmd->dirty_cursor);
  777. return r;
  778. }
  779. while (begin != end) {
  780. /*
  781. * We assume that unmapped blocks have their dirty bit
  782. * cleared.
  783. */
  784. dirty_flag = dm_bitset_cursor_get_value(&cmd->dirty_cursor);
  785. if (dirty_flag) {
  786. DMERR("%s: cache block %llu is dirty", __func__,
  787. (unsigned long long) from_cblock(begin));
  788. dm_bitset_cursor_end(&cmd->dirty_cursor);
  789. *result = false;
  790. return 0;
  791. }
  792. begin = to_cblock(from_cblock(begin) + 1);
  793. if (begin == end)
  794. break;
  795. r = dm_bitset_cursor_next(&cmd->dirty_cursor);
  796. if (r) {
  797. DMERR("%s: dm_bitset_cursor_next for dirty failed", __func__);
  798. dm_bitset_cursor_end(&cmd->dirty_cursor);
  799. return r;
  800. }
  801. }
  802. dm_bitset_cursor_end(&cmd->dirty_cursor);
  803. return 0;
  804. }
  805. static int blocks_are_unmapped_or_clean(struct dm_cache_metadata *cmd,
  806. dm_cblock_t begin, dm_cblock_t end,
  807. bool *result)
  808. {
  809. if (separate_dirty_bits(cmd))
  810. return blocks_are_clean_separate_dirty(cmd, begin, end, result);
  811. else
  812. return blocks_are_clean_combined_dirty(cmd, begin, end, result);
  813. }
  814. static bool cmd_write_lock(struct dm_cache_metadata *cmd)
  815. {
  816. down_write(&cmd->root_lock);
  817. if (cmd->fail_io || dm_bm_is_read_only(cmd->bm)) {
  818. up_write(&cmd->root_lock);
  819. return false;
  820. }
  821. return true;
  822. }
  823. #define WRITE_LOCK(cmd) \
  824. do { \
  825. if (!cmd_write_lock((cmd))) \
  826. return -EINVAL; \
  827. } while(0)
  828. #define WRITE_LOCK_VOID(cmd) \
  829. do { \
  830. if (!cmd_write_lock((cmd))) \
  831. return; \
  832. } while(0)
  833. #define WRITE_UNLOCK(cmd) \
  834. up_write(&(cmd)->root_lock)
  835. static bool cmd_read_lock(struct dm_cache_metadata *cmd)
  836. {
  837. down_read(&cmd->root_lock);
  838. if (cmd->fail_io) {
  839. up_read(&cmd->root_lock);
  840. return false;
  841. }
  842. return true;
  843. }
  844. #define READ_LOCK(cmd) \
  845. do { \
  846. if (!cmd_read_lock((cmd))) \
  847. return -EINVAL; \
  848. } while(0)
  849. #define READ_LOCK_VOID(cmd) \
  850. do { \
  851. if (!cmd_read_lock((cmd))) \
  852. return; \
  853. } while(0)
  854. #define READ_UNLOCK(cmd) \
  855. up_read(&(cmd)->root_lock)
  856. int dm_cache_resize(struct dm_cache_metadata *cmd, dm_cblock_t new_cache_size)
  857. {
  858. int r;
  859. bool clean;
  860. __le64 null_mapping = pack_value(0, 0);
  861. WRITE_LOCK(cmd);
  862. __dm_bless_for_disk(&null_mapping);
  863. if (from_cblock(new_cache_size) < from_cblock(cmd->cache_blocks)) {
  864. r = blocks_are_unmapped_or_clean(cmd, new_cache_size, cmd->cache_blocks, &clean);
  865. if (r) {
  866. __dm_unbless_for_disk(&null_mapping);
  867. goto out;
  868. }
  869. if (!clean) {
  870. DMERR("unable to shrink cache due to dirty blocks");
  871. r = -EINVAL;
  872. __dm_unbless_for_disk(&null_mapping);
  873. goto out;
  874. }
  875. }
  876. r = dm_array_resize(&cmd->info, cmd->root, from_cblock(cmd->cache_blocks),
  877. from_cblock(new_cache_size),
  878. &null_mapping, &cmd->root);
  879. if (r)
  880. goto out;
  881. if (separate_dirty_bits(cmd)) {
  882. r = dm_bitset_resize(&cmd->dirty_info, cmd->dirty_root,
  883. from_cblock(cmd->cache_blocks), from_cblock(new_cache_size),
  884. false, &cmd->dirty_root);
  885. if (r)
  886. goto out;
  887. }
  888. cmd->cache_blocks = new_cache_size;
  889. cmd->changed = true;
  890. out:
  891. WRITE_UNLOCK(cmd);
  892. return r;
  893. }
  894. int dm_cache_discard_bitset_resize(struct dm_cache_metadata *cmd,
  895. sector_t discard_block_size,
  896. dm_dblock_t new_nr_entries)
  897. {
  898. int r;
  899. WRITE_LOCK(cmd);
  900. r = dm_bitset_resize(&cmd->discard_info,
  901. cmd->discard_root,
  902. from_dblock(cmd->discard_nr_blocks),
  903. from_dblock(new_nr_entries),
  904. false, &cmd->discard_root);
  905. if (!r) {
  906. cmd->discard_block_size = discard_block_size;
  907. cmd->discard_nr_blocks = new_nr_entries;
  908. }
  909. cmd->changed = true;
  910. WRITE_UNLOCK(cmd);
  911. return r;
  912. }
  913. static int __set_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
  914. {
  915. return dm_bitset_set_bit(&cmd->discard_info, cmd->discard_root,
  916. from_dblock(b), &cmd->discard_root);
  917. }
  918. static int __clear_discard(struct dm_cache_metadata *cmd, dm_dblock_t b)
  919. {
  920. return dm_bitset_clear_bit(&cmd->discard_info, cmd->discard_root,
  921. from_dblock(b), &cmd->discard_root);
  922. }
  923. static int __discard(struct dm_cache_metadata *cmd,
  924. dm_dblock_t dblock, bool discard)
  925. {
  926. int r;
  927. r = (discard ? __set_discard : __clear_discard)(cmd, dblock);
  928. if (r)
  929. return r;
  930. cmd->changed = true;
  931. return 0;
  932. }
  933. int dm_cache_set_discard(struct dm_cache_metadata *cmd,
  934. dm_dblock_t dblock, bool discard)
  935. {
  936. int r;
  937. WRITE_LOCK(cmd);
  938. r = __discard(cmd, dblock, discard);
  939. WRITE_UNLOCK(cmd);
  940. return r;
  941. }
  942. static int __load_discards(struct dm_cache_metadata *cmd,
  943. load_discard_fn fn, void *context)
  944. {
  945. int r = 0;
  946. uint32_t b;
  947. struct dm_bitset_cursor c;
  948. if (from_dblock(cmd->discard_nr_blocks) == 0)
  949. /* nothing to do */
  950. return 0;
  951. if (cmd->clean_when_opened) {
  952. r = dm_bitset_flush(&cmd->discard_info, cmd->discard_root, &cmd->discard_root);
  953. if (r)
  954. return r;
  955. r = dm_bitset_cursor_begin(&cmd->discard_info, cmd->discard_root,
  956. from_dblock(cmd->discard_nr_blocks), &c);
  957. if (r)
  958. return r;
  959. for (b = 0; ; b++) {
  960. r = fn(context, cmd->discard_block_size, to_dblock(b),
  961. dm_bitset_cursor_get_value(&c));
  962. if (r)
  963. break;
  964. if (b >= (from_dblock(cmd->discard_nr_blocks) - 1))
  965. break;
  966. r = dm_bitset_cursor_next(&c);
  967. if (r)
  968. break;
  969. }
  970. dm_bitset_cursor_end(&c);
  971. } else {
  972. for (b = 0; b < from_dblock(cmd->discard_nr_blocks); b++) {
  973. r = fn(context, cmd->discard_block_size, to_dblock(b), false);
  974. if (r)
  975. return r;
  976. }
  977. }
  978. return r;
  979. }
  980. int dm_cache_load_discards(struct dm_cache_metadata *cmd,
  981. load_discard_fn fn, void *context)
  982. {
  983. int r;
  984. READ_LOCK(cmd);
  985. r = __load_discards(cmd, fn, context);
  986. READ_UNLOCK(cmd);
  987. return r;
  988. }
  989. int dm_cache_size(struct dm_cache_metadata *cmd, dm_cblock_t *result)
  990. {
  991. READ_LOCK(cmd);
  992. *result = cmd->cache_blocks;
  993. READ_UNLOCK(cmd);
  994. return 0;
  995. }
  996. static int __remove(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
  997. {
  998. int r;
  999. __le64 value = pack_value(0, 0);
  1000. __dm_bless_for_disk(&value);
  1001. r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
  1002. &value, &cmd->root);
  1003. if (r)
  1004. return r;
  1005. cmd->changed = true;
  1006. return 0;
  1007. }
  1008. int dm_cache_remove_mapping(struct dm_cache_metadata *cmd, dm_cblock_t cblock)
  1009. {
  1010. int r;
  1011. WRITE_LOCK(cmd);
  1012. r = __remove(cmd, cblock);
  1013. WRITE_UNLOCK(cmd);
  1014. return r;
  1015. }
  1016. static int __insert(struct dm_cache_metadata *cmd,
  1017. dm_cblock_t cblock, dm_oblock_t oblock)
  1018. {
  1019. int r;
  1020. __le64 value = pack_value(oblock, M_VALID);
  1021. __dm_bless_for_disk(&value);
  1022. r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
  1023. &value, &cmd->root);
  1024. if (r)
  1025. return r;
  1026. cmd->changed = true;
  1027. return 0;
  1028. }
  1029. int dm_cache_insert_mapping(struct dm_cache_metadata *cmd,
  1030. dm_cblock_t cblock, dm_oblock_t oblock)
  1031. {
  1032. int r;
  1033. WRITE_LOCK(cmd);
  1034. r = __insert(cmd, cblock, oblock);
  1035. WRITE_UNLOCK(cmd);
  1036. return r;
  1037. }
  1038. struct thunk {
  1039. load_mapping_fn fn;
  1040. void *context;
  1041. struct dm_cache_metadata *cmd;
  1042. bool respect_dirty_flags;
  1043. bool hints_valid;
  1044. };
  1045. static bool policy_unchanged(struct dm_cache_metadata *cmd,
  1046. struct dm_cache_policy *policy)
  1047. {
  1048. const char *policy_name = dm_cache_policy_get_name(policy);
  1049. const unsigned *policy_version = dm_cache_policy_get_version(policy);
  1050. size_t policy_hint_size = dm_cache_policy_get_hint_size(policy);
  1051. /*
  1052. * Ensure policy names match.
  1053. */
  1054. if (strncmp(cmd->policy_name, policy_name, sizeof(cmd->policy_name)))
  1055. return false;
  1056. /*
  1057. * Ensure policy major versions match.
  1058. */
  1059. if (cmd->policy_version[0] != policy_version[0])
  1060. return false;
  1061. /*
  1062. * Ensure policy hint sizes match.
  1063. */
  1064. if (cmd->policy_hint_size != policy_hint_size)
  1065. return false;
  1066. return true;
  1067. }
  1068. static bool hints_array_initialized(struct dm_cache_metadata *cmd)
  1069. {
  1070. return cmd->hint_root && cmd->policy_hint_size;
  1071. }
  1072. static bool hints_array_available(struct dm_cache_metadata *cmd,
  1073. struct dm_cache_policy *policy)
  1074. {
  1075. return cmd->clean_when_opened && policy_unchanged(cmd, policy) &&
  1076. hints_array_initialized(cmd);
  1077. }
  1078. static int __load_mapping_v1(struct dm_cache_metadata *cmd,
  1079. uint64_t cb, bool hints_valid,
  1080. struct dm_array_cursor *mapping_cursor,
  1081. struct dm_array_cursor *hint_cursor,
  1082. load_mapping_fn fn, void *context)
  1083. {
  1084. int r = 0;
  1085. __le64 mapping;
  1086. __le32 hint = 0;
  1087. __le64 *mapping_value_le;
  1088. __le32 *hint_value_le;
  1089. dm_oblock_t oblock;
  1090. unsigned flags;
  1091. bool dirty = true;
  1092. dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
  1093. memcpy(&mapping, mapping_value_le, sizeof(mapping));
  1094. unpack_value(mapping, &oblock, &flags);
  1095. if (flags & M_VALID) {
  1096. if (hints_valid) {
  1097. dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
  1098. memcpy(&hint, hint_value_le, sizeof(hint));
  1099. }
  1100. if (cmd->clean_when_opened)
  1101. dirty = flags & M_DIRTY;
  1102. r = fn(context, oblock, to_cblock(cb), dirty,
  1103. le32_to_cpu(hint), hints_valid);
  1104. if (r) {
  1105. DMERR("policy couldn't load cache block %llu",
  1106. (unsigned long long) from_cblock(to_cblock(cb)));
  1107. }
  1108. }
  1109. return r;
  1110. }
  1111. static int __load_mapping_v2(struct dm_cache_metadata *cmd,
  1112. uint64_t cb, bool hints_valid,
  1113. struct dm_array_cursor *mapping_cursor,
  1114. struct dm_array_cursor *hint_cursor,
  1115. struct dm_bitset_cursor *dirty_cursor,
  1116. load_mapping_fn fn, void *context)
  1117. {
  1118. int r = 0;
  1119. __le64 mapping;
  1120. __le32 hint = 0;
  1121. __le64 *mapping_value_le;
  1122. __le32 *hint_value_le;
  1123. dm_oblock_t oblock;
  1124. unsigned flags;
  1125. bool dirty = true;
  1126. dm_array_cursor_get_value(mapping_cursor, (void **) &mapping_value_le);
  1127. memcpy(&mapping, mapping_value_le, sizeof(mapping));
  1128. unpack_value(mapping, &oblock, &flags);
  1129. if (flags & M_VALID) {
  1130. if (hints_valid) {
  1131. dm_array_cursor_get_value(hint_cursor, (void **) &hint_value_le);
  1132. memcpy(&hint, hint_value_le, sizeof(hint));
  1133. }
  1134. if (cmd->clean_when_opened)
  1135. dirty = dm_bitset_cursor_get_value(dirty_cursor);
  1136. r = fn(context, oblock, to_cblock(cb), dirty,
  1137. le32_to_cpu(hint), hints_valid);
  1138. if (r) {
  1139. DMERR("policy couldn't load cache block %llu",
  1140. (unsigned long long) from_cblock(to_cblock(cb)));
  1141. }
  1142. }
  1143. return r;
  1144. }
  1145. static int __load_mappings(struct dm_cache_metadata *cmd,
  1146. struct dm_cache_policy *policy,
  1147. load_mapping_fn fn, void *context)
  1148. {
  1149. int r;
  1150. uint64_t cb;
  1151. bool hints_valid = hints_array_available(cmd, policy);
  1152. if (from_cblock(cmd->cache_blocks) == 0)
  1153. /* Nothing to do */
  1154. return 0;
  1155. r = dm_array_cursor_begin(&cmd->info, cmd->root, &cmd->mapping_cursor);
  1156. if (r)
  1157. return r;
  1158. if (hints_valid) {
  1159. r = dm_array_cursor_begin(&cmd->hint_info, cmd->hint_root, &cmd->hint_cursor);
  1160. if (r) {
  1161. dm_array_cursor_end(&cmd->mapping_cursor);
  1162. return r;
  1163. }
  1164. }
  1165. if (separate_dirty_bits(cmd)) {
  1166. r = dm_bitset_cursor_begin(&cmd->dirty_info, cmd->dirty_root,
  1167. from_cblock(cmd->cache_blocks),
  1168. &cmd->dirty_cursor);
  1169. if (r) {
  1170. dm_array_cursor_end(&cmd->hint_cursor);
  1171. dm_array_cursor_end(&cmd->mapping_cursor);
  1172. return r;
  1173. }
  1174. }
  1175. for (cb = 0; ; cb++) {
  1176. if (separate_dirty_bits(cmd))
  1177. r = __load_mapping_v2(cmd, cb, hints_valid,
  1178. &cmd->mapping_cursor,
  1179. &cmd->hint_cursor,
  1180. &cmd->dirty_cursor,
  1181. fn, context);
  1182. else
  1183. r = __load_mapping_v1(cmd, cb, hints_valid,
  1184. &cmd->mapping_cursor, &cmd->hint_cursor,
  1185. fn, context);
  1186. if (r)
  1187. goto out;
  1188. /*
  1189. * We need to break out before we move the cursors.
  1190. */
  1191. if (cb >= (from_cblock(cmd->cache_blocks) - 1))
  1192. break;
  1193. r = dm_array_cursor_next(&cmd->mapping_cursor);
  1194. if (r) {
  1195. DMERR("dm_array_cursor_next for mapping failed");
  1196. goto out;
  1197. }
  1198. if (hints_valid) {
  1199. r = dm_array_cursor_next(&cmd->hint_cursor);
  1200. if (r) {
  1201. dm_array_cursor_end(&cmd->hint_cursor);
  1202. hints_valid = false;
  1203. }
  1204. }
  1205. if (separate_dirty_bits(cmd)) {
  1206. r = dm_bitset_cursor_next(&cmd->dirty_cursor);
  1207. if (r) {
  1208. DMERR("dm_bitset_cursor_next for dirty failed");
  1209. goto out;
  1210. }
  1211. }
  1212. }
  1213. out:
  1214. dm_array_cursor_end(&cmd->mapping_cursor);
  1215. if (hints_valid)
  1216. dm_array_cursor_end(&cmd->hint_cursor);
  1217. if (separate_dirty_bits(cmd))
  1218. dm_bitset_cursor_end(&cmd->dirty_cursor);
  1219. return r;
  1220. }
  1221. int dm_cache_load_mappings(struct dm_cache_metadata *cmd,
  1222. struct dm_cache_policy *policy,
  1223. load_mapping_fn fn, void *context)
  1224. {
  1225. int r;
  1226. READ_LOCK(cmd);
  1227. r = __load_mappings(cmd, policy, fn, context);
  1228. READ_UNLOCK(cmd);
  1229. return r;
  1230. }
  1231. static int __dump_mapping(void *context, uint64_t cblock, void *leaf)
  1232. {
  1233. int r = 0;
  1234. __le64 value;
  1235. dm_oblock_t oblock;
  1236. unsigned flags;
  1237. memcpy(&value, leaf, sizeof(value));
  1238. unpack_value(value, &oblock, &flags);
  1239. return r;
  1240. }
  1241. static int __dump_mappings(struct dm_cache_metadata *cmd)
  1242. {
  1243. return dm_array_walk(&cmd->info, cmd->root, __dump_mapping, NULL);
  1244. }
  1245. void dm_cache_dump(struct dm_cache_metadata *cmd)
  1246. {
  1247. READ_LOCK_VOID(cmd);
  1248. __dump_mappings(cmd);
  1249. READ_UNLOCK(cmd);
  1250. }
  1251. int dm_cache_changed_this_transaction(struct dm_cache_metadata *cmd)
  1252. {
  1253. int r;
  1254. READ_LOCK(cmd);
  1255. r = cmd->changed;
  1256. READ_UNLOCK(cmd);
  1257. return r;
  1258. }
  1259. static int __dirty(struct dm_cache_metadata *cmd, dm_cblock_t cblock, bool dirty)
  1260. {
  1261. int r;
  1262. unsigned flags;
  1263. dm_oblock_t oblock;
  1264. __le64 value;
  1265. r = dm_array_get_value(&cmd->info, cmd->root, from_cblock(cblock), &value);
  1266. if (r)
  1267. return r;
  1268. unpack_value(value, &oblock, &flags);
  1269. if (((flags & M_DIRTY) && dirty) || (!(flags & M_DIRTY) && !dirty))
  1270. /* nothing to be done */
  1271. return 0;
  1272. value = pack_value(oblock, (flags & ~M_DIRTY) | (dirty ? M_DIRTY : 0));
  1273. __dm_bless_for_disk(&value);
  1274. r = dm_array_set_value(&cmd->info, cmd->root, from_cblock(cblock),
  1275. &value, &cmd->root);
  1276. if (r)
  1277. return r;
  1278. cmd->changed = true;
  1279. return 0;
  1280. }
  1281. static int __set_dirty_bits_v1(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
  1282. {
  1283. int r;
  1284. unsigned i;
  1285. for (i = 0; i < nr_bits; i++) {
  1286. r = __dirty(cmd, to_cblock(i), test_bit(i, bits));
  1287. if (r)
  1288. return r;
  1289. }
  1290. return 0;
  1291. }
  1292. static int is_dirty_callback(uint32_t index, bool *value, void *context)
  1293. {
  1294. unsigned long *bits = context;
  1295. *value = test_bit(index, bits);
  1296. return 0;
  1297. }
  1298. static int __set_dirty_bits_v2(struct dm_cache_metadata *cmd, unsigned nr_bits, unsigned long *bits)
  1299. {
  1300. int r = 0;
  1301. /* nr_bits is really just a sanity check */
  1302. if (nr_bits != from_cblock(cmd->cache_blocks)) {
  1303. DMERR("dirty bitset is wrong size");
  1304. return -EINVAL;
  1305. }
  1306. r = dm_bitset_del(&cmd->dirty_info, cmd->dirty_root);
  1307. if (r)
  1308. return r;
  1309. cmd->changed = true;
  1310. return dm_bitset_new(&cmd->dirty_info, &cmd->dirty_root, nr_bits, is_dirty_callback, bits);
  1311. }
  1312. int dm_cache_set_dirty_bits(struct dm_cache_metadata *cmd,
  1313. unsigned nr_bits,
  1314. unsigned long *bits)
  1315. {
  1316. int r;
  1317. WRITE_LOCK(cmd);
  1318. if (separate_dirty_bits(cmd))
  1319. r = __set_dirty_bits_v2(cmd, nr_bits, bits);
  1320. else
  1321. r = __set_dirty_bits_v1(cmd, nr_bits, bits);
  1322. WRITE_UNLOCK(cmd);
  1323. return r;
  1324. }
  1325. void dm_cache_metadata_get_stats(struct dm_cache_metadata *cmd,
  1326. struct dm_cache_statistics *stats)
  1327. {
  1328. READ_LOCK_VOID(cmd);
  1329. *stats = cmd->stats;
  1330. READ_UNLOCK(cmd);
  1331. }
  1332. void dm_cache_metadata_set_stats(struct dm_cache_metadata *cmd,
  1333. struct dm_cache_statistics *stats)
  1334. {
  1335. WRITE_LOCK_VOID(cmd);
  1336. cmd->stats = *stats;
  1337. WRITE_UNLOCK(cmd);
  1338. }
  1339. int dm_cache_commit(struct dm_cache_metadata *cmd, bool clean_shutdown)
  1340. {
  1341. int r = -EINVAL;
  1342. flags_mutator mutator = (clean_shutdown ? set_clean_shutdown :
  1343. clear_clean_shutdown);
  1344. WRITE_LOCK(cmd);
  1345. if (cmd->fail_io)
  1346. goto out;
  1347. r = __commit_transaction(cmd, mutator);
  1348. if (r)
  1349. goto out;
  1350. r = __begin_transaction(cmd);
  1351. out:
  1352. WRITE_UNLOCK(cmd);
  1353. return r;
  1354. }
  1355. int dm_cache_get_free_metadata_block_count(struct dm_cache_metadata *cmd,
  1356. dm_block_t *result)
  1357. {
  1358. int r = -EINVAL;
  1359. READ_LOCK(cmd);
  1360. if (!cmd->fail_io)
  1361. r = dm_sm_get_nr_free(cmd->metadata_sm, result);
  1362. READ_UNLOCK(cmd);
  1363. return r;
  1364. }
  1365. int dm_cache_get_metadata_dev_size(struct dm_cache_metadata *cmd,
  1366. dm_block_t *result)
  1367. {
  1368. int r = -EINVAL;
  1369. READ_LOCK(cmd);
  1370. if (!cmd->fail_io)
  1371. r = dm_sm_get_nr_blocks(cmd->metadata_sm, result);
  1372. READ_UNLOCK(cmd);
  1373. return r;
  1374. }
  1375. /*----------------------------------------------------------------*/
  1376. static int get_hint(uint32_t index, void *value_le, void *context)
  1377. {
  1378. uint32_t value;
  1379. struct dm_cache_policy *policy = context;
  1380. value = policy_get_hint(policy, to_cblock(index));
  1381. *((__le32 *) value_le) = cpu_to_le32(value);
  1382. return 0;
  1383. }
  1384. /*
  1385. * It's quicker to always delete the hint array, and recreate with
  1386. * dm_array_new().
  1387. */
  1388. static int write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
  1389. {
  1390. int r;
  1391. size_t hint_size;
  1392. const char *policy_name = dm_cache_policy_get_name(policy);
  1393. const unsigned *policy_version = dm_cache_policy_get_version(policy);
  1394. if (!policy_name[0] ||
  1395. (strlen(policy_name) > sizeof(cmd->policy_name) - 1))
  1396. return -EINVAL;
  1397. strncpy(cmd->policy_name, policy_name, sizeof(cmd->policy_name));
  1398. memcpy(cmd->policy_version, policy_version, sizeof(cmd->policy_version));
  1399. hint_size = dm_cache_policy_get_hint_size(policy);
  1400. if (!hint_size)
  1401. return 0; /* short-circuit hints initialization */
  1402. cmd->policy_hint_size = hint_size;
  1403. if (cmd->hint_root) {
  1404. r = dm_array_del(&cmd->hint_info, cmd->hint_root);
  1405. if (r)
  1406. return r;
  1407. }
  1408. return dm_array_new(&cmd->hint_info, &cmd->hint_root,
  1409. from_cblock(cmd->cache_blocks),
  1410. get_hint, policy);
  1411. }
  1412. int dm_cache_write_hints(struct dm_cache_metadata *cmd, struct dm_cache_policy *policy)
  1413. {
  1414. int r;
  1415. WRITE_LOCK(cmd);
  1416. r = write_hints(cmd, policy);
  1417. WRITE_UNLOCK(cmd);
  1418. return r;
  1419. }
  1420. int dm_cache_metadata_all_clean(struct dm_cache_metadata *cmd, bool *result)
  1421. {
  1422. int r;
  1423. READ_LOCK(cmd);
  1424. r = blocks_are_unmapped_or_clean(cmd, 0, cmd->cache_blocks, result);
  1425. READ_UNLOCK(cmd);
  1426. return r;
  1427. }
  1428. void dm_cache_metadata_set_read_only(struct dm_cache_metadata *cmd)
  1429. {
  1430. WRITE_LOCK_VOID(cmd);
  1431. dm_bm_set_read_only(cmd->bm);
  1432. WRITE_UNLOCK(cmd);
  1433. }
  1434. void dm_cache_metadata_set_read_write(struct dm_cache_metadata *cmd)
  1435. {
  1436. WRITE_LOCK_VOID(cmd);
  1437. dm_bm_set_read_write(cmd->bm);
  1438. WRITE_UNLOCK(cmd);
  1439. }
  1440. int dm_cache_metadata_set_needs_check(struct dm_cache_metadata *cmd)
  1441. {
  1442. int r;
  1443. struct dm_block *sblock;
  1444. struct cache_disk_superblock *disk_super;
  1445. WRITE_LOCK(cmd);
  1446. set_bit(NEEDS_CHECK, &cmd->flags);
  1447. r = superblock_lock(cmd, &sblock);
  1448. if (r) {
  1449. DMERR("couldn't read superblock");
  1450. goto out;
  1451. }
  1452. disk_super = dm_block_data(sblock);
  1453. disk_super->flags = cpu_to_le32(cmd->flags);
  1454. dm_bm_unlock(sblock);
  1455. out:
  1456. WRITE_UNLOCK(cmd);
  1457. return r;
  1458. }
  1459. int dm_cache_metadata_needs_check(struct dm_cache_metadata *cmd, bool *result)
  1460. {
  1461. READ_LOCK(cmd);
  1462. *result = !!test_bit(NEEDS_CHECK, &cmd->flags);
  1463. READ_UNLOCK(cmd);
  1464. return 0;
  1465. }
  1466. int dm_cache_metadata_abort(struct dm_cache_metadata *cmd)
  1467. {
  1468. int r;
  1469. WRITE_LOCK(cmd);
  1470. __destroy_persistent_data_objects(cmd);
  1471. r = __create_persistent_data_objects(cmd, false);
  1472. if (r)
  1473. cmd->fail_io = true;
  1474. WRITE_UNLOCK(cmd);
  1475. return r;
  1476. }