dm-verity-fec.c 21 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C) 2015 Google, Inc.
  4. *
  5. * Author: Sami Tolvanen <samitolvanen@google.com>
  6. */
  7. #include "dm-verity-fec.h"
  8. #include <linux/math64.h>
  9. #define DM_MSG_PREFIX "verity-fec"
  10. /*
  11. * If error correction has been configured, returns true.
  12. */
  13. bool verity_fec_is_enabled(struct dm_verity *v)
  14. {
  15. return v->fec && v->fec->dev;
  16. }
  17. /*
  18. * Return a pointer to dm_verity_fec_io after dm_verity_io and its variable
  19. * length fields.
  20. */
  21. static inline struct dm_verity_fec_io *fec_io(struct dm_verity_io *io)
  22. {
  23. return (struct dm_verity_fec_io *) verity_io_digest_end(io->v, io);
  24. }
  25. /*
  26. * Return an interleaved offset for a byte in RS block.
  27. */
  28. static inline u64 fec_interleave(struct dm_verity *v, u64 offset)
  29. {
  30. u32 mod;
  31. mod = do_div(offset, v->fec->rsn);
  32. return offset + mod * (v->fec->rounds << v->data_dev_block_bits);
  33. }
  34. /*
  35. * Decode an RS block using Reed-Solomon.
  36. */
  37. static int fec_decode_rs8(struct dm_verity *v, struct dm_verity_fec_io *fio,
  38. u8 *data, u8 *fec, int neras)
  39. {
  40. int i;
  41. uint16_t par[DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN];
  42. for (i = 0; i < v->fec->roots; i++)
  43. par[i] = fec[i];
  44. return decode_rs8(fio->rs, data, par, v->fec->rsn, NULL, neras,
  45. fio->erasures, 0, NULL);
  46. }
  47. /*
  48. * Read error-correcting codes for the requested RS block. Returns a pointer
  49. * to the data block. Caller is responsible for releasing buf.
  50. */
  51. static u8 *fec_read_parity(struct dm_verity *v, u64 rsb, int index,
  52. unsigned *offset, struct dm_buffer **buf)
  53. {
  54. u64 position, block, rem;
  55. u8 *res;
  56. position = (index + rsb) * v->fec->roots;
  57. block = div64_u64_rem(position, v->fec->io_size, &rem);
  58. *offset = (unsigned)rem;
  59. res = dm_bufio_read(v->fec->bufio, block, buf);
  60. if (IS_ERR(res)) {
  61. DMERR("%s: FEC %llu: parity read failed (block %llu): %ld",
  62. v->data_dev->name, (unsigned long long)rsb,
  63. (unsigned long long)block, PTR_ERR(res));
  64. *buf = NULL;
  65. }
  66. return res;
  67. }
  68. /* Loop over each preallocated buffer slot. */
  69. #define fec_for_each_prealloc_buffer(__i) \
  70. for (__i = 0; __i < DM_VERITY_FEC_BUF_PREALLOC; __i++)
  71. /* Loop over each extra buffer slot. */
  72. #define fec_for_each_extra_buffer(io, __i) \
  73. for (__i = DM_VERITY_FEC_BUF_PREALLOC; __i < DM_VERITY_FEC_BUF_MAX; __i++)
  74. /* Loop over each allocated buffer. */
  75. #define fec_for_each_buffer(io, __i) \
  76. for (__i = 0; __i < (io)->nbufs; __i++)
  77. /* Loop over each RS block in each allocated buffer. */
  78. #define fec_for_each_buffer_rs_block(io, __i, __j) \
  79. fec_for_each_buffer(io, __i) \
  80. for (__j = 0; __j < 1 << DM_VERITY_FEC_BUF_RS_BITS; __j++)
  81. /*
  82. * Return a pointer to the current RS block when called inside
  83. * fec_for_each_buffer_rs_block.
  84. */
  85. static inline u8 *fec_buffer_rs_block(struct dm_verity *v,
  86. struct dm_verity_fec_io *fio,
  87. unsigned i, unsigned j)
  88. {
  89. return &fio->bufs[i][j * v->fec->rsn];
  90. }
  91. /*
  92. * Return an index to the current RS block when called inside
  93. * fec_for_each_buffer_rs_block.
  94. */
  95. static inline unsigned fec_buffer_rs_index(unsigned i, unsigned j)
  96. {
  97. return (i << DM_VERITY_FEC_BUF_RS_BITS) + j;
  98. }
  99. /*
  100. * Decode all RS blocks from buffers and copy corrected bytes into fio->output
  101. * starting from block_offset.
  102. */
  103. static int fec_decode_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio,
  104. u64 rsb, int byte_index, unsigned block_offset,
  105. int neras)
  106. {
  107. int r, corrected = 0, res;
  108. struct dm_buffer *buf;
  109. unsigned n, i, offset;
  110. u8 *par, *block;
  111. par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
  112. if (IS_ERR(par))
  113. return PTR_ERR(par);
  114. /*
  115. * Decode the RS blocks we have in bufs. Each RS block results in
  116. * one corrected target byte and consumes fec->roots parity bytes.
  117. */
  118. fec_for_each_buffer_rs_block(fio, n, i) {
  119. block = fec_buffer_rs_block(v, fio, n, i);
  120. res = fec_decode_rs8(v, fio, block, &par[offset], neras);
  121. if (res < 0) {
  122. r = res;
  123. goto error;
  124. }
  125. corrected += res;
  126. fio->output[block_offset] = block[byte_index];
  127. block_offset++;
  128. if (block_offset >= 1 << v->data_dev_block_bits)
  129. goto done;
  130. /* read the next block when we run out of parity bytes */
  131. offset += v->fec->roots;
  132. if (offset >= v->fec->io_size) {
  133. dm_bufio_release(buf);
  134. par = fec_read_parity(v, rsb, block_offset, &offset, &buf);
  135. if (IS_ERR(par))
  136. return PTR_ERR(par);
  137. }
  138. }
  139. done:
  140. r = corrected;
  141. error:
  142. dm_bufio_release(buf);
  143. if (r < 0 && neras)
  144. DMERR_LIMIT("%s: FEC %llu: failed to correct: %d",
  145. v->data_dev->name, (unsigned long long)rsb, r);
  146. else if (r > 0)
  147. DMWARN_LIMIT("%s: FEC %llu: corrected %d errors",
  148. v->data_dev->name, (unsigned long long)rsb, r);
  149. return r;
  150. }
  151. /*
  152. * Locate data block erasures using verity hashes.
  153. */
  154. static int fec_is_erasure(struct dm_verity *v, struct dm_verity_io *io,
  155. u8 *want_digest, u8 *data)
  156. {
  157. if (unlikely(verity_hash(v, verity_io_hash_req(v, io),
  158. data, 1 << v->data_dev_block_bits,
  159. verity_io_real_digest(v, io))))
  160. return 0;
  161. return memcmp(verity_io_real_digest(v, io), want_digest,
  162. v->digest_size) != 0;
  163. }
  164. /*
  165. * Read data blocks that are part of the RS block and deinterleave as much as
  166. * fits into buffers. Check for erasure locations if @neras is non-NULL.
  167. */
  168. static int fec_read_bufs(struct dm_verity *v, struct dm_verity_io *io,
  169. u64 rsb, u64 target, unsigned block_offset,
  170. int *neras)
  171. {
  172. bool is_zero;
  173. int i, j, target_index = -1;
  174. struct dm_buffer *buf;
  175. struct dm_bufio_client *bufio;
  176. struct dm_verity_fec_io *fio = fec_io(io);
  177. u64 block, ileaved;
  178. u8 *bbuf, *rs_block;
  179. u8 want_digest[HASH_MAX_DIGESTSIZE];
  180. unsigned n, k;
  181. if (neras)
  182. *neras = 0;
  183. if (WARN_ON(v->digest_size > sizeof(want_digest)))
  184. return -EINVAL;
  185. /*
  186. * read each of the rsn data blocks that are part of the RS block, and
  187. * interleave contents to available bufs
  188. */
  189. for (i = 0; i < v->fec->rsn; i++) {
  190. ileaved = fec_interleave(v, rsb * v->fec->rsn + i);
  191. /*
  192. * target is the data block we want to correct, target_index is
  193. * the index of this block within the rsn RS blocks
  194. */
  195. if (ileaved == target)
  196. target_index = i;
  197. block = ileaved >> v->data_dev_block_bits;
  198. bufio = v->fec->data_bufio;
  199. if (block >= v->data_blocks) {
  200. block -= v->data_blocks;
  201. /*
  202. * blocks outside the area were assumed to contain
  203. * zeros when encoding data was generated
  204. */
  205. if (unlikely(block >= v->fec->hash_blocks))
  206. continue;
  207. block += v->hash_start;
  208. bufio = v->bufio;
  209. }
  210. bbuf = dm_bufio_read(bufio, block, &buf);
  211. if (IS_ERR(bbuf)) {
  212. DMWARN_LIMIT("%s: FEC %llu: read failed (%llu): %ld",
  213. v->data_dev->name,
  214. (unsigned long long)rsb,
  215. (unsigned long long)block, PTR_ERR(bbuf));
  216. /* assume the block is corrupted */
  217. if (neras && *neras <= v->fec->roots)
  218. fio->erasures[(*neras)++] = i;
  219. continue;
  220. }
  221. /* locate erasures if the block is on the data device */
  222. if (bufio == v->fec->data_bufio &&
  223. verity_hash_for_block(v, io, block, want_digest,
  224. &is_zero) == 0) {
  225. /* skip known zero blocks entirely */
  226. if (is_zero)
  227. goto done;
  228. /*
  229. * skip if we have already found the theoretical
  230. * maximum number (i.e. fec->roots) of erasures
  231. */
  232. if (neras && *neras <= v->fec->roots &&
  233. fec_is_erasure(v, io, want_digest, bbuf))
  234. fio->erasures[(*neras)++] = i;
  235. }
  236. /*
  237. * deinterleave and copy the bytes that fit into bufs,
  238. * starting from block_offset
  239. */
  240. fec_for_each_buffer_rs_block(fio, n, j) {
  241. k = fec_buffer_rs_index(n, j) + block_offset;
  242. if (k >= 1 << v->data_dev_block_bits)
  243. goto done;
  244. rs_block = fec_buffer_rs_block(v, fio, n, j);
  245. rs_block[i] = bbuf[k];
  246. }
  247. done:
  248. dm_bufio_release(buf);
  249. }
  250. return target_index;
  251. }
  252. /*
  253. * Allocate RS control structure and FEC buffers from preallocated mempools,
  254. * and attempt to allocate as many extra buffers as available.
  255. */
  256. static int fec_alloc_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
  257. {
  258. unsigned n;
  259. if (!fio->rs)
  260. fio->rs = mempool_alloc(&v->fec->rs_pool, GFP_NOIO);
  261. fec_for_each_prealloc_buffer(n) {
  262. if (fio->bufs[n])
  263. continue;
  264. fio->bufs[n] = mempool_alloc(&v->fec->prealloc_pool, GFP_NOWAIT);
  265. if (unlikely(!fio->bufs[n])) {
  266. DMERR("failed to allocate FEC buffer");
  267. return -ENOMEM;
  268. }
  269. }
  270. /* try to allocate the maximum number of buffers */
  271. fec_for_each_extra_buffer(fio, n) {
  272. if (fio->bufs[n])
  273. continue;
  274. fio->bufs[n] = mempool_alloc(&v->fec->extra_pool, GFP_NOWAIT);
  275. /* we can manage with even one buffer if necessary */
  276. if (unlikely(!fio->bufs[n]))
  277. break;
  278. }
  279. fio->nbufs = n;
  280. if (!fio->output)
  281. fio->output = mempool_alloc(&v->fec->output_pool, GFP_NOIO);
  282. return 0;
  283. }
  284. /*
  285. * Initialize buffers and clear erasures. fec_read_bufs() assumes buffers are
  286. * zeroed before deinterleaving.
  287. */
  288. static void fec_init_bufs(struct dm_verity *v, struct dm_verity_fec_io *fio)
  289. {
  290. unsigned n;
  291. fec_for_each_buffer(fio, n)
  292. memset(fio->bufs[n], 0, v->fec->rsn << DM_VERITY_FEC_BUF_RS_BITS);
  293. memset(fio->erasures, 0, sizeof(fio->erasures));
  294. }
  295. /*
  296. * Decode all RS blocks in a single data block and return the target block
  297. * (indicated by @offset) in fio->output. If @use_erasures is non-zero, uses
  298. * hashes to locate erasures.
  299. */
  300. static int fec_decode_rsb(struct dm_verity *v, struct dm_verity_io *io,
  301. struct dm_verity_fec_io *fio, u64 rsb, u64 offset,
  302. bool use_erasures)
  303. {
  304. int r, neras = 0;
  305. unsigned pos;
  306. r = fec_alloc_bufs(v, fio);
  307. if (unlikely(r < 0))
  308. return r;
  309. for (pos = 0; pos < 1 << v->data_dev_block_bits; ) {
  310. fec_init_bufs(v, fio);
  311. r = fec_read_bufs(v, io, rsb, offset, pos,
  312. use_erasures ? &neras : NULL);
  313. if (unlikely(r < 0))
  314. return r;
  315. r = fec_decode_bufs(v, fio, rsb, r, pos, neras);
  316. if (r < 0)
  317. return r;
  318. pos += fio->nbufs << DM_VERITY_FEC_BUF_RS_BITS;
  319. }
  320. /* Always re-validate the corrected block against the expected hash */
  321. r = verity_hash(v, verity_io_hash_req(v, io), fio->output,
  322. 1 << v->data_dev_block_bits,
  323. verity_io_real_digest(v, io));
  324. if (unlikely(r < 0))
  325. return r;
  326. if (memcmp(verity_io_real_digest(v, io), verity_io_want_digest(v, io),
  327. v->digest_size)) {
  328. DMERR_LIMIT("%s: FEC %llu: failed to correct (%d erasures)",
  329. v->data_dev->name, (unsigned long long)rsb, neras);
  330. return -EILSEQ;
  331. }
  332. return 0;
  333. }
  334. static int fec_bv_copy(struct dm_verity *v, struct dm_verity_io *io, u8 *data,
  335. size_t len)
  336. {
  337. struct dm_verity_fec_io *fio = fec_io(io);
  338. memcpy(data, &fio->output[fio->output_pos], len);
  339. fio->output_pos += len;
  340. return 0;
  341. }
  342. /*
  343. * Correct errors in a block. Copies corrected block to dest if non-NULL,
  344. * otherwise to a bio_vec starting from iter.
  345. */
  346. int verity_fec_decode(struct dm_verity *v, struct dm_verity_io *io,
  347. enum verity_block_type type, sector_t block, u8 *dest,
  348. struct bvec_iter *iter)
  349. {
  350. int r;
  351. struct dm_verity_fec_io *fio = fec_io(io);
  352. u64 offset, res, rsb;
  353. if (!verity_fec_is_enabled(v))
  354. return -EOPNOTSUPP;
  355. if (fio->level >= DM_VERITY_FEC_MAX_RECURSION) {
  356. DMWARN_LIMIT("%s: FEC: recursion too deep", v->data_dev->name);
  357. return -EIO;
  358. }
  359. fio->level++;
  360. if (type == DM_VERITY_BLOCK_TYPE_METADATA)
  361. block = block - v->hash_start + v->data_blocks;
  362. /*
  363. * For RS(M, N), the continuous FEC data is divided into blocks of N
  364. * bytes. Since block size may not be divisible by N, the last block
  365. * is zero padded when decoding.
  366. *
  367. * Each byte of the block is covered by a different RS(M, N) code,
  368. * and each code is interleaved over N blocks to make it less likely
  369. * that bursty corruption will leave us in unrecoverable state.
  370. */
  371. offset = block << v->data_dev_block_bits;
  372. res = div64_u64(offset, v->fec->rounds << v->data_dev_block_bits);
  373. /*
  374. * The base RS block we can feed to the interleaver to find out all
  375. * blocks required for decoding.
  376. */
  377. rsb = offset - res * (v->fec->rounds << v->data_dev_block_bits);
  378. /*
  379. * Locating erasures is slow, so attempt to recover the block without
  380. * them first. Do a second attempt with erasures if the corruption is
  381. * bad enough.
  382. */
  383. r = fec_decode_rsb(v, io, fio, rsb, offset, false);
  384. if (r < 0) {
  385. r = fec_decode_rsb(v, io, fio, rsb, offset, true);
  386. if (r < 0)
  387. goto done;
  388. }
  389. if (dest)
  390. memcpy(dest, fio->output, 1 << v->data_dev_block_bits);
  391. else if (iter) {
  392. fio->output_pos = 0;
  393. r = verity_for_bv_block(v, io, iter, fec_bv_copy);
  394. }
  395. done:
  396. fio->level--;
  397. return r;
  398. }
  399. /*
  400. * Clean up per-bio data.
  401. */
  402. void verity_fec_finish_io(struct dm_verity_io *io)
  403. {
  404. unsigned n;
  405. struct dm_verity_fec *f = io->v->fec;
  406. struct dm_verity_fec_io *fio = fec_io(io);
  407. if (!verity_fec_is_enabled(io->v))
  408. return;
  409. mempool_free(fio->rs, &f->rs_pool);
  410. fec_for_each_prealloc_buffer(n)
  411. mempool_free(fio->bufs[n], &f->prealloc_pool);
  412. fec_for_each_extra_buffer(fio, n)
  413. mempool_free(fio->bufs[n], &f->extra_pool);
  414. mempool_free(fio->output, &f->output_pool);
  415. }
  416. /*
  417. * Initialize per-bio data.
  418. */
  419. void verity_fec_init_io(struct dm_verity_io *io)
  420. {
  421. struct dm_verity_fec_io *fio = fec_io(io);
  422. if (!verity_fec_is_enabled(io->v))
  423. return;
  424. fio->rs = NULL;
  425. memset(fio->bufs, 0, sizeof(fio->bufs));
  426. fio->nbufs = 0;
  427. fio->output = NULL;
  428. fio->level = 0;
  429. }
  430. /*
  431. * Append feature arguments and values to the status table.
  432. */
  433. unsigned verity_fec_status_table(struct dm_verity *v, unsigned sz,
  434. char *result, unsigned maxlen)
  435. {
  436. if (!verity_fec_is_enabled(v))
  437. return sz;
  438. DMEMIT(" " DM_VERITY_OPT_FEC_DEV " %s "
  439. DM_VERITY_OPT_FEC_BLOCKS " %llu "
  440. DM_VERITY_OPT_FEC_START " %llu "
  441. DM_VERITY_OPT_FEC_ROOTS " %d",
  442. v->fec->dev->name,
  443. (unsigned long long)v->fec->blocks,
  444. (unsigned long long)v->fec->start,
  445. v->fec->roots);
  446. return sz;
  447. }
  448. void verity_fec_dtr(struct dm_verity *v)
  449. {
  450. struct dm_verity_fec *f = v->fec;
  451. if (!verity_fec_is_enabled(v))
  452. goto out;
  453. mempool_exit(&f->rs_pool);
  454. mempool_exit(&f->prealloc_pool);
  455. mempool_exit(&f->extra_pool);
  456. mempool_exit(&f->output_pool);
  457. kmem_cache_destroy(f->cache);
  458. if (f->data_bufio)
  459. dm_bufio_client_destroy(f->data_bufio);
  460. if (f->bufio)
  461. dm_bufio_client_destroy(f->bufio);
  462. if (f->dev)
  463. dm_put_device(v->ti, f->dev);
  464. out:
  465. kfree(f);
  466. v->fec = NULL;
  467. }
  468. static void *fec_rs_alloc(gfp_t gfp_mask, void *pool_data)
  469. {
  470. struct dm_verity *v = (struct dm_verity *)pool_data;
  471. return init_rs_gfp(8, 0x11d, 0, 1, v->fec->roots, gfp_mask);
  472. }
  473. static void fec_rs_free(void *element, void *pool_data)
  474. {
  475. struct rs_control *rs = (struct rs_control *)element;
  476. if (rs)
  477. free_rs(rs);
  478. }
  479. bool verity_is_fec_opt_arg(const char *arg_name)
  480. {
  481. return (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV) ||
  482. !strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS) ||
  483. !strcasecmp(arg_name, DM_VERITY_OPT_FEC_START) ||
  484. !strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS));
  485. }
  486. int verity_fec_parse_opt_args(struct dm_arg_set *as, struct dm_verity *v,
  487. unsigned *argc, const char *arg_name)
  488. {
  489. int r;
  490. struct dm_target *ti = v->ti;
  491. const char *arg_value;
  492. unsigned long long num_ll;
  493. unsigned char num_c;
  494. char dummy;
  495. if (!*argc) {
  496. ti->error = "FEC feature arguments require a value";
  497. return -EINVAL;
  498. }
  499. arg_value = dm_shift_arg(as);
  500. (*argc)--;
  501. if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_DEV)) {
  502. r = dm_get_device(ti, arg_value, FMODE_READ, &v->fec->dev);
  503. if (r) {
  504. ti->error = "FEC device lookup failed";
  505. return r;
  506. }
  507. } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_BLOCKS)) {
  508. if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
  509. ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT))
  510. >> (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
  511. ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
  512. return -EINVAL;
  513. }
  514. v->fec->blocks = num_ll;
  515. } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_START)) {
  516. if (sscanf(arg_value, "%llu%c", &num_ll, &dummy) != 1 ||
  517. ((sector_t)(num_ll << (v->data_dev_block_bits - SECTOR_SHIFT)) >>
  518. (v->data_dev_block_bits - SECTOR_SHIFT) != num_ll)) {
  519. ti->error = "Invalid " DM_VERITY_OPT_FEC_START;
  520. return -EINVAL;
  521. }
  522. v->fec->start = num_ll;
  523. } else if (!strcasecmp(arg_name, DM_VERITY_OPT_FEC_ROOTS)) {
  524. if (sscanf(arg_value, "%hhu%c", &num_c, &dummy) != 1 || !num_c ||
  525. num_c < (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MAX_RSN) ||
  526. num_c > (DM_VERITY_FEC_RSM - DM_VERITY_FEC_MIN_RSN)) {
  527. ti->error = "Invalid " DM_VERITY_OPT_FEC_ROOTS;
  528. return -EINVAL;
  529. }
  530. v->fec->roots = num_c;
  531. } else {
  532. ti->error = "Unrecognized verity FEC feature request";
  533. return -EINVAL;
  534. }
  535. return 0;
  536. }
  537. /*
  538. * Allocate dm_verity_fec for v->fec. Must be called before verity_fec_ctr.
  539. */
  540. int verity_fec_ctr_alloc(struct dm_verity *v)
  541. {
  542. struct dm_verity_fec *f;
  543. f = kzalloc(sizeof(struct dm_verity_fec), GFP_KERNEL);
  544. if (!f) {
  545. v->ti->error = "Cannot allocate FEC structure";
  546. return -ENOMEM;
  547. }
  548. v->fec = f;
  549. return 0;
  550. }
  551. /*
  552. * Validate arguments and preallocate memory. Must be called after arguments
  553. * have been parsed using verity_fec_parse_opt_args.
  554. */
  555. int verity_fec_ctr(struct dm_verity *v)
  556. {
  557. struct dm_verity_fec *f = v->fec;
  558. struct dm_target *ti = v->ti;
  559. u64 hash_blocks, fec_blocks;
  560. int ret;
  561. if (!verity_fec_is_enabled(v)) {
  562. verity_fec_dtr(v);
  563. return 0;
  564. }
  565. /*
  566. * FEC is computed over data blocks, possible metadata, and
  567. * hash blocks. In other words, FEC covers total of fec_blocks
  568. * blocks consisting of the following:
  569. *
  570. * data blocks | hash blocks | metadata (optional)
  571. *
  572. * We allow metadata after hash blocks to support a use case
  573. * where all data is stored on the same device and FEC covers
  574. * the entire area.
  575. *
  576. * If metadata is included, we require it to be available on the
  577. * hash device after the hash blocks.
  578. */
  579. hash_blocks = v->hash_blocks - v->hash_start;
  580. /*
  581. * Require matching block sizes for data and hash devices for
  582. * simplicity.
  583. */
  584. if (v->data_dev_block_bits != v->hash_dev_block_bits) {
  585. ti->error = "Block sizes must match to use FEC";
  586. return -EINVAL;
  587. }
  588. if (!f->roots) {
  589. ti->error = "Missing " DM_VERITY_OPT_FEC_ROOTS;
  590. return -EINVAL;
  591. }
  592. f->rsn = DM_VERITY_FEC_RSM - f->roots;
  593. if (!f->blocks) {
  594. ti->error = "Missing " DM_VERITY_OPT_FEC_BLOCKS;
  595. return -EINVAL;
  596. }
  597. f->rounds = f->blocks;
  598. if (sector_div(f->rounds, f->rsn))
  599. f->rounds++;
  600. /*
  601. * Due to optional metadata, f->blocks can be larger than
  602. * data_blocks and hash_blocks combined.
  603. */
  604. if (f->blocks < v->data_blocks + hash_blocks || !f->rounds) {
  605. ti->error = "Invalid " DM_VERITY_OPT_FEC_BLOCKS;
  606. return -EINVAL;
  607. }
  608. /*
  609. * Metadata is accessed through the hash device, so we require
  610. * it to be large enough.
  611. */
  612. f->hash_blocks = f->blocks - v->data_blocks;
  613. if (dm_bufio_get_device_size(v->bufio) < f->hash_blocks) {
  614. ti->error = "Hash device is too small for "
  615. DM_VERITY_OPT_FEC_BLOCKS;
  616. return -E2BIG;
  617. }
  618. if ((f->roots << SECTOR_SHIFT) & ((1 << v->data_dev_block_bits) - 1))
  619. f->io_size = 1 << v->data_dev_block_bits;
  620. else
  621. f->io_size = v->fec->roots << SECTOR_SHIFT;
  622. f->bufio = dm_bufio_client_create(f->dev->bdev,
  623. f->io_size,
  624. 1, 0, NULL, NULL);
  625. if (IS_ERR(f->bufio)) {
  626. ti->error = "Cannot initialize FEC bufio client";
  627. return PTR_ERR(f->bufio);
  628. }
  629. dm_bufio_set_sector_offset(f->bufio, f->start << (v->data_dev_block_bits - SECTOR_SHIFT));
  630. fec_blocks = div64_u64(f->rounds * f->roots, v->fec->roots << SECTOR_SHIFT);
  631. if (dm_bufio_get_device_size(f->bufio) < fec_blocks) {
  632. ti->error = "FEC device is too small";
  633. return -E2BIG;
  634. }
  635. f->data_bufio = dm_bufio_client_create(v->data_dev->bdev,
  636. 1 << v->data_dev_block_bits,
  637. 1, 0, NULL, NULL);
  638. if (IS_ERR(f->data_bufio)) {
  639. ti->error = "Cannot initialize FEC data bufio client";
  640. return PTR_ERR(f->data_bufio);
  641. }
  642. if (dm_bufio_get_device_size(f->data_bufio) < v->data_blocks) {
  643. ti->error = "Data device is too small";
  644. return -E2BIG;
  645. }
  646. /* Preallocate an rs_control structure for each worker thread */
  647. ret = mempool_init(&f->rs_pool, num_online_cpus(), fec_rs_alloc,
  648. fec_rs_free, (void *) v);
  649. if (ret) {
  650. ti->error = "Cannot allocate RS pool";
  651. return ret;
  652. }
  653. f->cache = kmem_cache_create("dm_verity_fec_buffers",
  654. f->rsn << DM_VERITY_FEC_BUF_RS_BITS,
  655. 0, 0, NULL);
  656. if (!f->cache) {
  657. ti->error = "Cannot create FEC buffer cache";
  658. return -ENOMEM;
  659. }
  660. /* Preallocate DM_VERITY_FEC_BUF_PREALLOC buffers for each thread */
  661. ret = mempool_init_slab_pool(&f->prealloc_pool, num_online_cpus() *
  662. DM_VERITY_FEC_BUF_PREALLOC,
  663. f->cache);
  664. if (ret) {
  665. ti->error = "Cannot allocate FEC buffer prealloc pool";
  666. return ret;
  667. }
  668. ret = mempool_init_slab_pool(&f->extra_pool, 0, f->cache);
  669. if (ret) {
  670. ti->error = "Cannot allocate FEC buffer extra pool";
  671. return ret;
  672. }
  673. /* Preallocate an output buffer for each thread */
  674. ret = mempool_init_kmalloc_pool(&f->output_pool, num_online_cpus(),
  675. 1 << v->data_dev_block_bits);
  676. if (ret) {
  677. ti->error = "Cannot allocate FEC output pool";
  678. return ret;
  679. }
  680. /* Reserve space for our per-bio data */
  681. ti->per_io_data_size += sizeof(struct dm_verity_fec_io);
  682. return 0;
  683. }