bio-integrity.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * bio-integrity.c - bio data integrity extensions
  4. *
  5. * Copyright (C) 2007, 2008, 2009 Oracle Corporation
  6. * Written by: Martin K. Petersen <martin.petersen@oracle.com>
  7. */
  8. #include <linux/blkdev.h>
  9. #include <linux/mempool.h>
  10. #include <linux/export.h>
  11. #include <linux/bio.h>
  12. #include <linux/workqueue.h>
  13. #include <linux/slab.h>
  14. #include "blk.h"
  15. #define BIP_INLINE_VECS 4
  16. static struct kmem_cache *bip_slab;
  17. static struct workqueue_struct *kintegrityd_wq;
  18. void blk_flush_integrity(void)
  19. {
  20. flush_workqueue(kintegrityd_wq);
  21. }
  22. static void __bio_integrity_free(struct bio_set *bs,
  23. struct bio_integrity_payload *bip)
  24. {
  25. if (bs && mempool_initialized(&bs->bio_integrity_pool)) {
  26. if (bip->bip_vec)
  27. bvec_free(&bs->bvec_integrity_pool, bip->bip_vec,
  28. bip->bip_slab);
  29. mempool_free(bip, &bs->bio_integrity_pool);
  30. } else {
  31. kfree(bip);
  32. }
  33. }
  34. /**
  35. * bio_integrity_alloc - Allocate integrity payload and attach it to bio
  36. * @bio: bio to attach integrity metadata to
  37. * @gfp_mask: Memory allocation mask
  38. * @nr_vecs: Number of integrity metadata scatter-gather elements
  39. *
  40. * Description: This function prepares a bio for attaching integrity
  41. * metadata. nr_vecs specifies the maximum number of pages containing
  42. * integrity metadata that can be attached.
  43. */
  44. struct bio_integrity_payload *bio_integrity_alloc(struct bio *bio,
  45. gfp_t gfp_mask,
  46. unsigned int nr_vecs)
  47. {
  48. struct bio_integrity_payload *bip;
  49. struct bio_set *bs = bio->bi_pool;
  50. unsigned inline_vecs;
  51. if (WARN_ON_ONCE(bio_has_crypt_ctx(bio)))
  52. return ERR_PTR(-EOPNOTSUPP);
  53. if (!bs || !mempool_initialized(&bs->bio_integrity_pool)) {
  54. bip = kmalloc(struct_size(bip, bip_inline_vecs, nr_vecs), gfp_mask);
  55. inline_vecs = nr_vecs;
  56. } else {
  57. bip = mempool_alloc(&bs->bio_integrity_pool, gfp_mask);
  58. inline_vecs = BIP_INLINE_VECS;
  59. }
  60. if (unlikely(!bip))
  61. return ERR_PTR(-ENOMEM);
  62. memset(bip, 0, sizeof(*bip));
  63. if (nr_vecs > inline_vecs) {
  64. unsigned long idx = 0;
  65. bip->bip_vec = bvec_alloc(gfp_mask, nr_vecs, &idx,
  66. &bs->bvec_integrity_pool);
  67. if (!bip->bip_vec)
  68. goto err;
  69. bip->bip_max_vcnt = bvec_nr_vecs(idx);
  70. bip->bip_slab = idx;
  71. } else {
  72. bip->bip_vec = bip->bip_inline_vecs;
  73. bip->bip_max_vcnt = inline_vecs;
  74. }
  75. bip->bip_bio = bio;
  76. bio->bi_integrity = bip;
  77. bio->bi_opf |= REQ_INTEGRITY;
  78. return bip;
  79. err:
  80. __bio_integrity_free(bs, bip);
  81. return ERR_PTR(-ENOMEM);
  82. }
  83. EXPORT_SYMBOL(bio_integrity_alloc);
  84. /**
  85. * bio_integrity_free - Free bio integrity payload
  86. * @bio: bio containing bip to be freed
  87. *
  88. * Description: Used to free the integrity portion of a bio. Usually
  89. * called from bio_free().
  90. */
  91. void bio_integrity_free(struct bio *bio)
  92. {
  93. struct bio_integrity_payload *bip = bio_integrity(bio);
  94. struct bio_set *bs = bio->bi_pool;
  95. if (bip->bip_flags & BIP_BLOCK_INTEGRITY)
  96. kfree(page_address(bip->bip_vec->bv_page) +
  97. bip->bip_vec->bv_offset);
  98. __bio_integrity_free(bs, bip);
  99. bio->bi_integrity = NULL;
  100. bio->bi_opf &= ~REQ_INTEGRITY;
  101. }
  102. /**
  103. * bio_integrity_add_page - Attach integrity metadata
  104. * @bio: bio to update
  105. * @page: page containing integrity metadata
  106. * @len: number of bytes of integrity metadata in page
  107. * @offset: start offset within page
  108. *
  109. * Description: Attach a page containing integrity metadata to bio.
  110. */
  111. int bio_integrity_add_page(struct bio *bio, struct page *page,
  112. unsigned int len, unsigned int offset)
  113. {
  114. struct bio_integrity_payload *bip = bio_integrity(bio);
  115. struct bio_vec *iv;
  116. if (bip->bip_vcnt >= bip->bip_max_vcnt) {
  117. printk(KERN_ERR "%s: bip_vec full\n", __func__);
  118. return 0;
  119. }
  120. iv = bip->bip_vec + bip->bip_vcnt;
  121. if (bip->bip_vcnt &&
  122. bvec_gap_to_prev(bio->bi_disk->queue,
  123. &bip->bip_vec[bip->bip_vcnt - 1], offset))
  124. return 0;
  125. iv->bv_page = page;
  126. iv->bv_len = len;
  127. iv->bv_offset = offset;
  128. bip->bip_vcnt++;
  129. return len;
  130. }
  131. EXPORT_SYMBOL(bio_integrity_add_page);
  132. /**
  133. * bio_integrity_process - Process integrity metadata for a bio
  134. * @bio: bio to generate/verify integrity metadata for
  135. * @proc_iter: iterator to process
  136. * @proc_fn: Pointer to the relevant processing function
  137. */
  138. static blk_status_t bio_integrity_process(struct bio *bio,
  139. struct bvec_iter *proc_iter, integrity_processing_fn *proc_fn)
  140. {
  141. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  142. struct blk_integrity_iter iter;
  143. struct bvec_iter bviter;
  144. struct bio_vec bv;
  145. struct bio_integrity_payload *bip = bio_integrity(bio);
  146. blk_status_t ret = BLK_STS_OK;
  147. void *prot_buf = page_address(bip->bip_vec->bv_page) +
  148. bip->bip_vec->bv_offset;
  149. iter.disk_name = bio->bi_disk->disk_name;
  150. iter.interval = 1 << bi->interval_exp;
  151. iter.seed = proc_iter->bi_sector;
  152. iter.prot_buf = prot_buf;
  153. __bio_for_each_segment(bv, bio, bviter, *proc_iter) {
  154. void *kaddr = kmap_atomic(bv.bv_page);
  155. iter.data_buf = kaddr + bv.bv_offset;
  156. iter.data_size = bv.bv_len;
  157. ret = proc_fn(&iter);
  158. if (ret) {
  159. kunmap_atomic(kaddr);
  160. return ret;
  161. }
  162. kunmap_atomic(kaddr);
  163. }
  164. return ret;
  165. }
  166. /**
  167. * bio_integrity_prep - Prepare bio for integrity I/O
  168. * @bio: bio to prepare
  169. *
  170. * Description: Checks if the bio already has an integrity payload attached.
  171. * If it does, the payload has been generated by another kernel subsystem,
  172. * and we just pass it through. Otherwise allocates integrity payload.
  173. * The bio must have data direction, target device and start sector set priot
  174. * to calling. In the WRITE case, integrity metadata will be generated using
  175. * the block device's integrity function. In the READ case, the buffer
  176. * will be prepared for DMA and a suitable end_io handler set up.
  177. */
  178. bool bio_integrity_prep(struct bio *bio)
  179. {
  180. struct bio_integrity_payload *bip;
  181. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  182. struct request_queue *q = bio->bi_disk->queue;
  183. void *buf;
  184. unsigned long start, end;
  185. unsigned int len, nr_pages;
  186. unsigned int bytes, offset, i;
  187. unsigned int intervals;
  188. blk_status_t status;
  189. if (!bi)
  190. return true;
  191. if (bio_op(bio) != REQ_OP_READ && bio_op(bio) != REQ_OP_WRITE)
  192. return true;
  193. if (!bio_sectors(bio))
  194. return true;
  195. /* Already protected? */
  196. if (bio_integrity(bio))
  197. return true;
  198. if (bio_data_dir(bio) == READ) {
  199. if (!bi->profile->verify_fn ||
  200. !(bi->flags & BLK_INTEGRITY_VERIFY))
  201. return true;
  202. } else {
  203. if (!bi->profile->generate_fn ||
  204. !(bi->flags & BLK_INTEGRITY_GENERATE))
  205. return true;
  206. }
  207. intervals = bio_integrity_intervals(bi, bio_sectors(bio));
  208. /* Allocate kernel buffer for protection data */
  209. len = intervals * bi->tuple_size;
  210. buf = kmalloc(len, GFP_NOIO | q->bounce_gfp);
  211. status = BLK_STS_RESOURCE;
  212. if (unlikely(buf == NULL)) {
  213. printk(KERN_ERR "could not allocate integrity buffer\n");
  214. goto err_end_io;
  215. }
  216. end = (((unsigned long) buf) + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  217. start = ((unsigned long) buf) >> PAGE_SHIFT;
  218. nr_pages = end - start;
  219. /* Allocate bio integrity payload and integrity vectors */
  220. bip = bio_integrity_alloc(bio, GFP_NOIO, nr_pages);
  221. if (IS_ERR(bip)) {
  222. printk(KERN_ERR "could not allocate data integrity bioset\n");
  223. kfree(buf);
  224. status = BLK_STS_RESOURCE;
  225. goto err_end_io;
  226. }
  227. bip->bip_flags |= BIP_BLOCK_INTEGRITY;
  228. bip->bip_iter.bi_size = len;
  229. bip_set_seed(bip, bio->bi_iter.bi_sector);
  230. if (bi->flags & BLK_INTEGRITY_IP_CHECKSUM)
  231. bip->bip_flags |= BIP_IP_CHECKSUM;
  232. /* Map it */
  233. offset = offset_in_page(buf);
  234. for (i = 0 ; i < nr_pages ; i++) {
  235. int ret;
  236. bytes = PAGE_SIZE - offset;
  237. if (len <= 0)
  238. break;
  239. if (bytes > len)
  240. bytes = len;
  241. ret = bio_integrity_add_page(bio, virt_to_page(buf),
  242. bytes, offset);
  243. if (ret == 0) {
  244. printk(KERN_ERR "could not attach integrity payload\n");
  245. status = BLK_STS_RESOURCE;
  246. goto err_end_io;
  247. }
  248. if (ret < bytes)
  249. break;
  250. buf += bytes;
  251. len -= bytes;
  252. offset = 0;
  253. }
  254. /* Auto-generate integrity metadata if this is a write */
  255. if (bio_data_dir(bio) == WRITE) {
  256. bio_integrity_process(bio, &bio->bi_iter,
  257. bi->profile->generate_fn);
  258. } else {
  259. bip->bio_iter = bio->bi_iter;
  260. }
  261. return true;
  262. err_end_io:
  263. bio->bi_status = status;
  264. bio_endio(bio);
  265. return false;
  266. }
  267. EXPORT_SYMBOL(bio_integrity_prep);
  268. /**
  269. * bio_integrity_verify_fn - Integrity I/O completion worker
  270. * @work: Work struct stored in bio to be verified
  271. *
  272. * Description: This workqueue function is called to complete a READ
  273. * request. The function verifies the transferred integrity metadata
  274. * and then calls the original bio end_io function.
  275. */
  276. static void bio_integrity_verify_fn(struct work_struct *work)
  277. {
  278. struct bio_integrity_payload *bip =
  279. container_of(work, struct bio_integrity_payload, bip_work);
  280. struct bio *bio = bip->bip_bio;
  281. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  282. /*
  283. * At the moment verify is called bio's iterator was advanced
  284. * during split and completion, we need to rewind iterator to
  285. * it's original position.
  286. */
  287. bio->bi_status = bio_integrity_process(bio, &bip->bio_iter,
  288. bi->profile->verify_fn);
  289. bio_integrity_free(bio);
  290. bio_endio(bio);
  291. }
  292. /**
  293. * __bio_integrity_endio - Integrity I/O completion function
  294. * @bio: Protected bio
  295. *
  296. * Description: Completion for integrity I/O
  297. *
  298. * Normally I/O completion is done in interrupt context. However,
  299. * verifying I/O integrity is a time-consuming task which must be run
  300. * in process context. This function postpones completion
  301. * accordingly.
  302. */
  303. bool __bio_integrity_endio(struct bio *bio)
  304. {
  305. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  306. struct bio_integrity_payload *bip = bio_integrity(bio);
  307. if (bio_op(bio) == REQ_OP_READ && !bio->bi_status &&
  308. (bip->bip_flags & BIP_BLOCK_INTEGRITY) && bi->profile->verify_fn) {
  309. INIT_WORK(&bip->bip_work, bio_integrity_verify_fn);
  310. queue_work(kintegrityd_wq, &bip->bip_work);
  311. return false;
  312. }
  313. bio_integrity_free(bio);
  314. return true;
  315. }
  316. /**
  317. * bio_integrity_advance - Advance integrity vector
  318. * @bio: bio whose integrity vector to update
  319. * @bytes_done: number of data bytes that have been completed
  320. *
  321. * Description: This function calculates how many integrity bytes the
  322. * number of completed data bytes correspond to and advances the
  323. * integrity vector accordingly.
  324. */
  325. void bio_integrity_advance(struct bio *bio, unsigned int bytes_done)
  326. {
  327. struct bio_integrity_payload *bip = bio_integrity(bio);
  328. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  329. unsigned bytes = bio_integrity_bytes(bi, bytes_done >> 9);
  330. bip->bip_iter.bi_sector += bio_integrity_intervals(bi, bytes_done >> 9);
  331. bvec_iter_advance(bip->bip_vec, &bip->bip_iter, bytes);
  332. }
  333. /**
  334. * bio_integrity_trim - Trim integrity vector
  335. * @bio: bio whose integrity vector to update
  336. *
  337. * Description: Used to trim the integrity vector in a cloned bio.
  338. */
  339. void bio_integrity_trim(struct bio *bio)
  340. {
  341. struct bio_integrity_payload *bip = bio_integrity(bio);
  342. struct blk_integrity *bi = blk_get_integrity(bio->bi_disk);
  343. bip->bip_iter.bi_size = bio_integrity_bytes(bi, bio_sectors(bio));
  344. }
  345. EXPORT_SYMBOL(bio_integrity_trim);
  346. /**
  347. * bio_integrity_clone - Callback for cloning bios with integrity metadata
  348. * @bio: New bio
  349. * @bio_src: Original bio
  350. * @gfp_mask: Memory allocation mask
  351. *
  352. * Description: Called to allocate a bip when cloning a bio
  353. */
  354. int bio_integrity_clone(struct bio *bio, struct bio *bio_src,
  355. gfp_t gfp_mask)
  356. {
  357. struct bio_integrity_payload *bip_src = bio_integrity(bio_src);
  358. struct bio_integrity_payload *bip;
  359. BUG_ON(bip_src == NULL);
  360. bip = bio_integrity_alloc(bio, gfp_mask, bip_src->bip_vcnt);
  361. if (IS_ERR(bip))
  362. return PTR_ERR(bip);
  363. memcpy(bip->bip_vec, bip_src->bip_vec,
  364. bip_src->bip_vcnt * sizeof(struct bio_vec));
  365. bip->bip_vcnt = bip_src->bip_vcnt;
  366. bip->bip_iter = bip_src->bip_iter;
  367. return 0;
  368. }
  369. EXPORT_SYMBOL(bio_integrity_clone);
  370. int bioset_integrity_create(struct bio_set *bs, int pool_size)
  371. {
  372. if (mempool_initialized(&bs->bio_integrity_pool))
  373. return 0;
  374. if (mempool_init_slab_pool(&bs->bio_integrity_pool,
  375. pool_size, bip_slab))
  376. return -1;
  377. if (biovec_init_pool(&bs->bvec_integrity_pool, pool_size)) {
  378. mempool_exit(&bs->bio_integrity_pool);
  379. return -1;
  380. }
  381. return 0;
  382. }
  383. EXPORT_SYMBOL(bioset_integrity_create);
  384. void bioset_integrity_free(struct bio_set *bs)
  385. {
  386. mempool_exit(&bs->bio_integrity_pool);
  387. mempool_exit(&bs->bvec_integrity_pool);
  388. }
  389. void __init bio_integrity_init(void)
  390. {
  391. /*
  392. * kintegrityd won't block much but may burn a lot of CPU cycles.
  393. * Make it highpri CPU intensive wq with max concurrency of 1.
  394. */
  395. kintegrityd_wq = alloc_workqueue("kintegrityd", WQ_MEM_RECLAIM |
  396. WQ_HIGHPRI | WQ_CPU_INTENSIVE, 1);
  397. if (!kintegrityd_wq)
  398. panic("Failed to create kintegrityd\n");
  399. bip_slab = kmem_cache_create("bio_integrity_payload",
  400. sizeof(struct bio_integrity_payload) +
  401. sizeof(struct bio_vec) * BIP_INLINE_VECS,
  402. 0, SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
  403. }