dm-ebs-target.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471
  1. /*
  2. * Copyright (C) 2020 Red Hat GmbH
  3. *
  4. * This file is released under the GPL.
  5. *
  6. * Device-mapper target to emulate smaller logical block
  7. * size on backing devices exposing (natively) larger ones.
  8. *
  9. * E.g. 512 byte sector emulation on 4K native disks.
  10. */
  11. #include "dm.h"
  12. #include <linux/module.h>
  13. #include <linux/workqueue.h>
  14. #include <linux/dm-bufio.h>
  15. #define DM_MSG_PREFIX "ebs"
  16. static void ebs_dtr(struct dm_target *ti);
  17. /* Emulated block size context. */
  18. struct ebs_c {
  19. struct dm_dev *dev; /* Underlying device to emulate block size on. */
  20. struct dm_bufio_client *bufio; /* Use dm-bufio for read and read-modify-write processing. */
  21. struct workqueue_struct *wq; /* Workqueue for ^ processing of bios. */
  22. struct work_struct ws; /* Work item used for ^. */
  23. struct bio_list bios_in; /* Worker bios input list. */
  24. spinlock_t lock; /* Guard bios input list above. */
  25. sector_t start; /* <start> table line argument, see ebs_ctr below. */
  26. unsigned int e_bs; /* Emulated block size in sectors exposed to upper layer. */
  27. unsigned int u_bs; /* Underlying block size in sectors retrievd from/set on lower layer device. */
  28. unsigned char block_shift; /* bitshift sectors -> blocks used in dm-bufio API. */
  29. bool u_bs_set:1; /* Flag to indicate underlying block size is set on table line. */
  30. };
  31. static inline sector_t __sector_to_block(struct ebs_c *ec, sector_t sector)
  32. {
  33. return sector >> ec->block_shift;
  34. }
  35. static inline sector_t __block_mod(sector_t sector, unsigned int bs)
  36. {
  37. return sector & (bs - 1);
  38. }
  39. /* Return number of blocks for a bio, accounting for misalignement of start and end sectors. */
  40. static inline unsigned int __nr_blocks(struct ebs_c *ec, struct bio *bio)
  41. {
  42. sector_t end_sector = __block_mod(bio->bi_iter.bi_sector, ec->u_bs) + bio_sectors(bio);
  43. return __sector_to_block(ec, end_sector) + (__block_mod(end_sector, ec->u_bs) ? 1 : 0);
  44. }
  45. static inline bool __ebs_check_bs(unsigned int bs)
  46. {
  47. return bs && is_power_of_2(bs);
  48. }
  49. /*
  50. * READ/WRITE:
  51. *
  52. * copy blocks between bufio blocks and bio vector's (partial/overlapping) pages.
  53. */
  54. static int __ebs_rw_bvec(struct ebs_c *ec, int rw, struct bio_vec *bv, struct bvec_iter *iter)
  55. {
  56. int r = 0;
  57. unsigned char *ba, *pa;
  58. unsigned int cur_len;
  59. unsigned int bv_len = bv->bv_len;
  60. unsigned int buf_off = to_bytes(__block_mod(iter->bi_sector, ec->u_bs));
  61. sector_t block = __sector_to_block(ec, iter->bi_sector);
  62. struct dm_buffer *b;
  63. if (unlikely(!bv->bv_page || !bv_len))
  64. return -EIO;
  65. pa = page_address(bv->bv_page) + bv->bv_offset;
  66. /* Handle overlapping page <-> blocks */
  67. while (bv_len) {
  68. cur_len = min(dm_bufio_get_block_size(ec->bufio) - buf_off, bv_len);
  69. /* Avoid reading for writes in case bio vector's page overwrites block completely. */
  70. if (rw == READ || buf_off || bv_len < dm_bufio_get_block_size(ec->bufio))
  71. ba = dm_bufio_read(ec->bufio, block, &b);
  72. else
  73. ba = dm_bufio_new(ec->bufio, block, &b);
  74. if (unlikely(IS_ERR(ba))) {
  75. /*
  76. * Carry on with next buffer, if any, to issue all possible
  77. * data but return error.
  78. */
  79. r = PTR_ERR(ba);
  80. } else {
  81. /* Copy data to/from bio to buffer if read/new was successful above. */
  82. ba += buf_off;
  83. if (rw == READ) {
  84. memcpy(pa, ba, cur_len);
  85. flush_dcache_page(bv->bv_page);
  86. } else {
  87. flush_dcache_page(bv->bv_page);
  88. memcpy(ba, pa, cur_len);
  89. dm_bufio_mark_partial_buffer_dirty(b, buf_off, buf_off + cur_len);
  90. }
  91. dm_bufio_release(b);
  92. }
  93. pa += cur_len;
  94. bv_len -= cur_len;
  95. buf_off = 0;
  96. block++;
  97. }
  98. return r;
  99. }
  100. /* READ/WRITE: iterate bio vector's copying between (partial) pages and bufio blocks. */
  101. static int __ebs_rw_bio(struct ebs_c *ec, int rw, struct bio *bio)
  102. {
  103. int r = 0, rr;
  104. struct bio_vec bv;
  105. struct bvec_iter iter;
  106. bio_for_each_bvec(bv, bio, iter) {
  107. rr = __ebs_rw_bvec(ec, rw, &bv, &iter);
  108. if (rr)
  109. r = rr;
  110. }
  111. return r;
  112. }
  113. /*
  114. * Discard bio's blocks, i.e. pass discards down.
  115. *
  116. * Avoid discarding partial blocks at beginning and end;
  117. * return 0 in case no blocks can be discarded as a result.
  118. */
  119. static int __ebs_discard_bio(struct ebs_c *ec, struct bio *bio)
  120. {
  121. sector_t block, blocks, sector = bio->bi_iter.bi_sector;
  122. block = __sector_to_block(ec, sector);
  123. blocks = __nr_blocks(ec, bio);
  124. /*
  125. * Partial first underlying block (__nr_blocks() may have
  126. * resulted in one block).
  127. */
  128. if (__block_mod(sector, ec->u_bs)) {
  129. block++;
  130. blocks--;
  131. }
  132. /* Partial last underlying block if any. */
  133. if (blocks && __block_mod(bio_end_sector(bio), ec->u_bs))
  134. blocks--;
  135. return blocks ? dm_bufio_issue_discard(ec->bufio, block, blocks) : 0;
  136. }
  137. /* Release blocks them from the bufio cache. */
  138. static void __ebs_forget_bio(struct ebs_c *ec, struct bio *bio)
  139. {
  140. sector_t blocks, sector = bio->bi_iter.bi_sector;
  141. blocks = __nr_blocks(ec, bio);
  142. dm_bufio_forget_buffers(ec->bufio, __sector_to_block(ec, sector), blocks);
  143. }
  144. /* Worker funtion to process incoming bios. */
  145. static void __ebs_process_bios(struct work_struct *ws)
  146. {
  147. int r;
  148. bool write = false;
  149. sector_t block1, block2;
  150. struct ebs_c *ec = container_of(ws, struct ebs_c, ws);
  151. struct bio *bio;
  152. struct bio_list bios;
  153. bio_list_init(&bios);
  154. spin_lock_irq(&ec->lock);
  155. bios = ec->bios_in;
  156. bio_list_init(&ec->bios_in);
  157. spin_unlock_irq(&ec->lock);
  158. /* Prefetch all read and any mis-aligned write buffers */
  159. bio_list_for_each(bio, &bios) {
  160. block1 = __sector_to_block(ec, bio->bi_iter.bi_sector);
  161. if (bio_op(bio) == REQ_OP_READ)
  162. dm_bufio_prefetch(ec->bufio, block1, __nr_blocks(ec, bio));
  163. else if (bio_op(bio) == REQ_OP_WRITE && !(bio->bi_opf & REQ_PREFLUSH)) {
  164. block2 = __sector_to_block(ec, bio_end_sector(bio));
  165. if (__block_mod(bio->bi_iter.bi_sector, ec->u_bs))
  166. dm_bufio_prefetch(ec->bufio, block1, 1);
  167. if (__block_mod(bio_end_sector(bio), ec->u_bs) && block2 != block1)
  168. dm_bufio_prefetch(ec->bufio, block2, 1);
  169. }
  170. }
  171. bio_list_for_each(bio, &bios) {
  172. r = -EIO;
  173. if (bio_op(bio) == REQ_OP_READ)
  174. r = __ebs_rw_bio(ec, READ, bio);
  175. else if (bio_op(bio) == REQ_OP_WRITE) {
  176. write = true;
  177. r = __ebs_rw_bio(ec, WRITE, bio);
  178. } else if (bio_op(bio) == REQ_OP_DISCARD) {
  179. __ebs_forget_bio(ec, bio);
  180. r = __ebs_discard_bio(ec, bio);
  181. }
  182. if (r < 0)
  183. bio->bi_status = errno_to_blk_status(r);
  184. }
  185. /*
  186. * We write dirty buffers after processing I/O on them
  187. * but before we endio thus addressing REQ_FUA/REQ_SYNC.
  188. */
  189. r = write ? dm_bufio_write_dirty_buffers(ec->bufio) : 0;
  190. while ((bio = bio_list_pop(&bios))) {
  191. /* Any other request is endioed. */
  192. if (unlikely(r && bio_op(bio) == REQ_OP_WRITE))
  193. bio_io_error(bio);
  194. else
  195. bio_endio(bio);
  196. }
  197. }
  198. /*
  199. * Construct an emulated block size mapping: <dev_path> <offset> <ebs> [<ubs>]
  200. *
  201. * <dev_path>: path of the underlying device
  202. * <offset>: offset in 512 bytes sectors into <dev_path>
  203. * <ebs>: emulated block size in units of 512 bytes exposed to the upper layer
  204. * [<ubs>]: underlying block size in units of 512 bytes imposed on the lower layer;
  205. * optional, if not supplied, retrieve logical block size from underlying device
  206. */
  207. static int ebs_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  208. {
  209. int r;
  210. unsigned short tmp1;
  211. unsigned long long tmp;
  212. char dummy;
  213. struct ebs_c *ec;
  214. if (argc < 3 || argc > 4) {
  215. ti->error = "Invalid argument count";
  216. return -EINVAL;
  217. }
  218. ec = ti->private = kzalloc(sizeof(*ec), GFP_KERNEL);
  219. if (!ec) {
  220. ti->error = "Cannot allocate ebs context";
  221. return -ENOMEM;
  222. }
  223. r = -EINVAL;
  224. if (sscanf(argv[1], "%llu%c", &tmp, &dummy) != 1 ||
  225. tmp != (sector_t)tmp ||
  226. (sector_t)tmp >= ti->len) {
  227. ti->error = "Invalid device offset sector";
  228. goto bad;
  229. }
  230. ec->start = tmp;
  231. if (sscanf(argv[2], "%hu%c", &tmp1, &dummy) != 1 ||
  232. !__ebs_check_bs(tmp1) ||
  233. to_bytes(tmp1) > PAGE_SIZE) {
  234. ti->error = "Invalid emulated block size";
  235. goto bad;
  236. }
  237. ec->e_bs = tmp1;
  238. if (argc > 3) {
  239. if (sscanf(argv[3], "%hu%c", &tmp1, &dummy) != 1 || !__ebs_check_bs(tmp1)) {
  240. ti->error = "Invalid underlying block size";
  241. goto bad;
  242. }
  243. ec->u_bs = tmp1;
  244. ec->u_bs_set = true;
  245. } else
  246. ec->u_bs_set = false;
  247. r = dm_get_device(ti, argv[0], dm_table_get_mode(ti->table), &ec->dev);
  248. if (r) {
  249. ti->error = "Device lookup failed";
  250. ec->dev = NULL;
  251. goto bad;
  252. }
  253. r = -EINVAL;
  254. if (!ec->u_bs_set) {
  255. ec->u_bs = to_sector(bdev_logical_block_size(ec->dev->bdev));
  256. if (!__ebs_check_bs(ec->u_bs)) {
  257. ti->error = "Invalid retrieved underlying block size";
  258. goto bad;
  259. }
  260. }
  261. if (!ec->u_bs_set && ec->e_bs == ec->u_bs)
  262. DMINFO("Emulation superfluous: emulated equal to underlying block size");
  263. if (__block_mod(ec->start, ec->u_bs)) {
  264. ti->error = "Device offset must be multiple of underlying block size";
  265. goto bad;
  266. }
  267. ec->bufio = dm_bufio_client_create(ec->dev->bdev, to_bytes(ec->u_bs), 1, 0, NULL, NULL);
  268. if (IS_ERR(ec->bufio)) {
  269. ti->error = "Cannot create dm bufio client";
  270. r = PTR_ERR(ec->bufio);
  271. ec->bufio = NULL;
  272. goto bad;
  273. }
  274. ec->wq = alloc_ordered_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM);
  275. if (!ec->wq) {
  276. ti->error = "Cannot create dm-" DM_MSG_PREFIX " workqueue";
  277. r = -ENOMEM;
  278. goto bad;
  279. }
  280. ec->block_shift = __ffs(ec->u_bs);
  281. INIT_WORK(&ec->ws, &__ebs_process_bios);
  282. bio_list_init(&ec->bios_in);
  283. spin_lock_init(&ec->lock);
  284. ti->num_flush_bios = 1;
  285. ti->num_discard_bios = 1;
  286. ti->num_secure_erase_bios = 0;
  287. ti->num_write_same_bios = 0;
  288. ti->num_write_zeroes_bios = 0;
  289. return 0;
  290. bad:
  291. ebs_dtr(ti);
  292. return r;
  293. }
  294. static void ebs_dtr(struct dm_target *ti)
  295. {
  296. struct ebs_c *ec = ti->private;
  297. if (ec->wq)
  298. destroy_workqueue(ec->wq);
  299. if (ec->bufio)
  300. dm_bufio_client_destroy(ec->bufio);
  301. if (ec->dev)
  302. dm_put_device(ti, ec->dev);
  303. kfree(ec);
  304. }
  305. static int ebs_map(struct dm_target *ti, struct bio *bio)
  306. {
  307. struct ebs_c *ec = ti->private;
  308. bio_set_dev(bio, ec->dev->bdev);
  309. bio->bi_iter.bi_sector = ec->start + dm_target_offset(ti, bio->bi_iter.bi_sector);
  310. if (unlikely(bio_op(bio) == REQ_OP_FLUSH))
  311. return DM_MAPIO_REMAPPED;
  312. /*
  313. * Only queue for bufio processing in case of partial or overlapping buffers
  314. * -or-
  315. * emulation with ebs == ubs aiming for tests of dm-bufio overhead.
  316. */
  317. if (likely(__block_mod(bio->bi_iter.bi_sector, ec->u_bs) ||
  318. __block_mod(bio_end_sector(bio), ec->u_bs) ||
  319. ec->e_bs == ec->u_bs)) {
  320. spin_lock_irq(&ec->lock);
  321. bio_list_add(&ec->bios_in, bio);
  322. spin_unlock_irq(&ec->lock);
  323. queue_work(ec->wq, &ec->ws);
  324. return DM_MAPIO_SUBMITTED;
  325. }
  326. /* Forget any buffer content relative to this direct backing device I/O. */
  327. __ebs_forget_bio(ec, bio);
  328. return DM_MAPIO_REMAPPED;
  329. }
  330. static void ebs_status(struct dm_target *ti, status_type_t type,
  331. unsigned status_flags, char *result, unsigned maxlen)
  332. {
  333. struct ebs_c *ec = ti->private;
  334. switch (type) {
  335. case STATUSTYPE_INFO:
  336. *result = '\0';
  337. break;
  338. case STATUSTYPE_TABLE:
  339. snprintf(result, maxlen, ec->u_bs_set ? "%s %llu %u %u" : "%s %llu %u",
  340. ec->dev->name, (unsigned long long) ec->start, ec->e_bs, ec->u_bs);
  341. break;
  342. }
  343. }
  344. static int ebs_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
  345. {
  346. struct ebs_c *ec = ti->private;
  347. struct dm_dev *dev = ec->dev;
  348. /*
  349. * Only pass ioctls through if the device sizes match exactly.
  350. */
  351. *bdev = dev->bdev;
  352. return !!(ec->start || ti->len != i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT);
  353. }
  354. static void ebs_io_hints(struct dm_target *ti, struct queue_limits *limits)
  355. {
  356. struct ebs_c *ec = ti->private;
  357. limits->logical_block_size = to_bytes(ec->e_bs);
  358. limits->physical_block_size = to_bytes(ec->u_bs);
  359. limits->alignment_offset = limits->physical_block_size;
  360. blk_limits_io_min(limits, limits->logical_block_size);
  361. }
  362. static int ebs_iterate_devices(struct dm_target *ti,
  363. iterate_devices_callout_fn fn, void *data)
  364. {
  365. struct ebs_c *ec = ti->private;
  366. return fn(ti, ec->dev, ec->start, ti->len, data);
  367. }
  368. static struct target_type ebs_target = {
  369. .name = "ebs",
  370. .version = {1, 0, 1},
  371. .features = DM_TARGET_PASSES_INTEGRITY,
  372. .module = THIS_MODULE,
  373. .ctr = ebs_ctr,
  374. .dtr = ebs_dtr,
  375. .map = ebs_map,
  376. .status = ebs_status,
  377. .io_hints = ebs_io_hints,
  378. .prepare_ioctl = ebs_prepare_ioctl,
  379. .iterate_devices = ebs_iterate_devices,
  380. };
  381. static int __init dm_ebs_init(void)
  382. {
  383. int r = dm_register_target(&ebs_target);
  384. if (r < 0)
  385. DMERR("register failed %d", r);
  386. return r;
  387. }
  388. static void dm_ebs_exit(void)
  389. {
  390. dm_unregister_target(&ebs_target);
  391. }
  392. module_init(dm_ebs_init);
  393. module_exit(dm_ebs_exit);
  394. MODULE_AUTHOR("Heinz Mauelshagen <dm-devel@redhat.com>");
  395. MODULE_DESCRIPTION(DM_NAME " emulated block size target");
  396. MODULE_LICENSE("GPL");