brd.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Ram backed block device driver.
  4. *
  5. * Copyright (C) 2007 Nick Piggin
  6. * Copyright (C) 2007 Novell Inc.
  7. *
  8. * Parts derived from drivers/block/rd.c, and drivers/block/loop.c, copyright
  9. * of their respective owners.
  10. */
  11. #include <linux/init.h>
  12. #include <linux/initrd.h>
  13. #include <linux/module.h>
  14. #include <linux/moduleparam.h>
  15. #include <linux/major.h>
  16. #include <linux/blkdev.h>
  17. #include <linux/bio.h>
  18. #include <linux/highmem.h>
  19. #include <linux/mutex.h>
  20. #include <linux/radix-tree.h>
  21. #include <linux/fs.h>
  22. #include <linux/slab.h>
  23. #include <linux/backing-dev.h>
  24. #include <linux/uaccess.h>
  25. #define PAGE_SECTORS_SHIFT (PAGE_SHIFT - SECTOR_SHIFT)
  26. #define PAGE_SECTORS (1 << PAGE_SECTORS_SHIFT)
  27. /*
  28. * Each block ramdisk device has a radix_tree brd_pages of pages that stores
  29. * the pages containing the block device's contents. A brd page's ->index is
  30. * its offset in PAGE_SIZE units. This is similar to, but in no way connected
  31. * with, the kernel's pagecache or buffer cache (which sit above our block
  32. * device).
  33. */
  34. struct brd_device {
  35. int brd_number;
  36. struct request_queue *brd_queue;
  37. struct gendisk *brd_disk;
  38. struct list_head brd_list;
  39. /*
  40. * Backing store of pages and lock to protect it. This is the contents
  41. * of the block device.
  42. */
  43. spinlock_t brd_lock;
  44. struct radix_tree_root brd_pages;
  45. };
  46. /*
  47. * Look up and return a brd's page for a given sector.
  48. */
  49. static struct page *brd_lookup_page(struct brd_device *brd, sector_t sector)
  50. {
  51. pgoff_t idx;
  52. struct page *page;
  53. /*
  54. * The page lifetime is protected by the fact that we have opened the
  55. * device node -- brd pages will never be deleted under us, so we
  56. * don't need any further locking or refcounting.
  57. *
  58. * This is strictly true for the radix-tree nodes as well (ie. we
  59. * don't actually need the rcu_read_lock()), however that is not a
  60. * documented feature of the radix-tree API so it is better to be
  61. * safe here (we don't have total exclusion from radix tree updates
  62. * here, only deletes).
  63. */
  64. rcu_read_lock();
  65. idx = sector >> PAGE_SECTORS_SHIFT; /* sector to page index */
  66. page = radix_tree_lookup(&brd->brd_pages, idx);
  67. rcu_read_unlock();
  68. BUG_ON(page && page->index != idx);
  69. return page;
  70. }
  71. /*
  72. * Look up and return a brd's page for a given sector.
  73. * If one does not exist, allocate an empty page, and insert that. Then
  74. * return it.
  75. */
  76. static struct page *brd_insert_page(struct brd_device *brd, sector_t sector)
  77. {
  78. pgoff_t idx;
  79. struct page *page;
  80. gfp_t gfp_flags;
  81. page = brd_lookup_page(brd, sector);
  82. if (page)
  83. return page;
  84. /*
  85. * Must use NOIO because we don't want to recurse back into the
  86. * block or filesystem layers from page reclaim.
  87. */
  88. gfp_flags = GFP_NOIO | __GFP_ZERO | __GFP_HIGHMEM;
  89. page = alloc_page(gfp_flags);
  90. if (!page)
  91. return NULL;
  92. if (radix_tree_preload(GFP_NOIO)) {
  93. __free_page(page);
  94. return NULL;
  95. }
  96. spin_lock(&brd->brd_lock);
  97. idx = sector >> PAGE_SECTORS_SHIFT;
  98. page->index = idx;
  99. if (radix_tree_insert(&brd->brd_pages, idx, page)) {
  100. __free_page(page);
  101. page = radix_tree_lookup(&brd->brd_pages, idx);
  102. BUG_ON(!page);
  103. BUG_ON(page->index != idx);
  104. }
  105. spin_unlock(&brd->brd_lock);
  106. radix_tree_preload_end();
  107. return page;
  108. }
  109. /*
  110. * Free all backing store pages and radix tree. This must only be called when
  111. * there are no other users of the device.
  112. */
  113. #define FREE_BATCH 16
  114. static void brd_free_pages(struct brd_device *brd)
  115. {
  116. unsigned long pos = 0;
  117. struct page *pages[FREE_BATCH];
  118. int nr_pages;
  119. do {
  120. int i;
  121. nr_pages = radix_tree_gang_lookup(&brd->brd_pages,
  122. (void **)pages, pos, FREE_BATCH);
  123. for (i = 0; i < nr_pages; i++) {
  124. void *ret;
  125. BUG_ON(pages[i]->index < pos);
  126. pos = pages[i]->index;
  127. ret = radix_tree_delete(&brd->brd_pages, pos);
  128. BUG_ON(!ret || ret != pages[i]);
  129. __free_page(pages[i]);
  130. }
  131. pos++;
  132. /*
  133. * It takes 3.4 seconds to remove 80GiB ramdisk.
  134. * So, we need cond_resched to avoid stalling the CPU.
  135. */
  136. cond_resched();
  137. /*
  138. * This assumes radix_tree_gang_lookup always returns as
  139. * many pages as possible. If the radix-tree code changes,
  140. * so will this have to.
  141. */
  142. } while (nr_pages == FREE_BATCH);
  143. }
  144. /*
  145. * copy_to_brd_setup must be called before copy_to_brd. It may sleep.
  146. */
  147. static int copy_to_brd_setup(struct brd_device *brd, sector_t sector, size_t n)
  148. {
  149. unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
  150. size_t copy;
  151. copy = min_t(size_t, n, PAGE_SIZE - offset);
  152. if (!brd_insert_page(brd, sector))
  153. return -ENOSPC;
  154. if (copy < n) {
  155. sector += copy >> SECTOR_SHIFT;
  156. if (!brd_insert_page(brd, sector))
  157. return -ENOSPC;
  158. }
  159. return 0;
  160. }
  161. /*
  162. * Copy n bytes from src to the brd starting at sector. Does not sleep.
  163. */
  164. static void copy_to_brd(struct brd_device *brd, const void *src,
  165. sector_t sector, size_t n)
  166. {
  167. struct page *page;
  168. void *dst;
  169. unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
  170. size_t copy;
  171. copy = min_t(size_t, n, PAGE_SIZE - offset);
  172. page = brd_lookup_page(brd, sector);
  173. BUG_ON(!page);
  174. dst = kmap_atomic(page);
  175. memcpy(dst + offset, src, copy);
  176. kunmap_atomic(dst);
  177. if (copy < n) {
  178. src += copy;
  179. sector += copy >> SECTOR_SHIFT;
  180. copy = n - copy;
  181. page = brd_lookup_page(brd, sector);
  182. BUG_ON(!page);
  183. dst = kmap_atomic(page);
  184. memcpy(dst, src, copy);
  185. kunmap_atomic(dst);
  186. }
  187. }
  188. /*
  189. * Copy n bytes to dst from the brd starting at sector. Does not sleep.
  190. */
  191. static void copy_from_brd(void *dst, struct brd_device *brd,
  192. sector_t sector, size_t n)
  193. {
  194. struct page *page;
  195. void *src;
  196. unsigned int offset = (sector & (PAGE_SECTORS-1)) << SECTOR_SHIFT;
  197. size_t copy;
  198. copy = min_t(size_t, n, PAGE_SIZE - offset);
  199. page = brd_lookup_page(brd, sector);
  200. if (page) {
  201. src = kmap_atomic(page);
  202. memcpy(dst, src + offset, copy);
  203. kunmap_atomic(src);
  204. } else
  205. memset(dst, 0, copy);
  206. if (copy < n) {
  207. dst += copy;
  208. sector += copy >> SECTOR_SHIFT;
  209. copy = n - copy;
  210. page = brd_lookup_page(brd, sector);
  211. if (page) {
  212. src = kmap_atomic(page);
  213. memcpy(dst, src, copy);
  214. kunmap_atomic(src);
  215. } else
  216. memset(dst, 0, copy);
  217. }
  218. }
  219. /*
  220. * Process a single bvec of a bio.
  221. */
  222. static int brd_do_bvec(struct brd_device *brd, struct page *page,
  223. unsigned int len, unsigned int off, unsigned int op,
  224. sector_t sector)
  225. {
  226. void *mem;
  227. int err = 0;
  228. if (op_is_write(op)) {
  229. err = copy_to_brd_setup(brd, sector, len);
  230. if (err)
  231. goto out;
  232. }
  233. mem = kmap_atomic(page);
  234. if (!op_is_write(op)) {
  235. copy_from_brd(mem + off, brd, sector, len);
  236. flush_dcache_page(page);
  237. } else {
  238. flush_dcache_page(page);
  239. copy_to_brd(brd, mem + off, sector, len);
  240. }
  241. kunmap_atomic(mem);
  242. out:
  243. return err;
  244. }
  245. static blk_qc_t brd_submit_bio(struct bio *bio)
  246. {
  247. struct brd_device *brd = bio->bi_disk->private_data;
  248. struct bio_vec bvec;
  249. sector_t sector;
  250. struct bvec_iter iter;
  251. sector = bio->bi_iter.bi_sector;
  252. if (bio_end_sector(bio) > get_capacity(bio->bi_disk))
  253. goto io_error;
  254. bio_for_each_segment(bvec, bio, iter) {
  255. unsigned int len = bvec.bv_len;
  256. int err;
  257. /* Don't support un-aligned buffer */
  258. WARN_ON_ONCE((bvec.bv_offset & (SECTOR_SIZE - 1)) ||
  259. (len & (SECTOR_SIZE - 1)));
  260. err = brd_do_bvec(brd, bvec.bv_page, len, bvec.bv_offset,
  261. bio_op(bio), sector);
  262. if (err)
  263. goto io_error;
  264. sector += len >> SECTOR_SHIFT;
  265. }
  266. bio_endio(bio);
  267. return BLK_QC_T_NONE;
  268. io_error:
  269. bio_io_error(bio);
  270. return BLK_QC_T_NONE;
  271. }
  272. static int brd_rw_page(struct block_device *bdev, sector_t sector,
  273. struct page *page, unsigned int op)
  274. {
  275. struct brd_device *brd = bdev->bd_disk->private_data;
  276. int err;
  277. if (PageTransHuge(page))
  278. return -ENOTSUPP;
  279. err = brd_do_bvec(brd, page, PAGE_SIZE, 0, op, sector);
  280. page_endio(page, op_is_write(op), err);
  281. return err;
  282. }
  283. static const struct block_device_operations brd_fops = {
  284. .owner = THIS_MODULE,
  285. .submit_bio = brd_submit_bio,
  286. .rw_page = brd_rw_page,
  287. };
  288. /*
  289. * And now the modules code and kernel interface.
  290. */
  291. static int rd_nr = CONFIG_BLK_DEV_RAM_COUNT;
  292. module_param(rd_nr, int, 0444);
  293. MODULE_PARM_DESC(rd_nr, "Maximum number of brd devices");
  294. unsigned long rd_size = CONFIG_BLK_DEV_RAM_SIZE;
  295. module_param(rd_size, ulong, 0444);
  296. MODULE_PARM_DESC(rd_size, "Size of each RAM disk in kbytes.");
  297. static int max_part = 1;
  298. module_param(max_part, int, 0444);
  299. MODULE_PARM_DESC(max_part, "Num Minors to reserve between devices");
  300. MODULE_LICENSE("GPL");
  301. MODULE_ALIAS_BLOCKDEV_MAJOR(RAMDISK_MAJOR);
  302. MODULE_ALIAS("rd");
  303. #ifndef MODULE
  304. /* Legacy boot options - nonmodular */
  305. static int __init ramdisk_size(char *str)
  306. {
  307. rd_size = simple_strtol(str, NULL, 0);
  308. return 1;
  309. }
  310. __setup("ramdisk_size=", ramdisk_size);
  311. #endif
  312. /*
  313. * The device scheme is derived from loop.c. Keep them in synch where possible
  314. * (should share code eventually).
  315. */
  316. static LIST_HEAD(brd_devices);
  317. static DEFINE_MUTEX(brd_devices_mutex);
  318. static struct brd_device *brd_alloc(int i)
  319. {
  320. struct brd_device *brd;
  321. struct gendisk *disk;
  322. brd = kzalloc(sizeof(*brd), GFP_KERNEL);
  323. if (!brd)
  324. goto out;
  325. brd->brd_number = i;
  326. spin_lock_init(&brd->brd_lock);
  327. INIT_RADIX_TREE(&brd->brd_pages, GFP_ATOMIC);
  328. brd->brd_queue = blk_alloc_queue(NUMA_NO_NODE);
  329. if (!brd->brd_queue)
  330. goto out_free_dev;
  331. /* This is so fdisk will align partitions on 4k, because of
  332. * direct_access API needing 4k alignment, returning a PFN
  333. * (This is only a problem on very small devices <= 4M,
  334. * otherwise fdisk will align on 1M. Regardless this call
  335. * is harmless)
  336. */
  337. blk_queue_physical_block_size(brd->brd_queue, PAGE_SIZE);
  338. disk = brd->brd_disk = alloc_disk(max_part);
  339. if (!disk)
  340. goto out_free_queue;
  341. disk->major = RAMDISK_MAJOR;
  342. disk->first_minor = i * max_part;
  343. disk->fops = &brd_fops;
  344. disk->private_data = brd;
  345. disk->flags = GENHD_FL_EXT_DEVT;
  346. sprintf(disk->disk_name, "ram%d", i);
  347. set_capacity(disk, rd_size * 2);
  348. /* Tell the block layer that this is not a rotational device */
  349. blk_queue_flag_set(QUEUE_FLAG_NONROT, brd->brd_queue);
  350. blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, brd->brd_queue);
  351. return brd;
  352. out_free_queue:
  353. blk_cleanup_queue(brd->brd_queue);
  354. out_free_dev:
  355. kfree(brd);
  356. out:
  357. return NULL;
  358. }
  359. static void brd_free(struct brd_device *brd)
  360. {
  361. put_disk(brd->brd_disk);
  362. blk_cleanup_queue(brd->brd_queue);
  363. brd_free_pages(brd);
  364. kfree(brd);
  365. }
  366. static struct brd_device *brd_init_one(int i, bool *new)
  367. {
  368. struct brd_device *brd;
  369. *new = false;
  370. list_for_each_entry(brd, &brd_devices, brd_list) {
  371. if (brd->brd_number == i)
  372. goto out;
  373. }
  374. brd = brd_alloc(i);
  375. if (brd) {
  376. brd->brd_disk->queue = brd->brd_queue;
  377. add_disk(brd->brd_disk);
  378. list_add_tail(&brd->brd_list, &brd_devices);
  379. }
  380. *new = true;
  381. out:
  382. return brd;
  383. }
  384. static void brd_del_one(struct brd_device *brd)
  385. {
  386. list_del(&brd->brd_list);
  387. del_gendisk(brd->brd_disk);
  388. brd_free(brd);
  389. }
  390. static struct kobject *brd_probe(dev_t dev, int *part, void *data)
  391. {
  392. struct brd_device *brd;
  393. struct kobject *kobj;
  394. bool new;
  395. mutex_lock(&brd_devices_mutex);
  396. brd = brd_init_one(MINOR(dev) / max_part, &new);
  397. kobj = brd ? get_disk_and_module(brd->brd_disk) : NULL;
  398. mutex_unlock(&brd_devices_mutex);
  399. if (new)
  400. *part = 0;
  401. return kobj;
  402. }
  403. static inline void brd_check_and_reset_par(void)
  404. {
  405. if (unlikely(!max_part))
  406. max_part = 1;
  407. /*
  408. * make sure 'max_part' can be divided exactly by (1U << MINORBITS),
  409. * otherwise, it is possiable to get same dev_t when adding partitions.
  410. */
  411. if ((1U << MINORBITS) % max_part != 0)
  412. max_part = 1UL << fls(max_part);
  413. if (max_part > DISK_MAX_PARTS) {
  414. pr_info("brd: max_part can't be larger than %d, reset max_part = %d.\n",
  415. DISK_MAX_PARTS, DISK_MAX_PARTS);
  416. max_part = DISK_MAX_PARTS;
  417. }
  418. }
  419. static int __init brd_init(void)
  420. {
  421. struct brd_device *brd, *next;
  422. int i;
  423. /*
  424. * brd module now has a feature to instantiate underlying device
  425. * structure on-demand, provided that there is an access dev node.
  426. *
  427. * (1) if rd_nr is specified, create that many upfront. else
  428. * it defaults to CONFIG_BLK_DEV_RAM_COUNT
  429. * (2) User can further extend brd devices by create dev node themselves
  430. * and have kernel automatically instantiate actual device
  431. * on-demand. Example:
  432. * mknod /path/devnod_name b 1 X # 1 is the rd major
  433. * fdisk -l /path/devnod_name
  434. * If (X / max_part) was not already created it will be created
  435. * dynamically.
  436. */
  437. if (register_blkdev(RAMDISK_MAJOR, "ramdisk"))
  438. return -EIO;
  439. brd_check_and_reset_par();
  440. for (i = 0; i < rd_nr; i++) {
  441. brd = brd_alloc(i);
  442. if (!brd)
  443. goto out_free;
  444. list_add_tail(&brd->brd_list, &brd_devices);
  445. }
  446. /* point of no return */
  447. list_for_each_entry(brd, &brd_devices, brd_list) {
  448. /*
  449. * associate with queue just before adding disk for
  450. * avoiding to mess up failure path
  451. */
  452. brd->brd_disk->queue = brd->brd_queue;
  453. add_disk(brd->brd_disk);
  454. }
  455. blk_register_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS,
  456. THIS_MODULE, brd_probe, NULL, NULL);
  457. pr_info("brd: module loaded\n");
  458. return 0;
  459. out_free:
  460. list_for_each_entry_safe(brd, next, &brd_devices, brd_list) {
  461. list_del(&brd->brd_list);
  462. brd_free(brd);
  463. }
  464. unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
  465. pr_info("brd: module NOT loaded !!!\n");
  466. return -ENOMEM;
  467. }
  468. static void __exit brd_exit(void)
  469. {
  470. struct brd_device *brd, *next;
  471. list_for_each_entry_safe(brd, next, &brd_devices, brd_list)
  472. brd_del_one(brd);
  473. blk_unregister_region(MKDEV(RAMDISK_MAJOR, 0), 1UL << MINORBITS);
  474. unregister_blkdev(RAMDISK_MAJOR, "ramdisk");
  475. pr_info("brd: module unloaded\n");
  476. }
  477. module_init(brd_init);
  478. module_exit(brd_exit);