block_dev.c 54 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * linux/fs/block_dev.c
  4. *
  5. * Copyright (C) 1991, 1992 Linus Torvalds
  6. * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
  7. */
  8. #include <linux/init.h>
  9. #include <linux/mm.h>
  10. #include <linux/fcntl.h>
  11. #include <linux/slab.h>
  12. #include <linux/kmod.h>
  13. #include <linux/major.h>
  14. #include <linux/device_cgroup.h>
  15. #include <linux/highmem.h>
  16. #include <linux/blkdev.h>
  17. #include <linux/backing-dev.h>
  18. #include <linux/module.h>
  19. #include <linux/blkpg.h>
  20. #include <linux/magic.h>
  21. #include <linux/buffer_head.h>
  22. #include <linux/swap.h>
  23. #include <linux/pagevec.h>
  24. #include <linux/writeback.h>
  25. #include <linux/mpage.h>
  26. #include <linux/mount.h>
  27. #include <linux/pseudo_fs.h>
  28. #include <linux/uio.h>
  29. #include <linux/namei.h>
  30. #include <linux/log2.h>
  31. #include <linux/cleancache.h>
  32. #include <linux/task_io_accounting_ops.h>
  33. #include <linux/falloc.h>
  34. #include <linux/uaccess.h>
  35. #include <linux/suspend.h>
  36. #include "internal.h"
  37. struct bdev_inode {
  38. struct block_device bdev;
  39. struct inode vfs_inode;
  40. };
  41. static const struct address_space_operations def_blk_aops;
  42. static inline struct bdev_inode *BDEV_I(struct inode *inode)
  43. {
  44. return container_of(inode, struct bdev_inode, vfs_inode);
  45. }
  46. struct block_device *I_BDEV(struct inode *inode)
  47. {
  48. return &BDEV_I(inode)->bdev;
  49. }
  50. EXPORT_SYMBOL(I_BDEV);
  51. static void bdev_write_inode(struct block_device *bdev)
  52. {
  53. struct inode *inode = bdev->bd_inode;
  54. int ret;
  55. spin_lock(&inode->i_lock);
  56. while (inode->i_state & I_DIRTY) {
  57. spin_unlock(&inode->i_lock);
  58. ret = write_inode_now(inode, true);
  59. if (ret) {
  60. char name[BDEVNAME_SIZE];
  61. pr_warn_ratelimited("VFS: Dirty inode writeback failed "
  62. "for block device %s (err=%d).\n",
  63. bdevname(bdev, name), ret);
  64. }
  65. spin_lock(&inode->i_lock);
  66. }
  67. spin_unlock(&inode->i_lock);
  68. }
  69. /* Kill _all_ buffers and pagecache , dirty or not.. */
  70. static void kill_bdev(struct block_device *bdev)
  71. {
  72. struct address_space *mapping = bdev->bd_inode->i_mapping;
  73. if (mapping->nrpages == 0 && mapping->nrexceptional == 0)
  74. return;
  75. invalidate_bh_lrus();
  76. truncate_inode_pages(mapping, 0);
  77. }
  78. /* Invalidate clean unused buffers and pagecache. */
  79. void invalidate_bdev(struct block_device *bdev)
  80. {
  81. struct address_space *mapping = bdev->bd_inode->i_mapping;
  82. if (mapping->nrpages) {
  83. invalidate_bh_lrus();
  84. lru_add_drain_all(); /* make sure all lru add caches are flushed */
  85. invalidate_mapping_pages(mapping, 0, -1);
  86. }
  87. /* 99% of the time, we don't need to flush the cleancache on the bdev.
  88. * But, for the strange corners, lets be cautious
  89. */
  90. cleancache_invalidate_inode(mapping);
  91. }
  92. EXPORT_SYMBOL(invalidate_bdev);
  93. /*
  94. * Drop all buffers & page cache for given bdev range. This function bails
  95. * with error if bdev has other exclusive owner (such as filesystem).
  96. */
  97. int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
  98. loff_t lstart, loff_t lend)
  99. {
  100. struct block_device *claimed_bdev = NULL;
  101. int err;
  102. /*
  103. * If we don't hold exclusive handle for the device, upgrade to it
  104. * while we discard the buffer cache to avoid discarding buffers
  105. * under live filesystem.
  106. */
  107. if (!(mode & FMODE_EXCL)) {
  108. claimed_bdev = bdev->bd_contains;
  109. err = bd_prepare_to_claim(bdev, claimed_bdev,
  110. truncate_bdev_range);
  111. if (err)
  112. goto invalidate;
  113. }
  114. truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
  115. if (claimed_bdev)
  116. bd_abort_claiming(bdev, claimed_bdev, truncate_bdev_range);
  117. return 0;
  118. invalidate:
  119. /*
  120. * Someone else has handle exclusively open. Try invalidating instead.
  121. * The 'end' argument is inclusive so the rounding is safe.
  122. */
  123. return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
  124. lstart >> PAGE_SHIFT,
  125. lend >> PAGE_SHIFT);
  126. }
  127. EXPORT_SYMBOL(truncate_bdev_range);
  128. static void set_init_blocksize(struct block_device *bdev)
  129. {
  130. unsigned int bsize = bdev_logical_block_size(bdev);
  131. loff_t size = i_size_read(bdev->bd_inode);
  132. while (bsize < PAGE_SIZE) {
  133. if (size & bsize)
  134. break;
  135. bsize <<= 1;
  136. }
  137. bdev->bd_inode->i_blkbits = blksize_bits(bsize);
  138. }
  139. int set_blocksize(struct block_device *bdev, int size)
  140. {
  141. /* Size must be a power of two, and between 512 and PAGE_SIZE */
  142. if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
  143. return -EINVAL;
  144. /* Size cannot be smaller than the size supported by the device */
  145. if (size < bdev_logical_block_size(bdev))
  146. return -EINVAL;
  147. /* Don't change the size if it is same as current */
  148. if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {
  149. sync_blockdev(bdev);
  150. bdev->bd_inode->i_blkbits = blksize_bits(size);
  151. kill_bdev(bdev);
  152. }
  153. return 0;
  154. }
  155. EXPORT_SYMBOL(set_blocksize);
  156. int sb_set_blocksize(struct super_block *sb, int size)
  157. {
  158. if (set_blocksize(sb->s_bdev, size))
  159. return 0;
  160. /* If we get here, we know size is power of two
  161. * and it's value is between 512 and PAGE_SIZE */
  162. sb->s_blocksize = size;
  163. sb->s_blocksize_bits = blksize_bits(size);
  164. return sb->s_blocksize;
  165. }
  166. EXPORT_SYMBOL_NS(sb_set_blocksize, ANDROID_GKI_VFS_EXPORT_ONLY);
  167. int sb_min_blocksize(struct super_block *sb, int size)
  168. {
  169. int minsize = bdev_logical_block_size(sb->s_bdev);
  170. if (size < minsize)
  171. size = minsize;
  172. return sb_set_blocksize(sb, size);
  173. }
  174. EXPORT_SYMBOL_NS(sb_min_blocksize, ANDROID_GKI_VFS_EXPORT_ONLY);
  175. static int
  176. blkdev_get_block(struct inode *inode, sector_t iblock,
  177. struct buffer_head *bh, int create)
  178. {
  179. bh->b_bdev = I_BDEV(inode);
  180. bh->b_blocknr = iblock;
  181. set_buffer_mapped(bh);
  182. return 0;
  183. }
  184. static struct inode *bdev_file_inode(struct file *file)
  185. {
  186. return file->f_mapping->host;
  187. }
  188. static unsigned int dio_bio_write_op(struct kiocb *iocb)
  189. {
  190. unsigned int op = REQ_OP_WRITE | REQ_SYNC | REQ_IDLE;
  191. /* avoid the need for a I/O completion work item */
  192. if (iocb->ki_flags & IOCB_DSYNC)
  193. op |= REQ_FUA;
  194. return op;
  195. }
  196. #define DIO_INLINE_BIO_VECS 4
  197. static void blkdev_bio_end_io_simple(struct bio *bio)
  198. {
  199. struct task_struct *waiter = bio->bi_private;
  200. WRITE_ONCE(bio->bi_private, NULL);
  201. blk_wake_io_task(waiter);
  202. }
  203. static ssize_t
  204. __blkdev_direct_IO_simple(struct kiocb *iocb, struct iov_iter *iter,
  205. int nr_pages)
  206. {
  207. struct file *file = iocb->ki_filp;
  208. struct block_device *bdev = I_BDEV(bdev_file_inode(file));
  209. struct bio_vec inline_vecs[DIO_INLINE_BIO_VECS], *vecs;
  210. loff_t pos = iocb->ki_pos;
  211. bool should_dirty = false;
  212. struct bio bio;
  213. ssize_t ret;
  214. blk_qc_t qc;
  215. if ((pos | iov_iter_alignment(iter)) &
  216. (bdev_logical_block_size(bdev) - 1))
  217. return -EINVAL;
  218. if (nr_pages <= DIO_INLINE_BIO_VECS)
  219. vecs = inline_vecs;
  220. else {
  221. vecs = kmalloc_array(nr_pages, sizeof(struct bio_vec),
  222. GFP_KERNEL);
  223. if (!vecs)
  224. return -ENOMEM;
  225. }
  226. bio_init(&bio, vecs, nr_pages);
  227. bio_set_dev(&bio, bdev);
  228. bio.bi_iter.bi_sector = pos >> 9;
  229. bio.bi_write_hint = iocb->ki_hint;
  230. bio.bi_private = current;
  231. bio.bi_end_io = blkdev_bio_end_io_simple;
  232. bio.bi_ioprio = iocb->ki_ioprio;
  233. ret = bio_iov_iter_get_pages(&bio, iter);
  234. if (unlikely(ret))
  235. goto out;
  236. ret = bio.bi_iter.bi_size;
  237. if (iov_iter_rw(iter) == READ) {
  238. bio.bi_opf = REQ_OP_READ;
  239. if (iter_is_iovec(iter))
  240. should_dirty = true;
  241. } else {
  242. bio.bi_opf = dio_bio_write_op(iocb);
  243. task_io_account_write(ret);
  244. }
  245. if (iocb->ki_flags & IOCB_NOWAIT)
  246. bio.bi_opf |= REQ_NOWAIT;
  247. if (iocb->ki_flags & IOCB_HIPRI)
  248. bio_set_polled(&bio, iocb);
  249. qc = submit_bio(&bio);
  250. for (;;) {
  251. set_current_state(TASK_UNINTERRUPTIBLE);
  252. if (!READ_ONCE(bio.bi_private))
  253. break;
  254. if (!(iocb->ki_flags & IOCB_HIPRI) ||
  255. !blk_poll(bdev_get_queue(bdev), qc, true))
  256. blk_io_schedule();
  257. }
  258. __set_current_state(TASK_RUNNING);
  259. bio_release_pages(&bio, should_dirty);
  260. if (unlikely(bio.bi_status))
  261. ret = blk_status_to_errno(bio.bi_status);
  262. out:
  263. if (vecs != inline_vecs)
  264. kfree(vecs);
  265. bio_uninit(&bio);
  266. return ret;
  267. }
  268. struct blkdev_dio {
  269. union {
  270. struct kiocb *iocb;
  271. struct task_struct *waiter;
  272. };
  273. size_t size;
  274. atomic_t ref;
  275. bool multi_bio : 1;
  276. bool should_dirty : 1;
  277. bool is_sync : 1;
  278. struct bio bio;
  279. };
  280. static struct bio_set blkdev_dio_pool;
  281. static int blkdev_iopoll(struct kiocb *kiocb, bool wait)
  282. {
  283. struct block_device *bdev = I_BDEV(kiocb->ki_filp->f_mapping->host);
  284. struct request_queue *q = bdev_get_queue(bdev);
  285. return blk_poll(q, READ_ONCE(kiocb->ki_cookie), wait);
  286. }
  287. static void blkdev_bio_end_io(struct bio *bio)
  288. {
  289. struct blkdev_dio *dio = bio->bi_private;
  290. bool should_dirty = dio->should_dirty;
  291. if (bio->bi_status && !dio->bio.bi_status)
  292. dio->bio.bi_status = bio->bi_status;
  293. if (!dio->multi_bio || atomic_dec_and_test(&dio->ref)) {
  294. if (!dio->is_sync) {
  295. struct kiocb *iocb = dio->iocb;
  296. ssize_t ret;
  297. if (likely(!dio->bio.bi_status)) {
  298. ret = dio->size;
  299. iocb->ki_pos += ret;
  300. } else {
  301. ret = blk_status_to_errno(dio->bio.bi_status);
  302. }
  303. dio->iocb->ki_complete(iocb, ret, 0);
  304. if (dio->multi_bio)
  305. bio_put(&dio->bio);
  306. } else {
  307. struct task_struct *waiter = dio->waiter;
  308. WRITE_ONCE(dio->waiter, NULL);
  309. blk_wake_io_task(waiter);
  310. }
  311. }
  312. if (should_dirty) {
  313. bio_check_pages_dirty(bio);
  314. } else {
  315. bio_release_pages(bio, false);
  316. bio_put(bio);
  317. }
  318. }
  319. static ssize_t
  320. __blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter, int nr_pages)
  321. {
  322. struct file *file = iocb->ki_filp;
  323. struct inode *inode = bdev_file_inode(file);
  324. struct block_device *bdev = I_BDEV(inode);
  325. struct blk_plug plug;
  326. struct blkdev_dio *dio;
  327. struct bio *bio;
  328. bool is_poll = (iocb->ki_flags & IOCB_HIPRI) != 0;
  329. bool is_read = (iov_iter_rw(iter) == READ), is_sync;
  330. loff_t pos = iocb->ki_pos;
  331. blk_qc_t qc = BLK_QC_T_NONE;
  332. int ret = 0;
  333. if ((pos | iov_iter_alignment(iter)) &
  334. (bdev_logical_block_size(bdev) - 1))
  335. return -EINVAL;
  336. bio = bio_alloc_bioset(GFP_KERNEL, nr_pages, &blkdev_dio_pool);
  337. dio = container_of(bio, struct blkdev_dio, bio);
  338. dio->is_sync = is_sync = is_sync_kiocb(iocb);
  339. if (dio->is_sync) {
  340. dio->waiter = current;
  341. bio_get(bio);
  342. } else {
  343. dio->iocb = iocb;
  344. }
  345. dio->size = 0;
  346. dio->multi_bio = false;
  347. dio->should_dirty = is_read && iter_is_iovec(iter);
  348. /*
  349. * Don't plug for HIPRI/polled IO, as those should go straight
  350. * to issue
  351. */
  352. if (!is_poll)
  353. blk_start_plug(&plug);
  354. for (;;) {
  355. bio_set_dev(bio, bdev);
  356. bio->bi_iter.bi_sector = pos >> 9;
  357. bio->bi_write_hint = iocb->ki_hint;
  358. bio->bi_private = dio;
  359. bio->bi_end_io = blkdev_bio_end_io;
  360. bio->bi_ioprio = iocb->ki_ioprio;
  361. ret = bio_iov_iter_get_pages(bio, iter);
  362. if (unlikely(ret)) {
  363. bio->bi_status = BLK_STS_IOERR;
  364. bio_endio(bio);
  365. break;
  366. }
  367. if (is_read) {
  368. bio->bi_opf = REQ_OP_READ;
  369. if (dio->should_dirty)
  370. bio_set_pages_dirty(bio);
  371. } else {
  372. bio->bi_opf = dio_bio_write_op(iocb);
  373. task_io_account_write(bio->bi_iter.bi_size);
  374. }
  375. if (iocb->ki_flags & IOCB_NOWAIT)
  376. bio->bi_opf |= REQ_NOWAIT;
  377. dio->size += bio->bi_iter.bi_size;
  378. pos += bio->bi_iter.bi_size;
  379. nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES);
  380. if (!nr_pages) {
  381. bool polled = false;
  382. if (iocb->ki_flags & IOCB_HIPRI) {
  383. bio_set_polled(bio, iocb);
  384. polled = true;
  385. }
  386. qc = submit_bio(bio);
  387. if (polled)
  388. WRITE_ONCE(iocb->ki_cookie, qc);
  389. break;
  390. }
  391. if (!dio->multi_bio) {
  392. /*
  393. * AIO needs an extra reference to ensure the dio
  394. * structure which is embedded into the first bio
  395. * stays around.
  396. */
  397. if (!is_sync)
  398. bio_get(bio);
  399. dio->multi_bio = true;
  400. atomic_set(&dio->ref, 2);
  401. } else {
  402. atomic_inc(&dio->ref);
  403. }
  404. submit_bio(bio);
  405. bio = bio_alloc(GFP_KERNEL, nr_pages);
  406. }
  407. if (!is_poll)
  408. blk_finish_plug(&plug);
  409. if (!is_sync)
  410. return -EIOCBQUEUED;
  411. for (;;) {
  412. set_current_state(TASK_UNINTERRUPTIBLE);
  413. if (!READ_ONCE(dio->waiter))
  414. break;
  415. if (!(iocb->ki_flags & IOCB_HIPRI) ||
  416. !blk_poll(bdev_get_queue(bdev), qc, true))
  417. blk_io_schedule();
  418. }
  419. __set_current_state(TASK_RUNNING);
  420. if (!ret)
  421. ret = blk_status_to_errno(dio->bio.bi_status);
  422. if (likely(!ret))
  423. ret = dio->size;
  424. bio_put(&dio->bio);
  425. return ret;
  426. }
  427. static ssize_t
  428. blkdev_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  429. {
  430. int nr_pages;
  431. nr_pages = iov_iter_npages(iter, BIO_MAX_PAGES + 1);
  432. if (!nr_pages)
  433. return 0;
  434. if (is_sync_kiocb(iocb) && nr_pages <= BIO_MAX_PAGES)
  435. return __blkdev_direct_IO_simple(iocb, iter, nr_pages);
  436. return __blkdev_direct_IO(iocb, iter, min(nr_pages, BIO_MAX_PAGES));
  437. }
  438. static __init int blkdev_init(void)
  439. {
  440. return bioset_init(&blkdev_dio_pool, 4, offsetof(struct blkdev_dio, bio), BIOSET_NEED_BVECS);
  441. }
  442. module_init(blkdev_init);
  443. int __sync_blockdev(struct block_device *bdev, int wait)
  444. {
  445. if (!bdev)
  446. return 0;
  447. if (!wait)
  448. return filemap_flush(bdev->bd_inode->i_mapping);
  449. return filemap_write_and_wait(bdev->bd_inode->i_mapping);
  450. }
  451. /*
  452. * Write out and wait upon all the dirty data associated with a block
  453. * device via its mapping. Does not take the superblock lock.
  454. */
  455. int sync_blockdev(struct block_device *bdev)
  456. {
  457. return __sync_blockdev(bdev, 1);
  458. }
  459. EXPORT_SYMBOL(sync_blockdev);
  460. /*
  461. * Write out and wait upon all dirty data associated with this
  462. * device. Filesystem data as well as the underlying block
  463. * device. Takes the superblock lock.
  464. */
  465. int fsync_bdev(struct block_device *bdev)
  466. {
  467. struct super_block *sb = get_super(bdev);
  468. if (sb) {
  469. int res = sync_filesystem(sb);
  470. drop_super(sb);
  471. return res;
  472. }
  473. return sync_blockdev(bdev);
  474. }
  475. EXPORT_SYMBOL(fsync_bdev);
  476. /**
  477. * freeze_bdev -- lock a filesystem and force it into a consistent state
  478. * @bdev: blockdevice to lock
  479. *
  480. * If a superblock is found on this device, we take the s_umount semaphore
  481. * on it to make sure nobody unmounts until the snapshot creation is done.
  482. * The reference counter (bd_fsfreeze_count) guarantees that only the last
  483. * unfreeze process can unfreeze the frozen filesystem actually when multiple
  484. * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
  485. * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
  486. * actually.
  487. */
  488. int freeze_bdev(struct block_device *bdev)
  489. {
  490. struct super_block *sb;
  491. int error = 0;
  492. mutex_lock(&bdev->bd_fsfreeze_mutex);
  493. if (++bdev->bd_fsfreeze_count > 1)
  494. goto done;
  495. sb = get_active_super(bdev);
  496. if (!sb)
  497. goto sync;
  498. if (sb->s_op->freeze_super)
  499. error = sb->s_op->freeze_super(sb);
  500. else
  501. error = freeze_super(sb);
  502. deactivate_super(sb);
  503. if (error) {
  504. bdev->bd_fsfreeze_count--;
  505. goto done;
  506. }
  507. bdev->bd_fsfreeze_sb = sb;
  508. sync:
  509. sync_blockdev(bdev);
  510. done:
  511. mutex_unlock(&bdev->bd_fsfreeze_mutex);
  512. return error;
  513. }
  514. EXPORT_SYMBOL(freeze_bdev);
  515. /**
  516. * thaw_bdev -- unlock filesystem
  517. * @bdev: blockdevice to unlock
  518. *
  519. * Unlocks the filesystem and marks it writeable again after freeze_bdev().
  520. */
  521. int thaw_bdev(struct block_device *bdev)
  522. {
  523. struct super_block *sb;
  524. int error = -EINVAL;
  525. mutex_lock(&bdev->bd_fsfreeze_mutex);
  526. if (!bdev->bd_fsfreeze_count)
  527. goto out;
  528. error = 0;
  529. if (--bdev->bd_fsfreeze_count > 0)
  530. goto out;
  531. sb = bdev->bd_fsfreeze_sb;
  532. if (!sb)
  533. goto out;
  534. if (sb->s_op->thaw_super)
  535. error = sb->s_op->thaw_super(sb);
  536. else
  537. error = thaw_super(sb);
  538. if (error)
  539. bdev->bd_fsfreeze_count++;
  540. out:
  541. mutex_unlock(&bdev->bd_fsfreeze_mutex);
  542. return error;
  543. }
  544. EXPORT_SYMBOL(thaw_bdev);
  545. static int blkdev_writepage(struct page *page, struct writeback_control *wbc)
  546. {
  547. return block_write_full_page(page, blkdev_get_block, wbc);
  548. }
  549. static int blkdev_readpage(struct file * file, struct page * page)
  550. {
  551. return block_read_full_page(page, blkdev_get_block);
  552. }
  553. static void blkdev_readahead(struct readahead_control *rac)
  554. {
  555. mpage_readahead(rac, blkdev_get_block);
  556. }
  557. static int blkdev_write_begin(struct file *file, struct address_space *mapping,
  558. loff_t pos, unsigned len, unsigned flags,
  559. struct page **pagep, void **fsdata)
  560. {
  561. return block_write_begin(mapping, pos, len, flags, pagep,
  562. blkdev_get_block);
  563. }
  564. static int blkdev_write_end(struct file *file, struct address_space *mapping,
  565. loff_t pos, unsigned len, unsigned copied,
  566. struct page *page, void *fsdata)
  567. {
  568. int ret;
  569. ret = block_write_end(file, mapping, pos, len, copied, page, fsdata);
  570. unlock_page(page);
  571. put_page(page);
  572. return ret;
  573. }
  574. /*
  575. * private llseek:
  576. * for a block special file file_inode(file)->i_size is zero
  577. * so we compute the size by hand (just as in block_read/write above)
  578. */
  579. static loff_t block_llseek(struct file *file, loff_t offset, int whence)
  580. {
  581. struct inode *bd_inode = bdev_file_inode(file);
  582. loff_t retval;
  583. inode_lock(bd_inode);
  584. retval = fixed_size_llseek(file, offset, whence, i_size_read(bd_inode));
  585. inode_unlock(bd_inode);
  586. return retval;
  587. }
  588. int blkdev_fsync(struct file *filp, loff_t start, loff_t end, int datasync)
  589. {
  590. struct inode *bd_inode = bdev_file_inode(filp);
  591. struct block_device *bdev = I_BDEV(bd_inode);
  592. int error;
  593. error = file_write_and_wait_range(filp, start, end);
  594. if (error)
  595. return error;
  596. /*
  597. * There is no need to serialise calls to blkdev_issue_flush with
  598. * i_mutex and doing so causes performance issues with concurrent
  599. * O_SYNC writers to a block device.
  600. */
  601. error = blkdev_issue_flush(bdev, GFP_KERNEL);
  602. if (error == -EOPNOTSUPP)
  603. error = 0;
  604. return error;
  605. }
  606. EXPORT_SYMBOL(blkdev_fsync);
  607. /**
  608. * bdev_read_page() - Start reading a page from a block device
  609. * @bdev: The device to read the page from
  610. * @sector: The offset on the device to read the page to (need not be aligned)
  611. * @page: The page to read
  612. *
  613. * On entry, the page should be locked. It will be unlocked when the page
  614. * has been read. If the block driver implements rw_page synchronously,
  615. * that will be true on exit from this function, but it need not be.
  616. *
  617. * Errors returned by this function are usually "soft", eg out of memory, or
  618. * queue full; callers should try a different route to read this page rather
  619. * than propagate an error back up the stack.
  620. *
  621. * Return: negative errno if an error occurs, 0 if submission was successful.
  622. */
  623. int bdev_read_page(struct block_device *bdev, sector_t sector,
  624. struct page *page)
  625. {
  626. const struct block_device_operations *ops = bdev->bd_disk->fops;
  627. int result = -EOPNOTSUPP;
  628. if (!ops->rw_page || bdev_get_integrity(bdev))
  629. return result;
  630. result = blk_queue_enter(bdev->bd_disk->queue, 0);
  631. if (result)
  632. return result;
  633. result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
  634. REQ_OP_READ);
  635. blk_queue_exit(bdev->bd_disk->queue);
  636. return result;
  637. }
  638. /**
  639. * bdev_write_page() - Start writing a page to a block device
  640. * @bdev: The device to write the page to
  641. * @sector: The offset on the device to write the page to (need not be aligned)
  642. * @page: The page to write
  643. * @wbc: The writeback_control for the write
  644. *
  645. * On entry, the page should be locked and not currently under writeback.
  646. * On exit, if the write started successfully, the page will be unlocked and
  647. * under writeback. If the write failed already (eg the driver failed to
  648. * queue the page to the device), the page will still be locked. If the
  649. * caller is a ->writepage implementation, it will need to unlock the page.
  650. *
  651. * Errors returned by this function are usually "soft", eg out of memory, or
  652. * queue full; callers should try a different route to write this page rather
  653. * than propagate an error back up the stack.
  654. *
  655. * Return: negative errno if an error occurs, 0 if submission was successful.
  656. */
  657. int bdev_write_page(struct block_device *bdev, sector_t sector,
  658. struct page *page, struct writeback_control *wbc)
  659. {
  660. int result;
  661. const struct block_device_operations *ops = bdev->bd_disk->fops;
  662. if (!ops->rw_page || bdev_get_integrity(bdev))
  663. return -EOPNOTSUPP;
  664. result = blk_queue_enter(bdev->bd_disk->queue, 0);
  665. if (result)
  666. return result;
  667. set_page_writeback(page);
  668. result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
  669. REQ_OP_WRITE);
  670. if (result) {
  671. end_page_writeback(page);
  672. } else {
  673. clean_page_buffers(page);
  674. unlock_page(page);
  675. }
  676. blk_queue_exit(bdev->bd_disk->queue);
  677. return result;
  678. }
  679. /*
  680. * pseudo-fs
  681. */
  682. static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
  683. static struct kmem_cache * bdev_cachep __read_mostly;
  684. static struct inode *bdev_alloc_inode(struct super_block *sb)
  685. {
  686. struct bdev_inode *ei = kmem_cache_alloc(bdev_cachep, GFP_KERNEL);
  687. if (!ei)
  688. return NULL;
  689. return &ei->vfs_inode;
  690. }
  691. static void bdev_free_inode(struct inode *inode)
  692. {
  693. kmem_cache_free(bdev_cachep, BDEV_I(inode));
  694. }
  695. static void init_once(void *foo)
  696. {
  697. struct bdev_inode *ei = (struct bdev_inode *) foo;
  698. struct block_device *bdev = &ei->bdev;
  699. memset(bdev, 0, sizeof(*bdev));
  700. mutex_init(&bdev->bd_mutex);
  701. #ifdef CONFIG_SYSFS
  702. INIT_LIST_HEAD(&bdev->bd_holder_disks);
  703. #endif
  704. bdev->bd_bdi = &noop_backing_dev_info;
  705. inode_init_once(&ei->vfs_inode);
  706. /* Initialize mutex for freeze. */
  707. mutex_init(&bdev->bd_fsfreeze_mutex);
  708. }
  709. static void bdev_evict_inode(struct inode *inode)
  710. {
  711. struct block_device *bdev = &BDEV_I(inode)->bdev;
  712. truncate_inode_pages_final(&inode->i_data);
  713. invalidate_inode_buffers(inode); /* is it needed here? */
  714. clear_inode(inode);
  715. /* Detach inode from wb early as bdi_put() may free bdi->wb */
  716. inode_detach_wb(inode);
  717. if (bdev->bd_bdi != &noop_backing_dev_info) {
  718. bdi_put(bdev->bd_bdi);
  719. bdev->bd_bdi = &noop_backing_dev_info;
  720. }
  721. }
  722. static const struct super_operations bdev_sops = {
  723. .statfs = simple_statfs,
  724. .alloc_inode = bdev_alloc_inode,
  725. .free_inode = bdev_free_inode,
  726. .drop_inode = generic_delete_inode,
  727. .evict_inode = bdev_evict_inode,
  728. };
  729. static int bd_init_fs_context(struct fs_context *fc)
  730. {
  731. struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
  732. if (!ctx)
  733. return -ENOMEM;
  734. fc->s_iflags |= SB_I_CGROUPWB;
  735. ctx->ops = &bdev_sops;
  736. return 0;
  737. }
  738. static struct file_system_type bd_type = {
  739. .name = "bdev",
  740. .init_fs_context = bd_init_fs_context,
  741. .kill_sb = kill_anon_super,
  742. };
  743. struct super_block *blockdev_superblock __read_mostly;
  744. EXPORT_SYMBOL_GPL(blockdev_superblock);
  745. void __init bdev_cache_init(void)
  746. {
  747. int err;
  748. static struct vfsmount *bd_mnt;
  749. bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
  750. 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
  751. SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
  752. init_once);
  753. err = register_filesystem(&bd_type);
  754. if (err)
  755. panic("Cannot register bdev pseudo-fs");
  756. bd_mnt = kern_mount(&bd_type);
  757. if (IS_ERR(bd_mnt))
  758. panic("Cannot create bdev pseudo-fs");
  759. blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */
  760. }
  761. /*
  762. * Most likely _very_ bad one - but then it's hardly critical for small
  763. * /dev and can be fixed when somebody will need really large one.
  764. * Keep in mind that it will be fed through icache hash function too.
  765. */
  766. static inline unsigned long hash(dev_t dev)
  767. {
  768. return MAJOR(dev)+MINOR(dev);
  769. }
  770. static int bdev_test(struct inode *inode, void *data)
  771. {
  772. return BDEV_I(inode)->bdev.bd_dev == *(dev_t *)data;
  773. }
  774. static int bdev_set(struct inode *inode, void *data)
  775. {
  776. BDEV_I(inode)->bdev.bd_dev = *(dev_t *)data;
  777. return 0;
  778. }
  779. static struct block_device *bdget(dev_t dev)
  780. {
  781. struct block_device *bdev;
  782. struct inode *inode;
  783. inode = iget5_locked(blockdev_superblock, hash(dev),
  784. bdev_test, bdev_set, &dev);
  785. if (!inode)
  786. return NULL;
  787. bdev = &BDEV_I(inode)->bdev;
  788. if (inode->i_state & I_NEW) {
  789. spin_lock_init(&bdev->bd_size_lock);
  790. bdev->bd_contains = NULL;
  791. bdev->bd_super = NULL;
  792. bdev->bd_inode = inode;
  793. bdev->bd_part_count = 0;
  794. inode->i_mode = S_IFBLK;
  795. inode->i_rdev = dev;
  796. inode->i_bdev = bdev;
  797. inode->i_data.a_ops = &def_blk_aops;
  798. mapping_set_gfp_mask(&inode->i_data, GFP_USER);
  799. unlock_new_inode(inode);
  800. }
  801. return bdev;
  802. }
  803. /**
  804. * bdgrab -- Grab a reference to an already referenced block device
  805. * @bdev: Block device to grab a reference to.
  806. */
  807. struct block_device *bdgrab(struct block_device *bdev)
  808. {
  809. ihold(bdev->bd_inode);
  810. return bdev;
  811. }
  812. EXPORT_SYMBOL(bdgrab);
  813. struct block_device *bdget_part(struct hd_struct *part)
  814. {
  815. return bdget(part_devt(part));
  816. }
  817. long nr_blockdev_pages(void)
  818. {
  819. struct inode *inode;
  820. long ret = 0;
  821. spin_lock(&blockdev_superblock->s_inode_list_lock);
  822. list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
  823. ret += inode->i_mapping->nrpages;
  824. spin_unlock(&blockdev_superblock->s_inode_list_lock);
  825. return ret;
  826. }
  827. void bdput(struct block_device *bdev)
  828. {
  829. iput(bdev->bd_inode);
  830. }
  831. EXPORT_SYMBOL(bdput);
  832. static struct block_device *bd_acquire(struct inode *inode)
  833. {
  834. struct block_device *bdev;
  835. spin_lock(&bdev_lock);
  836. bdev = inode->i_bdev;
  837. if (bdev && !inode_unhashed(bdev->bd_inode)) {
  838. bdgrab(bdev);
  839. spin_unlock(&bdev_lock);
  840. return bdev;
  841. }
  842. spin_unlock(&bdev_lock);
  843. /*
  844. * i_bdev references block device inode that was already shut down
  845. * (corresponding device got removed). Remove the reference and look
  846. * up block device inode again just in case new device got
  847. * reestablished under the same device number.
  848. */
  849. if (bdev)
  850. bd_forget(inode);
  851. bdev = bdget(inode->i_rdev);
  852. if (bdev) {
  853. spin_lock(&bdev_lock);
  854. if (!inode->i_bdev) {
  855. /*
  856. * We take an additional reference to bd_inode,
  857. * and it's released in clear_inode() of inode.
  858. * So, we can access it via ->i_mapping always
  859. * without igrab().
  860. */
  861. bdgrab(bdev);
  862. inode->i_bdev = bdev;
  863. inode->i_mapping = bdev->bd_inode->i_mapping;
  864. }
  865. spin_unlock(&bdev_lock);
  866. }
  867. return bdev;
  868. }
  869. /* Call when you free inode */
  870. void bd_forget(struct inode *inode)
  871. {
  872. struct block_device *bdev = NULL;
  873. spin_lock(&bdev_lock);
  874. if (!sb_is_blkdev_sb(inode->i_sb))
  875. bdev = inode->i_bdev;
  876. inode->i_bdev = NULL;
  877. inode->i_mapping = &inode->i_data;
  878. spin_unlock(&bdev_lock);
  879. if (bdev)
  880. bdput(bdev);
  881. }
  882. /**
  883. * bd_may_claim - test whether a block device can be claimed
  884. * @bdev: block device of interest
  885. * @whole: whole block device containing @bdev, may equal @bdev
  886. * @holder: holder trying to claim @bdev
  887. *
  888. * Test whether @bdev can be claimed by @holder.
  889. *
  890. * CONTEXT:
  891. * spin_lock(&bdev_lock).
  892. *
  893. * RETURNS:
  894. * %true if @bdev can be claimed, %false otherwise.
  895. */
  896. static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
  897. void *holder)
  898. {
  899. if (bdev->bd_holder == holder)
  900. return true; /* already a holder */
  901. else if (bdev->bd_holder != NULL)
  902. return false; /* held by someone else */
  903. else if (whole == bdev)
  904. return true; /* is a whole device which isn't held */
  905. else if (whole->bd_holder == bd_may_claim)
  906. return true; /* is a partition of a device that is being partitioned */
  907. else if (whole->bd_holder != NULL)
  908. return false; /* is a partition of a held device */
  909. else
  910. return true; /* is a partition of an un-held device */
  911. }
  912. /**
  913. * bd_prepare_to_claim - claim a block device
  914. * @bdev: block device of interest
  915. * @whole: the whole device containing @bdev, may equal @bdev
  916. * @holder: holder trying to claim @bdev
  917. *
  918. * Claim @bdev. This function fails if @bdev is already claimed by another
  919. * holder and waits if another claiming is in progress. return, the caller
  920. * has ownership of bd_claiming and bd_holder[s].
  921. *
  922. * RETURNS:
  923. * 0 if @bdev can be claimed, -EBUSY otherwise.
  924. */
  925. int bd_prepare_to_claim(struct block_device *bdev, struct block_device *whole,
  926. void *holder)
  927. {
  928. retry:
  929. spin_lock(&bdev_lock);
  930. /* if someone else claimed, fail */
  931. if (!bd_may_claim(bdev, whole, holder)) {
  932. spin_unlock(&bdev_lock);
  933. return -EBUSY;
  934. }
  935. /* if claiming is already in progress, wait for it to finish */
  936. if (whole->bd_claiming) {
  937. wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
  938. DEFINE_WAIT(wait);
  939. prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
  940. spin_unlock(&bdev_lock);
  941. schedule();
  942. finish_wait(wq, &wait);
  943. goto retry;
  944. }
  945. /* yay, all mine */
  946. whole->bd_claiming = holder;
  947. spin_unlock(&bdev_lock);
  948. return 0;
  949. }
  950. EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
  951. static struct gendisk *bdev_get_gendisk(struct block_device *bdev, int *partno)
  952. {
  953. struct gendisk *disk = get_gendisk(bdev->bd_dev, partno);
  954. if (!disk)
  955. return NULL;
  956. /*
  957. * Now that we hold gendisk reference we make sure bdev we looked up is
  958. * not stale. If it is, it means device got removed and created before
  959. * we looked up gendisk and we fail open in such case. Associating
  960. * unhashed bdev with newly created gendisk could lead to two bdevs
  961. * (and thus two independent caches) being associated with one device
  962. * which is bad.
  963. */
  964. if (inode_unhashed(bdev->bd_inode)) {
  965. put_disk_and_module(disk);
  966. return NULL;
  967. }
  968. return disk;
  969. }
  970. static void bd_clear_claiming(struct block_device *whole, void *holder)
  971. {
  972. lockdep_assert_held(&bdev_lock);
  973. /* tell others that we're done */
  974. BUG_ON(whole->bd_claiming != holder);
  975. whole->bd_claiming = NULL;
  976. wake_up_bit(&whole->bd_claiming, 0);
  977. }
  978. /**
  979. * bd_finish_claiming - finish claiming of a block device
  980. * @bdev: block device of interest
  981. * @whole: whole block device
  982. * @holder: holder that has claimed @bdev
  983. *
  984. * Finish exclusive open of a block device. Mark the device as exlusively
  985. * open by the holder and wake up all waiters for exclusive open to finish.
  986. */
  987. static void bd_finish_claiming(struct block_device *bdev,
  988. struct block_device *whole, void *holder)
  989. {
  990. spin_lock(&bdev_lock);
  991. BUG_ON(!bd_may_claim(bdev, whole, holder));
  992. /*
  993. * Note that for a whole device bd_holders will be incremented twice,
  994. * and bd_holder will be set to bd_may_claim before being set to holder
  995. */
  996. whole->bd_holders++;
  997. whole->bd_holder = bd_may_claim;
  998. bdev->bd_holders++;
  999. bdev->bd_holder = holder;
  1000. bd_clear_claiming(whole, holder);
  1001. spin_unlock(&bdev_lock);
  1002. }
  1003. /**
  1004. * bd_abort_claiming - abort claiming of a block device
  1005. * @bdev: block device of interest
  1006. * @whole: whole block device
  1007. * @holder: holder that has claimed @bdev
  1008. *
  1009. * Abort claiming of a block device when the exclusive open failed. This can be
  1010. * also used when exclusive open is not actually desired and we just needed
  1011. * to block other exclusive openers for a while.
  1012. */
  1013. void bd_abort_claiming(struct block_device *bdev, struct block_device *whole,
  1014. void *holder)
  1015. {
  1016. spin_lock(&bdev_lock);
  1017. bd_clear_claiming(whole, holder);
  1018. spin_unlock(&bdev_lock);
  1019. }
  1020. EXPORT_SYMBOL(bd_abort_claiming);
  1021. #ifdef CONFIG_SYSFS
  1022. struct bd_holder_disk {
  1023. struct list_head list;
  1024. struct gendisk *disk;
  1025. int refcnt;
  1026. };
  1027. static struct bd_holder_disk *bd_find_holder_disk(struct block_device *bdev,
  1028. struct gendisk *disk)
  1029. {
  1030. struct bd_holder_disk *holder;
  1031. list_for_each_entry(holder, &bdev->bd_holder_disks, list)
  1032. if (holder->disk == disk)
  1033. return holder;
  1034. return NULL;
  1035. }
  1036. static int add_symlink(struct kobject *from, struct kobject *to)
  1037. {
  1038. return sysfs_create_link(from, to, kobject_name(to));
  1039. }
  1040. static void del_symlink(struct kobject *from, struct kobject *to)
  1041. {
  1042. sysfs_remove_link(from, kobject_name(to));
  1043. }
  1044. /**
  1045. * bd_link_disk_holder - create symlinks between holding disk and slave bdev
  1046. * @bdev: the claimed slave bdev
  1047. * @disk: the holding disk
  1048. *
  1049. * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
  1050. *
  1051. * This functions creates the following sysfs symlinks.
  1052. *
  1053. * - from "slaves" directory of the holder @disk to the claimed @bdev
  1054. * - from "holders" directory of the @bdev to the holder @disk
  1055. *
  1056. * For example, if /dev/dm-0 maps to /dev/sda and disk for dm-0 is
  1057. * passed to bd_link_disk_holder(), then:
  1058. *
  1059. * /sys/block/dm-0/slaves/sda --> /sys/block/sda
  1060. * /sys/block/sda/holders/dm-0 --> /sys/block/dm-0
  1061. *
  1062. * The caller must have claimed @bdev before calling this function and
  1063. * ensure that both @bdev and @disk are valid during the creation and
  1064. * lifetime of these symlinks.
  1065. *
  1066. * CONTEXT:
  1067. * Might sleep.
  1068. *
  1069. * RETURNS:
  1070. * 0 on success, -errno on failure.
  1071. */
  1072. int bd_link_disk_holder(struct block_device *bdev, struct gendisk *disk)
  1073. {
  1074. struct bd_holder_disk *holder;
  1075. int ret = 0;
  1076. mutex_lock(&bdev->bd_mutex);
  1077. WARN_ON_ONCE(!bdev->bd_holder);
  1078. /* FIXME: remove the following once add_disk() handles errors */
  1079. if (WARN_ON(!disk->slave_dir || !bdev->bd_part->holder_dir))
  1080. goto out_unlock;
  1081. holder = bd_find_holder_disk(bdev, disk);
  1082. if (holder) {
  1083. holder->refcnt++;
  1084. goto out_unlock;
  1085. }
  1086. holder = kzalloc(sizeof(*holder), GFP_KERNEL);
  1087. if (!holder) {
  1088. ret = -ENOMEM;
  1089. goto out_unlock;
  1090. }
  1091. INIT_LIST_HEAD(&holder->list);
  1092. holder->disk = disk;
  1093. holder->refcnt = 1;
  1094. ret = add_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
  1095. if (ret)
  1096. goto out_free;
  1097. ret = add_symlink(bdev->bd_part->holder_dir, &disk_to_dev(disk)->kobj);
  1098. if (ret)
  1099. goto out_del;
  1100. /*
  1101. * bdev could be deleted beneath us which would implicitly destroy
  1102. * the holder directory. Hold on to it.
  1103. */
  1104. kobject_get(bdev->bd_part->holder_dir);
  1105. list_add(&holder->list, &bdev->bd_holder_disks);
  1106. goto out_unlock;
  1107. out_del:
  1108. del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
  1109. out_free:
  1110. kfree(holder);
  1111. out_unlock:
  1112. mutex_unlock(&bdev->bd_mutex);
  1113. return ret;
  1114. }
  1115. EXPORT_SYMBOL_GPL(bd_link_disk_holder);
  1116. /**
  1117. * bd_unlink_disk_holder - destroy symlinks created by bd_link_disk_holder()
  1118. * @bdev: the calimed slave bdev
  1119. * @disk: the holding disk
  1120. *
  1121. * DON'T USE THIS UNLESS YOU'RE ALREADY USING IT.
  1122. *
  1123. * CONTEXT:
  1124. * Might sleep.
  1125. */
  1126. void bd_unlink_disk_holder(struct block_device *bdev, struct gendisk *disk)
  1127. {
  1128. struct bd_holder_disk *holder;
  1129. mutex_lock(&bdev->bd_mutex);
  1130. holder = bd_find_holder_disk(bdev, disk);
  1131. if (!WARN_ON_ONCE(holder == NULL) && !--holder->refcnt) {
  1132. del_symlink(disk->slave_dir, &part_to_dev(bdev->bd_part)->kobj);
  1133. del_symlink(bdev->bd_part->holder_dir,
  1134. &disk_to_dev(disk)->kobj);
  1135. kobject_put(bdev->bd_part->holder_dir);
  1136. list_del_init(&holder->list);
  1137. kfree(holder);
  1138. }
  1139. mutex_unlock(&bdev->bd_mutex);
  1140. }
  1141. EXPORT_SYMBOL_GPL(bd_unlink_disk_holder);
  1142. #endif
  1143. /**
  1144. * check_disk_size_change - checks for disk size change and adjusts bdev size.
  1145. * @disk: struct gendisk to check
  1146. * @bdev: struct bdev to adjust.
  1147. * @verbose: if %true log a message about a size change if there is any
  1148. *
  1149. * This routine checks to see if the bdev size does not match the disk size
  1150. * and adjusts it if it differs. When shrinking the bdev size, its all caches
  1151. * are freed.
  1152. */
  1153. static void check_disk_size_change(struct gendisk *disk,
  1154. struct block_device *bdev, bool verbose)
  1155. {
  1156. loff_t disk_size, bdev_size;
  1157. spin_lock(&bdev->bd_size_lock);
  1158. disk_size = (loff_t)get_capacity(disk) << 9;
  1159. bdev_size = i_size_read(bdev->bd_inode);
  1160. if (disk_size != bdev_size) {
  1161. if (verbose) {
  1162. printk(KERN_INFO
  1163. "%s: detected capacity change from %lld to %lld\n",
  1164. disk->disk_name, bdev_size, disk_size);
  1165. }
  1166. i_size_write(bdev->bd_inode, disk_size);
  1167. }
  1168. spin_unlock(&bdev->bd_size_lock);
  1169. if (bdev_size > disk_size) {
  1170. if (__invalidate_device(bdev, false))
  1171. pr_warn("VFS: busy inodes on resized disk %s\n",
  1172. disk->disk_name);
  1173. }
  1174. }
  1175. /**
  1176. * revalidate_disk_size - checks for disk size change and adjusts bdev size.
  1177. * @disk: struct gendisk to check
  1178. * @verbose: if %true log a message about a size change if there is any
  1179. *
  1180. * This routine checks to see if the bdev size does not match the disk size
  1181. * and adjusts it if it differs. When shrinking the bdev size, its all caches
  1182. * are freed.
  1183. */
  1184. void revalidate_disk_size(struct gendisk *disk, bool verbose)
  1185. {
  1186. struct block_device *bdev;
  1187. /*
  1188. * Hidden disks don't have associated bdev so there's no point in
  1189. * revalidating them.
  1190. */
  1191. if (disk->flags & GENHD_FL_HIDDEN)
  1192. return;
  1193. bdev = bdget_disk(disk, 0);
  1194. if (bdev) {
  1195. check_disk_size_change(disk, bdev, verbose);
  1196. bdput(bdev);
  1197. }
  1198. }
  1199. EXPORT_SYMBOL(revalidate_disk_size);
  1200. void bd_set_nr_sectors(struct block_device *bdev, sector_t sectors)
  1201. {
  1202. spin_lock(&bdev->bd_size_lock);
  1203. i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
  1204. spin_unlock(&bdev->bd_size_lock);
  1205. }
  1206. EXPORT_SYMBOL(bd_set_nr_sectors);
  1207. static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part);
  1208. int bdev_disk_changed(struct block_device *bdev, bool invalidate)
  1209. {
  1210. struct gendisk *disk = bdev->bd_disk;
  1211. int ret;
  1212. lockdep_assert_held(&bdev->bd_mutex);
  1213. if (!(disk->flags & GENHD_FL_UP))
  1214. return -ENXIO;
  1215. rescan:
  1216. ret = blk_drop_partitions(bdev);
  1217. if (ret)
  1218. return ret;
  1219. clear_bit(GD_NEED_PART_SCAN, &disk->state);
  1220. /*
  1221. * Historically we only set the capacity to zero for devices that
  1222. * support partitions (independ of actually having partitions created).
  1223. * Doing that is rather inconsistent, but changing it broke legacy
  1224. * udisks polling for legacy ide-cdrom devices. Use the crude check
  1225. * below to get the sane behavior for most device while not breaking
  1226. * userspace for this particular setup.
  1227. */
  1228. if (invalidate) {
  1229. if (disk_part_scan_enabled(disk) ||
  1230. !(disk->flags & GENHD_FL_REMOVABLE))
  1231. set_capacity(disk, 0);
  1232. } else {
  1233. if (disk->fops->revalidate_disk)
  1234. disk->fops->revalidate_disk(disk);
  1235. }
  1236. check_disk_size_change(disk, bdev, !invalidate);
  1237. if (get_capacity(disk)) {
  1238. ret = blk_add_partitions(disk, bdev);
  1239. if (ret == -EAGAIN)
  1240. goto rescan;
  1241. } else if (invalidate) {
  1242. /*
  1243. * Tell userspace that the media / partition table may have
  1244. * changed.
  1245. */
  1246. kobject_uevent(&disk_to_dev(disk)->kobj, KOBJ_CHANGE);
  1247. }
  1248. return ret;
  1249. }
  1250. /*
  1251. * Only exported for for loop and dasd for historic reasons. Don't use in new
  1252. * code!
  1253. */
  1254. EXPORT_SYMBOL_GPL(bdev_disk_changed);
  1255. /*
  1256. * bd_mutex locking:
  1257. *
  1258. * mutex_lock(part->bd_mutex)
  1259. * mutex_lock_nested(whole->bd_mutex, 1)
  1260. */
  1261. static int __blkdev_get(struct block_device *bdev, fmode_t mode, void *holder,
  1262. int for_part)
  1263. {
  1264. struct block_device *whole = NULL, *claiming = NULL;
  1265. struct gendisk *disk;
  1266. int ret;
  1267. int partno;
  1268. bool first_open = false, unblock_events = true, need_restart;
  1269. restart:
  1270. need_restart = false;
  1271. ret = -ENXIO;
  1272. disk = bdev_get_gendisk(bdev, &partno);
  1273. if (!disk)
  1274. goto out;
  1275. if (partno) {
  1276. whole = bdget_disk(disk, 0);
  1277. if (!whole) {
  1278. ret = -ENOMEM;
  1279. goto out_put_disk;
  1280. }
  1281. }
  1282. if (!for_part && (mode & FMODE_EXCL)) {
  1283. WARN_ON_ONCE(!holder);
  1284. if (whole)
  1285. claiming = whole;
  1286. else
  1287. claiming = bdev;
  1288. ret = bd_prepare_to_claim(bdev, claiming, holder);
  1289. if (ret)
  1290. goto out_put_whole;
  1291. }
  1292. disk_block_events(disk);
  1293. mutex_lock_nested(&bdev->bd_mutex, for_part);
  1294. if (!bdev->bd_openers) {
  1295. first_open = true;
  1296. bdev->bd_disk = disk;
  1297. bdev->bd_contains = bdev;
  1298. bdev->bd_partno = partno;
  1299. if (!partno) {
  1300. ret = -ENXIO;
  1301. bdev->bd_part = disk_get_part(disk, partno);
  1302. if (!bdev->bd_part)
  1303. goto out_clear;
  1304. ret = 0;
  1305. if (disk->fops->open) {
  1306. ret = disk->fops->open(bdev, mode);
  1307. /*
  1308. * If we lost a race with 'disk' being deleted,
  1309. * try again. See md.c
  1310. */
  1311. if (ret == -ERESTARTSYS)
  1312. need_restart = true;
  1313. }
  1314. if (!ret) {
  1315. bd_set_nr_sectors(bdev, get_capacity(disk));
  1316. set_init_blocksize(bdev);
  1317. }
  1318. /*
  1319. * If the device is invalidated, rescan partition
  1320. * if open succeeded or failed with -ENOMEDIUM.
  1321. * The latter is necessary to prevent ghost
  1322. * partitions on a removed medium.
  1323. */
  1324. if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
  1325. (!ret || ret == -ENOMEDIUM))
  1326. bdev_disk_changed(bdev, ret == -ENOMEDIUM);
  1327. if (ret)
  1328. goto out_clear;
  1329. } else {
  1330. BUG_ON(for_part);
  1331. ret = __blkdev_get(whole, mode, NULL, 1);
  1332. if (ret)
  1333. goto out_clear;
  1334. bdev->bd_contains = bdgrab(whole);
  1335. bdev->bd_part = disk_get_part(disk, partno);
  1336. if (!(disk->flags & GENHD_FL_UP) ||
  1337. !bdev->bd_part || !bdev->bd_part->nr_sects) {
  1338. ret = -ENXIO;
  1339. goto out_clear;
  1340. }
  1341. bd_set_nr_sectors(bdev, bdev->bd_part->nr_sects);
  1342. set_init_blocksize(bdev);
  1343. }
  1344. if (bdev->bd_bdi == &noop_backing_dev_info)
  1345. bdev->bd_bdi = bdi_get(disk->queue->backing_dev_info);
  1346. } else {
  1347. if (bdev->bd_contains == bdev) {
  1348. ret = 0;
  1349. if (bdev->bd_disk->fops->open)
  1350. ret = bdev->bd_disk->fops->open(bdev, mode);
  1351. /* the same as first opener case, read comment there */
  1352. if (test_bit(GD_NEED_PART_SCAN, &disk->state) &&
  1353. (!ret || ret == -ENOMEDIUM))
  1354. bdev_disk_changed(bdev, ret == -ENOMEDIUM);
  1355. if (ret)
  1356. goto out_unlock_bdev;
  1357. }
  1358. }
  1359. bdev->bd_openers++;
  1360. if (for_part)
  1361. bdev->bd_part_count++;
  1362. if (claiming)
  1363. bd_finish_claiming(bdev, claiming, holder);
  1364. /*
  1365. * Block event polling for write claims if requested. Any write holder
  1366. * makes the write_holder state stick until all are released. This is
  1367. * good enough and tracking individual writeable reference is too
  1368. * fragile given the way @mode is used in blkdev_get/put().
  1369. */
  1370. if (claiming && (mode & FMODE_WRITE) && !bdev->bd_write_holder &&
  1371. (disk->flags & GENHD_FL_BLOCK_EVENTS_ON_EXCL_WRITE)) {
  1372. bdev->bd_write_holder = true;
  1373. unblock_events = false;
  1374. }
  1375. mutex_unlock(&bdev->bd_mutex);
  1376. if (unblock_events)
  1377. disk_unblock_events(disk);
  1378. /* only one opener holds refs to the module and disk */
  1379. if (!first_open)
  1380. put_disk_and_module(disk);
  1381. if (whole)
  1382. bdput(whole);
  1383. return 0;
  1384. out_clear:
  1385. disk_put_part(bdev->bd_part);
  1386. bdev->bd_disk = NULL;
  1387. bdev->bd_part = NULL;
  1388. if (bdev != bdev->bd_contains)
  1389. __blkdev_put(bdev->bd_contains, mode, 1);
  1390. bdev->bd_contains = NULL;
  1391. out_unlock_bdev:
  1392. if (claiming)
  1393. bd_abort_claiming(bdev, claiming, holder);
  1394. mutex_unlock(&bdev->bd_mutex);
  1395. disk_unblock_events(disk);
  1396. out_put_whole:
  1397. if (whole)
  1398. bdput(whole);
  1399. out_put_disk:
  1400. put_disk_and_module(disk);
  1401. if (need_restart)
  1402. goto restart;
  1403. out:
  1404. return ret;
  1405. }
  1406. /**
  1407. * blkdev_get - open a block device
  1408. * @bdev: block_device to open
  1409. * @mode: FMODE_* mask
  1410. * @holder: exclusive holder identifier
  1411. *
  1412. * Open @bdev with @mode. If @mode includes %FMODE_EXCL, @bdev is
  1413. * open with exclusive access. Specifying %FMODE_EXCL with %NULL
  1414. * @holder is invalid. Exclusive opens may nest for the same @holder.
  1415. *
  1416. * On success, the reference count of @bdev is unchanged. On failure,
  1417. * @bdev is put.
  1418. *
  1419. * CONTEXT:
  1420. * Might sleep.
  1421. *
  1422. * RETURNS:
  1423. * 0 on success, -errno on failure.
  1424. */
  1425. static int blkdev_get(struct block_device *bdev, fmode_t mode, void *holder)
  1426. {
  1427. int ret, perm = 0;
  1428. if (mode & FMODE_READ)
  1429. perm |= MAY_READ;
  1430. if (mode & FMODE_WRITE)
  1431. perm |= MAY_WRITE;
  1432. ret = devcgroup_inode_permission(bdev->bd_inode, perm);
  1433. if (ret)
  1434. goto bdput;
  1435. ret =__blkdev_get(bdev, mode, holder, 0);
  1436. if (ret)
  1437. goto bdput;
  1438. return 0;
  1439. bdput:
  1440. bdput(bdev);
  1441. return ret;
  1442. }
  1443. /**
  1444. * blkdev_get_by_path - open a block device by name
  1445. * @path: path to the block device to open
  1446. * @mode: FMODE_* mask
  1447. * @holder: exclusive holder identifier
  1448. *
  1449. * Open the blockdevice described by the device file at @path. @mode
  1450. * and @holder are identical to blkdev_get().
  1451. *
  1452. * On success, the returned block_device has reference count of one.
  1453. *
  1454. * CONTEXT:
  1455. * Might sleep.
  1456. *
  1457. * RETURNS:
  1458. * Pointer to block_device on success, ERR_PTR(-errno) on failure.
  1459. */
  1460. struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
  1461. void *holder)
  1462. {
  1463. struct block_device *bdev;
  1464. int err;
  1465. bdev = lookup_bdev(path);
  1466. if (IS_ERR(bdev))
  1467. return bdev;
  1468. err = blkdev_get(bdev, mode, holder);
  1469. if (err)
  1470. return ERR_PTR(err);
  1471. if ((mode & FMODE_WRITE) && bdev_read_only(bdev)) {
  1472. blkdev_put(bdev, mode);
  1473. return ERR_PTR(-EACCES);
  1474. }
  1475. return bdev;
  1476. }
  1477. EXPORT_SYMBOL(blkdev_get_by_path);
  1478. /**
  1479. * blkdev_get_by_dev - open a block device by device number
  1480. * @dev: device number of block device to open
  1481. * @mode: FMODE_* mask
  1482. * @holder: exclusive holder identifier
  1483. *
  1484. * Open the blockdevice described by device number @dev. @mode and
  1485. * @holder are identical to blkdev_get().
  1486. *
  1487. * Use it ONLY if you really do not have anything better - i.e. when
  1488. * you are behind a truly sucky interface and all you are given is a
  1489. * device number. _Never_ to be used for internal purposes. If you
  1490. * ever need it - reconsider your API.
  1491. *
  1492. * On success, the returned block_device has reference count of one.
  1493. *
  1494. * CONTEXT:
  1495. * Might sleep.
  1496. *
  1497. * RETURNS:
  1498. * Pointer to block_device on success, ERR_PTR(-errno) on failure.
  1499. */
  1500. struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
  1501. {
  1502. struct block_device *bdev;
  1503. int err;
  1504. bdev = bdget(dev);
  1505. if (!bdev)
  1506. return ERR_PTR(-ENOMEM);
  1507. err = blkdev_get(bdev, mode, holder);
  1508. if (err)
  1509. return ERR_PTR(err);
  1510. return bdev;
  1511. }
  1512. EXPORT_SYMBOL(blkdev_get_by_dev);
  1513. static int blkdev_open(struct inode * inode, struct file * filp)
  1514. {
  1515. struct block_device *bdev;
  1516. /*
  1517. * Preserve backwards compatibility and allow large file access
  1518. * even if userspace doesn't ask for it explicitly. Some mkfs
  1519. * binary needs it. We might want to drop this workaround
  1520. * during an unstable branch.
  1521. */
  1522. filp->f_flags |= O_LARGEFILE;
  1523. filp->f_mode |= FMODE_NOWAIT | FMODE_BUF_RASYNC;
  1524. if (filp->f_flags & O_NDELAY)
  1525. filp->f_mode |= FMODE_NDELAY;
  1526. if (filp->f_flags & O_EXCL)
  1527. filp->f_mode |= FMODE_EXCL;
  1528. if ((filp->f_flags & O_ACCMODE) == 3)
  1529. filp->f_mode |= FMODE_WRITE_IOCTL;
  1530. bdev = bd_acquire(inode);
  1531. if (bdev == NULL)
  1532. return -ENOMEM;
  1533. filp->f_mapping = bdev->bd_inode->i_mapping;
  1534. filp->f_wb_err = filemap_sample_wb_err(filp->f_mapping);
  1535. return blkdev_get(bdev, filp->f_mode, filp);
  1536. }
  1537. static void __blkdev_put(struct block_device *bdev, fmode_t mode, int for_part)
  1538. {
  1539. struct gendisk *disk = bdev->bd_disk;
  1540. struct block_device *victim = NULL;
  1541. /*
  1542. * Sync early if it looks like we're the last one. If someone else
  1543. * opens the block device between now and the decrement of bd_openers
  1544. * then we did a sync that we didn't need to, but that's not the end
  1545. * of the world and we want to avoid long (could be several minute)
  1546. * syncs while holding the mutex.
  1547. */
  1548. if (bdev->bd_openers == 1)
  1549. sync_blockdev(bdev);
  1550. mutex_lock_nested(&bdev->bd_mutex, for_part);
  1551. if (for_part)
  1552. bdev->bd_part_count--;
  1553. if (!--bdev->bd_openers) {
  1554. WARN_ON_ONCE(bdev->bd_holders);
  1555. sync_blockdev(bdev);
  1556. kill_bdev(bdev);
  1557. bdev_write_inode(bdev);
  1558. }
  1559. if (bdev->bd_contains == bdev) {
  1560. if (disk->fops->release)
  1561. disk->fops->release(disk, mode);
  1562. }
  1563. if (!bdev->bd_openers) {
  1564. disk_put_part(bdev->bd_part);
  1565. bdev->bd_part = NULL;
  1566. bdev->bd_disk = NULL;
  1567. if (bdev != bdev->bd_contains)
  1568. victim = bdev->bd_contains;
  1569. bdev->bd_contains = NULL;
  1570. put_disk_and_module(disk);
  1571. }
  1572. mutex_unlock(&bdev->bd_mutex);
  1573. bdput(bdev);
  1574. if (victim)
  1575. __blkdev_put(victim, mode, 1);
  1576. }
  1577. void blkdev_put(struct block_device *bdev, fmode_t mode)
  1578. {
  1579. mutex_lock(&bdev->bd_mutex);
  1580. if (mode & FMODE_EXCL) {
  1581. bool bdev_free;
  1582. /*
  1583. * Release a claim on the device. The holder fields
  1584. * are protected with bdev_lock. bd_mutex is to
  1585. * synchronize disk_holder unlinking.
  1586. */
  1587. spin_lock(&bdev_lock);
  1588. WARN_ON_ONCE(--bdev->bd_holders < 0);
  1589. WARN_ON_ONCE(--bdev->bd_contains->bd_holders < 0);
  1590. /* bd_contains might point to self, check in a separate step */
  1591. if ((bdev_free = !bdev->bd_holders))
  1592. bdev->bd_holder = NULL;
  1593. if (!bdev->bd_contains->bd_holders)
  1594. bdev->bd_contains->bd_holder = NULL;
  1595. spin_unlock(&bdev_lock);
  1596. /*
  1597. * If this was the last claim, remove holder link and
  1598. * unblock evpoll if it was a write holder.
  1599. */
  1600. if (bdev_free && bdev->bd_write_holder) {
  1601. disk_unblock_events(bdev->bd_disk);
  1602. bdev->bd_write_holder = false;
  1603. }
  1604. }
  1605. /*
  1606. * Trigger event checking and tell drivers to flush MEDIA_CHANGE
  1607. * event. This is to ensure detection of media removal commanded
  1608. * from userland - e.g. eject(1).
  1609. */
  1610. disk_flush_events(bdev->bd_disk, DISK_EVENT_MEDIA_CHANGE);
  1611. mutex_unlock(&bdev->bd_mutex);
  1612. __blkdev_put(bdev, mode, 0);
  1613. }
  1614. EXPORT_SYMBOL(blkdev_put);
  1615. static int blkdev_close(struct inode * inode, struct file * filp)
  1616. {
  1617. struct block_device *bdev = I_BDEV(bdev_file_inode(filp));
  1618. blkdev_put(bdev, filp->f_mode);
  1619. return 0;
  1620. }
  1621. static long block_ioctl(struct file *file, unsigned cmd, unsigned long arg)
  1622. {
  1623. struct block_device *bdev = I_BDEV(bdev_file_inode(file));
  1624. fmode_t mode = file->f_mode;
  1625. /*
  1626. * O_NDELAY can be altered using fcntl(.., F_SETFL, ..), so we have
  1627. * to updated it before every ioctl.
  1628. */
  1629. if (file->f_flags & O_NDELAY)
  1630. mode |= FMODE_NDELAY;
  1631. else
  1632. mode &= ~FMODE_NDELAY;
  1633. return blkdev_ioctl(bdev, mode, cmd, arg);
  1634. }
  1635. /*
  1636. * Write data to the block device. Only intended for the block device itself
  1637. * and the raw driver which basically is a fake block device.
  1638. *
  1639. * Does not take i_mutex for the write and thus is not for general purpose
  1640. * use.
  1641. */
  1642. ssize_t blkdev_write_iter(struct kiocb *iocb, struct iov_iter *from)
  1643. {
  1644. struct file *file = iocb->ki_filp;
  1645. struct inode *bd_inode = bdev_file_inode(file);
  1646. loff_t size = i_size_read(bd_inode);
  1647. struct blk_plug plug;
  1648. size_t shorted = 0;
  1649. ssize_t ret;
  1650. if (bdev_read_only(I_BDEV(bd_inode)))
  1651. return -EPERM;
  1652. if (IS_SWAPFILE(bd_inode) && !is_hibernate_resume_dev(bd_inode->i_rdev))
  1653. return -ETXTBSY;
  1654. if (!iov_iter_count(from))
  1655. return 0;
  1656. if (iocb->ki_pos >= size)
  1657. return -ENOSPC;
  1658. if ((iocb->ki_flags & (IOCB_NOWAIT | IOCB_DIRECT)) == IOCB_NOWAIT)
  1659. return -EOPNOTSUPP;
  1660. size -= iocb->ki_pos;
  1661. if (iov_iter_count(from) > size) {
  1662. shorted = iov_iter_count(from) - size;
  1663. iov_iter_truncate(from, size);
  1664. }
  1665. blk_start_plug(&plug);
  1666. ret = __generic_file_write_iter(iocb, from);
  1667. if (ret > 0)
  1668. ret = generic_write_sync(iocb, ret);
  1669. iov_iter_reexpand(from, iov_iter_count(from) + shorted);
  1670. blk_finish_plug(&plug);
  1671. return ret;
  1672. }
  1673. EXPORT_SYMBOL_GPL(blkdev_write_iter);
  1674. ssize_t blkdev_read_iter(struct kiocb *iocb, struct iov_iter *to)
  1675. {
  1676. struct file *file = iocb->ki_filp;
  1677. struct inode *bd_inode = bdev_file_inode(file);
  1678. loff_t size = i_size_read(bd_inode);
  1679. loff_t pos = iocb->ki_pos;
  1680. size_t shorted = 0;
  1681. ssize_t ret;
  1682. if (pos >= size)
  1683. return 0;
  1684. size -= pos;
  1685. if (iov_iter_count(to) > size) {
  1686. shorted = iov_iter_count(to) - size;
  1687. iov_iter_truncate(to, size);
  1688. }
  1689. ret = generic_file_read_iter(iocb, to);
  1690. iov_iter_reexpand(to, iov_iter_count(to) + shorted);
  1691. return ret;
  1692. }
  1693. EXPORT_SYMBOL_GPL(blkdev_read_iter);
  1694. /*
  1695. * Try to release a page associated with block device when the system
  1696. * is under memory pressure.
  1697. */
  1698. static int blkdev_releasepage(struct page *page, gfp_t wait)
  1699. {
  1700. struct super_block *super = BDEV_I(page->mapping->host)->bdev.bd_super;
  1701. if (super && super->s_op->bdev_try_to_free_page)
  1702. return super->s_op->bdev_try_to_free_page(super, page, wait);
  1703. return try_to_free_buffers(page);
  1704. }
  1705. static int blkdev_writepages(struct address_space *mapping,
  1706. struct writeback_control *wbc)
  1707. {
  1708. return generic_writepages(mapping, wbc);
  1709. }
  1710. static const struct address_space_operations def_blk_aops = {
  1711. .readpage = blkdev_readpage,
  1712. .readahead = blkdev_readahead,
  1713. .writepage = blkdev_writepage,
  1714. .write_begin = blkdev_write_begin,
  1715. .write_end = blkdev_write_end,
  1716. .writepages = blkdev_writepages,
  1717. .releasepage = blkdev_releasepage,
  1718. .direct_IO = blkdev_direct_IO,
  1719. .migratepage = buffer_migrate_page_norefs,
  1720. .is_dirty_writeback = buffer_check_dirty_writeback,
  1721. };
  1722. #define BLKDEV_FALLOC_FL_SUPPORTED \
  1723. (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE | \
  1724. FALLOC_FL_ZERO_RANGE | FALLOC_FL_NO_HIDE_STALE)
  1725. static long blkdev_fallocate(struct file *file, int mode, loff_t start,
  1726. loff_t len)
  1727. {
  1728. struct block_device *bdev = I_BDEV(bdev_file_inode(file));
  1729. loff_t end = start + len - 1;
  1730. loff_t isize;
  1731. int error;
  1732. /* Fail if we don't recognize the flags. */
  1733. if (mode & ~BLKDEV_FALLOC_FL_SUPPORTED)
  1734. return -EOPNOTSUPP;
  1735. /* Don't go off the end of the device. */
  1736. isize = i_size_read(bdev->bd_inode);
  1737. if (start >= isize)
  1738. return -EINVAL;
  1739. if (end >= isize) {
  1740. if (mode & FALLOC_FL_KEEP_SIZE) {
  1741. len = isize - start;
  1742. end = start + len - 1;
  1743. } else
  1744. return -EINVAL;
  1745. }
  1746. /*
  1747. * Don't allow IO that isn't aligned to logical block size.
  1748. */
  1749. if ((start | len) & (bdev_logical_block_size(bdev) - 1))
  1750. return -EINVAL;
  1751. /* Invalidate the page cache, including dirty pages. */
  1752. error = truncate_bdev_range(bdev, file->f_mode, start, end);
  1753. if (error)
  1754. return error;
  1755. switch (mode) {
  1756. case FALLOC_FL_ZERO_RANGE:
  1757. case FALLOC_FL_ZERO_RANGE | FALLOC_FL_KEEP_SIZE:
  1758. error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
  1759. GFP_KERNEL, BLKDEV_ZERO_NOUNMAP);
  1760. break;
  1761. case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE:
  1762. error = blkdev_issue_zeroout(bdev, start >> 9, len >> 9,
  1763. GFP_KERNEL, BLKDEV_ZERO_NOFALLBACK);
  1764. break;
  1765. case FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE | FALLOC_FL_NO_HIDE_STALE:
  1766. error = blkdev_issue_discard(bdev, start >> 9, len >> 9,
  1767. GFP_KERNEL, 0);
  1768. break;
  1769. default:
  1770. return -EOPNOTSUPP;
  1771. }
  1772. if (error)
  1773. return error;
  1774. /*
  1775. * Invalidate again; if someone wandered in and dirtied a page,
  1776. * the caller will be given -EBUSY. The third argument is
  1777. * inclusive, so the rounding here is safe.
  1778. */
  1779. return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
  1780. start >> PAGE_SHIFT,
  1781. end >> PAGE_SHIFT);
  1782. }
  1783. const struct file_operations def_blk_fops = {
  1784. .open = blkdev_open,
  1785. .release = blkdev_close,
  1786. .llseek = block_llseek,
  1787. .read_iter = blkdev_read_iter,
  1788. .write_iter = blkdev_write_iter,
  1789. .iopoll = blkdev_iopoll,
  1790. .mmap = generic_file_mmap,
  1791. .fsync = blkdev_fsync,
  1792. .unlocked_ioctl = block_ioctl,
  1793. #ifdef CONFIG_COMPAT
  1794. .compat_ioctl = compat_blkdev_ioctl,
  1795. #endif
  1796. .splice_read = generic_file_splice_read,
  1797. .splice_write = iter_file_splice_write,
  1798. .fallocate = blkdev_fallocate,
  1799. };
  1800. /**
  1801. * lookup_bdev - lookup a struct block_device by name
  1802. * @pathname: special file representing the block device
  1803. *
  1804. * Get a reference to the blockdevice at @pathname in the current
  1805. * namespace if possible and return it. Return ERR_PTR(error)
  1806. * otherwise.
  1807. */
  1808. struct block_device *lookup_bdev(const char *pathname)
  1809. {
  1810. struct block_device *bdev;
  1811. struct inode *inode;
  1812. struct path path;
  1813. int error;
  1814. if (!pathname || !*pathname)
  1815. return ERR_PTR(-EINVAL);
  1816. error = kern_path(pathname, LOOKUP_FOLLOW, &path);
  1817. if (error)
  1818. return ERR_PTR(error);
  1819. inode = d_backing_inode(path.dentry);
  1820. error = -ENOTBLK;
  1821. if (!S_ISBLK(inode->i_mode))
  1822. goto fail;
  1823. error = -EACCES;
  1824. if (!may_open_dev(&path))
  1825. goto fail;
  1826. error = -ENOMEM;
  1827. bdev = bd_acquire(inode);
  1828. if (!bdev)
  1829. goto fail;
  1830. out:
  1831. path_put(&path);
  1832. return bdev;
  1833. fail:
  1834. bdev = ERR_PTR(error);
  1835. goto out;
  1836. }
  1837. EXPORT_SYMBOL(lookup_bdev);
  1838. int __invalidate_device(struct block_device *bdev, bool kill_dirty)
  1839. {
  1840. struct super_block *sb = get_super(bdev);
  1841. int res = 0;
  1842. if (sb) {
  1843. /*
  1844. * no need to lock the super, get_super holds the
  1845. * read mutex so the filesystem cannot go away
  1846. * under us (->put_super runs with the write lock
  1847. * hold).
  1848. */
  1849. shrink_dcache_sb(sb);
  1850. res = invalidate_inodes(sb, kill_dirty);
  1851. drop_super(sb);
  1852. }
  1853. invalidate_bdev(bdev);
  1854. return res;
  1855. }
  1856. EXPORT_SYMBOL(__invalidate_device);
  1857. void iterate_bdevs(void (*func)(struct block_device *, void *), void *arg)
  1858. {
  1859. struct inode *inode, *old_inode = NULL;
  1860. spin_lock(&blockdev_superblock->s_inode_list_lock);
  1861. list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
  1862. struct address_space *mapping = inode->i_mapping;
  1863. struct block_device *bdev;
  1864. spin_lock(&inode->i_lock);
  1865. if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
  1866. mapping->nrpages == 0) {
  1867. spin_unlock(&inode->i_lock);
  1868. continue;
  1869. }
  1870. __iget(inode);
  1871. spin_unlock(&inode->i_lock);
  1872. spin_unlock(&blockdev_superblock->s_inode_list_lock);
  1873. /*
  1874. * We hold a reference to 'inode' so it couldn't have been
  1875. * removed from s_inodes list while we dropped the
  1876. * s_inode_list_lock We cannot iput the inode now as we can
  1877. * be holding the last reference and we cannot iput it under
  1878. * s_inode_list_lock. So we keep the reference and iput it
  1879. * later.
  1880. */
  1881. iput(old_inode);
  1882. old_inode = inode;
  1883. bdev = I_BDEV(inode);
  1884. mutex_lock(&bdev->bd_mutex);
  1885. if (bdev->bd_openers)
  1886. func(bdev, arg);
  1887. mutex_unlock(&bdev->bd_mutex);
  1888. spin_lock(&blockdev_superblock->s_inode_list_lock);
  1889. }
  1890. spin_unlock(&blockdev_superblock->s_inode_list_lock);
  1891. iput(old_inode);
  1892. }