md-bitmap.c 70 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * bitmap.c two-level bitmap (C) Peter T. Breuer (ptb@ot.uc3m.es) 2003
  4. *
  5. * bitmap_create - sets up the bitmap structure
  6. * bitmap_destroy - destroys the bitmap structure
  7. *
  8. * additions, Copyright (C) 2003-2004, Paul Clements, SteelEye Technology, Inc.:
  9. * - added disk storage for bitmap
  10. * - changes to allow various bitmap chunk sizes
  11. */
  12. /*
  13. * Still to do:
  14. *
  15. * flush after percent set rather than just time based. (maybe both).
  16. */
  17. #include <linux/blkdev.h>
  18. #include <linux/module.h>
  19. #include <linux/errno.h>
  20. #include <linux/slab.h>
  21. #include <linux/init.h>
  22. #include <linux/timer.h>
  23. #include <linux/sched.h>
  24. #include <linux/list.h>
  25. #include <linux/file.h>
  26. #include <linux/mount.h>
  27. #include <linux/buffer_head.h>
  28. #include <linux/seq_file.h>
  29. #include <trace/events/block.h>
  30. #include "md.h"
  31. #include "md-bitmap.h"
  32. static inline char *bmname(struct bitmap *bitmap)
  33. {
  34. return bitmap->mddev ? mdname(bitmap->mddev) : "mdX";
  35. }
  36. /*
  37. * check a page and, if necessary, allocate it (or hijack it if the alloc fails)
  38. *
  39. * 1) check to see if this page is allocated, if it's not then try to alloc
  40. * 2) if the alloc fails, set the page's hijacked flag so we'll use the
  41. * page pointer directly as a counter
  42. *
  43. * if we find our page, we increment the page's refcount so that it stays
  44. * allocated while we're using it
  45. */
  46. static int md_bitmap_checkpage(struct bitmap_counts *bitmap,
  47. unsigned long page, int create, int no_hijack)
  48. __releases(bitmap->lock)
  49. __acquires(bitmap->lock)
  50. {
  51. unsigned char *mappage;
  52. if (page >= bitmap->pages) {
  53. /* This can happen if bitmap_start_sync goes beyond
  54. * End-of-device while looking for a whole page.
  55. * It is harmless.
  56. */
  57. return -EINVAL;
  58. }
  59. if (bitmap->bp[page].hijacked) /* it's hijacked, don't try to alloc */
  60. return 0;
  61. if (bitmap->bp[page].map) /* page is already allocated, just return */
  62. return 0;
  63. if (!create)
  64. return -ENOENT;
  65. /* this page has not been allocated yet */
  66. spin_unlock_irq(&bitmap->lock);
  67. /* It is possible that this is being called inside a
  68. * prepare_to_wait/finish_wait loop from raid5c:make_request().
  69. * In general it is not permitted to sleep in that context as it
  70. * can cause the loop to spin freely.
  71. * That doesn't apply here as we can only reach this point
  72. * once with any loop.
  73. * When this function completes, either bp[page].map or
  74. * bp[page].hijacked. In either case, this function will
  75. * abort before getting to this point again. So there is
  76. * no risk of a free-spin, and so it is safe to assert
  77. * that sleeping here is allowed.
  78. */
  79. sched_annotate_sleep();
  80. mappage = kzalloc(PAGE_SIZE, GFP_NOIO);
  81. spin_lock_irq(&bitmap->lock);
  82. if (mappage == NULL) {
  83. pr_debug("md/bitmap: map page allocation failed, hijacking\n");
  84. /* We don't support hijack for cluster raid */
  85. if (no_hijack)
  86. return -ENOMEM;
  87. /* failed - set the hijacked flag so that we can use the
  88. * pointer as a counter */
  89. if (!bitmap->bp[page].map)
  90. bitmap->bp[page].hijacked = 1;
  91. } else if (bitmap->bp[page].map ||
  92. bitmap->bp[page].hijacked) {
  93. /* somebody beat us to getting the page */
  94. kfree(mappage);
  95. } else {
  96. /* no page was in place and we have one, so install it */
  97. bitmap->bp[page].map = mappage;
  98. bitmap->missing_pages--;
  99. }
  100. return 0;
  101. }
  102. /* if page is completely empty, put it back on the free list, or dealloc it */
  103. /* if page was hijacked, unmark the flag so it might get alloced next time */
  104. /* Note: lock should be held when calling this */
  105. static void md_bitmap_checkfree(struct bitmap_counts *bitmap, unsigned long page)
  106. {
  107. char *ptr;
  108. if (bitmap->bp[page].count) /* page is still busy */
  109. return;
  110. /* page is no longer in use, it can be released */
  111. if (bitmap->bp[page].hijacked) { /* page was hijacked, undo this now */
  112. bitmap->bp[page].hijacked = 0;
  113. bitmap->bp[page].map = NULL;
  114. } else {
  115. /* normal case, free the page */
  116. ptr = bitmap->bp[page].map;
  117. bitmap->bp[page].map = NULL;
  118. bitmap->missing_pages++;
  119. kfree(ptr);
  120. }
  121. }
  122. /*
  123. * bitmap file handling - read and write the bitmap file and its superblock
  124. */
  125. /*
  126. * basic page I/O operations
  127. */
  128. /* IO operations when bitmap is stored near all superblocks */
  129. static int read_sb_page(struct mddev *mddev, loff_t offset,
  130. struct page *page,
  131. unsigned long index, int size)
  132. {
  133. /* choose a good rdev and read the page from there */
  134. struct md_rdev *rdev;
  135. sector_t target;
  136. rdev_for_each(rdev, mddev) {
  137. if (! test_bit(In_sync, &rdev->flags)
  138. || test_bit(Faulty, &rdev->flags)
  139. || test_bit(Bitmap_sync, &rdev->flags))
  140. continue;
  141. target = offset + index * (PAGE_SIZE/512);
  142. if (sync_page_io(rdev, target,
  143. roundup(size, bdev_logical_block_size(rdev->bdev)),
  144. page, REQ_OP_READ, 0, true)) {
  145. page->index = index;
  146. return 0;
  147. }
  148. }
  149. return -EIO;
  150. }
  151. static struct md_rdev *next_active_rdev(struct md_rdev *rdev, struct mddev *mddev)
  152. {
  153. /* Iterate the disks of an mddev, using rcu to protect access to the
  154. * linked list, and raising the refcount of devices we return to ensure
  155. * they don't disappear while in use.
  156. * As devices are only added or removed when raid_disk is < 0 and
  157. * nr_pending is 0 and In_sync is clear, the entries we return will
  158. * still be in the same position on the list when we re-enter
  159. * list_for_each_entry_continue_rcu.
  160. *
  161. * Note that if entered with 'rdev == NULL' to start at the
  162. * beginning, we temporarily assign 'rdev' to an address which
  163. * isn't really an rdev, but which can be used by
  164. * list_for_each_entry_continue_rcu() to find the first entry.
  165. */
  166. rcu_read_lock();
  167. if (rdev == NULL)
  168. /* start at the beginning */
  169. rdev = list_entry(&mddev->disks, struct md_rdev, same_set);
  170. else {
  171. /* release the previous rdev and start from there. */
  172. rdev_dec_pending(rdev, mddev);
  173. }
  174. list_for_each_entry_continue_rcu(rdev, &mddev->disks, same_set) {
  175. if (rdev->raid_disk >= 0 &&
  176. !test_bit(Faulty, &rdev->flags)) {
  177. /* this is a usable devices */
  178. atomic_inc(&rdev->nr_pending);
  179. rcu_read_unlock();
  180. return rdev;
  181. }
  182. }
  183. rcu_read_unlock();
  184. return NULL;
  185. }
  186. static int write_sb_page(struct bitmap *bitmap, struct page *page, int wait)
  187. {
  188. struct md_rdev *rdev;
  189. struct block_device *bdev;
  190. struct mddev *mddev = bitmap->mddev;
  191. struct bitmap_storage *store = &bitmap->storage;
  192. restart:
  193. rdev = NULL;
  194. while ((rdev = next_active_rdev(rdev, mddev)) != NULL) {
  195. int size = PAGE_SIZE;
  196. loff_t offset = mddev->bitmap_info.offset;
  197. bdev = (rdev->meta_bdev) ? rdev->meta_bdev : rdev->bdev;
  198. if (page->index == store->file_pages-1) {
  199. int last_page_size = store->bytes & (PAGE_SIZE-1);
  200. if (last_page_size == 0)
  201. last_page_size = PAGE_SIZE;
  202. size = roundup(last_page_size,
  203. bdev_logical_block_size(bdev));
  204. }
  205. /* Just make sure we aren't corrupting data or
  206. * metadata
  207. */
  208. if (mddev->external) {
  209. /* Bitmap could be anywhere. */
  210. if (rdev->sb_start + offset + (page->index
  211. * (PAGE_SIZE/512))
  212. > rdev->data_offset
  213. &&
  214. rdev->sb_start + offset
  215. < (rdev->data_offset + mddev->dev_sectors
  216. + (PAGE_SIZE/512)))
  217. goto bad_alignment;
  218. } else if (offset < 0) {
  219. /* DATA BITMAP METADATA */
  220. if (offset
  221. + (long)(page->index * (PAGE_SIZE/512))
  222. + size/512 > 0)
  223. /* bitmap runs in to metadata */
  224. goto bad_alignment;
  225. if (rdev->data_offset + mddev->dev_sectors
  226. > rdev->sb_start + offset)
  227. /* data runs in to bitmap */
  228. goto bad_alignment;
  229. } else if (rdev->sb_start < rdev->data_offset) {
  230. /* METADATA BITMAP DATA */
  231. if (rdev->sb_start
  232. + offset
  233. + page->index*(PAGE_SIZE/512) + size/512
  234. > rdev->data_offset)
  235. /* bitmap runs in to data */
  236. goto bad_alignment;
  237. } else {
  238. /* DATA METADATA BITMAP - no problems */
  239. }
  240. md_super_write(mddev, rdev,
  241. rdev->sb_start + offset
  242. + page->index * (PAGE_SIZE/512),
  243. size,
  244. page);
  245. }
  246. if (wait && md_super_wait(mddev) < 0)
  247. goto restart;
  248. return 0;
  249. bad_alignment:
  250. return -EINVAL;
  251. }
  252. static void md_bitmap_file_kick(struct bitmap *bitmap);
  253. /*
  254. * write out a page to a file
  255. */
  256. static void write_page(struct bitmap *bitmap, struct page *page, int wait)
  257. {
  258. struct buffer_head *bh;
  259. if (bitmap->storage.file == NULL) {
  260. switch (write_sb_page(bitmap, page, wait)) {
  261. case -EINVAL:
  262. set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
  263. }
  264. } else {
  265. bh = page_buffers(page);
  266. while (bh && bh->b_blocknr) {
  267. atomic_inc(&bitmap->pending_writes);
  268. set_buffer_locked(bh);
  269. set_buffer_mapped(bh);
  270. submit_bh(REQ_OP_WRITE, REQ_SYNC, bh);
  271. bh = bh->b_this_page;
  272. }
  273. if (wait)
  274. wait_event(bitmap->write_wait,
  275. atomic_read(&bitmap->pending_writes)==0);
  276. }
  277. if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
  278. md_bitmap_file_kick(bitmap);
  279. }
  280. static void end_bitmap_write(struct buffer_head *bh, int uptodate)
  281. {
  282. struct bitmap *bitmap = bh->b_private;
  283. if (!uptodate)
  284. set_bit(BITMAP_WRITE_ERROR, &bitmap->flags);
  285. if (atomic_dec_and_test(&bitmap->pending_writes))
  286. wake_up(&bitmap->write_wait);
  287. }
  288. static void free_buffers(struct page *page)
  289. {
  290. struct buffer_head *bh;
  291. if (!PagePrivate(page))
  292. return;
  293. bh = page_buffers(page);
  294. while (bh) {
  295. struct buffer_head *next = bh->b_this_page;
  296. free_buffer_head(bh);
  297. bh = next;
  298. }
  299. detach_page_private(page);
  300. put_page(page);
  301. }
  302. /* read a page from a file.
  303. * We both read the page, and attach buffers to the page to record the
  304. * address of each block (using bmap). These addresses will be used
  305. * to write the block later, completely bypassing the filesystem.
  306. * This usage is similar to how swap files are handled, and allows us
  307. * to write to a file with no concerns of memory allocation failing.
  308. */
  309. static int read_page(struct file *file, unsigned long index,
  310. struct bitmap *bitmap,
  311. unsigned long count,
  312. struct page *page)
  313. {
  314. int ret = 0;
  315. struct inode *inode = file_inode(file);
  316. struct buffer_head *bh;
  317. sector_t block, blk_cur;
  318. unsigned long blocksize = i_blocksize(inode);
  319. pr_debug("read bitmap file (%dB @ %llu)\n", (int)PAGE_SIZE,
  320. (unsigned long long)index << PAGE_SHIFT);
  321. bh = alloc_page_buffers(page, blocksize, false);
  322. if (!bh) {
  323. ret = -ENOMEM;
  324. goto out;
  325. }
  326. attach_page_private(page, bh);
  327. blk_cur = index << (PAGE_SHIFT - inode->i_blkbits);
  328. while (bh) {
  329. block = blk_cur;
  330. if (count == 0)
  331. bh->b_blocknr = 0;
  332. else {
  333. ret = bmap(inode, &block);
  334. if (ret || !block) {
  335. ret = -EINVAL;
  336. bh->b_blocknr = 0;
  337. goto out;
  338. }
  339. bh->b_blocknr = block;
  340. bh->b_bdev = inode->i_sb->s_bdev;
  341. if (count < blocksize)
  342. count = 0;
  343. else
  344. count -= blocksize;
  345. bh->b_end_io = end_bitmap_write;
  346. bh->b_private = bitmap;
  347. atomic_inc(&bitmap->pending_writes);
  348. set_buffer_locked(bh);
  349. set_buffer_mapped(bh);
  350. submit_bh(REQ_OP_READ, 0, bh);
  351. }
  352. blk_cur++;
  353. bh = bh->b_this_page;
  354. }
  355. page->index = index;
  356. wait_event(bitmap->write_wait,
  357. atomic_read(&bitmap->pending_writes)==0);
  358. if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
  359. ret = -EIO;
  360. out:
  361. if (ret)
  362. pr_err("md: bitmap read error: (%dB @ %llu): %d\n",
  363. (int)PAGE_SIZE,
  364. (unsigned long long)index << PAGE_SHIFT,
  365. ret);
  366. return ret;
  367. }
  368. /*
  369. * bitmap file superblock operations
  370. */
  371. /*
  372. * md_bitmap_wait_writes() should be called before writing any bitmap
  373. * blocks, to ensure previous writes, particularly from
  374. * md_bitmap_daemon_work(), have completed.
  375. */
  376. static void md_bitmap_wait_writes(struct bitmap *bitmap)
  377. {
  378. if (bitmap->storage.file)
  379. wait_event(bitmap->write_wait,
  380. atomic_read(&bitmap->pending_writes)==0);
  381. else
  382. /* Note that we ignore the return value. The writes
  383. * might have failed, but that would just mean that
  384. * some bits which should be cleared haven't been,
  385. * which is safe. The relevant bitmap blocks will
  386. * probably get written again, but there is no great
  387. * loss if they aren't.
  388. */
  389. md_super_wait(bitmap->mddev);
  390. }
  391. /* update the event counter and sync the superblock to disk */
  392. void md_bitmap_update_sb(struct bitmap *bitmap)
  393. {
  394. bitmap_super_t *sb;
  395. if (!bitmap || !bitmap->mddev) /* no bitmap for this array */
  396. return;
  397. if (bitmap->mddev->bitmap_info.external)
  398. return;
  399. if (!bitmap->storage.sb_page) /* no superblock */
  400. return;
  401. sb = kmap_atomic(bitmap->storage.sb_page);
  402. sb->events = cpu_to_le64(bitmap->mddev->events);
  403. if (bitmap->mddev->events < bitmap->events_cleared)
  404. /* rocking back to read-only */
  405. bitmap->events_cleared = bitmap->mddev->events;
  406. sb->events_cleared = cpu_to_le64(bitmap->events_cleared);
  407. /*
  408. * clear BITMAP_WRITE_ERROR bit to protect against the case that
  409. * a bitmap write error occurred but the later writes succeeded.
  410. */
  411. sb->state = cpu_to_le32(bitmap->flags & ~BIT(BITMAP_WRITE_ERROR));
  412. /* Just in case these have been changed via sysfs: */
  413. sb->daemon_sleep = cpu_to_le32(bitmap->mddev->bitmap_info.daemon_sleep/HZ);
  414. sb->write_behind = cpu_to_le32(bitmap->mddev->bitmap_info.max_write_behind);
  415. /* This might have been changed by a reshape */
  416. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  417. sb->chunksize = cpu_to_le32(bitmap->mddev->bitmap_info.chunksize);
  418. sb->nodes = cpu_to_le32(bitmap->mddev->bitmap_info.nodes);
  419. sb->sectors_reserved = cpu_to_le32(bitmap->mddev->
  420. bitmap_info.space);
  421. kunmap_atomic(sb);
  422. write_page(bitmap, bitmap->storage.sb_page, 1);
  423. }
  424. EXPORT_SYMBOL(md_bitmap_update_sb);
  425. /* print out the bitmap file superblock */
  426. void md_bitmap_print_sb(struct bitmap *bitmap)
  427. {
  428. bitmap_super_t *sb;
  429. if (!bitmap || !bitmap->storage.sb_page)
  430. return;
  431. sb = kmap_atomic(bitmap->storage.sb_page);
  432. pr_debug("%s: bitmap file superblock:\n", bmname(bitmap));
  433. pr_debug(" magic: %08x\n", le32_to_cpu(sb->magic));
  434. pr_debug(" version: %d\n", le32_to_cpu(sb->version));
  435. pr_debug(" uuid: %08x.%08x.%08x.%08x\n",
  436. le32_to_cpu(*(__le32 *)(sb->uuid+0)),
  437. le32_to_cpu(*(__le32 *)(sb->uuid+4)),
  438. le32_to_cpu(*(__le32 *)(sb->uuid+8)),
  439. le32_to_cpu(*(__le32 *)(sb->uuid+12)));
  440. pr_debug(" events: %llu\n",
  441. (unsigned long long) le64_to_cpu(sb->events));
  442. pr_debug("events cleared: %llu\n",
  443. (unsigned long long) le64_to_cpu(sb->events_cleared));
  444. pr_debug(" state: %08x\n", le32_to_cpu(sb->state));
  445. pr_debug(" chunksize: %d B\n", le32_to_cpu(sb->chunksize));
  446. pr_debug(" daemon sleep: %ds\n", le32_to_cpu(sb->daemon_sleep));
  447. pr_debug(" sync size: %llu KB\n",
  448. (unsigned long long)le64_to_cpu(sb->sync_size)/2);
  449. pr_debug("max write behind: %d\n", le32_to_cpu(sb->write_behind));
  450. kunmap_atomic(sb);
  451. }
  452. /*
  453. * bitmap_new_disk_sb
  454. * @bitmap
  455. *
  456. * This function is somewhat the reverse of bitmap_read_sb. bitmap_read_sb
  457. * reads and verifies the on-disk bitmap superblock and populates bitmap_info.
  458. * This function verifies 'bitmap_info' and populates the on-disk bitmap
  459. * structure, which is to be written to disk.
  460. *
  461. * Returns: 0 on success, -Exxx on error
  462. */
  463. static int md_bitmap_new_disk_sb(struct bitmap *bitmap)
  464. {
  465. bitmap_super_t *sb;
  466. unsigned long chunksize, daemon_sleep, write_behind;
  467. bitmap->storage.sb_page = alloc_page(GFP_KERNEL | __GFP_ZERO);
  468. if (bitmap->storage.sb_page == NULL)
  469. return -ENOMEM;
  470. bitmap->storage.sb_page->index = 0;
  471. sb = kmap_atomic(bitmap->storage.sb_page);
  472. sb->magic = cpu_to_le32(BITMAP_MAGIC);
  473. sb->version = cpu_to_le32(BITMAP_MAJOR_HI);
  474. chunksize = bitmap->mddev->bitmap_info.chunksize;
  475. BUG_ON(!chunksize);
  476. if (!is_power_of_2(chunksize)) {
  477. kunmap_atomic(sb);
  478. pr_warn("bitmap chunksize not a power of 2\n");
  479. return -EINVAL;
  480. }
  481. sb->chunksize = cpu_to_le32(chunksize);
  482. daemon_sleep = bitmap->mddev->bitmap_info.daemon_sleep;
  483. if (!daemon_sleep || (daemon_sleep > MAX_SCHEDULE_TIMEOUT)) {
  484. pr_debug("Choosing daemon_sleep default (5 sec)\n");
  485. daemon_sleep = 5 * HZ;
  486. }
  487. sb->daemon_sleep = cpu_to_le32(daemon_sleep);
  488. bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
  489. /*
  490. * FIXME: write_behind for RAID1. If not specified, what
  491. * is a good choice? We choose COUNTER_MAX / 2 arbitrarily.
  492. */
  493. write_behind = bitmap->mddev->bitmap_info.max_write_behind;
  494. if (write_behind > COUNTER_MAX)
  495. write_behind = COUNTER_MAX / 2;
  496. sb->write_behind = cpu_to_le32(write_behind);
  497. bitmap->mddev->bitmap_info.max_write_behind = write_behind;
  498. /* keep the array size field of the bitmap superblock up to date */
  499. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  500. memcpy(sb->uuid, bitmap->mddev->uuid, 16);
  501. set_bit(BITMAP_STALE, &bitmap->flags);
  502. sb->state = cpu_to_le32(bitmap->flags);
  503. bitmap->events_cleared = bitmap->mddev->events;
  504. sb->events_cleared = cpu_to_le64(bitmap->mddev->events);
  505. bitmap->mddev->bitmap_info.nodes = 0;
  506. kunmap_atomic(sb);
  507. return 0;
  508. }
  509. /* read the superblock from the bitmap file and initialize some bitmap fields */
  510. static int md_bitmap_read_sb(struct bitmap *bitmap)
  511. {
  512. char *reason = NULL;
  513. bitmap_super_t *sb;
  514. unsigned long chunksize, daemon_sleep, write_behind;
  515. unsigned long long events;
  516. int nodes = 0;
  517. unsigned long sectors_reserved = 0;
  518. int err = -EINVAL;
  519. struct page *sb_page;
  520. loff_t offset = bitmap->mddev->bitmap_info.offset;
  521. if (!bitmap->storage.file && !bitmap->mddev->bitmap_info.offset) {
  522. chunksize = 128 * 1024 * 1024;
  523. daemon_sleep = 5 * HZ;
  524. write_behind = 0;
  525. set_bit(BITMAP_STALE, &bitmap->flags);
  526. err = 0;
  527. goto out_no_sb;
  528. }
  529. /* page 0 is the superblock, read it... */
  530. sb_page = alloc_page(GFP_KERNEL);
  531. if (!sb_page)
  532. return -ENOMEM;
  533. bitmap->storage.sb_page = sb_page;
  534. re_read:
  535. /* If cluster_slot is set, the cluster is setup */
  536. if (bitmap->cluster_slot >= 0) {
  537. sector_t bm_blocks = bitmap->mddev->resync_max_sectors;
  538. bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks,
  539. (bitmap->mddev->bitmap_info.chunksize >> 9));
  540. /* bits to bytes */
  541. bm_blocks = ((bm_blocks+7) >> 3) + sizeof(bitmap_super_t);
  542. /* to 4k blocks */
  543. bm_blocks = DIV_ROUND_UP_SECTOR_T(bm_blocks, 4096);
  544. offset = bitmap->mddev->bitmap_info.offset + (bitmap->cluster_slot * (bm_blocks << 3));
  545. pr_debug("%s:%d bm slot: %d offset: %llu\n", __func__, __LINE__,
  546. bitmap->cluster_slot, offset);
  547. }
  548. if (bitmap->storage.file) {
  549. loff_t isize = i_size_read(bitmap->storage.file->f_mapping->host);
  550. int bytes = isize > PAGE_SIZE ? PAGE_SIZE : isize;
  551. err = read_page(bitmap->storage.file, 0,
  552. bitmap, bytes, sb_page);
  553. } else {
  554. err = read_sb_page(bitmap->mddev,
  555. offset,
  556. sb_page,
  557. 0, sizeof(bitmap_super_t));
  558. }
  559. if (err)
  560. return err;
  561. err = -EINVAL;
  562. sb = kmap_atomic(sb_page);
  563. chunksize = le32_to_cpu(sb->chunksize);
  564. daemon_sleep = le32_to_cpu(sb->daemon_sleep) * HZ;
  565. write_behind = le32_to_cpu(sb->write_behind);
  566. sectors_reserved = le32_to_cpu(sb->sectors_reserved);
  567. /* Setup nodes/clustername only if bitmap version is
  568. * cluster-compatible
  569. */
  570. if (sb->version == cpu_to_le32(BITMAP_MAJOR_CLUSTERED)) {
  571. nodes = le32_to_cpu(sb->nodes);
  572. strlcpy(bitmap->mddev->bitmap_info.cluster_name,
  573. sb->cluster_name, 64);
  574. }
  575. /* verify that the bitmap-specific fields are valid */
  576. if (sb->magic != cpu_to_le32(BITMAP_MAGIC))
  577. reason = "bad magic";
  578. else if (le32_to_cpu(sb->version) < BITMAP_MAJOR_LO ||
  579. le32_to_cpu(sb->version) > BITMAP_MAJOR_CLUSTERED)
  580. reason = "unrecognized superblock version";
  581. else if (chunksize < 512)
  582. reason = "bitmap chunksize too small";
  583. else if (!is_power_of_2(chunksize))
  584. reason = "bitmap chunksize not a power of 2";
  585. else if (daemon_sleep < 1 || daemon_sleep > MAX_SCHEDULE_TIMEOUT)
  586. reason = "daemon sleep period out of range";
  587. else if (write_behind > COUNTER_MAX)
  588. reason = "write-behind limit out of range (0 - 16383)";
  589. if (reason) {
  590. pr_warn("%s: invalid bitmap file superblock: %s\n",
  591. bmname(bitmap), reason);
  592. goto out;
  593. }
  594. /* keep the array size field of the bitmap superblock up to date */
  595. sb->sync_size = cpu_to_le64(bitmap->mddev->resync_max_sectors);
  596. if (bitmap->mddev->persistent) {
  597. /*
  598. * We have a persistent array superblock, so compare the
  599. * bitmap's UUID and event counter to the mddev's
  600. */
  601. if (memcmp(sb->uuid, bitmap->mddev->uuid, 16)) {
  602. pr_warn("%s: bitmap superblock UUID mismatch\n",
  603. bmname(bitmap));
  604. goto out;
  605. }
  606. events = le64_to_cpu(sb->events);
  607. if (!nodes && (events < bitmap->mddev->events)) {
  608. pr_warn("%s: bitmap file is out of date (%llu < %llu) -- forcing full recovery\n",
  609. bmname(bitmap), events,
  610. (unsigned long long) bitmap->mddev->events);
  611. set_bit(BITMAP_STALE, &bitmap->flags);
  612. }
  613. }
  614. /* assign fields using values from superblock */
  615. bitmap->flags |= le32_to_cpu(sb->state);
  616. if (le32_to_cpu(sb->version) == BITMAP_MAJOR_HOSTENDIAN)
  617. set_bit(BITMAP_HOSTENDIAN, &bitmap->flags);
  618. bitmap->events_cleared = le64_to_cpu(sb->events_cleared);
  619. strlcpy(bitmap->mddev->bitmap_info.cluster_name, sb->cluster_name, 64);
  620. err = 0;
  621. out:
  622. kunmap_atomic(sb);
  623. /* Assigning chunksize is required for "re_read" */
  624. bitmap->mddev->bitmap_info.chunksize = chunksize;
  625. if (err == 0 && nodes && (bitmap->cluster_slot < 0)) {
  626. err = md_setup_cluster(bitmap->mddev, nodes);
  627. if (err) {
  628. pr_warn("%s: Could not setup cluster service (%d)\n",
  629. bmname(bitmap), err);
  630. goto out_no_sb;
  631. }
  632. bitmap->cluster_slot = md_cluster_ops->slot_number(bitmap->mddev);
  633. goto re_read;
  634. }
  635. out_no_sb:
  636. if (test_bit(BITMAP_STALE, &bitmap->flags))
  637. bitmap->events_cleared = bitmap->mddev->events;
  638. bitmap->mddev->bitmap_info.chunksize = chunksize;
  639. bitmap->mddev->bitmap_info.daemon_sleep = daemon_sleep;
  640. bitmap->mddev->bitmap_info.max_write_behind = write_behind;
  641. bitmap->mddev->bitmap_info.nodes = nodes;
  642. if (bitmap->mddev->bitmap_info.space == 0 ||
  643. bitmap->mddev->bitmap_info.space > sectors_reserved)
  644. bitmap->mddev->bitmap_info.space = sectors_reserved;
  645. if (err) {
  646. md_bitmap_print_sb(bitmap);
  647. if (bitmap->cluster_slot < 0)
  648. md_cluster_stop(bitmap->mddev);
  649. }
  650. return err;
  651. }
  652. /*
  653. * general bitmap file operations
  654. */
  655. /*
  656. * on-disk bitmap:
  657. *
  658. * Use one bit per "chunk" (block set). We do the disk I/O on the bitmap
  659. * file a page at a time. There's a superblock at the start of the file.
  660. */
  661. /* calculate the index of the page that contains this bit */
  662. static inline unsigned long file_page_index(struct bitmap_storage *store,
  663. unsigned long chunk)
  664. {
  665. if (store->sb_page)
  666. chunk += sizeof(bitmap_super_t) << 3;
  667. return chunk >> PAGE_BIT_SHIFT;
  668. }
  669. /* calculate the (bit) offset of this bit within a page */
  670. static inline unsigned long file_page_offset(struct bitmap_storage *store,
  671. unsigned long chunk)
  672. {
  673. if (store->sb_page)
  674. chunk += sizeof(bitmap_super_t) << 3;
  675. return chunk & (PAGE_BITS - 1);
  676. }
  677. /*
  678. * return a pointer to the page in the filemap that contains the given bit
  679. *
  680. */
  681. static inline struct page *filemap_get_page(struct bitmap_storage *store,
  682. unsigned long chunk)
  683. {
  684. if (file_page_index(store, chunk) >= store->file_pages)
  685. return NULL;
  686. return store->filemap[file_page_index(store, chunk)];
  687. }
  688. static int md_bitmap_storage_alloc(struct bitmap_storage *store,
  689. unsigned long chunks, int with_super,
  690. int slot_number)
  691. {
  692. int pnum, offset = 0;
  693. unsigned long num_pages;
  694. unsigned long bytes;
  695. bytes = DIV_ROUND_UP(chunks, 8);
  696. if (with_super)
  697. bytes += sizeof(bitmap_super_t);
  698. num_pages = DIV_ROUND_UP(bytes, PAGE_SIZE);
  699. offset = slot_number * num_pages;
  700. store->filemap = kmalloc_array(num_pages, sizeof(struct page *),
  701. GFP_KERNEL);
  702. if (!store->filemap)
  703. return -ENOMEM;
  704. if (with_super && !store->sb_page) {
  705. store->sb_page = alloc_page(GFP_KERNEL|__GFP_ZERO);
  706. if (store->sb_page == NULL)
  707. return -ENOMEM;
  708. }
  709. pnum = 0;
  710. if (store->sb_page) {
  711. store->filemap[0] = store->sb_page;
  712. pnum = 1;
  713. store->sb_page->index = offset;
  714. }
  715. for ( ; pnum < num_pages; pnum++) {
  716. store->filemap[pnum] = alloc_page(GFP_KERNEL|__GFP_ZERO);
  717. if (!store->filemap[pnum]) {
  718. store->file_pages = pnum;
  719. return -ENOMEM;
  720. }
  721. store->filemap[pnum]->index = pnum + offset;
  722. }
  723. store->file_pages = pnum;
  724. /* We need 4 bits per page, rounded up to a multiple
  725. * of sizeof(unsigned long) */
  726. store->filemap_attr = kzalloc(
  727. roundup(DIV_ROUND_UP(num_pages*4, 8), sizeof(unsigned long)),
  728. GFP_KERNEL);
  729. if (!store->filemap_attr)
  730. return -ENOMEM;
  731. store->bytes = bytes;
  732. return 0;
  733. }
  734. static void md_bitmap_file_unmap(struct bitmap_storage *store)
  735. {
  736. struct page **map, *sb_page;
  737. int pages;
  738. struct file *file;
  739. file = store->file;
  740. map = store->filemap;
  741. pages = store->file_pages;
  742. sb_page = store->sb_page;
  743. while (pages--)
  744. if (map[pages] != sb_page) /* 0 is sb_page, release it below */
  745. free_buffers(map[pages]);
  746. kfree(map);
  747. kfree(store->filemap_attr);
  748. if (sb_page)
  749. free_buffers(sb_page);
  750. if (file) {
  751. struct inode *inode = file_inode(file);
  752. invalidate_mapping_pages(inode->i_mapping, 0, -1);
  753. fput(file);
  754. }
  755. }
  756. /*
  757. * bitmap_file_kick - if an error occurs while manipulating the bitmap file
  758. * then it is no longer reliable, so we stop using it and we mark the file
  759. * as failed in the superblock
  760. */
  761. static void md_bitmap_file_kick(struct bitmap *bitmap)
  762. {
  763. char *path, *ptr = NULL;
  764. if (!test_and_set_bit(BITMAP_STALE, &bitmap->flags)) {
  765. md_bitmap_update_sb(bitmap);
  766. if (bitmap->storage.file) {
  767. path = kmalloc(PAGE_SIZE, GFP_KERNEL);
  768. if (path)
  769. ptr = file_path(bitmap->storage.file,
  770. path, PAGE_SIZE);
  771. pr_warn("%s: kicking failed bitmap file %s from array!\n",
  772. bmname(bitmap), IS_ERR(ptr) ? "" : ptr);
  773. kfree(path);
  774. } else
  775. pr_warn("%s: disabling internal bitmap due to errors\n",
  776. bmname(bitmap));
  777. }
  778. }
  779. enum bitmap_page_attr {
  780. BITMAP_PAGE_DIRTY = 0, /* there are set bits that need to be synced */
  781. BITMAP_PAGE_PENDING = 1, /* there are bits that are being cleaned.
  782. * i.e. counter is 1 or 2. */
  783. BITMAP_PAGE_NEEDWRITE = 2, /* there are cleared bits that need to be synced */
  784. };
  785. static inline void set_page_attr(struct bitmap *bitmap, int pnum,
  786. enum bitmap_page_attr attr)
  787. {
  788. set_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
  789. }
  790. static inline void clear_page_attr(struct bitmap *bitmap, int pnum,
  791. enum bitmap_page_attr attr)
  792. {
  793. clear_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
  794. }
  795. static inline int test_page_attr(struct bitmap *bitmap, int pnum,
  796. enum bitmap_page_attr attr)
  797. {
  798. return test_bit((pnum<<2) + attr, bitmap->storage.filemap_attr);
  799. }
  800. static inline int test_and_clear_page_attr(struct bitmap *bitmap, int pnum,
  801. enum bitmap_page_attr attr)
  802. {
  803. return test_and_clear_bit((pnum<<2) + attr,
  804. bitmap->storage.filemap_attr);
  805. }
  806. /*
  807. * bitmap_file_set_bit -- called before performing a write to the md device
  808. * to set (and eventually sync) a particular bit in the bitmap file
  809. *
  810. * we set the bit immediately, then we record the page number so that
  811. * when an unplug occurs, we can flush the dirty pages out to disk
  812. */
  813. static void md_bitmap_file_set_bit(struct bitmap *bitmap, sector_t block)
  814. {
  815. unsigned long bit;
  816. struct page *page;
  817. void *kaddr;
  818. unsigned long chunk = block >> bitmap->counts.chunkshift;
  819. struct bitmap_storage *store = &bitmap->storage;
  820. unsigned long node_offset = 0;
  821. if (mddev_is_clustered(bitmap->mddev))
  822. node_offset = bitmap->cluster_slot * store->file_pages;
  823. page = filemap_get_page(&bitmap->storage, chunk);
  824. if (!page)
  825. return;
  826. bit = file_page_offset(&bitmap->storage, chunk);
  827. /* set the bit */
  828. kaddr = kmap_atomic(page);
  829. if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
  830. set_bit(bit, kaddr);
  831. else
  832. set_bit_le(bit, kaddr);
  833. kunmap_atomic(kaddr);
  834. pr_debug("set file bit %lu page %lu\n", bit, page->index);
  835. /* record page number so it gets flushed to disk when unplug occurs */
  836. set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_DIRTY);
  837. }
  838. static void md_bitmap_file_clear_bit(struct bitmap *bitmap, sector_t block)
  839. {
  840. unsigned long bit;
  841. struct page *page;
  842. void *paddr;
  843. unsigned long chunk = block >> bitmap->counts.chunkshift;
  844. struct bitmap_storage *store = &bitmap->storage;
  845. unsigned long node_offset = 0;
  846. if (mddev_is_clustered(bitmap->mddev))
  847. node_offset = bitmap->cluster_slot * store->file_pages;
  848. page = filemap_get_page(&bitmap->storage, chunk);
  849. if (!page)
  850. return;
  851. bit = file_page_offset(&bitmap->storage, chunk);
  852. paddr = kmap_atomic(page);
  853. if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
  854. clear_bit(bit, paddr);
  855. else
  856. clear_bit_le(bit, paddr);
  857. kunmap_atomic(paddr);
  858. if (!test_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_NEEDWRITE)) {
  859. set_page_attr(bitmap, page->index - node_offset, BITMAP_PAGE_PENDING);
  860. bitmap->allclean = 0;
  861. }
  862. }
  863. static int md_bitmap_file_test_bit(struct bitmap *bitmap, sector_t block)
  864. {
  865. unsigned long bit;
  866. struct page *page;
  867. void *paddr;
  868. unsigned long chunk = block >> bitmap->counts.chunkshift;
  869. int set = 0;
  870. page = filemap_get_page(&bitmap->storage, chunk);
  871. if (!page)
  872. return -EINVAL;
  873. bit = file_page_offset(&bitmap->storage, chunk);
  874. paddr = kmap_atomic(page);
  875. if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
  876. set = test_bit(bit, paddr);
  877. else
  878. set = test_bit_le(bit, paddr);
  879. kunmap_atomic(paddr);
  880. return set;
  881. }
  882. /* this gets called when the md device is ready to unplug its underlying
  883. * (slave) device queues -- before we let any writes go down, we need to
  884. * sync the dirty pages of the bitmap file to disk */
  885. void md_bitmap_unplug(struct bitmap *bitmap)
  886. {
  887. unsigned long i;
  888. int dirty, need_write;
  889. int writing = 0;
  890. if (!bitmap || !bitmap->storage.filemap ||
  891. test_bit(BITMAP_STALE, &bitmap->flags))
  892. return;
  893. /* look at each page to see if there are any set bits that need to be
  894. * flushed out to disk */
  895. for (i = 0; i < bitmap->storage.file_pages; i++) {
  896. dirty = test_and_clear_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
  897. need_write = test_and_clear_page_attr(bitmap, i,
  898. BITMAP_PAGE_NEEDWRITE);
  899. if (dirty || need_write) {
  900. if (!writing) {
  901. md_bitmap_wait_writes(bitmap);
  902. if (bitmap->mddev->queue)
  903. blk_add_trace_msg(bitmap->mddev->queue,
  904. "md bitmap_unplug");
  905. }
  906. clear_page_attr(bitmap, i, BITMAP_PAGE_PENDING);
  907. write_page(bitmap, bitmap->storage.filemap[i], 0);
  908. writing = 1;
  909. }
  910. }
  911. if (writing)
  912. md_bitmap_wait_writes(bitmap);
  913. if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
  914. md_bitmap_file_kick(bitmap);
  915. }
  916. EXPORT_SYMBOL(md_bitmap_unplug);
  917. static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed);
  918. /* * bitmap_init_from_disk -- called at bitmap_create time to initialize
  919. * the in-memory bitmap from the on-disk bitmap -- also, sets up the
  920. * memory mapping of the bitmap file
  921. * Special cases:
  922. * if there's no bitmap file, or if the bitmap file had been
  923. * previously kicked from the array, we mark all the bits as
  924. * 1's in order to cause a full resync.
  925. *
  926. * We ignore all bits for sectors that end earlier than 'start'.
  927. * This is used when reading an out-of-date bitmap...
  928. */
  929. static int md_bitmap_init_from_disk(struct bitmap *bitmap, sector_t start)
  930. {
  931. unsigned long i, chunks, index, oldindex, bit, node_offset = 0;
  932. struct page *page = NULL;
  933. unsigned long bit_cnt = 0;
  934. struct file *file;
  935. unsigned long offset;
  936. int outofdate;
  937. int ret = -ENOSPC;
  938. void *paddr;
  939. struct bitmap_storage *store = &bitmap->storage;
  940. chunks = bitmap->counts.chunks;
  941. file = store->file;
  942. if (!file && !bitmap->mddev->bitmap_info.offset) {
  943. /* No permanent bitmap - fill with '1s'. */
  944. store->filemap = NULL;
  945. store->file_pages = 0;
  946. for (i = 0; i < chunks ; i++) {
  947. /* if the disk bit is set, set the memory bit */
  948. int needed = ((sector_t)(i+1) << (bitmap->counts.chunkshift)
  949. >= start);
  950. md_bitmap_set_memory_bits(bitmap,
  951. (sector_t)i << bitmap->counts.chunkshift,
  952. needed);
  953. }
  954. return 0;
  955. }
  956. outofdate = test_bit(BITMAP_STALE, &bitmap->flags);
  957. if (outofdate)
  958. pr_warn("%s: bitmap file is out of date, doing full recovery\n", bmname(bitmap));
  959. if (file && i_size_read(file->f_mapping->host) < store->bytes) {
  960. pr_warn("%s: bitmap file too short %lu < %lu\n",
  961. bmname(bitmap),
  962. (unsigned long) i_size_read(file->f_mapping->host),
  963. store->bytes);
  964. goto err;
  965. }
  966. oldindex = ~0L;
  967. offset = 0;
  968. if (!bitmap->mddev->bitmap_info.external)
  969. offset = sizeof(bitmap_super_t);
  970. if (mddev_is_clustered(bitmap->mddev))
  971. node_offset = bitmap->cluster_slot * (DIV_ROUND_UP(store->bytes, PAGE_SIZE));
  972. for (i = 0; i < chunks; i++) {
  973. int b;
  974. index = file_page_index(&bitmap->storage, i);
  975. bit = file_page_offset(&bitmap->storage, i);
  976. if (index != oldindex) { /* this is a new page, read it in */
  977. int count;
  978. /* unmap the old page, we're done with it */
  979. if (index == store->file_pages-1)
  980. count = store->bytes - index * PAGE_SIZE;
  981. else
  982. count = PAGE_SIZE;
  983. page = store->filemap[index];
  984. if (file)
  985. ret = read_page(file, index, bitmap,
  986. count, page);
  987. else
  988. ret = read_sb_page(
  989. bitmap->mddev,
  990. bitmap->mddev->bitmap_info.offset,
  991. page,
  992. index + node_offset, count);
  993. if (ret)
  994. goto err;
  995. oldindex = index;
  996. if (outofdate) {
  997. /*
  998. * if bitmap is out of date, dirty the
  999. * whole page and write it out
  1000. */
  1001. paddr = kmap_atomic(page);
  1002. memset(paddr + offset, 0xff,
  1003. PAGE_SIZE - offset);
  1004. kunmap_atomic(paddr);
  1005. write_page(bitmap, page, 1);
  1006. ret = -EIO;
  1007. if (test_bit(BITMAP_WRITE_ERROR,
  1008. &bitmap->flags))
  1009. goto err;
  1010. }
  1011. }
  1012. paddr = kmap_atomic(page);
  1013. if (test_bit(BITMAP_HOSTENDIAN, &bitmap->flags))
  1014. b = test_bit(bit, paddr);
  1015. else
  1016. b = test_bit_le(bit, paddr);
  1017. kunmap_atomic(paddr);
  1018. if (b) {
  1019. /* if the disk bit is set, set the memory bit */
  1020. int needed = ((sector_t)(i+1) << bitmap->counts.chunkshift
  1021. >= start);
  1022. md_bitmap_set_memory_bits(bitmap,
  1023. (sector_t)i << bitmap->counts.chunkshift,
  1024. needed);
  1025. bit_cnt++;
  1026. }
  1027. offset = 0;
  1028. }
  1029. pr_debug("%s: bitmap initialized from disk: read %lu pages, set %lu of %lu bits\n",
  1030. bmname(bitmap), store->file_pages,
  1031. bit_cnt, chunks);
  1032. return 0;
  1033. err:
  1034. pr_warn("%s: bitmap initialisation failed: %d\n",
  1035. bmname(bitmap), ret);
  1036. return ret;
  1037. }
  1038. void md_bitmap_write_all(struct bitmap *bitmap)
  1039. {
  1040. /* We don't actually write all bitmap blocks here,
  1041. * just flag them as needing to be written
  1042. */
  1043. int i;
  1044. if (!bitmap || !bitmap->storage.filemap)
  1045. return;
  1046. if (bitmap->storage.file)
  1047. /* Only one copy, so nothing needed */
  1048. return;
  1049. for (i = 0; i < bitmap->storage.file_pages; i++)
  1050. set_page_attr(bitmap, i,
  1051. BITMAP_PAGE_NEEDWRITE);
  1052. bitmap->allclean = 0;
  1053. }
  1054. static void md_bitmap_count_page(struct bitmap_counts *bitmap,
  1055. sector_t offset, int inc)
  1056. {
  1057. sector_t chunk = offset >> bitmap->chunkshift;
  1058. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  1059. bitmap->bp[page].count += inc;
  1060. md_bitmap_checkfree(bitmap, page);
  1061. }
  1062. static void md_bitmap_set_pending(struct bitmap_counts *bitmap, sector_t offset)
  1063. {
  1064. sector_t chunk = offset >> bitmap->chunkshift;
  1065. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  1066. struct bitmap_page *bp = &bitmap->bp[page];
  1067. if (!bp->pending)
  1068. bp->pending = 1;
  1069. }
  1070. static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
  1071. sector_t offset, sector_t *blocks,
  1072. int create);
  1073. /*
  1074. * bitmap daemon -- periodically wakes up to clean bits and flush pages
  1075. * out to disk
  1076. */
  1077. void md_bitmap_daemon_work(struct mddev *mddev)
  1078. {
  1079. struct bitmap *bitmap;
  1080. unsigned long j;
  1081. unsigned long nextpage;
  1082. sector_t blocks;
  1083. struct bitmap_counts *counts;
  1084. /* Use a mutex to guard daemon_work against
  1085. * bitmap_destroy.
  1086. */
  1087. mutex_lock(&mddev->bitmap_info.mutex);
  1088. bitmap = mddev->bitmap;
  1089. if (bitmap == NULL) {
  1090. mutex_unlock(&mddev->bitmap_info.mutex);
  1091. return;
  1092. }
  1093. if (time_before(jiffies, bitmap->daemon_lastrun
  1094. + mddev->bitmap_info.daemon_sleep))
  1095. goto done;
  1096. bitmap->daemon_lastrun = jiffies;
  1097. if (bitmap->allclean) {
  1098. mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
  1099. goto done;
  1100. }
  1101. bitmap->allclean = 1;
  1102. if (bitmap->mddev->queue)
  1103. blk_add_trace_msg(bitmap->mddev->queue,
  1104. "md bitmap_daemon_work");
  1105. /* Any file-page which is PENDING now needs to be written.
  1106. * So set NEEDWRITE now, then after we make any last-minute changes
  1107. * we will write it.
  1108. */
  1109. for (j = 0; j < bitmap->storage.file_pages; j++)
  1110. if (test_and_clear_page_attr(bitmap, j,
  1111. BITMAP_PAGE_PENDING))
  1112. set_page_attr(bitmap, j,
  1113. BITMAP_PAGE_NEEDWRITE);
  1114. if (bitmap->need_sync &&
  1115. mddev->bitmap_info.external == 0) {
  1116. /* Arrange for superblock update as well as
  1117. * other changes */
  1118. bitmap_super_t *sb;
  1119. bitmap->need_sync = 0;
  1120. if (bitmap->storage.filemap) {
  1121. sb = kmap_atomic(bitmap->storage.sb_page);
  1122. sb->events_cleared =
  1123. cpu_to_le64(bitmap->events_cleared);
  1124. kunmap_atomic(sb);
  1125. set_page_attr(bitmap, 0,
  1126. BITMAP_PAGE_NEEDWRITE);
  1127. }
  1128. }
  1129. /* Now look at the bitmap counters and if any are '2' or '1',
  1130. * decrement and handle accordingly.
  1131. */
  1132. counts = &bitmap->counts;
  1133. spin_lock_irq(&counts->lock);
  1134. nextpage = 0;
  1135. for (j = 0; j < counts->chunks; j++) {
  1136. bitmap_counter_t *bmc;
  1137. sector_t block = (sector_t)j << counts->chunkshift;
  1138. if (j == nextpage) {
  1139. nextpage += PAGE_COUNTER_RATIO;
  1140. if (!counts->bp[j >> PAGE_COUNTER_SHIFT].pending) {
  1141. j |= PAGE_COUNTER_MASK;
  1142. continue;
  1143. }
  1144. counts->bp[j >> PAGE_COUNTER_SHIFT].pending = 0;
  1145. }
  1146. bmc = md_bitmap_get_counter(counts, block, &blocks, 0);
  1147. if (!bmc) {
  1148. j |= PAGE_COUNTER_MASK;
  1149. continue;
  1150. }
  1151. if (*bmc == 1 && !bitmap->need_sync) {
  1152. /* We can clear the bit */
  1153. *bmc = 0;
  1154. md_bitmap_count_page(counts, block, -1);
  1155. md_bitmap_file_clear_bit(bitmap, block);
  1156. } else if (*bmc && *bmc <= 2) {
  1157. *bmc = 1;
  1158. md_bitmap_set_pending(counts, block);
  1159. bitmap->allclean = 0;
  1160. }
  1161. }
  1162. spin_unlock_irq(&counts->lock);
  1163. md_bitmap_wait_writes(bitmap);
  1164. /* Now start writeout on any page in NEEDWRITE that isn't DIRTY.
  1165. * DIRTY pages need to be written by bitmap_unplug so it can wait
  1166. * for them.
  1167. * If we find any DIRTY page we stop there and let bitmap_unplug
  1168. * handle all the rest. This is important in the case where
  1169. * the first blocking holds the superblock and it has been updated.
  1170. * We mustn't write any other blocks before the superblock.
  1171. */
  1172. for (j = 0;
  1173. j < bitmap->storage.file_pages
  1174. && !test_bit(BITMAP_STALE, &bitmap->flags);
  1175. j++) {
  1176. if (test_page_attr(bitmap, j,
  1177. BITMAP_PAGE_DIRTY))
  1178. /* bitmap_unplug will handle the rest */
  1179. break;
  1180. if (bitmap->storage.filemap &&
  1181. test_and_clear_page_attr(bitmap, j,
  1182. BITMAP_PAGE_NEEDWRITE)) {
  1183. write_page(bitmap, bitmap->storage.filemap[j], 0);
  1184. }
  1185. }
  1186. done:
  1187. if (bitmap->allclean == 0)
  1188. mddev->thread->timeout =
  1189. mddev->bitmap_info.daemon_sleep;
  1190. mutex_unlock(&mddev->bitmap_info.mutex);
  1191. }
  1192. static bitmap_counter_t *md_bitmap_get_counter(struct bitmap_counts *bitmap,
  1193. sector_t offset, sector_t *blocks,
  1194. int create)
  1195. __releases(bitmap->lock)
  1196. __acquires(bitmap->lock)
  1197. {
  1198. /* If 'create', we might release the lock and reclaim it.
  1199. * The lock must have been taken with interrupts enabled.
  1200. * If !create, we don't release the lock.
  1201. */
  1202. sector_t chunk = offset >> bitmap->chunkshift;
  1203. unsigned long page = chunk >> PAGE_COUNTER_SHIFT;
  1204. unsigned long pageoff = (chunk & PAGE_COUNTER_MASK) << COUNTER_BYTE_SHIFT;
  1205. sector_t csize;
  1206. int err;
  1207. err = md_bitmap_checkpage(bitmap, page, create, 0);
  1208. if (bitmap->bp[page].hijacked ||
  1209. bitmap->bp[page].map == NULL)
  1210. csize = ((sector_t)1) << (bitmap->chunkshift +
  1211. PAGE_COUNTER_SHIFT);
  1212. else
  1213. csize = ((sector_t)1) << bitmap->chunkshift;
  1214. *blocks = csize - (offset & (csize - 1));
  1215. if (err < 0)
  1216. return NULL;
  1217. /* now locked ... */
  1218. if (bitmap->bp[page].hijacked) { /* hijacked pointer */
  1219. /* should we use the first or second counter field
  1220. * of the hijacked pointer? */
  1221. int hi = (pageoff > PAGE_COUNTER_MASK);
  1222. return &((bitmap_counter_t *)
  1223. &bitmap->bp[page].map)[hi];
  1224. } else /* page is allocated */
  1225. return (bitmap_counter_t *)
  1226. &(bitmap->bp[page].map[pageoff]);
  1227. }
  1228. int md_bitmap_startwrite(struct bitmap *bitmap, sector_t offset, unsigned long sectors, int behind)
  1229. {
  1230. if (!bitmap)
  1231. return 0;
  1232. if (behind) {
  1233. int bw;
  1234. atomic_inc(&bitmap->behind_writes);
  1235. bw = atomic_read(&bitmap->behind_writes);
  1236. if (bw > bitmap->behind_writes_used)
  1237. bitmap->behind_writes_used = bw;
  1238. pr_debug("inc write-behind count %d/%lu\n",
  1239. bw, bitmap->mddev->bitmap_info.max_write_behind);
  1240. }
  1241. while (sectors) {
  1242. sector_t blocks;
  1243. bitmap_counter_t *bmc;
  1244. spin_lock_irq(&bitmap->counts.lock);
  1245. bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 1);
  1246. if (!bmc) {
  1247. spin_unlock_irq(&bitmap->counts.lock);
  1248. return 0;
  1249. }
  1250. if (unlikely(COUNTER(*bmc) == COUNTER_MAX)) {
  1251. DEFINE_WAIT(__wait);
  1252. /* note that it is safe to do the prepare_to_wait
  1253. * after the test as long as we do it before dropping
  1254. * the spinlock.
  1255. */
  1256. prepare_to_wait(&bitmap->overflow_wait, &__wait,
  1257. TASK_UNINTERRUPTIBLE);
  1258. spin_unlock_irq(&bitmap->counts.lock);
  1259. schedule();
  1260. finish_wait(&bitmap->overflow_wait, &__wait);
  1261. continue;
  1262. }
  1263. switch (*bmc) {
  1264. case 0:
  1265. md_bitmap_file_set_bit(bitmap, offset);
  1266. md_bitmap_count_page(&bitmap->counts, offset, 1);
  1267. fallthrough;
  1268. case 1:
  1269. *bmc = 2;
  1270. }
  1271. (*bmc)++;
  1272. spin_unlock_irq(&bitmap->counts.lock);
  1273. offset += blocks;
  1274. if (sectors > blocks)
  1275. sectors -= blocks;
  1276. else
  1277. sectors = 0;
  1278. }
  1279. return 0;
  1280. }
  1281. EXPORT_SYMBOL(md_bitmap_startwrite);
  1282. void md_bitmap_endwrite(struct bitmap *bitmap, sector_t offset,
  1283. unsigned long sectors, int success, int behind)
  1284. {
  1285. if (!bitmap)
  1286. return;
  1287. if (behind) {
  1288. if (atomic_dec_and_test(&bitmap->behind_writes))
  1289. wake_up(&bitmap->behind_wait);
  1290. pr_debug("dec write-behind count %d/%lu\n",
  1291. atomic_read(&bitmap->behind_writes),
  1292. bitmap->mddev->bitmap_info.max_write_behind);
  1293. }
  1294. while (sectors) {
  1295. sector_t blocks;
  1296. unsigned long flags;
  1297. bitmap_counter_t *bmc;
  1298. spin_lock_irqsave(&bitmap->counts.lock, flags);
  1299. bmc = md_bitmap_get_counter(&bitmap->counts, offset, &blocks, 0);
  1300. if (!bmc) {
  1301. spin_unlock_irqrestore(&bitmap->counts.lock, flags);
  1302. return;
  1303. }
  1304. if (success && !bitmap->mddev->degraded &&
  1305. bitmap->events_cleared < bitmap->mddev->events) {
  1306. bitmap->events_cleared = bitmap->mddev->events;
  1307. bitmap->need_sync = 1;
  1308. sysfs_notify_dirent_safe(bitmap->sysfs_can_clear);
  1309. }
  1310. if (!success && !NEEDED(*bmc))
  1311. *bmc |= NEEDED_MASK;
  1312. if (COUNTER(*bmc) == COUNTER_MAX)
  1313. wake_up(&bitmap->overflow_wait);
  1314. (*bmc)--;
  1315. if (*bmc <= 2) {
  1316. md_bitmap_set_pending(&bitmap->counts, offset);
  1317. bitmap->allclean = 0;
  1318. }
  1319. spin_unlock_irqrestore(&bitmap->counts.lock, flags);
  1320. offset += blocks;
  1321. if (sectors > blocks)
  1322. sectors -= blocks;
  1323. else
  1324. sectors = 0;
  1325. }
  1326. }
  1327. EXPORT_SYMBOL(md_bitmap_endwrite);
  1328. static int __bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
  1329. int degraded)
  1330. {
  1331. bitmap_counter_t *bmc;
  1332. int rv;
  1333. if (bitmap == NULL) {/* FIXME or bitmap set as 'failed' */
  1334. *blocks = 1024;
  1335. return 1; /* always resync if no bitmap */
  1336. }
  1337. spin_lock_irq(&bitmap->counts.lock);
  1338. bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
  1339. rv = 0;
  1340. if (bmc) {
  1341. /* locked */
  1342. if (RESYNC(*bmc))
  1343. rv = 1;
  1344. else if (NEEDED(*bmc)) {
  1345. rv = 1;
  1346. if (!degraded) { /* don't set/clear bits if degraded */
  1347. *bmc |= RESYNC_MASK;
  1348. *bmc &= ~NEEDED_MASK;
  1349. }
  1350. }
  1351. }
  1352. spin_unlock_irq(&bitmap->counts.lock);
  1353. return rv;
  1354. }
  1355. int md_bitmap_start_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks,
  1356. int degraded)
  1357. {
  1358. /* bitmap_start_sync must always report on multiples of whole
  1359. * pages, otherwise resync (which is very PAGE_SIZE based) will
  1360. * get confused.
  1361. * So call __bitmap_start_sync repeatedly (if needed) until
  1362. * At least PAGE_SIZE>>9 blocks are covered.
  1363. * Return the 'or' of the result.
  1364. */
  1365. int rv = 0;
  1366. sector_t blocks1;
  1367. *blocks = 0;
  1368. while (*blocks < (PAGE_SIZE>>9)) {
  1369. rv |= __bitmap_start_sync(bitmap, offset,
  1370. &blocks1, degraded);
  1371. offset += blocks1;
  1372. *blocks += blocks1;
  1373. }
  1374. return rv;
  1375. }
  1376. EXPORT_SYMBOL(md_bitmap_start_sync);
  1377. void md_bitmap_end_sync(struct bitmap *bitmap, sector_t offset, sector_t *blocks, int aborted)
  1378. {
  1379. bitmap_counter_t *bmc;
  1380. unsigned long flags;
  1381. if (bitmap == NULL) {
  1382. *blocks = 1024;
  1383. return;
  1384. }
  1385. spin_lock_irqsave(&bitmap->counts.lock, flags);
  1386. bmc = md_bitmap_get_counter(&bitmap->counts, offset, blocks, 0);
  1387. if (bmc == NULL)
  1388. goto unlock;
  1389. /* locked */
  1390. if (RESYNC(*bmc)) {
  1391. *bmc &= ~RESYNC_MASK;
  1392. if (!NEEDED(*bmc) && aborted)
  1393. *bmc |= NEEDED_MASK;
  1394. else {
  1395. if (*bmc <= 2) {
  1396. md_bitmap_set_pending(&bitmap->counts, offset);
  1397. bitmap->allclean = 0;
  1398. }
  1399. }
  1400. }
  1401. unlock:
  1402. spin_unlock_irqrestore(&bitmap->counts.lock, flags);
  1403. }
  1404. EXPORT_SYMBOL(md_bitmap_end_sync);
  1405. void md_bitmap_close_sync(struct bitmap *bitmap)
  1406. {
  1407. /* Sync has finished, and any bitmap chunks that weren't synced
  1408. * properly have been aborted. It remains to us to clear the
  1409. * RESYNC bit wherever it is still on
  1410. */
  1411. sector_t sector = 0;
  1412. sector_t blocks;
  1413. if (!bitmap)
  1414. return;
  1415. while (sector < bitmap->mddev->resync_max_sectors) {
  1416. md_bitmap_end_sync(bitmap, sector, &blocks, 0);
  1417. sector += blocks;
  1418. }
  1419. }
  1420. EXPORT_SYMBOL(md_bitmap_close_sync);
  1421. void md_bitmap_cond_end_sync(struct bitmap *bitmap, sector_t sector, bool force)
  1422. {
  1423. sector_t s = 0;
  1424. sector_t blocks;
  1425. if (!bitmap)
  1426. return;
  1427. if (sector == 0) {
  1428. bitmap->last_end_sync = jiffies;
  1429. return;
  1430. }
  1431. if (!force && time_before(jiffies, (bitmap->last_end_sync
  1432. + bitmap->mddev->bitmap_info.daemon_sleep)))
  1433. return;
  1434. wait_event(bitmap->mddev->recovery_wait,
  1435. atomic_read(&bitmap->mddev->recovery_active) == 0);
  1436. bitmap->mddev->curr_resync_completed = sector;
  1437. set_bit(MD_SB_CHANGE_CLEAN, &bitmap->mddev->sb_flags);
  1438. sector &= ~((1ULL << bitmap->counts.chunkshift) - 1);
  1439. s = 0;
  1440. while (s < sector && s < bitmap->mddev->resync_max_sectors) {
  1441. md_bitmap_end_sync(bitmap, s, &blocks, 0);
  1442. s += blocks;
  1443. }
  1444. bitmap->last_end_sync = jiffies;
  1445. sysfs_notify_dirent_safe(bitmap->mddev->sysfs_completed);
  1446. }
  1447. EXPORT_SYMBOL(md_bitmap_cond_end_sync);
  1448. void md_bitmap_sync_with_cluster(struct mddev *mddev,
  1449. sector_t old_lo, sector_t old_hi,
  1450. sector_t new_lo, sector_t new_hi)
  1451. {
  1452. struct bitmap *bitmap = mddev->bitmap;
  1453. sector_t sector, blocks = 0;
  1454. for (sector = old_lo; sector < new_lo; ) {
  1455. md_bitmap_end_sync(bitmap, sector, &blocks, 0);
  1456. sector += blocks;
  1457. }
  1458. WARN((blocks > new_lo) && old_lo, "alignment is not correct for lo\n");
  1459. for (sector = old_hi; sector < new_hi; ) {
  1460. md_bitmap_start_sync(bitmap, sector, &blocks, 0);
  1461. sector += blocks;
  1462. }
  1463. WARN((blocks > new_hi) && old_hi, "alignment is not correct for hi\n");
  1464. }
  1465. EXPORT_SYMBOL(md_bitmap_sync_with_cluster);
  1466. static void md_bitmap_set_memory_bits(struct bitmap *bitmap, sector_t offset, int needed)
  1467. {
  1468. /* For each chunk covered by any of these sectors, set the
  1469. * counter to 2 and possibly set resync_needed. They should all
  1470. * be 0 at this point
  1471. */
  1472. sector_t secs;
  1473. bitmap_counter_t *bmc;
  1474. spin_lock_irq(&bitmap->counts.lock);
  1475. bmc = md_bitmap_get_counter(&bitmap->counts, offset, &secs, 1);
  1476. if (!bmc) {
  1477. spin_unlock_irq(&bitmap->counts.lock);
  1478. return;
  1479. }
  1480. if (!*bmc) {
  1481. *bmc = 2;
  1482. md_bitmap_count_page(&bitmap->counts, offset, 1);
  1483. md_bitmap_set_pending(&bitmap->counts, offset);
  1484. bitmap->allclean = 0;
  1485. }
  1486. if (needed)
  1487. *bmc |= NEEDED_MASK;
  1488. spin_unlock_irq(&bitmap->counts.lock);
  1489. }
  1490. /* dirty the memory and file bits for bitmap chunks "s" to "e" */
  1491. void md_bitmap_dirty_bits(struct bitmap *bitmap, unsigned long s, unsigned long e)
  1492. {
  1493. unsigned long chunk;
  1494. for (chunk = s; chunk <= e; chunk++) {
  1495. sector_t sec = (sector_t)chunk << bitmap->counts.chunkshift;
  1496. md_bitmap_set_memory_bits(bitmap, sec, 1);
  1497. md_bitmap_file_set_bit(bitmap, sec);
  1498. if (sec < bitmap->mddev->recovery_cp)
  1499. /* We are asserting that the array is dirty,
  1500. * so move the recovery_cp address back so
  1501. * that it is obvious that it is dirty
  1502. */
  1503. bitmap->mddev->recovery_cp = sec;
  1504. }
  1505. }
  1506. /*
  1507. * flush out any pending updates
  1508. */
  1509. void md_bitmap_flush(struct mddev *mddev)
  1510. {
  1511. struct bitmap *bitmap = mddev->bitmap;
  1512. long sleep;
  1513. if (!bitmap) /* there was no bitmap */
  1514. return;
  1515. /* run the daemon_work three time to ensure everything is flushed
  1516. * that can be
  1517. */
  1518. sleep = mddev->bitmap_info.daemon_sleep * 2;
  1519. bitmap->daemon_lastrun -= sleep;
  1520. md_bitmap_daemon_work(mddev);
  1521. bitmap->daemon_lastrun -= sleep;
  1522. md_bitmap_daemon_work(mddev);
  1523. bitmap->daemon_lastrun -= sleep;
  1524. md_bitmap_daemon_work(mddev);
  1525. if (mddev->bitmap_info.external)
  1526. md_super_wait(mddev);
  1527. md_bitmap_update_sb(bitmap);
  1528. }
  1529. /*
  1530. * free memory that was allocated
  1531. */
  1532. void md_bitmap_free(struct bitmap *bitmap)
  1533. {
  1534. unsigned long k, pages;
  1535. struct bitmap_page *bp;
  1536. if (!bitmap) /* there was no bitmap */
  1537. return;
  1538. if (bitmap->sysfs_can_clear)
  1539. sysfs_put(bitmap->sysfs_can_clear);
  1540. if (mddev_is_clustered(bitmap->mddev) && bitmap->mddev->cluster_info &&
  1541. bitmap->cluster_slot == md_cluster_ops->slot_number(bitmap->mddev))
  1542. md_cluster_stop(bitmap->mddev);
  1543. /* Shouldn't be needed - but just in case.... */
  1544. wait_event(bitmap->write_wait,
  1545. atomic_read(&bitmap->pending_writes) == 0);
  1546. /* release the bitmap file */
  1547. md_bitmap_file_unmap(&bitmap->storage);
  1548. bp = bitmap->counts.bp;
  1549. pages = bitmap->counts.pages;
  1550. /* free all allocated memory */
  1551. if (bp) /* deallocate the page memory */
  1552. for (k = 0; k < pages; k++)
  1553. if (bp[k].map && !bp[k].hijacked)
  1554. kfree(bp[k].map);
  1555. kfree(bp);
  1556. kfree(bitmap);
  1557. }
  1558. EXPORT_SYMBOL(md_bitmap_free);
  1559. void md_bitmap_wait_behind_writes(struct mddev *mddev)
  1560. {
  1561. struct bitmap *bitmap = mddev->bitmap;
  1562. /* wait for behind writes to complete */
  1563. if (bitmap && atomic_read(&bitmap->behind_writes) > 0) {
  1564. pr_debug("md:%s: behind writes in progress - waiting to stop.\n",
  1565. mdname(mddev));
  1566. /* need to kick something here to make sure I/O goes? */
  1567. wait_event(bitmap->behind_wait,
  1568. atomic_read(&bitmap->behind_writes) == 0);
  1569. }
  1570. }
  1571. void md_bitmap_destroy(struct mddev *mddev)
  1572. {
  1573. struct bitmap *bitmap = mddev->bitmap;
  1574. if (!bitmap) /* there was no bitmap */
  1575. return;
  1576. md_bitmap_wait_behind_writes(mddev);
  1577. if (!mddev->serialize_policy)
  1578. mddev_destroy_serial_pool(mddev, NULL, true);
  1579. mutex_lock(&mddev->bitmap_info.mutex);
  1580. spin_lock(&mddev->lock);
  1581. mddev->bitmap = NULL; /* disconnect from the md device */
  1582. spin_unlock(&mddev->lock);
  1583. mutex_unlock(&mddev->bitmap_info.mutex);
  1584. if (mddev->thread)
  1585. mddev->thread->timeout = MAX_SCHEDULE_TIMEOUT;
  1586. md_bitmap_free(bitmap);
  1587. }
  1588. /*
  1589. * initialize the bitmap structure
  1590. * if this returns an error, bitmap_destroy must be called to do clean up
  1591. * once mddev->bitmap is set
  1592. */
  1593. struct bitmap *md_bitmap_create(struct mddev *mddev, int slot)
  1594. {
  1595. struct bitmap *bitmap;
  1596. sector_t blocks = mddev->resync_max_sectors;
  1597. struct file *file = mddev->bitmap_info.file;
  1598. int err;
  1599. struct kernfs_node *bm = NULL;
  1600. BUILD_BUG_ON(sizeof(bitmap_super_t) != 256);
  1601. BUG_ON(file && mddev->bitmap_info.offset);
  1602. if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
  1603. pr_notice("md/raid:%s: array with journal cannot have bitmap\n",
  1604. mdname(mddev));
  1605. return ERR_PTR(-EBUSY);
  1606. }
  1607. bitmap = kzalloc(sizeof(*bitmap), GFP_KERNEL);
  1608. if (!bitmap)
  1609. return ERR_PTR(-ENOMEM);
  1610. spin_lock_init(&bitmap->counts.lock);
  1611. atomic_set(&bitmap->pending_writes, 0);
  1612. init_waitqueue_head(&bitmap->write_wait);
  1613. init_waitqueue_head(&bitmap->overflow_wait);
  1614. init_waitqueue_head(&bitmap->behind_wait);
  1615. bitmap->mddev = mddev;
  1616. bitmap->cluster_slot = slot;
  1617. if (mddev->kobj.sd)
  1618. bm = sysfs_get_dirent(mddev->kobj.sd, "bitmap");
  1619. if (bm) {
  1620. bitmap->sysfs_can_clear = sysfs_get_dirent(bm, "can_clear");
  1621. sysfs_put(bm);
  1622. } else
  1623. bitmap->sysfs_can_clear = NULL;
  1624. bitmap->storage.file = file;
  1625. if (file) {
  1626. get_file(file);
  1627. /* As future accesses to this file will use bmap,
  1628. * and bypass the page cache, we must sync the file
  1629. * first.
  1630. */
  1631. vfs_fsync(file, 1);
  1632. }
  1633. /* read superblock from bitmap file (this sets mddev->bitmap_info.chunksize) */
  1634. if (!mddev->bitmap_info.external) {
  1635. /*
  1636. * If 'MD_ARRAY_FIRST_USE' is set, then device-mapper is
  1637. * instructing us to create a new on-disk bitmap instance.
  1638. */
  1639. if (test_and_clear_bit(MD_ARRAY_FIRST_USE, &mddev->flags))
  1640. err = md_bitmap_new_disk_sb(bitmap);
  1641. else
  1642. err = md_bitmap_read_sb(bitmap);
  1643. } else {
  1644. err = 0;
  1645. if (mddev->bitmap_info.chunksize == 0 ||
  1646. mddev->bitmap_info.daemon_sleep == 0)
  1647. /* chunksize and time_base need to be
  1648. * set first. */
  1649. err = -EINVAL;
  1650. }
  1651. if (err)
  1652. goto error;
  1653. bitmap->daemon_lastrun = jiffies;
  1654. err = md_bitmap_resize(bitmap, blocks, mddev->bitmap_info.chunksize, 1);
  1655. if (err)
  1656. goto error;
  1657. pr_debug("created bitmap (%lu pages) for device %s\n",
  1658. bitmap->counts.pages, bmname(bitmap));
  1659. err = test_bit(BITMAP_WRITE_ERROR, &bitmap->flags) ? -EIO : 0;
  1660. if (err)
  1661. goto error;
  1662. return bitmap;
  1663. error:
  1664. md_bitmap_free(bitmap);
  1665. return ERR_PTR(err);
  1666. }
  1667. int md_bitmap_load(struct mddev *mddev)
  1668. {
  1669. int err = 0;
  1670. sector_t start = 0;
  1671. sector_t sector = 0;
  1672. struct bitmap *bitmap = mddev->bitmap;
  1673. struct md_rdev *rdev;
  1674. if (!bitmap)
  1675. goto out;
  1676. rdev_for_each(rdev, mddev)
  1677. mddev_create_serial_pool(mddev, rdev, true);
  1678. if (mddev_is_clustered(mddev))
  1679. md_cluster_ops->load_bitmaps(mddev, mddev->bitmap_info.nodes);
  1680. /* Clear out old bitmap info first: Either there is none, or we
  1681. * are resuming after someone else has possibly changed things,
  1682. * so we should forget old cached info.
  1683. * All chunks should be clean, but some might need_sync.
  1684. */
  1685. while (sector < mddev->resync_max_sectors) {
  1686. sector_t blocks;
  1687. md_bitmap_start_sync(bitmap, sector, &blocks, 0);
  1688. sector += blocks;
  1689. }
  1690. md_bitmap_close_sync(bitmap);
  1691. if (mddev->degraded == 0
  1692. || bitmap->events_cleared == mddev->events)
  1693. /* no need to keep dirty bits to optimise a
  1694. * re-add of a missing device */
  1695. start = mddev->recovery_cp;
  1696. mutex_lock(&mddev->bitmap_info.mutex);
  1697. err = md_bitmap_init_from_disk(bitmap, start);
  1698. mutex_unlock(&mddev->bitmap_info.mutex);
  1699. if (err)
  1700. goto out;
  1701. clear_bit(BITMAP_STALE, &bitmap->flags);
  1702. /* Kick recovery in case any bits were set */
  1703. set_bit(MD_RECOVERY_NEEDED, &bitmap->mddev->recovery);
  1704. mddev->thread->timeout = mddev->bitmap_info.daemon_sleep;
  1705. md_wakeup_thread(mddev->thread);
  1706. md_bitmap_update_sb(bitmap);
  1707. if (test_bit(BITMAP_WRITE_ERROR, &bitmap->flags))
  1708. err = -EIO;
  1709. out:
  1710. return err;
  1711. }
  1712. EXPORT_SYMBOL_GPL(md_bitmap_load);
  1713. /* caller need to free returned bitmap with md_bitmap_free() */
  1714. struct bitmap *get_bitmap_from_slot(struct mddev *mddev, int slot)
  1715. {
  1716. int rv = 0;
  1717. struct bitmap *bitmap;
  1718. bitmap = md_bitmap_create(mddev, slot);
  1719. if (IS_ERR(bitmap)) {
  1720. rv = PTR_ERR(bitmap);
  1721. return ERR_PTR(rv);
  1722. }
  1723. rv = md_bitmap_init_from_disk(bitmap, 0);
  1724. if (rv) {
  1725. md_bitmap_free(bitmap);
  1726. return ERR_PTR(rv);
  1727. }
  1728. return bitmap;
  1729. }
  1730. EXPORT_SYMBOL(get_bitmap_from_slot);
  1731. /* Loads the bitmap associated with slot and copies the resync information
  1732. * to our bitmap
  1733. */
  1734. int md_bitmap_copy_from_slot(struct mddev *mddev, int slot,
  1735. sector_t *low, sector_t *high, bool clear_bits)
  1736. {
  1737. int rv = 0, i, j;
  1738. sector_t block, lo = 0, hi = 0;
  1739. struct bitmap_counts *counts;
  1740. struct bitmap *bitmap;
  1741. bitmap = get_bitmap_from_slot(mddev, slot);
  1742. if (IS_ERR(bitmap)) {
  1743. pr_err("%s can't get bitmap from slot %d\n", __func__, slot);
  1744. return -1;
  1745. }
  1746. counts = &bitmap->counts;
  1747. for (j = 0; j < counts->chunks; j++) {
  1748. block = (sector_t)j << counts->chunkshift;
  1749. if (md_bitmap_file_test_bit(bitmap, block)) {
  1750. if (!lo)
  1751. lo = block;
  1752. hi = block;
  1753. md_bitmap_file_clear_bit(bitmap, block);
  1754. md_bitmap_set_memory_bits(mddev->bitmap, block, 1);
  1755. md_bitmap_file_set_bit(mddev->bitmap, block);
  1756. }
  1757. }
  1758. if (clear_bits) {
  1759. md_bitmap_update_sb(bitmap);
  1760. /* BITMAP_PAGE_PENDING is set, but bitmap_unplug needs
  1761. * BITMAP_PAGE_DIRTY or _NEEDWRITE to write ... */
  1762. for (i = 0; i < bitmap->storage.file_pages; i++)
  1763. if (test_page_attr(bitmap, i, BITMAP_PAGE_PENDING))
  1764. set_page_attr(bitmap, i, BITMAP_PAGE_NEEDWRITE);
  1765. md_bitmap_unplug(bitmap);
  1766. }
  1767. md_bitmap_unplug(mddev->bitmap);
  1768. *low = lo;
  1769. *high = hi;
  1770. md_bitmap_free(bitmap);
  1771. return rv;
  1772. }
  1773. EXPORT_SYMBOL_GPL(md_bitmap_copy_from_slot);
  1774. void md_bitmap_status(struct seq_file *seq, struct bitmap *bitmap)
  1775. {
  1776. unsigned long chunk_kb;
  1777. struct bitmap_counts *counts;
  1778. if (!bitmap)
  1779. return;
  1780. counts = &bitmap->counts;
  1781. chunk_kb = bitmap->mddev->bitmap_info.chunksize >> 10;
  1782. seq_printf(seq, "bitmap: %lu/%lu pages [%luKB], "
  1783. "%lu%s chunk",
  1784. counts->pages - counts->missing_pages,
  1785. counts->pages,
  1786. (counts->pages - counts->missing_pages)
  1787. << (PAGE_SHIFT - 10),
  1788. chunk_kb ? chunk_kb : bitmap->mddev->bitmap_info.chunksize,
  1789. chunk_kb ? "KB" : "B");
  1790. if (bitmap->storage.file) {
  1791. seq_printf(seq, ", file: ");
  1792. seq_file_path(seq, bitmap->storage.file, " \t\n");
  1793. }
  1794. seq_printf(seq, "\n");
  1795. }
  1796. int md_bitmap_resize(struct bitmap *bitmap, sector_t blocks,
  1797. int chunksize, int init)
  1798. {
  1799. /* If chunk_size is 0, choose an appropriate chunk size.
  1800. * Then possibly allocate new storage space.
  1801. * Then quiesce, copy bits, replace bitmap, and re-start
  1802. *
  1803. * This function is called both to set up the initial bitmap
  1804. * and to resize the bitmap while the array is active.
  1805. * If this happens as a result of the array being resized,
  1806. * chunksize will be zero, and we need to choose a suitable
  1807. * chunksize, otherwise we use what we are given.
  1808. */
  1809. struct bitmap_storage store;
  1810. struct bitmap_counts old_counts;
  1811. unsigned long chunks;
  1812. sector_t block;
  1813. sector_t old_blocks, new_blocks;
  1814. int chunkshift;
  1815. int ret = 0;
  1816. long pages;
  1817. struct bitmap_page *new_bp;
  1818. if (bitmap->storage.file && !init) {
  1819. pr_info("md: cannot resize file-based bitmap\n");
  1820. return -EINVAL;
  1821. }
  1822. if (chunksize == 0) {
  1823. /* If there is enough space, leave the chunk size unchanged,
  1824. * else increase by factor of two until there is enough space.
  1825. */
  1826. long bytes;
  1827. long space = bitmap->mddev->bitmap_info.space;
  1828. if (space == 0) {
  1829. /* We don't know how much space there is, so limit
  1830. * to current size - in sectors.
  1831. */
  1832. bytes = DIV_ROUND_UP(bitmap->counts.chunks, 8);
  1833. if (!bitmap->mddev->bitmap_info.external)
  1834. bytes += sizeof(bitmap_super_t);
  1835. space = DIV_ROUND_UP(bytes, 512);
  1836. bitmap->mddev->bitmap_info.space = space;
  1837. }
  1838. chunkshift = bitmap->counts.chunkshift;
  1839. chunkshift--;
  1840. do {
  1841. /* 'chunkshift' is shift from block size to chunk size */
  1842. chunkshift++;
  1843. chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
  1844. bytes = DIV_ROUND_UP(chunks, 8);
  1845. if (!bitmap->mddev->bitmap_info.external)
  1846. bytes += sizeof(bitmap_super_t);
  1847. } while (bytes > (space << 9));
  1848. } else
  1849. chunkshift = ffz(~chunksize) - BITMAP_BLOCK_SHIFT;
  1850. chunks = DIV_ROUND_UP_SECTOR_T(blocks, 1 << chunkshift);
  1851. memset(&store, 0, sizeof(store));
  1852. if (bitmap->mddev->bitmap_info.offset || bitmap->mddev->bitmap_info.file)
  1853. ret = md_bitmap_storage_alloc(&store, chunks,
  1854. !bitmap->mddev->bitmap_info.external,
  1855. mddev_is_clustered(bitmap->mddev)
  1856. ? bitmap->cluster_slot : 0);
  1857. if (ret) {
  1858. md_bitmap_file_unmap(&store);
  1859. goto err;
  1860. }
  1861. pages = DIV_ROUND_UP(chunks, PAGE_COUNTER_RATIO);
  1862. new_bp = kcalloc(pages, sizeof(*new_bp), GFP_KERNEL);
  1863. ret = -ENOMEM;
  1864. if (!new_bp) {
  1865. md_bitmap_file_unmap(&store);
  1866. goto err;
  1867. }
  1868. if (!init)
  1869. bitmap->mddev->pers->quiesce(bitmap->mddev, 1);
  1870. store.file = bitmap->storage.file;
  1871. bitmap->storage.file = NULL;
  1872. if (store.sb_page && bitmap->storage.sb_page)
  1873. memcpy(page_address(store.sb_page),
  1874. page_address(bitmap->storage.sb_page),
  1875. sizeof(bitmap_super_t));
  1876. spin_lock_irq(&bitmap->counts.lock);
  1877. md_bitmap_file_unmap(&bitmap->storage);
  1878. bitmap->storage = store;
  1879. old_counts = bitmap->counts;
  1880. bitmap->counts.bp = new_bp;
  1881. bitmap->counts.pages = pages;
  1882. bitmap->counts.missing_pages = pages;
  1883. bitmap->counts.chunkshift = chunkshift;
  1884. bitmap->counts.chunks = chunks;
  1885. bitmap->mddev->bitmap_info.chunksize = 1 << (chunkshift +
  1886. BITMAP_BLOCK_SHIFT);
  1887. blocks = min(old_counts.chunks << old_counts.chunkshift,
  1888. chunks << chunkshift);
  1889. /* For cluster raid, need to pre-allocate bitmap */
  1890. if (mddev_is_clustered(bitmap->mddev)) {
  1891. unsigned long page;
  1892. for (page = 0; page < pages; page++) {
  1893. ret = md_bitmap_checkpage(&bitmap->counts, page, 1, 1);
  1894. if (ret) {
  1895. unsigned long k;
  1896. /* deallocate the page memory */
  1897. for (k = 0; k < page; k++) {
  1898. kfree(new_bp[k].map);
  1899. }
  1900. kfree(new_bp);
  1901. /* restore some fields from old_counts */
  1902. bitmap->counts.bp = old_counts.bp;
  1903. bitmap->counts.pages = old_counts.pages;
  1904. bitmap->counts.missing_pages = old_counts.pages;
  1905. bitmap->counts.chunkshift = old_counts.chunkshift;
  1906. bitmap->counts.chunks = old_counts.chunks;
  1907. bitmap->mddev->bitmap_info.chunksize = 1 << (old_counts.chunkshift +
  1908. BITMAP_BLOCK_SHIFT);
  1909. blocks = old_counts.chunks << old_counts.chunkshift;
  1910. pr_warn("Could not pre-allocate in-memory bitmap for cluster raid\n");
  1911. break;
  1912. } else
  1913. bitmap->counts.bp[page].count += 1;
  1914. }
  1915. }
  1916. for (block = 0; block < blocks; ) {
  1917. bitmap_counter_t *bmc_old, *bmc_new;
  1918. int set;
  1919. bmc_old = md_bitmap_get_counter(&old_counts, block, &old_blocks, 0);
  1920. set = bmc_old && NEEDED(*bmc_old);
  1921. if (set) {
  1922. bmc_new = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
  1923. if (*bmc_new == 0) {
  1924. /* need to set on-disk bits too. */
  1925. sector_t end = block + new_blocks;
  1926. sector_t start = block >> chunkshift;
  1927. start <<= chunkshift;
  1928. while (start < end) {
  1929. md_bitmap_file_set_bit(bitmap, block);
  1930. start += 1 << chunkshift;
  1931. }
  1932. *bmc_new = 2;
  1933. md_bitmap_count_page(&bitmap->counts, block, 1);
  1934. md_bitmap_set_pending(&bitmap->counts, block);
  1935. }
  1936. *bmc_new |= NEEDED_MASK;
  1937. if (new_blocks < old_blocks)
  1938. old_blocks = new_blocks;
  1939. }
  1940. block += old_blocks;
  1941. }
  1942. if (bitmap->counts.bp != old_counts.bp) {
  1943. unsigned long k;
  1944. for (k = 0; k < old_counts.pages; k++)
  1945. if (!old_counts.bp[k].hijacked)
  1946. kfree(old_counts.bp[k].map);
  1947. kfree(old_counts.bp);
  1948. }
  1949. if (!init) {
  1950. int i;
  1951. while (block < (chunks << chunkshift)) {
  1952. bitmap_counter_t *bmc;
  1953. bmc = md_bitmap_get_counter(&bitmap->counts, block, &new_blocks, 1);
  1954. if (bmc) {
  1955. /* new space. It needs to be resynced, so
  1956. * we set NEEDED_MASK.
  1957. */
  1958. if (*bmc == 0) {
  1959. *bmc = NEEDED_MASK | 2;
  1960. md_bitmap_count_page(&bitmap->counts, block, 1);
  1961. md_bitmap_set_pending(&bitmap->counts, block);
  1962. }
  1963. }
  1964. block += new_blocks;
  1965. }
  1966. for (i = 0; i < bitmap->storage.file_pages; i++)
  1967. set_page_attr(bitmap, i, BITMAP_PAGE_DIRTY);
  1968. }
  1969. spin_unlock_irq(&bitmap->counts.lock);
  1970. if (!init) {
  1971. md_bitmap_unplug(bitmap);
  1972. bitmap->mddev->pers->quiesce(bitmap->mddev, 0);
  1973. }
  1974. ret = 0;
  1975. err:
  1976. return ret;
  1977. }
  1978. EXPORT_SYMBOL_GPL(md_bitmap_resize);
  1979. static ssize_t
  1980. location_show(struct mddev *mddev, char *page)
  1981. {
  1982. ssize_t len;
  1983. if (mddev->bitmap_info.file)
  1984. len = sprintf(page, "file");
  1985. else if (mddev->bitmap_info.offset)
  1986. len = sprintf(page, "%+lld", (long long)mddev->bitmap_info.offset);
  1987. else
  1988. len = sprintf(page, "none");
  1989. len += sprintf(page+len, "\n");
  1990. return len;
  1991. }
  1992. static ssize_t
  1993. location_store(struct mddev *mddev, const char *buf, size_t len)
  1994. {
  1995. int rv;
  1996. rv = mddev_lock(mddev);
  1997. if (rv)
  1998. return rv;
  1999. if (mddev->pers) {
  2000. if (!mddev->pers->quiesce) {
  2001. rv = -EBUSY;
  2002. goto out;
  2003. }
  2004. if (mddev->recovery || mddev->sync_thread) {
  2005. rv = -EBUSY;
  2006. goto out;
  2007. }
  2008. }
  2009. if (mddev->bitmap || mddev->bitmap_info.file ||
  2010. mddev->bitmap_info.offset) {
  2011. /* bitmap already configured. Only option is to clear it */
  2012. if (strncmp(buf, "none", 4) != 0) {
  2013. rv = -EBUSY;
  2014. goto out;
  2015. }
  2016. if (mddev->pers) {
  2017. mddev_suspend(mddev);
  2018. md_bitmap_destroy(mddev);
  2019. mddev_resume(mddev);
  2020. }
  2021. mddev->bitmap_info.offset = 0;
  2022. if (mddev->bitmap_info.file) {
  2023. struct file *f = mddev->bitmap_info.file;
  2024. mddev->bitmap_info.file = NULL;
  2025. fput(f);
  2026. }
  2027. } else {
  2028. /* No bitmap, OK to set a location */
  2029. long long offset;
  2030. if (strncmp(buf, "none", 4) == 0)
  2031. /* nothing to be done */;
  2032. else if (strncmp(buf, "file:", 5) == 0) {
  2033. /* Not supported yet */
  2034. rv = -EINVAL;
  2035. goto out;
  2036. } else {
  2037. if (buf[0] == '+')
  2038. rv = kstrtoll(buf+1, 10, &offset);
  2039. else
  2040. rv = kstrtoll(buf, 10, &offset);
  2041. if (rv)
  2042. goto out;
  2043. if (offset == 0) {
  2044. rv = -EINVAL;
  2045. goto out;
  2046. }
  2047. if (mddev->bitmap_info.external == 0 &&
  2048. mddev->major_version == 0 &&
  2049. offset != mddev->bitmap_info.default_offset) {
  2050. rv = -EINVAL;
  2051. goto out;
  2052. }
  2053. mddev->bitmap_info.offset = offset;
  2054. if (mddev->pers) {
  2055. struct bitmap *bitmap;
  2056. bitmap = md_bitmap_create(mddev, -1);
  2057. mddev_suspend(mddev);
  2058. if (IS_ERR(bitmap))
  2059. rv = PTR_ERR(bitmap);
  2060. else {
  2061. mddev->bitmap = bitmap;
  2062. rv = md_bitmap_load(mddev);
  2063. if (rv)
  2064. mddev->bitmap_info.offset = 0;
  2065. }
  2066. if (rv) {
  2067. md_bitmap_destroy(mddev);
  2068. mddev_resume(mddev);
  2069. goto out;
  2070. }
  2071. mddev_resume(mddev);
  2072. }
  2073. }
  2074. }
  2075. if (!mddev->external) {
  2076. /* Ensure new bitmap info is stored in
  2077. * metadata promptly.
  2078. */
  2079. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  2080. md_wakeup_thread(mddev->thread);
  2081. }
  2082. rv = 0;
  2083. out:
  2084. mddev_unlock(mddev);
  2085. if (rv)
  2086. return rv;
  2087. return len;
  2088. }
  2089. static struct md_sysfs_entry bitmap_location =
  2090. __ATTR(location, S_IRUGO|S_IWUSR, location_show, location_store);
  2091. /* 'bitmap/space' is the space available at 'location' for the
  2092. * bitmap. This allows the kernel to know when it is safe to
  2093. * resize the bitmap to match a resized array.
  2094. */
  2095. static ssize_t
  2096. space_show(struct mddev *mddev, char *page)
  2097. {
  2098. return sprintf(page, "%lu\n", mddev->bitmap_info.space);
  2099. }
  2100. static ssize_t
  2101. space_store(struct mddev *mddev, const char *buf, size_t len)
  2102. {
  2103. unsigned long sectors;
  2104. int rv;
  2105. rv = kstrtoul(buf, 10, &sectors);
  2106. if (rv)
  2107. return rv;
  2108. if (sectors == 0)
  2109. return -EINVAL;
  2110. if (mddev->bitmap &&
  2111. sectors < (mddev->bitmap->storage.bytes + 511) >> 9)
  2112. return -EFBIG; /* Bitmap is too big for this small space */
  2113. /* could make sure it isn't too big, but that isn't really
  2114. * needed - user-space should be careful.
  2115. */
  2116. mddev->bitmap_info.space = sectors;
  2117. return len;
  2118. }
  2119. static struct md_sysfs_entry bitmap_space =
  2120. __ATTR(space, S_IRUGO|S_IWUSR, space_show, space_store);
  2121. static ssize_t
  2122. timeout_show(struct mddev *mddev, char *page)
  2123. {
  2124. ssize_t len;
  2125. unsigned long secs = mddev->bitmap_info.daemon_sleep / HZ;
  2126. unsigned long jifs = mddev->bitmap_info.daemon_sleep % HZ;
  2127. len = sprintf(page, "%lu", secs);
  2128. if (jifs)
  2129. len += sprintf(page+len, ".%03u", jiffies_to_msecs(jifs));
  2130. len += sprintf(page+len, "\n");
  2131. return len;
  2132. }
  2133. static ssize_t
  2134. timeout_store(struct mddev *mddev, const char *buf, size_t len)
  2135. {
  2136. /* timeout can be set at any time */
  2137. unsigned long timeout;
  2138. int rv = strict_strtoul_scaled(buf, &timeout, 4);
  2139. if (rv)
  2140. return rv;
  2141. /* just to make sure we don't overflow... */
  2142. if (timeout >= LONG_MAX / HZ)
  2143. return -EINVAL;
  2144. timeout = timeout * HZ / 10000;
  2145. if (timeout >= MAX_SCHEDULE_TIMEOUT)
  2146. timeout = MAX_SCHEDULE_TIMEOUT-1;
  2147. if (timeout < 1)
  2148. timeout = 1;
  2149. mddev->bitmap_info.daemon_sleep = timeout;
  2150. if (mddev->thread) {
  2151. /* if thread->timeout is MAX_SCHEDULE_TIMEOUT, then
  2152. * the bitmap is all clean and we don't need to
  2153. * adjust the timeout right now
  2154. */
  2155. if (mddev->thread->timeout < MAX_SCHEDULE_TIMEOUT) {
  2156. mddev->thread->timeout = timeout;
  2157. md_wakeup_thread(mddev->thread);
  2158. }
  2159. }
  2160. return len;
  2161. }
  2162. static struct md_sysfs_entry bitmap_timeout =
  2163. __ATTR(time_base, S_IRUGO|S_IWUSR, timeout_show, timeout_store);
  2164. static ssize_t
  2165. backlog_show(struct mddev *mddev, char *page)
  2166. {
  2167. return sprintf(page, "%lu\n", mddev->bitmap_info.max_write_behind);
  2168. }
  2169. static ssize_t
  2170. backlog_store(struct mddev *mddev, const char *buf, size_t len)
  2171. {
  2172. unsigned long backlog;
  2173. unsigned long old_mwb = mddev->bitmap_info.max_write_behind;
  2174. int rv = kstrtoul(buf, 10, &backlog);
  2175. if (rv)
  2176. return rv;
  2177. if (backlog > COUNTER_MAX)
  2178. return -EINVAL;
  2179. mddev->bitmap_info.max_write_behind = backlog;
  2180. if (!backlog && mddev->serial_info_pool) {
  2181. /* serial_info_pool is not needed if backlog is zero */
  2182. if (!mddev->serialize_policy)
  2183. mddev_destroy_serial_pool(mddev, NULL, false);
  2184. } else if (backlog && !mddev->serial_info_pool) {
  2185. /* serial_info_pool is needed since backlog is not zero */
  2186. struct md_rdev *rdev;
  2187. rdev_for_each(rdev, mddev)
  2188. mddev_create_serial_pool(mddev, rdev, false);
  2189. }
  2190. if (old_mwb != backlog)
  2191. md_bitmap_update_sb(mddev->bitmap);
  2192. return len;
  2193. }
  2194. static struct md_sysfs_entry bitmap_backlog =
  2195. __ATTR(backlog, S_IRUGO|S_IWUSR, backlog_show, backlog_store);
  2196. static ssize_t
  2197. chunksize_show(struct mddev *mddev, char *page)
  2198. {
  2199. return sprintf(page, "%lu\n", mddev->bitmap_info.chunksize);
  2200. }
  2201. static ssize_t
  2202. chunksize_store(struct mddev *mddev, const char *buf, size_t len)
  2203. {
  2204. /* Can only be changed when no bitmap is active */
  2205. int rv;
  2206. unsigned long csize;
  2207. if (mddev->bitmap)
  2208. return -EBUSY;
  2209. rv = kstrtoul(buf, 10, &csize);
  2210. if (rv)
  2211. return rv;
  2212. if (csize < 512 ||
  2213. !is_power_of_2(csize))
  2214. return -EINVAL;
  2215. mddev->bitmap_info.chunksize = csize;
  2216. return len;
  2217. }
  2218. static struct md_sysfs_entry bitmap_chunksize =
  2219. __ATTR(chunksize, S_IRUGO|S_IWUSR, chunksize_show, chunksize_store);
  2220. static ssize_t metadata_show(struct mddev *mddev, char *page)
  2221. {
  2222. if (mddev_is_clustered(mddev))
  2223. return sprintf(page, "clustered\n");
  2224. return sprintf(page, "%s\n", (mddev->bitmap_info.external
  2225. ? "external" : "internal"));
  2226. }
  2227. static ssize_t metadata_store(struct mddev *mddev, const char *buf, size_t len)
  2228. {
  2229. if (mddev->bitmap ||
  2230. mddev->bitmap_info.file ||
  2231. mddev->bitmap_info.offset)
  2232. return -EBUSY;
  2233. if (strncmp(buf, "external", 8) == 0)
  2234. mddev->bitmap_info.external = 1;
  2235. else if ((strncmp(buf, "internal", 8) == 0) ||
  2236. (strncmp(buf, "clustered", 9) == 0))
  2237. mddev->bitmap_info.external = 0;
  2238. else
  2239. return -EINVAL;
  2240. return len;
  2241. }
  2242. static struct md_sysfs_entry bitmap_metadata =
  2243. __ATTR(metadata, S_IRUGO|S_IWUSR, metadata_show, metadata_store);
  2244. static ssize_t can_clear_show(struct mddev *mddev, char *page)
  2245. {
  2246. int len;
  2247. spin_lock(&mddev->lock);
  2248. if (mddev->bitmap)
  2249. len = sprintf(page, "%s\n", (mddev->bitmap->need_sync ?
  2250. "false" : "true"));
  2251. else
  2252. len = sprintf(page, "\n");
  2253. spin_unlock(&mddev->lock);
  2254. return len;
  2255. }
  2256. static ssize_t can_clear_store(struct mddev *mddev, const char *buf, size_t len)
  2257. {
  2258. if (mddev->bitmap == NULL)
  2259. return -ENOENT;
  2260. if (strncmp(buf, "false", 5) == 0)
  2261. mddev->bitmap->need_sync = 1;
  2262. else if (strncmp(buf, "true", 4) == 0) {
  2263. if (mddev->degraded)
  2264. return -EBUSY;
  2265. mddev->bitmap->need_sync = 0;
  2266. } else
  2267. return -EINVAL;
  2268. return len;
  2269. }
  2270. static struct md_sysfs_entry bitmap_can_clear =
  2271. __ATTR(can_clear, S_IRUGO|S_IWUSR, can_clear_show, can_clear_store);
  2272. static ssize_t
  2273. behind_writes_used_show(struct mddev *mddev, char *page)
  2274. {
  2275. ssize_t ret;
  2276. spin_lock(&mddev->lock);
  2277. if (mddev->bitmap == NULL)
  2278. ret = sprintf(page, "0\n");
  2279. else
  2280. ret = sprintf(page, "%lu\n",
  2281. mddev->bitmap->behind_writes_used);
  2282. spin_unlock(&mddev->lock);
  2283. return ret;
  2284. }
  2285. static ssize_t
  2286. behind_writes_used_reset(struct mddev *mddev, const char *buf, size_t len)
  2287. {
  2288. if (mddev->bitmap)
  2289. mddev->bitmap->behind_writes_used = 0;
  2290. return len;
  2291. }
  2292. static struct md_sysfs_entry max_backlog_used =
  2293. __ATTR(max_backlog_used, S_IRUGO | S_IWUSR,
  2294. behind_writes_used_show, behind_writes_used_reset);
  2295. static struct attribute *md_bitmap_attrs[] = {
  2296. &bitmap_location.attr,
  2297. &bitmap_space.attr,
  2298. &bitmap_timeout.attr,
  2299. &bitmap_backlog.attr,
  2300. &bitmap_chunksize.attr,
  2301. &bitmap_metadata.attr,
  2302. &bitmap_can_clear.attr,
  2303. &max_backlog_used.attr,
  2304. NULL
  2305. };
  2306. struct attribute_group md_bitmap_group = {
  2307. .name = "bitmap",
  2308. .attrs = md_bitmap_attrs,
  2309. };