raid5-cache.c 88 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2015 Shaohua Li <shli@fb.com>
  4. * Copyright (C) 2016 Song Liu <songliubraving@fb.com>
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/wait.h>
  8. #include <linux/blkdev.h>
  9. #include <linux/slab.h>
  10. #include <linux/raid/md_p.h>
  11. #include <linux/crc32c.h>
  12. #include <linux/random.h>
  13. #include <linux/kthread.h>
  14. #include <linux/types.h>
  15. #include "md.h"
  16. #include "raid5.h"
  17. #include "md-bitmap.h"
  18. #include "raid5-log.h"
  19. /*
  20. * metadata/data stored in disk with 4k size unit (a block) regardless
  21. * underneath hardware sector size. only works with PAGE_SIZE == 4096
  22. */
  23. #define BLOCK_SECTORS (8)
  24. #define BLOCK_SECTOR_SHIFT (3)
  25. /*
  26. * log->max_free_space is min(1/4 disk size, 10G reclaimable space).
  27. *
  28. * In write through mode, the reclaim runs every log->max_free_space.
  29. * This can prevent the recovery scans for too long
  30. */
  31. #define RECLAIM_MAX_FREE_SPACE (10 * 1024 * 1024 * 2) /* sector */
  32. #define RECLAIM_MAX_FREE_SPACE_SHIFT (2)
  33. /* wake up reclaim thread periodically */
  34. #define R5C_RECLAIM_WAKEUP_INTERVAL (30 * HZ)
  35. /* start flush with these full stripes */
  36. #define R5C_FULL_STRIPE_FLUSH_BATCH(conf) (conf->max_nr_stripes / 4)
  37. /* reclaim stripes in groups */
  38. #define R5C_RECLAIM_STRIPE_GROUP (NR_STRIPE_HASH_LOCKS * 2)
  39. /*
  40. * We only need 2 bios per I/O unit to make progress, but ensure we
  41. * have a few more available to not get too tight.
  42. */
  43. #define R5L_POOL_SIZE 4
  44. static char *r5c_journal_mode_str[] = {"write-through",
  45. "write-back"};
  46. /*
  47. * raid5 cache state machine
  48. *
  49. * With the RAID cache, each stripe works in two phases:
  50. * - caching phase
  51. * - writing-out phase
  52. *
  53. * These two phases are controlled by bit STRIPE_R5C_CACHING:
  54. * if STRIPE_R5C_CACHING == 0, the stripe is in writing-out phase
  55. * if STRIPE_R5C_CACHING == 1, the stripe is in caching phase
  56. *
  57. * When there is no journal, or the journal is in write-through mode,
  58. * the stripe is always in writing-out phase.
  59. *
  60. * For write-back journal, the stripe is sent to caching phase on write
  61. * (r5c_try_caching_write). r5c_make_stripe_write_out() kicks off
  62. * the write-out phase by clearing STRIPE_R5C_CACHING.
  63. *
  64. * Stripes in caching phase do not write the raid disks. Instead, all
  65. * writes are committed from the log device. Therefore, a stripe in
  66. * caching phase handles writes as:
  67. * - write to log device
  68. * - return IO
  69. *
  70. * Stripes in writing-out phase handle writes as:
  71. * - calculate parity
  72. * - write pending data and parity to journal
  73. * - write data and parity to raid disks
  74. * - return IO for pending writes
  75. */
  76. struct r5l_log {
  77. struct md_rdev *rdev;
  78. u32 uuid_checksum;
  79. sector_t device_size; /* log device size, round to
  80. * BLOCK_SECTORS */
  81. sector_t max_free_space; /* reclaim run if free space is at
  82. * this size */
  83. sector_t last_checkpoint; /* log tail. where recovery scan
  84. * starts from */
  85. u64 last_cp_seq; /* log tail sequence */
  86. sector_t log_start; /* log head. where new data appends */
  87. u64 seq; /* log head sequence */
  88. sector_t next_checkpoint;
  89. struct mutex io_mutex;
  90. struct r5l_io_unit *current_io; /* current io_unit accepting new data */
  91. spinlock_t io_list_lock;
  92. struct list_head running_ios; /* io_units which are still running,
  93. * and have not yet been completely
  94. * written to the log */
  95. struct list_head io_end_ios; /* io_units which have been completely
  96. * written to the log but not yet written
  97. * to the RAID */
  98. struct list_head flushing_ios; /* io_units which are waiting for log
  99. * cache flush */
  100. struct list_head finished_ios; /* io_units which settle down in log disk */
  101. struct bio flush_bio;
  102. struct list_head no_mem_stripes; /* pending stripes, -ENOMEM */
  103. struct kmem_cache *io_kc;
  104. mempool_t io_pool;
  105. struct bio_set bs;
  106. mempool_t meta_pool;
  107. struct md_thread *reclaim_thread;
  108. unsigned long reclaim_target; /* number of space that need to be
  109. * reclaimed. if it's 0, reclaim spaces
  110. * used by io_units which are in
  111. * IO_UNIT_STRIPE_END state (eg, reclaim
  112. * dones't wait for specific io_unit
  113. * switching to IO_UNIT_STRIPE_END
  114. * state) */
  115. wait_queue_head_t iounit_wait;
  116. struct list_head no_space_stripes; /* pending stripes, log has no space */
  117. spinlock_t no_space_stripes_lock;
  118. bool need_cache_flush;
  119. /* for r5c_cache */
  120. enum r5c_journal_mode r5c_journal_mode;
  121. /* all stripes in r5cache, in the order of seq at sh->log_start */
  122. struct list_head stripe_in_journal_list;
  123. spinlock_t stripe_in_journal_lock;
  124. atomic_t stripe_in_journal_count;
  125. /* to submit async io_units, to fulfill ordering of flush */
  126. struct work_struct deferred_io_work;
  127. /* to disable write back during in degraded mode */
  128. struct work_struct disable_writeback_work;
  129. /* to for chunk_aligned_read in writeback mode, details below */
  130. spinlock_t tree_lock;
  131. struct radix_tree_root big_stripe_tree;
  132. };
  133. /*
  134. * Enable chunk_aligned_read() with write back cache.
  135. *
  136. * Each chunk may contain more than one stripe (for example, a 256kB
  137. * chunk contains 64 4kB-page, so this chunk contain 64 stripes). For
  138. * chunk_aligned_read, these stripes are grouped into one "big_stripe".
  139. * For each big_stripe, we count how many stripes of this big_stripe
  140. * are in the write back cache. These data are tracked in a radix tree
  141. * (big_stripe_tree). We use radix_tree item pointer as the counter.
  142. * r5c_tree_index() is used to calculate keys for the radix tree.
  143. *
  144. * chunk_aligned_read() calls r5c_big_stripe_cached() to look up
  145. * big_stripe of each chunk in the tree. If this big_stripe is in the
  146. * tree, chunk_aligned_read() aborts. This look up is protected by
  147. * rcu_read_lock().
  148. *
  149. * It is necessary to remember whether a stripe is counted in
  150. * big_stripe_tree. Instead of adding new flag, we reuses existing flags:
  151. * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE. If either of these
  152. * two flags are set, the stripe is counted in big_stripe_tree. This
  153. * requires moving set_bit(STRIPE_R5C_PARTIAL_STRIPE) to
  154. * r5c_try_caching_write(); and moving clear_bit of
  155. * STRIPE_R5C_PARTIAL_STRIPE and STRIPE_R5C_FULL_STRIPE to
  156. * r5c_finish_stripe_write_out().
  157. */
  158. /*
  159. * radix tree requests lowest 2 bits of data pointer to be 2b'00.
  160. * So it is necessary to left shift the counter by 2 bits before using it
  161. * as data pointer of the tree.
  162. */
  163. #define R5C_RADIX_COUNT_SHIFT 2
  164. /*
  165. * calculate key for big_stripe_tree
  166. *
  167. * sect: align_bi->bi_iter.bi_sector or sh->sector
  168. */
  169. static inline sector_t r5c_tree_index(struct r5conf *conf,
  170. sector_t sect)
  171. {
  172. sector_div(sect, conf->chunk_sectors);
  173. return sect;
  174. }
  175. /*
  176. * an IO range starts from a meta data block and end at the next meta data
  177. * block. The io unit's the meta data block tracks data/parity followed it. io
  178. * unit is written to log disk with normal write, as we always flush log disk
  179. * first and then start move data to raid disks, there is no requirement to
  180. * write io unit with FLUSH/FUA
  181. */
  182. struct r5l_io_unit {
  183. struct r5l_log *log;
  184. struct page *meta_page; /* store meta block */
  185. int meta_offset; /* current offset in meta_page */
  186. struct bio *current_bio;/* current_bio accepting new data */
  187. atomic_t pending_stripe;/* how many stripes not flushed to raid */
  188. u64 seq; /* seq number of the metablock */
  189. sector_t log_start; /* where the io_unit starts */
  190. sector_t log_end; /* where the io_unit ends */
  191. struct list_head log_sibling; /* log->running_ios */
  192. struct list_head stripe_list; /* stripes added to the io_unit */
  193. int state;
  194. bool need_split_bio;
  195. struct bio *split_bio;
  196. unsigned int has_flush:1; /* include flush request */
  197. unsigned int has_fua:1; /* include fua request */
  198. unsigned int has_null_flush:1; /* include null flush request */
  199. unsigned int has_flush_payload:1; /* include flush payload */
  200. /*
  201. * io isn't sent yet, flush/fua request can only be submitted till it's
  202. * the first IO in running_ios list
  203. */
  204. unsigned int io_deferred:1;
  205. struct bio_list flush_barriers; /* size == 0 flush bios */
  206. };
  207. /* r5l_io_unit state */
  208. enum r5l_io_unit_state {
  209. IO_UNIT_RUNNING = 0, /* accepting new IO */
  210. IO_UNIT_IO_START = 1, /* io_unit bio start writing to log,
  211. * don't accepting new bio */
  212. IO_UNIT_IO_END = 2, /* io_unit bio finish writing to log */
  213. IO_UNIT_STRIPE_END = 3, /* stripes data finished writing to raid */
  214. };
  215. bool r5c_is_writeback(struct r5l_log *log)
  216. {
  217. return (log != NULL &&
  218. log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK);
  219. }
  220. static sector_t r5l_ring_add(struct r5l_log *log, sector_t start, sector_t inc)
  221. {
  222. start += inc;
  223. if (start >= log->device_size)
  224. start = start - log->device_size;
  225. return start;
  226. }
  227. static sector_t r5l_ring_distance(struct r5l_log *log, sector_t start,
  228. sector_t end)
  229. {
  230. if (end >= start)
  231. return end - start;
  232. else
  233. return end + log->device_size - start;
  234. }
  235. static bool r5l_has_free_space(struct r5l_log *log, sector_t size)
  236. {
  237. sector_t used_size;
  238. used_size = r5l_ring_distance(log, log->last_checkpoint,
  239. log->log_start);
  240. return log->device_size > used_size + size;
  241. }
  242. static void __r5l_set_io_unit_state(struct r5l_io_unit *io,
  243. enum r5l_io_unit_state state)
  244. {
  245. if (WARN_ON(io->state >= state))
  246. return;
  247. io->state = state;
  248. }
  249. static void
  250. r5c_return_dev_pending_writes(struct r5conf *conf, struct r5dev *dev)
  251. {
  252. struct bio *wbi, *wbi2;
  253. wbi = dev->written;
  254. dev->written = NULL;
  255. while (wbi && wbi->bi_iter.bi_sector <
  256. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  257. wbi2 = r5_next_bio(conf, wbi, dev->sector);
  258. md_write_end(conf->mddev);
  259. bio_endio(wbi);
  260. wbi = wbi2;
  261. }
  262. }
  263. void r5c_handle_cached_data_endio(struct r5conf *conf,
  264. struct stripe_head *sh, int disks)
  265. {
  266. int i;
  267. for (i = sh->disks; i--; ) {
  268. if (sh->dev[i].written) {
  269. set_bit(R5_UPTODATE, &sh->dev[i].flags);
  270. r5c_return_dev_pending_writes(conf, &sh->dev[i]);
  271. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  272. RAID5_STRIPE_SECTORS(conf),
  273. !test_bit(STRIPE_DEGRADED, &sh->state),
  274. 0);
  275. }
  276. }
  277. }
  278. void r5l_wake_reclaim(struct r5l_log *log, sector_t space);
  279. /* Check whether we should flush some stripes to free up stripe cache */
  280. void r5c_check_stripe_cache_usage(struct r5conf *conf)
  281. {
  282. int total_cached;
  283. if (!r5c_is_writeback(conf->log))
  284. return;
  285. total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
  286. atomic_read(&conf->r5c_cached_full_stripes);
  287. /*
  288. * The following condition is true for either of the following:
  289. * - stripe cache pressure high:
  290. * total_cached > 3/4 min_nr_stripes ||
  291. * empty_inactive_list_nr > 0
  292. * - stripe cache pressure moderate:
  293. * total_cached > 1/2 min_nr_stripes
  294. */
  295. if (total_cached > conf->min_nr_stripes * 1 / 2 ||
  296. atomic_read(&conf->empty_inactive_list_nr) > 0)
  297. r5l_wake_reclaim(conf->log, 0);
  298. }
  299. /*
  300. * flush cache when there are R5C_FULL_STRIPE_FLUSH_BATCH or more full
  301. * stripes in the cache
  302. */
  303. void r5c_check_cached_full_stripe(struct r5conf *conf)
  304. {
  305. if (!r5c_is_writeback(conf->log))
  306. return;
  307. /*
  308. * wake up reclaim for R5C_FULL_STRIPE_FLUSH_BATCH cached stripes
  309. * or a full stripe (chunk size / 4k stripes).
  310. */
  311. if (atomic_read(&conf->r5c_cached_full_stripes) >=
  312. min(R5C_FULL_STRIPE_FLUSH_BATCH(conf),
  313. conf->chunk_sectors >> RAID5_STRIPE_SHIFT(conf)))
  314. r5l_wake_reclaim(conf->log, 0);
  315. }
  316. /*
  317. * Total log space (in sectors) needed to flush all data in cache
  318. *
  319. * To avoid deadlock due to log space, it is necessary to reserve log
  320. * space to flush critical stripes (stripes that occupying log space near
  321. * last_checkpoint). This function helps check how much log space is
  322. * required to flush all cached stripes.
  323. *
  324. * To reduce log space requirements, two mechanisms are used to give cache
  325. * flush higher priorities:
  326. * 1. In handle_stripe_dirtying() and schedule_reconstruction(),
  327. * stripes ALREADY in journal can be flushed w/o pending writes;
  328. * 2. In r5l_write_stripe() and r5c_cache_data(), stripes NOT in journal
  329. * can be delayed (r5l_add_no_space_stripe).
  330. *
  331. * In cache flush, the stripe goes through 1 and then 2. For a stripe that
  332. * already passed 1, flushing it requires at most (conf->max_degraded + 1)
  333. * pages of journal space. For stripes that has not passed 1, flushing it
  334. * requires (conf->raid_disks + 1) pages of journal space. There are at
  335. * most (conf->group_cnt + 1) stripe that passed 1. So total journal space
  336. * required to flush all cached stripes (in pages) is:
  337. *
  338. * (stripe_in_journal_count - group_cnt - 1) * (max_degraded + 1) +
  339. * (group_cnt + 1) * (raid_disks + 1)
  340. * or
  341. * (stripe_in_journal_count) * (max_degraded + 1) +
  342. * (group_cnt + 1) * (raid_disks - max_degraded)
  343. */
  344. static sector_t r5c_log_required_to_flush_cache(struct r5conf *conf)
  345. {
  346. struct r5l_log *log = conf->log;
  347. if (!r5c_is_writeback(log))
  348. return 0;
  349. return BLOCK_SECTORS *
  350. ((conf->max_degraded + 1) * atomic_read(&log->stripe_in_journal_count) +
  351. (conf->raid_disks - conf->max_degraded) * (conf->group_cnt + 1));
  352. }
  353. /*
  354. * evaluate log space usage and update R5C_LOG_TIGHT and R5C_LOG_CRITICAL
  355. *
  356. * R5C_LOG_TIGHT is set when free space on the log device is less than 3x of
  357. * reclaim_required_space. R5C_LOG_CRITICAL is set when free space on the log
  358. * device is less than 2x of reclaim_required_space.
  359. */
  360. static inline void r5c_update_log_state(struct r5l_log *log)
  361. {
  362. struct r5conf *conf = log->rdev->mddev->private;
  363. sector_t free_space;
  364. sector_t reclaim_space;
  365. bool wake_reclaim = false;
  366. if (!r5c_is_writeback(log))
  367. return;
  368. free_space = r5l_ring_distance(log, log->log_start,
  369. log->last_checkpoint);
  370. reclaim_space = r5c_log_required_to_flush_cache(conf);
  371. if (free_space < 2 * reclaim_space)
  372. set_bit(R5C_LOG_CRITICAL, &conf->cache_state);
  373. else {
  374. if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
  375. wake_reclaim = true;
  376. clear_bit(R5C_LOG_CRITICAL, &conf->cache_state);
  377. }
  378. if (free_space < 3 * reclaim_space)
  379. set_bit(R5C_LOG_TIGHT, &conf->cache_state);
  380. else
  381. clear_bit(R5C_LOG_TIGHT, &conf->cache_state);
  382. if (wake_reclaim)
  383. r5l_wake_reclaim(log, 0);
  384. }
  385. /*
  386. * Put the stripe into writing-out phase by clearing STRIPE_R5C_CACHING.
  387. * This function should only be called in write-back mode.
  388. */
  389. void r5c_make_stripe_write_out(struct stripe_head *sh)
  390. {
  391. struct r5conf *conf = sh->raid_conf;
  392. struct r5l_log *log = conf->log;
  393. BUG_ON(!r5c_is_writeback(log));
  394. WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
  395. clear_bit(STRIPE_R5C_CACHING, &sh->state);
  396. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  397. atomic_inc(&conf->preread_active_stripes);
  398. }
  399. static void r5c_handle_data_cached(struct stripe_head *sh)
  400. {
  401. int i;
  402. for (i = sh->disks; i--; )
  403. if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
  404. set_bit(R5_InJournal, &sh->dev[i].flags);
  405. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  406. }
  407. clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
  408. }
  409. /*
  410. * this journal write must contain full parity,
  411. * it may also contain some data pages
  412. */
  413. static void r5c_handle_parity_cached(struct stripe_head *sh)
  414. {
  415. int i;
  416. for (i = sh->disks; i--; )
  417. if (test_bit(R5_InJournal, &sh->dev[i].flags))
  418. set_bit(R5_Wantwrite, &sh->dev[i].flags);
  419. }
  420. /*
  421. * Setting proper flags after writing (or flushing) data and/or parity to the
  422. * log device. This is called from r5l_log_endio() or r5l_log_flush_endio().
  423. */
  424. static void r5c_finish_cache_stripe(struct stripe_head *sh)
  425. {
  426. struct r5l_log *log = sh->raid_conf->log;
  427. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
  428. BUG_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
  429. /*
  430. * Set R5_InJournal for parity dev[pd_idx]. This means
  431. * all data AND parity in the journal. For RAID 6, it is
  432. * NOT necessary to set the flag for dev[qd_idx], as the
  433. * two parities are written out together.
  434. */
  435. set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
  436. } else if (test_bit(STRIPE_R5C_CACHING, &sh->state)) {
  437. r5c_handle_data_cached(sh);
  438. } else {
  439. r5c_handle_parity_cached(sh);
  440. set_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
  441. }
  442. }
  443. static void r5l_io_run_stripes(struct r5l_io_unit *io)
  444. {
  445. struct stripe_head *sh, *next;
  446. list_for_each_entry_safe(sh, next, &io->stripe_list, log_list) {
  447. list_del_init(&sh->log_list);
  448. r5c_finish_cache_stripe(sh);
  449. set_bit(STRIPE_HANDLE, &sh->state);
  450. raid5_release_stripe(sh);
  451. }
  452. }
  453. static void r5l_log_run_stripes(struct r5l_log *log)
  454. {
  455. struct r5l_io_unit *io, *next;
  456. lockdep_assert_held(&log->io_list_lock);
  457. list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
  458. /* don't change list order */
  459. if (io->state < IO_UNIT_IO_END)
  460. break;
  461. list_move_tail(&io->log_sibling, &log->finished_ios);
  462. r5l_io_run_stripes(io);
  463. }
  464. }
  465. static void r5l_move_to_end_ios(struct r5l_log *log)
  466. {
  467. struct r5l_io_unit *io, *next;
  468. lockdep_assert_held(&log->io_list_lock);
  469. list_for_each_entry_safe(io, next, &log->running_ios, log_sibling) {
  470. /* don't change list order */
  471. if (io->state < IO_UNIT_IO_END)
  472. break;
  473. list_move_tail(&io->log_sibling, &log->io_end_ios);
  474. }
  475. }
  476. static void __r5l_stripe_write_finished(struct r5l_io_unit *io);
  477. static void r5l_log_endio(struct bio *bio)
  478. {
  479. struct r5l_io_unit *io = bio->bi_private;
  480. struct r5l_io_unit *io_deferred;
  481. struct r5l_log *log = io->log;
  482. unsigned long flags;
  483. bool has_null_flush;
  484. bool has_flush_payload;
  485. if (bio->bi_status)
  486. md_error(log->rdev->mddev, log->rdev);
  487. bio_put(bio);
  488. mempool_free(io->meta_page, &log->meta_pool);
  489. spin_lock_irqsave(&log->io_list_lock, flags);
  490. __r5l_set_io_unit_state(io, IO_UNIT_IO_END);
  491. /*
  492. * if the io doesn't not have null_flush or flush payload,
  493. * it is not safe to access it after releasing io_list_lock.
  494. * Therefore, it is necessary to check the condition with
  495. * the lock held.
  496. */
  497. has_null_flush = io->has_null_flush;
  498. has_flush_payload = io->has_flush_payload;
  499. if (log->need_cache_flush && !list_empty(&io->stripe_list))
  500. r5l_move_to_end_ios(log);
  501. else
  502. r5l_log_run_stripes(log);
  503. if (!list_empty(&log->running_ios)) {
  504. /*
  505. * FLUSH/FUA io_unit is deferred because of ordering, now we
  506. * can dispatch it
  507. */
  508. io_deferred = list_first_entry(&log->running_ios,
  509. struct r5l_io_unit, log_sibling);
  510. if (io_deferred->io_deferred)
  511. schedule_work(&log->deferred_io_work);
  512. }
  513. spin_unlock_irqrestore(&log->io_list_lock, flags);
  514. if (log->need_cache_flush)
  515. md_wakeup_thread(log->rdev->mddev->thread);
  516. /* finish flush only io_unit and PAYLOAD_FLUSH only io_unit */
  517. if (has_null_flush) {
  518. struct bio *bi;
  519. WARN_ON(bio_list_empty(&io->flush_barriers));
  520. while ((bi = bio_list_pop(&io->flush_barriers)) != NULL) {
  521. bio_endio(bi);
  522. if (atomic_dec_and_test(&io->pending_stripe)) {
  523. __r5l_stripe_write_finished(io);
  524. return;
  525. }
  526. }
  527. }
  528. /* decrease pending_stripe for flush payload */
  529. if (has_flush_payload)
  530. if (atomic_dec_and_test(&io->pending_stripe))
  531. __r5l_stripe_write_finished(io);
  532. }
  533. static void r5l_do_submit_io(struct r5l_log *log, struct r5l_io_unit *io)
  534. {
  535. unsigned long flags;
  536. spin_lock_irqsave(&log->io_list_lock, flags);
  537. __r5l_set_io_unit_state(io, IO_UNIT_IO_START);
  538. spin_unlock_irqrestore(&log->io_list_lock, flags);
  539. /*
  540. * In case of journal device failures, submit_bio will get error
  541. * and calls endio, then active stripes will continue write
  542. * process. Therefore, it is not necessary to check Faulty bit
  543. * of journal device here.
  544. *
  545. * We can't check split_bio after current_bio is submitted. If
  546. * io->split_bio is null, after current_bio is submitted, current_bio
  547. * might already be completed and the io_unit is freed. We submit
  548. * split_bio first to avoid the issue.
  549. */
  550. if (io->split_bio) {
  551. if (io->has_flush)
  552. io->split_bio->bi_opf |= REQ_PREFLUSH;
  553. if (io->has_fua)
  554. io->split_bio->bi_opf |= REQ_FUA;
  555. submit_bio(io->split_bio);
  556. }
  557. if (io->has_flush)
  558. io->current_bio->bi_opf |= REQ_PREFLUSH;
  559. if (io->has_fua)
  560. io->current_bio->bi_opf |= REQ_FUA;
  561. submit_bio(io->current_bio);
  562. }
  563. /* deferred io_unit will be dispatched here */
  564. static void r5l_submit_io_async(struct work_struct *work)
  565. {
  566. struct r5l_log *log = container_of(work, struct r5l_log,
  567. deferred_io_work);
  568. struct r5l_io_unit *io = NULL;
  569. unsigned long flags;
  570. spin_lock_irqsave(&log->io_list_lock, flags);
  571. if (!list_empty(&log->running_ios)) {
  572. io = list_first_entry(&log->running_ios, struct r5l_io_unit,
  573. log_sibling);
  574. if (!io->io_deferred)
  575. io = NULL;
  576. else
  577. io->io_deferred = 0;
  578. }
  579. spin_unlock_irqrestore(&log->io_list_lock, flags);
  580. if (io)
  581. r5l_do_submit_io(log, io);
  582. }
  583. static void r5c_disable_writeback_async(struct work_struct *work)
  584. {
  585. struct r5l_log *log = container_of(work, struct r5l_log,
  586. disable_writeback_work);
  587. struct mddev *mddev = log->rdev->mddev;
  588. struct r5conf *conf = mddev->private;
  589. int locked = 0;
  590. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
  591. return;
  592. pr_info("md/raid:%s: Disabling writeback cache for degraded array.\n",
  593. mdname(mddev));
  594. /* wait superblock change before suspend */
  595. wait_event(mddev->sb_wait,
  596. conf->log == NULL ||
  597. (!test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags) &&
  598. (locked = mddev_trylock(mddev))));
  599. if (locked) {
  600. mddev_suspend(mddev);
  601. log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
  602. mddev_resume(mddev);
  603. mddev_unlock(mddev);
  604. }
  605. }
  606. static void r5l_submit_current_io(struct r5l_log *log)
  607. {
  608. struct r5l_io_unit *io = log->current_io;
  609. struct r5l_meta_block *block;
  610. unsigned long flags;
  611. u32 crc;
  612. bool do_submit = true;
  613. if (!io)
  614. return;
  615. block = page_address(io->meta_page);
  616. block->meta_size = cpu_to_le32(io->meta_offset);
  617. crc = crc32c_le(log->uuid_checksum, block, PAGE_SIZE);
  618. block->checksum = cpu_to_le32(crc);
  619. log->current_io = NULL;
  620. spin_lock_irqsave(&log->io_list_lock, flags);
  621. if (io->has_flush || io->has_fua) {
  622. if (io != list_first_entry(&log->running_ios,
  623. struct r5l_io_unit, log_sibling)) {
  624. io->io_deferred = 1;
  625. do_submit = false;
  626. }
  627. }
  628. spin_unlock_irqrestore(&log->io_list_lock, flags);
  629. if (do_submit)
  630. r5l_do_submit_io(log, io);
  631. }
  632. static struct bio *r5l_bio_alloc(struct r5l_log *log)
  633. {
  634. struct bio *bio = bio_alloc_bioset(GFP_NOIO, BIO_MAX_PAGES, &log->bs);
  635. bio_set_op_attrs(bio, REQ_OP_WRITE, 0);
  636. bio_set_dev(bio, log->rdev->bdev);
  637. bio->bi_iter.bi_sector = log->rdev->data_offset + log->log_start;
  638. return bio;
  639. }
  640. static void r5_reserve_log_entry(struct r5l_log *log, struct r5l_io_unit *io)
  641. {
  642. log->log_start = r5l_ring_add(log, log->log_start, BLOCK_SECTORS);
  643. r5c_update_log_state(log);
  644. /*
  645. * If we filled up the log device start from the beginning again,
  646. * which will require a new bio.
  647. *
  648. * Note: for this to work properly the log size needs to me a multiple
  649. * of BLOCK_SECTORS.
  650. */
  651. if (log->log_start == 0)
  652. io->need_split_bio = true;
  653. io->log_end = log->log_start;
  654. }
  655. static struct r5l_io_unit *r5l_new_meta(struct r5l_log *log)
  656. {
  657. struct r5l_io_unit *io;
  658. struct r5l_meta_block *block;
  659. io = mempool_alloc(&log->io_pool, GFP_ATOMIC);
  660. if (!io)
  661. return NULL;
  662. memset(io, 0, sizeof(*io));
  663. io->log = log;
  664. INIT_LIST_HEAD(&io->log_sibling);
  665. INIT_LIST_HEAD(&io->stripe_list);
  666. bio_list_init(&io->flush_barriers);
  667. io->state = IO_UNIT_RUNNING;
  668. io->meta_page = mempool_alloc(&log->meta_pool, GFP_NOIO);
  669. block = page_address(io->meta_page);
  670. clear_page(block);
  671. block->magic = cpu_to_le32(R5LOG_MAGIC);
  672. block->version = R5LOG_VERSION;
  673. block->seq = cpu_to_le64(log->seq);
  674. block->position = cpu_to_le64(log->log_start);
  675. io->log_start = log->log_start;
  676. io->meta_offset = sizeof(struct r5l_meta_block);
  677. io->seq = log->seq++;
  678. io->current_bio = r5l_bio_alloc(log);
  679. io->current_bio->bi_end_io = r5l_log_endio;
  680. io->current_bio->bi_private = io;
  681. bio_add_page(io->current_bio, io->meta_page, PAGE_SIZE, 0);
  682. r5_reserve_log_entry(log, io);
  683. spin_lock_irq(&log->io_list_lock);
  684. list_add_tail(&io->log_sibling, &log->running_ios);
  685. spin_unlock_irq(&log->io_list_lock);
  686. return io;
  687. }
  688. static int r5l_get_meta(struct r5l_log *log, unsigned int payload_size)
  689. {
  690. if (log->current_io &&
  691. log->current_io->meta_offset + payload_size > PAGE_SIZE)
  692. r5l_submit_current_io(log);
  693. if (!log->current_io) {
  694. log->current_io = r5l_new_meta(log);
  695. if (!log->current_io)
  696. return -ENOMEM;
  697. }
  698. return 0;
  699. }
  700. static void r5l_append_payload_meta(struct r5l_log *log, u16 type,
  701. sector_t location,
  702. u32 checksum1, u32 checksum2,
  703. bool checksum2_valid)
  704. {
  705. struct r5l_io_unit *io = log->current_io;
  706. struct r5l_payload_data_parity *payload;
  707. payload = page_address(io->meta_page) + io->meta_offset;
  708. payload->header.type = cpu_to_le16(type);
  709. payload->header.flags = cpu_to_le16(0);
  710. payload->size = cpu_to_le32((1 + !!checksum2_valid) <<
  711. (PAGE_SHIFT - 9));
  712. payload->location = cpu_to_le64(location);
  713. payload->checksum[0] = cpu_to_le32(checksum1);
  714. if (checksum2_valid)
  715. payload->checksum[1] = cpu_to_le32(checksum2);
  716. io->meta_offset += sizeof(struct r5l_payload_data_parity) +
  717. sizeof(__le32) * (1 + !!checksum2_valid);
  718. }
  719. static void r5l_append_payload_page(struct r5l_log *log, struct page *page)
  720. {
  721. struct r5l_io_unit *io = log->current_io;
  722. if (io->need_split_bio) {
  723. BUG_ON(io->split_bio);
  724. io->split_bio = io->current_bio;
  725. io->current_bio = r5l_bio_alloc(log);
  726. bio_chain(io->current_bio, io->split_bio);
  727. io->need_split_bio = false;
  728. }
  729. if (!bio_add_page(io->current_bio, page, PAGE_SIZE, 0))
  730. BUG();
  731. r5_reserve_log_entry(log, io);
  732. }
  733. static void r5l_append_flush_payload(struct r5l_log *log, sector_t sect)
  734. {
  735. struct mddev *mddev = log->rdev->mddev;
  736. struct r5conf *conf = mddev->private;
  737. struct r5l_io_unit *io;
  738. struct r5l_payload_flush *payload;
  739. int meta_size;
  740. /*
  741. * payload_flush requires extra writes to the journal.
  742. * To avoid handling the extra IO in quiesce, just skip
  743. * flush_payload
  744. */
  745. if (conf->quiesce)
  746. return;
  747. mutex_lock(&log->io_mutex);
  748. meta_size = sizeof(struct r5l_payload_flush) + sizeof(__le64);
  749. if (r5l_get_meta(log, meta_size)) {
  750. mutex_unlock(&log->io_mutex);
  751. return;
  752. }
  753. /* current implementation is one stripe per flush payload */
  754. io = log->current_io;
  755. payload = page_address(io->meta_page) + io->meta_offset;
  756. payload->header.type = cpu_to_le16(R5LOG_PAYLOAD_FLUSH);
  757. payload->header.flags = cpu_to_le16(0);
  758. payload->size = cpu_to_le32(sizeof(__le64));
  759. payload->flush_stripes[0] = cpu_to_le64(sect);
  760. io->meta_offset += meta_size;
  761. /* multiple flush payloads count as one pending_stripe */
  762. if (!io->has_flush_payload) {
  763. io->has_flush_payload = 1;
  764. atomic_inc(&io->pending_stripe);
  765. }
  766. mutex_unlock(&log->io_mutex);
  767. }
  768. static int r5l_log_stripe(struct r5l_log *log, struct stripe_head *sh,
  769. int data_pages, int parity_pages)
  770. {
  771. int i;
  772. int meta_size;
  773. int ret;
  774. struct r5l_io_unit *io;
  775. meta_size =
  776. ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32))
  777. * data_pages) +
  778. sizeof(struct r5l_payload_data_parity) +
  779. sizeof(__le32) * parity_pages;
  780. ret = r5l_get_meta(log, meta_size);
  781. if (ret)
  782. return ret;
  783. io = log->current_io;
  784. if (test_and_clear_bit(STRIPE_R5C_PREFLUSH, &sh->state))
  785. io->has_flush = 1;
  786. for (i = 0; i < sh->disks; i++) {
  787. if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
  788. test_bit(R5_InJournal, &sh->dev[i].flags))
  789. continue;
  790. if (i == sh->pd_idx || i == sh->qd_idx)
  791. continue;
  792. if (test_bit(R5_WantFUA, &sh->dev[i].flags) &&
  793. log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK) {
  794. io->has_fua = 1;
  795. /*
  796. * we need to flush journal to make sure recovery can
  797. * reach the data with fua flag
  798. */
  799. io->has_flush = 1;
  800. }
  801. r5l_append_payload_meta(log, R5LOG_PAYLOAD_DATA,
  802. raid5_compute_blocknr(sh, i, 0),
  803. sh->dev[i].log_checksum, 0, false);
  804. r5l_append_payload_page(log, sh->dev[i].page);
  805. }
  806. if (parity_pages == 2) {
  807. r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
  808. sh->sector, sh->dev[sh->pd_idx].log_checksum,
  809. sh->dev[sh->qd_idx].log_checksum, true);
  810. r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
  811. r5l_append_payload_page(log, sh->dev[sh->qd_idx].page);
  812. } else if (parity_pages == 1) {
  813. r5l_append_payload_meta(log, R5LOG_PAYLOAD_PARITY,
  814. sh->sector, sh->dev[sh->pd_idx].log_checksum,
  815. 0, false);
  816. r5l_append_payload_page(log, sh->dev[sh->pd_idx].page);
  817. } else /* Just writing data, not parity, in caching phase */
  818. BUG_ON(parity_pages != 0);
  819. list_add_tail(&sh->log_list, &io->stripe_list);
  820. atomic_inc(&io->pending_stripe);
  821. sh->log_io = io;
  822. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
  823. return 0;
  824. if (sh->log_start == MaxSector) {
  825. BUG_ON(!list_empty(&sh->r5c));
  826. sh->log_start = io->log_start;
  827. spin_lock_irq(&log->stripe_in_journal_lock);
  828. list_add_tail(&sh->r5c,
  829. &log->stripe_in_journal_list);
  830. spin_unlock_irq(&log->stripe_in_journal_lock);
  831. atomic_inc(&log->stripe_in_journal_count);
  832. }
  833. return 0;
  834. }
  835. /* add stripe to no_space_stripes, and then wake up reclaim */
  836. static inline void r5l_add_no_space_stripe(struct r5l_log *log,
  837. struct stripe_head *sh)
  838. {
  839. spin_lock(&log->no_space_stripes_lock);
  840. list_add_tail(&sh->log_list, &log->no_space_stripes);
  841. spin_unlock(&log->no_space_stripes_lock);
  842. }
  843. /*
  844. * running in raid5d, where reclaim could wait for raid5d too (when it flushes
  845. * data from log to raid disks), so we shouldn't wait for reclaim here
  846. */
  847. int r5l_write_stripe(struct r5l_log *log, struct stripe_head *sh)
  848. {
  849. struct r5conf *conf = sh->raid_conf;
  850. int write_disks = 0;
  851. int data_pages, parity_pages;
  852. int reserve;
  853. int i;
  854. int ret = 0;
  855. bool wake_reclaim = false;
  856. if (!log)
  857. return -EAGAIN;
  858. /* Don't support stripe batch */
  859. if (sh->log_io || !test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags) ||
  860. test_bit(STRIPE_SYNCING, &sh->state)) {
  861. /* the stripe is written to log, we start writing it to raid */
  862. clear_bit(STRIPE_LOG_TRAPPED, &sh->state);
  863. return -EAGAIN;
  864. }
  865. WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
  866. for (i = 0; i < sh->disks; i++) {
  867. void *addr;
  868. if (!test_bit(R5_Wantwrite, &sh->dev[i].flags) ||
  869. test_bit(R5_InJournal, &sh->dev[i].flags))
  870. continue;
  871. write_disks++;
  872. /* checksum is already calculated in last run */
  873. if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
  874. continue;
  875. addr = kmap_atomic(sh->dev[i].page);
  876. sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
  877. addr, PAGE_SIZE);
  878. kunmap_atomic(addr);
  879. }
  880. parity_pages = 1 + !!(sh->qd_idx >= 0);
  881. data_pages = write_disks - parity_pages;
  882. set_bit(STRIPE_LOG_TRAPPED, &sh->state);
  883. /*
  884. * The stripe must enter state machine again to finish the write, so
  885. * don't delay.
  886. */
  887. clear_bit(STRIPE_DELAYED, &sh->state);
  888. atomic_inc(&sh->count);
  889. mutex_lock(&log->io_mutex);
  890. /* meta + data */
  891. reserve = (1 + write_disks) << (PAGE_SHIFT - 9);
  892. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
  893. if (!r5l_has_free_space(log, reserve)) {
  894. r5l_add_no_space_stripe(log, sh);
  895. wake_reclaim = true;
  896. } else {
  897. ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
  898. if (ret) {
  899. spin_lock_irq(&log->io_list_lock);
  900. list_add_tail(&sh->log_list,
  901. &log->no_mem_stripes);
  902. spin_unlock_irq(&log->io_list_lock);
  903. }
  904. }
  905. } else { /* R5C_JOURNAL_MODE_WRITE_BACK */
  906. /*
  907. * log space critical, do not process stripes that are
  908. * not in cache yet (sh->log_start == MaxSector).
  909. */
  910. if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
  911. sh->log_start == MaxSector) {
  912. r5l_add_no_space_stripe(log, sh);
  913. wake_reclaim = true;
  914. reserve = 0;
  915. } else if (!r5l_has_free_space(log, reserve)) {
  916. if (sh->log_start == log->last_checkpoint)
  917. BUG();
  918. else
  919. r5l_add_no_space_stripe(log, sh);
  920. } else {
  921. ret = r5l_log_stripe(log, sh, data_pages, parity_pages);
  922. if (ret) {
  923. spin_lock_irq(&log->io_list_lock);
  924. list_add_tail(&sh->log_list,
  925. &log->no_mem_stripes);
  926. spin_unlock_irq(&log->io_list_lock);
  927. }
  928. }
  929. }
  930. mutex_unlock(&log->io_mutex);
  931. if (wake_reclaim)
  932. r5l_wake_reclaim(log, reserve);
  933. return 0;
  934. }
  935. void r5l_write_stripe_run(struct r5l_log *log)
  936. {
  937. if (!log)
  938. return;
  939. mutex_lock(&log->io_mutex);
  940. r5l_submit_current_io(log);
  941. mutex_unlock(&log->io_mutex);
  942. }
  943. int r5l_handle_flush_request(struct r5l_log *log, struct bio *bio)
  944. {
  945. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH) {
  946. /*
  947. * in write through (journal only)
  948. * we flush log disk cache first, then write stripe data to
  949. * raid disks. So if bio is finished, the log disk cache is
  950. * flushed already. The recovery guarantees we can recovery
  951. * the bio from log disk, so we don't need to flush again
  952. */
  953. if (bio->bi_iter.bi_size == 0) {
  954. bio_endio(bio);
  955. return 0;
  956. }
  957. bio->bi_opf &= ~REQ_PREFLUSH;
  958. } else {
  959. /* write back (with cache) */
  960. if (bio->bi_iter.bi_size == 0) {
  961. mutex_lock(&log->io_mutex);
  962. r5l_get_meta(log, 0);
  963. bio_list_add(&log->current_io->flush_barriers, bio);
  964. log->current_io->has_flush = 1;
  965. log->current_io->has_null_flush = 1;
  966. atomic_inc(&log->current_io->pending_stripe);
  967. r5l_submit_current_io(log);
  968. mutex_unlock(&log->io_mutex);
  969. return 0;
  970. }
  971. }
  972. return -EAGAIN;
  973. }
  974. /* This will run after log space is reclaimed */
  975. static void r5l_run_no_space_stripes(struct r5l_log *log)
  976. {
  977. struct stripe_head *sh;
  978. spin_lock(&log->no_space_stripes_lock);
  979. while (!list_empty(&log->no_space_stripes)) {
  980. sh = list_first_entry(&log->no_space_stripes,
  981. struct stripe_head, log_list);
  982. list_del_init(&sh->log_list);
  983. set_bit(STRIPE_HANDLE, &sh->state);
  984. raid5_release_stripe(sh);
  985. }
  986. spin_unlock(&log->no_space_stripes_lock);
  987. }
  988. /*
  989. * calculate new last_checkpoint
  990. * for write through mode, returns log->next_checkpoint
  991. * for write back, returns log_start of first sh in stripe_in_journal_list
  992. */
  993. static sector_t r5c_calculate_new_cp(struct r5conf *conf)
  994. {
  995. struct stripe_head *sh;
  996. struct r5l_log *log = conf->log;
  997. sector_t new_cp;
  998. unsigned long flags;
  999. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
  1000. return log->next_checkpoint;
  1001. spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
  1002. if (list_empty(&conf->log->stripe_in_journal_list)) {
  1003. /* all stripes flushed */
  1004. spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
  1005. return log->next_checkpoint;
  1006. }
  1007. sh = list_first_entry(&conf->log->stripe_in_journal_list,
  1008. struct stripe_head, r5c);
  1009. new_cp = sh->log_start;
  1010. spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
  1011. return new_cp;
  1012. }
  1013. static sector_t r5l_reclaimable_space(struct r5l_log *log)
  1014. {
  1015. struct r5conf *conf = log->rdev->mddev->private;
  1016. return r5l_ring_distance(log, log->last_checkpoint,
  1017. r5c_calculate_new_cp(conf));
  1018. }
  1019. static void r5l_run_no_mem_stripe(struct r5l_log *log)
  1020. {
  1021. struct stripe_head *sh;
  1022. lockdep_assert_held(&log->io_list_lock);
  1023. if (!list_empty(&log->no_mem_stripes)) {
  1024. sh = list_first_entry(&log->no_mem_stripes,
  1025. struct stripe_head, log_list);
  1026. list_del_init(&sh->log_list);
  1027. set_bit(STRIPE_HANDLE, &sh->state);
  1028. raid5_release_stripe(sh);
  1029. }
  1030. }
  1031. static bool r5l_complete_finished_ios(struct r5l_log *log)
  1032. {
  1033. struct r5l_io_unit *io, *next;
  1034. bool found = false;
  1035. lockdep_assert_held(&log->io_list_lock);
  1036. list_for_each_entry_safe(io, next, &log->finished_ios, log_sibling) {
  1037. /* don't change list order */
  1038. if (io->state < IO_UNIT_STRIPE_END)
  1039. break;
  1040. log->next_checkpoint = io->log_start;
  1041. list_del(&io->log_sibling);
  1042. mempool_free(io, &log->io_pool);
  1043. r5l_run_no_mem_stripe(log);
  1044. found = true;
  1045. }
  1046. return found;
  1047. }
  1048. static void __r5l_stripe_write_finished(struct r5l_io_unit *io)
  1049. {
  1050. struct r5l_log *log = io->log;
  1051. struct r5conf *conf = log->rdev->mddev->private;
  1052. unsigned long flags;
  1053. spin_lock_irqsave(&log->io_list_lock, flags);
  1054. __r5l_set_io_unit_state(io, IO_UNIT_STRIPE_END);
  1055. if (!r5l_complete_finished_ios(log)) {
  1056. spin_unlock_irqrestore(&log->io_list_lock, flags);
  1057. return;
  1058. }
  1059. if (r5l_reclaimable_space(log) > log->max_free_space ||
  1060. test_bit(R5C_LOG_TIGHT, &conf->cache_state))
  1061. r5l_wake_reclaim(log, 0);
  1062. spin_unlock_irqrestore(&log->io_list_lock, flags);
  1063. wake_up(&log->iounit_wait);
  1064. }
  1065. void r5l_stripe_write_finished(struct stripe_head *sh)
  1066. {
  1067. struct r5l_io_unit *io;
  1068. io = sh->log_io;
  1069. sh->log_io = NULL;
  1070. if (io && atomic_dec_and_test(&io->pending_stripe))
  1071. __r5l_stripe_write_finished(io);
  1072. }
  1073. static void r5l_log_flush_endio(struct bio *bio)
  1074. {
  1075. struct r5l_log *log = container_of(bio, struct r5l_log,
  1076. flush_bio);
  1077. unsigned long flags;
  1078. struct r5l_io_unit *io;
  1079. if (bio->bi_status)
  1080. md_error(log->rdev->mddev, log->rdev);
  1081. spin_lock_irqsave(&log->io_list_lock, flags);
  1082. list_for_each_entry(io, &log->flushing_ios, log_sibling)
  1083. r5l_io_run_stripes(io);
  1084. list_splice_tail_init(&log->flushing_ios, &log->finished_ios);
  1085. spin_unlock_irqrestore(&log->io_list_lock, flags);
  1086. }
  1087. /*
  1088. * Starting dispatch IO to raid.
  1089. * io_unit(meta) consists of a log. There is one situation we want to avoid. A
  1090. * broken meta in the middle of a log causes recovery can't find meta at the
  1091. * head of log. If operations require meta at the head persistent in log, we
  1092. * must make sure meta before it persistent in log too. A case is:
  1093. *
  1094. * stripe data/parity is in log, we start write stripe to raid disks. stripe
  1095. * data/parity must be persistent in log before we do the write to raid disks.
  1096. *
  1097. * The solution is we restrictly maintain io_unit list order. In this case, we
  1098. * only write stripes of an io_unit to raid disks till the io_unit is the first
  1099. * one whose data/parity is in log.
  1100. */
  1101. void r5l_flush_stripe_to_raid(struct r5l_log *log)
  1102. {
  1103. bool do_flush;
  1104. if (!log || !log->need_cache_flush)
  1105. return;
  1106. spin_lock_irq(&log->io_list_lock);
  1107. /* flush bio is running */
  1108. if (!list_empty(&log->flushing_ios)) {
  1109. spin_unlock_irq(&log->io_list_lock);
  1110. return;
  1111. }
  1112. list_splice_tail_init(&log->io_end_ios, &log->flushing_ios);
  1113. do_flush = !list_empty(&log->flushing_ios);
  1114. spin_unlock_irq(&log->io_list_lock);
  1115. if (!do_flush)
  1116. return;
  1117. bio_reset(&log->flush_bio);
  1118. bio_set_dev(&log->flush_bio, log->rdev->bdev);
  1119. log->flush_bio.bi_end_io = r5l_log_flush_endio;
  1120. log->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
  1121. submit_bio(&log->flush_bio);
  1122. }
  1123. static void r5l_write_super(struct r5l_log *log, sector_t cp);
  1124. static void r5l_write_super_and_discard_space(struct r5l_log *log,
  1125. sector_t end)
  1126. {
  1127. struct block_device *bdev = log->rdev->bdev;
  1128. struct mddev *mddev;
  1129. r5l_write_super(log, end);
  1130. if (!blk_queue_discard(bdev_get_queue(bdev)))
  1131. return;
  1132. mddev = log->rdev->mddev;
  1133. /*
  1134. * Discard could zero data, so before discard we must make sure
  1135. * superblock is updated to new log tail. Updating superblock (either
  1136. * directly call md_update_sb() or depend on md thread) must hold
  1137. * reconfig mutex. On the other hand, raid5_quiesce is called with
  1138. * reconfig_mutex hold. The first step of raid5_quiesce() is waitting
  1139. * for all IO finish, hence waitting for reclaim thread, while reclaim
  1140. * thread is calling this function and waitting for reconfig mutex. So
  1141. * there is a deadlock. We workaround this issue with a trylock.
  1142. * FIXME: we could miss discard if we can't take reconfig mutex
  1143. */
  1144. set_mask_bits(&mddev->sb_flags, 0,
  1145. BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
  1146. if (!mddev_trylock(mddev))
  1147. return;
  1148. md_update_sb(mddev, 1);
  1149. mddev_unlock(mddev);
  1150. /* discard IO error really doesn't matter, ignore it */
  1151. if (log->last_checkpoint < end) {
  1152. blkdev_issue_discard(bdev,
  1153. log->last_checkpoint + log->rdev->data_offset,
  1154. end - log->last_checkpoint, GFP_NOIO, 0);
  1155. } else {
  1156. blkdev_issue_discard(bdev,
  1157. log->last_checkpoint + log->rdev->data_offset,
  1158. log->device_size - log->last_checkpoint,
  1159. GFP_NOIO, 0);
  1160. blkdev_issue_discard(bdev, log->rdev->data_offset, end,
  1161. GFP_NOIO, 0);
  1162. }
  1163. }
  1164. /*
  1165. * r5c_flush_stripe moves stripe from cached list to handle_list. When called,
  1166. * the stripe must be on r5c_cached_full_stripes or r5c_cached_partial_stripes.
  1167. *
  1168. * must hold conf->device_lock
  1169. */
  1170. static void r5c_flush_stripe(struct r5conf *conf, struct stripe_head *sh)
  1171. {
  1172. BUG_ON(list_empty(&sh->lru));
  1173. BUG_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
  1174. BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
  1175. /*
  1176. * The stripe is not ON_RELEASE_LIST, so it is safe to call
  1177. * raid5_release_stripe() while holding conf->device_lock
  1178. */
  1179. BUG_ON(test_bit(STRIPE_ON_RELEASE_LIST, &sh->state));
  1180. lockdep_assert_held(&conf->device_lock);
  1181. list_del_init(&sh->lru);
  1182. atomic_inc(&sh->count);
  1183. set_bit(STRIPE_HANDLE, &sh->state);
  1184. atomic_inc(&conf->active_stripes);
  1185. r5c_make_stripe_write_out(sh);
  1186. if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
  1187. atomic_inc(&conf->r5c_flushing_partial_stripes);
  1188. else
  1189. atomic_inc(&conf->r5c_flushing_full_stripes);
  1190. raid5_release_stripe(sh);
  1191. }
  1192. /*
  1193. * if num == 0, flush all full stripes
  1194. * if num > 0, flush all full stripes. If less than num full stripes are
  1195. * flushed, flush some partial stripes until totally num stripes are
  1196. * flushed or there is no more cached stripes.
  1197. */
  1198. void r5c_flush_cache(struct r5conf *conf, int num)
  1199. {
  1200. int count;
  1201. struct stripe_head *sh, *next;
  1202. lockdep_assert_held(&conf->device_lock);
  1203. if (!conf->log)
  1204. return;
  1205. count = 0;
  1206. list_for_each_entry_safe(sh, next, &conf->r5c_full_stripe_list, lru) {
  1207. r5c_flush_stripe(conf, sh);
  1208. count++;
  1209. }
  1210. if (count >= num)
  1211. return;
  1212. list_for_each_entry_safe(sh, next,
  1213. &conf->r5c_partial_stripe_list, lru) {
  1214. r5c_flush_stripe(conf, sh);
  1215. if (++count >= num)
  1216. break;
  1217. }
  1218. }
  1219. static void r5c_do_reclaim(struct r5conf *conf)
  1220. {
  1221. struct r5l_log *log = conf->log;
  1222. struct stripe_head *sh;
  1223. int count = 0;
  1224. unsigned long flags;
  1225. int total_cached;
  1226. int stripes_to_flush;
  1227. int flushing_partial, flushing_full;
  1228. if (!r5c_is_writeback(log))
  1229. return;
  1230. flushing_partial = atomic_read(&conf->r5c_flushing_partial_stripes);
  1231. flushing_full = atomic_read(&conf->r5c_flushing_full_stripes);
  1232. total_cached = atomic_read(&conf->r5c_cached_partial_stripes) +
  1233. atomic_read(&conf->r5c_cached_full_stripes) -
  1234. flushing_full - flushing_partial;
  1235. if (total_cached > conf->min_nr_stripes * 3 / 4 ||
  1236. atomic_read(&conf->empty_inactive_list_nr) > 0)
  1237. /*
  1238. * if stripe cache pressure high, flush all full stripes and
  1239. * some partial stripes
  1240. */
  1241. stripes_to_flush = R5C_RECLAIM_STRIPE_GROUP;
  1242. else if (total_cached > conf->min_nr_stripes * 1 / 2 ||
  1243. atomic_read(&conf->r5c_cached_full_stripes) - flushing_full >
  1244. R5C_FULL_STRIPE_FLUSH_BATCH(conf))
  1245. /*
  1246. * if stripe cache pressure moderate, or if there is many full
  1247. * stripes,flush all full stripes
  1248. */
  1249. stripes_to_flush = 0;
  1250. else
  1251. /* no need to flush */
  1252. stripes_to_flush = -1;
  1253. if (stripes_to_flush >= 0) {
  1254. spin_lock_irqsave(&conf->device_lock, flags);
  1255. r5c_flush_cache(conf, stripes_to_flush);
  1256. spin_unlock_irqrestore(&conf->device_lock, flags);
  1257. }
  1258. /* if log space is tight, flush stripes on stripe_in_journal_list */
  1259. if (test_bit(R5C_LOG_TIGHT, &conf->cache_state)) {
  1260. spin_lock_irqsave(&log->stripe_in_journal_lock, flags);
  1261. spin_lock(&conf->device_lock);
  1262. list_for_each_entry(sh, &log->stripe_in_journal_list, r5c) {
  1263. /*
  1264. * stripes on stripe_in_journal_list could be in any
  1265. * state of the stripe_cache state machine. In this
  1266. * case, we only want to flush stripe on
  1267. * r5c_cached_full/partial_stripes. The following
  1268. * condition makes sure the stripe is on one of the
  1269. * two lists.
  1270. */
  1271. if (!list_empty(&sh->lru) &&
  1272. !test_bit(STRIPE_HANDLE, &sh->state) &&
  1273. atomic_read(&sh->count) == 0) {
  1274. r5c_flush_stripe(conf, sh);
  1275. if (count++ >= R5C_RECLAIM_STRIPE_GROUP)
  1276. break;
  1277. }
  1278. }
  1279. spin_unlock(&conf->device_lock);
  1280. spin_unlock_irqrestore(&log->stripe_in_journal_lock, flags);
  1281. }
  1282. if (!test_bit(R5C_LOG_CRITICAL, &conf->cache_state))
  1283. r5l_run_no_space_stripes(log);
  1284. md_wakeup_thread(conf->mddev->thread);
  1285. }
  1286. static void r5l_do_reclaim(struct r5l_log *log)
  1287. {
  1288. struct r5conf *conf = log->rdev->mddev->private;
  1289. sector_t reclaim_target = xchg(&log->reclaim_target, 0);
  1290. sector_t reclaimable;
  1291. sector_t next_checkpoint;
  1292. bool write_super;
  1293. spin_lock_irq(&log->io_list_lock);
  1294. write_super = r5l_reclaimable_space(log) > log->max_free_space ||
  1295. reclaim_target != 0 || !list_empty(&log->no_space_stripes);
  1296. /*
  1297. * move proper io_unit to reclaim list. We should not change the order.
  1298. * reclaimable/unreclaimable io_unit can be mixed in the list, we
  1299. * shouldn't reuse space of an unreclaimable io_unit
  1300. */
  1301. while (1) {
  1302. reclaimable = r5l_reclaimable_space(log);
  1303. if (reclaimable >= reclaim_target ||
  1304. (list_empty(&log->running_ios) &&
  1305. list_empty(&log->io_end_ios) &&
  1306. list_empty(&log->flushing_ios) &&
  1307. list_empty(&log->finished_ios)))
  1308. break;
  1309. md_wakeup_thread(log->rdev->mddev->thread);
  1310. wait_event_lock_irq(log->iounit_wait,
  1311. r5l_reclaimable_space(log) > reclaimable,
  1312. log->io_list_lock);
  1313. }
  1314. next_checkpoint = r5c_calculate_new_cp(conf);
  1315. spin_unlock_irq(&log->io_list_lock);
  1316. if (reclaimable == 0 || !write_super)
  1317. return;
  1318. /*
  1319. * write_super will flush cache of each raid disk. We must write super
  1320. * here, because the log area might be reused soon and we don't want to
  1321. * confuse recovery
  1322. */
  1323. r5l_write_super_and_discard_space(log, next_checkpoint);
  1324. mutex_lock(&log->io_mutex);
  1325. log->last_checkpoint = next_checkpoint;
  1326. r5c_update_log_state(log);
  1327. mutex_unlock(&log->io_mutex);
  1328. r5l_run_no_space_stripes(log);
  1329. }
  1330. static void r5l_reclaim_thread(struct md_thread *thread)
  1331. {
  1332. struct mddev *mddev = thread->mddev;
  1333. struct r5conf *conf = mddev->private;
  1334. struct r5l_log *log = conf->log;
  1335. if (!log)
  1336. return;
  1337. r5c_do_reclaim(conf);
  1338. r5l_do_reclaim(log);
  1339. }
  1340. void r5l_wake_reclaim(struct r5l_log *log, sector_t space)
  1341. {
  1342. unsigned long target;
  1343. unsigned long new = (unsigned long)space; /* overflow in theory */
  1344. if (!log)
  1345. return;
  1346. do {
  1347. target = log->reclaim_target;
  1348. if (new < target)
  1349. return;
  1350. } while (cmpxchg(&log->reclaim_target, target, new) != target);
  1351. md_wakeup_thread(log->reclaim_thread);
  1352. }
  1353. void r5l_quiesce(struct r5l_log *log, int quiesce)
  1354. {
  1355. struct mddev *mddev;
  1356. if (quiesce) {
  1357. /* make sure r5l_write_super_and_discard_space exits */
  1358. mddev = log->rdev->mddev;
  1359. wake_up(&mddev->sb_wait);
  1360. kthread_park(log->reclaim_thread->tsk);
  1361. r5l_wake_reclaim(log, MaxSector);
  1362. r5l_do_reclaim(log);
  1363. } else
  1364. kthread_unpark(log->reclaim_thread->tsk);
  1365. }
  1366. bool r5l_log_disk_error(struct r5conf *conf)
  1367. {
  1368. struct r5l_log *log;
  1369. bool ret;
  1370. /* don't allow write if journal disk is missing */
  1371. rcu_read_lock();
  1372. log = rcu_dereference(conf->log);
  1373. if (!log)
  1374. ret = test_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
  1375. else
  1376. ret = test_bit(Faulty, &log->rdev->flags);
  1377. rcu_read_unlock();
  1378. return ret;
  1379. }
  1380. #define R5L_RECOVERY_PAGE_POOL_SIZE 256
  1381. struct r5l_recovery_ctx {
  1382. struct page *meta_page; /* current meta */
  1383. sector_t meta_total_blocks; /* total size of current meta and data */
  1384. sector_t pos; /* recovery position */
  1385. u64 seq; /* recovery position seq */
  1386. int data_parity_stripes; /* number of data_parity stripes */
  1387. int data_only_stripes; /* number of data_only stripes */
  1388. struct list_head cached_list;
  1389. /*
  1390. * read ahead page pool (ra_pool)
  1391. * in recovery, log is read sequentially. It is not efficient to
  1392. * read every page with sync_page_io(). The read ahead page pool
  1393. * reads multiple pages with one IO, so further log read can
  1394. * just copy data from the pool.
  1395. */
  1396. struct page *ra_pool[R5L_RECOVERY_PAGE_POOL_SIZE];
  1397. sector_t pool_offset; /* offset of first page in the pool */
  1398. int total_pages; /* total allocated pages */
  1399. int valid_pages; /* pages with valid data */
  1400. struct bio *ra_bio; /* bio to do the read ahead */
  1401. };
  1402. static int r5l_recovery_allocate_ra_pool(struct r5l_log *log,
  1403. struct r5l_recovery_ctx *ctx)
  1404. {
  1405. struct page *page;
  1406. ctx->ra_bio = bio_alloc_bioset(GFP_KERNEL, BIO_MAX_PAGES, &log->bs);
  1407. if (!ctx->ra_bio)
  1408. return -ENOMEM;
  1409. ctx->valid_pages = 0;
  1410. ctx->total_pages = 0;
  1411. while (ctx->total_pages < R5L_RECOVERY_PAGE_POOL_SIZE) {
  1412. page = alloc_page(GFP_KERNEL);
  1413. if (!page)
  1414. break;
  1415. ctx->ra_pool[ctx->total_pages] = page;
  1416. ctx->total_pages += 1;
  1417. }
  1418. if (ctx->total_pages == 0) {
  1419. bio_put(ctx->ra_bio);
  1420. return -ENOMEM;
  1421. }
  1422. ctx->pool_offset = 0;
  1423. return 0;
  1424. }
  1425. static void r5l_recovery_free_ra_pool(struct r5l_log *log,
  1426. struct r5l_recovery_ctx *ctx)
  1427. {
  1428. int i;
  1429. for (i = 0; i < ctx->total_pages; ++i)
  1430. put_page(ctx->ra_pool[i]);
  1431. bio_put(ctx->ra_bio);
  1432. }
  1433. /*
  1434. * fetch ctx->valid_pages pages from offset
  1435. * In normal cases, ctx->valid_pages == ctx->total_pages after the call.
  1436. * However, if the offset is close to the end of the journal device,
  1437. * ctx->valid_pages could be smaller than ctx->total_pages
  1438. */
  1439. static int r5l_recovery_fetch_ra_pool(struct r5l_log *log,
  1440. struct r5l_recovery_ctx *ctx,
  1441. sector_t offset)
  1442. {
  1443. bio_reset(ctx->ra_bio);
  1444. bio_set_dev(ctx->ra_bio, log->rdev->bdev);
  1445. bio_set_op_attrs(ctx->ra_bio, REQ_OP_READ, 0);
  1446. ctx->ra_bio->bi_iter.bi_sector = log->rdev->data_offset + offset;
  1447. ctx->valid_pages = 0;
  1448. ctx->pool_offset = offset;
  1449. while (ctx->valid_pages < ctx->total_pages) {
  1450. bio_add_page(ctx->ra_bio,
  1451. ctx->ra_pool[ctx->valid_pages], PAGE_SIZE, 0);
  1452. ctx->valid_pages += 1;
  1453. offset = r5l_ring_add(log, offset, BLOCK_SECTORS);
  1454. if (offset == 0) /* reached end of the device */
  1455. break;
  1456. }
  1457. return submit_bio_wait(ctx->ra_bio);
  1458. }
  1459. /*
  1460. * try read a page from the read ahead page pool, if the page is not in the
  1461. * pool, call r5l_recovery_fetch_ra_pool
  1462. */
  1463. static int r5l_recovery_read_page(struct r5l_log *log,
  1464. struct r5l_recovery_ctx *ctx,
  1465. struct page *page,
  1466. sector_t offset)
  1467. {
  1468. int ret;
  1469. if (offset < ctx->pool_offset ||
  1470. offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS) {
  1471. ret = r5l_recovery_fetch_ra_pool(log, ctx, offset);
  1472. if (ret)
  1473. return ret;
  1474. }
  1475. BUG_ON(offset < ctx->pool_offset ||
  1476. offset >= ctx->pool_offset + ctx->valid_pages * BLOCK_SECTORS);
  1477. memcpy(page_address(page),
  1478. page_address(ctx->ra_pool[(offset - ctx->pool_offset) >>
  1479. BLOCK_SECTOR_SHIFT]),
  1480. PAGE_SIZE);
  1481. return 0;
  1482. }
  1483. static int r5l_recovery_read_meta_block(struct r5l_log *log,
  1484. struct r5l_recovery_ctx *ctx)
  1485. {
  1486. struct page *page = ctx->meta_page;
  1487. struct r5l_meta_block *mb;
  1488. u32 crc, stored_crc;
  1489. int ret;
  1490. ret = r5l_recovery_read_page(log, ctx, page, ctx->pos);
  1491. if (ret != 0)
  1492. return ret;
  1493. mb = page_address(page);
  1494. stored_crc = le32_to_cpu(mb->checksum);
  1495. mb->checksum = 0;
  1496. if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
  1497. le64_to_cpu(mb->seq) != ctx->seq ||
  1498. mb->version != R5LOG_VERSION ||
  1499. le64_to_cpu(mb->position) != ctx->pos)
  1500. return -EINVAL;
  1501. crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
  1502. if (stored_crc != crc)
  1503. return -EINVAL;
  1504. if (le32_to_cpu(mb->meta_size) > PAGE_SIZE)
  1505. return -EINVAL;
  1506. ctx->meta_total_blocks = BLOCK_SECTORS;
  1507. return 0;
  1508. }
  1509. static void
  1510. r5l_recovery_create_empty_meta_block(struct r5l_log *log,
  1511. struct page *page,
  1512. sector_t pos, u64 seq)
  1513. {
  1514. struct r5l_meta_block *mb;
  1515. mb = page_address(page);
  1516. clear_page(mb);
  1517. mb->magic = cpu_to_le32(R5LOG_MAGIC);
  1518. mb->version = R5LOG_VERSION;
  1519. mb->meta_size = cpu_to_le32(sizeof(struct r5l_meta_block));
  1520. mb->seq = cpu_to_le64(seq);
  1521. mb->position = cpu_to_le64(pos);
  1522. }
  1523. static int r5l_log_write_empty_meta_block(struct r5l_log *log, sector_t pos,
  1524. u64 seq)
  1525. {
  1526. struct page *page;
  1527. struct r5l_meta_block *mb;
  1528. page = alloc_page(GFP_KERNEL);
  1529. if (!page)
  1530. return -ENOMEM;
  1531. r5l_recovery_create_empty_meta_block(log, page, pos, seq);
  1532. mb = page_address(page);
  1533. mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
  1534. mb, PAGE_SIZE));
  1535. if (!sync_page_io(log->rdev, pos, PAGE_SIZE, page, REQ_OP_WRITE,
  1536. REQ_SYNC | REQ_FUA, false)) {
  1537. __free_page(page);
  1538. return -EIO;
  1539. }
  1540. __free_page(page);
  1541. return 0;
  1542. }
  1543. /*
  1544. * r5l_recovery_load_data and r5l_recovery_load_parity uses flag R5_Wantwrite
  1545. * to mark valid (potentially not flushed) data in the journal.
  1546. *
  1547. * We already verified checksum in r5l_recovery_verify_data_checksum_for_mb,
  1548. * so there should not be any mismatch here.
  1549. */
  1550. static void r5l_recovery_load_data(struct r5l_log *log,
  1551. struct stripe_head *sh,
  1552. struct r5l_recovery_ctx *ctx,
  1553. struct r5l_payload_data_parity *payload,
  1554. sector_t log_offset)
  1555. {
  1556. struct mddev *mddev = log->rdev->mddev;
  1557. struct r5conf *conf = mddev->private;
  1558. int dd_idx;
  1559. raid5_compute_sector(conf,
  1560. le64_to_cpu(payload->location), 0,
  1561. &dd_idx, sh);
  1562. r5l_recovery_read_page(log, ctx, sh->dev[dd_idx].page, log_offset);
  1563. sh->dev[dd_idx].log_checksum =
  1564. le32_to_cpu(payload->checksum[0]);
  1565. ctx->meta_total_blocks += BLOCK_SECTORS;
  1566. set_bit(R5_Wantwrite, &sh->dev[dd_idx].flags);
  1567. set_bit(STRIPE_R5C_CACHING, &sh->state);
  1568. }
  1569. static void r5l_recovery_load_parity(struct r5l_log *log,
  1570. struct stripe_head *sh,
  1571. struct r5l_recovery_ctx *ctx,
  1572. struct r5l_payload_data_parity *payload,
  1573. sector_t log_offset)
  1574. {
  1575. struct mddev *mddev = log->rdev->mddev;
  1576. struct r5conf *conf = mddev->private;
  1577. ctx->meta_total_blocks += BLOCK_SECTORS * conf->max_degraded;
  1578. r5l_recovery_read_page(log, ctx, sh->dev[sh->pd_idx].page, log_offset);
  1579. sh->dev[sh->pd_idx].log_checksum =
  1580. le32_to_cpu(payload->checksum[0]);
  1581. set_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags);
  1582. if (sh->qd_idx >= 0) {
  1583. r5l_recovery_read_page(
  1584. log, ctx, sh->dev[sh->qd_idx].page,
  1585. r5l_ring_add(log, log_offset, BLOCK_SECTORS));
  1586. sh->dev[sh->qd_idx].log_checksum =
  1587. le32_to_cpu(payload->checksum[1]);
  1588. set_bit(R5_Wantwrite, &sh->dev[sh->qd_idx].flags);
  1589. }
  1590. clear_bit(STRIPE_R5C_CACHING, &sh->state);
  1591. }
  1592. static void r5l_recovery_reset_stripe(struct stripe_head *sh)
  1593. {
  1594. int i;
  1595. sh->state = 0;
  1596. sh->log_start = MaxSector;
  1597. for (i = sh->disks; i--; )
  1598. sh->dev[i].flags = 0;
  1599. }
  1600. static void
  1601. r5l_recovery_replay_one_stripe(struct r5conf *conf,
  1602. struct stripe_head *sh,
  1603. struct r5l_recovery_ctx *ctx)
  1604. {
  1605. struct md_rdev *rdev, *rrdev;
  1606. int disk_index;
  1607. int data_count = 0;
  1608. for (disk_index = 0; disk_index < sh->disks; disk_index++) {
  1609. if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
  1610. continue;
  1611. if (disk_index == sh->qd_idx || disk_index == sh->pd_idx)
  1612. continue;
  1613. data_count++;
  1614. }
  1615. /*
  1616. * stripes that only have parity must have been flushed
  1617. * before the crash that we are now recovering from, so
  1618. * there is nothing more to recovery.
  1619. */
  1620. if (data_count == 0)
  1621. goto out;
  1622. for (disk_index = 0; disk_index < sh->disks; disk_index++) {
  1623. if (!test_bit(R5_Wantwrite, &sh->dev[disk_index].flags))
  1624. continue;
  1625. /* in case device is broken */
  1626. rcu_read_lock();
  1627. rdev = rcu_dereference(conf->disks[disk_index].rdev);
  1628. if (rdev) {
  1629. atomic_inc(&rdev->nr_pending);
  1630. rcu_read_unlock();
  1631. sync_page_io(rdev, sh->sector, PAGE_SIZE,
  1632. sh->dev[disk_index].page, REQ_OP_WRITE, 0,
  1633. false);
  1634. rdev_dec_pending(rdev, rdev->mddev);
  1635. rcu_read_lock();
  1636. }
  1637. rrdev = rcu_dereference(conf->disks[disk_index].replacement);
  1638. if (rrdev) {
  1639. atomic_inc(&rrdev->nr_pending);
  1640. rcu_read_unlock();
  1641. sync_page_io(rrdev, sh->sector, PAGE_SIZE,
  1642. sh->dev[disk_index].page, REQ_OP_WRITE, 0,
  1643. false);
  1644. rdev_dec_pending(rrdev, rrdev->mddev);
  1645. rcu_read_lock();
  1646. }
  1647. rcu_read_unlock();
  1648. }
  1649. ctx->data_parity_stripes++;
  1650. out:
  1651. r5l_recovery_reset_stripe(sh);
  1652. }
  1653. static struct stripe_head *
  1654. r5c_recovery_alloc_stripe(
  1655. struct r5conf *conf,
  1656. sector_t stripe_sect,
  1657. int noblock)
  1658. {
  1659. struct stripe_head *sh;
  1660. sh = raid5_get_active_stripe(conf, stripe_sect, 0, noblock, 0);
  1661. if (!sh)
  1662. return NULL; /* no more stripe available */
  1663. r5l_recovery_reset_stripe(sh);
  1664. return sh;
  1665. }
  1666. static struct stripe_head *
  1667. r5c_recovery_lookup_stripe(struct list_head *list, sector_t sect)
  1668. {
  1669. struct stripe_head *sh;
  1670. list_for_each_entry(sh, list, lru)
  1671. if (sh->sector == sect)
  1672. return sh;
  1673. return NULL;
  1674. }
  1675. static void
  1676. r5c_recovery_drop_stripes(struct list_head *cached_stripe_list,
  1677. struct r5l_recovery_ctx *ctx)
  1678. {
  1679. struct stripe_head *sh, *next;
  1680. list_for_each_entry_safe(sh, next, cached_stripe_list, lru) {
  1681. r5l_recovery_reset_stripe(sh);
  1682. list_del_init(&sh->lru);
  1683. raid5_release_stripe(sh);
  1684. }
  1685. }
  1686. static void
  1687. r5c_recovery_replay_stripes(struct list_head *cached_stripe_list,
  1688. struct r5l_recovery_ctx *ctx)
  1689. {
  1690. struct stripe_head *sh, *next;
  1691. list_for_each_entry_safe(sh, next, cached_stripe_list, lru)
  1692. if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
  1693. r5l_recovery_replay_one_stripe(sh->raid_conf, sh, ctx);
  1694. list_del_init(&sh->lru);
  1695. raid5_release_stripe(sh);
  1696. }
  1697. }
  1698. /* if matches return 0; otherwise return -EINVAL */
  1699. static int
  1700. r5l_recovery_verify_data_checksum(struct r5l_log *log,
  1701. struct r5l_recovery_ctx *ctx,
  1702. struct page *page,
  1703. sector_t log_offset, __le32 log_checksum)
  1704. {
  1705. void *addr;
  1706. u32 checksum;
  1707. r5l_recovery_read_page(log, ctx, page, log_offset);
  1708. addr = kmap_atomic(page);
  1709. checksum = crc32c_le(log->uuid_checksum, addr, PAGE_SIZE);
  1710. kunmap_atomic(addr);
  1711. return (le32_to_cpu(log_checksum) == checksum) ? 0 : -EINVAL;
  1712. }
  1713. /*
  1714. * before loading data to stripe cache, we need verify checksum for all data,
  1715. * if there is mismatch for any data page, we drop all data in the mata block
  1716. */
  1717. static int
  1718. r5l_recovery_verify_data_checksum_for_mb(struct r5l_log *log,
  1719. struct r5l_recovery_ctx *ctx)
  1720. {
  1721. struct mddev *mddev = log->rdev->mddev;
  1722. struct r5conf *conf = mddev->private;
  1723. struct r5l_meta_block *mb = page_address(ctx->meta_page);
  1724. sector_t mb_offset = sizeof(struct r5l_meta_block);
  1725. sector_t log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
  1726. struct page *page;
  1727. struct r5l_payload_data_parity *payload;
  1728. struct r5l_payload_flush *payload_flush;
  1729. page = alloc_page(GFP_KERNEL);
  1730. if (!page)
  1731. return -ENOMEM;
  1732. while (mb_offset < le32_to_cpu(mb->meta_size)) {
  1733. payload = (void *)mb + mb_offset;
  1734. payload_flush = (void *)mb + mb_offset;
  1735. if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
  1736. if (r5l_recovery_verify_data_checksum(
  1737. log, ctx, page, log_offset,
  1738. payload->checksum[0]) < 0)
  1739. goto mismatch;
  1740. } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY) {
  1741. if (r5l_recovery_verify_data_checksum(
  1742. log, ctx, page, log_offset,
  1743. payload->checksum[0]) < 0)
  1744. goto mismatch;
  1745. if (conf->max_degraded == 2 && /* q for RAID 6 */
  1746. r5l_recovery_verify_data_checksum(
  1747. log, ctx, page,
  1748. r5l_ring_add(log, log_offset,
  1749. BLOCK_SECTORS),
  1750. payload->checksum[1]) < 0)
  1751. goto mismatch;
  1752. } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
  1753. /* nothing to do for R5LOG_PAYLOAD_FLUSH here */
  1754. } else /* not R5LOG_PAYLOAD_DATA/PARITY/FLUSH */
  1755. goto mismatch;
  1756. if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
  1757. mb_offset += sizeof(struct r5l_payload_flush) +
  1758. le32_to_cpu(payload_flush->size);
  1759. } else {
  1760. /* DATA or PARITY payload */
  1761. log_offset = r5l_ring_add(log, log_offset,
  1762. le32_to_cpu(payload->size));
  1763. mb_offset += sizeof(struct r5l_payload_data_parity) +
  1764. sizeof(__le32) *
  1765. (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
  1766. }
  1767. }
  1768. put_page(page);
  1769. return 0;
  1770. mismatch:
  1771. put_page(page);
  1772. return -EINVAL;
  1773. }
  1774. /*
  1775. * Analyze all data/parity pages in one meta block
  1776. * Returns:
  1777. * 0 for success
  1778. * -EINVAL for unknown playload type
  1779. * -EAGAIN for checksum mismatch of data page
  1780. * -ENOMEM for run out of memory (alloc_page failed or run out of stripes)
  1781. */
  1782. static int
  1783. r5c_recovery_analyze_meta_block(struct r5l_log *log,
  1784. struct r5l_recovery_ctx *ctx,
  1785. struct list_head *cached_stripe_list)
  1786. {
  1787. struct mddev *mddev = log->rdev->mddev;
  1788. struct r5conf *conf = mddev->private;
  1789. struct r5l_meta_block *mb;
  1790. struct r5l_payload_data_parity *payload;
  1791. struct r5l_payload_flush *payload_flush;
  1792. int mb_offset;
  1793. sector_t log_offset;
  1794. sector_t stripe_sect;
  1795. struct stripe_head *sh;
  1796. int ret;
  1797. /*
  1798. * for mismatch in data blocks, we will drop all data in this mb, but
  1799. * we will still read next mb for other data with FLUSH flag, as
  1800. * io_unit could finish out of order.
  1801. */
  1802. ret = r5l_recovery_verify_data_checksum_for_mb(log, ctx);
  1803. if (ret == -EINVAL)
  1804. return -EAGAIN;
  1805. else if (ret)
  1806. return ret; /* -ENOMEM duo to alloc_page() failed */
  1807. mb = page_address(ctx->meta_page);
  1808. mb_offset = sizeof(struct r5l_meta_block);
  1809. log_offset = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
  1810. while (mb_offset < le32_to_cpu(mb->meta_size)) {
  1811. int dd;
  1812. payload = (void *)mb + mb_offset;
  1813. payload_flush = (void *)mb + mb_offset;
  1814. if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_FLUSH) {
  1815. int i, count;
  1816. count = le32_to_cpu(payload_flush->size) / sizeof(__le64);
  1817. for (i = 0; i < count; ++i) {
  1818. stripe_sect = le64_to_cpu(payload_flush->flush_stripes[i]);
  1819. sh = r5c_recovery_lookup_stripe(cached_stripe_list,
  1820. stripe_sect);
  1821. if (sh) {
  1822. WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
  1823. r5l_recovery_reset_stripe(sh);
  1824. list_del_init(&sh->lru);
  1825. raid5_release_stripe(sh);
  1826. }
  1827. }
  1828. mb_offset += sizeof(struct r5l_payload_flush) +
  1829. le32_to_cpu(payload_flush->size);
  1830. continue;
  1831. }
  1832. /* DATA or PARITY payload */
  1833. stripe_sect = (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) ?
  1834. raid5_compute_sector(
  1835. conf, le64_to_cpu(payload->location), 0, &dd,
  1836. NULL)
  1837. : le64_to_cpu(payload->location);
  1838. sh = r5c_recovery_lookup_stripe(cached_stripe_list,
  1839. stripe_sect);
  1840. if (!sh) {
  1841. sh = r5c_recovery_alloc_stripe(conf, stripe_sect, 1);
  1842. /*
  1843. * cannot get stripe from raid5_get_active_stripe
  1844. * try replay some stripes
  1845. */
  1846. if (!sh) {
  1847. r5c_recovery_replay_stripes(
  1848. cached_stripe_list, ctx);
  1849. sh = r5c_recovery_alloc_stripe(
  1850. conf, stripe_sect, 1);
  1851. }
  1852. if (!sh) {
  1853. int new_size = conf->min_nr_stripes * 2;
  1854. pr_debug("md/raid:%s: Increasing stripe cache size to %d to recovery data on journal.\n",
  1855. mdname(mddev),
  1856. new_size);
  1857. ret = raid5_set_cache_size(mddev, new_size);
  1858. if (conf->min_nr_stripes <= new_size / 2) {
  1859. pr_err("md/raid:%s: Cannot increase cache size, ret=%d, new_size=%d, min_nr_stripes=%d, max_nr_stripes=%d\n",
  1860. mdname(mddev),
  1861. ret,
  1862. new_size,
  1863. conf->min_nr_stripes,
  1864. conf->max_nr_stripes);
  1865. return -ENOMEM;
  1866. }
  1867. sh = r5c_recovery_alloc_stripe(
  1868. conf, stripe_sect, 0);
  1869. }
  1870. if (!sh) {
  1871. pr_err("md/raid:%s: Cannot get enough stripes due to memory pressure. Recovery failed.\n",
  1872. mdname(mddev));
  1873. return -ENOMEM;
  1874. }
  1875. list_add_tail(&sh->lru, cached_stripe_list);
  1876. }
  1877. if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_DATA) {
  1878. if (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
  1879. test_bit(R5_Wantwrite, &sh->dev[sh->pd_idx].flags)) {
  1880. r5l_recovery_replay_one_stripe(conf, sh, ctx);
  1881. list_move_tail(&sh->lru, cached_stripe_list);
  1882. }
  1883. r5l_recovery_load_data(log, sh, ctx, payload,
  1884. log_offset);
  1885. } else if (le16_to_cpu(payload->header.type) == R5LOG_PAYLOAD_PARITY)
  1886. r5l_recovery_load_parity(log, sh, ctx, payload,
  1887. log_offset);
  1888. else
  1889. return -EINVAL;
  1890. log_offset = r5l_ring_add(log, log_offset,
  1891. le32_to_cpu(payload->size));
  1892. mb_offset += sizeof(struct r5l_payload_data_parity) +
  1893. sizeof(__le32) *
  1894. (le32_to_cpu(payload->size) >> (PAGE_SHIFT - 9));
  1895. }
  1896. return 0;
  1897. }
  1898. /*
  1899. * Load the stripe into cache. The stripe will be written out later by
  1900. * the stripe cache state machine.
  1901. */
  1902. static void r5c_recovery_load_one_stripe(struct r5l_log *log,
  1903. struct stripe_head *sh)
  1904. {
  1905. struct r5dev *dev;
  1906. int i;
  1907. for (i = sh->disks; i--; ) {
  1908. dev = sh->dev + i;
  1909. if (test_and_clear_bit(R5_Wantwrite, &dev->flags)) {
  1910. set_bit(R5_InJournal, &dev->flags);
  1911. set_bit(R5_UPTODATE, &dev->flags);
  1912. }
  1913. }
  1914. }
  1915. /*
  1916. * Scan through the log for all to-be-flushed data
  1917. *
  1918. * For stripes with data and parity, namely Data-Parity stripe
  1919. * (STRIPE_R5C_CACHING == 0), we simply replay all the writes.
  1920. *
  1921. * For stripes with only data, namely Data-Only stripe
  1922. * (STRIPE_R5C_CACHING == 1), we load them to stripe cache state machine.
  1923. *
  1924. * For a stripe, if we see data after parity, we should discard all previous
  1925. * data and parity for this stripe, as these data are already flushed to
  1926. * the array.
  1927. *
  1928. * At the end of the scan, we return the new journal_tail, which points to
  1929. * first data-only stripe on the journal device, or next invalid meta block.
  1930. */
  1931. static int r5c_recovery_flush_log(struct r5l_log *log,
  1932. struct r5l_recovery_ctx *ctx)
  1933. {
  1934. struct stripe_head *sh;
  1935. int ret = 0;
  1936. /* scan through the log */
  1937. while (1) {
  1938. if (r5l_recovery_read_meta_block(log, ctx))
  1939. break;
  1940. ret = r5c_recovery_analyze_meta_block(log, ctx,
  1941. &ctx->cached_list);
  1942. /*
  1943. * -EAGAIN means mismatch in data block, in this case, we still
  1944. * try scan the next metablock
  1945. */
  1946. if (ret && ret != -EAGAIN)
  1947. break; /* ret == -EINVAL or -ENOMEM */
  1948. ctx->seq++;
  1949. ctx->pos = r5l_ring_add(log, ctx->pos, ctx->meta_total_blocks);
  1950. }
  1951. if (ret == -ENOMEM) {
  1952. r5c_recovery_drop_stripes(&ctx->cached_list, ctx);
  1953. return ret;
  1954. }
  1955. /* replay data-parity stripes */
  1956. r5c_recovery_replay_stripes(&ctx->cached_list, ctx);
  1957. /* load data-only stripes to stripe cache */
  1958. list_for_each_entry(sh, &ctx->cached_list, lru) {
  1959. WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
  1960. r5c_recovery_load_one_stripe(log, sh);
  1961. ctx->data_only_stripes++;
  1962. }
  1963. return 0;
  1964. }
  1965. /*
  1966. * we did a recovery. Now ctx.pos points to an invalid meta block. New
  1967. * log will start here. but we can't let superblock point to last valid
  1968. * meta block. The log might looks like:
  1969. * | meta 1| meta 2| meta 3|
  1970. * meta 1 is valid, meta 2 is invalid. meta 3 could be valid. If
  1971. * superblock points to meta 1, we write a new valid meta 2n. if crash
  1972. * happens again, new recovery will start from meta 1. Since meta 2n is
  1973. * valid now, recovery will think meta 3 is valid, which is wrong.
  1974. * The solution is we create a new meta in meta2 with its seq == meta
  1975. * 1's seq + 10000 and let superblock points to meta2. The same recovery
  1976. * will not think meta 3 is a valid meta, because its seq doesn't match
  1977. */
  1978. /*
  1979. * Before recovery, the log looks like the following
  1980. *
  1981. * ---------------------------------------------
  1982. * | valid log | invalid log |
  1983. * ---------------------------------------------
  1984. * ^
  1985. * |- log->last_checkpoint
  1986. * |- log->last_cp_seq
  1987. *
  1988. * Now we scan through the log until we see invalid entry
  1989. *
  1990. * ---------------------------------------------
  1991. * | valid log | invalid log |
  1992. * ---------------------------------------------
  1993. * ^ ^
  1994. * |- log->last_checkpoint |- ctx->pos
  1995. * |- log->last_cp_seq |- ctx->seq
  1996. *
  1997. * From this point, we need to increase seq number by 10 to avoid
  1998. * confusing next recovery.
  1999. *
  2000. * ---------------------------------------------
  2001. * | valid log | invalid log |
  2002. * ---------------------------------------------
  2003. * ^ ^
  2004. * |- log->last_checkpoint |- ctx->pos+1
  2005. * |- log->last_cp_seq |- ctx->seq+10001
  2006. *
  2007. * However, it is not safe to start the state machine yet, because data only
  2008. * parities are not yet secured in RAID. To save these data only parities, we
  2009. * rewrite them from seq+11.
  2010. *
  2011. * -----------------------------------------------------------------
  2012. * | valid log | data only stripes | invalid log |
  2013. * -----------------------------------------------------------------
  2014. * ^ ^
  2015. * |- log->last_checkpoint |- ctx->pos+n
  2016. * |- log->last_cp_seq |- ctx->seq+10000+n
  2017. *
  2018. * If failure happens again during this process, the recovery can safe start
  2019. * again from log->last_checkpoint.
  2020. *
  2021. * Once data only stripes are rewritten to journal, we move log_tail
  2022. *
  2023. * -----------------------------------------------------------------
  2024. * | old log | data only stripes | invalid log |
  2025. * -----------------------------------------------------------------
  2026. * ^ ^
  2027. * |- log->last_checkpoint |- ctx->pos+n
  2028. * |- log->last_cp_seq |- ctx->seq+10000+n
  2029. *
  2030. * Then we can safely start the state machine. If failure happens from this
  2031. * point on, the recovery will start from new log->last_checkpoint.
  2032. */
  2033. static int
  2034. r5c_recovery_rewrite_data_only_stripes(struct r5l_log *log,
  2035. struct r5l_recovery_ctx *ctx)
  2036. {
  2037. struct stripe_head *sh;
  2038. struct mddev *mddev = log->rdev->mddev;
  2039. struct page *page;
  2040. sector_t next_checkpoint = MaxSector;
  2041. page = alloc_page(GFP_KERNEL);
  2042. if (!page) {
  2043. pr_err("md/raid:%s: cannot allocate memory to rewrite data only stripes\n",
  2044. mdname(mddev));
  2045. return -ENOMEM;
  2046. }
  2047. WARN_ON(list_empty(&ctx->cached_list));
  2048. list_for_each_entry(sh, &ctx->cached_list, lru) {
  2049. struct r5l_meta_block *mb;
  2050. int i;
  2051. int offset;
  2052. sector_t write_pos;
  2053. WARN_ON(!test_bit(STRIPE_R5C_CACHING, &sh->state));
  2054. r5l_recovery_create_empty_meta_block(log, page,
  2055. ctx->pos, ctx->seq);
  2056. mb = page_address(page);
  2057. offset = le32_to_cpu(mb->meta_size);
  2058. write_pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
  2059. for (i = sh->disks; i--; ) {
  2060. struct r5dev *dev = &sh->dev[i];
  2061. struct r5l_payload_data_parity *payload;
  2062. void *addr;
  2063. if (test_bit(R5_InJournal, &dev->flags)) {
  2064. payload = (void *)mb + offset;
  2065. payload->header.type = cpu_to_le16(
  2066. R5LOG_PAYLOAD_DATA);
  2067. payload->size = cpu_to_le32(BLOCK_SECTORS);
  2068. payload->location = cpu_to_le64(
  2069. raid5_compute_blocknr(sh, i, 0));
  2070. addr = kmap_atomic(dev->page);
  2071. payload->checksum[0] = cpu_to_le32(
  2072. crc32c_le(log->uuid_checksum, addr,
  2073. PAGE_SIZE));
  2074. kunmap_atomic(addr);
  2075. sync_page_io(log->rdev, write_pos, PAGE_SIZE,
  2076. dev->page, REQ_OP_WRITE, 0, false);
  2077. write_pos = r5l_ring_add(log, write_pos,
  2078. BLOCK_SECTORS);
  2079. offset += sizeof(__le32) +
  2080. sizeof(struct r5l_payload_data_parity);
  2081. }
  2082. }
  2083. mb->meta_size = cpu_to_le32(offset);
  2084. mb->checksum = cpu_to_le32(crc32c_le(log->uuid_checksum,
  2085. mb, PAGE_SIZE));
  2086. sync_page_io(log->rdev, ctx->pos, PAGE_SIZE, page,
  2087. REQ_OP_WRITE, REQ_SYNC | REQ_FUA, false);
  2088. sh->log_start = ctx->pos;
  2089. list_add_tail(&sh->r5c, &log->stripe_in_journal_list);
  2090. atomic_inc(&log->stripe_in_journal_count);
  2091. ctx->pos = write_pos;
  2092. ctx->seq += 1;
  2093. next_checkpoint = sh->log_start;
  2094. }
  2095. log->next_checkpoint = next_checkpoint;
  2096. __free_page(page);
  2097. return 0;
  2098. }
  2099. static void r5c_recovery_flush_data_only_stripes(struct r5l_log *log,
  2100. struct r5l_recovery_ctx *ctx)
  2101. {
  2102. struct mddev *mddev = log->rdev->mddev;
  2103. struct r5conf *conf = mddev->private;
  2104. struct stripe_head *sh, *next;
  2105. bool cleared_pending = false;
  2106. if (ctx->data_only_stripes == 0)
  2107. return;
  2108. if (test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags)) {
  2109. cleared_pending = true;
  2110. clear_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
  2111. }
  2112. log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_BACK;
  2113. list_for_each_entry_safe(sh, next, &ctx->cached_list, lru) {
  2114. r5c_make_stripe_write_out(sh);
  2115. set_bit(STRIPE_HANDLE, &sh->state);
  2116. list_del_init(&sh->lru);
  2117. raid5_release_stripe(sh);
  2118. }
  2119. /* reuse conf->wait_for_quiescent in recovery */
  2120. wait_event(conf->wait_for_quiescent,
  2121. atomic_read(&conf->active_stripes) == 0);
  2122. log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
  2123. if (cleared_pending)
  2124. set_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags);
  2125. }
  2126. static int r5l_recovery_log(struct r5l_log *log)
  2127. {
  2128. struct mddev *mddev = log->rdev->mddev;
  2129. struct r5l_recovery_ctx *ctx;
  2130. int ret;
  2131. sector_t pos;
  2132. ctx = kzalloc(sizeof(*ctx), GFP_KERNEL);
  2133. if (!ctx)
  2134. return -ENOMEM;
  2135. ctx->pos = log->last_checkpoint;
  2136. ctx->seq = log->last_cp_seq;
  2137. INIT_LIST_HEAD(&ctx->cached_list);
  2138. ctx->meta_page = alloc_page(GFP_KERNEL);
  2139. if (!ctx->meta_page) {
  2140. ret = -ENOMEM;
  2141. goto meta_page;
  2142. }
  2143. if (r5l_recovery_allocate_ra_pool(log, ctx) != 0) {
  2144. ret = -ENOMEM;
  2145. goto ra_pool;
  2146. }
  2147. ret = r5c_recovery_flush_log(log, ctx);
  2148. if (ret)
  2149. goto error;
  2150. pos = ctx->pos;
  2151. ctx->seq += 10000;
  2152. if ((ctx->data_only_stripes == 0) && (ctx->data_parity_stripes == 0))
  2153. pr_info("md/raid:%s: starting from clean shutdown\n",
  2154. mdname(mddev));
  2155. else
  2156. pr_info("md/raid:%s: recovering %d data-only stripes and %d data-parity stripes\n",
  2157. mdname(mddev), ctx->data_only_stripes,
  2158. ctx->data_parity_stripes);
  2159. if (ctx->data_only_stripes == 0) {
  2160. log->next_checkpoint = ctx->pos;
  2161. r5l_log_write_empty_meta_block(log, ctx->pos, ctx->seq++);
  2162. ctx->pos = r5l_ring_add(log, ctx->pos, BLOCK_SECTORS);
  2163. } else if (r5c_recovery_rewrite_data_only_stripes(log, ctx)) {
  2164. pr_err("md/raid:%s: failed to rewrite stripes to journal\n",
  2165. mdname(mddev));
  2166. ret = -EIO;
  2167. goto error;
  2168. }
  2169. log->log_start = ctx->pos;
  2170. log->seq = ctx->seq;
  2171. log->last_checkpoint = pos;
  2172. r5l_write_super(log, pos);
  2173. r5c_recovery_flush_data_only_stripes(log, ctx);
  2174. ret = 0;
  2175. error:
  2176. r5l_recovery_free_ra_pool(log, ctx);
  2177. ra_pool:
  2178. __free_page(ctx->meta_page);
  2179. meta_page:
  2180. kfree(ctx);
  2181. return ret;
  2182. }
  2183. static void r5l_write_super(struct r5l_log *log, sector_t cp)
  2184. {
  2185. struct mddev *mddev = log->rdev->mddev;
  2186. log->rdev->journal_tail = cp;
  2187. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  2188. }
  2189. static ssize_t r5c_journal_mode_show(struct mddev *mddev, char *page)
  2190. {
  2191. struct r5conf *conf;
  2192. int ret;
  2193. spin_lock(&mddev->lock);
  2194. conf = mddev->private;
  2195. if (!conf || !conf->log) {
  2196. spin_unlock(&mddev->lock);
  2197. return 0;
  2198. }
  2199. switch (conf->log->r5c_journal_mode) {
  2200. case R5C_JOURNAL_MODE_WRITE_THROUGH:
  2201. ret = snprintf(
  2202. page, PAGE_SIZE, "[%s] %s\n",
  2203. r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
  2204. r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
  2205. break;
  2206. case R5C_JOURNAL_MODE_WRITE_BACK:
  2207. ret = snprintf(
  2208. page, PAGE_SIZE, "%s [%s]\n",
  2209. r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_THROUGH],
  2210. r5c_journal_mode_str[R5C_JOURNAL_MODE_WRITE_BACK]);
  2211. break;
  2212. default:
  2213. ret = 0;
  2214. }
  2215. spin_unlock(&mddev->lock);
  2216. return ret;
  2217. }
  2218. /*
  2219. * Set journal cache mode on @mddev (external API initially needed by dm-raid).
  2220. *
  2221. * @mode as defined in 'enum r5c_journal_mode'.
  2222. *
  2223. */
  2224. int r5c_journal_mode_set(struct mddev *mddev, int mode)
  2225. {
  2226. struct r5conf *conf;
  2227. if (mode < R5C_JOURNAL_MODE_WRITE_THROUGH ||
  2228. mode > R5C_JOURNAL_MODE_WRITE_BACK)
  2229. return -EINVAL;
  2230. conf = mddev->private;
  2231. if (!conf || !conf->log)
  2232. return -ENODEV;
  2233. if (raid5_calc_degraded(conf) > 0 &&
  2234. mode == R5C_JOURNAL_MODE_WRITE_BACK)
  2235. return -EINVAL;
  2236. mddev_suspend(mddev);
  2237. conf->log->r5c_journal_mode = mode;
  2238. mddev_resume(mddev);
  2239. pr_debug("md/raid:%s: setting r5c cache mode to %d: %s\n",
  2240. mdname(mddev), mode, r5c_journal_mode_str[mode]);
  2241. return 0;
  2242. }
  2243. EXPORT_SYMBOL(r5c_journal_mode_set);
  2244. static ssize_t r5c_journal_mode_store(struct mddev *mddev,
  2245. const char *page, size_t length)
  2246. {
  2247. int mode = ARRAY_SIZE(r5c_journal_mode_str);
  2248. size_t len = length;
  2249. int ret;
  2250. if (len < 2)
  2251. return -EINVAL;
  2252. if (page[len - 1] == '\n')
  2253. len--;
  2254. while (mode--)
  2255. if (strlen(r5c_journal_mode_str[mode]) == len &&
  2256. !strncmp(page, r5c_journal_mode_str[mode], len))
  2257. break;
  2258. ret = mddev_lock(mddev);
  2259. if (ret)
  2260. return ret;
  2261. ret = r5c_journal_mode_set(mddev, mode);
  2262. mddev_unlock(mddev);
  2263. return ret ?: length;
  2264. }
  2265. struct md_sysfs_entry
  2266. r5c_journal_mode = __ATTR(journal_mode, 0644,
  2267. r5c_journal_mode_show, r5c_journal_mode_store);
  2268. /*
  2269. * Try handle write operation in caching phase. This function should only
  2270. * be called in write-back mode.
  2271. *
  2272. * If all outstanding writes can be handled in caching phase, returns 0
  2273. * If writes requires write-out phase, call r5c_make_stripe_write_out()
  2274. * and returns -EAGAIN
  2275. */
  2276. int r5c_try_caching_write(struct r5conf *conf,
  2277. struct stripe_head *sh,
  2278. struct stripe_head_state *s,
  2279. int disks)
  2280. {
  2281. struct r5l_log *log = conf->log;
  2282. int i;
  2283. struct r5dev *dev;
  2284. int to_cache = 0;
  2285. void **pslot;
  2286. sector_t tree_index;
  2287. int ret;
  2288. uintptr_t refcount;
  2289. BUG_ON(!r5c_is_writeback(log));
  2290. if (!test_bit(STRIPE_R5C_CACHING, &sh->state)) {
  2291. /*
  2292. * There are two different scenarios here:
  2293. * 1. The stripe has some data cached, and it is sent to
  2294. * write-out phase for reclaim
  2295. * 2. The stripe is clean, and this is the first write
  2296. *
  2297. * For 1, return -EAGAIN, so we continue with
  2298. * handle_stripe_dirtying().
  2299. *
  2300. * For 2, set STRIPE_R5C_CACHING and continue with caching
  2301. * write.
  2302. */
  2303. /* case 1: anything injournal or anything in written */
  2304. if (s->injournal > 0 || s->written > 0)
  2305. return -EAGAIN;
  2306. /* case 2 */
  2307. set_bit(STRIPE_R5C_CACHING, &sh->state);
  2308. }
  2309. /*
  2310. * When run in degraded mode, array is set to write-through mode.
  2311. * This check helps drain pending write safely in the transition to
  2312. * write-through mode.
  2313. *
  2314. * When a stripe is syncing, the write is also handled in write
  2315. * through mode.
  2316. */
  2317. if (s->failed || test_bit(STRIPE_SYNCING, &sh->state)) {
  2318. r5c_make_stripe_write_out(sh);
  2319. return -EAGAIN;
  2320. }
  2321. for (i = disks; i--; ) {
  2322. dev = &sh->dev[i];
  2323. /* if non-overwrite, use writing-out phase */
  2324. if (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags) &&
  2325. !test_bit(R5_InJournal, &dev->flags)) {
  2326. r5c_make_stripe_write_out(sh);
  2327. return -EAGAIN;
  2328. }
  2329. }
  2330. /* if the stripe is not counted in big_stripe_tree, add it now */
  2331. if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
  2332. !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
  2333. tree_index = r5c_tree_index(conf, sh->sector);
  2334. spin_lock(&log->tree_lock);
  2335. pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
  2336. tree_index);
  2337. if (pslot) {
  2338. refcount = (uintptr_t)radix_tree_deref_slot_protected(
  2339. pslot, &log->tree_lock) >>
  2340. R5C_RADIX_COUNT_SHIFT;
  2341. radix_tree_replace_slot(
  2342. &log->big_stripe_tree, pslot,
  2343. (void *)((refcount + 1) << R5C_RADIX_COUNT_SHIFT));
  2344. } else {
  2345. /*
  2346. * this radix_tree_insert can fail safely, so no
  2347. * need to call radix_tree_preload()
  2348. */
  2349. ret = radix_tree_insert(
  2350. &log->big_stripe_tree, tree_index,
  2351. (void *)(1 << R5C_RADIX_COUNT_SHIFT));
  2352. if (ret) {
  2353. spin_unlock(&log->tree_lock);
  2354. r5c_make_stripe_write_out(sh);
  2355. return -EAGAIN;
  2356. }
  2357. }
  2358. spin_unlock(&log->tree_lock);
  2359. /*
  2360. * set STRIPE_R5C_PARTIAL_STRIPE, this shows the stripe is
  2361. * counted in the radix tree
  2362. */
  2363. set_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state);
  2364. atomic_inc(&conf->r5c_cached_partial_stripes);
  2365. }
  2366. for (i = disks; i--; ) {
  2367. dev = &sh->dev[i];
  2368. if (dev->towrite) {
  2369. set_bit(R5_Wantwrite, &dev->flags);
  2370. set_bit(R5_Wantdrain, &dev->flags);
  2371. set_bit(R5_LOCKED, &dev->flags);
  2372. to_cache++;
  2373. }
  2374. }
  2375. if (to_cache) {
  2376. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  2377. /*
  2378. * set STRIPE_LOG_TRAPPED, which triggers r5c_cache_data()
  2379. * in ops_run_io(). STRIPE_LOG_TRAPPED will be cleared in
  2380. * r5c_handle_data_cached()
  2381. */
  2382. set_bit(STRIPE_LOG_TRAPPED, &sh->state);
  2383. }
  2384. return 0;
  2385. }
  2386. /*
  2387. * free extra pages (orig_page) we allocated for prexor
  2388. */
  2389. void r5c_release_extra_page(struct stripe_head *sh)
  2390. {
  2391. struct r5conf *conf = sh->raid_conf;
  2392. int i;
  2393. bool using_disk_info_extra_page;
  2394. using_disk_info_extra_page =
  2395. sh->dev[0].orig_page == conf->disks[0].extra_page;
  2396. for (i = sh->disks; i--; )
  2397. if (sh->dev[i].page != sh->dev[i].orig_page) {
  2398. struct page *p = sh->dev[i].orig_page;
  2399. sh->dev[i].orig_page = sh->dev[i].page;
  2400. clear_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
  2401. if (!using_disk_info_extra_page)
  2402. put_page(p);
  2403. }
  2404. if (using_disk_info_extra_page) {
  2405. clear_bit(R5C_EXTRA_PAGE_IN_USE, &conf->cache_state);
  2406. md_wakeup_thread(conf->mddev->thread);
  2407. }
  2408. }
  2409. void r5c_use_extra_page(struct stripe_head *sh)
  2410. {
  2411. struct r5conf *conf = sh->raid_conf;
  2412. int i;
  2413. struct r5dev *dev;
  2414. for (i = sh->disks; i--; ) {
  2415. dev = &sh->dev[i];
  2416. if (dev->orig_page != dev->page)
  2417. put_page(dev->orig_page);
  2418. dev->orig_page = conf->disks[i].extra_page;
  2419. }
  2420. }
  2421. /*
  2422. * clean up the stripe (clear R5_InJournal for dev[pd_idx] etc.) after the
  2423. * stripe is committed to RAID disks.
  2424. */
  2425. void r5c_finish_stripe_write_out(struct r5conf *conf,
  2426. struct stripe_head *sh,
  2427. struct stripe_head_state *s)
  2428. {
  2429. struct r5l_log *log = conf->log;
  2430. int i;
  2431. int do_wakeup = 0;
  2432. sector_t tree_index;
  2433. void **pslot;
  2434. uintptr_t refcount;
  2435. if (!log || !test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags))
  2436. return;
  2437. WARN_ON(test_bit(STRIPE_R5C_CACHING, &sh->state));
  2438. clear_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags);
  2439. if (log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_THROUGH)
  2440. return;
  2441. for (i = sh->disks; i--; ) {
  2442. clear_bit(R5_InJournal, &sh->dev[i].flags);
  2443. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  2444. do_wakeup = 1;
  2445. }
  2446. /*
  2447. * analyse_stripe() runs before r5c_finish_stripe_write_out(),
  2448. * We updated R5_InJournal, so we also update s->injournal.
  2449. */
  2450. s->injournal = 0;
  2451. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  2452. if (atomic_dec_and_test(&conf->pending_full_writes))
  2453. md_wakeup_thread(conf->mddev->thread);
  2454. if (do_wakeup)
  2455. wake_up(&conf->wait_for_overlap);
  2456. spin_lock_irq(&log->stripe_in_journal_lock);
  2457. list_del_init(&sh->r5c);
  2458. spin_unlock_irq(&log->stripe_in_journal_lock);
  2459. sh->log_start = MaxSector;
  2460. atomic_dec(&log->stripe_in_journal_count);
  2461. r5c_update_log_state(log);
  2462. /* stop counting this stripe in big_stripe_tree */
  2463. if (test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) ||
  2464. test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
  2465. tree_index = r5c_tree_index(conf, sh->sector);
  2466. spin_lock(&log->tree_lock);
  2467. pslot = radix_tree_lookup_slot(&log->big_stripe_tree,
  2468. tree_index);
  2469. BUG_ON(pslot == NULL);
  2470. refcount = (uintptr_t)radix_tree_deref_slot_protected(
  2471. pslot, &log->tree_lock) >>
  2472. R5C_RADIX_COUNT_SHIFT;
  2473. if (refcount == 1)
  2474. radix_tree_delete(&log->big_stripe_tree, tree_index);
  2475. else
  2476. radix_tree_replace_slot(
  2477. &log->big_stripe_tree, pslot,
  2478. (void *)((refcount - 1) << R5C_RADIX_COUNT_SHIFT));
  2479. spin_unlock(&log->tree_lock);
  2480. }
  2481. if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) {
  2482. BUG_ON(atomic_read(&conf->r5c_cached_partial_stripes) == 0);
  2483. atomic_dec(&conf->r5c_flushing_partial_stripes);
  2484. atomic_dec(&conf->r5c_cached_partial_stripes);
  2485. }
  2486. if (test_and_clear_bit(STRIPE_R5C_FULL_STRIPE, &sh->state)) {
  2487. BUG_ON(atomic_read(&conf->r5c_cached_full_stripes) == 0);
  2488. atomic_dec(&conf->r5c_flushing_full_stripes);
  2489. atomic_dec(&conf->r5c_cached_full_stripes);
  2490. }
  2491. r5l_append_flush_payload(log, sh->sector);
  2492. /* stripe is flused to raid disks, we can do resync now */
  2493. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
  2494. set_bit(STRIPE_HANDLE, &sh->state);
  2495. }
  2496. int r5c_cache_data(struct r5l_log *log, struct stripe_head *sh)
  2497. {
  2498. struct r5conf *conf = sh->raid_conf;
  2499. int pages = 0;
  2500. int reserve;
  2501. int i;
  2502. int ret = 0;
  2503. BUG_ON(!log);
  2504. for (i = 0; i < sh->disks; i++) {
  2505. void *addr;
  2506. if (!test_bit(R5_Wantwrite, &sh->dev[i].flags))
  2507. continue;
  2508. addr = kmap_atomic(sh->dev[i].page);
  2509. sh->dev[i].log_checksum = crc32c_le(log->uuid_checksum,
  2510. addr, PAGE_SIZE);
  2511. kunmap_atomic(addr);
  2512. pages++;
  2513. }
  2514. WARN_ON(pages == 0);
  2515. /*
  2516. * The stripe must enter state machine again to call endio, so
  2517. * don't delay.
  2518. */
  2519. clear_bit(STRIPE_DELAYED, &sh->state);
  2520. atomic_inc(&sh->count);
  2521. mutex_lock(&log->io_mutex);
  2522. /* meta + data */
  2523. reserve = (1 + pages) << (PAGE_SHIFT - 9);
  2524. if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
  2525. sh->log_start == MaxSector)
  2526. r5l_add_no_space_stripe(log, sh);
  2527. else if (!r5l_has_free_space(log, reserve)) {
  2528. if (sh->log_start == log->last_checkpoint)
  2529. BUG();
  2530. else
  2531. r5l_add_no_space_stripe(log, sh);
  2532. } else {
  2533. ret = r5l_log_stripe(log, sh, pages, 0);
  2534. if (ret) {
  2535. spin_lock_irq(&log->io_list_lock);
  2536. list_add_tail(&sh->log_list, &log->no_mem_stripes);
  2537. spin_unlock_irq(&log->io_list_lock);
  2538. }
  2539. }
  2540. mutex_unlock(&log->io_mutex);
  2541. return 0;
  2542. }
  2543. /* check whether this big stripe is in write back cache. */
  2544. bool r5c_big_stripe_cached(struct r5conf *conf, sector_t sect)
  2545. {
  2546. struct r5l_log *log = conf->log;
  2547. sector_t tree_index;
  2548. void *slot;
  2549. if (!log)
  2550. return false;
  2551. WARN_ON_ONCE(!rcu_read_lock_held());
  2552. tree_index = r5c_tree_index(conf, sect);
  2553. slot = radix_tree_lookup(&log->big_stripe_tree, tree_index);
  2554. return slot != NULL;
  2555. }
  2556. static int r5l_load_log(struct r5l_log *log)
  2557. {
  2558. struct md_rdev *rdev = log->rdev;
  2559. struct page *page;
  2560. struct r5l_meta_block *mb;
  2561. sector_t cp = log->rdev->journal_tail;
  2562. u32 stored_crc, expected_crc;
  2563. bool create_super = false;
  2564. int ret = 0;
  2565. /* Make sure it's valid */
  2566. if (cp >= rdev->sectors || round_down(cp, BLOCK_SECTORS) != cp)
  2567. cp = 0;
  2568. page = alloc_page(GFP_KERNEL);
  2569. if (!page)
  2570. return -ENOMEM;
  2571. if (!sync_page_io(rdev, cp, PAGE_SIZE, page, REQ_OP_READ, 0, false)) {
  2572. ret = -EIO;
  2573. goto ioerr;
  2574. }
  2575. mb = page_address(page);
  2576. if (le32_to_cpu(mb->magic) != R5LOG_MAGIC ||
  2577. mb->version != R5LOG_VERSION) {
  2578. create_super = true;
  2579. goto create;
  2580. }
  2581. stored_crc = le32_to_cpu(mb->checksum);
  2582. mb->checksum = 0;
  2583. expected_crc = crc32c_le(log->uuid_checksum, mb, PAGE_SIZE);
  2584. if (stored_crc != expected_crc) {
  2585. create_super = true;
  2586. goto create;
  2587. }
  2588. if (le64_to_cpu(mb->position) != cp) {
  2589. create_super = true;
  2590. goto create;
  2591. }
  2592. create:
  2593. if (create_super) {
  2594. log->last_cp_seq = prandom_u32();
  2595. cp = 0;
  2596. r5l_log_write_empty_meta_block(log, cp, log->last_cp_seq);
  2597. /*
  2598. * Make sure super points to correct address. Log might have
  2599. * data very soon. If super hasn't correct log tail address,
  2600. * recovery can't find the log
  2601. */
  2602. r5l_write_super(log, cp);
  2603. } else
  2604. log->last_cp_seq = le64_to_cpu(mb->seq);
  2605. log->device_size = round_down(rdev->sectors, BLOCK_SECTORS);
  2606. log->max_free_space = log->device_size >> RECLAIM_MAX_FREE_SPACE_SHIFT;
  2607. if (log->max_free_space > RECLAIM_MAX_FREE_SPACE)
  2608. log->max_free_space = RECLAIM_MAX_FREE_SPACE;
  2609. log->last_checkpoint = cp;
  2610. __free_page(page);
  2611. if (create_super) {
  2612. log->log_start = r5l_ring_add(log, cp, BLOCK_SECTORS);
  2613. log->seq = log->last_cp_seq + 1;
  2614. log->next_checkpoint = cp;
  2615. } else
  2616. ret = r5l_recovery_log(log);
  2617. r5c_update_log_state(log);
  2618. return ret;
  2619. ioerr:
  2620. __free_page(page);
  2621. return ret;
  2622. }
  2623. int r5l_start(struct r5l_log *log)
  2624. {
  2625. int ret;
  2626. if (!log)
  2627. return 0;
  2628. ret = r5l_load_log(log);
  2629. if (ret) {
  2630. struct mddev *mddev = log->rdev->mddev;
  2631. struct r5conf *conf = mddev->private;
  2632. r5l_exit_log(conf);
  2633. }
  2634. return ret;
  2635. }
  2636. void r5c_update_on_rdev_error(struct mddev *mddev, struct md_rdev *rdev)
  2637. {
  2638. struct r5conf *conf = mddev->private;
  2639. struct r5l_log *log = conf->log;
  2640. if (!log)
  2641. return;
  2642. if ((raid5_calc_degraded(conf) > 0 ||
  2643. test_bit(Journal, &rdev->flags)) &&
  2644. conf->log->r5c_journal_mode == R5C_JOURNAL_MODE_WRITE_BACK)
  2645. schedule_work(&log->disable_writeback_work);
  2646. }
  2647. int r5l_init_log(struct r5conf *conf, struct md_rdev *rdev)
  2648. {
  2649. struct request_queue *q = bdev_get_queue(rdev->bdev);
  2650. struct r5l_log *log;
  2651. char b[BDEVNAME_SIZE];
  2652. int ret;
  2653. pr_debug("md/raid:%s: using device %s as journal\n",
  2654. mdname(conf->mddev), bdevname(rdev->bdev, b));
  2655. if (PAGE_SIZE != 4096)
  2656. return -EINVAL;
  2657. /*
  2658. * The PAGE_SIZE must be big enough to hold 1 r5l_meta_block and
  2659. * raid_disks r5l_payload_data_parity.
  2660. *
  2661. * Write journal and cache does not work for very big array
  2662. * (raid_disks > 203)
  2663. */
  2664. if (sizeof(struct r5l_meta_block) +
  2665. ((sizeof(struct r5l_payload_data_parity) + sizeof(__le32)) *
  2666. conf->raid_disks) > PAGE_SIZE) {
  2667. pr_err("md/raid:%s: write journal/cache doesn't work for array with %d disks\n",
  2668. mdname(conf->mddev), conf->raid_disks);
  2669. return -EINVAL;
  2670. }
  2671. log = kzalloc(sizeof(*log), GFP_KERNEL);
  2672. if (!log)
  2673. return -ENOMEM;
  2674. log->rdev = rdev;
  2675. log->need_cache_flush = test_bit(QUEUE_FLAG_WC, &q->queue_flags) != 0;
  2676. log->uuid_checksum = crc32c_le(~0, rdev->mddev->uuid,
  2677. sizeof(rdev->mddev->uuid));
  2678. mutex_init(&log->io_mutex);
  2679. spin_lock_init(&log->io_list_lock);
  2680. INIT_LIST_HEAD(&log->running_ios);
  2681. INIT_LIST_HEAD(&log->io_end_ios);
  2682. INIT_LIST_HEAD(&log->flushing_ios);
  2683. INIT_LIST_HEAD(&log->finished_ios);
  2684. bio_init(&log->flush_bio, NULL, 0);
  2685. log->io_kc = KMEM_CACHE(r5l_io_unit, 0);
  2686. if (!log->io_kc)
  2687. goto io_kc;
  2688. ret = mempool_init_slab_pool(&log->io_pool, R5L_POOL_SIZE, log->io_kc);
  2689. if (ret)
  2690. goto io_pool;
  2691. ret = bioset_init(&log->bs, R5L_POOL_SIZE, 0, BIOSET_NEED_BVECS);
  2692. if (ret)
  2693. goto io_bs;
  2694. ret = mempool_init_page_pool(&log->meta_pool, R5L_POOL_SIZE, 0);
  2695. if (ret)
  2696. goto out_mempool;
  2697. spin_lock_init(&log->tree_lock);
  2698. INIT_RADIX_TREE(&log->big_stripe_tree, GFP_NOWAIT | __GFP_NOWARN);
  2699. log->reclaim_thread = md_register_thread(r5l_reclaim_thread,
  2700. log->rdev->mddev, "reclaim");
  2701. if (!log->reclaim_thread)
  2702. goto reclaim_thread;
  2703. log->reclaim_thread->timeout = R5C_RECLAIM_WAKEUP_INTERVAL;
  2704. init_waitqueue_head(&log->iounit_wait);
  2705. INIT_LIST_HEAD(&log->no_mem_stripes);
  2706. INIT_LIST_HEAD(&log->no_space_stripes);
  2707. spin_lock_init(&log->no_space_stripes_lock);
  2708. INIT_WORK(&log->deferred_io_work, r5l_submit_io_async);
  2709. INIT_WORK(&log->disable_writeback_work, r5c_disable_writeback_async);
  2710. log->r5c_journal_mode = R5C_JOURNAL_MODE_WRITE_THROUGH;
  2711. INIT_LIST_HEAD(&log->stripe_in_journal_list);
  2712. spin_lock_init(&log->stripe_in_journal_lock);
  2713. atomic_set(&log->stripe_in_journal_count, 0);
  2714. rcu_assign_pointer(conf->log, log);
  2715. set_bit(MD_HAS_JOURNAL, &conf->mddev->flags);
  2716. return 0;
  2717. reclaim_thread:
  2718. mempool_exit(&log->meta_pool);
  2719. out_mempool:
  2720. bioset_exit(&log->bs);
  2721. io_bs:
  2722. mempool_exit(&log->io_pool);
  2723. io_pool:
  2724. kmem_cache_destroy(log->io_kc);
  2725. io_kc:
  2726. kfree(log);
  2727. return -EINVAL;
  2728. }
  2729. void r5l_exit_log(struct r5conf *conf)
  2730. {
  2731. struct r5l_log *log = conf->log;
  2732. conf->log = NULL;
  2733. synchronize_rcu();
  2734. /* Ensure disable_writeback_work wakes up and exits */
  2735. wake_up(&conf->mddev->sb_wait);
  2736. flush_work(&log->disable_writeback_work);
  2737. md_unregister_thread(&log->reclaim_thread);
  2738. mempool_exit(&log->meta_pool);
  2739. bioset_exit(&log->bs);
  2740. mempool_exit(&log->io_pool);
  2741. kmem_cache_destroy(log->io_kc);
  2742. kfree(log);
  2743. }