raid5.c 247 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542354335443545354635473548354935503551355235533554355535563557355835593560356135623563356435653566356735683569357035713572357335743575357635773578357935803581358235833584358535863587358835893590359135923593359435953596359735983599360036013602360336043605360636073608360936103611361236133614361536163617361836193620362136223623362436253626362736283629363036313632363336343635363636373638363936403641364236433644364536463647364836493650365136523653365436553656365736583659366036613662366336643665366636673668366936703671367236733674367536763677367836793680368136823683368436853686368736883689369036913692369336943695369636973698369937003701370237033704370537063707370837093710371137123713371437153716371737183719372037213722372337243725372637273728372937303731373237333734373537363737373837393740374137423743374437453746374737483749375037513752375337543755375637573758375937603761376237633764376537663767376837693770377137723773377437753776377737783779378037813782378337843785378637873788378937903791379237933794379537963797379837993800380138023803380438053806380738083809381038113812381338143815381638173818381938203821382238233824382538263827382838293830383138323833383438353836383738383839384038413842384338443845384638473848384938503851385238533854385538563857385838593860386138623863386438653866386738683869387038713872387338743875387638773878387938803881388238833884388538863887388838893890389138923893389438953896389738983899390039013902390339043905390639073908390939103911391239133914391539163917391839193920392139223923392439253926392739283929393039313932393339343935393639373938393939403941394239433944394539463947394839493950395139523953395439553956395739583959396039613962396339643965396639673968396939703971397239733974397539763977397839793980398139823983398439853986398739883989399039913992399339943995399639973998399940004001400240034004400540064007400840094010401140124013401440154016401740184019402040214022402340244025402640274028402940304031403240334034403540364037403840394040404140424043404440454046404740484049405040514052405340544055405640574058405940604061406240634064406540664067406840694070407140724073407440754076407740784079408040814082408340844085408640874088408940904091409240934094409540964097409840994100410141024103410441054106410741084109411041114112411341144115411641174118411941204121412241234124412541264127412841294130413141324133413441354136413741384139414041414142414341444145414641474148414941504151415241534154415541564157415841594160416141624163416441654166416741684169417041714172417341744175417641774178417941804181418241834184418541864187418841894190419141924193419441954196419741984199420042014202420342044205420642074208420942104211421242134214421542164217421842194220422142224223422442254226422742284229423042314232423342344235423642374238423942404241424242434244424542464247424842494250425142524253425442554256425742584259426042614262426342644265426642674268426942704271427242734274427542764277427842794280428142824283428442854286428742884289429042914292429342944295429642974298429943004301430243034304430543064307430843094310431143124313431443154316431743184319432043214322432343244325432643274328432943304331433243334334433543364337433843394340434143424343434443454346434743484349435043514352435343544355435643574358435943604361436243634364436543664367436843694370437143724373437443754376437743784379438043814382438343844385438643874388438943904391439243934394439543964397439843994400440144024403440444054406440744084409441044114412441344144415441644174418441944204421442244234424442544264427442844294430443144324433443444354436443744384439444044414442444344444445444644474448444944504451445244534454445544564457445844594460446144624463446444654466446744684469447044714472447344744475447644774478447944804481448244834484448544864487448844894490449144924493449444954496449744984499450045014502450345044505450645074508450945104511451245134514451545164517451845194520452145224523452445254526452745284529453045314532453345344535453645374538453945404541454245434544454545464547454845494550455145524553455445554556455745584559456045614562456345644565456645674568456945704571457245734574457545764577457845794580458145824583458445854586458745884589459045914592459345944595459645974598459946004601460246034604460546064607460846094610461146124613461446154616461746184619462046214622462346244625462646274628462946304631463246334634463546364637463846394640464146424643464446454646464746484649465046514652465346544655465646574658465946604661466246634664466546664667466846694670467146724673467446754676467746784679468046814682468346844685468646874688468946904691469246934694469546964697469846994700470147024703470447054706470747084709471047114712471347144715471647174718471947204721472247234724472547264727472847294730473147324733473447354736473747384739474047414742474347444745474647474748474947504751475247534754475547564757475847594760476147624763476447654766476747684769477047714772477347744775477647774778477947804781478247834784478547864787478847894790479147924793479447954796479747984799480048014802480348044805480648074808480948104811481248134814481548164817481848194820482148224823482448254826482748284829483048314832483348344835483648374838483948404841484248434844484548464847484848494850485148524853485448554856485748584859486048614862486348644865486648674868486948704871487248734874487548764877487848794880488148824883488448854886488748884889489048914892489348944895489648974898489949004901490249034904490549064907490849094910491149124913491449154916491749184919492049214922492349244925492649274928492949304931493249334934493549364937493849394940494149424943494449454946494749484949495049514952495349544955495649574958495949604961496249634964496549664967496849694970497149724973497449754976497749784979498049814982498349844985498649874988498949904991499249934994499549964997499849995000500150025003500450055006500750085009501050115012501350145015501650175018501950205021502250235024502550265027502850295030503150325033503450355036503750385039504050415042504350445045504650475048504950505051505250535054505550565057505850595060506150625063506450655066506750685069507050715072507350745075507650775078507950805081508250835084508550865087508850895090509150925093509450955096509750985099510051015102510351045105510651075108510951105111511251135114511551165117511851195120512151225123512451255126512751285129513051315132513351345135513651375138513951405141514251435144514551465147514851495150515151525153515451555156515751585159516051615162516351645165516651675168516951705171517251735174517551765177517851795180518151825183518451855186518751885189519051915192519351945195519651975198519952005201520252035204520552065207520852095210521152125213521452155216521752185219522052215222522352245225522652275228522952305231523252335234523552365237523852395240524152425243524452455246524752485249525052515252525352545255525652575258525952605261526252635264526552665267526852695270527152725273527452755276527752785279528052815282528352845285528652875288528952905291529252935294529552965297529852995300530153025303530453055306530753085309531053115312531353145315531653175318531953205321532253235324532553265327532853295330533153325333533453355336533753385339534053415342534353445345534653475348534953505351535253535354535553565357535853595360536153625363536453655366536753685369537053715372537353745375537653775378537953805381538253835384538553865387538853895390539153925393539453955396539753985399540054015402540354045405540654075408540954105411541254135414541554165417541854195420542154225423542454255426542754285429543054315432543354345435543654375438543954405441544254435444544554465447544854495450545154525453545454555456545754585459546054615462546354645465546654675468546954705471547254735474547554765477547854795480548154825483548454855486548754885489549054915492549354945495549654975498549955005501550255035504550555065507550855095510551155125513551455155516551755185519552055215522552355245525552655275528552955305531553255335534553555365537553855395540554155425543554455455546554755485549555055515552555355545555555655575558555955605561556255635564556555665567556855695570557155725573557455755576557755785579558055815582558355845585558655875588558955905591559255935594559555965597559855995600560156025603560456055606560756085609561056115612561356145615561656175618561956205621562256235624562556265627562856295630563156325633563456355636563756385639564056415642564356445645564656475648564956505651565256535654565556565657565856595660566156625663566456655666566756685669567056715672567356745675567656775678567956805681568256835684568556865687568856895690569156925693569456955696569756985699570057015702570357045705570657075708570957105711571257135714571557165717571857195720572157225723572457255726572757285729573057315732573357345735573657375738573957405741574257435744574557465747574857495750575157525753575457555756575757585759576057615762576357645765576657675768576957705771577257735774577557765777577857795780578157825783578457855786578757885789579057915792579357945795579657975798579958005801580258035804580558065807580858095810581158125813581458155816581758185819582058215822582358245825582658275828582958305831583258335834583558365837583858395840584158425843584458455846584758485849585058515852585358545855585658575858585958605861586258635864586558665867586858695870587158725873587458755876587758785879588058815882588358845885588658875888588958905891589258935894589558965897589858995900590159025903590459055906590759085909591059115912591359145915591659175918591959205921592259235924592559265927592859295930593159325933593459355936593759385939594059415942594359445945594659475948594959505951595259535954595559565957595859595960596159625963596459655966596759685969597059715972597359745975597659775978597959805981598259835984598559865987598859895990599159925993599459955996599759985999600060016002600360046005600660076008600960106011601260136014601560166017601860196020602160226023602460256026602760286029603060316032603360346035603660376038603960406041604260436044604560466047604860496050605160526053605460556056605760586059606060616062606360646065606660676068606960706071607260736074607560766077607860796080608160826083608460856086608760886089609060916092609360946095609660976098609961006101610261036104610561066107610861096110611161126113611461156116611761186119612061216122612361246125612661276128612961306131613261336134613561366137613861396140614161426143614461456146614761486149615061516152615361546155615661576158615961606161616261636164616561666167616861696170617161726173617461756176617761786179618061816182618361846185618661876188618961906191619261936194619561966197619861996200620162026203620462056206620762086209621062116212621362146215621662176218621962206221622262236224622562266227622862296230623162326233623462356236623762386239624062416242624362446245624662476248624962506251625262536254625562566257625862596260626162626263626462656266626762686269627062716272627362746275627662776278627962806281628262836284628562866287628862896290629162926293629462956296629762986299630063016302630363046305630663076308630963106311631263136314631563166317631863196320632163226323632463256326632763286329633063316332633363346335633663376338633963406341634263436344634563466347634863496350635163526353635463556356635763586359636063616362636363646365636663676368636963706371637263736374637563766377637863796380638163826383638463856386638763886389639063916392639363946395639663976398639964006401640264036404640564066407640864096410641164126413641464156416641764186419642064216422642364246425642664276428642964306431643264336434643564366437643864396440644164426443644464456446644764486449645064516452645364546455645664576458645964606461646264636464646564666467646864696470647164726473647464756476647764786479648064816482648364846485648664876488648964906491649264936494649564966497649864996500650165026503650465056506650765086509651065116512651365146515651665176518651965206521652265236524652565266527652865296530653165326533653465356536653765386539654065416542654365446545654665476548654965506551655265536554655565566557655865596560656165626563656465656566656765686569657065716572657365746575657665776578657965806581658265836584658565866587658865896590659165926593659465956596659765986599660066016602660366046605660666076608660966106611661266136614661566166617661866196620662166226623662466256626662766286629663066316632663366346635663666376638663966406641664266436644664566466647664866496650665166526653665466556656665766586659666066616662666366646665666666676668666966706671667266736674667566766677667866796680668166826683668466856686668766886689669066916692669366946695669666976698669967006701670267036704670567066707670867096710671167126713671467156716671767186719672067216722672367246725672667276728672967306731673267336734673567366737673867396740674167426743674467456746674767486749675067516752675367546755675667576758675967606761676267636764676567666767676867696770677167726773677467756776677767786779678067816782678367846785678667876788678967906791679267936794679567966797679867996800680168026803680468056806680768086809681068116812681368146815681668176818681968206821682268236824682568266827682868296830683168326833683468356836683768386839684068416842684368446845684668476848684968506851685268536854685568566857685868596860686168626863686468656866686768686869687068716872687368746875687668776878687968806881688268836884688568866887688868896890689168926893689468956896689768986899690069016902690369046905690669076908690969106911691269136914691569166917691869196920692169226923692469256926692769286929693069316932693369346935693669376938693969406941694269436944694569466947694869496950695169526953695469556956695769586959696069616962696369646965696669676968696969706971697269736974697569766977697869796980698169826983698469856986698769886989699069916992699369946995699669976998699970007001700270037004700570067007700870097010701170127013701470157016701770187019702070217022702370247025702670277028702970307031703270337034703570367037703870397040704170427043704470457046704770487049705070517052705370547055705670577058705970607061706270637064706570667067706870697070707170727073707470757076707770787079708070817082708370847085708670877088708970907091709270937094709570967097709870997100710171027103710471057106710771087109711071117112711371147115711671177118711971207121712271237124712571267127712871297130713171327133713471357136713771387139714071417142714371447145714671477148714971507151715271537154715571567157715871597160716171627163716471657166716771687169717071717172717371747175717671777178717971807181718271837184718571867187718871897190719171927193719471957196719771987199720072017202720372047205720672077208720972107211721272137214721572167217721872197220722172227223722472257226722772287229723072317232723372347235723672377238723972407241724272437244724572467247724872497250725172527253725472557256725772587259726072617262726372647265726672677268726972707271727272737274727572767277727872797280728172827283728472857286728772887289729072917292729372947295729672977298729973007301730273037304730573067307730873097310731173127313731473157316731773187319732073217322732373247325732673277328732973307331733273337334733573367337733873397340734173427343734473457346734773487349735073517352735373547355735673577358735973607361736273637364736573667367736873697370737173727373737473757376737773787379738073817382738373847385738673877388738973907391739273937394739573967397739873997400740174027403740474057406740774087409741074117412741374147415741674177418741974207421742274237424742574267427742874297430743174327433743474357436743774387439744074417442744374447445744674477448744974507451745274537454745574567457745874597460746174627463746474657466746774687469747074717472747374747475747674777478747974807481748274837484748574867487748874897490749174927493749474957496749774987499750075017502750375047505750675077508750975107511751275137514751575167517751875197520752175227523752475257526752775287529753075317532753375347535753675377538753975407541754275437544754575467547754875497550755175527553755475557556755775587559756075617562756375647565756675677568756975707571757275737574757575767577757875797580758175827583758475857586758775887589759075917592759375947595759675977598759976007601760276037604760576067607760876097610761176127613761476157616761776187619762076217622762376247625762676277628762976307631763276337634763576367637763876397640764176427643764476457646764776487649765076517652765376547655765676577658765976607661766276637664766576667667766876697670767176727673767476757676767776787679768076817682768376847685768676877688768976907691769276937694769576967697769876997700770177027703770477057706770777087709771077117712771377147715771677177718771977207721772277237724772577267727772877297730773177327733773477357736773777387739774077417742774377447745774677477748774977507751775277537754775577567757775877597760776177627763776477657766776777687769777077717772777377747775777677777778777977807781778277837784778577867787778877897790779177927793779477957796779777987799780078017802780378047805780678077808780978107811781278137814781578167817781878197820782178227823782478257826782778287829783078317832783378347835783678377838783978407841784278437844784578467847784878497850785178527853785478557856785778587859786078617862786378647865786678677868786978707871787278737874787578767877787878797880788178827883788478857886788778887889789078917892789378947895789678977898789979007901790279037904790579067907790879097910791179127913791479157916791779187919792079217922792379247925792679277928792979307931793279337934793579367937793879397940794179427943794479457946794779487949795079517952795379547955795679577958795979607961796279637964796579667967796879697970797179727973797479757976797779787979798079817982798379847985798679877988798979907991799279937994799579967997799879998000800180028003800480058006800780088009801080118012801380148015801680178018801980208021802280238024802580268027802880298030803180328033803480358036803780388039804080418042804380448045804680478048804980508051805280538054805580568057805880598060806180628063806480658066806780688069807080718072807380748075807680778078807980808081808280838084808580868087808880898090809180928093809480958096809780988099810081018102810381048105810681078108810981108111811281138114811581168117811881198120812181228123812481258126812781288129813081318132813381348135813681378138813981408141814281438144814581468147814881498150815181528153815481558156815781588159816081618162816381648165816681678168816981708171817281738174817581768177817881798180818181828183818481858186818781888189819081918192819381948195819681978198819982008201820282038204820582068207820882098210821182128213821482158216821782188219822082218222822382248225822682278228822982308231823282338234823582368237823882398240824182428243824482458246824782488249825082518252825382548255825682578258825982608261826282638264826582668267826882698270827182728273827482758276827782788279828082818282828382848285828682878288828982908291829282938294829582968297829882998300830183028303830483058306830783088309831083118312831383148315831683178318831983208321832283238324832583268327832883298330833183328333833483358336833783388339834083418342834383448345834683478348834983508351835283538354835583568357835883598360836183628363836483658366836783688369837083718372837383748375837683778378837983808381838283838384838583868387838883898390839183928393839483958396839783988399840084018402840384048405840684078408840984108411841284138414841584168417841884198420842184228423842484258426842784288429843084318432843384348435843684378438843984408441844284438444844584468447844884498450845184528453845484558456845784588459846084618462846384648465846684678468846984708471847284738474847584768477847884798480848184828483848484858486848784888489849084918492849384948495849684978498849985008501850285038504850585068507850885098510851185128513851485158516851785188519852085218522852385248525852685278528852985308531853285338534853585368537853885398540854185428543854485458546854785488549855085518552855385548555855685578558855985608561856285638564856585668567856885698570857185728573857485758576857785788579858085818582858385848585858685878588858985908591859285938594859585968597859885998600860186028603860486058606860786088609861086118612861386148615861686178618861986208621862286238624862586268627862886298630863186328633863486358636863786388639864086418642864386448645864686478648864986508651865286538654865586568657865886598660866186628663866486658666866786688669867086718672867386748675867686778678867986808681868286838684868586868687868886898690869186928693869486958696869786988699870087018702870387048705870687078708870987108711871287138714871587168717871887198720872187228723872487258726872787288729873087318732873387348735873687378738873987408741874287438744874587468747874887498750875187528753875487558756875787588759876087618762876387648765876687678768876987708771877287738774877587768777877887798780878187828783878487858786878787888789879087918792879387948795879687978798879988008801880288038804880588068807
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * raid5.c : Multiple Devices driver for Linux
  4. * Copyright (C) 1996, 1997 Ingo Molnar, Miguel de Icaza, Gadi Oxman
  5. * Copyright (C) 1999, 2000 Ingo Molnar
  6. * Copyright (C) 2002, 2003 H. Peter Anvin
  7. *
  8. * RAID-4/5/6 management functions.
  9. * Thanks to Penguin Computing for making the RAID-6 development possible
  10. * by donating a test server!
  11. */
  12. /*
  13. * BITMAP UNPLUGGING:
  14. *
  15. * The sequencing for updating the bitmap reliably is a little
  16. * subtle (and I got it wrong the first time) so it deserves some
  17. * explanation.
  18. *
  19. * We group bitmap updates into batches. Each batch has a number.
  20. * We may write out several batches at once, but that isn't very important.
  21. * conf->seq_write is the number of the last batch successfully written.
  22. * conf->seq_flush is the number of the last batch that was closed to
  23. * new additions.
  24. * When we discover that we will need to write to any block in a stripe
  25. * (in add_stripe_bio) we update the in-memory bitmap and record in sh->bm_seq
  26. * the number of the batch it will be in. This is seq_flush+1.
  27. * When we are ready to do a write, if that batch hasn't been written yet,
  28. * we plug the array and queue the stripe for later.
  29. * When an unplug happens, we increment bm_flush, thus closing the current
  30. * batch.
  31. * When we notice that bm_flush > bm_write, we write out all pending updates
  32. * to the bitmap, and advance bm_write to where bm_flush was.
  33. * This may occasionally write a bit out twice, but is sure never to
  34. * miss any bits.
  35. */
  36. #include <linux/blkdev.h>
  37. #include <linux/kthread.h>
  38. #include <linux/raid/pq.h>
  39. #include <linux/async_tx.h>
  40. #include <linux/module.h>
  41. #include <linux/async.h>
  42. #include <linux/seq_file.h>
  43. #include <linux/cpu.h>
  44. #include <linux/slab.h>
  45. #include <linux/ratelimit.h>
  46. #include <linux/nodemask.h>
  47. #include <trace/events/block.h>
  48. #include <linux/list_sort.h>
  49. #include "md.h"
  50. #include "raid5.h"
  51. #include "raid0.h"
  52. #include "md-bitmap.h"
  53. #include "raid5-log.h"
  54. #define UNSUPPORTED_MDDEV_FLAGS (1L << MD_FAILFAST_SUPPORTED)
  55. #define cpu_to_group(cpu) cpu_to_node(cpu)
  56. #define ANY_GROUP NUMA_NO_NODE
  57. static bool devices_handle_discard_safely = false;
  58. module_param(devices_handle_discard_safely, bool, 0644);
  59. MODULE_PARM_DESC(devices_handle_discard_safely,
  60. "Set to Y if all devices in each array reliably return zeroes on reads from discarded regions");
  61. static struct workqueue_struct *raid5_wq;
  62. static inline struct hlist_head *stripe_hash(struct r5conf *conf, sector_t sect)
  63. {
  64. int hash = (sect >> RAID5_STRIPE_SHIFT(conf)) & HASH_MASK;
  65. return &conf->stripe_hashtbl[hash];
  66. }
  67. static inline int stripe_hash_locks_hash(struct r5conf *conf, sector_t sect)
  68. {
  69. return (sect >> RAID5_STRIPE_SHIFT(conf)) & STRIPE_HASH_LOCKS_MASK;
  70. }
  71. static inline void lock_device_hash_lock(struct r5conf *conf, int hash)
  72. {
  73. spin_lock_irq(conf->hash_locks + hash);
  74. spin_lock(&conf->device_lock);
  75. }
  76. static inline void unlock_device_hash_lock(struct r5conf *conf, int hash)
  77. {
  78. spin_unlock(&conf->device_lock);
  79. spin_unlock_irq(conf->hash_locks + hash);
  80. }
  81. static inline void lock_all_device_hash_locks_irq(struct r5conf *conf)
  82. {
  83. int i;
  84. spin_lock_irq(conf->hash_locks);
  85. for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
  86. spin_lock_nest_lock(conf->hash_locks + i, conf->hash_locks);
  87. spin_lock(&conf->device_lock);
  88. }
  89. static inline void unlock_all_device_hash_locks_irq(struct r5conf *conf)
  90. {
  91. int i;
  92. spin_unlock(&conf->device_lock);
  93. for (i = NR_STRIPE_HASH_LOCKS - 1; i; i--)
  94. spin_unlock(conf->hash_locks + i);
  95. spin_unlock_irq(conf->hash_locks);
  96. }
  97. /* Find first data disk in a raid6 stripe */
  98. static inline int raid6_d0(struct stripe_head *sh)
  99. {
  100. if (sh->ddf_layout)
  101. /* ddf always start from first device */
  102. return 0;
  103. /* md starts just after Q block */
  104. if (sh->qd_idx == sh->disks - 1)
  105. return 0;
  106. else
  107. return sh->qd_idx + 1;
  108. }
  109. static inline int raid6_next_disk(int disk, int raid_disks)
  110. {
  111. disk++;
  112. return (disk < raid_disks) ? disk : 0;
  113. }
  114. /* When walking through the disks in a raid5, starting at raid6_d0,
  115. * We need to map each disk to a 'slot', where the data disks are slot
  116. * 0 .. raid_disks-3, the parity disk is raid_disks-2 and the Q disk
  117. * is raid_disks-1. This help does that mapping.
  118. */
  119. static int raid6_idx_to_slot(int idx, struct stripe_head *sh,
  120. int *count, int syndrome_disks)
  121. {
  122. int slot = *count;
  123. if (sh->ddf_layout)
  124. (*count)++;
  125. if (idx == sh->pd_idx)
  126. return syndrome_disks;
  127. if (idx == sh->qd_idx)
  128. return syndrome_disks + 1;
  129. if (!sh->ddf_layout)
  130. (*count)++;
  131. return slot;
  132. }
  133. static void print_raid5_conf (struct r5conf *conf);
  134. static int stripe_operations_active(struct stripe_head *sh)
  135. {
  136. return sh->check_state || sh->reconstruct_state ||
  137. test_bit(STRIPE_BIOFILL_RUN, &sh->state) ||
  138. test_bit(STRIPE_COMPUTE_RUN, &sh->state);
  139. }
  140. static bool stripe_is_lowprio(struct stripe_head *sh)
  141. {
  142. return (test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) ||
  143. test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state)) &&
  144. !test_bit(STRIPE_R5C_CACHING, &sh->state);
  145. }
  146. static void raid5_wakeup_stripe_thread(struct stripe_head *sh)
  147. {
  148. struct r5conf *conf = sh->raid_conf;
  149. struct r5worker_group *group;
  150. int thread_cnt;
  151. int i, cpu = sh->cpu;
  152. if (!cpu_online(cpu)) {
  153. cpu = cpumask_any(cpu_online_mask);
  154. sh->cpu = cpu;
  155. }
  156. if (list_empty(&sh->lru)) {
  157. struct r5worker_group *group;
  158. group = conf->worker_groups + cpu_to_group(cpu);
  159. if (stripe_is_lowprio(sh))
  160. list_add_tail(&sh->lru, &group->loprio_list);
  161. else
  162. list_add_tail(&sh->lru, &group->handle_list);
  163. group->stripes_cnt++;
  164. sh->group = group;
  165. }
  166. if (conf->worker_cnt_per_group == 0) {
  167. md_wakeup_thread(conf->mddev->thread);
  168. return;
  169. }
  170. group = conf->worker_groups + cpu_to_group(sh->cpu);
  171. group->workers[0].working = true;
  172. /* at least one worker should run to avoid race */
  173. queue_work_on(sh->cpu, raid5_wq, &group->workers[0].work);
  174. thread_cnt = group->stripes_cnt / MAX_STRIPE_BATCH - 1;
  175. /* wakeup more workers */
  176. for (i = 1; i < conf->worker_cnt_per_group && thread_cnt > 0; i++) {
  177. if (group->workers[i].working == false) {
  178. group->workers[i].working = true;
  179. queue_work_on(sh->cpu, raid5_wq,
  180. &group->workers[i].work);
  181. thread_cnt--;
  182. }
  183. }
  184. }
  185. static void do_release_stripe(struct r5conf *conf, struct stripe_head *sh,
  186. struct list_head *temp_inactive_list)
  187. {
  188. int i;
  189. int injournal = 0; /* number of date pages with R5_InJournal */
  190. BUG_ON(!list_empty(&sh->lru));
  191. BUG_ON(atomic_read(&conf->active_stripes)==0);
  192. if (r5c_is_writeback(conf->log))
  193. for (i = sh->disks; i--; )
  194. if (test_bit(R5_InJournal, &sh->dev[i].flags))
  195. injournal++;
  196. /*
  197. * In the following cases, the stripe cannot be released to cached
  198. * lists. Therefore, we make the stripe write out and set
  199. * STRIPE_HANDLE:
  200. * 1. when quiesce in r5c write back;
  201. * 2. when resync is requested fot the stripe.
  202. */
  203. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) ||
  204. (conf->quiesce && r5c_is_writeback(conf->log) &&
  205. !test_bit(STRIPE_HANDLE, &sh->state) && injournal != 0)) {
  206. if (test_bit(STRIPE_R5C_CACHING, &sh->state))
  207. r5c_make_stripe_write_out(sh);
  208. set_bit(STRIPE_HANDLE, &sh->state);
  209. }
  210. if (test_bit(STRIPE_HANDLE, &sh->state)) {
  211. if (test_bit(STRIPE_DELAYED, &sh->state) &&
  212. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  213. list_add_tail(&sh->lru, &conf->delayed_list);
  214. else if (test_bit(STRIPE_BIT_DELAY, &sh->state) &&
  215. sh->bm_seq - conf->seq_write > 0)
  216. list_add_tail(&sh->lru, &conf->bitmap_list);
  217. else {
  218. clear_bit(STRIPE_DELAYED, &sh->state);
  219. clear_bit(STRIPE_BIT_DELAY, &sh->state);
  220. if (conf->worker_cnt_per_group == 0) {
  221. if (stripe_is_lowprio(sh))
  222. list_add_tail(&sh->lru,
  223. &conf->loprio_list);
  224. else
  225. list_add_tail(&sh->lru,
  226. &conf->handle_list);
  227. } else {
  228. raid5_wakeup_stripe_thread(sh);
  229. return;
  230. }
  231. }
  232. md_wakeup_thread(conf->mddev->thread);
  233. } else {
  234. BUG_ON(stripe_operations_active(sh));
  235. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  236. if (atomic_dec_return(&conf->preread_active_stripes)
  237. < IO_THRESHOLD)
  238. md_wakeup_thread(conf->mddev->thread);
  239. atomic_dec(&conf->active_stripes);
  240. if (!test_bit(STRIPE_EXPANDING, &sh->state)) {
  241. if (!r5c_is_writeback(conf->log))
  242. list_add_tail(&sh->lru, temp_inactive_list);
  243. else {
  244. WARN_ON(test_bit(R5_InJournal, &sh->dev[sh->pd_idx].flags));
  245. if (injournal == 0)
  246. list_add_tail(&sh->lru, temp_inactive_list);
  247. else if (injournal == conf->raid_disks - conf->max_degraded) {
  248. /* full stripe */
  249. if (!test_and_set_bit(STRIPE_R5C_FULL_STRIPE, &sh->state))
  250. atomic_inc(&conf->r5c_cached_full_stripes);
  251. if (test_and_clear_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state))
  252. atomic_dec(&conf->r5c_cached_partial_stripes);
  253. list_add_tail(&sh->lru, &conf->r5c_full_stripe_list);
  254. r5c_check_cached_full_stripe(conf);
  255. } else
  256. /*
  257. * STRIPE_R5C_PARTIAL_STRIPE is set in
  258. * r5c_try_caching_write(). No need to
  259. * set it again.
  260. */
  261. list_add_tail(&sh->lru, &conf->r5c_partial_stripe_list);
  262. }
  263. }
  264. }
  265. }
  266. static void __release_stripe(struct r5conf *conf, struct stripe_head *sh,
  267. struct list_head *temp_inactive_list)
  268. {
  269. if (atomic_dec_and_test(&sh->count))
  270. do_release_stripe(conf, sh, temp_inactive_list);
  271. }
  272. /*
  273. * @hash could be NR_STRIPE_HASH_LOCKS, then we have a list of inactive_list
  274. *
  275. * Be careful: Only one task can add/delete stripes from temp_inactive_list at
  276. * given time. Adding stripes only takes device lock, while deleting stripes
  277. * only takes hash lock.
  278. */
  279. static void release_inactive_stripe_list(struct r5conf *conf,
  280. struct list_head *temp_inactive_list,
  281. int hash)
  282. {
  283. int size;
  284. bool do_wakeup = false;
  285. unsigned long flags;
  286. if (hash == NR_STRIPE_HASH_LOCKS) {
  287. size = NR_STRIPE_HASH_LOCKS;
  288. hash = NR_STRIPE_HASH_LOCKS - 1;
  289. } else
  290. size = 1;
  291. while (size) {
  292. struct list_head *list = &temp_inactive_list[size - 1];
  293. /*
  294. * We don't hold any lock here yet, raid5_get_active_stripe() might
  295. * remove stripes from the list
  296. */
  297. if (!list_empty_careful(list)) {
  298. spin_lock_irqsave(conf->hash_locks + hash, flags);
  299. if (list_empty(conf->inactive_list + hash) &&
  300. !list_empty(list))
  301. atomic_dec(&conf->empty_inactive_list_nr);
  302. list_splice_tail_init(list, conf->inactive_list + hash);
  303. do_wakeup = true;
  304. spin_unlock_irqrestore(conf->hash_locks + hash, flags);
  305. }
  306. size--;
  307. hash--;
  308. }
  309. if (do_wakeup) {
  310. wake_up(&conf->wait_for_stripe);
  311. if (atomic_read(&conf->active_stripes) == 0)
  312. wake_up(&conf->wait_for_quiescent);
  313. if (conf->retry_read_aligned)
  314. md_wakeup_thread(conf->mddev->thread);
  315. }
  316. }
  317. /* should hold conf->device_lock already */
  318. static int release_stripe_list(struct r5conf *conf,
  319. struct list_head *temp_inactive_list)
  320. {
  321. struct stripe_head *sh, *t;
  322. int count = 0;
  323. struct llist_node *head;
  324. head = llist_del_all(&conf->released_stripes);
  325. head = llist_reverse_order(head);
  326. llist_for_each_entry_safe(sh, t, head, release_list) {
  327. int hash;
  328. /* sh could be readded after STRIPE_ON_RELEASE_LIST is cleard */
  329. smp_mb();
  330. clear_bit(STRIPE_ON_RELEASE_LIST, &sh->state);
  331. /*
  332. * Don't worry the bit is set here, because if the bit is set
  333. * again, the count is always > 1. This is true for
  334. * STRIPE_ON_UNPLUG_LIST bit too.
  335. */
  336. hash = sh->hash_lock_index;
  337. __release_stripe(conf, sh, &temp_inactive_list[hash]);
  338. count++;
  339. }
  340. return count;
  341. }
  342. void raid5_release_stripe(struct stripe_head *sh)
  343. {
  344. struct r5conf *conf = sh->raid_conf;
  345. unsigned long flags;
  346. struct list_head list;
  347. int hash;
  348. bool wakeup;
  349. /* Avoid release_list until the last reference.
  350. */
  351. if (atomic_add_unless(&sh->count, -1, 1))
  352. return;
  353. if (unlikely(!conf->mddev->thread) ||
  354. test_and_set_bit(STRIPE_ON_RELEASE_LIST, &sh->state))
  355. goto slow_path;
  356. wakeup = llist_add(&sh->release_list, &conf->released_stripes);
  357. if (wakeup)
  358. md_wakeup_thread(conf->mddev->thread);
  359. return;
  360. slow_path:
  361. /* we are ok here if STRIPE_ON_RELEASE_LIST is set or not */
  362. if (atomic_dec_and_lock_irqsave(&sh->count, &conf->device_lock, flags)) {
  363. INIT_LIST_HEAD(&list);
  364. hash = sh->hash_lock_index;
  365. do_release_stripe(conf, sh, &list);
  366. spin_unlock_irqrestore(&conf->device_lock, flags);
  367. release_inactive_stripe_list(conf, &list, hash);
  368. }
  369. }
  370. static inline void remove_hash(struct stripe_head *sh)
  371. {
  372. pr_debug("remove_hash(), stripe %llu\n",
  373. (unsigned long long)sh->sector);
  374. hlist_del_init(&sh->hash);
  375. }
  376. static inline void insert_hash(struct r5conf *conf, struct stripe_head *sh)
  377. {
  378. struct hlist_head *hp = stripe_hash(conf, sh->sector);
  379. pr_debug("insert_hash(), stripe %llu\n",
  380. (unsigned long long)sh->sector);
  381. hlist_add_head(&sh->hash, hp);
  382. }
  383. /* find an idle stripe, make sure it is unhashed, and return it. */
  384. static struct stripe_head *get_free_stripe(struct r5conf *conf, int hash)
  385. {
  386. struct stripe_head *sh = NULL;
  387. struct list_head *first;
  388. if (list_empty(conf->inactive_list + hash))
  389. goto out;
  390. first = (conf->inactive_list + hash)->next;
  391. sh = list_entry(first, struct stripe_head, lru);
  392. list_del_init(first);
  393. remove_hash(sh);
  394. atomic_inc(&conf->active_stripes);
  395. BUG_ON(hash != sh->hash_lock_index);
  396. if (list_empty(conf->inactive_list + hash))
  397. atomic_inc(&conf->empty_inactive_list_nr);
  398. out:
  399. return sh;
  400. }
  401. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  402. static void free_stripe_pages(struct stripe_head *sh)
  403. {
  404. int i;
  405. struct page *p;
  406. /* Have not allocate page pool */
  407. if (!sh->pages)
  408. return;
  409. for (i = 0; i < sh->nr_pages; i++) {
  410. p = sh->pages[i];
  411. if (p)
  412. put_page(p);
  413. sh->pages[i] = NULL;
  414. }
  415. }
  416. static int alloc_stripe_pages(struct stripe_head *sh, gfp_t gfp)
  417. {
  418. int i;
  419. struct page *p;
  420. for (i = 0; i < sh->nr_pages; i++) {
  421. /* The page have allocated. */
  422. if (sh->pages[i])
  423. continue;
  424. p = alloc_page(gfp);
  425. if (!p) {
  426. free_stripe_pages(sh);
  427. return -ENOMEM;
  428. }
  429. sh->pages[i] = p;
  430. }
  431. return 0;
  432. }
  433. static int
  434. init_stripe_shared_pages(struct stripe_head *sh, struct r5conf *conf, int disks)
  435. {
  436. int nr_pages, cnt;
  437. if (sh->pages)
  438. return 0;
  439. /* Each of the sh->dev[i] need one conf->stripe_size */
  440. cnt = PAGE_SIZE / conf->stripe_size;
  441. nr_pages = (disks + cnt - 1) / cnt;
  442. sh->pages = kcalloc(nr_pages, sizeof(struct page *), GFP_KERNEL);
  443. if (!sh->pages)
  444. return -ENOMEM;
  445. sh->nr_pages = nr_pages;
  446. sh->stripes_per_page = cnt;
  447. return 0;
  448. }
  449. #endif
  450. static void shrink_buffers(struct stripe_head *sh)
  451. {
  452. int i;
  453. int num = sh->raid_conf->pool_size;
  454. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  455. for (i = 0; i < num ; i++) {
  456. struct page *p;
  457. WARN_ON(sh->dev[i].page != sh->dev[i].orig_page);
  458. p = sh->dev[i].page;
  459. if (!p)
  460. continue;
  461. sh->dev[i].page = NULL;
  462. put_page(p);
  463. }
  464. #else
  465. for (i = 0; i < num; i++)
  466. sh->dev[i].page = NULL;
  467. free_stripe_pages(sh); /* Free pages */
  468. #endif
  469. }
  470. static int grow_buffers(struct stripe_head *sh, gfp_t gfp)
  471. {
  472. int i;
  473. int num = sh->raid_conf->pool_size;
  474. #if PAGE_SIZE == DEFAULT_STRIPE_SIZE
  475. for (i = 0; i < num; i++) {
  476. struct page *page;
  477. if (!(page = alloc_page(gfp))) {
  478. return 1;
  479. }
  480. sh->dev[i].page = page;
  481. sh->dev[i].orig_page = page;
  482. sh->dev[i].offset = 0;
  483. }
  484. #else
  485. if (alloc_stripe_pages(sh, gfp))
  486. return -ENOMEM;
  487. for (i = 0; i < num; i++) {
  488. sh->dev[i].page = raid5_get_dev_page(sh, i);
  489. sh->dev[i].orig_page = sh->dev[i].page;
  490. sh->dev[i].offset = raid5_get_page_offset(sh, i);
  491. }
  492. #endif
  493. return 0;
  494. }
  495. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  496. struct stripe_head *sh);
  497. static void init_stripe(struct stripe_head *sh, sector_t sector, int previous)
  498. {
  499. struct r5conf *conf = sh->raid_conf;
  500. int i, seq;
  501. BUG_ON(atomic_read(&sh->count) != 0);
  502. BUG_ON(test_bit(STRIPE_HANDLE, &sh->state));
  503. BUG_ON(stripe_operations_active(sh));
  504. BUG_ON(sh->batch_head);
  505. pr_debug("init_stripe called, stripe %llu\n",
  506. (unsigned long long)sector);
  507. retry:
  508. seq = read_seqcount_begin(&conf->gen_lock);
  509. sh->generation = conf->generation - previous;
  510. sh->disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  511. sh->sector = sector;
  512. stripe_set_idx(sector, conf, previous, sh);
  513. sh->state = 0;
  514. for (i = sh->disks; i--; ) {
  515. struct r5dev *dev = &sh->dev[i];
  516. if (dev->toread || dev->read || dev->towrite || dev->written ||
  517. test_bit(R5_LOCKED, &dev->flags)) {
  518. pr_err("sector=%llx i=%d %p %p %p %p %d\n",
  519. (unsigned long long)sh->sector, i, dev->toread,
  520. dev->read, dev->towrite, dev->written,
  521. test_bit(R5_LOCKED, &dev->flags));
  522. WARN_ON(1);
  523. }
  524. dev->flags = 0;
  525. dev->sector = raid5_compute_blocknr(sh, i, previous);
  526. }
  527. if (read_seqcount_retry(&conf->gen_lock, seq))
  528. goto retry;
  529. sh->overwrite_disks = 0;
  530. insert_hash(conf, sh);
  531. sh->cpu = smp_processor_id();
  532. set_bit(STRIPE_BATCH_READY, &sh->state);
  533. }
  534. static struct stripe_head *__find_stripe(struct r5conf *conf, sector_t sector,
  535. short generation)
  536. {
  537. struct stripe_head *sh;
  538. pr_debug("__find_stripe, sector %llu\n", (unsigned long long)sector);
  539. hlist_for_each_entry(sh, stripe_hash(conf, sector), hash)
  540. if (sh->sector == sector && sh->generation == generation)
  541. return sh;
  542. pr_debug("__stripe %llu not in cache\n", (unsigned long long)sector);
  543. return NULL;
  544. }
  545. /*
  546. * Need to check if array has failed when deciding whether to:
  547. * - start an array
  548. * - remove non-faulty devices
  549. * - add a spare
  550. * - allow a reshape
  551. * This determination is simple when no reshape is happening.
  552. * However if there is a reshape, we need to carefully check
  553. * both the before and after sections.
  554. * This is because some failed devices may only affect one
  555. * of the two sections, and some non-in_sync devices may
  556. * be insync in the section most affected by failed devices.
  557. */
  558. int raid5_calc_degraded(struct r5conf *conf)
  559. {
  560. int degraded, degraded2;
  561. int i;
  562. rcu_read_lock();
  563. degraded = 0;
  564. for (i = 0; i < conf->previous_raid_disks; i++) {
  565. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  566. if (rdev && test_bit(Faulty, &rdev->flags))
  567. rdev = rcu_dereference(conf->disks[i].replacement);
  568. if (!rdev || test_bit(Faulty, &rdev->flags))
  569. degraded++;
  570. else if (test_bit(In_sync, &rdev->flags))
  571. ;
  572. else
  573. /* not in-sync or faulty.
  574. * If the reshape increases the number of devices,
  575. * this is being recovered by the reshape, so
  576. * this 'previous' section is not in_sync.
  577. * If the number of devices is being reduced however,
  578. * the device can only be part of the array if
  579. * we are reverting a reshape, so this section will
  580. * be in-sync.
  581. */
  582. if (conf->raid_disks >= conf->previous_raid_disks)
  583. degraded++;
  584. }
  585. rcu_read_unlock();
  586. if (conf->raid_disks == conf->previous_raid_disks)
  587. return degraded;
  588. rcu_read_lock();
  589. degraded2 = 0;
  590. for (i = 0; i < conf->raid_disks; i++) {
  591. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  592. if (rdev && test_bit(Faulty, &rdev->flags))
  593. rdev = rcu_dereference(conf->disks[i].replacement);
  594. if (!rdev || test_bit(Faulty, &rdev->flags))
  595. degraded2++;
  596. else if (test_bit(In_sync, &rdev->flags))
  597. ;
  598. else
  599. /* not in-sync or faulty.
  600. * If reshape increases the number of devices, this
  601. * section has already been recovered, else it
  602. * almost certainly hasn't.
  603. */
  604. if (conf->raid_disks <= conf->previous_raid_disks)
  605. degraded2++;
  606. }
  607. rcu_read_unlock();
  608. if (degraded2 > degraded)
  609. return degraded2;
  610. return degraded;
  611. }
  612. static int has_failed(struct r5conf *conf)
  613. {
  614. int degraded;
  615. if (conf->mddev->reshape_position == MaxSector)
  616. return conf->mddev->degraded > conf->max_degraded;
  617. degraded = raid5_calc_degraded(conf);
  618. if (degraded > conf->max_degraded)
  619. return 1;
  620. return 0;
  621. }
  622. struct stripe_head *
  623. raid5_get_active_stripe(struct r5conf *conf, sector_t sector,
  624. int previous, int noblock, int noquiesce)
  625. {
  626. struct stripe_head *sh;
  627. int hash = stripe_hash_locks_hash(conf, sector);
  628. int inc_empty_inactive_list_flag;
  629. pr_debug("get_stripe, sector %llu\n", (unsigned long long)sector);
  630. spin_lock_irq(conf->hash_locks + hash);
  631. do {
  632. wait_event_lock_irq(conf->wait_for_quiescent,
  633. conf->quiesce == 0 || noquiesce,
  634. *(conf->hash_locks + hash));
  635. sh = __find_stripe(conf, sector, conf->generation - previous);
  636. if (!sh) {
  637. if (!test_bit(R5_INACTIVE_BLOCKED, &conf->cache_state)) {
  638. sh = get_free_stripe(conf, hash);
  639. if (!sh && !test_bit(R5_DID_ALLOC,
  640. &conf->cache_state))
  641. set_bit(R5_ALLOC_MORE,
  642. &conf->cache_state);
  643. }
  644. if (noblock && sh == NULL)
  645. break;
  646. r5c_check_stripe_cache_usage(conf);
  647. if (!sh) {
  648. set_bit(R5_INACTIVE_BLOCKED,
  649. &conf->cache_state);
  650. r5l_wake_reclaim(conf->log, 0);
  651. wait_event_lock_irq(
  652. conf->wait_for_stripe,
  653. !list_empty(conf->inactive_list + hash) &&
  654. (atomic_read(&conf->active_stripes)
  655. < (conf->max_nr_stripes * 3 / 4)
  656. || !test_bit(R5_INACTIVE_BLOCKED,
  657. &conf->cache_state)),
  658. *(conf->hash_locks + hash));
  659. clear_bit(R5_INACTIVE_BLOCKED,
  660. &conf->cache_state);
  661. } else {
  662. init_stripe(sh, sector, previous);
  663. atomic_inc(&sh->count);
  664. }
  665. } else if (!atomic_inc_not_zero(&sh->count)) {
  666. spin_lock(&conf->device_lock);
  667. if (!atomic_read(&sh->count)) {
  668. if (!test_bit(STRIPE_HANDLE, &sh->state))
  669. atomic_inc(&conf->active_stripes);
  670. BUG_ON(list_empty(&sh->lru) &&
  671. !test_bit(STRIPE_EXPANDING, &sh->state));
  672. inc_empty_inactive_list_flag = 0;
  673. if (!list_empty(conf->inactive_list + hash))
  674. inc_empty_inactive_list_flag = 1;
  675. list_del_init(&sh->lru);
  676. if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
  677. atomic_inc(&conf->empty_inactive_list_nr);
  678. if (sh->group) {
  679. sh->group->stripes_cnt--;
  680. sh->group = NULL;
  681. }
  682. }
  683. atomic_inc(&sh->count);
  684. spin_unlock(&conf->device_lock);
  685. }
  686. } while (sh == NULL);
  687. spin_unlock_irq(conf->hash_locks + hash);
  688. return sh;
  689. }
  690. static bool is_full_stripe_write(struct stripe_head *sh)
  691. {
  692. BUG_ON(sh->overwrite_disks > (sh->disks - sh->raid_conf->max_degraded));
  693. return sh->overwrite_disks == (sh->disks - sh->raid_conf->max_degraded);
  694. }
  695. static void lock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
  696. __acquires(&sh1->stripe_lock)
  697. __acquires(&sh2->stripe_lock)
  698. {
  699. if (sh1 > sh2) {
  700. spin_lock_irq(&sh2->stripe_lock);
  701. spin_lock_nested(&sh1->stripe_lock, 1);
  702. } else {
  703. spin_lock_irq(&sh1->stripe_lock);
  704. spin_lock_nested(&sh2->stripe_lock, 1);
  705. }
  706. }
  707. static void unlock_two_stripes(struct stripe_head *sh1, struct stripe_head *sh2)
  708. __releases(&sh1->stripe_lock)
  709. __releases(&sh2->stripe_lock)
  710. {
  711. spin_unlock(&sh1->stripe_lock);
  712. spin_unlock_irq(&sh2->stripe_lock);
  713. }
  714. /* Only freshly new full stripe normal write stripe can be added to a batch list */
  715. static bool stripe_can_batch(struct stripe_head *sh)
  716. {
  717. struct r5conf *conf = sh->raid_conf;
  718. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  719. return false;
  720. return test_bit(STRIPE_BATCH_READY, &sh->state) &&
  721. !test_bit(STRIPE_BITMAP_PENDING, &sh->state) &&
  722. is_full_stripe_write(sh);
  723. }
  724. /* we only do back search */
  725. static void stripe_add_to_batch_list(struct r5conf *conf, struct stripe_head *sh)
  726. {
  727. struct stripe_head *head;
  728. sector_t head_sector, tmp_sec;
  729. int hash;
  730. int dd_idx;
  731. int inc_empty_inactive_list_flag;
  732. /* Don't cross chunks, so stripe pd_idx/qd_idx is the same */
  733. tmp_sec = sh->sector;
  734. if (!sector_div(tmp_sec, conf->chunk_sectors))
  735. return;
  736. head_sector = sh->sector - RAID5_STRIPE_SECTORS(conf);
  737. hash = stripe_hash_locks_hash(conf, head_sector);
  738. spin_lock_irq(conf->hash_locks + hash);
  739. head = __find_stripe(conf, head_sector, conf->generation);
  740. if (head && !atomic_inc_not_zero(&head->count)) {
  741. spin_lock(&conf->device_lock);
  742. if (!atomic_read(&head->count)) {
  743. if (!test_bit(STRIPE_HANDLE, &head->state))
  744. atomic_inc(&conf->active_stripes);
  745. BUG_ON(list_empty(&head->lru) &&
  746. !test_bit(STRIPE_EXPANDING, &head->state));
  747. inc_empty_inactive_list_flag = 0;
  748. if (!list_empty(conf->inactive_list + hash))
  749. inc_empty_inactive_list_flag = 1;
  750. list_del_init(&head->lru);
  751. if (list_empty(conf->inactive_list + hash) && inc_empty_inactive_list_flag)
  752. atomic_inc(&conf->empty_inactive_list_nr);
  753. if (head->group) {
  754. head->group->stripes_cnt--;
  755. head->group = NULL;
  756. }
  757. }
  758. atomic_inc(&head->count);
  759. spin_unlock(&conf->device_lock);
  760. }
  761. spin_unlock_irq(conf->hash_locks + hash);
  762. if (!head)
  763. return;
  764. if (!stripe_can_batch(head))
  765. goto out;
  766. lock_two_stripes(head, sh);
  767. /* clear_batch_ready clear the flag */
  768. if (!stripe_can_batch(head) || !stripe_can_batch(sh))
  769. goto unlock_out;
  770. if (sh->batch_head)
  771. goto unlock_out;
  772. dd_idx = 0;
  773. while (dd_idx == sh->pd_idx || dd_idx == sh->qd_idx)
  774. dd_idx++;
  775. if (head->dev[dd_idx].towrite->bi_opf != sh->dev[dd_idx].towrite->bi_opf ||
  776. bio_op(head->dev[dd_idx].towrite) != bio_op(sh->dev[dd_idx].towrite))
  777. goto unlock_out;
  778. if (head->batch_head) {
  779. spin_lock(&head->batch_head->batch_lock);
  780. /* This batch list is already running */
  781. if (!stripe_can_batch(head)) {
  782. spin_unlock(&head->batch_head->batch_lock);
  783. goto unlock_out;
  784. }
  785. /*
  786. * We must assign batch_head of this stripe within the
  787. * batch_lock, otherwise clear_batch_ready of batch head
  788. * stripe could clear BATCH_READY bit of this stripe and
  789. * this stripe->batch_head doesn't get assigned, which
  790. * could confuse clear_batch_ready for this stripe
  791. */
  792. sh->batch_head = head->batch_head;
  793. /*
  794. * at this point, head's BATCH_READY could be cleared, but we
  795. * can still add the stripe to batch list
  796. */
  797. list_add(&sh->batch_list, &head->batch_list);
  798. spin_unlock(&head->batch_head->batch_lock);
  799. } else {
  800. head->batch_head = head;
  801. sh->batch_head = head->batch_head;
  802. spin_lock(&head->batch_lock);
  803. list_add_tail(&sh->batch_list, &head->batch_list);
  804. spin_unlock(&head->batch_lock);
  805. }
  806. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  807. if (atomic_dec_return(&conf->preread_active_stripes)
  808. < IO_THRESHOLD)
  809. md_wakeup_thread(conf->mddev->thread);
  810. if (test_and_clear_bit(STRIPE_BIT_DELAY, &sh->state)) {
  811. int seq = sh->bm_seq;
  812. if (test_bit(STRIPE_BIT_DELAY, &sh->batch_head->state) &&
  813. sh->batch_head->bm_seq > seq)
  814. seq = sh->batch_head->bm_seq;
  815. set_bit(STRIPE_BIT_DELAY, &sh->batch_head->state);
  816. sh->batch_head->bm_seq = seq;
  817. }
  818. atomic_inc(&sh->count);
  819. unlock_out:
  820. unlock_two_stripes(head, sh);
  821. out:
  822. raid5_release_stripe(head);
  823. }
  824. /* Determine if 'data_offset' or 'new_data_offset' should be used
  825. * in this stripe_head.
  826. */
  827. static int use_new_offset(struct r5conf *conf, struct stripe_head *sh)
  828. {
  829. sector_t progress = conf->reshape_progress;
  830. /* Need a memory barrier to make sure we see the value
  831. * of conf->generation, or ->data_offset that was set before
  832. * reshape_progress was updated.
  833. */
  834. smp_rmb();
  835. if (progress == MaxSector)
  836. return 0;
  837. if (sh->generation == conf->generation - 1)
  838. return 0;
  839. /* We are in a reshape, and this is a new-generation stripe,
  840. * so use new_data_offset.
  841. */
  842. return 1;
  843. }
  844. static void dispatch_bio_list(struct bio_list *tmp)
  845. {
  846. struct bio *bio;
  847. while ((bio = bio_list_pop(tmp)))
  848. submit_bio_noacct(bio);
  849. }
  850. static int cmp_stripe(void *priv, struct list_head *a, struct list_head *b)
  851. {
  852. const struct r5pending_data *da = list_entry(a,
  853. struct r5pending_data, sibling);
  854. const struct r5pending_data *db = list_entry(b,
  855. struct r5pending_data, sibling);
  856. if (da->sector > db->sector)
  857. return 1;
  858. if (da->sector < db->sector)
  859. return -1;
  860. return 0;
  861. }
  862. static void dispatch_defer_bios(struct r5conf *conf, int target,
  863. struct bio_list *list)
  864. {
  865. struct r5pending_data *data;
  866. struct list_head *first, *next = NULL;
  867. int cnt = 0;
  868. if (conf->pending_data_cnt == 0)
  869. return;
  870. list_sort(NULL, &conf->pending_list, cmp_stripe);
  871. first = conf->pending_list.next;
  872. /* temporarily move the head */
  873. if (conf->next_pending_data)
  874. list_move_tail(&conf->pending_list,
  875. &conf->next_pending_data->sibling);
  876. while (!list_empty(&conf->pending_list)) {
  877. data = list_first_entry(&conf->pending_list,
  878. struct r5pending_data, sibling);
  879. if (&data->sibling == first)
  880. first = data->sibling.next;
  881. next = data->sibling.next;
  882. bio_list_merge(list, &data->bios);
  883. list_move(&data->sibling, &conf->free_list);
  884. cnt++;
  885. if (cnt >= target)
  886. break;
  887. }
  888. conf->pending_data_cnt -= cnt;
  889. BUG_ON(conf->pending_data_cnt < 0 || cnt < target);
  890. if (next != &conf->pending_list)
  891. conf->next_pending_data = list_entry(next,
  892. struct r5pending_data, sibling);
  893. else
  894. conf->next_pending_data = NULL;
  895. /* list isn't empty */
  896. if (first != &conf->pending_list)
  897. list_move_tail(&conf->pending_list, first);
  898. }
  899. static void flush_deferred_bios(struct r5conf *conf)
  900. {
  901. struct bio_list tmp = BIO_EMPTY_LIST;
  902. if (conf->pending_data_cnt == 0)
  903. return;
  904. spin_lock(&conf->pending_bios_lock);
  905. dispatch_defer_bios(conf, conf->pending_data_cnt, &tmp);
  906. BUG_ON(conf->pending_data_cnt != 0);
  907. spin_unlock(&conf->pending_bios_lock);
  908. dispatch_bio_list(&tmp);
  909. }
  910. static void defer_issue_bios(struct r5conf *conf, sector_t sector,
  911. struct bio_list *bios)
  912. {
  913. struct bio_list tmp = BIO_EMPTY_LIST;
  914. struct r5pending_data *ent;
  915. spin_lock(&conf->pending_bios_lock);
  916. ent = list_first_entry(&conf->free_list, struct r5pending_data,
  917. sibling);
  918. list_move_tail(&ent->sibling, &conf->pending_list);
  919. ent->sector = sector;
  920. bio_list_init(&ent->bios);
  921. bio_list_merge(&ent->bios, bios);
  922. conf->pending_data_cnt++;
  923. if (conf->pending_data_cnt >= PENDING_IO_MAX)
  924. dispatch_defer_bios(conf, PENDING_IO_ONE_FLUSH, &tmp);
  925. spin_unlock(&conf->pending_bios_lock);
  926. dispatch_bio_list(&tmp);
  927. }
  928. static void
  929. raid5_end_read_request(struct bio *bi);
  930. static void
  931. raid5_end_write_request(struct bio *bi);
  932. static void ops_run_io(struct stripe_head *sh, struct stripe_head_state *s)
  933. {
  934. struct r5conf *conf = sh->raid_conf;
  935. int i, disks = sh->disks;
  936. struct stripe_head *head_sh = sh;
  937. struct bio_list pending_bios = BIO_EMPTY_LIST;
  938. bool should_defer;
  939. might_sleep();
  940. if (log_stripe(sh, s) == 0)
  941. return;
  942. should_defer = conf->batch_bio_dispatch && conf->group_cnt;
  943. for (i = disks; i--; ) {
  944. int op, op_flags = 0;
  945. int replace_only = 0;
  946. struct bio *bi, *rbi;
  947. struct md_rdev *rdev, *rrdev = NULL;
  948. sh = head_sh;
  949. if (test_and_clear_bit(R5_Wantwrite, &sh->dev[i].flags)) {
  950. op = REQ_OP_WRITE;
  951. if (test_and_clear_bit(R5_WantFUA, &sh->dev[i].flags))
  952. op_flags = REQ_FUA;
  953. if (test_bit(R5_Discard, &sh->dev[i].flags))
  954. op = REQ_OP_DISCARD;
  955. } else if (test_and_clear_bit(R5_Wantread, &sh->dev[i].flags))
  956. op = REQ_OP_READ;
  957. else if (test_and_clear_bit(R5_WantReplace,
  958. &sh->dev[i].flags)) {
  959. op = REQ_OP_WRITE;
  960. replace_only = 1;
  961. } else
  962. continue;
  963. if (test_and_clear_bit(R5_SyncIO, &sh->dev[i].flags))
  964. op_flags |= REQ_SYNC;
  965. again:
  966. bi = &sh->dev[i].req;
  967. rbi = &sh->dev[i].rreq; /* For writing to replacement */
  968. rcu_read_lock();
  969. rrdev = rcu_dereference(conf->disks[i].replacement);
  970. smp_mb(); /* Ensure that if rrdev is NULL, rdev won't be */
  971. rdev = rcu_dereference(conf->disks[i].rdev);
  972. if (!rdev) {
  973. rdev = rrdev;
  974. rrdev = NULL;
  975. }
  976. if (op_is_write(op)) {
  977. if (replace_only)
  978. rdev = NULL;
  979. if (rdev == rrdev)
  980. /* We raced and saw duplicates */
  981. rrdev = NULL;
  982. } else {
  983. if (test_bit(R5_ReadRepl, &head_sh->dev[i].flags) && rrdev)
  984. rdev = rrdev;
  985. rrdev = NULL;
  986. }
  987. if (rdev && test_bit(Faulty, &rdev->flags))
  988. rdev = NULL;
  989. if (rdev)
  990. atomic_inc(&rdev->nr_pending);
  991. if (rrdev && test_bit(Faulty, &rrdev->flags))
  992. rrdev = NULL;
  993. if (rrdev)
  994. atomic_inc(&rrdev->nr_pending);
  995. rcu_read_unlock();
  996. /* We have already checked bad blocks for reads. Now
  997. * need to check for writes. We never accept write errors
  998. * on the replacement, so we don't to check rrdev.
  999. */
  1000. while (op_is_write(op) && rdev &&
  1001. test_bit(WriteErrorSeen, &rdev->flags)) {
  1002. sector_t first_bad;
  1003. int bad_sectors;
  1004. int bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  1005. &first_bad, &bad_sectors);
  1006. if (!bad)
  1007. break;
  1008. if (bad < 0) {
  1009. set_bit(BlockedBadBlocks, &rdev->flags);
  1010. if (!conf->mddev->external &&
  1011. conf->mddev->sb_flags) {
  1012. /* It is very unlikely, but we might
  1013. * still need to write out the
  1014. * bad block log - better give it
  1015. * a chance*/
  1016. md_check_recovery(conf->mddev);
  1017. }
  1018. /*
  1019. * Because md_wait_for_blocked_rdev
  1020. * will dec nr_pending, we must
  1021. * increment it first.
  1022. */
  1023. atomic_inc(&rdev->nr_pending);
  1024. md_wait_for_blocked_rdev(rdev, conf->mddev);
  1025. } else {
  1026. /* Acknowledged bad block - skip the write */
  1027. rdev_dec_pending(rdev, conf->mddev);
  1028. rdev = NULL;
  1029. }
  1030. }
  1031. if (rdev) {
  1032. if (s->syncing || s->expanding || s->expanded
  1033. || s->replacing)
  1034. md_sync_acct(rdev->bdev, RAID5_STRIPE_SECTORS(conf));
  1035. set_bit(STRIPE_IO_STARTED, &sh->state);
  1036. bio_set_dev(bi, rdev->bdev);
  1037. bio_set_op_attrs(bi, op, op_flags);
  1038. bi->bi_end_io = op_is_write(op)
  1039. ? raid5_end_write_request
  1040. : raid5_end_read_request;
  1041. bi->bi_private = sh;
  1042. pr_debug("%s: for %llu schedule op %d on disc %d\n",
  1043. __func__, (unsigned long long)sh->sector,
  1044. bi->bi_opf, i);
  1045. atomic_inc(&sh->count);
  1046. if (sh != head_sh)
  1047. atomic_inc(&head_sh->count);
  1048. if (use_new_offset(conf, sh))
  1049. bi->bi_iter.bi_sector = (sh->sector
  1050. + rdev->new_data_offset);
  1051. else
  1052. bi->bi_iter.bi_sector = (sh->sector
  1053. + rdev->data_offset);
  1054. if (test_bit(R5_ReadNoMerge, &head_sh->dev[i].flags))
  1055. bi->bi_opf |= REQ_NOMERGE;
  1056. if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
  1057. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  1058. if (!op_is_write(op) &&
  1059. test_bit(R5_InJournal, &sh->dev[i].flags))
  1060. /*
  1061. * issuing read for a page in journal, this
  1062. * must be preparing for prexor in rmw; read
  1063. * the data into orig_page
  1064. */
  1065. sh->dev[i].vec.bv_page = sh->dev[i].orig_page;
  1066. else
  1067. sh->dev[i].vec.bv_page = sh->dev[i].page;
  1068. bi->bi_vcnt = 1;
  1069. bi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
  1070. bi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
  1071. bi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
  1072. bi->bi_write_hint = sh->dev[i].write_hint;
  1073. if (!rrdev)
  1074. sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
  1075. /*
  1076. * If this is discard request, set bi_vcnt 0. We don't
  1077. * want to confuse SCSI because SCSI will replace payload
  1078. */
  1079. if (op == REQ_OP_DISCARD)
  1080. bi->bi_vcnt = 0;
  1081. if (rrdev)
  1082. set_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags);
  1083. if (conf->mddev->gendisk)
  1084. trace_block_bio_remap(bi->bi_disk->queue,
  1085. bi, disk_devt(conf->mddev->gendisk),
  1086. sh->dev[i].sector);
  1087. if (should_defer && op_is_write(op))
  1088. bio_list_add(&pending_bios, bi);
  1089. else
  1090. submit_bio_noacct(bi);
  1091. }
  1092. if (rrdev) {
  1093. if (s->syncing || s->expanding || s->expanded
  1094. || s->replacing)
  1095. md_sync_acct(rrdev->bdev, RAID5_STRIPE_SECTORS(conf));
  1096. set_bit(STRIPE_IO_STARTED, &sh->state);
  1097. bio_set_dev(rbi, rrdev->bdev);
  1098. bio_set_op_attrs(rbi, op, op_flags);
  1099. BUG_ON(!op_is_write(op));
  1100. rbi->bi_end_io = raid5_end_write_request;
  1101. rbi->bi_private = sh;
  1102. pr_debug("%s: for %llu schedule op %d on "
  1103. "replacement disc %d\n",
  1104. __func__, (unsigned long long)sh->sector,
  1105. rbi->bi_opf, i);
  1106. atomic_inc(&sh->count);
  1107. if (sh != head_sh)
  1108. atomic_inc(&head_sh->count);
  1109. if (use_new_offset(conf, sh))
  1110. rbi->bi_iter.bi_sector = (sh->sector
  1111. + rrdev->new_data_offset);
  1112. else
  1113. rbi->bi_iter.bi_sector = (sh->sector
  1114. + rrdev->data_offset);
  1115. if (test_bit(R5_SkipCopy, &sh->dev[i].flags))
  1116. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  1117. sh->dev[i].rvec.bv_page = sh->dev[i].page;
  1118. rbi->bi_vcnt = 1;
  1119. rbi->bi_io_vec[0].bv_len = RAID5_STRIPE_SIZE(conf);
  1120. rbi->bi_io_vec[0].bv_offset = sh->dev[i].offset;
  1121. rbi->bi_iter.bi_size = RAID5_STRIPE_SIZE(conf);
  1122. rbi->bi_write_hint = sh->dev[i].write_hint;
  1123. sh->dev[i].write_hint = RWH_WRITE_LIFE_NOT_SET;
  1124. /*
  1125. * If this is discard request, set bi_vcnt 0. We don't
  1126. * want to confuse SCSI because SCSI will replace payload
  1127. */
  1128. if (op == REQ_OP_DISCARD)
  1129. rbi->bi_vcnt = 0;
  1130. if (conf->mddev->gendisk)
  1131. trace_block_bio_remap(rbi->bi_disk->queue,
  1132. rbi, disk_devt(conf->mddev->gendisk),
  1133. sh->dev[i].sector);
  1134. if (should_defer && op_is_write(op))
  1135. bio_list_add(&pending_bios, rbi);
  1136. else
  1137. submit_bio_noacct(rbi);
  1138. }
  1139. if (!rdev && !rrdev) {
  1140. if (op_is_write(op))
  1141. set_bit(STRIPE_DEGRADED, &sh->state);
  1142. pr_debug("skip op %d on disc %d for sector %llu\n",
  1143. bi->bi_opf, i, (unsigned long long)sh->sector);
  1144. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  1145. set_bit(STRIPE_HANDLE, &sh->state);
  1146. }
  1147. if (!head_sh->batch_head)
  1148. continue;
  1149. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1150. batch_list);
  1151. if (sh != head_sh)
  1152. goto again;
  1153. }
  1154. if (should_defer && !bio_list_empty(&pending_bios))
  1155. defer_issue_bios(conf, head_sh->sector, &pending_bios);
  1156. }
  1157. static struct dma_async_tx_descriptor *
  1158. async_copy_data(int frombio, struct bio *bio, struct page **page,
  1159. unsigned int poff, sector_t sector, struct dma_async_tx_descriptor *tx,
  1160. struct stripe_head *sh, int no_skipcopy)
  1161. {
  1162. struct bio_vec bvl;
  1163. struct bvec_iter iter;
  1164. struct page *bio_page;
  1165. int page_offset;
  1166. struct async_submit_ctl submit;
  1167. enum async_tx_flags flags = 0;
  1168. struct r5conf *conf = sh->raid_conf;
  1169. if (bio->bi_iter.bi_sector >= sector)
  1170. page_offset = (signed)(bio->bi_iter.bi_sector - sector) * 512;
  1171. else
  1172. page_offset = (signed)(sector - bio->bi_iter.bi_sector) * -512;
  1173. if (frombio)
  1174. flags |= ASYNC_TX_FENCE;
  1175. init_async_submit(&submit, flags, tx, NULL, NULL, NULL);
  1176. bio_for_each_segment(bvl, bio, iter) {
  1177. int len = bvl.bv_len;
  1178. int clen;
  1179. int b_offset = 0;
  1180. if (page_offset < 0) {
  1181. b_offset = -page_offset;
  1182. page_offset += b_offset;
  1183. len -= b_offset;
  1184. }
  1185. if (len > 0 && page_offset + len > RAID5_STRIPE_SIZE(conf))
  1186. clen = RAID5_STRIPE_SIZE(conf) - page_offset;
  1187. else
  1188. clen = len;
  1189. if (clen > 0) {
  1190. b_offset += bvl.bv_offset;
  1191. bio_page = bvl.bv_page;
  1192. if (frombio) {
  1193. if (conf->skip_copy &&
  1194. b_offset == 0 && page_offset == 0 &&
  1195. clen == RAID5_STRIPE_SIZE(conf) &&
  1196. !no_skipcopy)
  1197. *page = bio_page;
  1198. else
  1199. tx = async_memcpy(*page, bio_page, page_offset + poff,
  1200. b_offset, clen, &submit);
  1201. } else
  1202. tx = async_memcpy(bio_page, *page, b_offset,
  1203. page_offset + poff, clen, &submit);
  1204. }
  1205. /* chain the operations */
  1206. submit.depend_tx = tx;
  1207. if (clen < len) /* hit end of page */
  1208. break;
  1209. page_offset += len;
  1210. }
  1211. return tx;
  1212. }
  1213. static void ops_complete_biofill(void *stripe_head_ref)
  1214. {
  1215. struct stripe_head *sh = stripe_head_ref;
  1216. int i;
  1217. struct r5conf *conf = sh->raid_conf;
  1218. pr_debug("%s: stripe %llu\n", __func__,
  1219. (unsigned long long)sh->sector);
  1220. /* clear completed biofills */
  1221. for (i = sh->disks; i--; ) {
  1222. struct r5dev *dev = &sh->dev[i];
  1223. /* acknowledge completion of a biofill operation */
  1224. /* and check if we need to reply to a read request,
  1225. * new R5_Wantfill requests are held off until
  1226. * !STRIPE_BIOFILL_RUN
  1227. */
  1228. if (test_and_clear_bit(R5_Wantfill, &dev->flags)) {
  1229. struct bio *rbi, *rbi2;
  1230. BUG_ON(!dev->read);
  1231. rbi = dev->read;
  1232. dev->read = NULL;
  1233. while (rbi && rbi->bi_iter.bi_sector <
  1234. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1235. rbi2 = r5_next_bio(conf, rbi, dev->sector);
  1236. bio_endio(rbi);
  1237. rbi = rbi2;
  1238. }
  1239. }
  1240. }
  1241. clear_bit(STRIPE_BIOFILL_RUN, &sh->state);
  1242. set_bit(STRIPE_HANDLE, &sh->state);
  1243. raid5_release_stripe(sh);
  1244. }
  1245. static void ops_run_biofill(struct stripe_head *sh)
  1246. {
  1247. struct dma_async_tx_descriptor *tx = NULL;
  1248. struct async_submit_ctl submit;
  1249. int i;
  1250. struct r5conf *conf = sh->raid_conf;
  1251. BUG_ON(sh->batch_head);
  1252. pr_debug("%s: stripe %llu\n", __func__,
  1253. (unsigned long long)sh->sector);
  1254. for (i = sh->disks; i--; ) {
  1255. struct r5dev *dev = &sh->dev[i];
  1256. if (test_bit(R5_Wantfill, &dev->flags)) {
  1257. struct bio *rbi;
  1258. spin_lock_irq(&sh->stripe_lock);
  1259. dev->read = rbi = dev->toread;
  1260. dev->toread = NULL;
  1261. spin_unlock_irq(&sh->stripe_lock);
  1262. while (rbi && rbi->bi_iter.bi_sector <
  1263. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1264. tx = async_copy_data(0, rbi, &dev->page,
  1265. dev->offset,
  1266. dev->sector, tx, sh, 0);
  1267. rbi = r5_next_bio(conf, rbi, dev->sector);
  1268. }
  1269. }
  1270. }
  1271. atomic_inc(&sh->count);
  1272. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_biofill, sh, NULL);
  1273. async_trigger_callback(&submit);
  1274. }
  1275. static void mark_target_uptodate(struct stripe_head *sh, int target)
  1276. {
  1277. struct r5dev *tgt;
  1278. if (target < 0)
  1279. return;
  1280. tgt = &sh->dev[target];
  1281. set_bit(R5_UPTODATE, &tgt->flags);
  1282. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1283. clear_bit(R5_Wantcompute, &tgt->flags);
  1284. }
  1285. static void ops_complete_compute(void *stripe_head_ref)
  1286. {
  1287. struct stripe_head *sh = stripe_head_ref;
  1288. pr_debug("%s: stripe %llu\n", __func__,
  1289. (unsigned long long)sh->sector);
  1290. /* mark the computed target(s) as uptodate */
  1291. mark_target_uptodate(sh, sh->ops.target);
  1292. mark_target_uptodate(sh, sh->ops.target2);
  1293. clear_bit(STRIPE_COMPUTE_RUN, &sh->state);
  1294. if (sh->check_state == check_state_compute_run)
  1295. sh->check_state = check_state_compute_result;
  1296. set_bit(STRIPE_HANDLE, &sh->state);
  1297. raid5_release_stripe(sh);
  1298. }
  1299. /* return a pointer to the address conversion region of the scribble buffer */
  1300. static struct page **to_addr_page(struct raid5_percpu *percpu, int i)
  1301. {
  1302. return percpu->scribble + i * percpu->scribble_obj_size;
  1303. }
  1304. /* return a pointer to the address conversion region of the scribble buffer */
  1305. static addr_conv_t *to_addr_conv(struct stripe_head *sh,
  1306. struct raid5_percpu *percpu, int i)
  1307. {
  1308. return (void *) (to_addr_page(percpu, i) + sh->disks + 2);
  1309. }
  1310. /*
  1311. * Return a pointer to record offset address.
  1312. */
  1313. static unsigned int *
  1314. to_addr_offs(struct stripe_head *sh, struct raid5_percpu *percpu)
  1315. {
  1316. return (unsigned int *) (to_addr_conv(sh, percpu, 0) + sh->disks + 2);
  1317. }
  1318. static struct dma_async_tx_descriptor *
  1319. ops_run_compute5(struct stripe_head *sh, struct raid5_percpu *percpu)
  1320. {
  1321. int disks = sh->disks;
  1322. struct page **xor_srcs = to_addr_page(percpu, 0);
  1323. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1324. int target = sh->ops.target;
  1325. struct r5dev *tgt = &sh->dev[target];
  1326. struct page *xor_dest = tgt->page;
  1327. unsigned int off_dest = tgt->offset;
  1328. int count = 0;
  1329. struct dma_async_tx_descriptor *tx;
  1330. struct async_submit_ctl submit;
  1331. int i;
  1332. BUG_ON(sh->batch_head);
  1333. pr_debug("%s: stripe %llu block: %d\n",
  1334. __func__, (unsigned long long)sh->sector, target);
  1335. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1336. for (i = disks; i--; ) {
  1337. if (i != target) {
  1338. off_srcs[count] = sh->dev[i].offset;
  1339. xor_srcs[count++] = sh->dev[i].page;
  1340. }
  1341. }
  1342. atomic_inc(&sh->count);
  1343. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST, NULL,
  1344. ops_complete_compute, sh, to_addr_conv(sh, percpu, 0));
  1345. if (unlikely(count == 1))
  1346. tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
  1347. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1348. else
  1349. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1350. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1351. return tx;
  1352. }
  1353. /* set_syndrome_sources - populate source buffers for gen_syndrome
  1354. * @srcs - (struct page *) array of size sh->disks
  1355. * @offs - (unsigned int) array of offset for each page
  1356. * @sh - stripe_head to parse
  1357. *
  1358. * Populates srcs in proper layout order for the stripe and returns the
  1359. * 'count' of sources to be used in a call to async_gen_syndrome. The P
  1360. * destination buffer is recorded in srcs[count] and the Q destination
  1361. * is recorded in srcs[count+1]].
  1362. */
  1363. static int set_syndrome_sources(struct page **srcs,
  1364. unsigned int *offs,
  1365. struct stripe_head *sh,
  1366. int srctype)
  1367. {
  1368. int disks = sh->disks;
  1369. int syndrome_disks = sh->ddf_layout ? disks : (disks - 2);
  1370. int d0_idx = raid6_d0(sh);
  1371. int count;
  1372. int i;
  1373. for (i = 0; i < disks; i++)
  1374. srcs[i] = NULL;
  1375. count = 0;
  1376. i = d0_idx;
  1377. do {
  1378. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  1379. struct r5dev *dev = &sh->dev[i];
  1380. if (i == sh->qd_idx || i == sh->pd_idx ||
  1381. (srctype == SYNDROME_SRC_ALL) ||
  1382. (srctype == SYNDROME_SRC_WANT_DRAIN &&
  1383. (test_bit(R5_Wantdrain, &dev->flags) ||
  1384. test_bit(R5_InJournal, &dev->flags))) ||
  1385. (srctype == SYNDROME_SRC_WRITTEN &&
  1386. (dev->written ||
  1387. test_bit(R5_InJournal, &dev->flags)))) {
  1388. if (test_bit(R5_InJournal, &dev->flags))
  1389. srcs[slot] = sh->dev[i].orig_page;
  1390. else
  1391. srcs[slot] = sh->dev[i].page;
  1392. /*
  1393. * For R5_InJournal, PAGE_SIZE must be 4KB and will
  1394. * not shared page. In that case, dev[i].offset
  1395. * is 0.
  1396. */
  1397. offs[slot] = sh->dev[i].offset;
  1398. }
  1399. i = raid6_next_disk(i, disks);
  1400. } while (i != d0_idx);
  1401. return syndrome_disks;
  1402. }
  1403. static struct dma_async_tx_descriptor *
  1404. ops_run_compute6_1(struct stripe_head *sh, struct raid5_percpu *percpu)
  1405. {
  1406. int disks = sh->disks;
  1407. struct page **blocks = to_addr_page(percpu, 0);
  1408. unsigned int *offs = to_addr_offs(sh, percpu);
  1409. int target;
  1410. int qd_idx = sh->qd_idx;
  1411. struct dma_async_tx_descriptor *tx;
  1412. struct async_submit_ctl submit;
  1413. struct r5dev *tgt;
  1414. struct page *dest;
  1415. unsigned int dest_off;
  1416. int i;
  1417. int count;
  1418. BUG_ON(sh->batch_head);
  1419. if (sh->ops.target < 0)
  1420. target = sh->ops.target2;
  1421. else if (sh->ops.target2 < 0)
  1422. target = sh->ops.target;
  1423. else
  1424. /* we should only have one valid target */
  1425. BUG();
  1426. BUG_ON(target < 0);
  1427. pr_debug("%s: stripe %llu block: %d\n",
  1428. __func__, (unsigned long long)sh->sector, target);
  1429. tgt = &sh->dev[target];
  1430. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1431. dest = tgt->page;
  1432. dest_off = tgt->offset;
  1433. atomic_inc(&sh->count);
  1434. if (target == qd_idx) {
  1435. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
  1436. blocks[count] = NULL; /* regenerating p is not necessary */
  1437. BUG_ON(blocks[count+1] != dest); /* q should already be set */
  1438. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1439. ops_complete_compute, sh,
  1440. to_addr_conv(sh, percpu, 0));
  1441. tx = async_gen_syndrome(blocks, offs, count+2,
  1442. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1443. } else {
  1444. /* Compute any data- or p-drive using XOR */
  1445. count = 0;
  1446. for (i = disks; i-- ; ) {
  1447. if (i == target || i == qd_idx)
  1448. continue;
  1449. offs[count] = sh->dev[i].offset;
  1450. blocks[count++] = sh->dev[i].page;
  1451. }
  1452. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  1453. NULL, ops_complete_compute, sh,
  1454. to_addr_conv(sh, percpu, 0));
  1455. tx = async_xor_offs(dest, dest_off, blocks, offs, count,
  1456. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1457. }
  1458. return tx;
  1459. }
  1460. static struct dma_async_tx_descriptor *
  1461. ops_run_compute6_2(struct stripe_head *sh, struct raid5_percpu *percpu)
  1462. {
  1463. int i, count, disks = sh->disks;
  1464. int syndrome_disks = sh->ddf_layout ? disks : disks-2;
  1465. int d0_idx = raid6_d0(sh);
  1466. int faila = -1, failb = -1;
  1467. int target = sh->ops.target;
  1468. int target2 = sh->ops.target2;
  1469. struct r5dev *tgt = &sh->dev[target];
  1470. struct r5dev *tgt2 = &sh->dev[target2];
  1471. struct dma_async_tx_descriptor *tx;
  1472. struct page **blocks = to_addr_page(percpu, 0);
  1473. unsigned int *offs = to_addr_offs(sh, percpu);
  1474. struct async_submit_ctl submit;
  1475. BUG_ON(sh->batch_head);
  1476. pr_debug("%s: stripe %llu block1: %d block2: %d\n",
  1477. __func__, (unsigned long long)sh->sector, target, target2);
  1478. BUG_ON(target < 0 || target2 < 0);
  1479. BUG_ON(!test_bit(R5_Wantcompute, &tgt->flags));
  1480. BUG_ON(!test_bit(R5_Wantcompute, &tgt2->flags));
  1481. /* we need to open-code set_syndrome_sources to handle the
  1482. * slot number conversion for 'faila' and 'failb'
  1483. */
  1484. for (i = 0; i < disks ; i++) {
  1485. offs[i] = 0;
  1486. blocks[i] = NULL;
  1487. }
  1488. count = 0;
  1489. i = d0_idx;
  1490. do {
  1491. int slot = raid6_idx_to_slot(i, sh, &count, syndrome_disks);
  1492. offs[slot] = sh->dev[i].offset;
  1493. blocks[slot] = sh->dev[i].page;
  1494. if (i == target)
  1495. faila = slot;
  1496. if (i == target2)
  1497. failb = slot;
  1498. i = raid6_next_disk(i, disks);
  1499. } while (i != d0_idx);
  1500. BUG_ON(faila == failb);
  1501. if (failb < faila)
  1502. swap(faila, failb);
  1503. pr_debug("%s: stripe: %llu faila: %d failb: %d\n",
  1504. __func__, (unsigned long long)sh->sector, faila, failb);
  1505. atomic_inc(&sh->count);
  1506. if (failb == syndrome_disks+1) {
  1507. /* Q disk is one of the missing disks */
  1508. if (faila == syndrome_disks) {
  1509. /* Missing P+Q, just recompute */
  1510. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1511. ops_complete_compute, sh,
  1512. to_addr_conv(sh, percpu, 0));
  1513. return async_gen_syndrome(blocks, offs, syndrome_disks+2,
  1514. RAID5_STRIPE_SIZE(sh->raid_conf),
  1515. &submit);
  1516. } else {
  1517. struct page *dest;
  1518. unsigned int dest_off;
  1519. int data_target;
  1520. int qd_idx = sh->qd_idx;
  1521. /* Missing D+Q: recompute D from P, then recompute Q */
  1522. if (target == qd_idx)
  1523. data_target = target2;
  1524. else
  1525. data_target = target;
  1526. count = 0;
  1527. for (i = disks; i-- ; ) {
  1528. if (i == data_target || i == qd_idx)
  1529. continue;
  1530. offs[count] = sh->dev[i].offset;
  1531. blocks[count++] = sh->dev[i].page;
  1532. }
  1533. dest = sh->dev[data_target].page;
  1534. dest_off = sh->dev[data_target].offset;
  1535. init_async_submit(&submit,
  1536. ASYNC_TX_FENCE|ASYNC_TX_XOR_ZERO_DST,
  1537. NULL, NULL, NULL,
  1538. to_addr_conv(sh, percpu, 0));
  1539. tx = async_xor_offs(dest, dest_off, blocks, offs, count,
  1540. RAID5_STRIPE_SIZE(sh->raid_conf),
  1541. &submit);
  1542. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_ALL);
  1543. init_async_submit(&submit, ASYNC_TX_FENCE, tx,
  1544. ops_complete_compute, sh,
  1545. to_addr_conv(sh, percpu, 0));
  1546. return async_gen_syndrome(blocks, offs, count+2,
  1547. RAID5_STRIPE_SIZE(sh->raid_conf),
  1548. &submit);
  1549. }
  1550. } else {
  1551. init_async_submit(&submit, ASYNC_TX_FENCE, NULL,
  1552. ops_complete_compute, sh,
  1553. to_addr_conv(sh, percpu, 0));
  1554. if (failb == syndrome_disks) {
  1555. /* We're missing D+P. */
  1556. return async_raid6_datap_recov(syndrome_disks+2,
  1557. RAID5_STRIPE_SIZE(sh->raid_conf),
  1558. faila,
  1559. blocks, offs, &submit);
  1560. } else {
  1561. /* We're missing D+D. */
  1562. return async_raid6_2data_recov(syndrome_disks+2,
  1563. RAID5_STRIPE_SIZE(sh->raid_conf),
  1564. faila, failb,
  1565. blocks, offs, &submit);
  1566. }
  1567. }
  1568. }
  1569. static void ops_complete_prexor(void *stripe_head_ref)
  1570. {
  1571. struct stripe_head *sh = stripe_head_ref;
  1572. pr_debug("%s: stripe %llu\n", __func__,
  1573. (unsigned long long)sh->sector);
  1574. if (r5c_is_writeback(sh->raid_conf->log))
  1575. /*
  1576. * raid5-cache write back uses orig_page during prexor.
  1577. * After prexor, it is time to free orig_page
  1578. */
  1579. r5c_release_extra_page(sh);
  1580. }
  1581. static struct dma_async_tx_descriptor *
  1582. ops_run_prexor5(struct stripe_head *sh, struct raid5_percpu *percpu,
  1583. struct dma_async_tx_descriptor *tx)
  1584. {
  1585. int disks = sh->disks;
  1586. struct page **xor_srcs = to_addr_page(percpu, 0);
  1587. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1588. int count = 0, pd_idx = sh->pd_idx, i;
  1589. struct async_submit_ctl submit;
  1590. /* existing parity data subtracted */
  1591. unsigned int off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
  1592. struct page *xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1593. BUG_ON(sh->batch_head);
  1594. pr_debug("%s: stripe %llu\n", __func__,
  1595. (unsigned long long)sh->sector);
  1596. for (i = disks; i--; ) {
  1597. struct r5dev *dev = &sh->dev[i];
  1598. /* Only process blocks that are known to be uptodate */
  1599. if (test_bit(R5_InJournal, &dev->flags)) {
  1600. /*
  1601. * For this case, PAGE_SIZE must be equal to 4KB and
  1602. * page offset is zero.
  1603. */
  1604. off_srcs[count] = dev->offset;
  1605. xor_srcs[count++] = dev->orig_page;
  1606. } else if (test_bit(R5_Wantdrain, &dev->flags)) {
  1607. off_srcs[count] = dev->offset;
  1608. xor_srcs[count++] = dev->page;
  1609. }
  1610. }
  1611. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_XOR_DROP_DST, tx,
  1612. ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
  1613. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1614. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1615. return tx;
  1616. }
  1617. static struct dma_async_tx_descriptor *
  1618. ops_run_prexor6(struct stripe_head *sh, struct raid5_percpu *percpu,
  1619. struct dma_async_tx_descriptor *tx)
  1620. {
  1621. struct page **blocks = to_addr_page(percpu, 0);
  1622. unsigned int *offs = to_addr_offs(sh, percpu);
  1623. int count;
  1624. struct async_submit_ctl submit;
  1625. pr_debug("%s: stripe %llu\n", __func__,
  1626. (unsigned long long)sh->sector);
  1627. count = set_syndrome_sources(blocks, offs, sh, SYNDROME_SRC_WANT_DRAIN);
  1628. init_async_submit(&submit, ASYNC_TX_FENCE|ASYNC_TX_PQ_XOR_DST, tx,
  1629. ops_complete_prexor, sh, to_addr_conv(sh, percpu, 0));
  1630. tx = async_gen_syndrome(blocks, offs, count+2,
  1631. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1632. return tx;
  1633. }
  1634. static struct dma_async_tx_descriptor *
  1635. ops_run_biodrain(struct stripe_head *sh, struct dma_async_tx_descriptor *tx)
  1636. {
  1637. struct r5conf *conf = sh->raid_conf;
  1638. int disks = sh->disks;
  1639. int i;
  1640. struct stripe_head *head_sh = sh;
  1641. pr_debug("%s: stripe %llu\n", __func__,
  1642. (unsigned long long)sh->sector);
  1643. for (i = disks; i--; ) {
  1644. struct r5dev *dev;
  1645. struct bio *chosen;
  1646. sh = head_sh;
  1647. if (test_and_clear_bit(R5_Wantdrain, &head_sh->dev[i].flags)) {
  1648. struct bio *wbi;
  1649. again:
  1650. dev = &sh->dev[i];
  1651. /*
  1652. * clear R5_InJournal, so when rewriting a page in
  1653. * journal, it is not skipped by r5l_log_stripe()
  1654. */
  1655. clear_bit(R5_InJournal, &dev->flags);
  1656. spin_lock_irq(&sh->stripe_lock);
  1657. chosen = dev->towrite;
  1658. dev->towrite = NULL;
  1659. sh->overwrite_disks = 0;
  1660. BUG_ON(dev->written);
  1661. wbi = dev->written = chosen;
  1662. spin_unlock_irq(&sh->stripe_lock);
  1663. WARN_ON(dev->page != dev->orig_page);
  1664. while (wbi && wbi->bi_iter.bi_sector <
  1665. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  1666. if (wbi->bi_opf & REQ_FUA)
  1667. set_bit(R5_WantFUA, &dev->flags);
  1668. if (wbi->bi_opf & REQ_SYNC)
  1669. set_bit(R5_SyncIO, &dev->flags);
  1670. if (bio_op(wbi) == REQ_OP_DISCARD)
  1671. set_bit(R5_Discard, &dev->flags);
  1672. else {
  1673. tx = async_copy_data(1, wbi, &dev->page,
  1674. dev->offset,
  1675. dev->sector, tx, sh,
  1676. r5c_is_writeback(conf->log));
  1677. if (dev->page != dev->orig_page &&
  1678. !r5c_is_writeback(conf->log)) {
  1679. set_bit(R5_SkipCopy, &dev->flags);
  1680. clear_bit(R5_UPTODATE, &dev->flags);
  1681. clear_bit(R5_OVERWRITE, &dev->flags);
  1682. }
  1683. }
  1684. wbi = r5_next_bio(conf, wbi, dev->sector);
  1685. }
  1686. if (head_sh->batch_head) {
  1687. sh = list_first_entry(&sh->batch_list,
  1688. struct stripe_head,
  1689. batch_list);
  1690. if (sh == head_sh)
  1691. continue;
  1692. goto again;
  1693. }
  1694. }
  1695. }
  1696. return tx;
  1697. }
  1698. static void ops_complete_reconstruct(void *stripe_head_ref)
  1699. {
  1700. struct stripe_head *sh = stripe_head_ref;
  1701. int disks = sh->disks;
  1702. int pd_idx = sh->pd_idx;
  1703. int qd_idx = sh->qd_idx;
  1704. int i;
  1705. bool fua = false, sync = false, discard = false;
  1706. pr_debug("%s: stripe %llu\n", __func__,
  1707. (unsigned long long)sh->sector);
  1708. for (i = disks; i--; ) {
  1709. fua |= test_bit(R5_WantFUA, &sh->dev[i].flags);
  1710. sync |= test_bit(R5_SyncIO, &sh->dev[i].flags);
  1711. discard |= test_bit(R5_Discard, &sh->dev[i].flags);
  1712. }
  1713. for (i = disks; i--; ) {
  1714. struct r5dev *dev = &sh->dev[i];
  1715. if (dev->written || i == pd_idx || i == qd_idx) {
  1716. if (!discard && !test_bit(R5_SkipCopy, &dev->flags)) {
  1717. set_bit(R5_UPTODATE, &dev->flags);
  1718. if (test_bit(STRIPE_EXPAND_READY, &sh->state))
  1719. set_bit(R5_Expanded, &dev->flags);
  1720. }
  1721. if (fua)
  1722. set_bit(R5_WantFUA, &dev->flags);
  1723. if (sync)
  1724. set_bit(R5_SyncIO, &dev->flags);
  1725. }
  1726. }
  1727. if (sh->reconstruct_state == reconstruct_state_drain_run)
  1728. sh->reconstruct_state = reconstruct_state_drain_result;
  1729. else if (sh->reconstruct_state == reconstruct_state_prexor_drain_run)
  1730. sh->reconstruct_state = reconstruct_state_prexor_drain_result;
  1731. else {
  1732. BUG_ON(sh->reconstruct_state != reconstruct_state_run);
  1733. sh->reconstruct_state = reconstruct_state_result;
  1734. }
  1735. set_bit(STRIPE_HANDLE, &sh->state);
  1736. raid5_release_stripe(sh);
  1737. }
  1738. static void
  1739. ops_run_reconstruct5(struct stripe_head *sh, struct raid5_percpu *percpu,
  1740. struct dma_async_tx_descriptor *tx)
  1741. {
  1742. int disks = sh->disks;
  1743. struct page **xor_srcs;
  1744. unsigned int *off_srcs;
  1745. struct async_submit_ctl submit;
  1746. int count, pd_idx = sh->pd_idx, i;
  1747. struct page *xor_dest;
  1748. unsigned int off_dest;
  1749. int prexor = 0;
  1750. unsigned long flags;
  1751. int j = 0;
  1752. struct stripe_head *head_sh = sh;
  1753. int last_stripe;
  1754. pr_debug("%s: stripe %llu\n", __func__,
  1755. (unsigned long long)sh->sector);
  1756. for (i = 0; i < sh->disks; i++) {
  1757. if (pd_idx == i)
  1758. continue;
  1759. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1760. break;
  1761. }
  1762. if (i >= sh->disks) {
  1763. atomic_inc(&sh->count);
  1764. set_bit(R5_Discard, &sh->dev[pd_idx].flags);
  1765. ops_complete_reconstruct(sh);
  1766. return;
  1767. }
  1768. again:
  1769. count = 0;
  1770. xor_srcs = to_addr_page(percpu, j);
  1771. off_srcs = to_addr_offs(sh, percpu);
  1772. /* check if prexor is active which means only process blocks
  1773. * that are part of a read-modify-write (written)
  1774. */
  1775. if (head_sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
  1776. prexor = 1;
  1777. off_dest = off_srcs[count] = sh->dev[pd_idx].offset;
  1778. xor_dest = xor_srcs[count++] = sh->dev[pd_idx].page;
  1779. for (i = disks; i--; ) {
  1780. struct r5dev *dev = &sh->dev[i];
  1781. if (head_sh->dev[i].written ||
  1782. test_bit(R5_InJournal, &head_sh->dev[i].flags)) {
  1783. off_srcs[count] = dev->offset;
  1784. xor_srcs[count++] = dev->page;
  1785. }
  1786. }
  1787. } else {
  1788. xor_dest = sh->dev[pd_idx].page;
  1789. off_dest = sh->dev[pd_idx].offset;
  1790. for (i = disks; i--; ) {
  1791. struct r5dev *dev = &sh->dev[i];
  1792. if (i != pd_idx) {
  1793. off_srcs[count] = dev->offset;
  1794. xor_srcs[count++] = dev->page;
  1795. }
  1796. }
  1797. }
  1798. /* 1/ if we prexor'd then the dest is reused as a source
  1799. * 2/ if we did not prexor then we are redoing the parity
  1800. * set ASYNC_TX_XOR_DROP_DST and ASYNC_TX_XOR_ZERO_DST
  1801. * for the synchronous xor case
  1802. */
  1803. last_stripe = !head_sh->batch_head ||
  1804. list_first_entry(&sh->batch_list,
  1805. struct stripe_head, batch_list) == head_sh;
  1806. if (last_stripe) {
  1807. flags = ASYNC_TX_ACK |
  1808. (prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST);
  1809. atomic_inc(&head_sh->count);
  1810. init_async_submit(&submit, flags, tx, ops_complete_reconstruct, head_sh,
  1811. to_addr_conv(sh, percpu, j));
  1812. } else {
  1813. flags = prexor ? ASYNC_TX_XOR_DROP_DST : ASYNC_TX_XOR_ZERO_DST;
  1814. init_async_submit(&submit, flags, tx, NULL, NULL,
  1815. to_addr_conv(sh, percpu, j));
  1816. }
  1817. if (unlikely(count == 1))
  1818. tx = async_memcpy(xor_dest, xor_srcs[0], off_dest, off_srcs[0],
  1819. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1820. else
  1821. tx = async_xor_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1822. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1823. if (!last_stripe) {
  1824. j++;
  1825. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1826. batch_list);
  1827. goto again;
  1828. }
  1829. }
  1830. static void
  1831. ops_run_reconstruct6(struct stripe_head *sh, struct raid5_percpu *percpu,
  1832. struct dma_async_tx_descriptor *tx)
  1833. {
  1834. struct async_submit_ctl submit;
  1835. struct page **blocks;
  1836. unsigned int *offs;
  1837. int count, i, j = 0;
  1838. struct stripe_head *head_sh = sh;
  1839. int last_stripe;
  1840. int synflags;
  1841. unsigned long txflags;
  1842. pr_debug("%s: stripe %llu\n", __func__, (unsigned long long)sh->sector);
  1843. for (i = 0; i < sh->disks; i++) {
  1844. if (sh->pd_idx == i || sh->qd_idx == i)
  1845. continue;
  1846. if (!test_bit(R5_Discard, &sh->dev[i].flags))
  1847. break;
  1848. }
  1849. if (i >= sh->disks) {
  1850. atomic_inc(&sh->count);
  1851. set_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
  1852. set_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
  1853. ops_complete_reconstruct(sh);
  1854. return;
  1855. }
  1856. again:
  1857. blocks = to_addr_page(percpu, j);
  1858. offs = to_addr_offs(sh, percpu);
  1859. if (sh->reconstruct_state == reconstruct_state_prexor_drain_run) {
  1860. synflags = SYNDROME_SRC_WRITTEN;
  1861. txflags = ASYNC_TX_ACK | ASYNC_TX_PQ_XOR_DST;
  1862. } else {
  1863. synflags = SYNDROME_SRC_ALL;
  1864. txflags = ASYNC_TX_ACK;
  1865. }
  1866. count = set_syndrome_sources(blocks, offs, sh, synflags);
  1867. last_stripe = !head_sh->batch_head ||
  1868. list_first_entry(&sh->batch_list,
  1869. struct stripe_head, batch_list) == head_sh;
  1870. if (last_stripe) {
  1871. atomic_inc(&head_sh->count);
  1872. init_async_submit(&submit, txflags, tx, ops_complete_reconstruct,
  1873. head_sh, to_addr_conv(sh, percpu, j));
  1874. } else
  1875. init_async_submit(&submit, 0, tx, NULL, NULL,
  1876. to_addr_conv(sh, percpu, j));
  1877. tx = async_gen_syndrome(blocks, offs, count+2,
  1878. RAID5_STRIPE_SIZE(sh->raid_conf), &submit);
  1879. if (!last_stripe) {
  1880. j++;
  1881. sh = list_first_entry(&sh->batch_list, struct stripe_head,
  1882. batch_list);
  1883. goto again;
  1884. }
  1885. }
  1886. static void ops_complete_check(void *stripe_head_ref)
  1887. {
  1888. struct stripe_head *sh = stripe_head_ref;
  1889. pr_debug("%s: stripe %llu\n", __func__,
  1890. (unsigned long long)sh->sector);
  1891. sh->check_state = check_state_check_result;
  1892. set_bit(STRIPE_HANDLE, &sh->state);
  1893. raid5_release_stripe(sh);
  1894. }
  1895. static void ops_run_check_p(struct stripe_head *sh, struct raid5_percpu *percpu)
  1896. {
  1897. int disks = sh->disks;
  1898. int pd_idx = sh->pd_idx;
  1899. int qd_idx = sh->qd_idx;
  1900. struct page *xor_dest;
  1901. unsigned int off_dest;
  1902. struct page **xor_srcs = to_addr_page(percpu, 0);
  1903. unsigned int *off_srcs = to_addr_offs(sh, percpu);
  1904. struct dma_async_tx_descriptor *tx;
  1905. struct async_submit_ctl submit;
  1906. int count;
  1907. int i;
  1908. pr_debug("%s: stripe %llu\n", __func__,
  1909. (unsigned long long)sh->sector);
  1910. BUG_ON(sh->batch_head);
  1911. count = 0;
  1912. xor_dest = sh->dev[pd_idx].page;
  1913. off_dest = sh->dev[pd_idx].offset;
  1914. off_srcs[count] = off_dest;
  1915. xor_srcs[count++] = xor_dest;
  1916. for (i = disks; i--; ) {
  1917. if (i == pd_idx || i == qd_idx)
  1918. continue;
  1919. off_srcs[count] = sh->dev[i].offset;
  1920. xor_srcs[count++] = sh->dev[i].page;
  1921. }
  1922. init_async_submit(&submit, 0, NULL, NULL, NULL,
  1923. to_addr_conv(sh, percpu, 0));
  1924. tx = async_xor_val_offs(xor_dest, off_dest, xor_srcs, off_srcs, count,
  1925. RAID5_STRIPE_SIZE(sh->raid_conf),
  1926. &sh->ops.zero_sum_result, &submit);
  1927. atomic_inc(&sh->count);
  1928. init_async_submit(&submit, ASYNC_TX_ACK, tx, ops_complete_check, sh, NULL);
  1929. tx = async_trigger_callback(&submit);
  1930. }
  1931. static void ops_run_check_pq(struct stripe_head *sh, struct raid5_percpu *percpu, int checkp)
  1932. {
  1933. struct page **srcs = to_addr_page(percpu, 0);
  1934. unsigned int *offs = to_addr_offs(sh, percpu);
  1935. struct async_submit_ctl submit;
  1936. int count;
  1937. pr_debug("%s: stripe %llu checkp: %d\n", __func__,
  1938. (unsigned long long)sh->sector, checkp);
  1939. BUG_ON(sh->batch_head);
  1940. count = set_syndrome_sources(srcs, offs, sh, SYNDROME_SRC_ALL);
  1941. if (!checkp)
  1942. srcs[count] = NULL;
  1943. atomic_inc(&sh->count);
  1944. init_async_submit(&submit, ASYNC_TX_ACK, NULL, ops_complete_check,
  1945. sh, to_addr_conv(sh, percpu, 0));
  1946. async_syndrome_val(srcs, offs, count+2,
  1947. RAID5_STRIPE_SIZE(sh->raid_conf),
  1948. &sh->ops.zero_sum_result, percpu->spare_page, 0, &submit);
  1949. }
  1950. static void raid_run_ops(struct stripe_head *sh, unsigned long ops_request)
  1951. {
  1952. int overlap_clear = 0, i, disks = sh->disks;
  1953. struct dma_async_tx_descriptor *tx = NULL;
  1954. struct r5conf *conf = sh->raid_conf;
  1955. int level = conf->level;
  1956. struct raid5_percpu *percpu;
  1957. unsigned long cpu;
  1958. cpu = get_cpu();
  1959. percpu = per_cpu_ptr(conf->percpu, cpu);
  1960. if (test_bit(STRIPE_OP_BIOFILL, &ops_request)) {
  1961. ops_run_biofill(sh);
  1962. overlap_clear++;
  1963. }
  1964. if (test_bit(STRIPE_OP_COMPUTE_BLK, &ops_request)) {
  1965. if (level < 6)
  1966. tx = ops_run_compute5(sh, percpu);
  1967. else {
  1968. if (sh->ops.target2 < 0 || sh->ops.target < 0)
  1969. tx = ops_run_compute6_1(sh, percpu);
  1970. else
  1971. tx = ops_run_compute6_2(sh, percpu);
  1972. }
  1973. /* terminate the chain if reconstruct is not set to be run */
  1974. if (tx && !test_bit(STRIPE_OP_RECONSTRUCT, &ops_request))
  1975. async_tx_ack(tx);
  1976. }
  1977. if (test_bit(STRIPE_OP_PREXOR, &ops_request)) {
  1978. if (level < 6)
  1979. tx = ops_run_prexor5(sh, percpu, tx);
  1980. else
  1981. tx = ops_run_prexor6(sh, percpu, tx);
  1982. }
  1983. if (test_bit(STRIPE_OP_PARTIAL_PARITY, &ops_request))
  1984. tx = ops_run_partial_parity(sh, percpu, tx);
  1985. if (test_bit(STRIPE_OP_BIODRAIN, &ops_request)) {
  1986. tx = ops_run_biodrain(sh, tx);
  1987. overlap_clear++;
  1988. }
  1989. if (test_bit(STRIPE_OP_RECONSTRUCT, &ops_request)) {
  1990. if (level < 6)
  1991. ops_run_reconstruct5(sh, percpu, tx);
  1992. else
  1993. ops_run_reconstruct6(sh, percpu, tx);
  1994. }
  1995. if (test_bit(STRIPE_OP_CHECK, &ops_request)) {
  1996. if (sh->check_state == check_state_run)
  1997. ops_run_check_p(sh, percpu);
  1998. else if (sh->check_state == check_state_run_q)
  1999. ops_run_check_pq(sh, percpu, 0);
  2000. else if (sh->check_state == check_state_run_pq)
  2001. ops_run_check_pq(sh, percpu, 1);
  2002. else
  2003. BUG();
  2004. }
  2005. if (overlap_clear && !sh->batch_head)
  2006. for (i = disks; i--; ) {
  2007. struct r5dev *dev = &sh->dev[i];
  2008. if (test_and_clear_bit(R5_Overlap, &dev->flags))
  2009. wake_up(&sh->raid_conf->wait_for_overlap);
  2010. }
  2011. put_cpu();
  2012. }
  2013. static void free_stripe(struct kmem_cache *sc, struct stripe_head *sh)
  2014. {
  2015. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2016. kfree(sh->pages);
  2017. #endif
  2018. if (sh->ppl_page)
  2019. __free_page(sh->ppl_page);
  2020. kmem_cache_free(sc, sh);
  2021. }
  2022. static struct stripe_head *alloc_stripe(struct kmem_cache *sc, gfp_t gfp,
  2023. int disks, struct r5conf *conf)
  2024. {
  2025. struct stripe_head *sh;
  2026. int i;
  2027. sh = kmem_cache_zalloc(sc, gfp);
  2028. if (sh) {
  2029. spin_lock_init(&sh->stripe_lock);
  2030. spin_lock_init(&sh->batch_lock);
  2031. INIT_LIST_HEAD(&sh->batch_list);
  2032. INIT_LIST_HEAD(&sh->lru);
  2033. INIT_LIST_HEAD(&sh->r5c);
  2034. INIT_LIST_HEAD(&sh->log_list);
  2035. atomic_set(&sh->count, 1);
  2036. sh->raid_conf = conf;
  2037. sh->log_start = MaxSector;
  2038. for (i = 0; i < disks; i++) {
  2039. struct r5dev *dev = &sh->dev[i];
  2040. bio_init(&dev->req, &dev->vec, 1);
  2041. bio_init(&dev->rreq, &dev->rvec, 1);
  2042. }
  2043. if (raid5_has_ppl(conf)) {
  2044. sh->ppl_page = alloc_page(gfp);
  2045. if (!sh->ppl_page) {
  2046. free_stripe(sc, sh);
  2047. return NULL;
  2048. }
  2049. }
  2050. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2051. if (init_stripe_shared_pages(sh, conf, disks)) {
  2052. free_stripe(sc, sh);
  2053. return NULL;
  2054. }
  2055. #endif
  2056. }
  2057. return sh;
  2058. }
  2059. static int grow_one_stripe(struct r5conf *conf, gfp_t gfp)
  2060. {
  2061. struct stripe_head *sh;
  2062. sh = alloc_stripe(conf->slab_cache, gfp, conf->pool_size, conf);
  2063. if (!sh)
  2064. return 0;
  2065. if (grow_buffers(sh, gfp)) {
  2066. shrink_buffers(sh);
  2067. free_stripe(conf->slab_cache, sh);
  2068. return 0;
  2069. }
  2070. sh->hash_lock_index =
  2071. conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS;
  2072. /* we just created an active stripe so... */
  2073. atomic_inc(&conf->active_stripes);
  2074. raid5_release_stripe(sh);
  2075. conf->max_nr_stripes++;
  2076. return 1;
  2077. }
  2078. static int grow_stripes(struct r5conf *conf, int num)
  2079. {
  2080. struct kmem_cache *sc;
  2081. size_t namelen = sizeof(conf->cache_name[0]);
  2082. int devs = max(conf->raid_disks, conf->previous_raid_disks);
  2083. if (conf->mddev->gendisk)
  2084. snprintf(conf->cache_name[0], namelen,
  2085. "raid%d-%s", conf->level, mdname(conf->mddev));
  2086. else
  2087. snprintf(conf->cache_name[0], namelen,
  2088. "raid%d-%p", conf->level, conf->mddev);
  2089. snprintf(conf->cache_name[1], namelen, "%.27s-alt", conf->cache_name[0]);
  2090. conf->active_name = 0;
  2091. sc = kmem_cache_create(conf->cache_name[conf->active_name],
  2092. sizeof(struct stripe_head)+(devs-1)*sizeof(struct r5dev),
  2093. 0, 0, NULL);
  2094. if (!sc)
  2095. return 1;
  2096. conf->slab_cache = sc;
  2097. conf->pool_size = devs;
  2098. while (num--)
  2099. if (!grow_one_stripe(conf, GFP_KERNEL))
  2100. return 1;
  2101. return 0;
  2102. }
  2103. /**
  2104. * scribble_alloc - allocate percpu scribble buffer for required size
  2105. * of the scribble region
  2106. * @percpu: from for_each_present_cpu() of the caller
  2107. * @num: total number of disks in the array
  2108. * @cnt: scribble objs count for required size of the scribble region
  2109. *
  2110. * The scribble buffer size must be enough to contain:
  2111. * 1/ a struct page pointer for each device in the array +2
  2112. * 2/ room to convert each entry in (1) to its corresponding dma
  2113. * (dma_map_page()) or page (page_address()) address.
  2114. *
  2115. * Note: the +2 is for the destination buffers of the ddf/raid6 case where we
  2116. * calculate over all devices (not just the data blocks), using zeros in place
  2117. * of the P and Q blocks.
  2118. */
  2119. static int scribble_alloc(struct raid5_percpu *percpu,
  2120. int num, int cnt)
  2121. {
  2122. size_t obj_size =
  2123. sizeof(struct page *) * (num + 2) +
  2124. sizeof(addr_conv_t) * (num + 2) +
  2125. sizeof(unsigned int) * (num + 2);
  2126. void *scribble;
  2127. /*
  2128. * If here is in raid array suspend context, it is in memalloc noio
  2129. * context as well, there is no potential recursive memory reclaim
  2130. * I/Os with the GFP_KERNEL flag.
  2131. */
  2132. scribble = kvmalloc_array(cnt, obj_size, GFP_KERNEL);
  2133. if (!scribble)
  2134. return -ENOMEM;
  2135. kvfree(percpu->scribble);
  2136. percpu->scribble = scribble;
  2137. percpu->scribble_obj_size = obj_size;
  2138. return 0;
  2139. }
  2140. static int resize_chunks(struct r5conf *conf, int new_disks, int new_sectors)
  2141. {
  2142. unsigned long cpu;
  2143. int err = 0;
  2144. /*
  2145. * Never shrink. And mddev_suspend() could deadlock if this is called
  2146. * from raid5d. In that case, scribble_disks and scribble_sectors
  2147. * should equal to new_disks and new_sectors
  2148. */
  2149. if (conf->scribble_disks >= new_disks &&
  2150. conf->scribble_sectors >= new_sectors)
  2151. return 0;
  2152. mddev_suspend(conf->mddev);
  2153. get_online_cpus();
  2154. for_each_present_cpu(cpu) {
  2155. struct raid5_percpu *percpu;
  2156. percpu = per_cpu_ptr(conf->percpu, cpu);
  2157. err = scribble_alloc(percpu, new_disks,
  2158. new_sectors / RAID5_STRIPE_SECTORS(conf));
  2159. if (err)
  2160. break;
  2161. }
  2162. put_online_cpus();
  2163. mddev_resume(conf->mddev);
  2164. if (!err) {
  2165. conf->scribble_disks = new_disks;
  2166. conf->scribble_sectors = new_sectors;
  2167. }
  2168. return err;
  2169. }
  2170. static int resize_stripes(struct r5conf *conf, int newsize)
  2171. {
  2172. /* Make all the stripes able to hold 'newsize' devices.
  2173. * New slots in each stripe get 'page' set to a new page.
  2174. *
  2175. * This happens in stages:
  2176. * 1/ create a new kmem_cache and allocate the required number of
  2177. * stripe_heads.
  2178. * 2/ gather all the old stripe_heads and transfer the pages across
  2179. * to the new stripe_heads. This will have the side effect of
  2180. * freezing the array as once all stripe_heads have been collected,
  2181. * no IO will be possible. Old stripe heads are freed once their
  2182. * pages have been transferred over, and the old kmem_cache is
  2183. * freed when all stripes are done.
  2184. * 3/ reallocate conf->disks to be suitable bigger. If this fails,
  2185. * we simple return a failure status - no need to clean anything up.
  2186. * 4/ allocate new pages for the new slots in the new stripe_heads.
  2187. * If this fails, we don't bother trying the shrink the
  2188. * stripe_heads down again, we just leave them as they are.
  2189. * As each stripe_head is processed the new one is released into
  2190. * active service.
  2191. *
  2192. * Once step2 is started, we cannot afford to wait for a write,
  2193. * so we use GFP_NOIO allocations.
  2194. */
  2195. struct stripe_head *osh, *nsh;
  2196. LIST_HEAD(newstripes);
  2197. struct disk_info *ndisks;
  2198. int err = 0;
  2199. struct kmem_cache *sc;
  2200. int i;
  2201. int hash, cnt;
  2202. md_allow_write(conf->mddev);
  2203. /* Step 1 */
  2204. sc = kmem_cache_create(conf->cache_name[1-conf->active_name],
  2205. sizeof(struct stripe_head)+(newsize-1)*sizeof(struct r5dev),
  2206. 0, 0, NULL);
  2207. if (!sc)
  2208. return -ENOMEM;
  2209. /* Need to ensure auto-resizing doesn't interfere */
  2210. mutex_lock(&conf->cache_size_mutex);
  2211. for (i = conf->max_nr_stripes; i; i--) {
  2212. nsh = alloc_stripe(sc, GFP_KERNEL, newsize, conf);
  2213. if (!nsh)
  2214. break;
  2215. list_add(&nsh->lru, &newstripes);
  2216. }
  2217. if (i) {
  2218. /* didn't get enough, give up */
  2219. while (!list_empty(&newstripes)) {
  2220. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  2221. list_del(&nsh->lru);
  2222. free_stripe(sc, nsh);
  2223. }
  2224. kmem_cache_destroy(sc);
  2225. mutex_unlock(&conf->cache_size_mutex);
  2226. return -ENOMEM;
  2227. }
  2228. /* Step 2 - Must use GFP_NOIO now.
  2229. * OK, we have enough stripes, start collecting inactive
  2230. * stripes and copying them over
  2231. */
  2232. hash = 0;
  2233. cnt = 0;
  2234. list_for_each_entry(nsh, &newstripes, lru) {
  2235. lock_device_hash_lock(conf, hash);
  2236. wait_event_cmd(conf->wait_for_stripe,
  2237. !list_empty(conf->inactive_list + hash),
  2238. unlock_device_hash_lock(conf, hash),
  2239. lock_device_hash_lock(conf, hash));
  2240. osh = get_free_stripe(conf, hash);
  2241. unlock_device_hash_lock(conf, hash);
  2242. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2243. for (i = 0; i < osh->nr_pages; i++) {
  2244. nsh->pages[i] = osh->pages[i];
  2245. osh->pages[i] = NULL;
  2246. }
  2247. #endif
  2248. for(i=0; i<conf->pool_size; i++) {
  2249. nsh->dev[i].page = osh->dev[i].page;
  2250. nsh->dev[i].orig_page = osh->dev[i].page;
  2251. nsh->dev[i].offset = osh->dev[i].offset;
  2252. }
  2253. nsh->hash_lock_index = hash;
  2254. free_stripe(conf->slab_cache, osh);
  2255. cnt++;
  2256. if (cnt >= conf->max_nr_stripes / NR_STRIPE_HASH_LOCKS +
  2257. !!((conf->max_nr_stripes % NR_STRIPE_HASH_LOCKS) > hash)) {
  2258. hash++;
  2259. cnt = 0;
  2260. }
  2261. }
  2262. kmem_cache_destroy(conf->slab_cache);
  2263. /* Step 3.
  2264. * At this point, we are holding all the stripes so the array
  2265. * is completely stalled, so now is a good time to resize
  2266. * conf->disks and the scribble region
  2267. */
  2268. ndisks = kcalloc(newsize, sizeof(struct disk_info), GFP_NOIO);
  2269. if (ndisks) {
  2270. for (i = 0; i < conf->pool_size; i++)
  2271. ndisks[i] = conf->disks[i];
  2272. for (i = conf->pool_size; i < newsize; i++) {
  2273. ndisks[i].extra_page = alloc_page(GFP_NOIO);
  2274. if (!ndisks[i].extra_page)
  2275. err = -ENOMEM;
  2276. }
  2277. if (err) {
  2278. for (i = conf->pool_size; i < newsize; i++)
  2279. if (ndisks[i].extra_page)
  2280. put_page(ndisks[i].extra_page);
  2281. kfree(ndisks);
  2282. } else {
  2283. kfree(conf->disks);
  2284. conf->disks = ndisks;
  2285. }
  2286. } else
  2287. err = -ENOMEM;
  2288. conf->slab_cache = sc;
  2289. conf->active_name = 1-conf->active_name;
  2290. /* Step 4, return new stripes to service */
  2291. while(!list_empty(&newstripes)) {
  2292. nsh = list_entry(newstripes.next, struct stripe_head, lru);
  2293. list_del_init(&nsh->lru);
  2294. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  2295. for (i = 0; i < nsh->nr_pages; i++) {
  2296. if (nsh->pages[i])
  2297. continue;
  2298. nsh->pages[i] = alloc_page(GFP_NOIO);
  2299. if (!nsh->pages[i])
  2300. err = -ENOMEM;
  2301. }
  2302. for (i = conf->raid_disks; i < newsize; i++) {
  2303. if (nsh->dev[i].page)
  2304. continue;
  2305. nsh->dev[i].page = raid5_get_dev_page(nsh, i);
  2306. nsh->dev[i].orig_page = nsh->dev[i].page;
  2307. nsh->dev[i].offset = raid5_get_page_offset(nsh, i);
  2308. }
  2309. #else
  2310. for (i=conf->raid_disks; i < newsize; i++)
  2311. if (nsh->dev[i].page == NULL) {
  2312. struct page *p = alloc_page(GFP_NOIO);
  2313. nsh->dev[i].page = p;
  2314. nsh->dev[i].orig_page = p;
  2315. nsh->dev[i].offset = 0;
  2316. if (!p)
  2317. err = -ENOMEM;
  2318. }
  2319. #endif
  2320. raid5_release_stripe(nsh);
  2321. }
  2322. /* critical section pass, GFP_NOIO no longer needed */
  2323. if (!err)
  2324. conf->pool_size = newsize;
  2325. mutex_unlock(&conf->cache_size_mutex);
  2326. return err;
  2327. }
  2328. static int drop_one_stripe(struct r5conf *conf)
  2329. {
  2330. struct stripe_head *sh;
  2331. int hash = (conf->max_nr_stripes - 1) & STRIPE_HASH_LOCKS_MASK;
  2332. spin_lock_irq(conf->hash_locks + hash);
  2333. sh = get_free_stripe(conf, hash);
  2334. spin_unlock_irq(conf->hash_locks + hash);
  2335. if (!sh)
  2336. return 0;
  2337. BUG_ON(atomic_read(&sh->count));
  2338. shrink_buffers(sh);
  2339. free_stripe(conf->slab_cache, sh);
  2340. atomic_dec(&conf->active_stripes);
  2341. conf->max_nr_stripes--;
  2342. return 1;
  2343. }
  2344. static void shrink_stripes(struct r5conf *conf)
  2345. {
  2346. while (conf->max_nr_stripes &&
  2347. drop_one_stripe(conf))
  2348. ;
  2349. kmem_cache_destroy(conf->slab_cache);
  2350. conf->slab_cache = NULL;
  2351. }
  2352. static void raid5_end_read_request(struct bio * bi)
  2353. {
  2354. struct stripe_head *sh = bi->bi_private;
  2355. struct r5conf *conf = sh->raid_conf;
  2356. int disks = sh->disks, i;
  2357. char b[BDEVNAME_SIZE];
  2358. struct md_rdev *rdev = NULL;
  2359. sector_t s;
  2360. for (i=0 ; i<disks; i++)
  2361. if (bi == &sh->dev[i].req)
  2362. break;
  2363. pr_debug("end_read_request %llu/%d, count: %d, error %d.\n",
  2364. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  2365. bi->bi_status);
  2366. if (i == disks) {
  2367. bio_reset(bi);
  2368. BUG();
  2369. return;
  2370. }
  2371. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  2372. /* If replacement finished while this request was outstanding,
  2373. * 'replacement' might be NULL already.
  2374. * In that case it moved down to 'rdev'.
  2375. * rdev is not removed until all requests are finished.
  2376. */
  2377. rdev = conf->disks[i].replacement;
  2378. if (!rdev)
  2379. rdev = conf->disks[i].rdev;
  2380. if (use_new_offset(conf, sh))
  2381. s = sh->sector + rdev->new_data_offset;
  2382. else
  2383. s = sh->sector + rdev->data_offset;
  2384. if (!bi->bi_status) {
  2385. set_bit(R5_UPTODATE, &sh->dev[i].flags);
  2386. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  2387. /* Note that this cannot happen on a
  2388. * replacement device. We just fail those on
  2389. * any error
  2390. */
  2391. pr_info_ratelimited(
  2392. "md/raid:%s: read error corrected (%lu sectors at %llu on %s)\n",
  2393. mdname(conf->mddev), RAID5_STRIPE_SECTORS(conf),
  2394. (unsigned long long)s,
  2395. bdevname(rdev->bdev, b));
  2396. atomic_add(RAID5_STRIPE_SECTORS(conf), &rdev->corrected_errors);
  2397. clear_bit(R5_ReadError, &sh->dev[i].flags);
  2398. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  2399. } else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  2400. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2401. if (test_bit(R5_InJournal, &sh->dev[i].flags))
  2402. /*
  2403. * end read for a page in journal, this
  2404. * must be preparing for prexor in rmw
  2405. */
  2406. set_bit(R5_OrigPageUPTDODATE, &sh->dev[i].flags);
  2407. if (atomic_read(&rdev->read_errors))
  2408. atomic_set(&rdev->read_errors, 0);
  2409. } else {
  2410. const char *bdn = bdevname(rdev->bdev, b);
  2411. int retry = 0;
  2412. int set_bad = 0;
  2413. clear_bit(R5_UPTODATE, &sh->dev[i].flags);
  2414. if (!(bi->bi_status == BLK_STS_PROTECTION))
  2415. atomic_inc(&rdev->read_errors);
  2416. if (test_bit(R5_ReadRepl, &sh->dev[i].flags))
  2417. pr_warn_ratelimited(
  2418. "md/raid:%s: read error on replacement device (sector %llu on %s).\n",
  2419. mdname(conf->mddev),
  2420. (unsigned long long)s,
  2421. bdn);
  2422. else if (conf->mddev->degraded >= conf->max_degraded) {
  2423. set_bad = 1;
  2424. pr_warn_ratelimited(
  2425. "md/raid:%s: read error not correctable (sector %llu on %s).\n",
  2426. mdname(conf->mddev),
  2427. (unsigned long long)s,
  2428. bdn);
  2429. } else if (test_bit(R5_ReWrite, &sh->dev[i].flags)) {
  2430. /* Oh, no!!! */
  2431. set_bad = 1;
  2432. pr_warn_ratelimited(
  2433. "md/raid:%s: read error NOT corrected!! (sector %llu on %s).\n",
  2434. mdname(conf->mddev),
  2435. (unsigned long long)s,
  2436. bdn);
  2437. } else if (atomic_read(&rdev->read_errors)
  2438. > conf->max_nr_stripes) {
  2439. if (!test_bit(Faulty, &rdev->flags)) {
  2440. pr_warn("md/raid:%s: %d read_errors > %d stripes\n",
  2441. mdname(conf->mddev),
  2442. atomic_read(&rdev->read_errors),
  2443. conf->max_nr_stripes);
  2444. pr_warn("md/raid:%s: Too many read errors, failing device %s.\n",
  2445. mdname(conf->mddev), bdn);
  2446. }
  2447. } else
  2448. retry = 1;
  2449. if (set_bad && test_bit(In_sync, &rdev->flags)
  2450. && !test_bit(R5_ReadNoMerge, &sh->dev[i].flags))
  2451. retry = 1;
  2452. if (retry)
  2453. if (sh->qd_idx >= 0 && sh->pd_idx == i)
  2454. set_bit(R5_ReadError, &sh->dev[i].flags);
  2455. else if (test_bit(R5_ReadNoMerge, &sh->dev[i].flags)) {
  2456. set_bit(R5_ReadError, &sh->dev[i].flags);
  2457. clear_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2458. } else
  2459. set_bit(R5_ReadNoMerge, &sh->dev[i].flags);
  2460. else {
  2461. clear_bit(R5_ReadError, &sh->dev[i].flags);
  2462. clear_bit(R5_ReWrite, &sh->dev[i].flags);
  2463. if (!(set_bad
  2464. && test_bit(In_sync, &rdev->flags)
  2465. && rdev_set_badblocks(
  2466. rdev, sh->sector, RAID5_STRIPE_SECTORS(conf), 0)))
  2467. md_error(conf->mddev, rdev);
  2468. }
  2469. }
  2470. rdev_dec_pending(rdev, conf->mddev);
  2471. bio_reset(bi);
  2472. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  2473. set_bit(STRIPE_HANDLE, &sh->state);
  2474. raid5_release_stripe(sh);
  2475. }
  2476. static void raid5_end_write_request(struct bio *bi)
  2477. {
  2478. struct stripe_head *sh = bi->bi_private;
  2479. struct r5conf *conf = sh->raid_conf;
  2480. int disks = sh->disks, i;
  2481. struct md_rdev *rdev;
  2482. sector_t first_bad;
  2483. int bad_sectors;
  2484. int replacement = 0;
  2485. for (i = 0 ; i < disks; i++) {
  2486. if (bi == &sh->dev[i].req) {
  2487. rdev = conf->disks[i].rdev;
  2488. break;
  2489. }
  2490. if (bi == &sh->dev[i].rreq) {
  2491. rdev = conf->disks[i].replacement;
  2492. if (rdev)
  2493. replacement = 1;
  2494. else
  2495. /* rdev was removed and 'replacement'
  2496. * replaced it. rdev is not removed
  2497. * until all requests are finished.
  2498. */
  2499. rdev = conf->disks[i].rdev;
  2500. break;
  2501. }
  2502. }
  2503. pr_debug("end_write_request %llu/%d, count %d, error: %d.\n",
  2504. (unsigned long long)sh->sector, i, atomic_read(&sh->count),
  2505. bi->bi_status);
  2506. if (i == disks) {
  2507. bio_reset(bi);
  2508. BUG();
  2509. return;
  2510. }
  2511. if (replacement) {
  2512. if (bi->bi_status)
  2513. md_error(conf->mddev, rdev);
  2514. else if (is_badblock(rdev, sh->sector,
  2515. RAID5_STRIPE_SECTORS(conf),
  2516. &first_bad, &bad_sectors))
  2517. set_bit(R5_MadeGoodRepl, &sh->dev[i].flags);
  2518. } else {
  2519. if (bi->bi_status) {
  2520. set_bit(STRIPE_DEGRADED, &sh->state);
  2521. set_bit(WriteErrorSeen, &rdev->flags);
  2522. set_bit(R5_WriteError, &sh->dev[i].flags);
  2523. if (!test_and_set_bit(WantReplacement, &rdev->flags))
  2524. set_bit(MD_RECOVERY_NEEDED,
  2525. &rdev->mddev->recovery);
  2526. } else if (is_badblock(rdev, sh->sector,
  2527. RAID5_STRIPE_SECTORS(conf),
  2528. &first_bad, &bad_sectors)) {
  2529. set_bit(R5_MadeGood, &sh->dev[i].flags);
  2530. if (test_bit(R5_ReadError, &sh->dev[i].flags))
  2531. /* That was a successful write so make
  2532. * sure it looks like we already did
  2533. * a re-write.
  2534. */
  2535. set_bit(R5_ReWrite, &sh->dev[i].flags);
  2536. }
  2537. }
  2538. rdev_dec_pending(rdev, conf->mddev);
  2539. if (sh->batch_head && bi->bi_status && !replacement)
  2540. set_bit(STRIPE_BATCH_ERR, &sh->batch_head->state);
  2541. bio_reset(bi);
  2542. if (!test_and_clear_bit(R5_DOUBLE_LOCKED, &sh->dev[i].flags))
  2543. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  2544. set_bit(STRIPE_HANDLE, &sh->state);
  2545. raid5_release_stripe(sh);
  2546. if (sh->batch_head && sh != sh->batch_head)
  2547. raid5_release_stripe(sh->batch_head);
  2548. }
  2549. static void raid5_error(struct mddev *mddev, struct md_rdev *rdev)
  2550. {
  2551. char b[BDEVNAME_SIZE];
  2552. struct r5conf *conf = mddev->private;
  2553. unsigned long flags;
  2554. pr_debug("raid456: error called\n");
  2555. spin_lock_irqsave(&conf->device_lock, flags);
  2556. if (test_bit(In_sync, &rdev->flags) &&
  2557. mddev->degraded == conf->max_degraded) {
  2558. /*
  2559. * Don't allow to achieve failed state
  2560. * Don't try to recover this device
  2561. */
  2562. conf->recovery_disabled = mddev->recovery_disabled;
  2563. spin_unlock_irqrestore(&conf->device_lock, flags);
  2564. return;
  2565. }
  2566. set_bit(Faulty, &rdev->flags);
  2567. clear_bit(In_sync, &rdev->flags);
  2568. mddev->degraded = raid5_calc_degraded(conf);
  2569. spin_unlock_irqrestore(&conf->device_lock, flags);
  2570. set_bit(MD_RECOVERY_INTR, &mddev->recovery);
  2571. set_bit(Blocked, &rdev->flags);
  2572. set_mask_bits(&mddev->sb_flags, 0,
  2573. BIT(MD_SB_CHANGE_DEVS) | BIT(MD_SB_CHANGE_PENDING));
  2574. pr_crit("md/raid:%s: Disk failure on %s, disabling device.\n"
  2575. "md/raid:%s: Operation continuing on %d devices.\n",
  2576. mdname(mddev),
  2577. bdevname(rdev->bdev, b),
  2578. mdname(mddev),
  2579. conf->raid_disks - mddev->degraded);
  2580. r5c_update_on_rdev_error(mddev, rdev);
  2581. }
  2582. /*
  2583. * Input: a 'big' sector number,
  2584. * Output: index of the data and parity disk, and the sector # in them.
  2585. */
  2586. sector_t raid5_compute_sector(struct r5conf *conf, sector_t r_sector,
  2587. int previous, int *dd_idx,
  2588. struct stripe_head *sh)
  2589. {
  2590. sector_t stripe, stripe2;
  2591. sector_t chunk_number;
  2592. unsigned int chunk_offset;
  2593. int pd_idx, qd_idx;
  2594. int ddf_layout = 0;
  2595. sector_t new_sector;
  2596. int algorithm = previous ? conf->prev_algo
  2597. : conf->algorithm;
  2598. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  2599. : conf->chunk_sectors;
  2600. int raid_disks = previous ? conf->previous_raid_disks
  2601. : conf->raid_disks;
  2602. int data_disks = raid_disks - conf->max_degraded;
  2603. /* First compute the information on this sector */
  2604. /*
  2605. * Compute the chunk number and the sector offset inside the chunk
  2606. */
  2607. chunk_offset = sector_div(r_sector, sectors_per_chunk);
  2608. chunk_number = r_sector;
  2609. /*
  2610. * Compute the stripe number
  2611. */
  2612. stripe = chunk_number;
  2613. *dd_idx = sector_div(stripe, data_disks);
  2614. stripe2 = stripe;
  2615. /*
  2616. * Select the parity disk based on the user selected algorithm.
  2617. */
  2618. pd_idx = qd_idx = -1;
  2619. switch(conf->level) {
  2620. case 4:
  2621. pd_idx = data_disks;
  2622. break;
  2623. case 5:
  2624. switch (algorithm) {
  2625. case ALGORITHM_LEFT_ASYMMETRIC:
  2626. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  2627. if (*dd_idx >= pd_idx)
  2628. (*dd_idx)++;
  2629. break;
  2630. case ALGORITHM_RIGHT_ASYMMETRIC:
  2631. pd_idx = sector_div(stripe2, raid_disks);
  2632. if (*dd_idx >= pd_idx)
  2633. (*dd_idx)++;
  2634. break;
  2635. case ALGORITHM_LEFT_SYMMETRIC:
  2636. pd_idx = data_disks - sector_div(stripe2, raid_disks);
  2637. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2638. break;
  2639. case ALGORITHM_RIGHT_SYMMETRIC:
  2640. pd_idx = sector_div(stripe2, raid_disks);
  2641. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2642. break;
  2643. case ALGORITHM_PARITY_0:
  2644. pd_idx = 0;
  2645. (*dd_idx)++;
  2646. break;
  2647. case ALGORITHM_PARITY_N:
  2648. pd_idx = data_disks;
  2649. break;
  2650. default:
  2651. BUG();
  2652. }
  2653. break;
  2654. case 6:
  2655. switch (algorithm) {
  2656. case ALGORITHM_LEFT_ASYMMETRIC:
  2657. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2658. qd_idx = pd_idx + 1;
  2659. if (pd_idx == raid_disks-1) {
  2660. (*dd_idx)++; /* Q D D D P */
  2661. qd_idx = 0;
  2662. } else if (*dd_idx >= pd_idx)
  2663. (*dd_idx) += 2; /* D D P Q D */
  2664. break;
  2665. case ALGORITHM_RIGHT_ASYMMETRIC:
  2666. pd_idx = sector_div(stripe2, raid_disks);
  2667. qd_idx = pd_idx + 1;
  2668. if (pd_idx == raid_disks-1) {
  2669. (*dd_idx)++; /* Q D D D P */
  2670. qd_idx = 0;
  2671. } else if (*dd_idx >= pd_idx)
  2672. (*dd_idx) += 2; /* D D P Q D */
  2673. break;
  2674. case ALGORITHM_LEFT_SYMMETRIC:
  2675. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2676. qd_idx = (pd_idx + 1) % raid_disks;
  2677. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  2678. break;
  2679. case ALGORITHM_RIGHT_SYMMETRIC:
  2680. pd_idx = sector_div(stripe2, raid_disks);
  2681. qd_idx = (pd_idx + 1) % raid_disks;
  2682. *dd_idx = (pd_idx + 2 + *dd_idx) % raid_disks;
  2683. break;
  2684. case ALGORITHM_PARITY_0:
  2685. pd_idx = 0;
  2686. qd_idx = 1;
  2687. (*dd_idx) += 2;
  2688. break;
  2689. case ALGORITHM_PARITY_N:
  2690. pd_idx = data_disks;
  2691. qd_idx = data_disks + 1;
  2692. break;
  2693. case ALGORITHM_ROTATING_ZERO_RESTART:
  2694. /* Exactly the same as RIGHT_ASYMMETRIC, but or
  2695. * of blocks for computing Q is different.
  2696. */
  2697. pd_idx = sector_div(stripe2, raid_disks);
  2698. qd_idx = pd_idx + 1;
  2699. if (pd_idx == raid_disks-1) {
  2700. (*dd_idx)++; /* Q D D D P */
  2701. qd_idx = 0;
  2702. } else if (*dd_idx >= pd_idx)
  2703. (*dd_idx) += 2; /* D D P Q D */
  2704. ddf_layout = 1;
  2705. break;
  2706. case ALGORITHM_ROTATING_N_RESTART:
  2707. /* Same a left_asymmetric, by first stripe is
  2708. * D D D P Q rather than
  2709. * Q D D D P
  2710. */
  2711. stripe2 += 1;
  2712. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2713. qd_idx = pd_idx + 1;
  2714. if (pd_idx == raid_disks-1) {
  2715. (*dd_idx)++; /* Q D D D P */
  2716. qd_idx = 0;
  2717. } else if (*dd_idx >= pd_idx)
  2718. (*dd_idx) += 2; /* D D P Q D */
  2719. ddf_layout = 1;
  2720. break;
  2721. case ALGORITHM_ROTATING_N_CONTINUE:
  2722. /* Same as left_symmetric but Q is before P */
  2723. pd_idx = raid_disks - 1 - sector_div(stripe2, raid_disks);
  2724. qd_idx = (pd_idx + raid_disks - 1) % raid_disks;
  2725. *dd_idx = (pd_idx + 1 + *dd_idx) % raid_disks;
  2726. ddf_layout = 1;
  2727. break;
  2728. case ALGORITHM_LEFT_ASYMMETRIC_6:
  2729. /* RAID5 left_asymmetric, with Q on last device */
  2730. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  2731. if (*dd_idx >= pd_idx)
  2732. (*dd_idx)++;
  2733. qd_idx = raid_disks - 1;
  2734. break;
  2735. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  2736. pd_idx = sector_div(stripe2, raid_disks-1);
  2737. if (*dd_idx >= pd_idx)
  2738. (*dd_idx)++;
  2739. qd_idx = raid_disks - 1;
  2740. break;
  2741. case ALGORITHM_LEFT_SYMMETRIC_6:
  2742. pd_idx = data_disks - sector_div(stripe2, raid_disks-1);
  2743. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  2744. qd_idx = raid_disks - 1;
  2745. break;
  2746. case ALGORITHM_RIGHT_SYMMETRIC_6:
  2747. pd_idx = sector_div(stripe2, raid_disks-1);
  2748. *dd_idx = (pd_idx + 1 + *dd_idx) % (raid_disks-1);
  2749. qd_idx = raid_disks - 1;
  2750. break;
  2751. case ALGORITHM_PARITY_0_6:
  2752. pd_idx = 0;
  2753. (*dd_idx)++;
  2754. qd_idx = raid_disks - 1;
  2755. break;
  2756. default:
  2757. BUG();
  2758. }
  2759. break;
  2760. }
  2761. if (sh) {
  2762. sh->pd_idx = pd_idx;
  2763. sh->qd_idx = qd_idx;
  2764. sh->ddf_layout = ddf_layout;
  2765. }
  2766. /*
  2767. * Finally, compute the new sector number
  2768. */
  2769. new_sector = (sector_t)stripe * sectors_per_chunk + chunk_offset;
  2770. return new_sector;
  2771. }
  2772. sector_t raid5_compute_blocknr(struct stripe_head *sh, int i, int previous)
  2773. {
  2774. struct r5conf *conf = sh->raid_conf;
  2775. int raid_disks = sh->disks;
  2776. int data_disks = raid_disks - conf->max_degraded;
  2777. sector_t new_sector = sh->sector, check;
  2778. int sectors_per_chunk = previous ? conf->prev_chunk_sectors
  2779. : conf->chunk_sectors;
  2780. int algorithm = previous ? conf->prev_algo
  2781. : conf->algorithm;
  2782. sector_t stripe;
  2783. int chunk_offset;
  2784. sector_t chunk_number;
  2785. int dummy1, dd_idx = i;
  2786. sector_t r_sector;
  2787. struct stripe_head sh2;
  2788. chunk_offset = sector_div(new_sector, sectors_per_chunk);
  2789. stripe = new_sector;
  2790. if (i == sh->pd_idx)
  2791. return 0;
  2792. switch(conf->level) {
  2793. case 4: break;
  2794. case 5:
  2795. switch (algorithm) {
  2796. case ALGORITHM_LEFT_ASYMMETRIC:
  2797. case ALGORITHM_RIGHT_ASYMMETRIC:
  2798. if (i > sh->pd_idx)
  2799. i--;
  2800. break;
  2801. case ALGORITHM_LEFT_SYMMETRIC:
  2802. case ALGORITHM_RIGHT_SYMMETRIC:
  2803. if (i < sh->pd_idx)
  2804. i += raid_disks;
  2805. i -= (sh->pd_idx + 1);
  2806. break;
  2807. case ALGORITHM_PARITY_0:
  2808. i -= 1;
  2809. break;
  2810. case ALGORITHM_PARITY_N:
  2811. break;
  2812. default:
  2813. BUG();
  2814. }
  2815. break;
  2816. case 6:
  2817. if (i == sh->qd_idx)
  2818. return 0; /* It is the Q disk */
  2819. switch (algorithm) {
  2820. case ALGORITHM_LEFT_ASYMMETRIC:
  2821. case ALGORITHM_RIGHT_ASYMMETRIC:
  2822. case ALGORITHM_ROTATING_ZERO_RESTART:
  2823. case ALGORITHM_ROTATING_N_RESTART:
  2824. if (sh->pd_idx == raid_disks-1)
  2825. i--; /* Q D D D P */
  2826. else if (i > sh->pd_idx)
  2827. i -= 2; /* D D P Q D */
  2828. break;
  2829. case ALGORITHM_LEFT_SYMMETRIC:
  2830. case ALGORITHM_RIGHT_SYMMETRIC:
  2831. if (sh->pd_idx == raid_disks-1)
  2832. i--; /* Q D D D P */
  2833. else {
  2834. /* D D P Q D */
  2835. if (i < sh->pd_idx)
  2836. i += raid_disks;
  2837. i -= (sh->pd_idx + 2);
  2838. }
  2839. break;
  2840. case ALGORITHM_PARITY_0:
  2841. i -= 2;
  2842. break;
  2843. case ALGORITHM_PARITY_N:
  2844. break;
  2845. case ALGORITHM_ROTATING_N_CONTINUE:
  2846. /* Like left_symmetric, but P is before Q */
  2847. if (sh->pd_idx == 0)
  2848. i--; /* P D D D Q */
  2849. else {
  2850. /* D D Q P D */
  2851. if (i < sh->pd_idx)
  2852. i += raid_disks;
  2853. i -= (sh->pd_idx + 1);
  2854. }
  2855. break;
  2856. case ALGORITHM_LEFT_ASYMMETRIC_6:
  2857. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  2858. if (i > sh->pd_idx)
  2859. i--;
  2860. break;
  2861. case ALGORITHM_LEFT_SYMMETRIC_6:
  2862. case ALGORITHM_RIGHT_SYMMETRIC_6:
  2863. if (i < sh->pd_idx)
  2864. i += data_disks + 1;
  2865. i -= (sh->pd_idx + 1);
  2866. break;
  2867. case ALGORITHM_PARITY_0_6:
  2868. i -= 1;
  2869. break;
  2870. default:
  2871. BUG();
  2872. }
  2873. break;
  2874. }
  2875. chunk_number = stripe * data_disks + i;
  2876. r_sector = chunk_number * sectors_per_chunk + chunk_offset;
  2877. check = raid5_compute_sector(conf, r_sector,
  2878. previous, &dummy1, &sh2);
  2879. if (check != sh->sector || dummy1 != dd_idx || sh2.pd_idx != sh->pd_idx
  2880. || sh2.qd_idx != sh->qd_idx) {
  2881. pr_warn("md/raid:%s: compute_blocknr: map not correct\n",
  2882. mdname(conf->mddev));
  2883. return 0;
  2884. }
  2885. return r_sector;
  2886. }
  2887. /*
  2888. * There are cases where we want handle_stripe_dirtying() and
  2889. * schedule_reconstruction() to delay towrite to some dev of a stripe.
  2890. *
  2891. * This function checks whether we want to delay the towrite. Specifically,
  2892. * we delay the towrite when:
  2893. *
  2894. * 1. degraded stripe has a non-overwrite to the missing dev, AND this
  2895. * stripe has data in journal (for other devices).
  2896. *
  2897. * In this case, when reading data for the non-overwrite dev, it is
  2898. * necessary to handle complex rmw of write back cache (prexor with
  2899. * orig_page, and xor with page). To keep read path simple, we would
  2900. * like to flush data in journal to RAID disks first, so complex rmw
  2901. * is handled in the write patch (handle_stripe_dirtying).
  2902. *
  2903. * 2. when journal space is critical (R5C_LOG_CRITICAL=1)
  2904. *
  2905. * It is important to be able to flush all stripes in raid5-cache.
  2906. * Therefore, we need reserve some space on the journal device for
  2907. * these flushes. If flush operation includes pending writes to the
  2908. * stripe, we need to reserve (conf->raid_disk + 1) pages per stripe
  2909. * for the flush out. If we exclude these pending writes from flush
  2910. * operation, we only need (conf->max_degraded + 1) pages per stripe.
  2911. * Therefore, excluding pending writes in these cases enables more
  2912. * efficient use of the journal device.
  2913. *
  2914. * Note: To make sure the stripe makes progress, we only delay
  2915. * towrite for stripes with data already in journal (injournal > 0).
  2916. * When LOG_CRITICAL, stripes with injournal == 0 will be sent to
  2917. * no_space_stripes list.
  2918. *
  2919. * 3. during journal failure
  2920. * In journal failure, we try to flush all cached data to raid disks
  2921. * based on data in stripe cache. The array is read-only to upper
  2922. * layers, so we would skip all pending writes.
  2923. *
  2924. */
  2925. static inline bool delay_towrite(struct r5conf *conf,
  2926. struct r5dev *dev,
  2927. struct stripe_head_state *s)
  2928. {
  2929. /* case 1 above */
  2930. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  2931. !test_bit(R5_Insync, &dev->flags) && s->injournal)
  2932. return true;
  2933. /* case 2 above */
  2934. if (test_bit(R5C_LOG_CRITICAL, &conf->cache_state) &&
  2935. s->injournal > 0)
  2936. return true;
  2937. /* case 3 above */
  2938. if (s->log_failed && s->injournal)
  2939. return true;
  2940. return false;
  2941. }
  2942. static void
  2943. schedule_reconstruction(struct stripe_head *sh, struct stripe_head_state *s,
  2944. int rcw, int expand)
  2945. {
  2946. int i, pd_idx = sh->pd_idx, qd_idx = sh->qd_idx, disks = sh->disks;
  2947. struct r5conf *conf = sh->raid_conf;
  2948. int level = conf->level;
  2949. if (rcw) {
  2950. /*
  2951. * In some cases, handle_stripe_dirtying initially decided to
  2952. * run rmw and allocates extra page for prexor. However, rcw is
  2953. * cheaper later on. We need to free the extra page now,
  2954. * because we won't be able to do that in ops_complete_prexor().
  2955. */
  2956. r5c_release_extra_page(sh);
  2957. for (i = disks; i--; ) {
  2958. struct r5dev *dev = &sh->dev[i];
  2959. if (dev->towrite && !delay_towrite(conf, dev, s)) {
  2960. set_bit(R5_LOCKED, &dev->flags);
  2961. set_bit(R5_Wantdrain, &dev->flags);
  2962. if (!expand)
  2963. clear_bit(R5_UPTODATE, &dev->flags);
  2964. s->locked++;
  2965. } else if (test_bit(R5_InJournal, &dev->flags)) {
  2966. set_bit(R5_LOCKED, &dev->flags);
  2967. s->locked++;
  2968. }
  2969. }
  2970. /* if we are not expanding this is a proper write request, and
  2971. * there will be bios with new data to be drained into the
  2972. * stripe cache
  2973. */
  2974. if (!expand) {
  2975. if (!s->locked)
  2976. /* False alarm, nothing to do */
  2977. return;
  2978. sh->reconstruct_state = reconstruct_state_drain_run;
  2979. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  2980. } else
  2981. sh->reconstruct_state = reconstruct_state_run;
  2982. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  2983. if (s->locked + conf->max_degraded == disks)
  2984. if (!test_and_set_bit(STRIPE_FULL_WRITE, &sh->state))
  2985. atomic_inc(&conf->pending_full_writes);
  2986. } else {
  2987. BUG_ON(!(test_bit(R5_UPTODATE, &sh->dev[pd_idx].flags) ||
  2988. test_bit(R5_Wantcompute, &sh->dev[pd_idx].flags)));
  2989. BUG_ON(level == 6 &&
  2990. (!(test_bit(R5_UPTODATE, &sh->dev[qd_idx].flags) ||
  2991. test_bit(R5_Wantcompute, &sh->dev[qd_idx].flags))));
  2992. for (i = disks; i--; ) {
  2993. struct r5dev *dev = &sh->dev[i];
  2994. if (i == pd_idx || i == qd_idx)
  2995. continue;
  2996. if (dev->towrite &&
  2997. (test_bit(R5_UPTODATE, &dev->flags) ||
  2998. test_bit(R5_Wantcompute, &dev->flags))) {
  2999. set_bit(R5_Wantdrain, &dev->flags);
  3000. set_bit(R5_LOCKED, &dev->flags);
  3001. clear_bit(R5_UPTODATE, &dev->flags);
  3002. s->locked++;
  3003. } else if (test_bit(R5_InJournal, &dev->flags)) {
  3004. set_bit(R5_LOCKED, &dev->flags);
  3005. s->locked++;
  3006. }
  3007. }
  3008. if (!s->locked)
  3009. /* False alarm - nothing to do */
  3010. return;
  3011. sh->reconstruct_state = reconstruct_state_prexor_drain_run;
  3012. set_bit(STRIPE_OP_PREXOR, &s->ops_request);
  3013. set_bit(STRIPE_OP_BIODRAIN, &s->ops_request);
  3014. set_bit(STRIPE_OP_RECONSTRUCT, &s->ops_request);
  3015. }
  3016. /* keep the parity disk(s) locked while asynchronous operations
  3017. * are in flight
  3018. */
  3019. set_bit(R5_LOCKED, &sh->dev[pd_idx].flags);
  3020. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  3021. s->locked++;
  3022. if (level == 6) {
  3023. int qd_idx = sh->qd_idx;
  3024. struct r5dev *dev = &sh->dev[qd_idx];
  3025. set_bit(R5_LOCKED, &dev->flags);
  3026. clear_bit(R5_UPTODATE, &dev->flags);
  3027. s->locked++;
  3028. }
  3029. if (raid5_has_ppl(sh->raid_conf) && sh->ppl_page &&
  3030. test_bit(STRIPE_OP_BIODRAIN, &s->ops_request) &&
  3031. !test_bit(STRIPE_FULL_WRITE, &sh->state) &&
  3032. test_bit(R5_Insync, &sh->dev[pd_idx].flags))
  3033. set_bit(STRIPE_OP_PARTIAL_PARITY, &s->ops_request);
  3034. pr_debug("%s: stripe %llu locked: %d ops_request: %lx\n",
  3035. __func__, (unsigned long long)sh->sector,
  3036. s->locked, s->ops_request);
  3037. }
  3038. /*
  3039. * Each stripe/dev can have one or more bion attached.
  3040. * toread/towrite point to the first in a chain.
  3041. * The bi_next chain must be in order.
  3042. */
  3043. static int add_stripe_bio(struct stripe_head *sh, struct bio *bi, int dd_idx,
  3044. int forwrite, int previous)
  3045. {
  3046. struct bio **bip;
  3047. struct r5conf *conf = sh->raid_conf;
  3048. int firstwrite=0;
  3049. pr_debug("adding bi b#%llu to stripe s#%llu\n",
  3050. (unsigned long long)bi->bi_iter.bi_sector,
  3051. (unsigned long long)sh->sector);
  3052. spin_lock_irq(&sh->stripe_lock);
  3053. sh->dev[dd_idx].write_hint = bi->bi_write_hint;
  3054. /* Don't allow new IO added to stripes in batch list */
  3055. if (sh->batch_head)
  3056. goto overlap;
  3057. if (forwrite) {
  3058. bip = &sh->dev[dd_idx].towrite;
  3059. if (*bip == NULL)
  3060. firstwrite = 1;
  3061. } else
  3062. bip = &sh->dev[dd_idx].toread;
  3063. while (*bip && (*bip)->bi_iter.bi_sector < bi->bi_iter.bi_sector) {
  3064. if (bio_end_sector(*bip) > bi->bi_iter.bi_sector)
  3065. goto overlap;
  3066. bip = & (*bip)->bi_next;
  3067. }
  3068. if (*bip && (*bip)->bi_iter.bi_sector < bio_end_sector(bi))
  3069. goto overlap;
  3070. if (forwrite && raid5_has_ppl(conf)) {
  3071. /*
  3072. * With PPL only writes to consecutive data chunks within a
  3073. * stripe are allowed because for a single stripe_head we can
  3074. * only have one PPL entry at a time, which describes one data
  3075. * range. Not really an overlap, but wait_for_overlap can be
  3076. * used to handle this.
  3077. */
  3078. sector_t sector;
  3079. sector_t first = 0;
  3080. sector_t last = 0;
  3081. int count = 0;
  3082. int i;
  3083. for (i = 0; i < sh->disks; i++) {
  3084. if (i != sh->pd_idx &&
  3085. (i == dd_idx || sh->dev[i].towrite)) {
  3086. sector = sh->dev[i].sector;
  3087. if (count == 0 || sector < first)
  3088. first = sector;
  3089. if (sector > last)
  3090. last = sector;
  3091. count++;
  3092. }
  3093. }
  3094. if (first + conf->chunk_sectors * (count - 1) != last)
  3095. goto overlap;
  3096. }
  3097. if (!forwrite || previous)
  3098. clear_bit(STRIPE_BATCH_READY, &sh->state);
  3099. BUG_ON(*bip && bi->bi_next && (*bip) != bi->bi_next);
  3100. if (*bip)
  3101. bi->bi_next = *bip;
  3102. *bip = bi;
  3103. bio_inc_remaining(bi);
  3104. md_write_inc(conf->mddev, bi);
  3105. if (forwrite) {
  3106. /* check if page is covered */
  3107. sector_t sector = sh->dev[dd_idx].sector;
  3108. for (bi=sh->dev[dd_idx].towrite;
  3109. sector < sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf) &&
  3110. bi && bi->bi_iter.bi_sector <= sector;
  3111. bi = r5_next_bio(conf, bi, sh->dev[dd_idx].sector)) {
  3112. if (bio_end_sector(bi) >= sector)
  3113. sector = bio_end_sector(bi);
  3114. }
  3115. if (sector >= sh->dev[dd_idx].sector + RAID5_STRIPE_SECTORS(conf))
  3116. if (!test_and_set_bit(R5_OVERWRITE, &sh->dev[dd_idx].flags))
  3117. sh->overwrite_disks++;
  3118. }
  3119. pr_debug("added bi b#%llu to stripe s#%llu, disk %d.\n",
  3120. (unsigned long long)(*bip)->bi_iter.bi_sector,
  3121. (unsigned long long)sh->sector, dd_idx);
  3122. if (conf->mddev->bitmap && firstwrite) {
  3123. /* Cannot hold spinlock over bitmap_startwrite,
  3124. * but must ensure this isn't added to a batch until
  3125. * we have added to the bitmap and set bm_seq.
  3126. * So set STRIPE_BITMAP_PENDING to prevent
  3127. * batching.
  3128. * If multiple add_stripe_bio() calls race here they
  3129. * much all set STRIPE_BITMAP_PENDING. So only the first one
  3130. * to complete "bitmap_startwrite" gets to set
  3131. * STRIPE_BIT_DELAY. This is important as once a stripe
  3132. * is added to a batch, STRIPE_BIT_DELAY cannot be changed
  3133. * any more.
  3134. */
  3135. set_bit(STRIPE_BITMAP_PENDING, &sh->state);
  3136. spin_unlock_irq(&sh->stripe_lock);
  3137. md_bitmap_startwrite(conf->mddev->bitmap, sh->sector,
  3138. RAID5_STRIPE_SECTORS(conf), 0);
  3139. spin_lock_irq(&sh->stripe_lock);
  3140. clear_bit(STRIPE_BITMAP_PENDING, &sh->state);
  3141. if (!sh->batch_head) {
  3142. sh->bm_seq = conf->seq_flush+1;
  3143. set_bit(STRIPE_BIT_DELAY, &sh->state);
  3144. }
  3145. }
  3146. spin_unlock_irq(&sh->stripe_lock);
  3147. if (stripe_can_batch(sh))
  3148. stripe_add_to_batch_list(conf, sh);
  3149. return 1;
  3150. overlap:
  3151. set_bit(R5_Overlap, &sh->dev[dd_idx].flags);
  3152. spin_unlock_irq(&sh->stripe_lock);
  3153. return 0;
  3154. }
  3155. static void end_reshape(struct r5conf *conf);
  3156. static void stripe_set_idx(sector_t stripe, struct r5conf *conf, int previous,
  3157. struct stripe_head *sh)
  3158. {
  3159. int sectors_per_chunk =
  3160. previous ? conf->prev_chunk_sectors : conf->chunk_sectors;
  3161. int dd_idx;
  3162. int chunk_offset = sector_div(stripe, sectors_per_chunk);
  3163. int disks = previous ? conf->previous_raid_disks : conf->raid_disks;
  3164. raid5_compute_sector(conf,
  3165. stripe * (disks - conf->max_degraded)
  3166. *sectors_per_chunk + chunk_offset,
  3167. previous,
  3168. &dd_idx, sh);
  3169. }
  3170. static void
  3171. handle_failed_stripe(struct r5conf *conf, struct stripe_head *sh,
  3172. struct stripe_head_state *s, int disks)
  3173. {
  3174. int i;
  3175. BUG_ON(sh->batch_head);
  3176. for (i = disks; i--; ) {
  3177. struct bio *bi;
  3178. int bitmap_end = 0;
  3179. if (test_bit(R5_ReadError, &sh->dev[i].flags)) {
  3180. struct md_rdev *rdev;
  3181. rcu_read_lock();
  3182. rdev = rcu_dereference(conf->disks[i].rdev);
  3183. if (rdev && test_bit(In_sync, &rdev->flags) &&
  3184. !test_bit(Faulty, &rdev->flags))
  3185. atomic_inc(&rdev->nr_pending);
  3186. else
  3187. rdev = NULL;
  3188. rcu_read_unlock();
  3189. if (rdev) {
  3190. if (!rdev_set_badblocks(
  3191. rdev,
  3192. sh->sector,
  3193. RAID5_STRIPE_SECTORS(conf), 0))
  3194. md_error(conf->mddev, rdev);
  3195. rdev_dec_pending(rdev, conf->mddev);
  3196. }
  3197. }
  3198. spin_lock_irq(&sh->stripe_lock);
  3199. /* fail all writes first */
  3200. bi = sh->dev[i].towrite;
  3201. sh->dev[i].towrite = NULL;
  3202. sh->overwrite_disks = 0;
  3203. spin_unlock_irq(&sh->stripe_lock);
  3204. if (bi)
  3205. bitmap_end = 1;
  3206. log_stripe_write_finished(sh);
  3207. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  3208. wake_up(&conf->wait_for_overlap);
  3209. while (bi && bi->bi_iter.bi_sector <
  3210. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3211. struct bio *nextbi = r5_next_bio(conf, bi, sh->dev[i].sector);
  3212. md_write_end(conf->mddev);
  3213. bio_io_error(bi);
  3214. bi = nextbi;
  3215. }
  3216. if (bitmap_end)
  3217. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3218. RAID5_STRIPE_SECTORS(conf), 0, 0);
  3219. bitmap_end = 0;
  3220. /* and fail all 'written' */
  3221. bi = sh->dev[i].written;
  3222. sh->dev[i].written = NULL;
  3223. if (test_and_clear_bit(R5_SkipCopy, &sh->dev[i].flags)) {
  3224. WARN_ON(test_bit(R5_UPTODATE, &sh->dev[i].flags));
  3225. sh->dev[i].page = sh->dev[i].orig_page;
  3226. }
  3227. if (bi) bitmap_end = 1;
  3228. while (bi && bi->bi_iter.bi_sector <
  3229. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3230. struct bio *bi2 = r5_next_bio(conf, bi, sh->dev[i].sector);
  3231. md_write_end(conf->mddev);
  3232. bio_io_error(bi);
  3233. bi = bi2;
  3234. }
  3235. /* fail any reads if this device is non-operational and
  3236. * the data has not reached the cache yet.
  3237. */
  3238. if (!test_bit(R5_Wantfill, &sh->dev[i].flags) &&
  3239. s->failed > conf->max_degraded &&
  3240. (!test_bit(R5_Insync, &sh->dev[i].flags) ||
  3241. test_bit(R5_ReadError, &sh->dev[i].flags))) {
  3242. spin_lock_irq(&sh->stripe_lock);
  3243. bi = sh->dev[i].toread;
  3244. sh->dev[i].toread = NULL;
  3245. spin_unlock_irq(&sh->stripe_lock);
  3246. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  3247. wake_up(&conf->wait_for_overlap);
  3248. if (bi)
  3249. s->to_read--;
  3250. while (bi && bi->bi_iter.bi_sector <
  3251. sh->dev[i].sector + RAID5_STRIPE_SECTORS(conf)) {
  3252. struct bio *nextbi =
  3253. r5_next_bio(conf, bi, sh->dev[i].sector);
  3254. bio_io_error(bi);
  3255. bi = nextbi;
  3256. }
  3257. }
  3258. if (bitmap_end)
  3259. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3260. RAID5_STRIPE_SECTORS(conf), 0, 0);
  3261. /* If we were in the middle of a write the parity block might
  3262. * still be locked - so just clear all R5_LOCKED flags
  3263. */
  3264. clear_bit(R5_LOCKED, &sh->dev[i].flags);
  3265. }
  3266. s->to_write = 0;
  3267. s->written = 0;
  3268. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  3269. if (atomic_dec_and_test(&conf->pending_full_writes))
  3270. md_wakeup_thread(conf->mddev->thread);
  3271. }
  3272. static void
  3273. handle_failed_sync(struct r5conf *conf, struct stripe_head *sh,
  3274. struct stripe_head_state *s)
  3275. {
  3276. int abort = 0;
  3277. int i;
  3278. BUG_ON(sh->batch_head);
  3279. clear_bit(STRIPE_SYNCING, &sh->state);
  3280. if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
  3281. wake_up(&conf->wait_for_overlap);
  3282. s->syncing = 0;
  3283. s->replacing = 0;
  3284. /* There is nothing more to do for sync/check/repair.
  3285. * Don't even need to abort as that is handled elsewhere
  3286. * if needed, and not always wanted e.g. if there is a known
  3287. * bad block here.
  3288. * For recover/replace we need to record a bad block on all
  3289. * non-sync devices, or abort the recovery
  3290. */
  3291. if (test_bit(MD_RECOVERY_RECOVER, &conf->mddev->recovery)) {
  3292. /* During recovery devices cannot be removed, so
  3293. * locking and refcounting of rdevs is not needed
  3294. */
  3295. rcu_read_lock();
  3296. for (i = 0; i < conf->raid_disks; i++) {
  3297. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  3298. if (rdev
  3299. && !test_bit(Faulty, &rdev->flags)
  3300. && !test_bit(In_sync, &rdev->flags)
  3301. && !rdev_set_badblocks(rdev, sh->sector,
  3302. RAID5_STRIPE_SECTORS(conf), 0))
  3303. abort = 1;
  3304. rdev = rcu_dereference(conf->disks[i].replacement);
  3305. if (rdev
  3306. && !test_bit(Faulty, &rdev->flags)
  3307. && !test_bit(In_sync, &rdev->flags)
  3308. && !rdev_set_badblocks(rdev, sh->sector,
  3309. RAID5_STRIPE_SECTORS(conf), 0))
  3310. abort = 1;
  3311. }
  3312. rcu_read_unlock();
  3313. if (abort)
  3314. conf->recovery_disabled =
  3315. conf->mddev->recovery_disabled;
  3316. }
  3317. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), !abort);
  3318. }
  3319. static int want_replace(struct stripe_head *sh, int disk_idx)
  3320. {
  3321. struct md_rdev *rdev;
  3322. int rv = 0;
  3323. rcu_read_lock();
  3324. rdev = rcu_dereference(sh->raid_conf->disks[disk_idx].replacement);
  3325. if (rdev
  3326. && !test_bit(Faulty, &rdev->flags)
  3327. && !test_bit(In_sync, &rdev->flags)
  3328. && (rdev->recovery_offset <= sh->sector
  3329. || rdev->mddev->recovery_cp <= sh->sector))
  3330. rv = 1;
  3331. rcu_read_unlock();
  3332. return rv;
  3333. }
  3334. static int need_this_block(struct stripe_head *sh, struct stripe_head_state *s,
  3335. int disk_idx, int disks)
  3336. {
  3337. struct r5dev *dev = &sh->dev[disk_idx];
  3338. struct r5dev *fdev[2] = { &sh->dev[s->failed_num[0]],
  3339. &sh->dev[s->failed_num[1]] };
  3340. int i;
  3341. bool force_rcw = (sh->raid_conf->rmw_level == PARITY_DISABLE_RMW);
  3342. if (test_bit(R5_LOCKED, &dev->flags) ||
  3343. test_bit(R5_UPTODATE, &dev->flags))
  3344. /* No point reading this as we already have it or have
  3345. * decided to get it.
  3346. */
  3347. return 0;
  3348. if (dev->toread ||
  3349. (dev->towrite && !test_bit(R5_OVERWRITE, &dev->flags)))
  3350. /* We need this block to directly satisfy a request */
  3351. return 1;
  3352. if (s->syncing || s->expanding ||
  3353. (s->replacing && want_replace(sh, disk_idx)))
  3354. /* When syncing, or expanding we read everything.
  3355. * When replacing, we need the replaced block.
  3356. */
  3357. return 1;
  3358. if ((s->failed >= 1 && fdev[0]->toread) ||
  3359. (s->failed >= 2 && fdev[1]->toread))
  3360. /* If we want to read from a failed device, then
  3361. * we need to actually read every other device.
  3362. */
  3363. return 1;
  3364. /* Sometimes neither read-modify-write nor reconstruct-write
  3365. * cycles can work. In those cases we read every block we
  3366. * can. Then the parity-update is certain to have enough to
  3367. * work with.
  3368. * This can only be a problem when we need to write something,
  3369. * and some device has failed. If either of those tests
  3370. * fail we need look no further.
  3371. */
  3372. if (!s->failed || !s->to_write)
  3373. return 0;
  3374. if (test_bit(R5_Insync, &dev->flags) &&
  3375. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3376. /* Pre-reads at not permitted until after short delay
  3377. * to gather multiple requests. However if this
  3378. * device is no Insync, the block could only be computed
  3379. * and there is no need to delay that.
  3380. */
  3381. return 0;
  3382. for (i = 0; i < s->failed && i < 2; i++) {
  3383. if (fdev[i]->towrite &&
  3384. !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
  3385. !test_bit(R5_OVERWRITE, &fdev[i]->flags))
  3386. /* If we have a partial write to a failed
  3387. * device, then we will need to reconstruct
  3388. * the content of that device, so all other
  3389. * devices must be read.
  3390. */
  3391. return 1;
  3392. if (s->failed >= 2 &&
  3393. (fdev[i]->towrite ||
  3394. s->failed_num[i] == sh->pd_idx ||
  3395. s->failed_num[i] == sh->qd_idx) &&
  3396. !test_bit(R5_UPTODATE, &fdev[i]->flags))
  3397. /* In max degraded raid6, If the failed disk is P, Q,
  3398. * or we want to read the failed disk, we need to do
  3399. * reconstruct-write.
  3400. */
  3401. force_rcw = true;
  3402. }
  3403. /* If we are forced to do a reconstruct-write, because parity
  3404. * cannot be trusted and we are currently recovering it, there
  3405. * is extra need to be careful.
  3406. * If one of the devices that we would need to read, because
  3407. * it is not being overwritten (and maybe not written at all)
  3408. * is missing/faulty, then we need to read everything we can.
  3409. */
  3410. if (!force_rcw &&
  3411. sh->sector < sh->raid_conf->mddev->recovery_cp)
  3412. /* reconstruct-write isn't being forced */
  3413. return 0;
  3414. for (i = 0; i < s->failed && i < 2; i++) {
  3415. if (s->failed_num[i] != sh->pd_idx &&
  3416. s->failed_num[i] != sh->qd_idx &&
  3417. !test_bit(R5_UPTODATE, &fdev[i]->flags) &&
  3418. !test_bit(R5_OVERWRITE, &fdev[i]->flags))
  3419. return 1;
  3420. }
  3421. return 0;
  3422. }
  3423. /* fetch_block - checks the given member device to see if its data needs
  3424. * to be read or computed to satisfy a request.
  3425. *
  3426. * Returns 1 when no more member devices need to be checked, otherwise returns
  3427. * 0 to tell the loop in handle_stripe_fill to continue
  3428. */
  3429. static int fetch_block(struct stripe_head *sh, struct stripe_head_state *s,
  3430. int disk_idx, int disks)
  3431. {
  3432. struct r5dev *dev = &sh->dev[disk_idx];
  3433. /* is the data in this block needed, and can we get it? */
  3434. if (need_this_block(sh, s, disk_idx, disks)) {
  3435. /* we would like to get this block, possibly by computing it,
  3436. * otherwise read it if the backing disk is insync
  3437. */
  3438. BUG_ON(test_bit(R5_Wantcompute, &dev->flags));
  3439. BUG_ON(test_bit(R5_Wantread, &dev->flags));
  3440. BUG_ON(sh->batch_head);
  3441. /*
  3442. * In the raid6 case if the only non-uptodate disk is P
  3443. * then we already trusted P to compute the other failed
  3444. * drives. It is safe to compute rather than re-read P.
  3445. * In other cases we only compute blocks from failed
  3446. * devices, otherwise check/repair might fail to detect
  3447. * a real inconsistency.
  3448. */
  3449. if ((s->uptodate == disks - 1) &&
  3450. ((sh->qd_idx >= 0 && sh->pd_idx == disk_idx) ||
  3451. (s->failed && (disk_idx == s->failed_num[0] ||
  3452. disk_idx == s->failed_num[1])))) {
  3453. /* have disk failed, and we're requested to fetch it;
  3454. * do compute it
  3455. */
  3456. pr_debug("Computing stripe %llu block %d\n",
  3457. (unsigned long long)sh->sector, disk_idx);
  3458. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3459. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3460. set_bit(R5_Wantcompute, &dev->flags);
  3461. sh->ops.target = disk_idx;
  3462. sh->ops.target2 = -1; /* no 2nd target */
  3463. s->req_compute = 1;
  3464. /* Careful: from this point on 'uptodate' is in the eye
  3465. * of raid_run_ops which services 'compute' operations
  3466. * before writes. R5_Wantcompute flags a block that will
  3467. * be R5_UPTODATE by the time it is needed for a
  3468. * subsequent operation.
  3469. */
  3470. s->uptodate++;
  3471. return 1;
  3472. } else if (s->uptodate == disks-2 && s->failed >= 2) {
  3473. /* Computing 2-failure is *very* expensive; only
  3474. * do it if failed >= 2
  3475. */
  3476. int other;
  3477. for (other = disks; other--; ) {
  3478. if (other == disk_idx)
  3479. continue;
  3480. if (!test_bit(R5_UPTODATE,
  3481. &sh->dev[other].flags))
  3482. break;
  3483. }
  3484. BUG_ON(other < 0);
  3485. pr_debug("Computing stripe %llu blocks %d,%d\n",
  3486. (unsigned long long)sh->sector,
  3487. disk_idx, other);
  3488. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3489. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3490. set_bit(R5_Wantcompute, &sh->dev[disk_idx].flags);
  3491. set_bit(R5_Wantcompute, &sh->dev[other].flags);
  3492. sh->ops.target = disk_idx;
  3493. sh->ops.target2 = other;
  3494. s->uptodate += 2;
  3495. s->req_compute = 1;
  3496. return 1;
  3497. } else if (test_bit(R5_Insync, &dev->flags)) {
  3498. set_bit(R5_LOCKED, &dev->flags);
  3499. set_bit(R5_Wantread, &dev->flags);
  3500. s->locked++;
  3501. pr_debug("Reading block %d (sync=%d)\n",
  3502. disk_idx, s->syncing);
  3503. }
  3504. }
  3505. return 0;
  3506. }
  3507. /*
  3508. * handle_stripe_fill - read or compute data to satisfy pending requests.
  3509. */
  3510. static void handle_stripe_fill(struct stripe_head *sh,
  3511. struct stripe_head_state *s,
  3512. int disks)
  3513. {
  3514. int i;
  3515. /* look for blocks to read/compute, skip this if a compute
  3516. * is already in flight, or if the stripe contents are in the
  3517. * midst of changing due to a write
  3518. */
  3519. if (!test_bit(STRIPE_COMPUTE_RUN, &sh->state) && !sh->check_state &&
  3520. !sh->reconstruct_state) {
  3521. /*
  3522. * For degraded stripe with data in journal, do not handle
  3523. * read requests yet, instead, flush the stripe to raid
  3524. * disks first, this avoids handling complex rmw of write
  3525. * back cache (prexor with orig_page, and then xor with
  3526. * page) in the read path
  3527. */
  3528. if (s->injournal && s->failed) {
  3529. if (test_bit(STRIPE_R5C_CACHING, &sh->state))
  3530. r5c_make_stripe_write_out(sh);
  3531. goto out;
  3532. }
  3533. for (i = disks; i--; )
  3534. if (fetch_block(sh, s, i, disks))
  3535. break;
  3536. }
  3537. out:
  3538. set_bit(STRIPE_HANDLE, &sh->state);
  3539. }
  3540. static void break_stripe_batch_list(struct stripe_head *head_sh,
  3541. unsigned long handle_flags);
  3542. /* handle_stripe_clean_event
  3543. * any written block on an uptodate or failed drive can be returned.
  3544. * Note that if we 'wrote' to a failed drive, it will be UPTODATE, but
  3545. * never LOCKED, so we don't need to test 'failed' directly.
  3546. */
  3547. static void handle_stripe_clean_event(struct r5conf *conf,
  3548. struct stripe_head *sh, int disks)
  3549. {
  3550. int i;
  3551. struct r5dev *dev;
  3552. int discard_pending = 0;
  3553. struct stripe_head *head_sh = sh;
  3554. bool do_endio = false;
  3555. for (i = disks; i--; )
  3556. if (sh->dev[i].written) {
  3557. dev = &sh->dev[i];
  3558. if (!test_bit(R5_LOCKED, &dev->flags) &&
  3559. (test_bit(R5_UPTODATE, &dev->flags) ||
  3560. test_bit(R5_Discard, &dev->flags) ||
  3561. test_bit(R5_SkipCopy, &dev->flags))) {
  3562. /* We can return any write requests */
  3563. struct bio *wbi, *wbi2;
  3564. pr_debug("Return write for disc %d\n", i);
  3565. if (test_and_clear_bit(R5_Discard, &dev->flags))
  3566. clear_bit(R5_UPTODATE, &dev->flags);
  3567. if (test_and_clear_bit(R5_SkipCopy, &dev->flags)) {
  3568. WARN_ON(test_bit(R5_UPTODATE, &dev->flags));
  3569. }
  3570. do_endio = true;
  3571. returnbi:
  3572. dev->page = dev->orig_page;
  3573. wbi = dev->written;
  3574. dev->written = NULL;
  3575. while (wbi && wbi->bi_iter.bi_sector <
  3576. dev->sector + RAID5_STRIPE_SECTORS(conf)) {
  3577. wbi2 = r5_next_bio(conf, wbi, dev->sector);
  3578. md_write_end(conf->mddev);
  3579. bio_endio(wbi);
  3580. wbi = wbi2;
  3581. }
  3582. md_bitmap_endwrite(conf->mddev->bitmap, sh->sector,
  3583. RAID5_STRIPE_SECTORS(conf),
  3584. !test_bit(STRIPE_DEGRADED, &sh->state),
  3585. 0);
  3586. if (head_sh->batch_head) {
  3587. sh = list_first_entry(&sh->batch_list,
  3588. struct stripe_head,
  3589. batch_list);
  3590. if (sh != head_sh) {
  3591. dev = &sh->dev[i];
  3592. goto returnbi;
  3593. }
  3594. }
  3595. sh = head_sh;
  3596. dev = &sh->dev[i];
  3597. } else if (test_bit(R5_Discard, &dev->flags))
  3598. discard_pending = 1;
  3599. }
  3600. log_stripe_write_finished(sh);
  3601. if (!discard_pending &&
  3602. test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags)) {
  3603. int hash;
  3604. clear_bit(R5_Discard, &sh->dev[sh->pd_idx].flags);
  3605. clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
  3606. if (sh->qd_idx >= 0) {
  3607. clear_bit(R5_Discard, &sh->dev[sh->qd_idx].flags);
  3608. clear_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags);
  3609. }
  3610. /* now that discard is done we can proceed with any sync */
  3611. clear_bit(STRIPE_DISCARD, &sh->state);
  3612. /*
  3613. * SCSI discard will change some bio fields and the stripe has
  3614. * no updated data, so remove it from hash list and the stripe
  3615. * will be reinitialized
  3616. */
  3617. unhash:
  3618. hash = sh->hash_lock_index;
  3619. spin_lock_irq(conf->hash_locks + hash);
  3620. remove_hash(sh);
  3621. spin_unlock_irq(conf->hash_locks + hash);
  3622. if (head_sh->batch_head) {
  3623. sh = list_first_entry(&sh->batch_list,
  3624. struct stripe_head, batch_list);
  3625. if (sh != head_sh)
  3626. goto unhash;
  3627. }
  3628. sh = head_sh;
  3629. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state))
  3630. set_bit(STRIPE_HANDLE, &sh->state);
  3631. }
  3632. if (test_and_clear_bit(STRIPE_FULL_WRITE, &sh->state))
  3633. if (atomic_dec_and_test(&conf->pending_full_writes))
  3634. md_wakeup_thread(conf->mddev->thread);
  3635. if (head_sh->batch_head && do_endio)
  3636. break_stripe_batch_list(head_sh, STRIPE_EXPAND_SYNC_FLAGS);
  3637. }
  3638. /*
  3639. * For RMW in write back cache, we need extra page in prexor to store the
  3640. * old data. This page is stored in dev->orig_page.
  3641. *
  3642. * This function checks whether we have data for prexor. The exact logic
  3643. * is:
  3644. * R5_UPTODATE && (!R5_InJournal || R5_OrigPageUPTDODATE)
  3645. */
  3646. static inline bool uptodate_for_rmw(struct r5dev *dev)
  3647. {
  3648. return (test_bit(R5_UPTODATE, &dev->flags)) &&
  3649. (!test_bit(R5_InJournal, &dev->flags) ||
  3650. test_bit(R5_OrigPageUPTDODATE, &dev->flags));
  3651. }
  3652. static int handle_stripe_dirtying(struct r5conf *conf,
  3653. struct stripe_head *sh,
  3654. struct stripe_head_state *s,
  3655. int disks)
  3656. {
  3657. int rmw = 0, rcw = 0, i;
  3658. sector_t recovery_cp = conf->mddev->recovery_cp;
  3659. /* Check whether resync is now happening or should start.
  3660. * If yes, then the array is dirty (after unclean shutdown or
  3661. * initial creation), so parity in some stripes might be inconsistent.
  3662. * In this case, we need to always do reconstruct-write, to ensure
  3663. * that in case of drive failure or read-error correction, we
  3664. * generate correct data from the parity.
  3665. */
  3666. if (conf->rmw_level == PARITY_DISABLE_RMW ||
  3667. (recovery_cp < MaxSector && sh->sector >= recovery_cp &&
  3668. s->failed == 0)) {
  3669. /* Calculate the real rcw later - for now make it
  3670. * look like rcw is cheaper
  3671. */
  3672. rcw = 1; rmw = 2;
  3673. pr_debug("force RCW rmw_level=%u, recovery_cp=%llu sh->sector=%llu\n",
  3674. conf->rmw_level, (unsigned long long)recovery_cp,
  3675. (unsigned long long)sh->sector);
  3676. } else for (i = disks; i--; ) {
  3677. /* would I have to read this buffer for read_modify_write */
  3678. struct r5dev *dev = &sh->dev[i];
  3679. if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
  3680. i == sh->pd_idx || i == sh->qd_idx ||
  3681. test_bit(R5_InJournal, &dev->flags)) &&
  3682. !test_bit(R5_LOCKED, &dev->flags) &&
  3683. !(uptodate_for_rmw(dev) ||
  3684. test_bit(R5_Wantcompute, &dev->flags))) {
  3685. if (test_bit(R5_Insync, &dev->flags))
  3686. rmw++;
  3687. else
  3688. rmw += 2*disks; /* cannot read it */
  3689. }
  3690. /* Would I have to read this buffer for reconstruct_write */
  3691. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  3692. i != sh->pd_idx && i != sh->qd_idx &&
  3693. !test_bit(R5_LOCKED, &dev->flags) &&
  3694. !(test_bit(R5_UPTODATE, &dev->flags) ||
  3695. test_bit(R5_Wantcompute, &dev->flags))) {
  3696. if (test_bit(R5_Insync, &dev->flags))
  3697. rcw++;
  3698. else
  3699. rcw += 2*disks;
  3700. }
  3701. }
  3702. pr_debug("for sector %llu state 0x%lx, rmw=%d rcw=%d\n",
  3703. (unsigned long long)sh->sector, sh->state, rmw, rcw);
  3704. set_bit(STRIPE_HANDLE, &sh->state);
  3705. if ((rmw < rcw || (rmw == rcw && conf->rmw_level == PARITY_PREFER_RMW)) && rmw > 0) {
  3706. /* prefer read-modify-write, but need to get some data */
  3707. if (conf->mddev->queue)
  3708. blk_add_trace_msg(conf->mddev->queue,
  3709. "raid5 rmw %llu %d",
  3710. (unsigned long long)sh->sector, rmw);
  3711. for (i = disks; i--; ) {
  3712. struct r5dev *dev = &sh->dev[i];
  3713. if (test_bit(R5_InJournal, &dev->flags) &&
  3714. dev->page == dev->orig_page &&
  3715. !test_bit(R5_LOCKED, &sh->dev[sh->pd_idx].flags)) {
  3716. /* alloc page for prexor */
  3717. struct page *p = alloc_page(GFP_NOIO);
  3718. if (p) {
  3719. dev->orig_page = p;
  3720. continue;
  3721. }
  3722. /*
  3723. * alloc_page() failed, try use
  3724. * disk_info->extra_page
  3725. */
  3726. if (!test_and_set_bit(R5C_EXTRA_PAGE_IN_USE,
  3727. &conf->cache_state)) {
  3728. r5c_use_extra_page(sh);
  3729. break;
  3730. }
  3731. /* extra_page in use, add to delayed_list */
  3732. set_bit(STRIPE_DELAYED, &sh->state);
  3733. s->waiting_extra_page = 1;
  3734. return -EAGAIN;
  3735. }
  3736. }
  3737. for (i = disks; i--; ) {
  3738. struct r5dev *dev = &sh->dev[i];
  3739. if (((dev->towrite && !delay_towrite(conf, dev, s)) ||
  3740. i == sh->pd_idx || i == sh->qd_idx ||
  3741. test_bit(R5_InJournal, &dev->flags)) &&
  3742. !test_bit(R5_LOCKED, &dev->flags) &&
  3743. !(uptodate_for_rmw(dev) ||
  3744. test_bit(R5_Wantcompute, &dev->flags)) &&
  3745. test_bit(R5_Insync, &dev->flags)) {
  3746. if (test_bit(STRIPE_PREREAD_ACTIVE,
  3747. &sh->state)) {
  3748. pr_debug("Read_old block %d for r-m-w\n",
  3749. i);
  3750. set_bit(R5_LOCKED, &dev->flags);
  3751. set_bit(R5_Wantread, &dev->flags);
  3752. s->locked++;
  3753. } else
  3754. set_bit(STRIPE_DELAYED, &sh->state);
  3755. }
  3756. }
  3757. }
  3758. if ((rcw < rmw || (rcw == rmw && conf->rmw_level != PARITY_PREFER_RMW)) && rcw > 0) {
  3759. /* want reconstruct write, but need to get some data */
  3760. int qread =0;
  3761. rcw = 0;
  3762. for (i = disks; i--; ) {
  3763. struct r5dev *dev = &sh->dev[i];
  3764. if (!test_bit(R5_OVERWRITE, &dev->flags) &&
  3765. i != sh->pd_idx && i != sh->qd_idx &&
  3766. !test_bit(R5_LOCKED, &dev->flags) &&
  3767. !(test_bit(R5_UPTODATE, &dev->flags) ||
  3768. test_bit(R5_Wantcompute, &dev->flags))) {
  3769. rcw++;
  3770. if (test_bit(R5_Insync, &dev->flags) &&
  3771. test_bit(STRIPE_PREREAD_ACTIVE,
  3772. &sh->state)) {
  3773. pr_debug("Read_old block "
  3774. "%d for Reconstruct\n", i);
  3775. set_bit(R5_LOCKED, &dev->flags);
  3776. set_bit(R5_Wantread, &dev->flags);
  3777. s->locked++;
  3778. qread++;
  3779. } else
  3780. set_bit(STRIPE_DELAYED, &sh->state);
  3781. }
  3782. }
  3783. if (rcw && conf->mddev->queue)
  3784. blk_add_trace_msg(conf->mddev->queue, "raid5 rcw %llu %d %d %d",
  3785. (unsigned long long)sh->sector,
  3786. rcw, qread, test_bit(STRIPE_DELAYED, &sh->state));
  3787. }
  3788. if (rcw > disks && rmw > disks &&
  3789. !test_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  3790. set_bit(STRIPE_DELAYED, &sh->state);
  3791. /* now if nothing is locked, and if we have enough data,
  3792. * we can start a write request
  3793. */
  3794. /* since handle_stripe can be called at any time we need to handle the
  3795. * case where a compute block operation has been submitted and then a
  3796. * subsequent call wants to start a write request. raid_run_ops only
  3797. * handles the case where compute block and reconstruct are requested
  3798. * simultaneously. If this is not the case then new writes need to be
  3799. * held off until the compute completes.
  3800. */
  3801. if ((s->req_compute || !test_bit(STRIPE_COMPUTE_RUN, &sh->state)) &&
  3802. (s->locked == 0 && (rcw == 0 || rmw == 0) &&
  3803. !test_bit(STRIPE_BIT_DELAY, &sh->state)))
  3804. schedule_reconstruction(sh, s, rcw == 0, 0);
  3805. return 0;
  3806. }
  3807. static void handle_parity_checks5(struct r5conf *conf, struct stripe_head *sh,
  3808. struct stripe_head_state *s, int disks)
  3809. {
  3810. struct r5dev *dev = NULL;
  3811. BUG_ON(sh->batch_head);
  3812. set_bit(STRIPE_HANDLE, &sh->state);
  3813. switch (sh->check_state) {
  3814. case check_state_idle:
  3815. /* start a new check operation if there are no failures */
  3816. if (s->failed == 0) {
  3817. BUG_ON(s->uptodate != disks);
  3818. sh->check_state = check_state_run;
  3819. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  3820. clear_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags);
  3821. s->uptodate--;
  3822. break;
  3823. }
  3824. dev = &sh->dev[s->failed_num[0]];
  3825. fallthrough;
  3826. case check_state_compute_result:
  3827. sh->check_state = check_state_idle;
  3828. if (!dev)
  3829. dev = &sh->dev[sh->pd_idx];
  3830. /* check that a write has not made the stripe insync */
  3831. if (test_bit(STRIPE_INSYNC, &sh->state))
  3832. break;
  3833. /* either failed parity check, or recovery is happening */
  3834. BUG_ON(!test_bit(R5_UPTODATE, &dev->flags));
  3835. BUG_ON(s->uptodate != disks);
  3836. set_bit(R5_LOCKED, &dev->flags);
  3837. s->locked++;
  3838. set_bit(R5_Wantwrite, &dev->flags);
  3839. clear_bit(STRIPE_DEGRADED, &sh->state);
  3840. set_bit(STRIPE_INSYNC, &sh->state);
  3841. break;
  3842. case check_state_run:
  3843. break; /* we will be called again upon completion */
  3844. case check_state_check_result:
  3845. sh->check_state = check_state_idle;
  3846. /* if a failure occurred during the check operation, leave
  3847. * STRIPE_INSYNC not set and let the stripe be handled again
  3848. */
  3849. if (s->failed)
  3850. break;
  3851. /* handle a successful check operation, if parity is correct
  3852. * we are done. Otherwise update the mismatch count and repair
  3853. * parity if !MD_RECOVERY_CHECK
  3854. */
  3855. if ((sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) == 0)
  3856. /* parity is correct (on disc,
  3857. * not in buffer any more)
  3858. */
  3859. set_bit(STRIPE_INSYNC, &sh->state);
  3860. else {
  3861. atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
  3862. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
  3863. /* don't try to repair!! */
  3864. set_bit(STRIPE_INSYNC, &sh->state);
  3865. pr_warn_ratelimited("%s: mismatch sector in range "
  3866. "%llu-%llu\n", mdname(conf->mddev),
  3867. (unsigned long long) sh->sector,
  3868. (unsigned long long) sh->sector +
  3869. RAID5_STRIPE_SECTORS(conf));
  3870. } else {
  3871. sh->check_state = check_state_compute_run;
  3872. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  3873. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  3874. set_bit(R5_Wantcompute,
  3875. &sh->dev[sh->pd_idx].flags);
  3876. sh->ops.target = sh->pd_idx;
  3877. sh->ops.target2 = -1;
  3878. s->uptodate++;
  3879. }
  3880. }
  3881. break;
  3882. case check_state_compute_run:
  3883. break;
  3884. default:
  3885. pr_err("%s: unknown check_state: %d sector: %llu\n",
  3886. __func__, sh->check_state,
  3887. (unsigned long long) sh->sector);
  3888. BUG();
  3889. }
  3890. }
  3891. static void handle_parity_checks6(struct r5conf *conf, struct stripe_head *sh,
  3892. struct stripe_head_state *s,
  3893. int disks)
  3894. {
  3895. int pd_idx = sh->pd_idx;
  3896. int qd_idx = sh->qd_idx;
  3897. struct r5dev *dev;
  3898. BUG_ON(sh->batch_head);
  3899. set_bit(STRIPE_HANDLE, &sh->state);
  3900. BUG_ON(s->failed > 2);
  3901. /* Want to check and possibly repair P and Q.
  3902. * However there could be one 'failed' device, in which
  3903. * case we can only check one of them, possibly using the
  3904. * other to generate missing data
  3905. */
  3906. switch (sh->check_state) {
  3907. case check_state_idle:
  3908. /* start a new check operation if there are < 2 failures */
  3909. if (s->failed == s->q_failed) {
  3910. /* The only possible failed device holds Q, so it
  3911. * makes sense to check P (If anything else were failed,
  3912. * we would have used P to recreate it).
  3913. */
  3914. sh->check_state = check_state_run;
  3915. }
  3916. if (!s->q_failed && s->failed < 2) {
  3917. /* Q is not failed, and we didn't use it to generate
  3918. * anything, so it makes sense to check it
  3919. */
  3920. if (sh->check_state == check_state_run)
  3921. sh->check_state = check_state_run_pq;
  3922. else
  3923. sh->check_state = check_state_run_q;
  3924. }
  3925. /* discard potentially stale zero_sum_result */
  3926. sh->ops.zero_sum_result = 0;
  3927. if (sh->check_state == check_state_run) {
  3928. /* async_xor_zero_sum destroys the contents of P */
  3929. clear_bit(R5_UPTODATE, &sh->dev[pd_idx].flags);
  3930. s->uptodate--;
  3931. }
  3932. if (sh->check_state >= check_state_run &&
  3933. sh->check_state <= check_state_run_pq) {
  3934. /* async_syndrome_zero_sum preserves P and Q, so
  3935. * no need to mark them !uptodate here
  3936. */
  3937. set_bit(STRIPE_OP_CHECK, &s->ops_request);
  3938. break;
  3939. }
  3940. /* we have 2-disk failure */
  3941. BUG_ON(s->failed != 2);
  3942. fallthrough;
  3943. case check_state_compute_result:
  3944. sh->check_state = check_state_idle;
  3945. /* check that a write has not made the stripe insync */
  3946. if (test_bit(STRIPE_INSYNC, &sh->state))
  3947. break;
  3948. /* now write out any block on a failed drive,
  3949. * or P or Q if they were recomputed
  3950. */
  3951. dev = NULL;
  3952. if (s->failed == 2) {
  3953. dev = &sh->dev[s->failed_num[1]];
  3954. s->locked++;
  3955. set_bit(R5_LOCKED, &dev->flags);
  3956. set_bit(R5_Wantwrite, &dev->flags);
  3957. }
  3958. if (s->failed >= 1) {
  3959. dev = &sh->dev[s->failed_num[0]];
  3960. s->locked++;
  3961. set_bit(R5_LOCKED, &dev->flags);
  3962. set_bit(R5_Wantwrite, &dev->flags);
  3963. }
  3964. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  3965. dev = &sh->dev[pd_idx];
  3966. s->locked++;
  3967. set_bit(R5_LOCKED, &dev->flags);
  3968. set_bit(R5_Wantwrite, &dev->flags);
  3969. }
  3970. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  3971. dev = &sh->dev[qd_idx];
  3972. s->locked++;
  3973. set_bit(R5_LOCKED, &dev->flags);
  3974. set_bit(R5_Wantwrite, &dev->flags);
  3975. }
  3976. if (WARN_ONCE(dev && !test_bit(R5_UPTODATE, &dev->flags),
  3977. "%s: disk%td not up to date\n",
  3978. mdname(conf->mddev),
  3979. dev - (struct r5dev *) &sh->dev)) {
  3980. clear_bit(R5_LOCKED, &dev->flags);
  3981. clear_bit(R5_Wantwrite, &dev->flags);
  3982. s->locked--;
  3983. }
  3984. clear_bit(STRIPE_DEGRADED, &sh->state);
  3985. set_bit(STRIPE_INSYNC, &sh->state);
  3986. break;
  3987. case check_state_run:
  3988. case check_state_run_q:
  3989. case check_state_run_pq:
  3990. break; /* we will be called again upon completion */
  3991. case check_state_check_result:
  3992. sh->check_state = check_state_idle;
  3993. /* handle a successful check operation, if parity is correct
  3994. * we are done. Otherwise update the mismatch count and repair
  3995. * parity if !MD_RECOVERY_CHECK
  3996. */
  3997. if (sh->ops.zero_sum_result == 0) {
  3998. /* both parities are correct */
  3999. if (!s->failed)
  4000. set_bit(STRIPE_INSYNC, &sh->state);
  4001. else {
  4002. /* in contrast to the raid5 case we can validate
  4003. * parity, but still have a failure to write
  4004. * back
  4005. */
  4006. sh->check_state = check_state_compute_result;
  4007. /* Returning at this point means that we may go
  4008. * off and bring p and/or q uptodate again so
  4009. * we make sure to check zero_sum_result again
  4010. * to verify if p or q need writeback
  4011. */
  4012. }
  4013. } else {
  4014. atomic64_add(RAID5_STRIPE_SECTORS(conf), &conf->mddev->resync_mismatches);
  4015. if (test_bit(MD_RECOVERY_CHECK, &conf->mddev->recovery)) {
  4016. /* don't try to repair!! */
  4017. set_bit(STRIPE_INSYNC, &sh->state);
  4018. pr_warn_ratelimited("%s: mismatch sector in range "
  4019. "%llu-%llu\n", mdname(conf->mddev),
  4020. (unsigned long long) sh->sector,
  4021. (unsigned long long) sh->sector +
  4022. RAID5_STRIPE_SECTORS(conf));
  4023. } else {
  4024. int *target = &sh->ops.target;
  4025. sh->ops.target = -1;
  4026. sh->ops.target2 = -1;
  4027. sh->check_state = check_state_compute_run;
  4028. set_bit(STRIPE_COMPUTE_RUN, &sh->state);
  4029. set_bit(STRIPE_OP_COMPUTE_BLK, &s->ops_request);
  4030. if (sh->ops.zero_sum_result & SUM_CHECK_P_RESULT) {
  4031. set_bit(R5_Wantcompute,
  4032. &sh->dev[pd_idx].flags);
  4033. *target = pd_idx;
  4034. target = &sh->ops.target2;
  4035. s->uptodate++;
  4036. }
  4037. if (sh->ops.zero_sum_result & SUM_CHECK_Q_RESULT) {
  4038. set_bit(R5_Wantcompute,
  4039. &sh->dev[qd_idx].flags);
  4040. *target = qd_idx;
  4041. s->uptodate++;
  4042. }
  4043. }
  4044. }
  4045. break;
  4046. case check_state_compute_run:
  4047. break;
  4048. default:
  4049. pr_warn("%s: unknown check_state: %d sector: %llu\n",
  4050. __func__, sh->check_state,
  4051. (unsigned long long) sh->sector);
  4052. BUG();
  4053. }
  4054. }
  4055. static void handle_stripe_expansion(struct r5conf *conf, struct stripe_head *sh)
  4056. {
  4057. int i;
  4058. /* We have read all the blocks in this stripe and now we need to
  4059. * copy some of them into a target stripe for expand.
  4060. */
  4061. struct dma_async_tx_descriptor *tx = NULL;
  4062. BUG_ON(sh->batch_head);
  4063. clear_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  4064. for (i = 0; i < sh->disks; i++)
  4065. if (i != sh->pd_idx && i != sh->qd_idx) {
  4066. int dd_idx, j;
  4067. struct stripe_head *sh2;
  4068. struct async_submit_ctl submit;
  4069. sector_t bn = raid5_compute_blocknr(sh, i, 1);
  4070. sector_t s = raid5_compute_sector(conf, bn, 0,
  4071. &dd_idx, NULL);
  4072. sh2 = raid5_get_active_stripe(conf, s, 0, 1, 1);
  4073. if (sh2 == NULL)
  4074. /* so far only the early blocks of this stripe
  4075. * have been requested. When later blocks
  4076. * get requested, we will try again
  4077. */
  4078. continue;
  4079. if (!test_bit(STRIPE_EXPANDING, &sh2->state) ||
  4080. test_bit(R5_Expanded, &sh2->dev[dd_idx].flags)) {
  4081. /* must have already done this block */
  4082. raid5_release_stripe(sh2);
  4083. continue;
  4084. }
  4085. /* place all the copies on one channel */
  4086. init_async_submit(&submit, 0, tx, NULL, NULL, NULL);
  4087. tx = async_memcpy(sh2->dev[dd_idx].page,
  4088. sh->dev[i].page, sh2->dev[dd_idx].offset,
  4089. sh->dev[i].offset, RAID5_STRIPE_SIZE(conf),
  4090. &submit);
  4091. set_bit(R5_Expanded, &sh2->dev[dd_idx].flags);
  4092. set_bit(R5_UPTODATE, &sh2->dev[dd_idx].flags);
  4093. for (j = 0; j < conf->raid_disks; j++)
  4094. if (j != sh2->pd_idx &&
  4095. j != sh2->qd_idx &&
  4096. !test_bit(R5_Expanded, &sh2->dev[j].flags))
  4097. break;
  4098. if (j == conf->raid_disks) {
  4099. set_bit(STRIPE_EXPAND_READY, &sh2->state);
  4100. set_bit(STRIPE_HANDLE, &sh2->state);
  4101. }
  4102. raid5_release_stripe(sh2);
  4103. }
  4104. /* done submitting copies, wait for them to complete */
  4105. async_tx_quiesce(&tx);
  4106. }
  4107. /*
  4108. * handle_stripe - do things to a stripe.
  4109. *
  4110. * We lock the stripe by setting STRIPE_ACTIVE and then examine the
  4111. * state of various bits to see what needs to be done.
  4112. * Possible results:
  4113. * return some read requests which now have data
  4114. * return some write requests which are safely on storage
  4115. * schedule a read on some buffers
  4116. * schedule a write of some buffers
  4117. * return confirmation of parity correctness
  4118. *
  4119. */
  4120. static void analyse_stripe(struct stripe_head *sh, struct stripe_head_state *s)
  4121. {
  4122. struct r5conf *conf = sh->raid_conf;
  4123. int disks = sh->disks;
  4124. struct r5dev *dev;
  4125. int i;
  4126. int do_recovery = 0;
  4127. memset(s, 0, sizeof(*s));
  4128. s->expanding = test_bit(STRIPE_EXPAND_SOURCE, &sh->state) && !sh->batch_head;
  4129. s->expanded = test_bit(STRIPE_EXPAND_READY, &sh->state) && !sh->batch_head;
  4130. s->failed_num[0] = -1;
  4131. s->failed_num[1] = -1;
  4132. s->log_failed = r5l_log_disk_error(conf);
  4133. /* Now to look around and see what can be done */
  4134. rcu_read_lock();
  4135. for (i=disks; i--; ) {
  4136. struct md_rdev *rdev;
  4137. sector_t first_bad;
  4138. int bad_sectors;
  4139. int is_bad = 0;
  4140. dev = &sh->dev[i];
  4141. pr_debug("check %d: state 0x%lx read %p write %p written %p\n",
  4142. i, dev->flags,
  4143. dev->toread, dev->towrite, dev->written);
  4144. /* maybe we can reply to a read
  4145. *
  4146. * new wantfill requests are only permitted while
  4147. * ops_complete_biofill is guaranteed to be inactive
  4148. */
  4149. if (test_bit(R5_UPTODATE, &dev->flags) && dev->toread &&
  4150. !test_bit(STRIPE_BIOFILL_RUN, &sh->state))
  4151. set_bit(R5_Wantfill, &dev->flags);
  4152. /* now count some things */
  4153. if (test_bit(R5_LOCKED, &dev->flags))
  4154. s->locked++;
  4155. if (test_bit(R5_UPTODATE, &dev->flags))
  4156. s->uptodate++;
  4157. if (test_bit(R5_Wantcompute, &dev->flags)) {
  4158. s->compute++;
  4159. BUG_ON(s->compute > 2);
  4160. }
  4161. if (test_bit(R5_Wantfill, &dev->flags))
  4162. s->to_fill++;
  4163. else if (dev->toread)
  4164. s->to_read++;
  4165. if (dev->towrite) {
  4166. s->to_write++;
  4167. if (!test_bit(R5_OVERWRITE, &dev->flags))
  4168. s->non_overwrite++;
  4169. }
  4170. if (dev->written)
  4171. s->written++;
  4172. /* Prefer to use the replacement for reads, but only
  4173. * if it is recovered enough and has no bad blocks.
  4174. */
  4175. rdev = rcu_dereference(conf->disks[i].replacement);
  4176. if (rdev && !test_bit(Faulty, &rdev->flags) &&
  4177. rdev->recovery_offset >= sh->sector + RAID5_STRIPE_SECTORS(conf) &&
  4178. !is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  4179. &first_bad, &bad_sectors))
  4180. set_bit(R5_ReadRepl, &dev->flags);
  4181. else {
  4182. if (rdev && !test_bit(Faulty, &rdev->flags))
  4183. set_bit(R5_NeedReplace, &dev->flags);
  4184. else
  4185. clear_bit(R5_NeedReplace, &dev->flags);
  4186. rdev = rcu_dereference(conf->disks[i].rdev);
  4187. clear_bit(R5_ReadRepl, &dev->flags);
  4188. }
  4189. if (rdev && test_bit(Faulty, &rdev->flags))
  4190. rdev = NULL;
  4191. if (rdev) {
  4192. is_bad = is_badblock(rdev, sh->sector, RAID5_STRIPE_SECTORS(conf),
  4193. &first_bad, &bad_sectors);
  4194. if (s->blocked_rdev == NULL
  4195. && (test_bit(Blocked, &rdev->flags)
  4196. || is_bad < 0)) {
  4197. if (is_bad < 0)
  4198. set_bit(BlockedBadBlocks,
  4199. &rdev->flags);
  4200. s->blocked_rdev = rdev;
  4201. atomic_inc(&rdev->nr_pending);
  4202. }
  4203. }
  4204. clear_bit(R5_Insync, &dev->flags);
  4205. if (!rdev)
  4206. /* Not in-sync */;
  4207. else if (is_bad) {
  4208. /* also not in-sync */
  4209. if (!test_bit(WriteErrorSeen, &rdev->flags) &&
  4210. test_bit(R5_UPTODATE, &dev->flags)) {
  4211. /* treat as in-sync, but with a read error
  4212. * which we can now try to correct
  4213. */
  4214. set_bit(R5_Insync, &dev->flags);
  4215. set_bit(R5_ReadError, &dev->flags);
  4216. }
  4217. } else if (test_bit(In_sync, &rdev->flags))
  4218. set_bit(R5_Insync, &dev->flags);
  4219. else if (sh->sector + RAID5_STRIPE_SECTORS(conf) <= rdev->recovery_offset)
  4220. /* in sync if before recovery_offset */
  4221. set_bit(R5_Insync, &dev->flags);
  4222. else if (test_bit(R5_UPTODATE, &dev->flags) &&
  4223. test_bit(R5_Expanded, &dev->flags))
  4224. /* If we've reshaped into here, we assume it is Insync.
  4225. * We will shortly update recovery_offset to make
  4226. * it official.
  4227. */
  4228. set_bit(R5_Insync, &dev->flags);
  4229. if (test_bit(R5_WriteError, &dev->flags)) {
  4230. /* This flag does not apply to '.replacement'
  4231. * only to .rdev, so make sure to check that*/
  4232. struct md_rdev *rdev2 = rcu_dereference(
  4233. conf->disks[i].rdev);
  4234. if (rdev2 == rdev)
  4235. clear_bit(R5_Insync, &dev->flags);
  4236. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4237. s->handle_bad_blocks = 1;
  4238. atomic_inc(&rdev2->nr_pending);
  4239. } else
  4240. clear_bit(R5_WriteError, &dev->flags);
  4241. }
  4242. if (test_bit(R5_MadeGood, &dev->flags)) {
  4243. /* This flag does not apply to '.replacement'
  4244. * only to .rdev, so make sure to check that*/
  4245. struct md_rdev *rdev2 = rcu_dereference(
  4246. conf->disks[i].rdev);
  4247. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4248. s->handle_bad_blocks = 1;
  4249. atomic_inc(&rdev2->nr_pending);
  4250. } else
  4251. clear_bit(R5_MadeGood, &dev->flags);
  4252. }
  4253. if (test_bit(R5_MadeGoodRepl, &dev->flags)) {
  4254. struct md_rdev *rdev2 = rcu_dereference(
  4255. conf->disks[i].replacement);
  4256. if (rdev2 && !test_bit(Faulty, &rdev2->flags)) {
  4257. s->handle_bad_blocks = 1;
  4258. atomic_inc(&rdev2->nr_pending);
  4259. } else
  4260. clear_bit(R5_MadeGoodRepl, &dev->flags);
  4261. }
  4262. if (!test_bit(R5_Insync, &dev->flags)) {
  4263. /* The ReadError flag will just be confusing now */
  4264. clear_bit(R5_ReadError, &dev->flags);
  4265. clear_bit(R5_ReWrite, &dev->flags);
  4266. }
  4267. if (test_bit(R5_ReadError, &dev->flags))
  4268. clear_bit(R5_Insync, &dev->flags);
  4269. if (!test_bit(R5_Insync, &dev->flags)) {
  4270. if (s->failed < 2)
  4271. s->failed_num[s->failed] = i;
  4272. s->failed++;
  4273. if (rdev && !test_bit(Faulty, &rdev->flags))
  4274. do_recovery = 1;
  4275. else if (!rdev) {
  4276. rdev = rcu_dereference(
  4277. conf->disks[i].replacement);
  4278. if (rdev && !test_bit(Faulty, &rdev->flags))
  4279. do_recovery = 1;
  4280. }
  4281. }
  4282. if (test_bit(R5_InJournal, &dev->flags))
  4283. s->injournal++;
  4284. if (test_bit(R5_InJournal, &dev->flags) && dev->written)
  4285. s->just_cached++;
  4286. }
  4287. if (test_bit(STRIPE_SYNCING, &sh->state)) {
  4288. /* If there is a failed device being replaced,
  4289. * we must be recovering.
  4290. * else if we are after recovery_cp, we must be syncing
  4291. * else if MD_RECOVERY_REQUESTED is set, we also are syncing.
  4292. * else we can only be replacing
  4293. * sync and recovery both need to read all devices, and so
  4294. * use the same flag.
  4295. */
  4296. if (do_recovery ||
  4297. sh->sector >= conf->mddev->recovery_cp ||
  4298. test_bit(MD_RECOVERY_REQUESTED, &(conf->mddev->recovery)))
  4299. s->syncing = 1;
  4300. else
  4301. s->replacing = 1;
  4302. }
  4303. rcu_read_unlock();
  4304. }
  4305. /*
  4306. * Return '1' if this is a member of batch, or '0' if it is a lone stripe or
  4307. * a head which can now be handled.
  4308. */
  4309. static int clear_batch_ready(struct stripe_head *sh)
  4310. {
  4311. struct stripe_head *tmp;
  4312. if (!test_and_clear_bit(STRIPE_BATCH_READY, &sh->state))
  4313. return (sh->batch_head && sh->batch_head != sh);
  4314. spin_lock(&sh->stripe_lock);
  4315. if (!sh->batch_head) {
  4316. spin_unlock(&sh->stripe_lock);
  4317. return 0;
  4318. }
  4319. /*
  4320. * this stripe could be added to a batch list before we check
  4321. * BATCH_READY, skips it
  4322. */
  4323. if (sh->batch_head != sh) {
  4324. spin_unlock(&sh->stripe_lock);
  4325. return 1;
  4326. }
  4327. spin_lock(&sh->batch_lock);
  4328. list_for_each_entry(tmp, &sh->batch_list, batch_list)
  4329. clear_bit(STRIPE_BATCH_READY, &tmp->state);
  4330. spin_unlock(&sh->batch_lock);
  4331. spin_unlock(&sh->stripe_lock);
  4332. /*
  4333. * BATCH_READY is cleared, no new stripes can be added.
  4334. * batch_list can be accessed without lock
  4335. */
  4336. return 0;
  4337. }
  4338. static void break_stripe_batch_list(struct stripe_head *head_sh,
  4339. unsigned long handle_flags)
  4340. {
  4341. struct stripe_head *sh, *next;
  4342. int i;
  4343. int do_wakeup = 0;
  4344. list_for_each_entry_safe(sh, next, &head_sh->batch_list, batch_list) {
  4345. list_del_init(&sh->batch_list);
  4346. WARN_ONCE(sh->state & ((1 << STRIPE_ACTIVE) |
  4347. (1 << STRIPE_SYNCING) |
  4348. (1 << STRIPE_REPLACED) |
  4349. (1 << STRIPE_DELAYED) |
  4350. (1 << STRIPE_BIT_DELAY) |
  4351. (1 << STRIPE_FULL_WRITE) |
  4352. (1 << STRIPE_BIOFILL_RUN) |
  4353. (1 << STRIPE_COMPUTE_RUN) |
  4354. (1 << STRIPE_DISCARD) |
  4355. (1 << STRIPE_BATCH_READY) |
  4356. (1 << STRIPE_BATCH_ERR) |
  4357. (1 << STRIPE_BITMAP_PENDING)),
  4358. "stripe state: %lx\n", sh->state);
  4359. WARN_ONCE(head_sh->state & ((1 << STRIPE_DISCARD) |
  4360. (1 << STRIPE_REPLACED)),
  4361. "head stripe state: %lx\n", head_sh->state);
  4362. set_mask_bits(&sh->state, ~(STRIPE_EXPAND_SYNC_FLAGS |
  4363. (1 << STRIPE_PREREAD_ACTIVE) |
  4364. (1 << STRIPE_DEGRADED) |
  4365. (1 << STRIPE_ON_UNPLUG_LIST)),
  4366. head_sh->state & (1 << STRIPE_INSYNC));
  4367. sh->check_state = head_sh->check_state;
  4368. sh->reconstruct_state = head_sh->reconstruct_state;
  4369. spin_lock_irq(&sh->stripe_lock);
  4370. sh->batch_head = NULL;
  4371. spin_unlock_irq(&sh->stripe_lock);
  4372. for (i = 0; i < sh->disks; i++) {
  4373. if (test_and_clear_bit(R5_Overlap, &sh->dev[i].flags))
  4374. do_wakeup = 1;
  4375. sh->dev[i].flags = head_sh->dev[i].flags &
  4376. (~((1 << R5_WriteError) | (1 << R5_Overlap)));
  4377. }
  4378. if (handle_flags == 0 ||
  4379. sh->state & handle_flags)
  4380. set_bit(STRIPE_HANDLE, &sh->state);
  4381. raid5_release_stripe(sh);
  4382. }
  4383. spin_lock_irq(&head_sh->stripe_lock);
  4384. head_sh->batch_head = NULL;
  4385. spin_unlock_irq(&head_sh->stripe_lock);
  4386. for (i = 0; i < head_sh->disks; i++)
  4387. if (test_and_clear_bit(R5_Overlap, &head_sh->dev[i].flags))
  4388. do_wakeup = 1;
  4389. if (head_sh->state & handle_flags)
  4390. set_bit(STRIPE_HANDLE, &head_sh->state);
  4391. if (do_wakeup)
  4392. wake_up(&head_sh->raid_conf->wait_for_overlap);
  4393. }
  4394. static void handle_stripe(struct stripe_head *sh)
  4395. {
  4396. struct stripe_head_state s;
  4397. struct r5conf *conf = sh->raid_conf;
  4398. int i;
  4399. int prexor;
  4400. int disks = sh->disks;
  4401. struct r5dev *pdev, *qdev;
  4402. clear_bit(STRIPE_HANDLE, &sh->state);
  4403. /*
  4404. * handle_stripe should not continue handle the batched stripe, only
  4405. * the head of batch list or lone stripe can continue. Otherwise we
  4406. * could see break_stripe_batch_list warns about the STRIPE_ACTIVE
  4407. * is set for the batched stripe.
  4408. */
  4409. if (clear_batch_ready(sh))
  4410. return;
  4411. if (test_and_set_bit_lock(STRIPE_ACTIVE, &sh->state)) {
  4412. /* already being handled, ensure it gets handled
  4413. * again when current action finishes */
  4414. set_bit(STRIPE_HANDLE, &sh->state);
  4415. return;
  4416. }
  4417. if (test_and_clear_bit(STRIPE_BATCH_ERR, &sh->state))
  4418. break_stripe_batch_list(sh, 0);
  4419. if (test_bit(STRIPE_SYNC_REQUESTED, &sh->state) && !sh->batch_head) {
  4420. spin_lock(&sh->stripe_lock);
  4421. /*
  4422. * Cannot process 'sync' concurrently with 'discard'.
  4423. * Flush data in r5cache before 'sync'.
  4424. */
  4425. if (!test_bit(STRIPE_R5C_PARTIAL_STRIPE, &sh->state) &&
  4426. !test_bit(STRIPE_R5C_FULL_STRIPE, &sh->state) &&
  4427. !test_bit(STRIPE_DISCARD, &sh->state) &&
  4428. test_and_clear_bit(STRIPE_SYNC_REQUESTED, &sh->state)) {
  4429. set_bit(STRIPE_SYNCING, &sh->state);
  4430. clear_bit(STRIPE_INSYNC, &sh->state);
  4431. clear_bit(STRIPE_REPLACED, &sh->state);
  4432. }
  4433. spin_unlock(&sh->stripe_lock);
  4434. }
  4435. clear_bit(STRIPE_DELAYED, &sh->state);
  4436. pr_debug("handling stripe %llu, state=%#lx cnt=%d, "
  4437. "pd_idx=%d, qd_idx=%d\n, check:%d, reconstruct:%d\n",
  4438. (unsigned long long)sh->sector, sh->state,
  4439. atomic_read(&sh->count), sh->pd_idx, sh->qd_idx,
  4440. sh->check_state, sh->reconstruct_state);
  4441. analyse_stripe(sh, &s);
  4442. if (test_bit(STRIPE_LOG_TRAPPED, &sh->state))
  4443. goto finish;
  4444. if (s.handle_bad_blocks ||
  4445. test_bit(MD_SB_CHANGE_PENDING, &conf->mddev->sb_flags)) {
  4446. set_bit(STRIPE_HANDLE, &sh->state);
  4447. goto finish;
  4448. }
  4449. if (unlikely(s.blocked_rdev)) {
  4450. if (s.syncing || s.expanding || s.expanded ||
  4451. s.replacing || s.to_write || s.written) {
  4452. set_bit(STRIPE_HANDLE, &sh->state);
  4453. goto finish;
  4454. }
  4455. /* There is nothing for the blocked_rdev to block */
  4456. rdev_dec_pending(s.blocked_rdev, conf->mddev);
  4457. s.blocked_rdev = NULL;
  4458. }
  4459. if (s.to_fill && !test_bit(STRIPE_BIOFILL_RUN, &sh->state)) {
  4460. set_bit(STRIPE_OP_BIOFILL, &s.ops_request);
  4461. set_bit(STRIPE_BIOFILL_RUN, &sh->state);
  4462. }
  4463. pr_debug("locked=%d uptodate=%d to_read=%d"
  4464. " to_write=%d failed=%d failed_num=%d,%d\n",
  4465. s.locked, s.uptodate, s.to_read, s.to_write, s.failed,
  4466. s.failed_num[0], s.failed_num[1]);
  4467. /*
  4468. * check if the array has lost more than max_degraded devices and,
  4469. * if so, some requests might need to be failed.
  4470. *
  4471. * When journal device failed (log_failed), we will only process
  4472. * the stripe if there is data need write to raid disks
  4473. */
  4474. if (s.failed > conf->max_degraded ||
  4475. (s.log_failed && s.injournal == 0)) {
  4476. sh->check_state = 0;
  4477. sh->reconstruct_state = 0;
  4478. break_stripe_batch_list(sh, 0);
  4479. if (s.to_read+s.to_write+s.written)
  4480. handle_failed_stripe(conf, sh, &s, disks);
  4481. if (s.syncing + s.replacing)
  4482. handle_failed_sync(conf, sh, &s);
  4483. }
  4484. /* Now we check to see if any write operations have recently
  4485. * completed
  4486. */
  4487. prexor = 0;
  4488. if (sh->reconstruct_state == reconstruct_state_prexor_drain_result)
  4489. prexor = 1;
  4490. if (sh->reconstruct_state == reconstruct_state_drain_result ||
  4491. sh->reconstruct_state == reconstruct_state_prexor_drain_result) {
  4492. sh->reconstruct_state = reconstruct_state_idle;
  4493. /* All the 'written' buffers and the parity block are ready to
  4494. * be written back to disk
  4495. */
  4496. BUG_ON(!test_bit(R5_UPTODATE, &sh->dev[sh->pd_idx].flags) &&
  4497. !test_bit(R5_Discard, &sh->dev[sh->pd_idx].flags));
  4498. BUG_ON(sh->qd_idx >= 0 &&
  4499. !test_bit(R5_UPTODATE, &sh->dev[sh->qd_idx].flags) &&
  4500. !test_bit(R5_Discard, &sh->dev[sh->qd_idx].flags));
  4501. for (i = disks; i--; ) {
  4502. struct r5dev *dev = &sh->dev[i];
  4503. if (test_bit(R5_LOCKED, &dev->flags) &&
  4504. (i == sh->pd_idx || i == sh->qd_idx ||
  4505. dev->written || test_bit(R5_InJournal,
  4506. &dev->flags))) {
  4507. pr_debug("Writing block %d\n", i);
  4508. set_bit(R5_Wantwrite, &dev->flags);
  4509. if (prexor)
  4510. continue;
  4511. if (s.failed > 1)
  4512. continue;
  4513. if (!test_bit(R5_Insync, &dev->flags) ||
  4514. ((i == sh->pd_idx || i == sh->qd_idx) &&
  4515. s.failed == 0))
  4516. set_bit(STRIPE_INSYNC, &sh->state);
  4517. }
  4518. }
  4519. if (test_and_clear_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  4520. s.dec_preread_active = 1;
  4521. }
  4522. /*
  4523. * might be able to return some write requests if the parity blocks
  4524. * are safe, or on a failed drive
  4525. */
  4526. pdev = &sh->dev[sh->pd_idx];
  4527. s.p_failed = (s.failed >= 1 && s.failed_num[0] == sh->pd_idx)
  4528. || (s.failed >= 2 && s.failed_num[1] == sh->pd_idx);
  4529. qdev = &sh->dev[sh->qd_idx];
  4530. s.q_failed = (s.failed >= 1 && s.failed_num[0] == sh->qd_idx)
  4531. || (s.failed >= 2 && s.failed_num[1] == sh->qd_idx)
  4532. || conf->level < 6;
  4533. if (s.written &&
  4534. (s.p_failed || ((test_bit(R5_Insync, &pdev->flags)
  4535. && !test_bit(R5_LOCKED, &pdev->flags)
  4536. && (test_bit(R5_UPTODATE, &pdev->flags) ||
  4537. test_bit(R5_Discard, &pdev->flags))))) &&
  4538. (s.q_failed || ((test_bit(R5_Insync, &qdev->flags)
  4539. && !test_bit(R5_LOCKED, &qdev->flags)
  4540. && (test_bit(R5_UPTODATE, &qdev->flags) ||
  4541. test_bit(R5_Discard, &qdev->flags))))))
  4542. handle_stripe_clean_event(conf, sh, disks);
  4543. if (s.just_cached)
  4544. r5c_handle_cached_data_endio(conf, sh, disks);
  4545. log_stripe_write_finished(sh);
  4546. /* Now we might consider reading some blocks, either to check/generate
  4547. * parity, or to satisfy requests
  4548. * or to load a block that is being partially written.
  4549. */
  4550. if (s.to_read || s.non_overwrite
  4551. || (s.to_write && s.failed)
  4552. || (s.syncing && (s.uptodate + s.compute < disks))
  4553. || s.replacing
  4554. || s.expanding)
  4555. handle_stripe_fill(sh, &s, disks);
  4556. /*
  4557. * When the stripe finishes full journal write cycle (write to journal
  4558. * and raid disk), this is the clean up procedure so it is ready for
  4559. * next operation.
  4560. */
  4561. r5c_finish_stripe_write_out(conf, sh, &s);
  4562. /*
  4563. * Now to consider new write requests, cache write back and what else,
  4564. * if anything should be read. We do not handle new writes when:
  4565. * 1/ A 'write' operation (copy+xor) is already in flight.
  4566. * 2/ A 'check' operation is in flight, as it may clobber the parity
  4567. * block.
  4568. * 3/ A r5c cache log write is in flight.
  4569. */
  4570. if (!sh->reconstruct_state && !sh->check_state && !sh->log_io) {
  4571. if (!r5c_is_writeback(conf->log)) {
  4572. if (s.to_write)
  4573. handle_stripe_dirtying(conf, sh, &s, disks);
  4574. } else { /* write back cache */
  4575. int ret = 0;
  4576. /* First, try handle writes in caching phase */
  4577. if (s.to_write)
  4578. ret = r5c_try_caching_write(conf, sh, &s,
  4579. disks);
  4580. /*
  4581. * If caching phase failed: ret == -EAGAIN
  4582. * OR
  4583. * stripe under reclaim: !caching && injournal
  4584. *
  4585. * fall back to handle_stripe_dirtying()
  4586. */
  4587. if (ret == -EAGAIN ||
  4588. /* stripe under reclaim: !caching && injournal */
  4589. (!test_bit(STRIPE_R5C_CACHING, &sh->state) &&
  4590. s.injournal > 0)) {
  4591. ret = handle_stripe_dirtying(conf, sh, &s,
  4592. disks);
  4593. if (ret == -EAGAIN)
  4594. goto finish;
  4595. }
  4596. }
  4597. }
  4598. /* maybe we need to check and possibly fix the parity for this stripe
  4599. * Any reads will already have been scheduled, so we just see if enough
  4600. * data is available. The parity check is held off while parity
  4601. * dependent operations are in flight.
  4602. */
  4603. if (sh->check_state ||
  4604. (s.syncing && s.locked == 0 &&
  4605. !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
  4606. !test_bit(STRIPE_INSYNC, &sh->state))) {
  4607. if (conf->level == 6)
  4608. handle_parity_checks6(conf, sh, &s, disks);
  4609. else
  4610. handle_parity_checks5(conf, sh, &s, disks);
  4611. }
  4612. if ((s.replacing || s.syncing) && s.locked == 0
  4613. && !test_bit(STRIPE_COMPUTE_RUN, &sh->state)
  4614. && !test_bit(STRIPE_REPLACED, &sh->state)) {
  4615. /* Write out to replacement devices where possible */
  4616. for (i = 0; i < conf->raid_disks; i++)
  4617. if (test_bit(R5_NeedReplace, &sh->dev[i].flags)) {
  4618. WARN_ON(!test_bit(R5_UPTODATE, &sh->dev[i].flags));
  4619. set_bit(R5_WantReplace, &sh->dev[i].flags);
  4620. set_bit(R5_LOCKED, &sh->dev[i].flags);
  4621. s.locked++;
  4622. }
  4623. if (s.replacing)
  4624. set_bit(STRIPE_INSYNC, &sh->state);
  4625. set_bit(STRIPE_REPLACED, &sh->state);
  4626. }
  4627. if ((s.syncing || s.replacing) && s.locked == 0 &&
  4628. !test_bit(STRIPE_COMPUTE_RUN, &sh->state) &&
  4629. test_bit(STRIPE_INSYNC, &sh->state)) {
  4630. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
  4631. clear_bit(STRIPE_SYNCING, &sh->state);
  4632. if (test_and_clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags))
  4633. wake_up(&conf->wait_for_overlap);
  4634. }
  4635. /* If the failed drives are just a ReadError, then we might need
  4636. * to progress the repair/check process
  4637. */
  4638. if (s.failed <= conf->max_degraded && !conf->mddev->ro)
  4639. for (i = 0; i < s.failed; i++) {
  4640. struct r5dev *dev = &sh->dev[s.failed_num[i]];
  4641. if (test_bit(R5_ReadError, &dev->flags)
  4642. && !test_bit(R5_LOCKED, &dev->flags)
  4643. && test_bit(R5_UPTODATE, &dev->flags)
  4644. ) {
  4645. if (!test_bit(R5_ReWrite, &dev->flags)) {
  4646. set_bit(R5_Wantwrite, &dev->flags);
  4647. set_bit(R5_ReWrite, &dev->flags);
  4648. } else
  4649. /* let's read it back */
  4650. set_bit(R5_Wantread, &dev->flags);
  4651. set_bit(R5_LOCKED, &dev->flags);
  4652. s.locked++;
  4653. }
  4654. }
  4655. /* Finish reconstruct operations initiated by the expansion process */
  4656. if (sh->reconstruct_state == reconstruct_state_result) {
  4657. struct stripe_head *sh_src
  4658. = raid5_get_active_stripe(conf, sh->sector, 1, 1, 1);
  4659. if (sh_src && test_bit(STRIPE_EXPAND_SOURCE, &sh_src->state)) {
  4660. /* sh cannot be written until sh_src has been read.
  4661. * so arrange for sh to be delayed a little
  4662. */
  4663. set_bit(STRIPE_DELAYED, &sh->state);
  4664. set_bit(STRIPE_HANDLE, &sh->state);
  4665. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE,
  4666. &sh_src->state))
  4667. atomic_inc(&conf->preread_active_stripes);
  4668. raid5_release_stripe(sh_src);
  4669. goto finish;
  4670. }
  4671. if (sh_src)
  4672. raid5_release_stripe(sh_src);
  4673. sh->reconstruct_state = reconstruct_state_idle;
  4674. clear_bit(STRIPE_EXPANDING, &sh->state);
  4675. for (i = conf->raid_disks; i--; ) {
  4676. set_bit(R5_Wantwrite, &sh->dev[i].flags);
  4677. set_bit(R5_LOCKED, &sh->dev[i].flags);
  4678. s.locked++;
  4679. }
  4680. }
  4681. if (s.expanded && test_bit(STRIPE_EXPANDING, &sh->state) &&
  4682. !sh->reconstruct_state) {
  4683. /* Need to write out all blocks after computing parity */
  4684. sh->disks = conf->raid_disks;
  4685. stripe_set_idx(sh->sector, conf, 0, sh);
  4686. schedule_reconstruction(sh, &s, 1, 1);
  4687. } else if (s.expanded && !sh->reconstruct_state && s.locked == 0) {
  4688. clear_bit(STRIPE_EXPAND_READY, &sh->state);
  4689. atomic_dec(&conf->reshape_stripes);
  4690. wake_up(&conf->wait_for_overlap);
  4691. md_done_sync(conf->mddev, RAID5_STRIPE_SECTORS(conf), 1);
  4692. }
  4693. if (s.expanding && s.locked == 0 &&
  4694. !test_bit(STRIPE_COMPUTE_RUN, &sh->state))
  4695. handle_stripe_expansion(conf, sh);
  4696. finish:
  4697. /* wait for this device to become unblocked */
  4698. if (unlikely(s.blocked_rdev)) {
  4699. if (conf->mddev->external)
  4700. md_wait_for_blocked_rdev(s.blocked_rdev,
  4701. conf->mddev);
  4702. else
  4703. /* Internal metadata will immediately
  4704. * be written by raid5d, so we don't
  4705. * need to wait here.
  4706. */
  4707. rdev_dec_pending(s.blocked_rdev,
  4708. conf->mddev);
  4709. }
  4710. if (s.handle_bad_blocks)
  4711. for (i = disks; i--; ) {
  4712. struct md_rdev *rdev;
  4713. struct r5dev *dev = &sh->dev[i];
  4714. if (test_and_clear_bit(R5_WriteError, &dev->flags)) {
  4715. /* We own a safe reference to the rdev */
  4716. rdev = conf->disks[i].rdev;
  4717. if (!rdev_set_badblocks(rdev, sh->sector,
  4718. RAID5_STRIPE_SECTORS(conf), 0))
  4719. md_error(conf->mddev, rdev);
  4720. rdev_dec_pending(rdev, conf->mddev);
  4721. }
  4722. if (test_and_clear_bit(R5_MadeGood, &dev->flags)) {
  4723. rdev = conf->disks[i].rdev;
  4724. rdev_clear_badblocks(rdev, sh->sector,
  4725. RAID5_STRIPE_SECTORS(conf), 0);
  4726. rdev_dec_pending(rdev, conf->mddev);
  4727. }
  4728. if (test_and_clear_bit(R5_MadeGoodRepl, &dev->flags)) {
  4729. rdev = conf->disks[i].replacement;
  4730. if (!rdev)
  4731. /* rdev have been moved down */
  4732. rdev = conf->disks[i].rdev;
  4733. rdev_clear_badblocks(rdev, sh->sector,
  4734. RAID5_STRIPE_SECTORS(conf), 0);
  4735. rdev_dec_pending(rdev, conf->mddev);
  4736. }
  4737. }
  4738. if (s.ops_request)
  4739. raid_run_ops(sh, s.ops_request);
  4740. ops_run_io(sh, &s);
  4741. if (s.dec_preread_active) {
  4742. /* We delay this until after ops_run_io so that if make_request
  4743. * is waiting on a flush, it won't continue until the writes
  4744. * have actually been submitted.
  4745. */
  4746. atomic_dec(&conf->preread_active_stripes);
  4747. if (atomic_read(&conf->preread_active_stripes) <
  4748. IO_THRESHOLD)
  4749. md_wakeup_thread(conf->mddev->thread);
  4750. }
  4751. clear_bit_unlock(STRIPE_ACTIVE, &sh->state);
  4752. }
  4753. static void raid5_activate_delayed(struct r5conf *conf)
  4754. {
  4755. if (atomic_read(&conf->preread_active_stripes) < IO_THRESHOLD) {
  4756. while (!list_empty(&conf->delayed_list)) {
  4757. struct list_head *l = conf->delayed_list.next;
  4758. struct stripe_head *sh;
  4759. sh = list_entry(l, struct stripe_head, lru);
  4760. list_del_init(l);
  4761. clear_bit(STRIPE_DELAYED, &sh->state);
  4762. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  4763. atomic_inc(&conf->preread_active_stripes);
  4764. list_add_tail(&sh->lru, &conf->hold_list);
  4765. raid5_wakeup_stripe_thread(sh);
  4766. }
  4767. }
  4768. }
  4769. static void activate_bit_delay(struct r5conf *conf,
  4770. struct list_head *temp_inactive_list)
  4771. {
  4772. /* device_lock is held */
  4773. struct list_head head;
  4774. list_add(&head, &conf->bitmap_list);
  4775. list_del_init(&conf->bitmap_list);
  4776. while (!list_empty(&head)) {
  4777. struct stripe_head *sh = list_entry(head.next, struct stripe_head, lru);
  4778. int hash;
  4779. list_del_init(&sh->lru);
  4780. atomic_inc(&sh->count);
  4781. hash = sh->hash_lock_index;
  4782. __release_stripe(conf, sh, &temp_inactive_list[hash]);
  4783. }
  4784. }
  4785. static int in_chunk_boundary(struct mddev *mddev, struct bio *bio)
  4786. {
  4787. struct r5conf *conf = mddev->private;
  4788. sector_t sector = bio->bi_iter.bi_sector;
  4789. unsigned int chunk_sectors;
  4790. unsigned int bio_sectors = bio_sectors(bio);
  4791. WARN_ON_ONCE(bio->bi_partno);
  4792. chunk_sectors = min(conf->chunk_sectors, conf->prev_chunk_sectors);
  4793. return chunk_sectors >=
  4794. ((sector & (chunk_sectors - 1)) + bio_sectors);
  4795. }
  4796. /*
  4797. * add bio to the retry LIFO ( in O(1) ... we are in interrupt )
  4798. * later sampled by raid5d.
  4799. */
  4800. static void add_bio_to_retry(struct bio *bi,struct r5conf *conf)
  4801. {
  4802. unsigned long flags;
  4803. spin_lock_irqsave(&conf->device_lock, flags);
  4804. bi->bi_next = conf->retry_read_aligned_list;
  4805. conf->retry_read_aligned_list = bi;
  4806. spin_unlock_irqrestore(&conf->device_lock, flags);
  4807. md_wakeup_thread(conf->mddev->thread);
  4808. }
  4809. static struct bio *remove_bio_from_retry(struct r5conf *conf,
  4810. unsigned int *offset)
  4811. {
  4812. struct bio *bi;
  4813. bi = conf->retry_read_aligned;
  4814. if (bi) {
  4815. *offset = conf->retry_read_offset;
  4816. conf->retry_read_aligned = NULL;
  4817. return bi;
  4818. }
  4819. bi = conf->retry_read_aligned_list;
  4820. if(bi) {
  4821. conf->retry_read_aligned_list = bi->bi_next;
  4822. bi->bi_next = NULL;
  4823. *offset = 0;
  4824. }
  4825. return bi;
  4826. }
  4827. /*
  4828. * The "raid5_align_endio" should check if the read succeeded and if it
  4829. * did, call bio_endio on the original bio (having bio_put the new bio
  4830. * first).
  4831. * If the read failed..
  4832. */
  4833. static void raid5_align_endio(struct bio *bi)
  4834. {
  4835. struct bio* raid_bi = bi->bi_private;
  4836. struct mddev *mddev;
  4837. struct r5conf *conf;
  4838. struct md_rdev *rdev;
  4839. blk_status_t error = bi->bi_status;
  4840. bio_put(bi);
  4841. rdev = (void*)raid_bi->bi_next;
  4842. raid_bi->bi_next = NULL;
  4843. mddev = rdev->mddev;
  4844. conf = mddev->private;
  4845. rdev_dec_pending(rdev, conf->mddev);
  4846. if (!error) {
  4847. bio_endio(raid_bi);
  4848. if (atomic_dec_and_test(&conf->active_aligned_reads))
  4849. wake_up(&conf->wait_for_quiescent);
  4850. return;
  4851. }
  4852. pr_debug("raid5_align_endio : io error...handing IO for a retry\n");
  4853. add_bio_to_retry(raid_bi, conf);
  4854. }
  4855. static int raid5_read_one_chunk(struct mddev *mddev, struct bio *raid_bio)
  4856. {
  4857. struct r5conf *conf = mddev->private;
  4858. int dd_idx;
  4859. struct bio* align_bi;
  4860. struct md_rdev *rdev;
  4861. sector_t end_sector;
  4862. if (!in_chunk_boundary(mddev, raid_bio)) {
  4863. pr_debug("%s: non aligned\n", __func__);
  4864. return 0;
  4865. }
  4866. /*
  4867. * use bio_clone_fast to make a copy of the bio
  4868. */
  4869. align_bi = bio_clone_fast(raid_bio, GFP_NOIO, &mddev->bio_set);
  4870. if (!align_bi)
  4871. return 0;
  4872. /*
  4873. * set bi_end_io to a new function, and set bi_private to the
  4874. * original bio.
  4875. */
  4876. align_bi->bi_end_io = raid5_align_endio;
  4877. align_bi->bi_private = raid_bio;
  4878. /*
  4879. * compute position
  4880. */
  4881. align_bi->bi_iter.bi_sector =
  4882. raid5_compute_sector(conf, raid_bio->bi_iter.bi_sector,
  4883. 0, &dd_idx, NULL);
  4884. end_sector = bio_end_sector(align_bi);
  4885. rcu_read_lock();
  4886. rdev = rcu_dereference(conf->disks[dd_idx].replacement);
  4887. if (!rdev || test_bit(Faulty, &rdev->flags) ||
  4888. rdev->recovery_offset < end_sector) {
  4889. rdev = rcu_dereference(conf->disks[dd_idx].rdev);
  4890. if (rdev &&
  4891. (test_bit(Faulty, &rdev->flags) ||
  4892. !(test_bit(In_sync, &rdev->flags) ||
  4893. rdev->recovery_offset >= end_sector)))
  4894. rdev = NULL;
  4895. }
  4896. if (r5c_big_stripe_cached(conf, align_bi->bi_iter.bi_sector)) {
  4897. rcu_read_unlock();
  4898. bio_put(align_bi);
  4899. return 0;
  4900. }
  4901. if (rdev) {
  4902. sector_t first_bad;
  4903. int bad_sectors;
  4904. atomic_inc(&rdev->nr_pending);
  4905. rcu_read_unlock();
  4906. raid_bio->bi_next = (void*)rdev;
  4907. bio_set_dev(align_bi, rdev->bdev);
  4908. if (is_badblock(rdev, align_bi->bi_iter.bi_sector,
  4909. bio_sectors(align_bi),
  4910. &first_bad, &bad_sectors)) {
  4911. bio_put(align_bi);
  4912. rdev_dec_pending(rdev, mddev);
  4913. return 0;
  4914. }
  4915. /* No reshape active, so we can trust rdev->data_offset */
  4916. align_bi->bi_iter.bi_sector += rdev->data_offset;
  4917. spin_lock_irq(&conf->device_lock);
  4918. wait_event_lock_irq(conf->wait_for_quiescent,
  4919. conf->quiesce == 0,
  4920. conf->device_lock);
  4921. atomic_inc(&conf->active_aligned_reads);
  4922. spin_unlock_irq(&conf->device_lock);
  4923. if (mddev->gendisk)
  4924. trace_block_bio_remap(align_bi->bi_disk->queue,
  4925. align_bi, disk_devt(mddev->gendisk),
  4926. raid_bio->bi_iter.bi_sector);
  4927. submit_bio_noacct(align_bi);
  4928. return 1;
  4929. } else {
  4930. rcu_read_unlock();
  4931. bio_put(align_bi);
  4932. return 0;
  4933. }
  4934. }
  4935. static struct bio *chunk_aligned_read(struct mddev *mddev, struct bio *raid_bio)
  4936. {
  4937. struct bio *split;
  4938. sector_t sector = raid_bio->bi_iter.bi_sector;
  4939. unsigned chunk_sects = mddev->chunk_sectors;
  4940. unsigned sectors = chunk_sects - (sector & (chunk_sects-1));
  4941. if (sectors < bio_sectors(raid_bio)) {
  4942. struct r5conf *conf = mddev->private;
  4943. split = bio_split(raid_bio, sectors, GFP_NOIO, &conf->bio_split);
  4944. bio_chain(split, raid_bio);
  4945. submit_bio_noacct(raid_bio);
  4946. raid_bio = split;
  4947. }
  4948. if (!raid5_read_one_chunk(mddev, raid_bio))
  4949. return raid_bio;
  4950. return NULL;
  4951. }
  4952. /* __get_priority_stripe - get the next stripe to process
  4953. *
  4954. * Full stripe writes are allowed to pass preread active stripes up until
  4955. * the bypass_threshold is exceeded. In general the bypass_count
  4956. * increments when the handle_list is handled before the hold_list; however, it
  4957. * will not be incremented when STRIPE_IO_STARTED is sampled set signifying a
  4958. * stripe with in flight i/o. The bypass_count will be reset when the
  4959. * head of the hold_list has changed, i.e. the head was promoted to the
  4960. * handle_list.
  4961. */
  4962. static struct stripe_head *__get_priority_stripe(struct r5conf *conf, int group)
  4963. {
  4964. struct stripe_head *sh, *tmp;
  4965. struct list_head *handle_list = NULL;
  4966. struct r5worker_group *wg;
  4967. bool second_try = !r5c_is_writeback(conf->log) &&
  4968. !r5l_log_disk_error(conf);
  4969. bool try_loprio = test_bit(R5C_LOG_TIGHT, &conf->cache_state) ||
  4970. r5l_log_disk_error(conf);
  4971. again:
  4972. wg = NULL;
  4973. sh = NULL;
  4974. if (conf->worker_cnt_per_group == 0) {
  4975. handle_list = try_loprio ? &conf->loprio_list :
  4976. &conf->handle_list;
  4977. } else if (group != ANY_GROUP) {
  4978. handle_list = try_loprio ? &conf->worker_groups[group].loprio_list :
  4979. &conf->worker_groups[group].handle_list;
  4980. wg = &conf->worker_groups[group];
  4981. } else {
  4982. int i;
  4983. for (i = 0; i < conf->group_cnt; i++) {
  4984. handle_list = try_loprio ? &conf->worker_groups[i].loprio_list :
  4985. &conf->worker_groups[i].handle_list;
  4986. wg = &conf->worker_groups[i];
  4987. if (!list_empty(handle_list))
  4988. break;
  4989. }
  4990. }
  4991. pr_debug("%s: handle: %s hold: %s full_writes: %d bypass_count: %d\n",
  4992. __func__,
  4993. list_empty(handle_list) ? "empty" : "busy",
  4994. list_empty(&conf->hold_list) ? "empty" : "busy",
  4995. atomic_read(&conf->pending_full_writes), conf->bypass_count);
  4996. if (!list_empty(handle_list)) {
  4997. sh = list_entry(handle_list->next, typeof(*sh), lru);
  4998. if (list_empty(&conf->hold_list))
  4999. conf->bypass_count = 0;
  5000. else if (!test_bit(STRIPE_IO_STARTED, &sh->state)) {
  5001. if (conf->hold_list.next == conf->last_hold)
  5002. conf->bypass_count++;
  5003. else {
  5004. conf->last_hold = conf->hold_list.next;
  5005. conf->bypass_count -= conf->bypass_threshold;
  5006. if (conf->bypass_count < 0)
  5007. conf->bypass_count = 0;
  5008. }
  5009. }
  5010. } else if (!list_empty(&conf->hold_list) &&
  5011. ((conf->bypass_threshold &&
  5012. conf->bypass_count > conf->bypass_threshold) ||
  5013. atomic_read(&conf->pending_full_writes) == 0)) {
  5014. list_for_each_entry(tmp, &conf->hold_list, lru) {
  5015. if (conf->worker_cnt_per_group == 0 ||
  5016. group == ANY_GROUP ||
  5017. !cpu_online(tmp->cpu) ||
  5018. cpu_to_group(tmp->cpu) == group) {
  5019. sh = tmp;
  5020. break;
  5021. }
  5022. }
  5023. if (sh) {
  5024. conf->bypass_count -= conf->bypass_threshold;
  5025. if (conf->bypass_count < 0)
  5026. conf->bypass_count = 0;
  5027. }
  5028. wg = NULL;
  5029. }
  5030. if (!sh) {
  5031. if (second_try)
  5032. return NULL;
  5033. second_try = true;
  5034. try_loprio = !try_loprio;
  5035. goto again;
  5036. }
  5037. if (wg) {
  5038. wg->stripes_cnt--;
  5039. sh->group = NULL;
  5040. }
  5041. list_del_init(&sh->lru);
  5042. BUG_ON(atomic_inc_return(&sh->count) != 1);
  5043. return sh;
  5044. }
  5045. struct raid5_plug_cb {
  5046. struct blk_plug_cb cb;
  5047. struct list_head list;
  5048. struct list_head temp_inactive_list[NR_STRIPE_HASH_LOCKS];
  5049. };
  5050. static void raid5_unplug(struct blk_plug_cb *blk_cb, bool from_schedule)
  5051. {
  5052. struct raid5_plug_cb *cb = container_of(
  5053. blk_cb, struct raid5_plug_cb, cb);
  5054. struct stripe_head *sh;
  5055. struct mddev *mddev = cb->cb.data;
  5056. struct r5conf *conf = mddev->private;
  5057. int cnt = 0;
  5058. int hash;
  5059. if (cb->list.next && !list_empty(&cb->list)) {
  5060. spin_lock_irq(&conf->device_lock);
  5061. while (!list_empty(&cb->list)) {
  5062. sh = list_first_entry(&cb->list, struct stripe_head, lru);
  5063. list_del_init(&sh->lru);
  5064. /*
  5065. * avoid race release_stripe_plug() sees
  5066. * STRIPE_ON_UNPLUG_LIST clear but the stripe
  5067. * is still in our list
  5068. */
  5069. smp_mb__before_atomic();
  5070. clear_bit(STRIPE_ON_UNPLUG_LIST, &sh->state);
  5071. /*
  5072. * STRIPE_ON_RELEASE_LIST could be set here. In that
  5073. * case, the count is always > 1 here
  5074. */
  5075. hash = sh->hash_lock_index;
  5076. __release_stripe(conf, sh, &cb->temp_inactive_list[hash]);
  5077. cnt++;
  5078. }
  5079. spin_unlock_irq(&conf->device_lock);
  5080. }
  5081. release_inactive_stripe_list(conf, cb->temp_inactive_list,
  5082. NR_STRIPE_HASH_LOCKS);
  5083. if (mddev->queue)
  5084. trace_block_unplug(mddev->queue, cnt, !from_schedule);
  5085. kfree(cb);
  5086. }
  5087. static void release_stripe_plug(struct mddev *mddev,
  5088. struct stripe_head *sh)
  5089. {
  5090. struct blk_plug_cb *blk_cb = blk_check_plugged(
  5091. raid5_unplug, mddev,
  5092. sizeof(struct raid5_plug_cb));
  5093. struct raid5_plug_cb *cb;
  5094. if (!blk_cb) {
  5095. raid5_release_stripe(sh);
  5096. return;
  5097. }
  5098. cb = container_of(blk_cb, struct raid5_plug_cb, cb);
  5099. if (cb->list.next == NULL) {
  5100. int i;
  5101. INIT_LIST_HEAD(&cb->list);
  5102. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  5103. INIT_LIST_HEAD(cb->temp_inactive_list + i);
  5104. }
  5105. if (!test_and_set_bit(STRIPE_ON_UNPLUG_LIST, &sh->state))
  5106. list_add_tail(&sh->lru, &cb->list);
  5107. else
  5108. raid5_release_stripe(sh);
  5109. }
  5110. static void make_discard_request(struct mddev *mddev, struct bio *bi)
  5111. {
  5112. struct r5conf *conf = mddev->private;
  5113. sector_t logical_sector, last_sector;
  5114. struct stripe_head *sh;
  5115. int stripe_sectors;
  5116. if (mddev->reshape_position != MaxSector)
  5117. /* Skip discard while reshape is happening */
  5118. return;
  5119. logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5120. last_sector = bio_end_sector(bi);
  5121. bi->bi_next = NULL;
  5122. stripe_sectors = conf->chunk_sectors *
  5123. (conf->raid_disks - conf->max_degraded);
  5124. logical_sector = DIV_ROUND_UP_SECTOR_T(logical_sector,
  5125. stripe_sectors);
  5126. sector_div(last_sector, stripe_sectors);
  5127. logical_sector *= conf->chunk_sectors;
  5128. last_sector *= conf->chunk_sectors;
  5129. for (; logical_sector < last_sector;
  5130. logical_sector += RAID5_STRIPE_SECTORS(conf)) {
  5131. DEFINE_WAIT(w);
  5132. int d;
  5133. again:
  5134. sh = raid5_get_active_stripe(conf, logical_sector, 0, 0, 0);
  5135. prepare_to_wait(&conf->wait_for_overlap, &w,
  5136. TASK_UNINTERRUPTIBLE);
  5137. set_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
  5138. if (test_bit(STRIPE_SYNCING, &sh->state)) {
  5139. raid5_release_stripe(sh);
  5140. schedule();
  5141. goto again;
  5142. }
  5143. clear_bit(R5_Overlap, &sh->dev[sh->pd_idx].flags);
  5144. spin_lock_irq(&sh->stripe_lock);
  5145. for (d = 0; d < conf->raid_disks; d++) {
  5146. if (d == sh->pd_idx || d == sh->qd_idx)
  5147. continue;
  5148. if (sh->dev[d].towrite || sh->dev[d].toread) {
  5149. set_bit(R5_Overlap, &sh->dev[d].flags);
  5150. spin_unlock_irq(&sh->stripe_lock);
  5151. raid5_release_stripe(sh);
  5152. schedule();
  5153. goto again;
  5154. }
  5155. }
  5156. set_bit(STRIPE_DISCARD, &sh->state);
  5157. finish_wait(&conf->wait_for_overlap, &w);
  5158. sh->overwrite_disks = 0;
  5159. for (d = 0; d < conf->raid_disks; d++) {
  5160. if (d == sh->pd_idx || d == sh->qd_idx)
  5161. continue;
  5162. sh->dev[d].towrite = bi;
  5163. set_bit(R5_OVERWRITE, &sh->dev[d].flags);
  5164. bio_inc_remaining(bi);
  5165. md_write_inc(mddev, bi);
  5166. sh->overwrite_disks++;
  5167. }
  5168. spin_unlock_irq(&sh->stripe_lock);
  5169. if (conf->mddev->bitmap) {
  5170. for (d = 0;
  5171. d < conf->raid_disks - conf->max_degraded;
  5172. d++)
  5173. md_bitmap_startwrite(mddev->bitmap,
  5174. sh->sector,
  5175. RAID5_STRIPE_SECTORS(conf),
  5176. 0);
  5177. sh->bm_seq = conf->seq_flush + 1;
  5178. set_bit(STRIPE_BIT_DELAY, &sh->state);
  5179. }
  5180. set_bit(STRIPE_HANDLE, &sh->state);
  5181. clear_bit(STRIPE_DELAYED, &sh->state);
  5182. if (!test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  5183. atomic_inc(&conf->preread_active_stripes);
  5184. release_stripe_plug(mddev, sh);
  5185. }
  5186. bio_endio(bi);
  5187. }
  5188. static bool raid5_make_request(struct mddev *mddev, struct bio * bi)
  5189. {
  5190. struct r5conf *conf = mddev->private;
  5191. int dd_idx;
  5192. sector_t new_sector;
  5193. sector_t logical_sector, last_sector;
  5194. struct stripe_head *sh;
  5195. const int rw = bio_data_dir(bi);
  5196. DEFINE_WAIT(w);
  5197. bool do_prepare;
  5198. bool do_flush = false;
  5199. if (unlikely(bi->bi_opf & REQ_PREFLUSH)) {
  5200. int ret = log_handle_flush_request(conf, bi);
  5201. if (ret == 0)
  5202. return true;
  5203. if (ret == -ENODEV) {
  5204. if (md_flush_request(mddev, bi))
  5205. return true;
  5206. }
  5207. /* ret == -EAGAIN, fallback */
  5208. /*
  5209. * if r5l_handle_flush_request() didn't clear REQ_PREFLUSH,
  5210. * we need to flush journal device
  5211. */
  5212. do_flush = bi->bi_opf & REQ_PREFLUSH;
  5213. }
  5214. if (!md_write_start(mddev, bi))
  5215. return false;
  5216. /*
  5217. * If array is degraded, better not do chunk aligned read because
  5218. * later we might have to read it again in order to reconstruct
  5219. * data on failed drives.
  5220. */
  5221. if (rw == READ && mddev->degraded == 0 &&
  5222. mddev->reshape_position == MaxSector) {
  5223. bi = chunk_aligned_read(mddev, bi);
  5224. if (!bi)
  5225. return true;
  5226. }
  5227. if (unlikely(bio_op(bi) == REQ_OP_DISCARD)) {
  5228. make_discard_request(mddev, bi);
  5229. md_write_end(mddev);
  5230. return true;
  5231. }
  5232. logical_sector = bi->bi_iter.bi_sector & ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5233. last_sector = bio_end_sector(bi);
  5234. bi->bi_next = NULL;
  5235. prepare_to_wait(&conf->wait_for_overlap, &w, TASK_UNINTERRUPTIBLE);
  5236. for (; logical_sector < last_sector; logical_sector += RAID5_STRIPE_SECTORS(conf)) {
  5237. int previous;
  5238. int seq;
  5239. do_prepare = false;
  5240. retry:
  5241. seq = read_seqcount_begin(&conf->gen_lock);
  5242. previous = 0;
  5243. if (do_prepare)
  5244. prepare_to_wait(&conf->wait_for_overlap, &w,
  5245. TASK_UNINTERRUPTIBLE);
  5246. if (unlikely(conf->reshape_progress != MaxSector)) {
  5247. /* spinlock is needed as reshape_progress may be
  5248. * 64bit on a 32bit platform, and so it might be
  5249. * possible to see a half-updated value
  5250. * Of course reshape_progress could change after
  5251. * the lock is dropped, so once we get a reference
  5252. * to the stripe that we think it is, we will have
  5253. * to check again.
  5254. */
  5255. spin_lock_irq(&conf->device_lock);
  5256. if (mddev->reshape_backwards
  5257. ? logical_sector < conf->reshape_progress
  5258. : logical_sector >= conf->reshape_progress) {
  5259. previous = 1;
  5260. } else {
  5261. if (mddev->reshape_backwards
  5262. ? logical_sector < conf->reshape_safe
  5263. : logical_sector >= conf->reshape_safe) {
  5264. spin_unlock_irq(&conf->device_lock);
  5265. schedule();
  5266. do_prepare = true;
  5267. goto retry;
  5268. }
  5269. }
  5270. spin_unlock_irq(&conf->device_lock);
  5271. }
  5272. new_sector = raid5_compute_sector(conf, logical_sector,
  5273. previous,
  5274. &dd_idx, NULL);
  5275. pr_debug("raid456: raid5_make_request, sector %llu logical %llu\n",
  5276. (unsigned long long)new_sector,
  5277. (unsigned long long)logical_sector);
  5278. sh = raid5_get_active_stripe(conf, new_sector, previous,
  5279. (bi->bi_opf & REQ_RAHEAD), 0);
  5280. if (sh) {
  5281. if (unlikely(previous)) {
  5282. /* expansion might have moved on while waiting for a
  5283. * stripe, so we must do the range check again.
  5284. * Expansion could still move past after this
  5285. * test, but as we are holding a reference to
  5286. * 'sh', we know that if that happens,
  5287. * STRIPE_EXPANDING will get set and the expansion
  5288. * won't proceed until we finish with the stripe.
  5289. */
  5290. int must_retry = 0;
  5291. spin_lock_irq(&conf->device_lock);
  5292. if (mddev->reshape_backwards
  5293. ? logical_sector >= conf->reshape_progress
  5294. : logical_sector < conf->reshape_progress)
  5295. /* mismatch, need to try again */
  5296. must_retry = 1;
  5297. spin_unlock_irq(&conf->device_lock);
  5298. if (must_retry) {
  5299. raid5_release_stripe(sh);
  5300. schedule();
  5301. do_prepare = true;
  5302. goto retry;
  5303. }
  5304. }
  5305. if (read_seqcount_retry(&conf->gen_lock, seq)) {
  5306. /* Might have got the wrong stripe_head
  5307. * by accident
  5308. */
  5309. raid5_release_stripe(sh);
  5310. goto retry;
  5311. }
  5312. if (test_bit(STRIPE_EXPANDING, &sh->state) ||
  5313. !add_stripe_bio(sh, bi, dd_idx, rw, previous)) {
  5314. /* Stripe is busy expanding or
  5315. * add failed due to overlap. Flush everything
  5316. * and wait a while
  5317. */
  5318. md_wakeup_thread(mddev->thread);
  5319. raid5_release_stripe(sh);
  5320. schedule();
  5321. do_prepare = true;
  5322. goto retry;
  5323. }
  5324. if (do_flush) {
  5325. set_bit(STRIPE_R5C_PREFLUSH, &sh->state);
  5326. /* we only need flush for one stripe */
  5327. do_flush = false;
  5328. }
  5329. set_bit(STRIPE_HANDLE, &sh->state);
  5330. clear_bit(STRIPE_DELAYED, &sh->state);
  5331. if ((!sh->batch_head || sh == sh->batch_head) &&
  5332. (bi->bi_opf & REQ_SYNC) &&
  5333. !test_and_set_bit(STRIPE_PREREAD_ACTIVE, &sh->state))
  5334. atomic_inc(&conf->preread_active_stripes);
  5335. release_stripe_plug(mddev, sh);
  5336. } else {
  5337. /* cannot get stripe for read-ahead, just give-up */
  5338. bi->bi_status = BLK_STS_IOERR;
  5339. break;
  5340. }
  5341. }
  5342. finish_wait(&conf->wait_for_overlap, &w);
  5343. if (rw == WRITE)
  5344. md_write_end(mddev);
  5345. bio_endio(bi);
  5346. return true;
  5347. }
  5348. static sector_t raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks);
  5349. static sector_t reshape_request(struct mddev *mddev, sector_t sector_nr, int *skipped)
  5350. {
  5351. /* reshaping is quite different to recovery/resync so it is
  5352. * handled quite separately ... here.
  5353. *
  5354. * On each call to sync_request, we gather one chunk worth of
  5355. * destination stripes and flag them as expanding.
  5356. * Then we find all the source stripes and request reads.
  5357. * As the reads complete, handle_stripe will copy the data
  5358. * into the destination stripe and release that stripe.
  5359. */
  5360. struct r5conf *conf = mddev->private;
  5361. struct stripe_head *sh;
  5362. struct md_rdev *rdev;
  5363. sector_t first_sector, last_sector;
  5364. int raid_disks = conf->previous_raid_disks;
  5365. int data_disks = raid_disks - conf->max_degraded;
  5366. int new_data_disks = conf->raid_disks - conf->max_degraded;
  5367. int i;
  5368. int dd_idx;
  5369. sector_t writepos, readpos, safepos;
  5370. sector_t stripe_addr;
  5371. int reshape_sectors;
  5372. struct list_head stripes;
  5373. sector_t retn;
  5374. if (sector_nr == 0) {
  5375. /* If restarting in the middle, skip the initial sectors */
  5376. if (mddev->reshape_backwards &&
  5377. conf->reshape_progress < raid5_size(mddev, 0, 0)) {
  5378. sector_nr = raid5_size(mddev, 0, 0)
  5379. - conf->reshape_progress;
  5380. } else if (mddev->reshape_backwards &&
  5381. conf->reshape_progress == MaxSector) {
  5382. /* shouldn't happen, but just in case, finish up.*/
  5383. sector_nr = MaxSector;
  5384. } else if (!mddev->reshape_backwards &&
  5385. conf->reshape_progress > 0)
  5386. sector_nr = conf->reshape_progress;
  5387. sector_div(sector_nr, new_data_disks);
  5388. if (sector_nr) {
  5389. mddev->curr_resync_completed = sector_nr;
  5390. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5391. *skipped = 1;
  5392. retn = sector_nr;
  5393. goto finish;
  5394. }
  5395. }
  5396. /* We need to process a full chunk at a time.
  5397. * If old and new chunk sizes differ, we need to process the
  5398. * largest of these
  5399. */
  5400. reshape_sectors = max(conf->chunk_sectors, conf->prev_chunk_sectors);
  5401. /* We update the metadata at least every 10 seconds, or when
  5402. * the data about to be copied would over-write the source of
  5403. * the data at the front of the range. i.e. one new_stripe
  5404. * along from reshape_progress new_maps to after where
  5405. * reshape_safe old_maps to
  5406. */
  5407. writepos = conf->reshape_progress;
  5408. sector_div(writepos, new_data_disks);
  5409. readpos = conf->reshape_progress;
  5410. sector_div(readpos, data_disks);
  5411. safepos = conf->reshape_safe;
  5412. sector_div(safepos, data_disks);
  5413. if (mddev->reshape_backwards) {
  5414. BUG_ON(writepos < reshape_sectors);
  5415. writepos -= reshape_sectors;
  5416. readpos += reshape_sectors;
  5417. safepos += reshape_sectors;
  5418. } else {
  5419. writepos += reshape_sectors;
  5420. /* readpos and safepos are worst-case calculations.
  5421. * A negative number is overly pessimistic, and causes
  5422. * obvious problems for unsigned storage. So clip to 0.
  5423. */
  5424. readpos -= min_t(sector_t, reshape_sectors, readpos);
  5425. safepos -= min_t(sector_t, reshape_sectors, safepos);
  5426. }
  5427. /* Having calculated the 'writepos' possibly use it
  5428. * to set 'stripe_addr' which is where we will write to.
  5429. */
  5430. if (mddev->reshape_backwards) {
  5431. BUG_ON(conf->reshape_progress == 0);
  5432. stripe_addr = writepos;
  5433. BUG_ON((mddev->dev_sectors &
  5434. ~((sector_t)reshape_sectors - 1))
  5435. - reshape_sectors - stripe_addr
  5436. != sector_nr);
  5437. } else {
  5438. BUG_ON(writepos != sector_nr + reshape_sectors);
  5439. stripe_addr = sector_nr;
  5440. }
  5441. /* 'writepos' is the most advanced device address we might write.
  5442. * 'readpos' is the least advanced device address we might read.
  5443. * 'safepos' is the least address recorded in the metadata as having
  5444. * been reshaped.
  5445. * If there is a min_offset_diff, these are adjusted either by
  5446. * increasing the safepos/readpos if diff is negative, or
  5447. * increasing writepos if diff is positive.
  5448. * If 'readpos' is then behind 'writepos', there is no way that we can
  5449. * ensure safety in the face of a crash - that must be done by userspace
  5450. * making a backup of the data. So in that case there is no particular
  5451. * rush to update metadata.
  5452. * Otherwise if 'safepos' is behind 'writepos', then we really need to
  5453. * update the metadata to advance 'safepos' to match 'readpos' so that
  5454. * we can be safe in the event of a crash.
  5455. * So we insist on updating metadata if safepos is behind writepos and
  5456. * readpos is beyond writepos.
  5457. * In any case, update the metadata every 10 seconds.
  5458. * Maybe that number should be configurable, but I'm not sure it is
  5459. * worth it.... maybe it could be a multiple of safemode_delay???
  5460. */
  5461. if (conf->min_offset_diff < 0) {
  5462. safepos += -conf->min_offset_diff;
  5463. readpos += -conf->min_offset_diff;
  5464. } else
  5465. writepos += conf->min_offset_diff;
  5466. if ((mddev->reshape_backwards
  5467. ? (safepos > writepos && readpos < writepos)
  5468. : (safepos < writepos && readpos > writepos)) ||
  5469. time_after(jiffies, conf->reshape_checkpoint + 10*HZ)) {
  5470. /* Cannot proceed until we've updated the superblock... */
  5471. wait_event(conf->wait_for_overlap,
  5472. atomic_read(&conf->reshape_stripes)==0
  5473. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5474. if (atomic_read(&conf->reshape_stripes) != 0)
  5475. return 0;
  5476. mddev->reshape_position = conf->reshape_progress;
  5477. mddev->curr_resync_completed = sector_nr;
  5478. if (!mddev->reshape_backwards)
  5479. /* Can update recovery_offset */
  5480. rdev_for_each(rdev, mddev)
  5481. if (rdev->raid_disk >= 0 &&
  5482. !test_bit(Journal, &rdev->flags) &&
  5483. !test_bit(In_sync, &rdev->flags) &&
  5484. rdev->recovery_offset < sector_nr)
  5485. rdev->recovery_offset = sector_nr;
  5486. conf->reshape_checkpoint = jiffies;
  5487. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  5488. md_wakeup_thread(mddev->thread);
  5489. wait_event(mddev->sb_wait, mddev->sb_flags == 0 ||
  5490. test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5491. if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
  5492. return 0;
  5493. spin_lock_irq(&conf->device_lock);
  5494. conf->reshape_safe = mddev->reshape_position;
  5495. spin_unlock_irq(&conf->device_lock);
  5496. wake_up(&conf->wait_for_overlap);
  5497. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5498. }
  5499. INIT_LIST_HEAD(&stripes);
  5500. for (i = 0; i < reshape_sectors; i += RAID5_STRIPE_SECTORS(conf)) {
  5501. int j;
  5502. int skipped_disk = 0;
  5503. sh = raid5_get_active_stripe(conf, stripe_addr+i, 0, 0, 1);
  5504. set_bit(STRIPE_EXPANDING, &sh->state);
  5505. atomic_inc(&conf->reshape_stripes);
  5506. /* If any of this stripe is beyond the end of the old
  5507. * array, then we need to zero those blocks
  5508. */
  5509. for (j=sh->disks; j--;) {
  5510. sector_t s;
  5511. if (j == sh->pd_idx)
  5512. continue;
  5513. if (conf->level == 6 &&
  5514. j == sh->qd_idx)
  5515. continue;
  5516. s = raid5_compute_blocknr(sh, j, 0);
  5517. if (s < raid5_size(mddev, 0, 0)) {
  5518. skipped_disk = 1;
  5519. continue;
  5520. }
  5521. memset(page_address(sh->dev[j].page), 0, RAID5_STRIPE_SIZE(conf));
  5522. set_bit(R5_Expanded, &sh->dev[j].flags);
  5523. set_bit(R5_UPTODATE, &sh->dev[j].flags);
  5524. }
  5525. if (!skipped_disk) {
  5526. set_bit(STRIPE_EXPAND_READY, &sh->state);
  5527. set_bit(STRIPE_HANDLE, &sh->state);
  5528. }
  5529. list_add(&sh->lru, &stripes);
  5530. }
  5531. spin_lock_irq(&conf->device_lock);
  5532. if (mddev->reshape_backwards)
  5533. conf->reshape_progress -= reshape_sectors * new_data_disks;
  5534. else
  5535. conf->reshape_progress += reshape_sectors * new_data_disks;
  5536. spin_unlock_irq(&conf->device_lock);
  5537. /* Ok, those stripe are ready. We can start scheduling
  5538. * reads on the source stripes.
  5539. * The source stripes are determined by mapping the first and last
  5540. * block on the destination stripes.
  5541. */
  5542. first_sector =
  5543. raid5_compute_sector(conf, stripe_addr*(new_data_disks),
  5544. 1, &dd_idx, NULL);
  5545. last_sector =
  5546. raid5_compute_sector(conf, ((stripe_addr+reshape_sectors)
  5547. * new_data_disks - 1),
  5548. 1, &dd_idx, NULL);
  5549. if (last_sector >= mddev->dev_sectors)
  5550. last_sector = mddev->dev_sectors - 1;
  5551. while (first_sector <= last_sector) {
  5552. sh = raid5_get_active_stripe(conf, first_sector, 1, 0, 1);
  5553. set_bit(STRIPE_EXPAND_SOURCE, &sh->state);
  5554. set_bit(STRIPE_HANDLE, &sh->state);
  5555. raid5_release_stripe(sh);
  5556. first_sector += RAID5_STRIPE_SECTORS(conf);
  5557. }
  5558. /* Now that the sources are clearly marked, we can release
  5559. * the destination stripes
  5560. */
  5561. while (!list_empty(&stripes)) {
  5562. sh = list_entry(stripes.next, struct stripe_head, lru);
  5563. list_del_init(&sh->lru);
  5564. raid5_release_stripe(sh);
  5565. }
  5566. /* If this takes us to the resync_max point where we have to pause,
  5567. * then we need to write out the superblock.
  5568. */
  5569. sector_nr += reshape_sectors;
  5570. retn = reshape_sectors;
  5571. finish:
  5572. if (mddev->curr_resync_completed > mddev->resync_max ||
  5573. (sector_nr - mddev->curr_resync_completed) * 2
  5574. >= mddev->resync_max - mddev->curr_resync_completed) {
  5575. /* Cannot proceed until we've updated the superblock... */
  5576. wait_event(conf->wait_for_overlap,
  5577. atomic_read(&conf->reshape_stripes) == 0
  5578. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5579. if (atomic_read(&conf->reshape_stripes) != 0)
  5580. goto ret;
  5581. mddev->reshape_position = conf->reshape_progress;
  5582. mddev->curr_resync_completed = sector_nr;
  5583. if (!mddev->reshape_backwards)
  5584. /* Can update recovery_offset */
  5585. rdev_for_each(rdev, mddev)
  5586. if (rdev->raid_disk >= 0 &&
  5587. !test_bit(Journal, &rdev->flags) &&
  5588. !test_bit(In_sync, &rdev->flags) &&
  5589. rdev->recovery_offset < sector_nr)
  5590. rdev->recovery_offset = sector_nr;
  5591. conf->reshape_checkpoint = jiffies;
  5592. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  5593. md_wakeup_thread(mddev->thread);
  5594. wait_event(mddev->sb_wait,
  5595. !test_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags)
  5596. || test_bit(MD_RECOVERY_INTR, &mddev->recovery));
  5597. if (test_bit(MD_RECOVERY_INTR, &mddev->recovery))
  5598. goto ret;
  5599. spin_lock_irq(&conf->device_lock);
  5600. conf->reshape_safe = mddev->reshape_position;
  5601. spin_unlock_irq(&conf->device_lock);
  5602. wake_up(&conf->wait_for_overlap);
  5603. sysfs_notify_dirent_safe(mddev->sysfs_completed);
  5604. }
  5605. ret:
  5606. return retn;
  5607. }
  5608. static inline sector_t raid5_sync_request(struct mddev *mddev, sector_t sector_nr,
  5609. int *skipped)
  5610. {
  5611. struct r5conf *conf = mddev->private;
  5612. struct stripe_head *sh;
  5613. sector_t max_sector = mddev->dev_sectors;
  5614. sector_t sync_blocks;
  5615. int still_degraded = 0;
  5616. int i;
  5617. if (sector_nr >= max_sector) {
  5618. /* just being told to finish up .. nothing much to do */
  5619. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery)) {
  5620. end_reshape(conf);
  5621. return 0;
  5622. }
  5623. if (mddev->curr_resync < max_sector) /* aborted */
  5624. md_bitmap_end_sync(mddev->bitmap, mddev->curr_resync,
  5625. &sync_blocks, 1);
  5626. else /* completed sync */
  5627. conf->fullsync = 0;
  5628. md_bitmap_close_sync(mddev->bitmap);
  5629. return 0;
  5630. }
  5631. /* Allow raid5_quiesce to complete */
  5632. wait_event(conf->wait_for_overlap, conf->quiesce != 2);
  5633. if (test_bit(MD_RECOVERY_RESHAPE, &mddev->recovery))
  5634. return reshape_request(mddev, sector_nr, skipped);
  5635. /* No need to check resync_max as we never do more than one
  5636. * stripe, and as resync_max will always be on a chunk boundary,
  5637. * if the check in md_do_sync didn't fire, there is no chance
  5638. * of overstepping resync_max here
  5639. */
  5640. /* if there is too many failed drives and we are trying
  5641. * to resync, then assert that we are finished, because there is
  5642. * nothing we can do.
  5643. */
  5644. if (mddev->degraded >= conf->max_degraded &&
  5645. test_bit(MD_RECOVERY_SYNC, &mddev->recovery)) {
  5646. sector_t rv = mddev->dev_sectors - sector_nr;
  5647. *skipped = 1;
  5648. return rv;
  5649. }
  5650. if (!test_bit(MD_RECOVERY_REQUESTED, &mddev->recovery) &&
  5651. !conf->fullsync &&
  5652. !md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, 1) &&
  5653. sync_blocks >= RAID5_STRIPE_SECTORS(conf)) {
  5654. /* we can skip this block, and probably more */
  5655. do_div(sync_blocks, RAID5_STRIPE_SECTORS(conf));
  5656. *skipped = 1;
  5657. /* keep things rounded to whole stripes */
  5658. return sync_blocks * RAID5_STRIPE_SECTORS(conf);
  5659. }
  5660. md_bitmap_cond_end_sync(mddev->bitmap, sector_nr, false);
  5661. sh = raid5_get_active_stripe(conf, sector_nr, 0, 1, 0);
  5662. if (sh == NULL) {
  5663. sh = raid5_get_active_stripe(conf, sector_nr, 0, 0, 0);
  5664. /* make sure we don't swamp the stripe cache if someone else
  5665. * is trying to get access
  5666. */
  5667. schedule_timeout_uninterruptible(1);
  5668. }
  5669. /* Need to check if array will still be degraded after recovery/resync
  5670. * Note in case of > 1 drive failures it's possible we're rebuilding
  5671. * one drive while leaving another faulty drive in array.
  5672. */
  5673. rcu_read_lock();
  5674. for (i = 0; i < conf->raid_disks; i++) {
  5675. struct md_rdev *rdev = READ_ONCE(conf->disks[i].rdev);
  5676. if (rdev == NULL || test_bit(Faulty, &rdev->flags))
  5677. still_degraded = 1;
  5678. }
  5679. rcu_read_unlock();
  5680. md_bitmap_start_sync(mddev->bitmap, sector_nr, &sync_blocks, still_degraded);
  5681. set_bit(STRIPE_SYNC_REQUESTED, &sh->state);
  5682. set_bit(STRIPE_HANDLE, &sh->state);
  5683. raid5_release_stripe(sh);
  5684. return RAID5_STRIPE_SECTORS(conf);
  5685. }
  5686. static int retry_aligned_read(struct r5conf *conf, struct bio *raid_bio,
  5687. unsigned int offset)
  5688. {
  5689. /* We may not be able to submit a whole bio at once as there
  5690. * may not be enough stripe_heads available.
  5691. * We cannot pre-allocate enough stripe_heads as we may need
  5692. * more than exist in the cache (if we allow ever large chunks).
  5693. * So we do one stripe head at a time and record in
  5694. * ->bi_hw_segments how many have been done.
  5695. *
  5696. * We *know* that this entire raid_bio is in one chunk, so
  5697. * it will be only one 'dd_idx' and only need one call to raid5_compute_sector.
  5698. */
  5699. struct stripe_head *sh;
  5700. int dd_idx;
  5701. sector_t sector, logical_sector, last_sector;
  5702. int scnt = 0;
  5703. int handled = 0;
  5704. logical_sector = raid_bio->bi_iter.bi_sector &
  5705. ~((sector_t)RAID5_STRIPE_SECTORS(conf)-1);
  5706. sector = raid5_compute_sector(conf, logical_sector,
  5707. 0, &dd_idx, NULL);
  5708. last_sector = bio_end_sector(raid_bio);
  5709. for (; logical_sector < last_sector;
  5710. logical_sector += RAID5_STRIPE_SECTORS(conf),
  5711. sector += RAID5_STRIPE_SECTORS(conf),
  5712. scnt++) {
  5713. if (scnt < offset)
  5714. /* already done this stripe */
  5715. continue;
  5716. sh = raid5_get_active_stripe(conf, sector, 0, 1, 1);
  5717. if (!sh) {
  5718. /* failed to get a stripe - must wait */
  5719. conf->retry_read_aligned = raid_bio;
  5720. conf->retry_read_offset = scnt;
  5721. return handled;
  5722. }
  5723. if (!add_stripe_bio(sh, raid_bio, dd_idx, 0, 0)) {
  5724. raid5_release_stripe(sh);
  5725. conf->retry_read_aligned = raid_bio;
  5726. conf->retry_read_offset = scnt;
  5727. return handled;
  5728. }
  5729. set_bit(R5_ReadNoMerge, &sh->dev[dd_idx].flags);
  5730. handle_stripe(sh);
  5731. raid5_release_stripe(sh);
  5732. handled++;
  5733. }
  5734. bio_endio(raid_bio);
  5735. if (atomic_dec_and_test(&conf->active_aligned_reads))
  5736. wake_up(&conf->wait_for_quiescent);
  5737. return handled;
  5738. }
  5739. static int handle_active_stripes(struct r5conf *conf, int group,
  5740. struct r5worker *worker,
  5741. struct list_head *temp_inactive_list)
  5742. __releases(&conf->device_lock)
  5743. __acquires(&conf->device_lock)
  5744. {
  5745. struct stripe_head *batch[MAX_STRIPE_BATCH], *sh;
  5746. int i, batch_size = 0, hash;
  5747. bool release_inactive = false;
  5748. while (batch_size < MAX_STRIPE_BATCH &&
  5749. (sh = __get_priority_stripe(conf, group)) != NULL)
  5750. batch[batch_size++] = sh;
  5751. if (batch_size == 0) {
  5752. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  5753. if (!list_empty(temp_inactive_list + i))
  5754. break;
  5755. if (i == NR_STRIPE_HASH_LOCKS) {
  5756. spin_unlock_irq(&conf->device_lock);
  5757. log_flush_stripe_to_raid(conf);
  5758. spin_lock_irq(&conf->device_lock);
  5759. return batch_size;
  5760. }
  5761. release_inactive = true;
  5762. }
  5763. spin_unlock_irq(&conf->device_lock);
  5764. release_inactive_stripe_list(conf, temp_inactive_list,
  5765. NR_STRIPE_HASH_LOCKS);
  5766. r5l_flush_stripe_to_raid(conf->log);
  5767. if (release_inactive) {
  5768. spin_lock_irq(&conf->device_lock);
  5769. return 0;
  5770. }
  5771. for (i = 0; i < batch_size; i++)
  5772. handle_stripe(batch[i]);
  5773. log_write_stripe_run(conf);
  5774. cond_resched();
  5775. spin_lock_irq(&conf->device_lock);
  5776. for (i = 0; i < batch_size; i++) {
  5777. hash = batch[i]->hash_lock_index;
  5778. __release_stripe(conf, batch[i], &temp_inactive_list[hash]);
  5779. }
  5780. return batch_size;
  5781. }
  5782. static void raid5_do_work(struct work_struct *work)
  5783. {
  5784. struct r5worker *worker = container_of(work, struct r5worker, work);
  5785. struct r5worker_group *group = worker->group;
  5786. struct r5conf *conf = group->conf;
  5787. struct mddev *mddev = conf->mddev;
  5788. int group_id = group - conf->worker_groups;
  5789. int handled;
  5790. struct blk_plug plug;
  5791. pr_debug("+++ raid5worker active\n");
  5792. blk_start_plug(&plug);
  5793. handled = 0;
  5794. spin_lock_irq(&conf->device_lock);
  5795. while (1) {
  5796. int batch_size, released;
  5797. released = release_stripe_list(conf, worker->temp_inactive_list);
  5798. batch_size = handle_active_stripes(conf, group_id, worker,
  5799. worker->temp_inactive_list);
  5800. worker->working = false;
  5801. if (!batch_size && !released)
  5802. break;
  5803. handled += batch_size;
  5804. wait_event_lock_irq(mddev->sb_wait,
  5805. !test_bit(MD_SB_CHANGE_PENDING, &mddev->sb_flags),
  5806. conf->device_lock);
  5807. }
  5808. pr_debug("%d stripes handled\n", handled);
  5809. spin_unlock_irq(&conf->device_lock);
  5810. flush_deferred_bios(conf);
  5811. r5l_flush_stripe_to_raid(conf->log);
  5812. async_tx_issue_pending_all();
  5813. blk_finish_plug(&plug);
  5814. pr_debug("--- raid5worker inactive\n");
  5815. }
  5816. /*
  5817. * This is our raid5 kernel thread.
  5818. *
  5819. * We scan the hash table for stripes which can be handled now.
  5820. * During the scan, completed stripes are saved for us by the interrupt
  5821. * handler, so that they will not have to wait for our next wakeup.
  5822. */
  5823. static void raid5d(struct md_thread *thread)
  5824. {
  5825. struct mddev *mddev = thread->mddev;
  5826. struct r5conf *conf = mddev->private;
  5827. int handled;
  5828. struct blk_plug plug;
  5829. pr_debug("+++ raid5d active\n");
  5830. md_check_recovery(mddev);
  5831. blk_start_plug(&plug);
  5832. handled = 0;
  5833. spin_lock_irq(&conf->device_lock);
  5834. while (1) {
  5835. struct bio *bio;
  5836. int batch_size, released;
  5837. unsigned int offset;
  5838. released = release_stripe_list(conf, conf->temp_inactive_list);
  5839. if (released)
  5840. clear_bit(R5_DID_ALLOC, &conf->cache_state);
  5841. if (
  5842. !list_empty(&conf->bitmap_list)) {
  5843. /* Now is a good time to flush some bitmap updates */
  5844. conf->seq_flush++;
  5845. spin_unlock_irq(&conf->device_lock);
  5846. md_bitmap_unplug(mddev->bitmap);
  5847. spin_lock_irq(&conf->device_lock);
  5848. conf->seq_write = conf->seq_flush;
  5849. activate_bit_delay(conf, conf->temp_inactive_list);
  5850. }
  5851. raid5_activate_delayed(conf);
  5852. while ((bio = remove_bio_from_retry(conf, &offset))) {
  5853. int ok;
  5854. spin_unlock_irq(&conf->device_lock);
  5855. ok = retry_aligned_read(conf, bio, offset);
  5856. spin_lock_irq(&conf->device_lock);
  5857. if (!ok)
  5858. break;
  5859. handled++;
  5860. }
  5861. batch_size = handle_active_stripes(conf, ANY_GROUP, NULL,
  5862. conf->temp_inactive_list);
  5863. if (!batch_size && !released)
  5864. break;
  5865. handled += batch_size;
  5866. if (mddev->sb_flags & ~(1 << MD_SB_CHANGE_PENDING)) {
  5867. spin_unlock_irq(&conf->device_lock);
  5868. md_check_recovery(mddev);
  5869. spin_lock_irq(&conf->device_lock);
  5870. }
  5871. }
  5872. pr_debug("%d stripes handled\n", handled);
  5873. spin_unlock_irq(&conf->device_lock);
  5874. if (test_and_clear_bit(R5_ALLOC_MORE, &conf->cache_state) &&
  5875. mutex_trylock(&conf->cache_size_mutex)) {
  5876. grow_one_stripe(conf, __GFP_NOWARN);
  5877. /* Set flag even if allocation failed. This helps
  5878. * slow down allocation requests when mem is short
  5879. */
  5880. set_bit(R5_DID_ALLOC, &conf->cache_state);
  5881. mutex_unlock(&conf->cache_size_mutex);
  5882. }
  5883. flush_deferred_bios(conf);
  5884. r5l_flush_stripe_to_raid(conf->log);
  5885. async_tx_issue_pending_all();
  5886. blk_finish_plug(&plug);
  5887. pr_debug("--- raid5d inactive\n");
  5888. }
  5889. static ssize_t
  5890. raid5_show_stripe_cache_size(struct mddev *mddev, char *page)
  5891. {
  5892. struct r5conf *conf;
  5893. int ret = 0;
  5894. spin_lock(&mddev->lock);
  5895. conf = mddev->private;
  5896. if (conf)
  5897. ret = sprintf(page, "%d\n", conf->min_nr_stripes);
  5898. spin_unlock(&mddev->lock);
  5899. return ret;
  5900. }
  5901. int
  5902. raid5_set_cache_size(struct mddev *mddev, int size)
  5903. {
  5904. int result = 0;
  5905. struct r5conf *conf = mddev->private;
  5906. if (size <= 16 || size > 32768)
  5907. return -EINVAL;
  5908. conf->min_nr_stripes = size;
  5909. mutex_lock(&conf->cache_size_mutex);
  5910. while (size < conf->max_nr_stripes &&
  5911. drop_one_stripe(conf))
  5912. ;
  5913. mutex_unlock(&conf->cache_size_mutex);
  5914. md_allow_write(mddev);
  5915. mutex_lock(&conf->cache_size_mutex);
  5916. while (size > conf->max_nr_stripes)
  5917. if (!grow_one_stripe(conf, GFP_KERNEL)) {
  5918. conf->min_nr_stripes = conf->max_nr_stripes;
  5919. result = -ENOMEM;
  5920. break;
  5921. }
  5922. mutex_unlock(&conf->cache_size_mutex);
  5923. return result;
  5924. }
  5925. EXPORT_SYMBOL(raid5_set_cache_size);
  5926. static ssize_t
  5927. raid5_store_stripe_cache_size(struct mddev *mddev, const char *page, size_t len)
  5928. {
  5929. struct r5conf *conf;
  5930. unsigned long new;
  5931. int err;
  5932. if (len >= PAGE_SIZE)
  5933. return -EINVAL;
  5934. if (kstrtoul(page, 10, &new))
  5935. return -EINVAL;
  5936. err = mddev_lock(mddev);
  5937. if (err)
  5938. return err;
  5939. conf = mddev->private;
  5940. if (!conf)
  5941. err = -ENODEV;
  5942. else
  5943. err = raid5_set_cache_size(mddev, new);
  5944. mddev_unlock(mddev);
  5945. return err ?: len;
  5946. }
  5947. static struct md_sysfs_entry
  5948. raid5_stripecache_size = __ATTR(stripe_cache_size, S_IRUGO | S_IWUSR,
  5949. raid5_show_stripe_cache_size,
  5950. raid5_store_stripe_cache_size);
  5951. static ssize_t
  5952. raid5_show_rmw_level(struct mddev *mddev, char *page)
  5953. {
  5954. struct r5conf *conf = mddev->private;
  5955. if (conf)
  5956. return sprintf(page, "%d\n", conf->rmw_level);
  5957. else
  5958. return 0;
  5959. }
  5960. static ssize_t
  5961. raid5_store_rmw_level(struct mddev *mddev, const char *page, size_t len)
  5962. {
  5963. struct r5conf *conf = mddev->private;
  5964. unsigned long new;
  5965. if (!conf)
  5966. return -ENODEV;
  5967. if (len >= PAGE_SIZE)
  5968. return -EINVAL;
  5969. if (kstrtoul(page, 10, &new))
  5970. return -EINVAL;
  5971. if (new != PARITY_DISABLE_RMW && !raid6_call.xor_syndrome)
  5972. return -EINVAL;
  5973. if (new != PARITY_DISABLE_RMW &&
  5974. new != PARITY_ENABLE_RMW &&
  5975. new != PARITY_PREFER_RMW)
  5976. return -EINVAL;
  5977. conf->rmw_level = new;
  5978. return len;
  5979. }
  5980. static struct md_sysfs_entry
  5981. raid5_rmw_level = __ATTR(rmw_level, S_IRUGO | S_IWUSR,
  5982. raid5_show_rmw_level,
  5983. raid5_store_rmw_level);
  5984. static ssize_t
  5985. raid5_show_stripe_size(struct mddev *mddev, char *page)
  5986. {
  5987. struct r5conf *conf;
  5988. int ret = 0;
  5989. spin_lock(&mddev->lock);
  5990. conf = mddev->private;
  5991. if (conf)
  5992. ret = sprintf(page, "%lu\n", RAID5_STRIPE_SIZE(conf));
  5993. spin_unlock(&mddev->lock);
  5994. return ret;
  5995. }
  5996. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  5997. static ssize_t
  5998. raid5_store_stripe_size(struct mddev *mddev, const char *page, size_t len)
  5999. {
  6000. struct r5conf *conf;
  6001. unsigned long new;
  6002. int err;
  6003. int size;
  6004. if (len >= PAGE_SIZE)
  6005. return -EINVAL;
  6006. if (kstrtoul(page, 10, &new))
  6007. return -EINVAL;
  6008. /*
  6009. * The value should not be bigger than PAGE_SIZE. It requires to
  6010. * be multiple of DEFAULT_STRIPE_SIZE and the value should be power
  6011. * of two.
  6012. */
  6013. if (new % DEFAULT_STRIPE_SIZE != 0 ||
  6014. new > PAGE_SIZE || new == 0 ||
  6015. new != roundup_pow_of_two(new))
  6016. return -EINVAL;
  6017. err = mddev_lock(mddev);
  6018. if (err)
  6019. return err;
  6020. conf = mddev->private;
  6021. if (!conf) {
  6022. err = -ENODEV;
  6023. goto out_unlock;
  6024. }
  6025. if (new == conf->stripe_size)
  6026. goto out_unlock;
  6027. pr_debug("md/raid: change stripe_size from %lu to %lu\n",
  6028. conf->stripe_size, new);
  6029. if (mddev->sync_thread ||
  6030. test_bit(MD_RECOVERY_RUNNING, &mddev->recovery) ||
  6031. mddev->reshape_position != MaxSector ||
  6032. mddev->sysfs_active) {
  6033. err = -EBUSY;
  6034. goto out_unlock;
  6035. }
  6036. mddev_suspend(mddev);
  6037. mutex_lock(&conf->cache_size_mutex);
  6038. size = conf->max_nr_stripes;
  6039. shrink_stripes(conf);
  6040. conf->stripe_size = new;
  6041. conf->stripe_shift = ilog2(new) - 9;
  6042. conf->stripe_sectors = new >> 9;
  6043. if (grow_stripes(conf, size)) {
  6044. pr_warn("md/raid:%s: couldn't allocate buffers\n",
  6045. mdname(mddev));
  6046. err = -ENOMEM;
  6047. }
  6048. mutex_unlock(&conf->cache_size_mutex);
  6049. mddev_resume(mddev);
  6050. out_unlock:
  6051. mddev_unlock(mddev);
  6052. return err ?: len;
  6053. }
  6054. static struct md_sysfs_entry
  6055. raid5_stripe_size = __ATTR(stripe_size, 0644,
  6056. raid5_show_stripe_size,
  6057. raid5_store_stripe_size);
  6058. #else
  6059. static struct md_sysfs_entry
  6060. raid5_stripe_size = __ATTR(stripe_size, 0444,
  6061. raid5_show_stripe_size,
  6062. NULL);
  6063. #endif
  6064. static ssize_t
  6065. raid5_show_preread_threshold(struct mddev *mddev, char *page)
  6066. {
  6067. struct r5conf *conf;
  6068. int ret = 0;
  6069. spin_lock(&mddev->lock);
  6070. conf = mddev->private;
  6071. if (conf)
  6072. ret = sprintf(page, "%d\n", conf->bypass_threshold);
  6073. spin_unlock(&mddev->lock);
  6074. return ret;
  6075. }
  6076. static ssize_t
  6077. raid5_store_preread_threshold(struct mddev *mddev, const char *page, size_t len)
  6078. {
  6079. struct r5conf *conf;
  6080. unsigned long new;
  6081. int err;
  6082. if (len >= PAGE_SIZE)
  6083. return -EINVAL;
  6084. if (kstrtoul(page, 10, &new))
  6085. return -EINVAL;
  6086. err = mddev_lock(mddev);
  6087. if (err)
  6088. return err;
  6089. conf = mddev->private;
  6090. if (!conf)
  6091. err = -ENODEV;
  6092. else if (new > conf->min_nr_stripes)
  6093. err = -EINVAL;
  6094. else
  6095. conf->bypass_threshold = new;
  6096. mddev_unlock(mddev);
  6097. return err ?: len;
  6098. }
  6099. static struct md_sysfs_entry
  6100. raid5_preread_bypass_threshold = __ATTR(preread_bypass_threshold,
  6101. S_IRUGO | S_IWUSR,
  6102. raid5_show_preread_threshold,
  6103. raid5_store_preread_threshold);
  6104. static ssize_t
  6105. raid5_show_skip_copy(struct mddev *mddev, char *page)
  6106. {
  6107. struct r5conf *conf;
  6108. int ret = 0;
  6109. spin_lock(&mddev->lock);
  6110. conf = mddev->private;
  6111. if (conf)
  6112. ret = sprintf(page, "%d\n", conf->skip_copy);
  6113. spin_unlock(&mddev->lock);
  6114. return ret;
  6115. }
  6116. static ssize_t
  6117. raid5_store_skip_copy(struct mddev *mddev, const char *page, size_t len)
  6118. {
  6119. struct r5conf *conf;
  6120. unsigned long new;
  6121. int err;
  6122. if (len >= PAGE_SIZE)
  6123. return -EINVAL;
  6124. if (kstrtoul(page, 10, &new))
  6125. return -EINVAL;
  6126. new = !!new;
  6127. err = mddev_lock(mddev);
  6128. if (err)
  6129. return err;
  6130. conf = mddev->private;
  6131. if (!conf)
  6132. err = -ENODEV;
  6133. else if (new != conf->skip_copy) {
  6134. struct request_queue *q = mddev->queue;
  6135. mddev_suspend(mddev);
  6136. conf->skip_copy = new;
  6137. if (new)
  6138. blk_queue_flag_set(QUEUE_FLAG_STABLE_WRITES, q);
  6139. else
  6140. blk_queue_flag_clear(QUEUE_FLAG_STABLE_WRITES, q);
  6141. mddev_resume(mddev);
  6142. }
  6143. mddev_unlock(mddev);
  6144. return err ?: len;
  6145. }
  6146. static struct md_sysfs_entry
  6147. raid5_skip_copy = __ATTR(skip_copy, S_IRUGO | S_IWUSR,
  6148. raid5_show_skip_copy,
  6149. raid5_store_skip_copy);
  6150. static ssize_t
  6151. stripe_cache_active_show(struct mddev *mddev, char *page)
  6152. {
  6153. struct r5conf *conf = mddev->private;
  6154. if (conf)
  6155. return sprintf(page, "%d\n", atomic_read(&conf->active_stripes));
  6156. else
  6157. return 0;
  6158. }
  6159. static struct md_sysfs_entry
  6160. raid5_stripecache_active = __ATTR_RO(stripe_cache_active);
  6161. static ssize_t
  6162. raid5_show_group_thread_cnt(struct mddev *mddev, char *page)
  6163. {
  6164. struct r5conf *conf;
  6165. int ret = 0;
  6166. spin_lock(&mddev->lock);
  6167. conf = mddev->private;
  6168. if (conf)
  6169. ret = sprintf(page, "%d\n", conf->worker_cnt_per_group);
  6170. spin_unlock(&mddev->lock);
  6171. return ret;
  6172. }
  6173. static int alloc_thread_groups(struct r5conf *conf, int cnt,
  6174. int *group_cnt,
  6175. struct r5worker_group **worker_groups);
  6176. static ssize_t
  6177. raid5_store_group_thread_cnt(struct mddev *mddev, const char *page, size_t len)
  6178. {
  6179. struct r5conf *conf;
  6180. unsigned int new;
  6181. int err;
  6182. struct r5worker_group *new_groups, *old_groups;
  6183. int group_cnt;
  6184. if (len >= PAGE_SIZE)
  6185. return -EINVAL;
  6186. if (kstrtouint(page, 10, &new))
  6187. return -EINVAL;
  6188. /* 8192 should be big enough */
  6189. if (new > 8192)
  6190. return -EINVAL;
  6191. err = mddev_lock(mddev);
  6192. if (err)
  6193. return err;
  6194. conf = mddev->private;
  6195. if (!conf)
  6196. err = -ENODEV;
  6197. else if (new != conf->worker_cnt_per_group) {
  6198. mddev_suspend(mddev);
  6199. old_groups = conf->worker_groups;
  6200. if (old_groups)
  6201. flush_workqueue(raid5_wq);
  6202. err = alloc_thread_groups(conf, new, &group_cnt, &new_groups);
  6203. if (!err) {
  6204. spin_lock_irq(&conf->device_lock);
  6205. conf->group_cnt = group_cnt;
  6206. conf->worker_cnt_per_group = new;
  6207. conf->worker_groups = new_groups;
  6208. spin_unlock_irq(&conf->device_lock);
  6209. if (old_groups)
  6210. kfree(old_groups[0].workers);
  6211. kfree(old_groups);
  6212. }
  6213. mddev_resume(mddev);
  6214. }
  6215. mddev_unlock(mddev);
  6216. return err ?: len;
  6217. }
  6218. static struct md_sysfs_entry
  6219. raid5_group_thread_cnt = __ATTR(group_thread_cnt, S_IRUGO | S_IWUSR,
  6220. raid5_show_group_thread_cnt,
  6221. raid5_store_group_thread_cnt);
  6222. static struct attribute *raid5_attrs[] = {
  6223. &raid5_stripecache_size.attr,
  6224. &raid5_stripecache_active.attr,
  6225. &raid5_preread_bypass_threshold.attr,
  6226. &raid5_group_thread_cnt.attr,
  6227. &raid5_skip_copy.attr,
  6228. &raid5_rmw_level.attr,
  6229. &raid5_stripe_size.attr,
  6230. &r5c_journal_mode.attr,
  6231. &ppl_write_hint.attr,
  6232. NULL,
  6233. };
  6234. static struct attribute_group raid5_attrs_group = {
  6235. .name = NULL,
  6236. .attrs = raid5_attrs,
  6237. };
  6238. static int alloc_thread_groups(struct r5conf *conf, int cnt, int *group_cnt,
  6239. struct r5worker_group **worker_groups)
  6240. {
  6241. int i, j, k;
  6242. ssize_t size;
  6243. struct r5worker *workers;
  6244. if (cnt == 0) {
  6245. *group_cnt = 0;
  6246. *worker_groups = NULL;
  6247. return 0;
  6248. }
  6249. *group_cnt = num_possible_nodes();
  6250. size = sizeof(struct r5worker) * cnt;
  6251. workers = kcalloc(size, *group_cnt, GFP_NOIO);
  6252. *worker_groups = kcalloc(*group_cnt, sizeof(struct r5worker_group),
  6253. GFP_NOIO);
  6254. if (!*worker_groups || !workers) {
  6255. kfree(workers);
  6256. kfree(*worker_groups);
  6257. return -ENOMEM;
  6258. }
  6259. for (i = 0; i < *group_cnt; i++) {
  6260. struct r5worker_group *group;
  6261. group = &(*worker_groups)[i];
  6262. INIT_LIST_HEAD(&group->handle_list);
  6263. INIT_LIST_HEAD(&group->loprio_list);
  6264. group->conf = conf;
  6265. group->workers = workers + i * cnt;
  6266. for (j = 0; j < cnt; j++) {
  6267. struct r5worker *worker = group->workers + j;
  6268. worker->group = group;
  6269. INIT_WORK(&worker->work, raid5_do_work);
  6270. for (k = 0; k < NR_STRIPE_HASH_LOCKS; k++)
  6271. INIT_LIST_HEAD(worker->temp_inactive_list + k);
  6272. }
  6273. }
  6274. return 0;
  6275. }
  6276. static void free_thread_groups(struct r5conf *conf)
  6277. {
  6278. if (conf->worker_groups)
  6279. kfree(conf->worker_groups[0].workers);
  6280. kfree(conf->worker_groups);
  6281. conf->worker_groups = NULL;
  6282. }
  6283. static sector_t
  6284. raid5_size(struct mddev *mddev, sector_t sectors, int raid_disks)
  6285. {
  6286. struct r5conf *conf = mddev->private;
  6287. if (!sectors)
  6288. sectors = mddev->dev_sectors;
  6289. if (!raid_disks)
  6290. /* size is defined by the smallest of previous and new size */
  6291. raid_disks = min(conf->raid_disks, conf->previous_raid_disks);
  6292. sectors &= ~((sector_t)conf->chunk_sectors - 1);
  6293. sectors &= ~((sector_t)conf->prev_chunk_sectors - 1);
  6294. return sectors * (raid_disks - conf->max_degraded);
  6295. }
  6296. static void free_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
  6297. {
  6298. safe_put_page(percpu->spare_page);
  6299. percpu->spare_page = NULL;
  6300. kvfree(percpu->scribble);
  6301. percpu->scribble = NULL;
  6302. }
  6303. static int alloc_scratch_buffer(struct r5conf *conf, struct raid5_percpu *percpu)
  6304. {
  6305. if (conf->level == 6 && !percpu->spare_page) {
  6306. percpu->spare_page = alloc_page(GFP_KERNEL);
  6307. if (!percpu->spare_page)
  6308. return -ENOMEM;
  6309. }
  6310. if (scribble_alloc(percpu,
  6311. max(conf->raid_disks,
  6312. conf->previous_raid_disks),
  6313. max(conf->chunk_sectors,
  6314. conf->prev_chunk_sectors)
  6315. / RAID5_STRIPE_SECTORS(conf))) {
  6316. free_scratch_buffer(conf, percpu);
  6317. return -ENOMEM;
  6318. }
  6319. return 0;
  6320. }
  6321. static int raid456_cpu_dead(unsigned int cpu, struct hlist_node *node)
  6322. {
  6323. struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
  6324. free_scratch_buffer(conf, per_cpu_ptr(conf->percpu, cpu));
  6325. return 0;
  6326. }
  6327. static void raid5_free_percpu(struct r5conf *conf)
  6328. {
  6329. if (!conf->percpu)
  6330. return;
  6331. cpuhp_state_remove_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
  6332. free_percpu(conf->percpu);
  6333. }
  6334. static void free_conf(struct r5conf *conf)
  6335. {
  6336. int i;
  6337. log_exit(conf);
  6338. unregister_shrinker(&conf->shrinker);
  6339. free_thread_groups(conf);
  6340. shrink_stripes(conf);
  6341. raid5_free_percpu(conf);
  6342. for (i = 0; i < conf->pool_size; i++)
  6343. if (conf->disks[i].extra_page)
  6344. put_page(conf->disks[i].extra_page);
  6345. kfree(conf->disks);
  6346. bioset_exit(&conf->bio_split);
  6347. kfree(conf->stripe_hashtbl);
  6348. kfree(conf->pending_data);
  6349. kfree(conf);
  6350. }
  6351. static int raid456_cpu_up_prepare(unsigned int cpu, struct hlist_node *node)
  6352. {
  6353. struct r5conf *conf = hlist_entry_safe(node, struct r5conf, node);
  6354. struct raid5_percpu *percpu = per_cpu_ptr(conf->percpu, cpu);
  6355. if (alloc_scratch_buffer(conf, percpu)) {
  6356. pr_warn("%s: failed memory allocation for cpu%u\n",
  6357. __func__, cpu);
  6358. return -ENOMEM;
  6359. }
  6360. return 0;
  6361. }
  6362. static int raid5_alloc_percpu(struct r5conf *conf)
  6363. {
  6364. int err = 0;
  6365. conf->percpu = alloc_percpu(struct raid5_percpu);
  6366. if (!conf->percpu)
  6367. return -ENOMEM;
  6368. err = cpuhp_state_add_instance(CPUHP_MD_RAID5_PREPARE, &conf->node);
  6369. if (!err) {
  6370. conf->scribble_disks = max(conf->raid_disks,
  6371. conf->previous_raid_disks);
  6372. conf->scribble_sectors = max(conf->chunk_sectors,
  6373. conf->prev_chunk_sectors);
  6374. }
  6375. return err;
  6376. }
  6377. static unsigned long raid5_cache_scan(struct shrinker *shrink,
  6378. struct shrink_control *sc)
  6379. {
  6380. struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
  6381. unsigned long ret = SHRINK_STOP;
  6382. if (mutex_trylock(&conf->cache_size_mutex)) {
  6383. ret= 0;
  6384. while (ret < sc->nr_to_scan &&
  6385. conf->max_nr_stripes > conf->min_nr_stripes) {
  6386. if (drop_one_stripe(conf) == 0) {
  6387. ret = SHRINK_STOP;
  6388. break;
  6389. }
  6390. ret++;
  6391. }
  6392. mutex_unlock(&conf->cache_size_mutex);
  6393. }
  6394. return ret;
  6395. }
  6396. static unsigned long raid5_cache_count(struct shrinker *shrink,
  6397. struct shrink_control *sc)
  6398. {
  6399. struct r5conf *conf = container_of(shrink, struct r5conf, shrinker);
  6400. if (conf->max_nr_stripes < conf->min_nr_stripes)
  6401. /* unlikely, but not impossible */
  6402. return 0;
  6403. return conf->max_nr_stripes - conf->min_nr_stripes;
  6404. }
  6405. static struct r5conf *setup_conf(struct mddev *mddev)
  6406. {
  6407. struct r5conf *conf;
  6408. int raid_disk, memory, max_disks;
  6409. struct md_rdev *rdev;
  6410. struct disk_info *disk;
  6411. char pers_name[6];
  6412. int i;
  6413. int group_cnt;
  6414. struct r5worker_group *new_group;
  6415. int ret;
  6416. if (mddev->new_level != 5
  6417. && mddev->new_level != 4
  6418. && mddev->new_level != 6) {
  6419. pr_warn("md/raid:%s: raid level not set to 4/5/6 (%d)\n",
  6420. mdname(mddev), mddev->new_level);
  6421. return ERR_PTR(-EIO);
  6422. }
  6423. if ((mddev->new_level == 5
  6424. && !algorithm_valid_raid5(mddev->new_layout)) ||
  6425. (mddev->new_level == 6
  6426. && !algorithm_valid_raid6(mddev->new_layout))) {
  6427. pr_warn("md/raid:%s: layout %d not supported\n",
  6428. mdname(mddev), mddev->new_layout);
  6429. return ERR_PTR(-EIO);
  6430. }
  6431. if (mddev->new_level == 6 && mddev->raid_disks < 4) {
  6432. pr_warn("md/raid:%s: not enough configured devices (%d, minimum 4)\n",
  6433. mdname(mddev), mddev->raid_disks);
  6434. return ERR_PTR(-EINVAL);
  6435. }
  6436. if (!mddev->new_chunk_sectors ||
  6437. (mddev->new_chunk_sectors << 9) % PAGE_SIZE ||
  6438. !is_power_of_2(mddev->new_chunk_sectors)) {
  6439. pr_warn("md/raid:%s: invalid chunk size %d\n",
  6440. mdname(mddev), mddev->new_chunk_sectors << 9);
  6441. return ERR_PTR(-EINVAL);
  6442. }
  6443. conf = kzalloc(sizeof(struct r5conf), GFP_KERNEL);
  6444. if (conf == NULL)
  6445. goto abort;
  6446. #if PAGE_SIZE != DEFAULT_STRIPE_SIZE
  6447. conf->stripe_size = DEFAULT_STRIPE_SIZE;
  6448. conf->stripe_shift = ilog2(DEFAULT_STRIPE_SIZE) - 9;
  6449. conf->stripe_sectors = DEFAULT_STRIPE_SIZE >> 9;
  6450. #endif
  6451. INIT_LIST_HEAD(&conf->free_list);
  6452. INIT_LIST_HEAD(&conf->pending_list);
  6453. conf->pending_data = kcalloc(PENDING_IO_MAX,
  6454. sizeof(struct r5pending_data),
  6455. GFP_KERNEL);
  6456. if (!conf->pending_data)
  6457. goto abort;
  6458. for (i = 0; i < PENDING_IO_MAX; i++)
  6459. list_add(&conf->pending_data[i].sibling, &conf->free_list);
  6460. /* Don't enable multi-threading by default*/
  6461. if (!alloc_thread_groups(conf, 0, &group_cnt, &new_group)) {
  6462. conf->group_cnt = group_cnt;
  6463. conf->worker_cnt_per_group = 0;
  6464. conf->worker_groups = new_group;
  6465. } else
  6466. goto abort;
  6467. spin_lock_init(&conf->device_lock);
  6468. seqcount_spinlock_init(&conf->gen_lock, &conf->device_lock);
  6469. mutex_init(&conf->cache_size_mutex);
  6470. init_waitqueue_head(&conf->wait_for_quiescent);
  6471. init_waitqueue_head(&conf->wait_for_stripe);
  6472. init_waitqueue_head(&conf->wait_for_overlap);
  6473. INIT_LIST_HEAD(&conf->handle_list);
  6474. INIT_LIST_HEAD(&conf->loprio_list);
  6475. INIT_LIST_HEAD(&conf->hold_list);
  6476. INIT_LIST_HEAD(&conf->delayed_list);
  6477. INIT_LIST_HEAD(&conf->bitmap_list);
  6478. init_llist_head(&conf->released_stripes);
  6479. atomic_set(&conf->active_stripes, 0);
  6480. atomic_set(&conf->preread_active_stripes, 0);
  6481. atomic_set(&conf->active_aligned_reads, 0);
  6482. spin_lock_init(&conf->pending_bios_lock);
  6483. conf->batch_bio_dispatch = true;
  6484. rdev_for_each(rdev, mddev) {
  6485. if (test_bit(Journal, &rdev->flags))
  6486. continue;
  6487. if (blk_queue_nonrot(bdev_get_queue(rdev->bdev))) {
  6488. conf->batch_bio_dispatch = false;
  6489. break;
  6490. }
  6491. }
  6492. conf->bypass_threshold = BYPASS_THRESHOLD;
  6493. conf->recovery_disabled = mddev->recovery_disabled - 1;
  6494. conf->raid_disks = mddev->raid_disks;
  6495. if (mddev->reshape_position == MaxSector)
  6496. conf->previous_raid_disks = mddev->raid_disks;
  6497. else
  6498. conf->previous_raid_disks = mddev->raid_disks - mddev->delta_disks;
  6499. max_disks = max(conf->raid_disks, conf->previous_raid_disks);
  6500. conf->disks = kcalloc(max_disks, sizeof(struct disk_info),
  6501. GFP_KERNEL);
  6502. if (!conf->disks)
  6503. goto abort;
  6504. for (i = 0; i < max_disks; i++) {
  6505. conf->disks[i].extra_page = alloc_page(GFP_KERNEL);
  6506. if (!conf->disks[i].extra_page)
  6507. goto abort;
  6508. }
  6509. ret = bioset_init(&conf->bio_split, BIO_POOL_SIZE, 0, 0);
  6510. if (ret)
  6511. goto abort;
  6512. conf->mddev = mddev;
  6513. if ((conf->stripe_hashtbl = kzalloc(PAGE_SIZE, GFP_KERNEL)) == NULL)
  6514. goto abort;
  6515. /* We init hash_locks[0] separately to that it can be used
  6516. * as the reference lock in the spin_lock_nest_lock() call
  6517. * in lock_all_device_hash_locks_irq in order to convince
  6518. * lockdep that we know what we are doing.
  6519. */
  6520. spin_lock_init(conf->hash_locks);
  6521. for (i = 1; i < NR_STRIPE_HASH_LOCKS; i++)
  6522. spin_lock_init(conf->hash_locks + i);
  6523. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  6524. INIT_LIST_HEAD(conf->inactive_list + i);
  6525. for (i = 0; i < NR_STRIPE_HASH_LOCKS; i++)
  6526. INIT_LIST_HEAD(conf->temp_inactive_list + i);
  6527. atomic_set(&conf->r5c_cached_full_stripes, 0);
  6528. INIT_LIST_HEAD(&conf->r5c_full_stripe_list);
  6529. atomic_set(&conf->r5c_cached_partial_stripes, 0);
  6530. INIT_LIST_HEAD(&conf->r5c_partial_stripe_list);
  6531. atomic_set(&conf->r5c_flushing_full_stripes, 0);
  6532. atomic_set(&conf->r5c_flushing_partial_stripes, 0);
  6533. conf->level = mddev->new_level;
  6534. conf->chunk_sectors = mddev->new_chunk_sectors;
  6535. if (raid5_alloc_percpu(conf) != 0)
  6536. goto abort;
  6537. pr_debug("raid456: run(%s) called.\n", mdname(mddev));
  6538. rdev_for_each(rdev, mddev) {
  6539. raid_disk = rdev->raid_disk;
  6540. if (raid_disk >= max_disks
  6541. || raid_disk < 0 || test_bit(Journal, &rdev->flags))
  6542. continue;
  6543. disk = conf->disks + raid_disk;
  6544. if (test_bit(Replacement, &rdev->flags)) {
  6545. if (disk->replacement)
  6546. goto abort;
  6547. disk->replacement = rdev;
  6548. } else {
  6549. if (disk->rdev)
  6550. goto abort;
  6551. disk->rdev = rdev;
  6552. }
  6553. if (test_bit(In_sync, &rdev->flags)) {
  6554. char b[BDEVNAME_SIZE];
  6555. pr_info("md/raid:%s: device %s operational as raid disk %d\n",
  6556. mdname(mddev), bdevname(rdev->bdev, b), raid_disk);
  6557. } else if (rdev->saved_raid_disk != raid_disk)
  6558. /* Cannot rely on bitmap to complete recovery */
  6559. conf->fullsync = 1;
  6560. }
  6561. conf->level = mddev->new_level;
  6562. if (conf->level == 6) {
  6563. conf->max_degraded = 2;
  6564. if (raid6_call.xor_syndrome)
  6565. conf->rmw_level = PARITY_ENABLE_RMW;
  6566. else
  6567. conf->rmw_level = PARITY_DISABLE_RMW;
  6568. } else {
  6569. conf->max_degraded = 1;
  6570. conf->rmw_level = PARITY_ENABLE_RMW;
  6571. }
  6572. conf->algorithm = mddev->new_layout;
  6573. conf->reshape_progress = mddev->reshape_position;
  6574. if (conf->reshape_progress != MaxSector) {
  6575. conf->prev_chunk_sectors = mddev->chunk_sectors;
  6576. conf->prev_algo = mddev->layout;
  6577. } else {
  6578. conf->prev_chunk_sectors = conf->chunk_sectors;
  6579. conf->prev_algo = conf->algorithm;
  6580. }
  6581. conf->min_nr_stripes = NR_STRIPES;
  6582. if (mddev->reshape_position != MaxSector) {
  6583. int stripes = max_t(int,
  6584. ((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4,
  6585. ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4);
  6586. conf->min_nr_stripes = max(NR_STRIPES, stripes);
  6587. if (conf->min_nr_stripes != NR_STRIPES)
  6588. pr_info("md/raid:%s: force stripe size %d for reshape\n",
  6589. mdname(mddev), conf->min_nr_stripes);
  6590. }
  6591. memory = conf->min_nr_stripes * (sizeof(struct stripe_head) +
  6592. max_disks * ((sizeof(struct bio) + PAGE_SIZE))) / 1024;
  6593. atomic_set(&conf->empty_inactive_list_nr, NR_STRIPE_HASH_LOCKS);
  6594. if (grow_stripes(conf, conf->min_nr_stripes)) {
  6595. pr_warn("md/raid:%s: couldn't allocate %dkB for buffers\n",
  6596. mdname(mddev), memory);
  6597. goto abort;
  6598. } else
  6599. pr_debug("md/raid:%s: allocated %dkB\n", mdname(mddev), memory);
  6600. /*
  6601. * Losing a stripe head costs more than the time to refill it,
  6602. * it reduces the queue depth and so can hurt throughput.
  6603. * So set it rather large, scaled by number of devices.
  6604. */
  6605. conf->shrinker.seeks = DEFAULT_SEEKS * conf->raid_disks * 4;
  6606. conf->shrinker.scan_objects = raid5_cache_scan;
  6607. conf->shrinker.count_objects = raid5_cache_count;
  6608. conf->shrinker.batch = 128;
  6609. conf->shrinker.flags = 0;
  6610. if (register_shrinker(&conf->shrinker)) {
  6611. pr_warn("md/raid:%s: couldn't register shrinker.\n",
  6612. mdname(mddev));
  6613. goto abort;
  6614. }
  6615. sprintf(pers_name, "raid%d", mddev->new_level);
  6616. conf->thread = md_register_thread(raid5d, mddev, pers_name);
  6617. if (!conf->thread) {
  6618. pr_warn("md/raid:%s: couldn't allocate thread.\n",
  6619. mdname(mddev));
  6620. goto abort;
  6621. }
  6622. return conf;
  6623. abort:
  6624. if (conf) {
  6625. free_conf(conf);
  6626. return ERR_PTR(-EIO);
  6627. } else
  6628. return ERR_PTR(-ENOMEM);
  6629. }
  6630. static int only_parity(int raid_disk, int algo, int raid_disks, int max_degraded)
  6631. {
  6632. switch (algo) {
  6633. case ALGORITHM_PARITY_0:
  6634. if (raid_disk < max_degraded)
  6635. return 1;
  6636. break;
  6637. case ALGORITHM_PARITY_N:
  6638. if (raid_disk >= raid_disks - max_degraded)
  6639. return 1;
  6640. break;
  6641. case ALGORITHM_PARITY_0_6:
  6642. if (raid_disk == 0 ||
  6643. raid_disk == raid_disks - 1)
  6644. return 1;
  6645. break;
  6646. case ALGORITHM_LEFT_ASYMMETRIC_6:
  6647. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  6648. case ALGORITHM_LEFT_SYMMETRIC_6:
  6649. case ALGORITHM_RIGHT_SYMMETRIC_6:
  6650. if (raid_disk == raid_disks - 1)
  6651. return 1;
  6652. }
  6653. return 0;
  6654. }
  6655. static void raid5_set_io_opt(struct r5conf *conf)
  6656. {
  6657. blk_queue_io_opt(conf->mddev->queue, (conf->chunk_sectors << 9) *
  6658. (conf->raid_disks - conf->max_degraded));
  6659. }
  6660. static int raid5_run(struct mddev *mddev)
  6661. {
  6662. struct r5conf *conf;
  6663. int working_disks = 0;
  6664. int dirty_parity_disks = 0;
  6665. struct md_rdev *rdev;
  6666. struct md_rdev *journal_dev = NULL;
  6667. sector_t reshape_offset = 0;
  6668. int i;
  6669. long long min_offset_diff = 0;
  6670. int first = 1;
  6671. if (mddev_init_writes_pending(mddev) < 0)
  6672. return -ENOMEM;
  6673. if (mddev->recovery_cp != MaxSector)
  6674. pr_notice("md/raid:%s: not clean -- starting background reconstruction\n",
  6675. mdname(mddev));
  6676. rdev_for_each(rdev, mddev) {
  6677. long long diff;
  6678. if (test_bit(Journal, &rdev->flags)) {
  6679. journal_dev = rdev;
  6680. continue;
  6681. }
  6682. if (rdev->raid_disk < 0)
  6683. continue;
  6684. diff = (rdev->new_data_offset - rdev->data_offset);
  6685. if (first) {
  6686. min_offset_diff = diff;
  6687. first = 0;
  6688. } else if (mddev->reshape_backwards &&
  6689. diff < min_offset_diff)
  6690. min_offset_diff = diff;
  6691. else if (!mddev->reshape_backwards &&
  6692. diff > min_offset_diff)
  6693. min_offset_diff = diff;
  6694. }
  6695. if ((test_bit(MD_HAS_JOURNAL, &mddev->flags) || journal_dev) &&
  6696. (mddev->bitmap_info.offset || mddev->bitmap_info.file)) {
  6697. pr_notice("md/raid:%s: array cannot have both journal and bitmap\n",
  6698. mdname(mddev));
  6699. return -EINVAL;
  6700. }
  6701. if (mddev->reshape_position != MaxSector) {
  6702. /* Check that we can continue the reshape.
  6703. * Difficulties arise if the stripe we would write to
  6704. * next is at or after the stripe we would read from next.
  6705. * For a reshape that changes the number of devices, this
  6706. * is only possible for a very short time, and mdadm makes
  6707. * sure that time appears to have past before assembling
  6708. * the array. So we fail if that time hasn't passed.
  6709. * For a reshape that keeps the number of devices the same
  6710. * mdadm must be monitoring the reshape can keeping the
  6711. * critical areas read-only and backed up. It will start
  6712. * the array in read-only mode, so we check for that.
  6713. */
  6714. sector_t here_new, here_old;
  6715. int old_disks;
  6716. int max_degraded = (mddev->level == 6 ? 2 : 1);
  6717. int chunk_sectors;
  6718. int new_data_disks;
  6719. if (journal_dev) {
  6720. pr_warn("md/raid:%s: don't support reshape with journal - aborting.\n",
  6721. mdname(mddev));
  6722. return -EINVAL;
  6723. }
  6724. if (mddev->new_level != mddev->level) {
  6725. pr_warn("md/raid:%s: unsupported reshape required - aborting.\n",
  6726. mdname(mddev));
  6727. return -EINVAL;
  6728. }
  6729. old_disks = mddev->raid_disks - mddev->delta_disks;
  6730. /* reshape_position must be on a new-stripe boundary, and one
  6731. * further up in new geometry must map after here in old
  6732. * geometry.
  6733. * If the chunk sizes are different, then as we perform reshape
  6734. * in units of the largest of the two, reshape_position needs
  6735. * be a multiple of the largest chunk size times new data disks.
  6736. */
  6737. here_new = mddev->reshape_position;
  6738. chunk_sectors = max(mddev->chunk_sectors, mddev->new_chunk_sectors);
  6739. new_data_disks = mddev->raid_disks - max_degraded;
  6740. if (sector_div(here_new, chunk_sectors * new_data_disks)) {
  6741. pr_warn("md/raid:%s: reshape_position not on a stripe boundary\n",
  6742. mdname(mddev));
  6743. return -EINVAL;
  6744. }
  6745. reshape_offset = here_new * chunk_sectors;
  6746. /* here_new is the stripe we will write to */
  6747. here_old = mddev->reshape_position;
  6748. sector_div(here_old, chunk_sectors * (old_disks-max_degraded));
  6749. /* here_old is the first stripe that we might need to read
  6750. * from */
  6751. if (mddev->delta_disks == 0) {
  6752. /* We cannot be sure it is safe to start an in-place
  6753. * reshape. It is only safe if user-space is monitoring
  6754. * and taking constant backups.
  6755. * mdadm always starts a situation like this in
  6756. * readonly mode so it can take control before
  6757. * allowing any writes. So just check for that.
  6758. */
  6759. if (abs(min_offset_diff) >= mddev->chunk_sectors &&
  6760. abs(min_offset_diff) >= mddev->new_chunk_sectors)
  6761. /* not really in-place - so OK */;
  6762. else if (mddev->ro == 0) {
  6763. pr_warn("md/raid:%s: in-place reshape must be started in read-only mode - aborting\n",
  6764. mdname(mddev));
  6765. return -EINVAL;
  6766. }
  6767. } else if (mddev->reshape_backwards
  6768. ? (here_new * chunk_sectors + min_offset_diff <=
  6769. here_old * chunk_sectors)
  6770. : (here_new * chunk_sectors >=
  6771. here_old * chunk_sectors + (-min_offset_diff))) {
  6772. /* Reading from the same stripe as writing to - bad */
  6773. pr_warn("md/raid:%s: reshape_position too early for auto-recovery - aborting.\n",
  6774. mdname(mddev));
  6775. return -EINVAL;
  6776. }
  6777. pr_debug("md/raid:%s: reshape will continue\n", mdname(mddev));
  6778. /* OK, we should be able to continue; */
  6779. } else {
  6780. BUG_ON(mddev->level != mddev->new_level);
  6781. BUG_ON(mddev->layout != mddev->new_layout);
  6782. BUG_ON(mddev->chunk_sectors != mddev->new_chunk_sectors);
  6783. BUG_ON(mddev->delta_disks != 0);
  6784. }
  6785. if (test_bit(MD_HAS_JOURNAL, &mddev->flags) &&
  6786. test_bit(MD_HAS_PPL, &mddev->flags)) {
  6787. pr_warn("md/raid:%s: using journal device and PPL not allowed - disabling PPL\n",
  6788. mdname(mddev));
  6789. clear_bit(MD_HAS_PPL, &mddev->flags);
  6790. clear_bit(MD_HAS_MULTIPLE_PPLS, &mddev->flags);
  6791. }
  6792. if (mddev->private == NULL)
  6793. conf = setup_conf(mddev);
  6794. else
  6795. conf = mddev->private;
  6796. if (IS_ERR(conf))
  6797. return PTR_ERR(conf);
  6798. if (test_bit(MD_HAS_JOURNAL, &mddev->flags)) {
  6799. if (!journal_dev) {
  6800. pr_warn("md/raid:%s: journal disk is missing, force array readonly\n",
  6801. mdname(mddev));
  6802. mddev->ro = 1;
  6803. set_disk_ro(mddev->gendisk, 1);
  6804. } else if (mddev->recovery_cp == MaxSector)
  6805. set_bit(MD_JOURNAL_CLEAN, &mddev->flags);
  6806. }
  6807. conf->min_offset_diff = min_offset_diff;
  6808. mddev->thread = conf->thread;
  6809. conf->thread = NULL;
  6810. mddev->private = conf;
  6811. for (i = 0; i < conf->raid_disks && conf->previous_raid_disks;
  6812. i++) {
  6813. rdev = conf->disks[i].rdev;
  6814. if (!rdev && conf->disks[i].replacement) {
  6815. /* The replacement is all we have yet */
  6816. rdev = conf->disks[i].replacement;
  6817. conf->disks[i].replacement = NULL;
  6818. clear_bit(Replacement, &rdev->flags);
  6819. conf->disks[i].rdev = rdev;
  6820. }
  6821. if (!rdev)
  6822. continue;
  6823. if (conf->disks[i].replacement &&
  6824. conf->reshape_progress != MaxSector) {
  6825. /* replacements and reshape simply do not mix. */
  6826. pr_warn("md: cannot handle concurrent replacement and reshape.\n");
  6827. goto abort;
  6828. }
  6829. if (test_bit(In_sync, &rdev->flags)) {
  6830. working_disks++;
  6831. continue;
  6832. }
  6833. /* This disc is not fully in-sync. However if it
  6834. * just stored parity (beyond the recovery_offset),
  6835. * when we don't need to be concerned about the
  6836. * array being dirty.
  6837. * When reshape goes 'backwards', we never have
  6838. * partially completed devices, so we only need
  6839. * to worry about reshape going forwards.
  6840. */
  6841. /* Hack because v0.91 doesn't store recovery_offset properly. */
  6842. if (mddev->major_version == 0 &&
  6843. mddev->minor_version > 90)
  6844. rdev->recovery_offset = reshape_offset;
  6845. if (rdev->recovery_offset < reshape_offset) {
  6846. /* We need to check old and new layout */
  6847. if (!only_parity(rdev->raid_disk,
  6848. conf->algorithm,
  6849. conf->raid_disks,
  6850. conf->max_degraded))
  6851. continue;
  6852. }
  6853. if (!only_parity(rdev->raid_disk,
  6854. conf->prev_algo,
  6855. conf->previous_raid_disks,
  6856. conf->max_degraded))
  6857. continue;
  6858. dirty_parity_disks++;
  6859. }
  6860. /*
  6861. * 0 for a fully functional array, 1 or 2 for a degraded array.
  6862. */
  6863. mddev->degraded = raid5_calc_degraded(conf);
  6864. if (has_failed(conf)) {
  6865. pr_crit("md/raid:%s: not enough operational devices (%d/%d failed)\n",
  6866. mdname(mddev), mddev->degraded, conf->raid_disks);
  6867. goto abort;
  6868. }
  6869. /* device size must be a multiple of chunk size */
  6870. mddev->dev_sectors &= ~(mddev->chunk_sectors - 1);
  6871. mddev->resync_max_sectors = mddev->dev_sectors;
  6872. if (mddev->degraded > dirty_parity_disks &&
  6873. mddev->recovery_cp != MaxSector) {
  6874. if (test_bit(MD_HAS_PPL, &mddev->flags))
  6875. pr_crit("md/raid:%s: starting dirty degraded array with PPL.\n",
  6876. mdname(mddev));
  6877. else if (mddev->ok_start_degraded)
  6878. pr_crit("md/raid:%s: starting dirty degraded array - data corruption possible.\n",
  6879. mdname(mddev));
  6880. else {
  6881. pr_crit("md/raid:%s: cannot start dirty degraded array.\n",
  6882. mdname(mddev));
  6883. goto abort;
  6884. }
  6885. }
  6886. pr_info("md/raid:%s: raid level %d active with %d out of %d devices, algorithm %d\n",
  6887. mdname(mddev), conf->level,
  6888. mddev->raid_disks-mddev->degraded, mddev->raid_disks,
  6889. mddev->new_layout);
  6890. print_raid5_conf(conf);
  6891. if (conf->reshape_progress != MaxSector) {
  6892. conf->reshape_safe = conf->reshape_progress;
  6893. atomic_set(&conf->reshape_stripes, 0);
  6894. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  6895. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  6896. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  6897. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  6898. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  6899. "reshape");
  6900. if (!mddev->sync_thread)
  6901. goto abort;
  6902. }
  6903. /* Ok, everything is just fine now */
  6904. if (mddev->to_remove == &raid5_attrs_group)
  6905. mddev->to_remove = NULL;
  6906. else if (mddev->kobj.sd &&
  6907. sysfs_create_group(&mddev->kobj, &raid5_attrs_group))
  6908. pr_warn("raid5: failed to create sysfs attributes for %s\n",
  6909. mdname(mddev));
  6910. md_set_array_sectors(mddev, raid5_size(mddev, 0, 0));
  6911. if (mddev->queue) {
  6912. int chunk_size;
  6913. /* read-ahead size must cover two whole stripes, which
  6914. * is 2 * (datadisks) * chunksize where 'n' is the
  6915. * number of raid devices
  6916. */
  6917. int data_disks = conf->previous_raid_disks - conf->max_degraded;
  6918. int stripe = data_disks *
  6919. ((mddev->chunk_sectors << 9) / PAGE_SIZE);
  6920. chunk_size = mddev->chunk_sectors << 9;
  6921. blk_queue_io_min(mddev->queue, chunk_size);
  6922. raid5_set_io_opt(conf);
  6923. mddev->queue->limits.raid_partial_stripes_expensive = 1;
  6924. /*
  6925. * We can only discard a whole stripe. It doesn't make sense to
  6926. * discard data disk but write parity disk
  6927. */
  6928. stripe = stripe * PAGE_SIZE;
  6929. /* Round up to power of 2, as discard handling
  6930. * currently assumes that */
  6931. while ((stripe-1) & stripe)
  6932. stripe = (stripe | (stripe-1)) + 1;
  6933. mddev->queue->limits.discard_alignment = stripe;
  6934. mddev->queue->limits.discard_granularity = stripe;
  6935. blk_queue_max_write_same_sectors(mddev->queue, 0);
  6936. blk_queue_max_write_zeroes_sectors(mddev->queue, 0);
  6937. rdev_for_each(rdev, mddev) {
  6938. disk_stack_limits(mddev->gendisk, rdev->bdev,
  6939. rdev->data_offset << 9);
  6940. disk_stack_limits(mddev->gendisk, rdev->bdev,
  6941. rdev->new_data_offset << 9);
  6942. }
  6943. /*
  6944. * zeroing is required, otherwise data
  6945. * could be lost. Consider a scenario: discard a stripe
  6946. * (the stripe could be inconsistent if
  6947. * discard_zeroes_data is 0); write one disk of the
  6948. * stripe (the stripe could be inconsistent again
  6949. * depending on which disks are used to calculate
  6950. * parity); the disk is broken; The stripe data of this
  6951. * disk is lost.
  6952. *
  6953. * We only allow DISCARD if the sysadmin has confirmed that
  6954. * only safe devices are in use by setting a module parameter.
  6955. * A better idea might be to turn DISCARD into WRITE_ZEROES
  6956. * requests, as that is required to be safe.
  6957. */
  6958. if (devices_handle_discard_safely &&
  6959. mddev->queue->limits.max_discard_sectors >= (stripe >> 9) &&
  6960. mddev->queue->limits.discard_granularity >= stripe)
  6961. blk_queue_flag_set(QUEUE_FLAG_DISCARD,
  6962. mddev->queue);
  6963. else
  6964. blk_queue_flag_clear(QUEUE_FLAG_DISCARD,
  6965. mddev->queue);
  6966. blk_queue_max_hw_sectors(mddev->queue, UINT_MAX);
  6967. }
  6968. if (log_init(conf, journal_dev, raid5_has_ppl(conf)))
  6969. goto abort;
  6970. return 0;
  6971. abort:
  6972. md_unregister_thread(&mddev->thread);
  6973. print_raid5_conf(conf);
  6974. free_conf(conf);
  6975. mddev->private = NULL;
  6976. pr_warn("md/raid:%s: failed to run raid set.\n", mdname(mddev));
  6977. return -EIO;
  6978. }
  6979. static void raid5_free(struct mddev *mddev, void *priv)
  6980. {
  6981. struct r5conf *conf = priv;
  6982. free_conf(conf);
  6983. mddev->to_remove = &raid5_attrs_group;
  6984. }
  6985. static void raid5_status(struct seq_file *seq, struct mddev *mddev)
  6986. {
  6987. struct r5conf *conf = mddev->private;
  6988. int i;
  6989. seq_printf(seq, " level %d, %dk chunk, algorithm %d", mddev->level,
  6990. conf->chunk_sectors / 2, mddev->layout);
  6991. seq_printf (seq, " [%d/%d] [", conf->raid_disks, conf->raid_disks - mddev->degraded);
  6992. rcu_read_lock();
  6993. for (i = 0; i < conf->raid_disks; i++) {
  6994. struct md_rdev *rdev = rcu_dereference(conf->disks[i].rdev);
  6995. seq_printf (seq, "%s", rdev && test_bit(In_sync, &rdev->flags) ? "U" : "_");
  6996. }
  6997. rcu_read_unlock();
  6998. seq_printf (seq, "]");
  6999. }
  7000. static void print_raid5_conf (struct r5conf *conf)
  7001. {
  7002. int i;
  7003. struct disk_info *tmp;
  7004. pr_debug("RAID conf printout:\n");
  7005. if (!conf) {
  7006. pr_debug("(conf==NULL)\n");
  7007. return;
  7008. }
  7009. pr_debug(" --- level:%d rd:%d wd:%d\n", conf->level,
  7010. conf->raid_disks,
  7011. conf->raid_disks - conf->mddev->degraded);
  7012. for (i = 0; i < conf->raid_disks; i++) {
  7013. char b[BDEVNAME_SIZE];
  7014. tmp = conf->disks + i;
  7015. if (tmp->rdev)
  7016. pr_debug(" disk %d, o:%d, dev:%s\n",
  7017. i, !test_bit(Faulty, &tmp->rdev->flags),
  7018. bdevname(tmp->rdev->bdev, b));
  7019. }
  7020. }
  7021. static int raid5_spare_active(struct mddev *mddev)
  7022. {
  7023. int i;
  7024. struct r5conf *conf = mddev->private;
  7025. struct disk_info *tmp;
  7026. int count = 0;
  7027. unsigned long flags;
  7028. for (i = 0; i < conf->raid_disks; i++) {
  7029. tmp = conf->disks + i;
  7030. if (tmp->replacement
  7031. && tmp->replacement->recovery_offset == MaxSector
  7032. && !test_bit(Faulty, &tmp->replacement->flags)
  7033. && !test_and_set_bit(In_sync, &tmp->replacement->flags)) {
  7034. /* Replacement has just become active. */
  7035. if (!tmp->rdev
  7036. || !test_and_clear_bit(In_sync, &tmp->rdev->flags))
  7037. count++;
  7038. if (tmp->rdev) {
  7039. /* Replaced device not technically faulty,
  7040. * but we need to be sure it gets removed
  7041. * and never re-added.
  7042. */
  7043. set_bit(Faulty, &tmp->rdev->flags);
  7044. sysfs_notify_dirent_safe(
  7045. tmp->rdev->sysfs_state);
  7046. }
  7047. sysfs_notify_dirent_safe(tmp->replacement->sysfs_state);
  7048. } else if (tmp->rdev
  7049. && tmp->rdev->recovery_offset == MaxSector
  7050. && !test_bit(Faulty, &tmp->rdev->flags)
  7051. && !test_and_set_bit(In_sync, &tmp->rdev->flags)) {
  7052. count++;
  7053. sysfs_notify_dirent_safe(tmp->rdev->sysfs_state);
  7054. }
  7055. }
  7056. spin_lock_irqsave(&conf->device_lock, flags);
  7057. mddev->degraded = raid5_calc_degraded(conf);
  7058. spin_unlock_irqrestore(&conf->device_lock, flags);
  7059. print_raid5_conf(conf);
  7060. return count;
  7061. }
  7062. static int raid5_remove_disk(struct mddev *mddev, struct md_rdev *rdev)
  7063. {
  7064. struct r5conf *conf = mddev->private;
  7065. int err = 0;
  7066. int number = rdev->raid_disk;
  7067. struct md_rdev **rdevp;
  7068. struct disk_info *p = conf->disks + number;
  7069. print_raid5_conf(conf);
  7070. if (test_bit(Journal, &rdev->flags) && conf->log) {
  7071. /*
  7072. * we can't wait pending write here, as this is called in
  7073. * raid5d, wait will deadlock.
  7074. * neilb: there is no locking about new writes here,
  7075. * so this cannot be safe.
  7076. */
  7077. if (atomic_read(&conf->active_stripes) ||
  7078. atomic_read(&conf->r5c_cached_full_stripes) ||
  7079. atomic_read(&conf->r5c_cached_partial_stripes)) {
  7080. return -EBUSY;
  7081. }
  7082. log_exit(conf);
  7083. return 0;
  7084. }
  7085. if (rdev == p->rdev)
  7086. rdevp = &p->rdev;
  7087. else if (rdev == p->replacement)
  7088. rdevp = &p->replacement;
  7089. else
  7090. return 0;
  7091. if (number >= conf->raid_disks &&
  7092. conf->reshape_progress == MaxSector)
  7093. clear_bit(In_sync, &rdev->flags);
  7094. if (test_bit(In_sync, &rdev->flags) ||
  7095. atomic_read(&rdev->nr_pending)) {
  7096. err = -EBUSY;
  7097. goto abort;
  7098. }
  7099. /* Only remove non-faulty devices if recovery
  7100. * isn't possible.
  7101. */
  7102. if (!test_bit(Faulty, &rdev->flags) &&
  7103. mddev->recovery_disabled != conf->recovery_disabled &&
  7104. !has_failed(conf) &&
  7105. (!p->replacement || p->replacement == rdev) &&
  7106. number < conf->raid_disks) {
  7107. err = -EBUSY;
  7108. goto abort;
  7109. }
  7110. *rdevp = NULL;
  7111. if (!test_bit(RemoveSynchronized, &rdev->flags)) {
  7112. synchronize_rcu();
  7113. if (atomic_read(&rdev->nr_pending)) {
  7114. /* lost the race, try later */
  7115. err = -EBUSY;
  7116. *rdevp = rdev;
  7117. }
  7118. }
  7119. if (!err) {
  7120. err = log_modify(conf, rdev, false);
  7121. if (err)
  7122. goto abort;
  7123. }
  7124. if (p->replacement) {
  7125. /* We must have just cleared 'rdev' */
  7126. p->rdev = p->replacement;
  7127. clear_bit(Replacement, &p->replacement->flags);
  7128. smp_mb(); /* Make sure other CPUs may see both as identical
  7129. * but will never see neither - if they are careful
  7130. */
  7131. p->replacement = NULL;
  7132. if (!err)
  7133. err = log_modify(conf, p->rdev, true);
  7134. }
  7135. clear_bit(WantReplacement, &rdev->flags);
  7136. abort:
  7137. print_raid5_conf(conf);
  7138. return err;
  7139. }
  7140. static int raid5_add_disk(struct mddev *mddev, struct md_rdev *rdev)
  7141. {
  7142. struct r5conf *conf = mddev->private;
  7143. int ret, err = -EEXIST;
  7144. int disk;
  7145. struct disk_info *p;
  7146. int first = 0;
  7147. int last = conf->raid_disks - 1;
  7148. if (test_bit(Journal, &rdev->flags)) {
  7149. if (conf->log)
  7150. return -EBUSY;
  7151. rdev->raid_disk = 0;
  7152. /*
  7153. * The array is in readonly mode if journal is missing, so no
  7154. * write requests running. We should be safe
  7155. */
  7156. ret = log_init(conf, rdev, false);
  7157. if (ret)
  7158. return ret;
  7159. ret = r5l_start(conf->log);
  7160. if (ret)
  7161. return ret;
  7162. return 0;
  7163. }
  7164. if (mddev->recovery_disabled == conf->recovery_disabled)
  7165. return -EBUSY;
  7166. if (rdev->saved_raid_disk < 0 && has_failed(conf))
  7167. /* no point adding a device */
  7168. return -EINVAL;
  7169. if (rdev->raid_disk >= 0)
  7170. first = last = rdev->raid_disk;
  7171. /*
  7172. * find the disk ... but prefer rdev->saved_raid_disk
  7173. * if possible.
  7174. */
  7175. if (rdev->saved_raid_disk >= 0 &&
  7176. rdev->saved_raid_disk >= first &&
  7177. conf->disks[rdev->saved_raid_disk].rdev == NULL)
  7178. first = rdev->saved_raid_disk;
  7179. for (disk = first; disk <= last; disk++) {
  7180. p = conf->disks + disk;
  7181. if (p->rdev == NULL) {
  7182. clear_bit(In_sync, &rdev->flags);
  7183. rdev->raid_disk = disk;
  7184. if (rdev->saved_raid_disk != disk)
  7185. conf->fullsync = 1;
  7186. rcu_assign_pointer(p->rdev, rdev);
  7187. err = log_modify(conf, rdev, true);
  7188. goto out;
  7189. }
  7190. }
  7191. for (disk = first; disk <= last; disk++) {
  7192. p = conf->disks + disk;
  7193. if (test_bit(WantReplacement, &p->rdev->flags) &&
  7194. p->replacement == NULL) {
  7195. clear_bit(In_sync, &rdev->flags);
  7196. set_bit(Replacement, &rdev->flags);
  7197. rdev->raid_disk = disk;
  7198. err = 0;
  7199. conf->fullsync = 1;
  7200. rcu_assign_pointer(p->replacement, rdev);
  7201. break;
  7202. }
  7203. }
  7204. out:
  7205. print_raid5_conf(conf);
  7206. return err;
  7207. }
  7208. static int raid5_resize(struct mddev *mddev, sector_t sectors)
  7209. {
  7210. /* no resync is happening, and there is enough space
  7211. * on all devices, so we can resize.
  7212. * We need to make sure resync covers any new space.
  7213. * If the array is shrinking we should possibly wait until
  7214. * any io in the removed space completes, but it hardly seems
  7215. * worth it.
  7216. */
  7217. sector_t newsize;
  7218. struct r5conf *conf = mddev->private;
  7219. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  7220. return -EINVAL;
  7221. sectors &= ~((sector_t)conf->chunk_sectors - 1);
  7222. newsize = raid5_size(mddev, sectors, mddev->raid_disks);
  7223. if (mddev->external_size &&
  7224. mddev->array_sectors > newsize)
  7225. return -EINVAL;
  7226. if (mddev->bitmap) {
  7227. int ret = md_bitmap_resize(mddev->bitmap, sectors, 0, 0);
  7228. if (ret)
  7229. return ret;
  7230. }
  7231. md_set_array_sectors(mddev, newsize);
  7232. if (sectors > mddev->dev_sectors &&
  7233. mddev->recovery_cp > mddev->dev_sectors) {
  7234. mddev->recovery_cp = mddev->dev_sectors;
  7235. set_bit(MD_RECOVERY_NEEDED, &mddev->recovery);
  7236. }
  7237. mddev->dev_sectors = sectors;
  7238. mddev->resync_max_sectors = sectors;
  7239. return 0;
  7240. }
  7241. static int check_stripe_cache(struct mddev *mddev)
  7242. {
  7243. /* Can only proceed if there are plenty of stripe_heads.
  7244. * We need a minimum of one full stripe,, and for sensible progress
  7245. * it is best to have about 4 times that.
  7246. * If we require 4 times, then the default 256 4K stripe_heads will
  7247. * allow for chunk sizes up to 256K, which is probably OK.
  7248. * If the chunk size is greater, user-space should request more
  7249. * stripe_heads first.
  7250. */
  7251. struct r5conf *conf = mddev->private;
  7252. if (((mddev->chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
  7253. > conf->min_nr_stripes ||
  7254. ((mddev->new_chunk_sectors << 9) / RAID5_STRIPE_SIZE(conf)) * 4
  7255. > conf->min_nr_stripes) {
  7256. pr_warn("md/raid:%s: reshape: not enough stripes. Needed %lu\n",
  7257. mdname(mddev),
  7258. ((max(mddev->chunk_sectors, mddev->new_chunk_sectors) << 9)
  7259. / RAID5_STRIPE_SIZE(conf))*4);
  7260. return 0;
  7261. }
  7262. return 1;
  7263. }
  7264. static int check_reshape(struct mddev *mddev)
  7265. {
  7266. struct r5conf *conf = mddev->private;
  7267. if (raid5_has_log(conf) || raid5_has_ppl(conf))
  7268. return -EINVAL;
  7269. if (mddev->delta_disks == 0 &&
  7270. mddev->new_layout == mddev->layout &&
  7271. mddev->new_chunk_sectors == mddev->chunk_sectors)
  7272. return 0; /* nothing to do */
  7273. if (has_failed(conf))
  7274. return -EINVAL;
  7275. if (mddev->delta_disks < 0 && mddev->reshape_position == MaxSector) {
  7276. /* We might be able to shrink, but the devices must
  7277. * be made bigger first.
  7278. * For raid6, 4 is the minimum size.
  7279. * Otherwise 2 is the minimum
  7280. */
  7281. int min = 2;
  7282. if (mddev->level == 6)
  7283. min = 4;
  7284. if (mddev->raid_disks + mddev->delta_disks < min)
  7285. return -EINVAL;
  7286. }
  7287. if (!check_stripe_cache(mddev))
  7288. return -ENOSPC;
  7289. if (mddev->new_chunk_sectors > mddev->chunk_sectors ||
  7290. mddev->delta_disks > 0)
  7291. if (resize_chunks(conf,
  7292. conf->previous_raid_disks
  7293. + max(0, mddev->delta_disks),
  7294. max(mddev->new_chunk_sectors,
  7295. mddev->chunk_sectors)
  7296. ) < 0)
  7297. return -ENOMEM;
  7298. if (conf->previous_raid_disks + mddev->delta_disks <= conf->pool_size)
  7299. return 0; /* never bother to shrink */
  7300. return resize_stripes(conf, (conf->previous_raid_disks
  7301. + mddev->delta_disks));
  7302. }
  7303. static int raid5_start_reshape(struct mddev *mddev)
  7304. {
  7305. struct r5conf *conf = mddev->private;
  7306. struct md_rdev *rdev;
  7307. int spares = 0;
  7308. unsigned long flags;
  7309. if (test_bit(MD_RECOVERY_RUNNING, &mddev->recovery))
  7310. return -EBUSY;
  7311. if (!check_stripe_cache(mddev))
  7312. return -ENOSPC;
  7313. if (has_failed(conf))
  7314. return -EINVAL;
  7315. rdev_for_each(rdev, mddev) {
  7316. if (!test_bit(In_sync, &rdev->flags)
  7317. && !test_bit(Faulty, &rdev->flags))
  7318. spares++;
  7319. }
  7320. if (spares - mddev->degraded < mddev->delta_disks - conf->max_degraded)
  7321. /* Not enough devices even to make a degraded array
  7322. * of that size
  7323. */
  7324. return -EINVAL;
  7325. /* Refuse to reduce size of the array. Any reductions in
  7326. * array size must be through explicit setting of array_size
  7327. * attribute.
  7328. */
  7329. if (raid5_size(mddev, 0, conf->raid_disks + mddev->delta_disks)
  7330. < mddev->array_sectors) {
  7331. pr_warn("md/raid:%s: array size must be reduced before number of disks\n",
  7332. mdname(mddev));
  7333. return -EINVAL;
  7334. }
  7335. atomic_set(&conf->reshape_stripes, 0);
  7336. spin_lock_irq(&conf->device_lock);
  7337. write_seqcount_begin(&conf->gen_lock);
  7338. conf->previous_raid_disks = conf->raid_disks;
  7339. conf->raid_disks += mddev->delta_disks;
  7340. conf->prev_chunk_sectors = conf->chunk_sectors;
  7341. conf->chunk_sectors = mddev->new_chunk_sectors;
  7342. conf->prev_algo = conf->algorithm;
  7343. conf->algorithm = mddev->new_layout;
  7344. conf->generation++;
  7345. /* Code that selects data_offset needs to see the generation update
  7346. * if reshape_progress has been set - so a memory barrier needed.
  7347. */
  7348. smp_mb();
  7349. if (mddev->reshape_backwards)
  7350. conf->reshape_progress = raid5_size(mddev, 0, 0);
  7351. else
  7352. conf->reshape_progress = 0;
  7353. conf->reshape_safe = conf->reshape_progress;
  7354. write_seqcount_end(&conf->gen_lock);
  7355. spin_unlock_irq(&conf->device_lock);
  7356. /* Now make sure any requests that proceeded on the assumption
  7357. * the reshape wasn't running - like Discard or Read - have
  7358. * completed.
  7359. */
  7360. mddev_suspend(mddev);
  7361. mddev_resume(mddev);
  7362. /* Add some new drives, as many as will fit.
  7363. * We know there are enough to make the newly sized array work.
  7364. * Don't add devices if we are reducing the number of
  7365. * devices in the array. This is because it is not possible
  7366. * to correctly record the "partially reconstructed" state of
  7367. * such devices during the reshape and confusion could result.
  7368. */
  7369. if (mddev->delta_disks >= 0) {
  7370. rdev_for_each(rdev, mddev)
  7371. if (rdev->raid_disk < 0 &&
  7372. !test_bit(Faulty, &rdev->flags)) {
  7373. if (raid5_add_disk(mddev, rdev) == 0) {
  7374. if (rdev->raid_disk
  7375. >= conf->previous_raid_disks)
  7376. set_bit(In_sync, &rdev->flags);
  7377. else
  7378. rdev->recovery_offset = 0;
  7379. /* Failure here is OK */
  7380. sysfs_link_rdev(mddev, rdev);
  7381. }
  7382. } else if (rdev->raid_disk >= conf->previous_raid_disks
  7383. && !test_bit(Faulty, &rdev->flags)) {
  7384. /* This is a spare that was manually added */
  7385. set_bit(In_sync, &rdev->flags);
  7386. }
  7387. /* When a reshape changes the number of devices,
  7388. * ->degraded is measured against the larger of the
  7389. * pre and post number of devices.
  7390. */
  7391. spin_lock_irqsave(&conf->device_lock, flags);
  7392. mddev->degraded = raid5_calc_degraded(conf);
  7393. spin_unlock_irqrestore(&conf->device_lock, flags);
  7394. }
  7395. mddev->raid_disks = conf->raid_disks;
  7396. mddev->reshape_position = conf->reshape_progress;
  7397. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  7398. clear_bit(MD_RECOVERY_SYNC, &mddev->recovery);
  7399. clear_bit(MD_RECOVERY_CHECK, &mddev->recovery);
  7400. clear_bit(MD_RECOVERY_DONE, &mddev->recovery);
  7401. set_bit(MD_RECOVERY_RESHAPE, &mddev->recovery);
  7402. set_bit(MD_RECOVERY_RUNNING, &mddev->recovery);
  7403. mddev->sync_thread = md_register_thread(md_do_sync, mddev,
  7404. "reshape");
  7405. if (!mddev->sync_thread) {
  7406. mddev->recovery = 0;
  7407. spin_lock_irq(&conf->device_lock);
  7408. write_seqcount_begin(&conf->gen_lock);
  7409. mddev->raid_disks = conf->raid_disks = conf->previous_raid_disks;
  7410. mddev->new_chunk_sectors =
  7411. conf->chunk_sectors = conf->prev_chunk_sectors;
  7412. mddev->new_layout = conf->algorithm = conf->prev_algo;
  7413. rdev_for_each(rdev, mddev)
  7414. rdev->new_data_offset = rdev->data_offset;
  7415. smp_wmb();
  7416. conf->generation --;
  7417. conf->reshape_progress = MaxSector;
  7418. mddev->reshape_position = MaxSector;
  7419. write_seqcount_end(&conf->gen_lock);
  7420. spin_unlock_irq(&conf->device_lock);
  7421. return -EAGAIN;
  7422. }
  7423. conf->reshape_checkpoint = jiffies;
  7424. md_wakeup_thread(mddev->sync_thread);
  7425. md_new_event(mddev);
  7426. return 0;
  7427. }
  7428. /* This is called from the reshape thread and should make any
  7429. * changes needed in 'conf'
  7430. */
  7431. static void end_reshape(struct r5conf *conf)
  7432. {
  7433. if (!test_bit(MD_RECOVERY_INTR, &conf->mddev->recovery)) {
  7434. struct md_rdev *rdev;
  7435. spin_lock_irq(&conf->device_lock);
  7436. conf->previous_raid_disks = conf->raid_disks;
  7437. md_finish_reshape(conf->mddev);
  7438. smp_wmb();
  7439. conf->reshape_progress = MaxSector;
  7440. conf->mddev->reshape_position = MaxSector;
  7441. rdev_for_each(rdev, conf->mddev)
  7442. if (rdev->raid_disk >= 0 &&
  7443. !test_bit(Journal, &rdev->flags) &&
  7444. !test_bit(In_sync, &rdev->flags))
  7445. rdev->recovery_offset = MaxSector;
  7446. spin_unlock_irq(&conf->device_lock);
  7447. wake_up(&conf->wait_for_overlap);
  7448. if (conf->mddev->queue)
  7449. raid5_set_io_opt(conf);
  7450. }
  7451. }
  7452. /* This is called from the raid5d thread with mddev_lock held.
  7453. * It makes config changes to the device.
  7454. */
  7455. static void raid5_finish_reshape(struct mddev *mddev)
  7456. {
  7457. struct r5conf *conf = mddev->private;
  7458. if (!test_bit(MD_RECOVERY_INTR, &mddev->recovery)) {
  7459. if (mddev->delta_disks <= 0) {
  7460. int d;
  7461. spin_lock_irq(&conf->device_lock);
  7462. mddev->degraded = raid5_calc_degraded(conf);
  7463. spin_unlock_irq(&conf->device_lock);
  7464. for (d = conf->raid_disks ;
  7465. d < conf->raid_disks - mddev->delta_disks;
  7466. d++) {
  7467. struct md_rdev *rdev = conf->disks[d].rdev;
  7468. if (rdev)
  7469. clear_bit(In_sync, &rdev->flags);
  7470. rdev = conf->disks[d].replacement;
  7471. if (rdev)
  7472. clear_bit(In_sync, &rdev->flags);
  7473. }
  7474. }
  7475. mddev->layout = conf->algorithm;
  7476. mddev->chunk_sectors = conf->chunk_sectors;
  7477. mddev->reshape_position = MaxSector;
  7478. mddev->delta_disks = 0;
  7479. mddev->reshape_backwards = 0;
  7480. }
  7481. }
  7482. static void raid5_quiesce(struct mddev *mddev, int quiesce)
  7483. {
  7484. struct r5conf *conf = mddev->private;
  7485. if (quiesce) {
  7486. /* stop all writes */
  7487. lock_all_device_hash_locks_irq(conf);
  7488. /* '2' tells resync/reshape to pause so that all
  7489. * active stripes can drain
  7490. */
  7491. r5c_flush_cache(conf, INT_MAX);
  7492. conf->quiesce = 2;
  7493. wait_event_cmd(conf->wait_for_quiescent,
  7494. atomic_read(&conf->active_stripes) == 0 &&
  7495. atomic_read(&conf->active_aligned_reads) == 0,
  7496. unlock_all_device_hash_locks_irq(conf),
  7497. lock_all_device_hash_locks_irq(conf));
  7498. conf->quiesce = 1;
  7499. unlock_all_device_hash_locks_irq(conf);
  7500. /* allow reshape to continue */
  7501. wake_up(&conf->wait_for_overlap);
  7502. } else {
  7503. /* re-enable writes */
  7504. lock_all_device_hash_locks_irq(conf);
  7505. conf->quiesce = 0;
  7506. wake_up(&conf->wait_for_quiescent);
  7507. wake_up(&conf->wait_for_overlap);
  7508. unlock_all_device_hash_locks_irq(conf);
  7509. }
  7510. log_quiesce(conf, quiesce);
  7511. }
  7512. static void *raid45_takeover_raid0(struct mddev *mddev, int level)
  7513. {
  7514. struct r0conf *raid0_conf = mddev->private;
  7515. sector_t sectors;
  7516. /* for raid0 takeover only one zone is supported */
  7517. if (raid0_conf->nr_strip_zones > 1) {
  7518. pr_warn("md/raid:%s: cannot takeover raid0 with more than one zone.\n",
  7519. mdname(mddev));
  7520. return ERR_PTR(-EINVAL);
  7521. }
  7522. sectors = raid0_conf->strip_zone[0].zone_end;
  7523. sector_div(sectors, raid0_conf->strip_zone[0].nb_dev);
  7524. mddev->dev_sectors = sectors;
  7525. mddev->new_level = level;
  7526. mddev->new_layout = ALGORITHM_PARITY_N;
  7527. mddev->new_chunk_sectors = mddev->chunk_sectors;
  7528. mddev->raid_disks += 1;
  7529. mddev->delta_disks = 1;
  7530. /* make sure it will be not marked as dirty */
  7531. mddev->recovery_cp = MaxSector;
  7532. return setup_conf(mddev);
  7533. }
  7534. static void *raid5_takeover_raid1(struct mddev *mddev)
  7535. {
  7536. int chunksect;
  7537. void *ret;
  7538. if (mddev->raid_disks != 2 ||
  7539. mddev->degraded > 1)
  7540. return ERR_PTR(-EINVAL);
  7541. /* Should check if there are write-behind devices? */
  7542. chunksect = 64*2; /* 64K by default */
  7543. /* The array must be an exact multiple of chunksize */
  7544. while (chunksect && (mddev->array_sectors & (chunksect-1)))
  7545. chunksect >>= 1;
  7546. if ((chunksect<<9) < RAID5_STRIPE_SIZE((struct r5conf *)mddev->private))
  7547. /* array size does not allow a suitable chunk size */
  7548. return ERR_PTR(-EINVAL);
  7549. mddev->new_level = 5;
  7550. mddev->new_layout = ALGORITHM_LEFT_SYMMETRIC;
  7551. mddev->new_chunk_sectors = chunksect;
  7552. ret = setup_conf(mddev);
  7553. if (!IS_ERR(ret))
  7554. mddev_clear_unsupported_flags(mddev,
  7555. UNSUPPORTED_MDDEV_FLAGS);
  7556. return ret;
  7557. }
  7558. static void *raid5_takeover_raid6(struct mddev *mddev)
  7559. {
  7560. int new_layout;
  7561. switch (mddev->layout) {
  7562. case ALGORITHM_LEFT_ASYMMETRIC_6:
  7563. new_layout = ALGORITHM_LEFT_ASYMMETRIC;
  7564. break;
  7565. case ALGORITHM_RIGHT_ASYMMETRIC_6:
  7566. new_layout = ALGORITHM_RIGHT_ASYMMETRIC;
  7567. break;
  7568. case ALGORITHM_LEFT_SYMMETRIC_6:
  7569. new_layout = ALGORITHM_LEFT_SYMMETRIC;
  7570. break;
  7571. case ALGORITHM_RIGHT_SYMMETRIC_6:
  7572. new_layout = ALGORITHM_RIGHT_SYMMETRIC;
  7573. break;
  7574. case ALGORITHM_PARITY_0_6:
  7575. new_layout = ALGORITHM_PARITY_0;
  7576. break;
  7577. case ALGORITHM_PARITY_N:
  7578. new_layout = ALGORITHM_PARITY_N;
  7579. break;
  7580. default:
  7581. return ERR_PTR(-EINVAL);
  7582. }
  7583. mddev->new_level = 5;
  7584. mddev->new_layout = new_layout;
  7585. mddev->delta_disks = -1;
  7586. mddev->raid_disks -= 1;
  7587. return setup_conf(mddev);
  7588. }
  7589. static int raid5_check_reshape(struct mddev *mddev)
  7590. {
  7591. /* For a 2-drive array, the layout and chunk size can be changed
  7592. * immediately as not restriping is needed.
  7593. * For larger arrays we record the new value - after validation
  7594. * to be used by a reshape pass.
  7595. */
  7596. struct r5conf *conf = mddev->private;
  7597. int new_chunk = mddev->new_chunk_sectors;
  7598. if (mddev->new_layout >= 0 && !algorithm_valid_raid5(mddev->new_layout))
  7599. return -EINVAL;
  7600. if (new_chunk > 0) {
  7601. if (!is_power_of_2(new_chunk))
  7602. return -EINVAL;
  7603. if (new_chunk < (PAGE_SIZE>>9))
  7604. return -EINVAL;
  7605. if (mddev->array_sectors & (new_chunk-1))
  7606. /* not factor of array size */
  7607. return -EINVAL;
  7608. }
  7609. /* They look valid */
  7610. if (mddev->raid_disks == 2) {
  7611. /* can make the change immediately */
  7612. if (mddev->new_layout >= 0) {
  7613. conf->algorithm = mddev->new_layout;
  7614. mddev->layout = mddev->new_layout;
  7615. }
  7616. if (new_chunk > 0) {
  7617. conf->chunk_sectors = new_chunk ;
  7618. mddev->chunk_sectors = new_chunk;
  7619. }
  7620. set_bit(MD_SB_CHANGE_DEVS, &mddev->sb_flags);
  7621. md_wakeup_thread(mddev->thread);
  7622. }
  7623. return check_reshape(mddev);
  7624. }
  7625. static int raid6_check_reshape(struct mddev *mddev)
  7626. {
  7627. int new_chunk = mddev->new_chunk_sectors;
  7628. if (mddev->new_layout >= 0 && !algorithm_valid_raid6(mddev->new_layout))
  7629. return -EINVAL;
  7630. if (new_chunk > 0) {
  7631. if (!is_power_of_2(new_chunk))
  7632. return -EINVAL;
  7633. if (new_chunk < (PAGE_SIZE >> 9))
  7634. return -EINVAL;
  7635. if (mddev->array_sectors & (new_chunk-1))
  7636. /* not factor of array size */
  7637. return -EINVAL;
  7638. }
  7639. /* They look valid */
  7640. return check_reshape(mddev);
  7641. }
  7642. static void *raid5_takeover(struct mddev *mddev)
  7643. {
  7644. /* raid5 can take over:
  7645. * raid0 - if there is only one strip zone - make it a raid4 layout
  7646. * raid1 - if there are two drives. We need to know the chunk size
  7647. * raid4 - trivial - just use a raid4 layout.
  7648. * raid6 - Providing it is a *_6 layout
  7649. */
  7650. if (mddev->level == 0)
  7651. return raid45_takeover_raid0(mddev, 5);
  7652. if (mddev->level == 1)
  7653. return raid5_takeover_raid1(mddev);
  7654. if (mddev->level == 4) {
  7655. mddev->new_layout = ALGORITHM_PARITY_N;
  7656. mddev->new_level = 5;
  7657. return setup_conf(mddev);
  7658. }
  7659. if (mddev->level == 6)
  7660. return raid5_takeover_raid6(mddev);
  7661. return ERR_PTR(-EINVAL);
  7662. }
  7663. static void *raid4_takeover(struct mddev *mddev)
  7664. {
  7665. /* raid4 can take over:
  7666. * raid0 - if there is only one strip zone
  7667. * raid5 - if layout is right
  7668. */
  7669. if (mddev->level == 0)
  7670. return raid45_takeover_raid0(mddev, 4);
  7671. if (mddev->level == 5 &&
  7672. mddev->layout == ALGORITHM_PARITY_N) {
  7673. mddev->new_layout = 0;
  7674. mddev->new_level = 4;
  7675. return setup_conf(mddev);
  7676. }
  7677. return ERR_PTR(-EINVAL);
  7678. }
  7679. static struct md_personality raid5_personality;
  7680. static void *raid6_takeover(struct mddev *mddev)
  7681. {
  7682. /* Currently can only take over a raid5. We map the
  7683. * personality to an equivalent raid6 personality
  7684. * with the Q block at the end.
  7685. */
  7686. int new_layout;
  7687. if (mddev->pers != &raid5_personality)
  7688. return ERR_PTR(-EINVAL);
  7689. if (mddev->degraded > 1)
  7690. return ERR_PTR(-EINVAL);
  7691. if (mddev->raid_disks > 253)
  7692. return ERR_PTR(-EINVAL);
  7693. if (mddev->raid_disks < 3)
  7694. return ERR_PTR(-EINVAL);
  7695. switch (mddev->layout) {
  7696. case ALGORITHM_LEFT_ASYMMETRIC:
  7697. new_layout = ALGORITHM_LEFT_ASYMMETRIC_6;
  7698. break;
  7699. case ALGORITHM_RIGHT_ASYMMETRIC:
  7700. new_layout = ALGORITHM_RIGHT_ASYMMETRIC_6;
  7701. break;
  7702. case ALGORITHM_LEFT_SYMMETRIC:
  7703. new_layout = ALGORITHM_LEFT_SYMMETRIC_6;
  7704. break;
  7705. case ALGORITHM_RIGHT_SYMMETRIC:
  7706. new_layout = ALGORITHM_RIGHT_SYMMETRIC_6;
  7707. break;
  7708. case ALGORITHM_PARITY_0:
  7709. new_layout = ALGORITHM_PARITY_0_6;
  7710. break;
  7711. case ALGORITHM_PARITY_N:
  7712. new_layout = ALGORITHM_PARITY_N;
  7713. break;
  7714. default:
  7715. return ERR_PTR(-EINVAL);
  7716. }
  7717. mddev->new_level = 6;
  7718. mddev->new_layout = new_layout;
  7719. mddev->delta_disks = 1;
  7720. mddev->raid_disks += 1;
  7721. return setup_conf(mddev);
  7722. }
  7723. static int raid5_change_consistency_policy(struct mddev *mddev, const char *buf)
  7724. {
  7725. struct r5conf *conf;
  7726. int err;
  7727. err = mddev_lock(mddev);
  7728. if (err)
  7729. return err;
  7730. conf = mddev->private;
  7731. if (!conf) {
  7732. mddev_unlock(mddev);
  7733. return -ENODEV;
  7734. }
  7735. if (strncmp(buf, "ppl", 3) == 0) {
  7736. /* ppl only works with RAID 5 */
  7737. if (!raid5_has_ppl(conf) && conf->level == 5) {
  7738. err = log_init(conf, NULL, true);
  7739. if (!err) {
  7740. err = resize_stripes(conf, conf->pool_size);
  7741. if (err)
  7742. log_exit(conf);
  7743. }
  7744. } else
  7745. err = -EINVAL;
  7746. } else if (strncmp(buf, "resync", 6) == 0) {
  7747. if (raid5_has_ppl(conf)) {
  7748. mddev_suspend(mddev);
  7749. log_exit(conf);
  7750. mddev_resume(mddev);
  7751. err = resize_stripes(conf, conf->pool_size);
  7752. } else if (test_bit(MD_HAS_JOURNAL, &conf->mddev->flags) &&
  7753. r5l_log_disk_error(conf)) {
  7754. bool journal_dev_exists = false;
  7755. struct md_rdev *rdev;
  7756. rdev_for_each(rdev, mddev)
  7757. if (test_bit(Journal, &rdev->flags)) {
  7758. journal_dev_exists = true;
  7759. break;
  7760. }
  7761. if (!journal_dev_exists) {
  7762. mddev_suspend(mddev);
  7763. clear_bit(MD_HAS_JOURNAL, &mddev->flags);
  7764. mddev_resume(mddev);
  7765. } else /* need remove journal device first */
  7766. err = -EBUSY;
  7767. } else
  7768. err = -EINVAL;
  7769. } else {
  7770. err = -EINVAL;
  7771. }
  7772. if (!err)
  7773. md_update_sb(mddev, 1);
  7774. mddev_unlock(mddev);
  7775. return err;
  7776. }
  7777. static int raid5_start(struct mddev *mddev)
  7778. {
  7779. struct r5conf *conf = mddev->private;
  7780. return r5l_start(conf->log);
  7781. }
  7782. static struct md_personality raid6_personality =
  7783. {
  7784. .name = "raid6",
  7785. .level = 6,
  7786. .owner = THIS_MODULE,
  7787. .make_request = raid5_make_request,
  7788. .run = raid5_run,
  7789. .start = raid5_start,
  7790. .free = raid5_free,
  7791. .status = raid5_status,
  7792. .error_handler = raid5_error,
  7793. .hot_add_disk = raid5_add_disk,
  7794. .hot_remove_disk= raid5_remove_disk,
  7795. .spare_active = raid5_spare_active,
  7796. .sync_request = raid5_sync_request,
  7797. .resize = raid5_resize,
  7798. .size = raid5_size,
  7799. .check_reshape = raid6_check_reshape,
  7800. .start_reshape = raid5_start_reshape,
  7801. .finish_reshape = raid5_finish_reshape,
  7802. .quiesce = raid5_quiesce,
  7803. .takeover = raid6_takeover,
  7804. .change_consistency_policy = raid5_change_consistency_policy,
  7805. };
  7806. static struct md_personality raid5_personality =
  7807. {
  7808. .name = "raid5",
  7809. .level = 5,
  7810. .owner = THIS_MODULE,
  7811. .make_request = raid5_make_request,
  7812. .run = raid5_run,
  7813. .start = raid5_start,
  7814. .free = raid5_free,
  7815. .status = raid5_status,
  7816. .error_handler = raid5_error,
  7817. .hot_add_disk = raid5_add_disk,
  7818. .hot_remove_disk= raid5_remove_disk,
  7819. .spare_active = raid5_spare_active,
  7820. .sync_request = raid5_sync_request,
  7821. .resize = raid5_resize,
  7822. .size = raid5_size,
  7823. .check_reshape = raid5_check_reshape,
  7824. .start_reshape = raid5_start_reshape,
  7825. .finish_reshape = raid5_finish_reshape,
  7826. .quiesce = raid5_quiesce,
  7827. .takeover = raid5_takeover,
  7828. .change_consistency_policy = raid5_change_consistency_policy,
  7829. };
  7830. static struct md_personality raid4_personality =
  7831. {
  7832. .name = "raid4",
  7833. .level = 4,
  7834. .owner = THIS_MODULE,
  7835. .make_request = raid5_make_request,
  7836. .run = raid5_run,
  7837. .start = raid5_start,
  7838. .free = raid5_free,
  7839. .status = raid5_status,
  7840. .error_handler = raid5_error,
  7841. .hot_add_disk = raid5_add_disk,
  7842. .hot_remove_disk= raid5_remove_disk,
  7843. .spare_active = raid5_spare_active,
  7844. .sync_request = raid5_sync_request,
  7845. .resize = raid5_resize,
  7846. .size = raid5_size,
  7847. .check_reshape = raid5_check_reshape,
  7848. .start_reshape = raid5_start_reshape,
  7849. .finish_reshape = raid5_finish_reshape,
  7850. .quiesce = raid5_quiesce,
  7851. .takeover = raid4_takeover,
  7852. .change_consistency_policy = raid5_change_consistency_policy,
  7853. };
  7854. static int __init raid5_init(void)
  7855. {
  7856. int ret;
  7857. raid5_wq = alloc_workqueue("raid5wq",
  7858. WQ_UNBOUND|WQ_MEM_RECLAIM|WQ_CPU_INTENSIVE|WQ_SYSFS, 0);
  7859. if (!raid5_wq)
  7860. return -ENOMEM;
  7861. ret = cpuhp_setup_state_multi(CPUHP_MD_RAID5_PREPARE,
  7862. "md/raid5:prepare",
  7863. raid456_cpu_up_prepare,
  7864. raid456_cpu_dead);
  7865. if (ret) {
  7866. destroy_workqueue(raid5_wq);
  7867. return ret;
  7868. }
  7869. register_md_personality(&raid6_personality);
  7870. register_md_personality(&raid5_personality);
  7871. register_md_personality(&raid4_personality);
  7872. return 0;
  7873. }
  7874. static void raid5_exit(void)
  7875. {
  7876. unregister_md_personality(&raid6_personality);
  7877. unregister_md_personality(&raid5_personality);
  7878. unregister_md_personality(&raid4_personality);
  7879. cpuhp_remove_multi_state(CPUHP_MD_RAID5_PREPARE);
  7880. destroy_workqueue(raid5_wq);
  7881. }
  7882. module_init(raid5_init);
  7883. module_exit(raid5_exit);
  7884. MODULE_LICENSE("GPL");
  7885. MODULE_DESCRIPTION("RAID4/5/6 (striping with parity) personality for MD");
  7886. MODULE_ALIAS("md-personality-4"); /* RAID5 */
  7887. MODULE_ALIAS("md-raid5");
  7888. MODULE_ALIAS("md-raid4");
  7889. MODULE_ALIAS("md-level-5");
  7890. MODULE_ALIAS("md-level-4");
  7891. MODULE_ALIAS("md-personality-8"); /* RAID6 */
  7892. MODULE_ALIAS("md-raid6");
  7893. MODULE_ALIAS("md-level-6");
  7894. /* This used to be two separate modules, they were: */
  7895. MODULE_ALIAS("raid5");
  7896. MODULE_ALIAS("raid6");