inode.c 291 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362536353645365536653675368536953705371537253735374537553765377537853795380538153825383538453855386538753885389539053915392539353945395539653975398539954005401540254035404540554065407540854095410541154125413541454155416541754185419542054215422542354245425542654275428542954305431543254335434543554365437543854395440544154425443544454455446544754485449545054515452545354545455545654575458545954605461546254635464546554665467546854695470547154725473547454755476547754785479548054815482548354845485548654875488548954905491549254935494549554965497549854995500550155025503550455055506550755085509551055115512551355145515551655175518551955205521552255235524552555265527552855295530553155325533553455355536553755385539554055415542554355445545554655475548554955505551555255535554555555565557555855595560556155625563556455655566556755685569557055715572557355745575557655775578557955805581558255835584558555865587558855895590559155925593559455955596559755985599560056015602560356045605560656075608560956105611561256135614561556165617561856195620562156225623562456255626562756285629563056315632563356345635563656375638563956405641564256435644564556465647564856495650565156525653565456555656565756585659566056615662566356645665566656675668566956705671567256735674567556765677567856795680568156825683568456855686568756885689569056915692569356945695569656975698569957005701570257035704570557065707570857095710571157125713571457155716571757185719572057215722572357245725572657275728572957305731573257335734573557365737573857395740574157425743574457455746574757485749575057515752575357545755575657575758575957605761576257635764576557665767576857695770577157725773577457755776577757785779578057815782578357845785578657875788578957905791579257935794579557965797579857995800580158025803580458055806580758085809581058115812581358145815581658175818581958205821582258235824582558265827582858295830583158325833583458355836583758385839584058415842584358445845584658475848584958505851585258535854585558565857585858595860586158625863586458655866586758685869587058715872587358745875587658775878587958805881588258835884588558865887588858895890589158925893589458955896589758985899590059015902590359045905590659075908590959105911591259135914591559165917591859195920592159225923592459255926592759285929593059315932593359345935593659375938593959405941594259435944594559465947594859495950595159525953595459555956595759585959596059615962596359645965596659675968596959705971597259735974597559765977597859795980598159825983598459855986598759885989599059915992599359945995599659975998599960006001600260036004600560066007600860096010601160126013601460156016601760186019602060216022602360246025602660276028602960306031603260336034603560366037603860396040604160426043604460456046604760486049605060516052605360546055605660576058605960606061606260636064606560666067606860696070607160726073607460756076607760786079608060816082608360846085608660876088608960906091609260936094609560966097609860996100610161026103610461056106610761086109611061116112611361146115611661176118611961206121612261236124612561266127612861296130613161326133613461356136613761386139614061416142614361446145614661476148614961506151615261536154615561566157615861596160616161626163616461656166616761686169617061716172617361746175617661776178617961806181618261836184618561866187618861896190619161926193619461956196619761986199620062016202620362046205620662076208620962106211621262136214621562166217621862196220622162226223622462256226622762286229623062316232623362346235623662376238623962406241624262436244624562466247624862496250625162526253625462556256625762586259626062616262626362646265626662676268626962706271627262736274627562766277627862796280628162826283628462856286628762886289629062916292629362946295629662976298629963006301630263036304630563066307630863096310631163126313631463156316631763186319632063216322632363246325632663276328632963306331633263336334633563366337633863396340634163426343634463456346634763486349635063516352635363546355635663576358635963606361636263636364636563666367636863696370637163726373637463756376637763786379638063816382638363846385638663876388638963906391639263936394639563966397639863996400640164026403640464056406640764086409641064116412641364146415641664176418641964206421642264236424642564266427642864296430643164326433643464356436643764386439644064416442644364446445644664476448644964506451645264536454645564566457645864596460646164626463646464656466646764686469647064716472647364746475647664776478647964806481648264836484648564866487648864896490649164926493649464956496649764986499650065016502650365046505650665076508650965106511651265136514651565166517651865196520652165226523652465256526652765286529653065316532653365346535653665376538653965406541654265436544654565466547654865496550655165526553655465556556655765586559656065616562656365646565656665676568656965706571657265736574657565766577657865796580658165826583658465856586658765886589659065916592659365946595659665976598659966006601660266036604660566066607660866096610661166126613661466156616661766186619662066216622662366246625662666276628662966306631663266336634663566366637663866396640664166426643664466456646664766486649665066516652665366546655665666576658665966606661666266636664666566666667666866696670667166726673667466756676667766786679668066816682668366846685668666876688668966906691669266936694669566966697669866996700670167026703670467056706670767086709671067116712671367146715671667176718671967206721672267236724672567266727672867296730673167326733673467356736673767386739674067416742674367446745674667476748674967506751675267536754675567566757675867596760676167626763676467656766676767686769677067716772677367746775677667776778677967806781678267836784678567866787678867896790679167926793679467956796679767986799680068016802680368046805680668076808680968106811681268136814681568166817681868196820682168226823682468256826682768286829683068316832683368346835683668376838683968406841684268436844684568466847684868496850685168526853685468556856685768586859686068616862686368646865686668676868686968706871687268736874687568766877687868796880688168826883688468856886688768886889689068916892689368946895689668976898689969006901690269036904690569066907690869096910691169126913691469156916691769186919692069216922692369246925692669276928692969306931693269336934693569366937693869396940694169426943694469456946694769486949695069516952695369546955695669576958695969606961696269636964696569666967696869696970697169726973697469756976697769786979698069816982698369846985698669876988698969906991699269936994699569966997699869997000700170027003700470057006700770087009701070117012701370147015701670177018701970207021702270237024702570267027702870297030703170327033703470357036703770387039704070417042704370447045704670477048704970507051705270537054705570567057705870597060706170627063706470657066706770687069707070717072707370747075707670777078707970807081708270837084708570867087708870897090709170927093709470957096709770987099710071017102710371047105710671077108710971107111711271137114711571167117711871197120712171227123712471257126712771287129713071317132713371347135713671377138713971407141714271437144714571467147714871497150715171527153715471557156715771587159716071617162716371647165716671677168716971707171717271737174717571767177717871797180718171827183718471857186718771887189719071917192719371947195719671977198719972007201720272037204720572067207720872097210721172127213721472157216721772187219722072217222722372247225722672277228722972307231723272337234723572367237723872397240724172427243724472457246724772487249725072517252725372547255725672577258725972607261726272637264726572667267726872697270727172727273727472757276727772787279728072817282728372847285728672877288728972907291729272937294729572967297729872997300730173027303730473057306730773087309731073117312731373147315731673177318731973207321732273237324732573267327732873297330733173327333733473357336733773387339734073417342734373447345734673477348734973507351735273537354735573567357735873597360736173627363736473657366736773687369737073717372737373747375737673777378737973807381738273837384738573867387738873897390739173927393739473957396739773987399740074017402740374047405740674077408740974107411741274137414741574167417741874197420742174227423742474257426742774287429743074317432743374347435743674377438743974407441744274437444744574467447744874497450745174527453745474557456745774587459746074617462746374647465746674677468746974707471747274737474747574767477747874797480748174827483748474857486748774887489749074917492749374947495749674977498749975007501750275037504750575067507750875097510751175127513751475157516751775187519752075217522752375247525752675277528752975307531753275337534753575367537753875397540754175427543754475457546754775487549755075517552755375547555755675577558755975607561756275637564756575667567756875697570757175727573757475757576757775787579758075817582758375847585758675877588758975907591759275937594759575967597759875997600760176027603760476057606760776087609761076117612761376147615761676177618761976207621762276237624762576267627762876297630763176327633763476357636763776387639764076417642764376447645764676477648764976507651765276537654765576567657765876597660766176627663766476657666766776687669767076717672767376747675767676777678767976807681768276837684768576867687768876897690769176927693769476957696769776987699770077017702770377047705770677077708770977107711771277137714771577167717771877197720772177227723772477257726772777287729773077317732773377347735773677377738773977407741774277437744774577467747774877497750775177527753775477557756775777587759776077617762776377647765776677677768776977707771777277737774777577767777777877797780778177827783778477857786778777887789779077917792779377947795779677977798779978007801780278037804780578067807780878097810781178127813781478157816781778187819782078217822782378247825782678277828782978307831783278337834783578367837783878397840784178427843784478457846784778487849785078517852785378547855785678577858785978607861786278637864786578667867786878697870787178727873787478757876787778787879788078817882788378847885788678877888788978907891789278937894789578967897789878997900790179027903790479057906790779087909791079117912791379147915791679177918791979207921792279237924792579267927792879297930793179327933793479357936793779387939794079417942794379447945794679477948794979507951795279537954795579567957795879597960796179627963796479657966796779687969797079717972797379747975797679777978797979807981798279837984798579867987798879897990799179927993799479957996799779987999800080018002800380048005800680078008800980108011801280138014801580168017801880198020802180228023802480258026802780288029803080318032803380348035803680378038803980408041804280438044804580468047804880498050805180528053805480558056805780588059806080618062806380648065806680678068806980708071807280738074807580768077807880798080808180828083808480858086808780888089809080918092809380948095809680978098809981008101810281038104810581068107810881098110811181128113811481158116811781188119812081218122812381248125812681278128812981308131813281338134813581368137813881398140814181428143814481458146814781488149815081518152815381548155815681578158815981608161816281638164816581668167816881698170817181728173817481758176817781788179818081818182818381848185818681878188818981908191819281938194819581968197819881998200820182028203820482058206820782088209821082118212821382148215821682178218821982208221822282238224822582268227822882298230823182328233823482358236823782388239824082418242824382448245824682478248824982508251825282538254825582568257825882598260826182628263826482658266826782688269827082718272827382748275827682778278827982808281828282838284828582868287828882898290829182928293829482958296829782988299830083018302830383048305830683078308830983108311831283138314831583168317831883198320832183228323832483258326832783288329833083318332833383348335833683378338833983408341834283438344834583468347834883498350835183528353835483558356835783588359836083618362836383648365836683678368836983708371837283738374837583768377837883798380838183828383838483858386838783888389839083918392839383948395839683978398839984008401840284038404840584068407840884098410841184128413841484158416841784188419842084218422842384248425842684278428842984308431843284338434843584368437843884398440844184428443844484458446844784488449845084518452845384548455845684578458845984608461846284638464846584668467846884698470847184728473847484758476847784788479848084818482848384848485848684878488848984908491849284938494849584968497849884998500850185028503850485058506850785088509851085118512851385148515851685178518851985208521852285238524852585268527852885298530853185328533853485358536853785388539854085418542854385448545854685478548854985508551855285538554855585568557855885598560856185628563856485658566856785688569857085718572857385748575857685778578857985808581858285838584858585868587858885898590859185928593859485958596859785988599860086018602860386048605860686078608860986108611861286138614861586168617861886198620862186228623862486258626862786288629863086318632863386348635863686378638863986408641864286438644864586468647864886498650865186528653865486558656865786588659866086618662866386648665866686678668866986708671867286738674867586768677867886798680868186828683868486858686868786888689869086918692869386948695869686978698869987008701870287038704870587068707870887098710871187128713871487158716871787188719872087218722872387248725872687278728872987308731873287338734873587368737873887398740874187428743874487458746874787488749875087518752875387548755875687578758875987608761876287638764876587668767876887698770877187728773877487758776877787788779878087818782878387848785878687878788878987908791879287938794879587968797879887998800880188028803880488058806880788088809881088118812881388148815881688178818881988208821882288238824882588268827882888298830883188328833883488358836883788388839884088418842884388448845884688478848884988508851885288538854885588568857885888598860886188628863886488658866886788688869887088718872887388748875887688778878887988808881888288838884888588868887888888898890889188928893889488958896889788988899890089018902890389048905890689078908890989108911891289138914891589168917891889198920892189228923892489258926892789288929893089318932893389348935893689378938893989408941894289438944894589468947894889498950895189528953895489558956895789588959896089618962896389648965896689678968896989708971897289738974897589768977897889798980898189828983898489858986898789888989899089918992899389948995899689978998899990009001900290039004900590069007900890099010901190129013901490159016901790189019902090219022902390249025902690279028902990309031903290339034903590369037903890399040904190429043904490459046904790489049905090519052905390549055905690579058905990609061906290639064906590669067906890699070907190729073907490759076907790789079908090819082908390849085908690879088908990909091909290939094909590969097909890999100910191029103910491059106910791089109911091119112911391149115911691179118911991209121912291239124912591269127912891299130913191329133913491359136913791389139914091419142914391449145914691479148914991509151915291539154915591569157915891599160916191629163916491659166916791689169917091719172917391749175917691779178917991809181918291839184918591869187918891899190919191929193919491959196919791989199920092019202920392049205920692079208920992109211921292139214921592169217921892199220922192229223922492259226922792289229923092319232923392349235923692379238923992409241924292439244924592469247924892499250925192529253925492559256925792589259926092619262926392649265926692679268926992709271927292739274927592769277927892799280928192829283928492859286928792889289929092919292929392949295929692979298929993009301930293039304930593069307930893099310931193129313931493159316931793189319932093219322932393249325932693279328932993309331933293339334933593369337933893399340934193429343934493459346934793489349935093519352935393549355935693579358935993609361936293639364936593669367936893699370937193729373937493759376937793789379938093819382938393849385938693879388938993909391939293939394939593969397939893999400940194029403940494059406940794089409941094119412941394149415941694179418941994209421942294239424942594269427942894299430943194329433943494359436943794389439944094419442944394449445944694479448944994509451945294539454945594569457945894599460946194629463946494659466946794689469947094719472947394749475947694779478947994809481948294839484948594869487948894899490949194929493949494959496949794989499950095019502950395049505950695079508950995109511951295139514951595169517951895199520952195229523952495259526952795289529953095319532953395349535953695379538953995409541954295439544954595469547954895499550955195529553955495559556955795589559956095619562956395649565956695679568956995709571957295739574957595769577957895799580958195829583958495859586958795889589959095919592959395949595959695979598959996009601960296039604960596069607960896099610961196129613961496159616961796189619962096219622962396249625962696279628962996309631963296339634963596369637963896399640964196429643964496459646964796489649965096519652965396549655965696579658965996609661966296639664966596669667966896699670967196729673967496759676967796789679968096819682968396849685968696879688968996909691969296939694969596969697969896999700970197029703970497059706970797089709971097119712971397149715971697179718971997209721972297239724972597269727972897299730973197329733973497359736973797389739974097419742974397449745974697479748974997509751975297539754975597569757975897599760976197629763976497659766976797689769977097719772977397749775977697779778977997809781978297839784978597869787978897899790979197929793979497959796979797989799980098019802980398049805980698079808980998109811981298139814981598169817981898199820982198229823982498259826982798289829983098319832983398349835983698379838983998409841984298439844984598469847984898499850985198529853985498559856985798589859986098619862986398649865986698679868986998709871987298739874987598769877987898799880988198829883988498859886988798889889989098919892989398949895989698979898989999009901990299039904990599069907990899099910991199129913991499159916991799189919992099219922992399249925992699279928992999309931993299339934993599369937993899399940994199429943994499459946994799489949995099519952995399549955995699579958995999609961996299639964996599669967996899699970997199729973997499759976997799789979998099819982998399849985998699879988998999909991999299939994999599969997999899991000010001100021000310004100051000610007100081000910010100111001210013100141001510016100171001810019100201002110022100231002410025100261002710028100291003010031100321003310034100351003610037100381003910040100411004210043100441004510046100471004810049100501005110052100531005410055100561005710058100591006010061100621006310064100651006610067100681006910070100711007210073100741007510076100771007810079100801008110082100831008410085100861008710088100891009010091100921009310094100951009610097100981009910100101011010210103101041010510106101071010810109101101011110112101131011410115101161011710118101191012010121101221012310124101251012610127101281012910130101311013210133101341013510136101371013810139101401014110142101431014410145101461014710148101491015010151101521015310154101551015610157101581015910160101611016210163101641016510166101671016810169101701017110172101731017410175101761017710178101791018010181101821018310184101851018610187101881018910190101911019210193101941019510196101971019810199102001020110202102031020410205102061020710208102091021010211102121021310214102151021610217102181021910220102211022210223102241022510226102271022810229102301023110232102331023410235102361023710238102391024010241102421024310244102451024610247102481024910250102511025210253102541025510256102571025810259102601026110262102631026410265102661026710268102691027010271102721027310274102751027610277102781027910280102811028210283102841028510286102871028810289102901029110292102931029410295102961029710298102991030010301103021030310304103051030610307103081030910310103111031210313103141031510316103171031810319103201032110322103231032410325103261032710328103291033010331103321033310334103351033610337103381033910340103411034210343103441034510346103471034810349103501035110352103531035410355103561035710358103591036010361103621036310364103651036610367103681036910370103711037210373103741037510376103771037810379103801038110382103831038410385103861038710388103891039010391103921039310394103951039610397103981039910400104011040210403104041040510406104071040810409104101041110412104131041410415104161041710418104191042010421104221042310424104251042610427104281042910430104311043210433104341043510436104371043810439104401044110442104431044410445104461044710448104491045010451104521045310454104551045610457104581045910460104611046210463104641046510466104671046810469104701047110472104731047410475104761047710478104791048010481104821048310484104851048610487104881048910490104911049210493104941049510496104971049810499105001050110502105031050410505
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2007 Oracle. All rights reserved.
  4. */
  5. #include <crypto/hash.h>
  6. #include <linux/kernel.h>
  7. #include <linux/bio.h>
  8. #include <linux/file.h>
  9. #include <linux/fs.h>
  10. #include <linux/pagemap.h>
  11. #include <linux/highmem.h>
  12. #include <linux/time.h>
  13. #include <linux/init.h>
  14. #include <linux/string.h>
  15. #include <linux/backing-dev.h>
  16. #include <linux/writeback.h>
  17. #include <linux/compat.h>
  18. #include <linux/xattr.h>
  19. #include <linux/posix_acl.h>
  20. #include <linux/falloc.h>
  21. #include <linux/slab.h>
  22. #include <linux/ratelimit.h>
  23. #include <linux/btrfs.h>
  24. #include <linux/blkdev.h>
  25. #include <linux/posix_acl_xattr.h>
  26. #include <linux/uio.h>
  27. #include <linux/magic.h>
  28. #include <linux/iversion.h>
  29. #include <linux/swap.h>
  30. #include <linux/migrate.h>
  31. #include <linux/sched/mm.h>
  32. #include <linux/iomap.h>
  33. #include <asm/unaligned.h>
  34. #include "misc.h"
  35. #include "ctree.h"
  36. #include "disk-io.h"
  37. #include "transaction.h"
  38. #include "btrfs_inode.h"
  39. #include "print-tree.h"
  40. #include "ordered-data.h"
  41. #include "xattr.h"
  42. #include "tree-log.h"
  43. #include "volumes.h"
  44. #include "compression.h"
  45. #include "locking.h"
  46. #include "free-space-cache.h"
  47. #include "inode-map.h"
  48. #include "props.h"
  49. #include "qgroup.h"
  50. #include "delalloc-space.h"
  51. #include "block-group.h"
  52. #include "space-info.h"
  53. struct btrfs_iget_args {
  54. u64 ino;
  55. struct btrfs_root *root;
  56. };
  57. struct btrfs_dio_data {
  58. u64 reserve;
  59. loff_t length;
  60. ssize_t submitted;
  61. struct extent_changeset *data_reserved;
  62. bool sync;
  63. };
  64. static const struct inode_operations btrfs_dir_inode_operations;
  65. static const struct inode_operations btrfs_symlink_inode_operations;
  66. static const struct inode_operations btrfs_special_inode_operations;
  67. static const struct inode_operations btrfs_file_inode_operations;
  68. static const struct address_space_operations btrfs_aops;
  69. static const struct file_operations btrfs_dir_file_operations;
  70. static struct kmem_cache *btrfs_inode_cachep;
  71. struct kmem_cache *btrfs_trans_handle_cachep;
  72. struct kmem_cache *btrfs_path_cachep;
  73. struct kmem_cache *btrfs_free_space_cachep;
  74. struct kmem_cache *btrfs_free_space_bitmap_cachep;
  75. static int btrfs_setsize(struct inode *inode, struct iattr *attr);
  76. static int btrfs_truncate(struct inode *inode, bool skip_writeback);
  77. static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent);
  78. static noinline int cow_file_range(struct btrfs_inode *inode,
  79. struct page *locked_page,
  80. u64 start, u64 end, int *page_started,
  81. unsigned long *nr_written, int unlock);
  82. static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start,
  83. u64 len, u64 orig_start, u64 block_start,
  84. u64 block_len, u64 orig_block_len,
  85. u64 ram_bytes, int compress_type,
  86. int type);
  87. static void __endio_write_update_ordered(struct btrfs_inode *inode,
  88. const u64 offset, const u64 bytes,
  89. const bool uptodate);
  90. /*
  91. * Cleanup all submitted ordered extents in specified range to handle errors
  92. * from the btrfs_run_delalloc_range() callback.
  93. *
  94. * NOTE: caller must ensure that when an error happens, it can not call
  95. * extent_clear_unlock_delalloc() to clear both the bits EXTENT_DO_ACCOUNTING
  96. * and EXTENT_DELALLOC simultaneously, because that causes the reserved metadata
  97. * to be released, which we want to happen only when finishing the ordered
  98. * extent (btrfs_finish_ordered_io()).
  99. */
  100. static inline void btrfs_cleanup_ordered_extents(struct btrfs_inode *inode,
  101. struct page *locked_page,
  102. u64 offset, u64 bytes)
  103. {
  104. unsigned long index = offset >> PAGE_SHIFT;
  105. unsigned long end_index = (offset + bytes - 1) >> PAGE_SHIFT;
  106. u64 page_start = page_offset(locked_page);
  107. u64 page_end = page_start + PAGE_SIZE - 1;
  108. struct page *page;
  109. while (index <= end_index) {
  110. page = find_get_page(inode->vfs_inode.i_mapping, index);
  111. index++;
  112. if (!page)
  113. continue;
  114. ClearPagePrivate2(page);
  115. put_page(page);
  116. }
  117. /*
  118. * In case this page belongs to the delalloc range being instantiated
  119. * then skip it, since the first page of a range is going to be
  120. * properly cleaned up by the caller of run_delalloc_range
  121. */
  122. if (page_start >= offset && page_end <= (offset + bytes - 1)) {
  123. offset += PAGE_SIZE;
  124. bytes -= PAGE_SIZE;
  125. }
  126. return __endio_write_update_ordered(inode, offset, bytes, false);
  127. }
  128. static int btrfs_dirty_inode(struct inode *inode);
  129. static int btrfs_init_inode_security(struct btrfs_trans_handle *trans,
  130. struct inode *inode, struct inode *dir,
  131. const struct qstr *qstr)
  132. {
  133. int err;
  134. err = btrfs_init_acl(trans, inode, dir);
  135. if (!err)
  136. err = btrfs_xattr_security_init(trans, inode, dir, qstr);
  137. return err;
  138. }
  139. /*
  140. * this does all the hard work for inserting an inline extent into
  141. * the btree. The caller should have done a btrfs_drop_extents so that
  142. * no overlapping inline items exist in the btree
  143. */
  144. static int insert_inline_extent(struct btrfs_trans_handle *trans,
  145. struct btrfs_path *path, int extent_inserted,
  146. struct btrfs_root *root, struct inode *inode,
  147. u64 start, size_t size, size_t compressed_size,
  148. int compress_type,
  149. struct page **compressed_pages)
  150. {
  151. struct extent_buffer *leaf;
  152. struct page *page = NULL;
  153. char *kaddr;
  154. unsigned long ptr;
  155. struct btrfs_file_extent_item *ei;
  156. int ret;
  157. size_t cur_size = size;
  158. unsigned long offset;
  159. ASSERT((compressed_size > 0 && compressed_pages) ||
  160. (compressed_size == 0 && !compressed_pages));
  161. if (compressed_size && compressed_pages)
  162. cur_size = compressed_size;
  163. inode_add_bytes(inode, size);
  164. if (!extent_inserted) {
  165. struct btrfs_key key;
  166. size_t datasize;
  167. key.objectid = btrfs_ino(BTRFS_I(inode));
  168. key.offset = start;
  169. key.type = BTRFS_EXTENT_DATA_KEY;
  170. datasize = btrfs_file_extent_calc_inline_size(cur_size);
  171. path->leave_spinning = 1;
  172. ret = btrfs_insert_empty_item(trans, root, path, &key,
  173. datasize);
  174. if (ret)
  175. goto fail;
  176. }
  177. leaf = path->nodes[0];
  178. ei = btrfs_item_ptr(leaf, path->slots[0],
  179. struct btrfs_file_extent_item);
  180. btrfs_set_file_extent_generation(leaf, ei, trans->transid);
  181. btrfs_set_file_extent_type(leaf, ei, BTRFS_FILE_EXTENT_INLINE);
  182. btrfs_set_file_extent_encryption(leaf, ei, 0);
  183. btrfs_set_file_extent_other_encoding(leaf, ei, 0);
  184. btrfs_set_file_extent_ram_bytes(leaf, ei, size);
  185. ptr = btrfs_file_extent_inline_start(ei);
  186. if (compress_type != BTRFS_COMPRESS_NONE) {
  187. struct page *cpage;
  188. int i = 0;
  189. while (compressed_size > 0) {
  190. cpage = compressed_pages[i];
  191. cur_size = min_t(unsigned long, compressed_size,
  192. PAGE_SIZE);
  193. kaddr = kmap_atomic(cpage);
  194. write_extent_buffer(leaf, kaddr, ptr, cur_size);
  195. kunmap_atomic(kaddr);
  196. i++;
  197. ptr += cur_size;
  198. compressed_size -= cur_size;
  199. }
  200. btrfs_set_file_extent_compression(leaf, ei,
  201. compress_type);
  202. } else {
  203. page = find_get_page(inode->i_mapping,
  204. start >> PAGE_SHIFT);
  205. btrfs_set_file_extent_compression(leaf, ei, 0);
  206. kaddr = kmap_atomic(page);
  207. offset = offset_in_page(start);
  208. write_extent_buffer(leaf, kaddr + offset, ptr, size);
  209. kunmap_atomic(kaddr);
  210. put_page(page);
  211. }
  212. btrfs_mark_buffer_dirty(leaf);
  213. btrfs_release_path(path);
  214. /*
  215. * We align size to sectorsize for inline extents just for simplicity
  216. * sake.
  217. */
  218. size = ALIGN(size, root->fs_info->sectorsize);
  219. ret = btrfs_inode_set_file_extent_range(BTRFS_I(inode), start, size);
  220. if (ret)
  221. goto fail;
  222. /*
  223. * we're an inline extent, so nobody can
  224. * extend the file past i_size without locking
  225. * a page we already have locked.
  226. *
  227. * We must do any isize and inode updates
  228. * before we unlock the pages. Otherwise we
  229. * could end up racing with unlink.
  230. */
  231. BTRFS_I(inode)->disk_i_size = inode->i_size;
  232. ret = btrfs_update_inode(trans, root, inode);
  233. fail:
  234. return ret;
  235. }
  236. /*
  237. * conditionally insert an inline extent into the file. This
  238. * does the checks required to make sure the data is small enough
  239. * to fit as an inline extent.
  240. */
  241. static noinline int cow_file_range_inline(struct btrfs_inode *inode, u64 start,
  242. u64 end, size_t compressed_size,
  243. int compress_type,
  244. struct page **compressed_pages)
  245. {
  246. struct btrfs_root *root = inode->root;
  247. struct btrfs_fs_info *fs_info = root->fs_info;
  248. struct btrfs_trans_handle *trans;
  249. u64 isize = i_size_read(&inode->vfs_inode);
  250. u64 actual_end = min(end + 1, isize);
  251. u64 inline_len = actual_end - start;
  252. u64 aligned_end = ALIGN(end, fs_info->sectorsize);
  253. u64 data_len = inline_len;
  254. int ret;
  255. struct btrfs_path *path;
  256. int extent_inserted = 0;
  257. u32 extent_item_size;
  258. if (compressed_size)
  259. data_len = compressed_size;
  260. if (start > 0 ||
  261. actual_end > fs_info->sectorsize ||
  262. data_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info) ||
  263. (!compressed_size &&
  264. (actual_end & (fs_info->sectorsize - 1)) == 0) ||
  265. end + 1 < isize ||
  266. data_len > fs_info->max_inline) {
  267. return 1;
  268. }
  269. path = btrfs_alloc_path();
  270. if (!path)
  271. return -ENOMEM;
  272. trans = btrfs_join_transaction(root);
  273. if (IS_ERR(trans)) {
  274. btrfs_free_path(path);
  275. return PTR_ERR(trans);
  276. }
  277. trans->block_rsv = &inode->block_rsv;
  278. if (compressed_size && compressed_pages)
  279. extent_item_size = btrfs_file_extent_calc_inline_size(
  280. compressed_size);
  281. else
  282. extent_item_size = btrfs_file_extent_calc_inline_size(
  283. inline_len);
  284. ret = __btrfs_drop_extents(trans, root, inode, path, start, aligned_end,
  285. NULL, 1, 1, extent_item_size,
  286. &extent_inserted);
  287. if (ret) {
  288. btrfs_abort_transaction(trans, ret);
  289. goto out;
  290. }
  291. if (isize > actual_end)
  292. inline_len = min_t(u64, isize, actual_end);
  293. ret = insert_inline_extent(trans, path, extent_inserted,
  294. root, &inode->vfs_inode, start,
  295. inline_len, compressed_size,
  296. compress_type, compressed_pages);
  297. if (ret && ret != -ENOSPC) {
  298. btrfs_abort_transaction(trans, ret);
  299. goto out;
  300. } else if (ret == -ENOSPC) {
  301. ret = 1;
  302. goto out;
  303. }
  304. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &inode->runtime_flags);
  305. btrfs_drop_extent_cache(inode, start, aligned_end - 1, 0);
  306. out:
  307. /*
  308. * Don't forget to free the reserved space, as for inlined extent
  309. * it won't count as data extent, free them directly here.
  310. * And at reserve time, it's always aligned to page size, so
  311. * just free one page here.
  312. */
  313. btrfs_qgroup_free_data(inode, NULL, 0, PAGE_SIZE);
  314. btrfs_free_path(path);
  315. btrfs_end_transaction(trans);
  316. return ret;
  317. }
  318. struct async_extent {
  319. u64 start;
  320. u64 ram_size;
  321. u64 compressed_size;
  322. struct page **pages;
  323. unsigned long nr_pages;
  324. int compress_type;
  325. struct list_head list;
  326. };
  327. struct async_chunk {
  328. struct inode *inode;
  329. struct page *locked_page;
  330. u64 start;
  331. u64 end;
  332. unsigned int write_flags;
  333. struct list_head extents;
  334. struct cgroup_subsys_state *blkcg_css;
  335. struct btrfs_work work;
  336. atomic_t *pending;
  337. };
  338. struct async_cow {
  339. /* Number of chunks in flight; must be first in the structure */
  340. atomic_t num_chunks;
  341. struct async_chunk chunks[];
  342. };
  343. static noinline int add_async_extent(struct async_chunk *cow,
  344. u64 start, u64 ram_size,
  345. u64 compressed_size,
  346. struct page **pages,
  347. unsigned long nr_pages,
  348. int compress_type)
  349. {
  350. struct async_extent *async_extent;
  351. async_extent = kmalloc(sizeof(*async_extent), GFP_NOFS);
  352. BUG_ON(!async_extent); /* -ENOMEM */
  353. async_extent->start = start;
  354. async_extent->ram_size = ram_size;
  355. async_extent->compressed_size = compressed_size;
  356. async_extent->pages = pages;
  357. async_extent->nr_pages = nr_pages;
  358. async_extent->compress_type = compress_type;
  359. list_add_tail(&async_extent->list, &cow->extents);
  360. return 0;
  361. }
  362. /*
  363. * Check if the inode has flags compatible with compression
  364. */
  365. static inline bool inode_can_compress(struct btrfs_inode *inode)
  366. {
  367. if (inode->flags & BTRFS_INODE_NODATACOW ||
  368. inode->flags & BTRFS_INODE_NODATASUM)
  369. return false;
  370. return true;
  371. }
  372. /*
  373. * Check if the inode needs to be submitted to compression, based on mount
  374. * options, defragmentation, properties or heuristics.
  375. */
  376. static inline int inode_need_compress(struct btrfs_inode *inode, u64 start,
  377. u64 end)
  378. {
  379. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  380. if (!inode_can_compress(inode)) {
  381. WARN(IS_ENABLED(CONFIG_BTRFS_DEBUG),
  382. KERN_ERR "BTRFS: unexpected compression for ino %llu\n",
  383. btrfs_ino(inode));
  384. return 0;
  385. }
  386. /* force compress */
  387. if (btrfs_test_opt(fs_info, FORCE_COMPRESS))
  388. return 1;
  389. /* defrag ioctl */
  390. if (inode->defrag_compress)
  391. return 1;
  392. /* bad compression ratios */
  393. if (inode->flags & BTRFS_INODE_NOCOMPRESS)
  394. return 0;
  395. if (btrfs_test_opt(fs_info, COMPRESS) ||
  396. inode->flags & BTRFS_INODE_COMPRESS ||
  397. inode->prop_compress)
  398. return btrfs_compress_heuristic(&inode->vfs_inode, start, end);
  399. return 0;
  400. }
  401. static inline void inode_should_defrag(struct btrfs_inode *inode,
  402. u64 start, u64 end, u64 num_bytes, u64 small_write)
  403. {
  404. /* If this is a small write inside eof, kick off a defrag */
  405. if (num_bytes < small_write &&
  406. (start > 0 || end + 1 < inode->disk_i_size))
  407. btrfs_add_inode_defrag(NULL, inode);
  408. }
  409. /*
  410. * we create compressed extents in two phases. The first
  411. * phase compresses a range of pages that have already been
  412. * locked (both pages and state bits are locked).
  413. *
  414. * This is done inside an ordered work queue, and the compression
  415. * is spread across many cpus. The actual IO submission is step
  416. * two, and the ordered work queue takes care of making sure that
  417. * happens in the same order things were put onto the queue by
  418. * writepages and friends.
  419. *
  420. * If this code finds it can't get good compression, it puts an
  421. * entry onto the work queue to write the uncompressed bytes. This
  422. * makes sure that both compressed inodes and uncompressed inodes
  423. * are written in the same order that the flusher thread sent them
  424. * down.
  425. */
  426. static noinline int compress_file_range(struct async_chunk *async_chunk)
  427. {
  428. struct inode *inode = async_chunk->inode;
  429. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  430. u64 blocksize = fs_info->sectorsize;
  431. u64 start = async_chunk->start;
  432. u64 end = async_chunk->end;
  433. u64 actual_end;
  434. u64 i_size;
  435. int ret = 0;
  436. struct page **pages = NULL;
  437. unsigned long nr_pages;
  438. unsigned long total_compressed = 0;
  439. unsigned long total_in = 0;
  440. int i;
  441. int will_compress;
  442. int compress_type = fs_info->compress_type;
  443. int compressed_extents = 0;
  444. int redirty = 0;
  445. inode_should_defrag(BTRFS_I(inode), start, end, end - start + 1,
  446. SZ_16K);
  447. /*
  448. * We need to save i_size before now because it could change in between
  449. * us evaluating the size and assigning it. This is because we lock and
  450. * unlock the page in truncate and fallocate, and then modify the i_size
  451. * later on.
  452. *
  453. * The barriers are to emulate READ_ONCE, remove that once i_size_read
  454. * does that for us.
  455. */
  456. barrier();
  457. i_size = i_size_read(inode);
  458. barrier();
  459. actual_end = min_t(u64, i_size, end + 1);
  460. again:
  461. will_compress = 0;
  462. nr_pages = (end >> PAGE_SHIFT) - (start >> PAGE_SHIFT) + 1;
  463. BUILD_BUG_ON((BTRFS_MAX_COMPRESSED % PAGE_SIZE) != 0);
  464. nr_pages = min_t(unsigned long, nr_pages,
  465. BTRFS_MAX_COMPRESSED / PAGE_SIZE);
  466. /*
  467. * we don't want to send crud past the end of i_size through
  468. * compression, that's just a waste of CPU time. So, if the
  469. * end of the file is before the start of our current
  470. * requested range of bytes, we bail out to the uncompressed
  471. * cleanup code that can deal with all of this.
  472. *
  473. * It isn't really the fastest way to fix things, but this is a
  474. * very uncommon corner.
  475. */
  476. if (actual_end <= start)
  477. goto cleanup_and_bail_uncompressed;
  478. total_compressed = actual_end - start;
  479. /*
  480. * skip compression for a small file range(<=blocksize) that
  481. * isn't an inline extent, since it doesn't save disk space at all.
  482. */
  483. if (total_compressed <= blocksize &&
  484. (start > 0 || end + 1 < BTRFS_I(inode)->disk_i_size))
  485. goto cleanup_and_bail_uncompressed;
  486. total_compressed = min_t(unsigned long, total_compressed,
  487. BTRFS_MAX_UNCOMPRESSED);
  488. total_in = 0;
  489. ret = 0;
  490. /*
  491. * we do compression for mount -o compress and when the
  492. * inode has not been flagged as nocompress. This flag can
  493. * change at any time if we discover bad compression ratios.
  494. */
  495. if (inode_need_compress(BTRFS_I(inode), start, end)) {
  496. WARN_ON(pages);
  497. pages = kcalloc(nr_pages, sizeof(struct page *), GFP_NOFS);
  498. if (!pages) {
  499. /* just bail out to the uncompressed code */
  500. nr_pages = 0;
  501. goto cont;
  502. }
  503. if (BTRFS_I(inode)->defrag_compress)
  504. compress_type = BTRFS_I(inode)->defrag_compress;
  505. else if (BTRFS_I(inode)->prop_compress)
  506. compress_type = BTRFS_I(inode)->prop_compress;
  507. /*
  508. * we need to call clear_page_dirty_for_io on each
  509. * page in the range. Otherwise applications with the file
  510. * mmap'd can wander in and change the page contents while
  511. * we are compressing them.
  512. *
  513. * If the compression fails for any reason, we set the pages
  514. * dirty again later on.
  515. *
  516. * Note that the remaining part is redirtied, the start pointer
  517. * has moved, the end is the original one.
  518. */
  519. if (!redirty) {
  520. extent_range_clear_dirty_for_io(inode, start, end);
  521. redirty = 1;
  522. }
  523. /* Compression level is applied here and only here */
  524. ret = btrfs_compress_pages(
  525. compress_type | (fs_info->compress_level << 4),
  526. inode->i_mapping, start,
  527. pages,
  528. &nr_pages,
  529. &total_in,
  530. &total_compressed);
  531. if (!ret) {
  532. unsigned long offset = offset_in_page(total_compressed);
  533. struct page *page = pages[nr_pages - 1];
  534. char *kaddr;
  535. /* zero the tail end of the last page, we might be
  536. * sending it down to disk
  537. */
  538. if (offset) {
  539. kaddr = kmap_atomic(page);
  540. memset(kaddr + offset, 0,
  541. PAGE_SIZE - offset);
  542. kunmap_atomic(kaddr);
  543. }
  544. will_compress = 1;
  545. }
  546. }
  547. cont:
  548. if (start == 0) {
  549. /* lets try to make an inline extent */
  550. if (ret || total_in < actual_end) {
  551. /* we didn't compress the entire range, try
  552. * to make an uncompressed inline extent.
  553. */
  554. ret = cow_file_range_inline(BTRFS_I(inode), start, end,
  555. 0, BTRFS_COMPRESS_NONE,
  556. NULL);
  557. } else {
  558. /* try making a compressed inline extent */
  559. ret = cow_file_range_inline(BTRFS_I(inode), start, end,
  560. total_compressed,
  561. compress_type, pages);
  562. }
  563. if (ret <= 0) {
  564. unsigned long clear_flags = EXTENT_DELALLOC |
  565. EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
  566. EXTENT_DO_ACCOUNTING;
  567. unsigned long page_error_op;
  568. page_error_op = ret < 0 ? PAGE_SET_ERROR : 0;
  569. /*
  570. * inline extent creation worked or returned error,
  571. * we don't need to create any more async work items.
  572. * Unlock and free up our temp pages.
  573. *
  574. * We use DO_ACCOUNTING here because we need the
  575. * delalloc_release_metadata to be done _after_ we drop
  576. * our outstanding extent for clearing delalloc for this
  577. * range.
  578. */
  579. extent_clear_unlock_delalloc(BTRFS_I(inode), start, end,
  580. NULL,
  581. clear_flags,
  582. PAGE_UNLOCK |
  583. PAGE_CLEAR_DIRTY |
  584. PAGE_SET_WRITEBACK |
  585. page_error_op |
  586. PAGE_END_WRITEBACK);
  587. /*
  588. * Ensure we only free the compressed pages if we have
  589. * them allocated, as we can still reach here with
  590. * inode_need_compress() == false.
  591. */
  592. if (pages) {
  593. for (i = 0; i < nr_pages; i++) {
  594. WARN_ON(pages[i]->mapping);
  595. put_page(pages[i]);
  596. }
  597. kfree(pages);
  598. }
  599. return 0;
  600. }
  601. }
  602. if (will_compress) {
  603. /*
  604. * we aren't doing an inline extent round the compressed size
  605. * up to a block size boundary so the allocator does sane
  606. * things
  607. */
  608. total_compressed = ALIGN(total_compressed, blocksize);
  609. /*
  610. * one last check to make sure the compression is really a
  611. * win, compare the page count read with the blocks on disk,
  612. * compression must free at least one sector size
  613. */
  614. total_in = ALIGN(total_in, PAGE_SIZE);
  615. if (total_compressed + blocksize <= total_in) {
  616. compressed_extents++;
  617. /*
  618. * The async work queues will take care of doing actual
  619. * allocation on disk for these compressed pages, and
  620. * will submit them to the elevator.
  621. */
  622. add_async_extent(async_chunk, start, total_in,
  623. total_compressed, pages, nr_pages,
  624. compress_type);
  625. if (start + total_in < end) {
  626. start += total_in;
  627. pages = NULL;
  628. cond_resched();
  629. goto again;
  630. }
  631. return compressed_extents;
  632. }
  633. }
  634. if (pages) {
  635. /*
  636. * the compression code ran but failed to make things smaller,
  637. * free any pages it allocated and our page pointer array
  638. */
  639. for (i = 0; i < nr_pages; i++) {
  640. WARN_ON(pages[i]->mapping);
  641. put_page(pages[i]);
  642. }
  643. kfree(pages);
  644. pages = NULL;
  645. total_compressed = 0;
  646. nr_pages = 0;
  647. /* flag the file so we don't compress in the future */
  648. if (!btrfs_test_opt(fs_info, FORCE_COMPRESS) &&
  649. !(BTRFS_I(inode)->prop_compress)) {
  650. BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
  651. }
  652. }
  653. cleanup_and_bail_uncompressed:
  654. /*
  655. * No compression, but we still need to write the pages in the file
  656. * we've been given so far. redirty the locked page if it corresponds
  657. * to our extent and set things up for the async work queue to run
  658. * cow_file_range to do the normal delalloc dance.
  659. */
  660. if (async_chunk->locked_page &&
  661. (page_offset(async_chunk->locked_page) >= start &&
  662. page_offset(async_chunk->locked_page)) <= end) {
  663. __set_page_dirty_nobuffers(async_chunk->locked_page);
  664. /* unlocked later on in the async handlers */
  665. }
  666. if (redirty)
  667. extent_range_redirty_for_io(inode, start, end);
  668. add_async_extent(async_chunk, start, end - start + 1, 0, NULL, 0,
  669. BTRFS_COMPRESS_NONE);
  670. compressed_extents++;
  671. return compressed_extents;
  672. }
  673. static void free_async_extent_pages(struct async_extent *async_extent)
  674. {
  675. int i;
  676. if (!async_extent->pages)
  677. return;
  678. for (i = 0; i < async_extent->nr_pages; i++) {
  679. WARN_ON(async_extent->pages[i]->mapping);
  680. put_page(async_extent->pages[i]);
  681. }
  682. kfree(async_extent->pages);
  683. async_extent->nr_pages = 0;
  684. async_extent->pages = NULL;
  685. }
  686. /*
  687. * phase two of compressed writeback. This is the ordered portion
  688. * of the code, which only gets called in the order the work was
  689. * queued. We walk all the async extents created by compress_file_range
  690. * and send them down to the disk.
  691. */
  692. static noinline void submit_compressed_extents(struct async_chunk *async_chunk)
  693. {
  694. struct btrfs_inode *inode = BTRFS_I(async_chunk->inode);
  695. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  696. struct async_extent *async_extent;
  697. u64 alloc_hint = 0;
  698. struct btrfs_key ins;
  699. struct extent_map *em;
  700. struct btrfs_root *root = inode->root;
  701. struct extent_io_tree *io_tree = &inode->io_tree;
  702. int ret = 0;
  703. again:
  704. while (!list_empty(&async_chunk->extents)) {
  705. async_extent = list_entry(async_chunk->extents.next,
  706. struct async_extent, list);
  707. list_del(&async_extent->list);
  708. retry:
  709. lock_extent(io_tree, async_extent->start,
  710. async_extent->start + async_extent->ram_size - 1);
  711. /* did the compression code fall back to uncompressed IO? */
  712. if (!async_extent->pages) {
  713. int page_started = 0;
  714. unsigned long nr_written = 0;
  715. /* allocate blocks */
  716. ret = cow_file_range(inode, async_chunk->locked_page,
  717. async_extent->start,
  718. async_extent->start +
  719. async_extent->ram_size - 1,
  720. &page_started, &nr_written, 0);
  721. /* JDM XXX */
  722. /*
  723. * if page_started, cow_file_range inserted an
  724. * inline extent and took care of all the unlocking
  725. * and IO for us. Otherwise, we need to submit
  726. * all those pages down to the drive.
  727. */
  728. if (!page_started && !ret)
  729. extent_write_locked_range(&inode->vfs_inode,
  730. async_extent->start,
  731. async_extent->start +
  732. async_extent->ram_size - 1,
  733. WB_SYNC_ALL);
  734. else if (ret && async_chunk->locked_page)
  735. unlock_page(async_chunk->locked_page);
  736. kfree(async_extent);
  737. cond_resched();
  738. continue;
  739. }
  740. ret = btrfs_reserve_extent(root, async_extent->ram_size,
  741. async_extent->compressed_size,
  742. async_extent->compressed_size,
  743. 0, alloc_hint, &ins, 1, 1);
  744. if (ret) {
  745. free_async_extent_pages(async_extent);
  746. if (ret == -ENOSPC) {
  747. unlock_extent(io_tree, async_extent->start,
  748. async_extent->start +
  749. async_extent->ram_size - 1);
  750. /*
  751. * we need to redirty the pages if we decide to
  752. * fallback to uncompressed IO, otherwise we
  753. * will not submit these pages down to lower
  754. * layers.
  755. */
  756. extent_range_redirty_for_io(&inode->vfs_inode,
  757. async_extent->start,
  758. async_extent->start +
  759. async_extent->ram_size - 1);
  760. goto retry;
  761. }
  762. goto out_free;
  763. }
  764. /*
  765. * here we're doing allocation and writeback of the
  766. * compressed pages
  767. */
  768. em = create_io_em(inode, async_extent->start,
  769. async_extent->ram_size, /* len */
  770. async_extent->start, /* orig_start */
  771. ins.objectid, /* block_start */
  772. ins.offset, /* block_len */
  773. ins.offset, /* orig_block_len */
  774. async_extent->ram_size, /* ram_bytes */
  775. async_extent->compress_type,
  776. BTRFS_ORDERED_COMPRESSED);
  777. if (IS_ERR(em))
  778. /* ret value is not necessary due to void function */
  779. goto out_free_reserve;
  780. free_extent_map(em);
  781. ret = btrfs_add_ordered_extent_compress(inode,
  782. async_extent->start,
  783. ins.objectid,
  784. async_extent->ram_size,
  785. ins.offset,
  786. BTRFS_ORDERED_COMPRESSED,
  787. async_extent->compress_type);
  788. if (ret) {
  789. btrfs_drop_extent_cache(inode, async_extent->start,
  790. async_extent->start +
  791. async_extent->ram_size - 1, 0);
  792. goto out_free_reserve;
  793. }
  794. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  795. /*
  796. * clear dirty, set writeback and unlock the pages.
  797. */
  798. extent_clear_unlock_delalloc(inode, async_extent->start,
  799. async_extent->start +
  800. async_extent->ram_size - 1,
  801. NULL, EXTENT_LOCKED | EXTENT_DELALLOC,
  802. PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
  803. PAGE_SET_WRITEBACK);
  804. if (btrfs_submit_compressed_write(inode, async_extent->start,
  805. async_extent->ram_size,
  806. ins.objectid,
  807. ins.offset, async_extent->pages,
  808. async_extent->nr_pages,
  809. async_chunk->write_flags,
  810. async_chunk->blkcg_css)) {
  811. struct page *p = async_extent->pages[0];
  812. const u64 start = async_extent->start;
  813. const u64 end = start + async_extent->ram_size - 1;
  814. p->mapping = inode->vfs_inode.i_mapping;
  815. btrfs_writepage_endio_finish_ordered(p, start, end, 0);
  816. p->mapping = NULL;
  817. extent_clear_unlock_delalloc(inode, start, end, NULL, 0,
  818. PAGE_END_WRITEBACK |
  819. PAGE_SET_ERROR);
  820. free_async_extent_pages(async_extent);
  821. }
  822. alloc_hint = ins.objectid + ins.offset;
  823. kfree(async_extent);
  824. cond_resched();
  825. }
  826. return;
  827. out_free_reserve:
  828. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  829. btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
  830. out_free:
  831. extent_clear_unlock_delalloc(inode, async_extent->start,
  832. async_extent->start +
  833. async_extent->ram_size - 1,
  834. NULL, EXTENT_LOCKED | EXTENT_DELALLOC |
  835. EXTENT_DELALLOC_NEW |
  836. EXTENT_DEFRAG | EXTENT_DO_ACCOUNTING,
  837. PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
  838. PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
  839. PAGE_SET_ERROR);
  840. free_async_extent_pages(async_extent);
  841. kfree(async_extent);
  842. goto again;
  843. }
  844. static u64 get_extent_allocation_hint(struct btrfs_inode *inode, u64 start,
  845. u64 num_bytes)
  846. {
  847. struct extent_map_tree *em_tree = &inode->extent_tree;
  848. struct extent_map *em;
  849. u64 alloc_hint = 0;
  850. read_lock(&em_tree->lock);
  851. em = search_extent_mapping(em_tree, start, num_bytes);
  852. if (em) {
  853. /*
  854. * if block start isn't an actual block number then find the
  855. * first block in this inode and use that as a hint. If that
  856. * block is also bogus then just don't worry about it.
  857. */
  858. if (em->block_start >= EXTENT_MAP_LAST_BYTE) {
  859. free_extent_map(em);
  860. em = search_extent_mapping(em_tree, 0, 0);
  861. if (em && em->block_start < EXTENT_MAP_LAST_BYTE)
  862. alloc_hint = em->block_start;
  863. if (em)
  864. free_extent_map(em);
  865. } else {
  866. alloc_hint = em->block_start;
  867. free_extent_map(em);
  868. }
  869. }
  870. read_unlock(&em_tree->lock);
  871. return alloc_hint;
  872. }
  873. /*
  874. * when extent_io.c finds a delayed allocation range in the file,
  875. * the call backs end up in this code. The basic idea is to
  876. * allocate extents on disk for the range, and create ordered data structs
  877. * in ram to track those extents.
  878. *
  879. * locked_page is the page that writepage had locked already. We use
  880. * it to make sure we don't do extra locks or unlocks.
  881. *
  882. * *page_started is set to one if we unlock locked_page and do everything
  883. * required to start IO on it. It may be clean and already done with
  884. * IO when we return.
  885. */
  886. static noinline int cow_file_range(struct btrfs_inode *inode,
  887. struct page *locked_page,
  888. u64 start, u64 end, int *page_started,
  889. unsigned long *nr_written, int unlock)
  890. {
  891. struct btrfs_root *root = inode->root;
  892. struct btrfs_fs_info *fs_info = root->fs_info;
  893. u64 alloc_hint = 0;
  894. u64 num_bytes;
  895. unsigned long ram_size;
  896. u64 cur_alloc_size = 0;
  897. u64 min_alloc_size;
  898. u64 blocksize = fs_info->sectorsize;
  899. struct btrfs_key ins;
  900. struct extent_map *em;
  901. unsigned clear_bits;
  902. unsigned long page_ops;
  903. bool extent_reserved = false;
  904. int ret = 0;
  905. if (btrfs_is_free_space_inode(inode)) {
  906. ret = -EINVAL;
  907. goto out_unlock;
  908. }
  909. num_bytes = ALIGN(end - start + 1, blocksize);
  910. num_bytes = max(blocksize, num_bytes);
  911. ASSERT(num_bytes <= btrfs_super_total_bytes(fs_info->super_copy));
  912. inode_should_defrag(inode, start, end, num_bytes, SZ_64K);
  913. if (start == 0) {
  914. /* lets try to make an inline extent */
  915. ret = cow_file_range_inline(inode, start, end, 0,
  916. BTRFS_COMPRESS_NONE, NULL);
  917. if (ret == 0) {
  918. /*
  919. * We use DO_ACCOUNTING here because we need the
  920. * delalloc_release_metadata to be run _after_ we drop
  921. * our outstanding extent for clearing delalloc for this
  922. * range.
  923. */
  924. extent_clear_unlock_delalloc(inode, start, end, NULL,
  925. EXTENT_LOCKED | EXTENT_DELALLOC |
  926. EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
  927. EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
  928. PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
  929. PAGE_END_WRITEBACK);
  930. *nr_written = *nr_written +
  931. (end - start + PAGE_SIZE) / PAGE_SIZE;
  932. *page_started = 1;
  933. goto out;
  934. } else if (ret < 0) {
  935. goto out_unlock;
  936. }
  937. }
  938. alloc_hint = get_extent_allocation_hint(inode, start, num_bytes);
  939. btrfs_drop_extent_cache(inode, start, start + num_bytes - 1, 0);
  940. /*
  941. * Relocation relies on the relocated extents to have exactly the same
  942. * size as the original extents. Normally writeback for relocation data
  943. * extents follows a NOCOW path because relocation preallocates the
  944. * extents. However, due to an operation such as scrub turning a block
  945. * group to RO mode, it may fallback to COW mode, so we must make sure
  946. * an extent allocated during COW has exactly the requested size and can
  947. * not be split into smaller extents, otherwise relocation breaks and
  948. * fails during the stage where it updates the bytenr of file extent
  949. * items.
  950. */
  951. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
  952. min_alloc_size = num_bytes;
  953. else
  954. min_alloc_size = fs_info->sectorsize;
  955. while (num_bytes > 0) {
  956. cur_alloc_size = num_bytes;
  957. ret = btrfs_reserve_extent(root, cur_alloc_size, cur_alloc_size,
  958. min_alloc_size, 0, alloc_hint,
  959. &ins, 1, 1);
  960. if (ret < 0)
  961. goto out_unlock;
  962. cur_alloc_size = ins.offset;
  963. extent_reserved = true;
  964. ram_size = ins.offset;
  965. em = create_io_em(inode, start, ins.offset, /* len */
  966. start, /* orig_start */
  967. ins.objectid, /* block_start */
  968. ins.offset, /* block_len */
  969. ins.offset, /* orig_block_len */
  970. ram_size, /* ram_bytes */
  971. BTRFS_COMPRESS_NONE, /* compress_type */
  972. BTRFS_ORDERED_REGULAR /* type */);
  973. if (IS_ERR(em)) {
  974. ret = PTR_ERR(em);
  975. goto out_reserve;
  976. }
  977. free_extent_map(em);
  978. ret = btrfs_add_ordered_extent(inode, start, ins.objectid,
  979. ram_size, cur_alloc_size, 0);
  980. if (ret)
  981. goto out_drop_extent_cache;
  982. if (root->root_key.objectid ==
  983. BTRFS_DATA_RELOC_TREE_OBJECTID) {
  984. ret = btrfs_reloc_clone_csums(inode, start,
  985. cur_alloc_size);
  986. /*
  987. * Only drop cache here, and process as normal.
  988. *
  989. * We must not allow extent_clear_unlock_delalloc()
  990. * at out_unlock label to free meta of this ordered
  991. * extent, as its meta should be freed by
  992. * btrfs_finish_ordered_io().
  993. *
  994. * So we must continue until @start is increased to
  995. * skip current ordered extent.
  996. */
  997. if (ret)
  998. btrfs_drop_extent_cache(inode, start,
  999. start + ram_size - 1, 0);
  1000. }
  1001. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  1002. /* we're not doing compressed IO, don't unlock the first
  1003. * page (which the caller expects to stay locked), don't
  1004. * clear any dirty bits and don't set any writeback bits
  1005. *
  1006. * Do set the Private2 bit so we know this page was properly
  1007. * setup for writepage
  1008. */
  1009. page_ops = unlock ? PAGE_UNLOCK : 0;
  1010. page_ops |= PAGE_SET_PRIVATE2;
  1011. extent_clear_unlock_delalloc(inode, start, start + ram_size - 1,
  1012. locked_page,
  1013. EXTENT_LOCKED | EXTENT_DELALLOC,
  1014. page_ops);
  1015. if (num_bytes < cur_alloc_size)
  1016. num_bytes = 0;
  1017. else
  1018. num_bytes -= cur_alloc_size;
  1019. alloc_hint = ins.objectid + ins.offset;
  1020. start += cur_alloc_size;
  1021. extent_reserved = false;
  1022. /*
  1023. * btrfs_reloc_clone_csums() error, since start is increased
  1024. * extent_clear_unlock_delalloc() at out_unlock label won't
  1025. * free metadata of current ordered extent, we're OK to exit.
  1026. */
  1027. if (ret)
  1028. goto out_unlock;
  1029. }
  1030. out:
  1031. return ret;
  1032. out_drop_extent_cache:
  1033. btrfs_drop_extent_cache(inode, start, start + ram_size - 1, 0);
  1034. out_reserve:
  1035. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  1036. btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset, 1);
  1037. out_unlock:
  1038. clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
  1039. EXTENT_DEFRAG | EXTENT_CLEAR_META_RESV;
  1040. page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY | PAGE_SET_WRITEBACK |
  1041. PAGE_END_WRITEBACK;
  1042. /*
  1043. * If we reserved an extent for our delalloc range (or a subrange) and
  1044. * failed to create the respective ordered extent, then it means that
  1045. * when we reserved the extent we decremented the extent's size from
  1046. * the data space_info's bytes_may_use counter and incremented the
  1047. * space_info's bytes_reserved counter by the same amount. We must make
  1048. * sure extent_clear_unlock_delalloc() does not try to decrement again
  1049. * the data space_info's bytes_may_use counter, therefore we do not pass
  1050. * it the flag EXTENT_CLEAR_DATA_RESV.
  1051. */
  1052. if (extent_reserved) {
  1053. extent_clear_unlock_delalloc(inode, start,
  1054. start + cur_alloc_size - 1,
  1055. locked_page,
  1056. clear_bits,
  1057. page_ops);
  1058. start += cur_alloc_size;
  1059. if (start >= end)
  1060. goto out;
  1061. }
  1062. extent_clear_unlock_delalloc(inode, start, end, locked_page,
  1063. clear_bits | EXTENT_CLEAR_DATA_RESV,
  1064. page_ops);
  1065. goto out;
  1066. }
  1067. /*
  1068. * work queue call back to started compression on a file and pages
  1069. */
  1070. static noinline void async_cow_start(struct btrfs_work *work)
  1071. {
  1072. struct async_chunk *async_chunk;
  1073. int compressed_extents;
  1074. async_chunk = container_of(work, struct async_chunk, work);
  1075. compressed_extents = compress_file_range(async_chunk);
  1076. if (compressed_extents == 0) {
  1077. btrfs_add_delayed_iput(async_chunk->inode);
  1078. async_chunk->inode = NULL;
  1079. }
  1080. }
  1081. /*
  1082. * work queue call back to submit previously compressed pages
  1083. */
  1084. static noinline void async_cow_submit(struct btrfs_work *work)
  1085. {
  1086. struct async_chunk *async_chunk = container_of(work, struct async_chunk,
  1087. work);
  1088. struct btrfs_fs_info *fs_info = btrfs_work_owner(work);
  1089. unsigned long nr_pages;
  1090. nr_pages = (async_chunk->end - async_chunk->start + PAGE_SIZE) >>
  1091. PAGE_SHIFT;
  1092. /*
  1093. * ->inode could be NULL if async_chunk_start has failed to compress,
  1094. * in which case we don't have anything to submit, yet we need to
  1095. * always adjust ->async_delalloc_pages as its paired with the init
  1096. * happening in cow_file_range_async
  1097. */
  1098. if (async_chunk->inode)
  1099. submit_compressed_extents(async_chunk);
  1100. /* atomic_sub_return implies a barrier */
  1101. if (atomic_sub_return(nr_pages, &fs_info->async_delalloc_pages) <
  1102. 5 * SZ_1M)
  1103. cond_wake_up_nomb(&fs_info->async_submit_wait);
  1104. }
  1105. static noinline void async_cow_free(struct btrfs_work *work)
  1106. {
  1107. struct async_chunk *async_chunk;
  1108. async_chunk = container_of(work, struct async_chunk, work);
  1109. if (async_chunk->inode)
  1110. btrfs_add_delayed_iput(async_chunk->inode);
  1111. if (async_chunk->blkcg_css)
  1112. css_put(async_chunk->blkcg_css);
  1113. /*
  1114. * Since the pointer to 'pending' is at the beginning of the array of
  1115. * async_chunk's, freeing it ensures the whole array has been freed.
  1116. */
  1117. if (atomic_dec_and_test(async_chunk->pending))
  1118. kvfree(async_chunk->pending);
  1119. }
  1120. static int cow_file_range_async(struct btrfs_inode *inode,
  1121. struct writeback_control *wbc,
  1122. struct page *locked_page,
  1123. u64 start, u64 end, int *page_started,
  1124. unsigned long *nr_written)
  1125. {
  1126. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  1127. struct cgroup_subsys_state *blkcg_css = wbc_blkcg_css(wbc);
  1128. struct async_cow *ctx;
  1129. struct async_chunk *async_chunk;
  1130. unsigned long nr_pages;
  1131. u64 cur_end;
  1132. u64 num_chunks = DIV_ROUND_UP(end - start, SZ_512K);
  1133. int i;
  1134. bool should_compress;
  1135. unsigned nofs_flag;
  1136. const unsigned int write_flags = wbc_to_write_flags(wbc);
  1137. unlock_extent(&inode->io_tree, start, end);
  1138. if (inode->flags & BTRFS_INODE_NOCOMPRESS &&
  1139. !btrfs_test_opt(fs_info, FORCE_COMPRESS)) {
  1140. num_chunks = 1;
  1141. should_compress = false;
  1142. } else {
  1143. should_compress = true;
  1144. }
  1145. nofs_flag = memalloc_nofs_save();
  1146. ctx = kvmalloc(struct_size(ctx, chunks, num_chunks), GFP_KERNEL);
  1147. memalloc_nofs_restore(nofs_flag);
  1148. if (!ctx) {
  1149. unsigned clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC |
  1150. EXTENT_DELALLOC_NEW | EXTENT_DEFRAG |
  1151. EXTENT_DO_ACCOUNTING;
  1152. unsigned long page_ops = PAGE_UNLOCK | PAGE_CLEAR_DIRTY |
  1153. PAGE_SET_WRITEBACK | PAGE_END_WRITEBACK |
  1154. PAGE_SET_ERROR;
  1155. extent_clear_unlock_delalloc(inode, start, end, locked_page,
  1156. clear_bits, page_ops);
  1157. return -ENOMEM;
  1158. }
  1159. async_chunk = ctx->chunks;
  1160. atomic_set(&ctx->num_chunks, num_chunks);
  1161. for (i = 0; i < num_chunks; i++) {
  1162. if (should_compress)
  1163. cur_end = min(end, start + SZ_512K - 1);
  1164. else
  1165. cur_end = end;
  1166. /*
  1167. * igrab is called higher up in the call chain, take only the
  1168. * lightweight reference for the callback lifetime
  1169. */
  1170. ihold(&inode->vfs_inode);
  1171. async_chunk[i].pending = &ctx->num_chunks;
  1172. async_chunk[i].inode = &inode->vfs_inode;
  1173. async_chunk[i].start = start;
  1174. async_chunk[i].end = cur_end;
  1175. async_chunk[i].write_flags = write_flags;
  1176. INIT_LIST_HEAD(&async_chunk[i].extents);
  1177. /*
  1178. * The locked_page comes all the way from writepage and its
  1179. * the original page we were actually given. As we spread
  1180. * this large delalloc region across multiple async_chunk
  1181. * structs, only the first struct needs a pointer to locked_page
  1182. *
  1183. * This way we don't need racey decisions about who is supposed
  1184. * to unlock it.
  1185. */
  1186. if (locked_page) {
  1187. /*
  1188. * Depending on the compressibility, the pages might or
  1189. * might not go through async. We want all of them to
  1190. * be accounted against wbc once. Let's do it here
  1191. * before the paths diverge. wbc accounting is used
  1192. * only for foreign writeback detection and doesn't
  1193. * need full accuracy. Just account the whole thing
  1194. * against the first page.
  1195. */
  1196. wbc_account_cgroup_owner(wbc, locked_page,
  1197. cur_end - start);
  1198. async_chunk[i].locked_page = locked_page;
  1199. locked_page = NULL;
  1200. } else {
  1201. async_chunk[i].locked_page = NULL;
  1202. }
  1203. if (blkcg_css != blkcg_root_css) {
  1204. css_get(blkcg_css);
  1205. async_chunk[i].blkcg_css = blkcg_css;
  1206. } else {
  1207. async_chunk[i].blkcg_css = NULL;
  1208. }
  1209. btrfs_init_work(&async_chunk[i].work, async_cow_start,
  1210. async_cow_submit, async_cow_free);
  1211. nr_pages = DIV_ROUND_UP(cur_end - start, PAGE_SIZE);
  1212. atomic_add(nr_pages, &fs_info->async_delalloc_pages);
  1213. btrfs_queue_work(fs_info->delalloc_workers, &async_chunk[i].work);
  1214. *nr_written += nr_pages;
  1215. start = cur_end + 1;
  1216. }
  1217. *page_started = 1;
  1218. return 0;
  1219. }
  1220. static noinline int csum_exist_in_range(struct btrfs_fs_info *fs_info,
  1221. u64 bytenr, u64 num_bytes)
  1222. {
  1223. int ret;
  1224. struct btrfs_ordered_sum *sums;
  1225. LIST_HEAD(list);
  1226. ret = btrfs_lookup_csums_range(fs_info->csum_root, bytenr,
  1227. bytenr + num_bytes - 1, &list, 0);
  1228. if (ret == 0 && list_empty(&list))
  1229. return 0;
  1230. while (!list_empty(&list)) {
  1231. sums = list_entry(list.next, struct btrfs_ordered_sum, list);
  1232. list_del(&sums->list);
  1233. kfree(sums);
  1234. }
  1235. if (ret < 0)
  1236. return ret;
  1237. return 1;
  1238. }
  1239. static int fallback_to_cow(struct btrfs_inode *inode, struct page *locked_page,
  1240. const u64 start, const u64 end,
  1241. int *page_started, unsigned long *nr_written)
  1242. {
  1243. const bool is_space_ino = btrfs_is_free_space_inode(inode);
  1244. const bool is_reloc_ino = (inode->root->root_key.objectid ==
  1245. BTRFS_DATA_RELOC_TREE_OBJECTID);
  1246. const u64 range_bytes = end + 1 - start;
  1247. struct extent_io_tree *io_tree = &inode->io_tree;
  1248. u64 range_start = start;
  1249. u64 count;
  1250. /*
  1251. * If EXTENT_NORESERVE is set it means that when the buffered write was
  1252. * made we had not enough available data space and therefore we did not
  1253. * reserve data space for it, since we though we could do NOCOW for the
  1254. * respective file range (either there is prealloc extent or the inode
  1255. * has the NOCOW bit set).
  1256. *
  1257. * However when we need to fallback to COW mode (because for example the
  1258. * block group for the corresponding extent was turned to RO mode by a
  1259. * scrub or relocation) we need to do the following:
  1260. *
  1261. * 1) We increment the bytes_may_use counter of the data space info.
  1262. * If COW succeeds, it allocates a new data extent and after doing
  1263. * that it decrements the space info's bytes_may_use counter and
  1264. * increments its bytes_reserved counter by the same amount (we do
  1265. * this at btrfs_add_reserved_bytes()). So we need to increment the
  1266. * bytes_may_use counter to compensate (when space is reserved at
  1267. * buffered write time, the bytes_may_use counter is incremented);
  1268. *
  1269. * 2) We clear the EXTENT_NORESERVE bit from the range. We do this so
  1270. * that if the COW path fails for any reason, it decrements (through
  1271. * extent_clear_unlock_delalloc()) the bytes_may_use counter of the
  1272. * data space info, which we incremented in the step above.
  1273. *
  1274. * If we need to fallback to cow and the inode corresponds to a free
  1275. * space cache inode or an inode of the data relocation tree, we must
  1276. * also increment bytes_may_use of the data space_info for the same
  1277. * reason. Space caches and relocated data extents always get a prealloc
  1278. * extent for them, however scrub or balance may have set the block
  1279. * group that contains that extent to RO mode and therefore force COW
  1280. * when starting writeback.
  1281. */
  1282. count = count_range_bits(io_tree, &range_start, end, range_bytes,
  1283. EXTENT_NORESERVE, 0);
  1284. if (count > 0 || is_space_ino || is_reloc_ino) {
  1285. u64 bytes = count;
  1286. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  1287. struct btrfs_space_info *sinfo = fs_info->data_sinfo;
  1288. if (is_space_ino || is_reloc_ino)
  1289. bytes = range_bytes;
  1290. spin_lock(&sinfo->lock);
  1291. btrfs_space_info_update_bytes_may_use(fs_info, sinfo, bytes);
  1292. spin_unlock(&sinfo->lock);
  1293. if (count > 0)
  1294. clear_extent_bit(io_tree, start, end, EXTENT_NORESERVE,
  1295. 0, 0, NULL);
  1296. }
  1297. return cow_file_range(inode, locked_page, start, end, page_started,
  1298. nr_written, 1);
  1299. }
  1300. /*
  1301. * when nowcow writeback call back. This checks for snapshots or COW copies
  1302. * of the extents that exist in the file, and COWs the file as required.
  1303. *
  1304. * If no cow copies or snapshots exist, we write directly to the existing
  1305. * blocks on disk
  1306. */
  1307. static noinline int run_delalloc_nocow(struct btrfs_inode *inode,
  1308. struct page *locked_page,
  1309. const u64 start, const u64 end,
  1310. int *page_started, int force,
  1311. unsigned long *nr_written)
  1312. {
  1313. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  1314. struct btrfs_root *root = inode->root;
  1315. struct btrfs_path *path;
  1316. u64 cow_start = (u64)-1;
  1317. u64 cur_offset = start;
  1318. int ret;
  1319. bool check_prev = true;
  1320. const bool freespace_inode = btrfs_is_free_space_inode(inode);
  1321. u64 ino = btrfs_ino(inode);
  1322. bool nocow = false;
  1323. u64 disk_bytenr = 0;
  1324. path = btrfs_alloc_path();
  1325. if (!path) {
  1326. extent_clear_unlock_delalloc(inode, start, end, locked_page,
  1327. EXTENT_LOCKED | EXTENT_DELALLOC |
  1328. EXTENT_DO_ACCOUNTING |
  1329. EXTENT_DEFRAG, PAGE_UNLOCK |
  1330. PAGE_CLEAR_DIRTY |
  1331. PAGE_SET_WRITEBACK |
  1332. PAGE_END_WRITEBACK);
  1333. return -ENOMEM;
  1334. }
  1335. while (1) {
  1336. struct btrfs_key found_key;
  1337. struct btrfs_file_extent_item *fi;
  1338. struct extent_buffer *leaf;
  1339. u64 extent_end;
  1340. u64 extent_offset;
  1341. u64 num_bytes = 0;
  1342. u64 disk_num_bytes;
  1343. u64 ram_bytes;
  1344. int extent_type;
  1345. nocow = false;
  1346. ret = btrfs_lookup_file_extent(NULL, root, path, ino,
  1347. cur_offset, 0);
  1348. if (ret < 0)
  1349. goto error;
  1350. /*
  1351. * If there is no extent for our range when doing the initial
  1352. * search, then go back to the previous slot as it will be the
  1353. * one containing the search offset
  1354. */
  1355. if (ret > 0 && path->slots[0] > 0 && check_prev) {
  1356. leaf = path->nodes[0];
  1357. btrfs_item_key_to_cpu(leaf, &found_key,
  1358. path->slots[0] - 1);
  1359. if (found_key.objectid == ino &&
  1360. found_key.type == BTRFS_EXTENT_DATA_KEY)
  1361. path->slots[0]--;
  1362. }
  1363. check_prev = false;
  1364. next_slot:
  1365. /* Go to next leaf if we have exhausted the current one */
  1366. leaf = path->nodes[0];
  1367. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  1368. ret = btrfs_next_leaf(root, path);
  1369. if (ret < 0) {
  1370. if (cow_start != (u64)-1)
  1371. cur_offset = cow_start;
  1372. goto error;
  1373. }
  1374. if (ret > 0)
  1375. break;
  1376. leaf = path->nodes[0];
  1377. }
  1378. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  1379. /* Didn't find anything for our INO */
  1380. if (found_key.objectid > ino)
  1381. break;
  1382. /*
  1383. * Keep searching until we find an EXTENT_ITEM or there are no
  1384. * more extents for this inode
  1385. */
  1386. if (WARN_ON_ONCE(found_key.objectid < ino) ||
  1387. found_key.type < BTRFS_EXTENT_DATA_KEY) {
  1388. path->slots[0]++;
  1389. goto next_slot;
  1390. }
  1391. /* Found key is not EXTENT_DATA_KEY or starts after req range */
  1392. if (found_key.type > BTRFS_EXTENT_DATA_KEY ||
  1393. found_key.offset > end)
  1394. break;
  1395. /*
  1396. * If the found extent starts after requested offset, then
  1397. * adjust extent_end to be right before this extent begins
  1398. */
  1399. if (found_key.offset > cur_offset) {
  1400. extent_end = found_key.offset;
  1401. extent_type = 0;
  1402. goto out_check;
  1403. }
  1404. /*
  1405. * Found extent which begins before our range and potentially
  1406. * intersect it
  1407. */
  1408. fi = btrfs_item_ptr(leaf, path->slots[0],
  1409. struct btrfs_file_extent_item);
  1410. extent_type = btrfs_file_extent_type(leaf, fi);
  1411. ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  1412. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  1413. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  1414. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  1415. extent_offset = btrfs_file_extent_offset(leaf, fi);
  1416. extent_end = found_key.offset +
  1417. btrfs_file_extent_num_bytes(leaf, fi);
  1418. disk_num_bytes =
  1419. btrfs_file_extent_disk_num_bytes(leaf, fi);
  1420. /*
  1421. * If the extent we got ends before our current offset,
  1422. * skip to the next extent.
  1423. */
  1424. if (extent_end <= cur_offset) {
  1425. path->slots[0]++;
  1426. goto next_slot;
  1427. }
  1428. /* Skip holes */
  1429. if (disk_bytenr == 0)
  1430. goto out_check;
  1431. /* Skip compressed/encrypted/encoded extents */
  1432. if (btrfs_file_extent_compression(leaf, fi) ||
  1433. btrfs_file_extent_encryption(leaf, fi) ||
  1434. btrfs_file_extent_other_encoding(leaf, fi))
  1435. goto out_check;
  1436. /*
  1437. * If extent is created before the last volume's snapshot
  1438. * this implies the extent is shared, hence we can't do
  1439. * nocow. This is the same check as in
  1440. * btrfs_cross_ref_exist but without calling
  1441. * btrfs_search_slot.
  1442. */
  1443. if (!freespace_inode &&
  1444. btrfs_file_extent_generation(leaf, fi) <=
  1445. btrfs_root_last_snapshot(&root->root_item))
  1446. goto out_check;
  1447. if (extent_type == BTRFS_FILE_EXTENT_REG && !force)
  1448. goto out_check;
  1449. /* If extent is RO, we must COW it */
  1450. if (btrfs_extent_readonly(fs_info, disk_bytenr))
  1451. goto out_check;
  1452. ret = btrfs_cross_ref_exist(root, ino,
  1453. found_key.offset -
  1454. extent_offset, disk_bytenr, false);
  1455. if (ret) {
  1456. /*
  1457. * ret could be -EIO if the above fails to read
  1458. * metadata.
  1459. */
  1460. if (ret < 0) {
  1461. if (cow_start != (u64)-1)
  1462. cur_offset = cow_start;
  1463. goto error;
  1464. }
  1465. WARN_ON_ONCE(freespace_inode);
  1466. goto out_check;
  1467. }
  1468. disk_bytenr += extent_offset;
  1469. disk_bytenr += cur_offset - found_key.offset;
  1470. num_bytes = min(end + 1, extent_end) - cur_offset;
  1471. /*
  1472. * If there are pending snapshots for this root, we
  1473. * fall into common COW way
  1474. */
  1475. if (!freespace_inode && atomic_read(&root->snapshot_force_cow))
  1476. goto out_check;
  1477. /*
  1478. * force cow if csum exists in the range.
  1479. * this ensure that csum for a given extent are
  1480. * either valid or do not exist.
  1481. */
  1482. ret = csum_exist_in_range(fs_info, disk_bytenr,
  1483. num_bytes);
  1484. if (ret) {
  1485. /*
  1486. * ret could be -EIO if the above fails to read
  1487. * metadata.
  1488. */
  1489. if (ret < 0) {
  1490. if (cow_start != (u64)-1)
  1491. cur_offset = cow_start;
  1492. goto error;
  1493. }
  1494. WARN_ON_ONCE(freespace_inode);
  1495. goto out_check;
  1496. }
  1497. if (!btrfs_inc_nocow_writers(fs_info, disk_bytenr))
  1498. goto out_check;
  1499. nocow = true;
  1500. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  1501. extent_end = found_key.offset + ram_bytes;
  1502. extent_end = ALIGN(extent_end, fs_info->sectorsize);
  1503. /* Skip extents outside of our requested range */
  1504. if (extent_end <= start) {
  1505. path->slots[0]++;
  1506. goto next_slot;
  1507. }
  1508. } else {
  1509. /* If this triggers then we have a memory corruption */
  1510. BUG();
  1511. }
  1512. out_check:
  1513. /*
  1514. * If nocow is false then record the beginning of the range
  1515. * that needs to be COWed
  1516. */
  1517. if (!nocow) {
  1518. if (cow_start == (u64)-1)
  1519. cow_start = cur_offset;
  1520. cur_offset = extent_end;
  1521. if (cur_offset > end)
  1522. break;
  1523. path->slots[0]++;
  1524. goto next_slot;
  1525. }
  1526. btrfs_release_path(path);
  1527. /*
  1528. * COW range from cow_start to found_key.offset - 1. As the key
  1529. * will contain the beginning of the first extent that can be
  1530. * NOCOW, following one which needs to be COW'ed
  1531. */
  1532. if (cow_start != (u64)-1) {
  1533. ret = fallback_to_cow(inode, locked_page,
  1534. cow_start, found_key.offset - 1,
  1535. page_started, nr_written);
  1536. if (ret)
  1537. goto error;
  1538. cow_start = (u64)-1;
  1539. }
  1540. if (extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  1541. u64 orig_start = found_key.offset - extent_offset;
  1542. struct extent_map *em;
  1543. em = create_io_em(inode, cur_offset, num_bytes,
  1544. orig_start,
  1545. disk_bytenr, /* block_start */
  1546. num_bytes, /* block_len */
  1547. disk_num_bytes, /* orig_block_len */
  1548. ram_bytes, BTRFS_COMPRESS_NONE,
  1549. BTRFS_ORDERED_PREALLOC);
  1550. if (IS_ERR(em)) {
  1551. ret = PTR_ERR(em);
  1552. goto error;
  1553. }
  1554. free_extent_map(em);
  1555. ret = btrfs_add_ordered_extent(inode, cur_offset,
  1556. disk_bytenr, num_bytes,
  1557. num_bytes,
  1558. BTRFS_ORDERED_PREALLOC);
  1559. if (ret) {
  1560. btrfs_drop_extent_cache(inode, cur_offset,
  1561. cur_offset + num_bytes - 1,
  1562. 0);
  1563. goto error;
  1564. }
  1565. } else {
  1566. ret = btrfs_add_ordered_extent(inode, cur_offset,
  1567. disk_bytenr, num_bytes,
  1568. num_bytes,
  1569. BTRFS_ORDERED_NOCOW);
  1570. if (ret)
  1571. goto error;
  1572. }
  1573. if (nocow)
  1574. btrfs_dec_nocow_writers(fs_info, disk_bytenr);
  1575. nocow = false;
  1576. if (root->root_key.objectid ==
  1577. BTRFS_DATA_RELOC_TREE_OBJECTID)
  1578. /*
  1579. * Error handled later, as we must prevent
  1580. * extent_clear_unlock_delalloc() in error handler
  1581. * from freeing metadata of created ordered extent.
  1582. */
  1583. ret = btrfs_reloc_clone_csums(inode, cur_offset,
  1584. num_bytes);
  1585. extent_clear_unlock_delalloc(inode, cur_offset,
  1586. cur_offset + num_bytes - 1,
  1587. locked_page, EXTENT_LOCKED |
  1588. EXTENT_DELALLOC |
  1589. EXTENT_CLEAR_DATA_RESV,
  1590. PAGE_UNLOCK | PAGE_SET_PRIVATE2);
  1591. cur_offset = extent_end;
  1592. /*
  1593. * btrfs_reloc_clone_csums() error, now we're OK to call error
  1594. * handler, as metadata for created ordered extent will only
  1595. * be freed by btrfs_finish_ordered_io().
  1596. */
  1597. if (ret)
  1598. goto error;
  1599. if (cur_offset > end)
  1600. break;
  1601. }
  1602. btrfs_release_path(path);
  1603. if (cur_offset <= end && cow_start == (u64)-1)
  1604. cow_start = cur_offset;
  1605. if (cow_start != (u64)-1) {
  1606. cur_offset = end;
  1607. ret = fallback_to_cow(inode, locked_page, cow_start, end,
  1608. page_started, nr_written);
  1609. if (ret)
  1610. goto error;
  1611. }
  1612. error:
  1613. if (nocow)
  1614. btrfs_dec_nocow_writers(fs_info, disk_bytenr);
  1615. if (ret && cur_offset < end)
  1616. extent_clear_unlock_delalloc(inode, cur_offset, end,
  1617. locked_page, EXTENT_LOCKED |
  1618. EXTENT_DELALLOC | EXTENT_DEFRAG |
  1619. EXTENT_DO_ACCOUNTING, PAGE_UNLOCK |
  1620. PAGE_CLEAR_DIRTY |
  1621. PAGE_SET_WRITEBACK |
  1622. PAGE_END_WRITEBACK);
  1623. btrfs_free_path(path);
  1624. return ret;
  1625. }
  1626. static inline int need_force_cow(struct btrfs_inode *inode, u64 start, u64 end)
  1627. {
  1628. if (!(inode->flags & BTRFS_INODE_NODATACOW) &&
  1629. !(inode->flags & BTRFS_INODE_PREALLOC))
  1630. return 0;
  1631. /*
  1632. * @defrag_bytes is a hint value, no spinlock held here,
  1633. * if is not zero, it means the file is defragging.
  1634. * Force cow if given extent needs to be defragged.
  1635. */
  1636. if (inode->defrag_bytes &&
  1637. test_range_bit(&inode->io_tree, start, end, EXTENT_DEFRAG, 0, NULL))
  1638. return 1;
  1639. return 0;
  1640. }
  1641. /*
  1642. * Function to process delayed allocation (create CoW) for ranges which are
  1643. * being touched for the first time.
  1644. */
  1645. int btrfs_run_delalloc_range(struct btrfs_inode *inode, struct page *locked_page,
  1646. u64 start, u64 end, int *page_started, unsigned long *nr_written,
  1647. struct writeback_control *wbc)
  1648. {
  1649. int ret;
  1650. int force_cow = need_force_cow(inode, start, end);
  1651. if (inode->flags & BTRFS_INODE_NODATACOW && !force_cow) {
  1652. ret = run_delalloc_nocow(inode, locked_page, start, end,
  1653. page_started, 1, nr_written);
  1654. } else if (inode->flags & BTRFS_INODE_PREALLOC && !force_cow) {
  1655. ret = run_delalloc_nocow(inode, locked_page, start, end,
  1656. page_started, 0, nr_written);
  1657. } else if (!inode_can_compress(inode) ||
  1658. !inode_need_compress(inode, start, end)) {
  1659. ret = cow_file_range(inode, locked_page, start, end,
  1660. page_started, nr_written, 1);
  1661. } else {
  1662. set_bit(BTRFS_INODE_HAS_ASYNC_EXTENT, &inode->runtime_flags);
  1663. ret = cow_file_range_async(inode, wbc, locked_page, start, end,
  1664. page_started, nr_written);
  1665. }
  1666. if (ret)
  1667. btrfs_cleanup_ordered_extents(inode, locked_page, start,
  1668. end - start + 1);
  1669. return ret;
  1670. }
  1671. void btrfs_split_delalloc_extent(struct inode *inode,
  1672. struct extent_state *orig, u64 split)
  1673. {
  1674. u64 size;
  1675. /* not delalloc, ignore it */
  1676. if (!(orig->state & EXTENT_DELALLOC))
  1677. return;
  1678. size = orig->end - orig->start + 1;
  1679. if (size > BTRFS_MAX_EXTENT_SIZE) {
  1680. u32 num_extents;
  1681. u64 new_size;
  1682. /*
  1683. * See the explanation in btrfs_merge_delalloc_extent, the same
  1684. * applies here, just in reverse.
  1685. */
  1686. new_size = orig->end - split + 1;
  1687. num_extents = count_max_extents(new_size);
  1688. new_size = split - orig->start;
  1689. num_extents += count_max_extents(new_size);
  1690. if (count_max_extents(size) >= num_extents)
  1691. return;
  1692. }
  1693. spin_lock(&BTRFS_I(inode)->lock);
  1694. btrfs_mod_outstanding_extents(BTRFS_I(inode), 1);
  1695. spin_unlock(&BTRFS_I(inode)->lock);
  1696. }
  1697. /*
  1698. * Handle merged delayed allocation extents so we can keep track of new extents
  1699. * that are just merged onto old extents, such as when we are doing sequential
  1700. * writes, so we can properly account for the metadata space we'll need.
  1701. */
  1702. void btrfs_merge_delalloc_extent(struct inode *inode, struct extent_state *new,
  1703. struct extent_state *other)
  1704. {
  1705. u64 new_size, old_size;
  1706. u32 num_extents;
  1707. /* not delalloc, ignore it */
  1708. if (!(other->state & EXTENT_DELALLOC))
  1709. return;
  1710. if (new->start > other->start)
  1711. new_size = new->end - other->start + 1;
  1712. else
  1713. new_size = other->end - new->start + 1;
  1714. /* we're not bigger than the max, unreserve the space and go */
  1715. if (new_size <= BTRFS_MAX_EXTENT_SIZE) {
  1716. spin_lock(&BTRFS_I(inode)->lock);
  1717. btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
  1718. spin_unlock(&BTRFS_I(inode)->lock);
  1719. return;
  1720. }
  1721. /*
  1722. * We have to add up either side to figure out how many extents were
  1723. * accounted for before we merged into one big extent. If the number of
  1724. * extents we accounted for is <= the amount we need for the new range
  1725. * then we can return, otherwise drop. Think of it like this
  1726. *
  1727. * [ 4k][MAX_SIZE]
  1728. *
  1729. * So we've grown the extent by a MAX_SIZE extent, this would mean we
  1730. * need 2 outstanding extents, on one side we have 1 and the other side
  1731. * we have 1 so they are == and we can return. But in this case
  1732. *
  1733. * [MAX_SIZE+4k][MAX_SIZE+4k]
  1734. *
  1735. * Each range on their own accounts for 2 extents, but merged together
  1736. * they are only 3 extents worth of accounting, so we need to drop in
  1737. * this case.
  1738. */
  1739. old_size = other->end - other->start + 1;
  1740. num_extents = count_max_extents(old_size);
  1741. old_size = new->end - new->start + 1;
  1742. num_extents += count_max_extents(old_size);
  1743. if (count_max_extents(new_size) >= num_extents)
  1744. return;
  1745. spin_lock(&BTRFS_I(inode)->lock);
  1746. btrfs_mod_outstanding_extents(BTRFS_I(inode), -1);
  1747. spin_unlock(&BTRFS_I(inode)->lock);
  1748. }
  1749. static void btrfs_add_delalloc_inodes(struct btrfs_root *root,
  1750. struct inode *inode)
  1751. {
  1752. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1753. spin_lock(&root->delalloc_lock);
  1754. if (list_empty(&BTRFS_I(inode)->delalloc_inodes)) {
  1755. list_add_tail(&BTRFS_I(inode)->delalloc_inodes,
  1756. &root->delalloc_inodes);
  1757. set_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1758. &BTRFS_I(inode)->runtime_flags);
  1759. root->nr_delalloc_inodes++;
  1760. if (root->nr_delalloc_inodes == 1) {
  1761. spin_lock(&fs_info->delalloc_root_lock);
  1762. BUG_ON(!list_empty(&root->delalloc_root));
  1763. list_add_tail(&root->delalloc_root,
  1764. &fs_info->delalloc_roots);
  1765. spin_unlock(&fs_info->delalloc_root_lock);
  1766. }
  1767. }
  1768. spin_unlock(&root->delalloc_lock);
  1769. }
  1770. void __btrfs_del_delalloc_inode(struct btrfs_root *root,
  1771. struct btrfs_inode *inode)
  1772. {
  1773. struct btrfs_fs_info *fs_info = root->fs_info;
  1774. if (!list_empty(&inode->delalloc_inodes)) {
  1775. list_del_init(&inode->delalloc_inodes);
  1776. clear_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1777. &inode->runtime_flags);
  1778. root->nr_delalloc_inodes--;
  1779. if (!root->nr_delalloc_inodes) {
  1780. ASSERT(list_empty(&root->delalloc_inodes));
  1781. spin_lock(&fs_info->delalloc_root_lock);
  1782. BUG_ON(list_empty(&root->delalloc_root));
  1783. list_del_init(&root->delalloc_root);
  1784. spin_unlock(&fs_info->delalloc_root_lock);
  1785. }
  1786. }
  1787. }
  1788. static void btrfs_del_delalloc_inode(struct btrfs_root *root,
  1789. struct btrfs_inode *inode)
  1790. {
  1791. spin_lock(&root->delalloc_lock);
  1792. __btrfs_del_delalloc_inode(root, inode);
  1793. spin_unlock(&root->delalloc_lock);
  1794. }
  1795. /*
  1796. * Properly track delayed allocation bytes in the inode and to maintain the
  1797. * list of inodes that have pending delalloc work to be done.
  1798. */
  1799. void btrfs_set_delalloc_extent(struct inode *inode, struct extent_state *state,
  1800. unsigned *bits)
  1801. {
  1802. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1803. if ((*bits & EXTENT_DEFRAG) && !(*bits & EXTENT_DELALLOC))
  1804. WARN_ON(1);
  1805. /*
  1806. * set_bit and clear bit hooks normally require _irqsave/restore
  1807. * but in this case, we are only testing for the DELALLOC
  1808. * bit, which is only set or cleared with irqs on
  1809. */
  1810. if (!(state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
  1811. struct btrfs_root *root = BTRFS_I(inode)->root;
  1812. u64 len = state->end + 1 - state->start;
  1813. u32 num_extents = count_max_extents(len);
  1814. bool do_list = !btrfs_is_free_space_inode(BTRFS_I(inode));
  1815. spin_lock(&BTRFS_I(inode)->lock);
  1816. btrfs_mod_outstanding_extents(BTRFS_I(inode), num_extents);
  1817. spin_unlock(&BTRFS_I(inode)->lock);
  1818. /* For sanity tests */
  1819. if (btrfs_is_testing(fs_info))
  1820. return;
  1821. percpu_counter_add_batch(&fs_info->delalloc_bytes, len,
  1822. fs_info->delalloc_batch);
  1823. spin_lock(&BTRFS_I(inode)->lock);
  1824. BTRFS_I(inode)->delalloc_bytes += len;
  1825. if (*bits & EXTENT_DEFRAG)
  1826. BTRFS_I(inode)->defrag_bytes += len;
  1827. if (do_list && !test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1828. &BTRFS_I(inode)->runtime_flags))
  1829. btrfs_add_delalloc_inodes(root, inode);
  1830. spin_unlock(&BTRFS_I(inode)->lock);
  1831. }
  1832. if (!(state->state & EXTENT_DELALLOC_NEW) &&
  1833. (*bits & EXTENT_DELALLOC_NEW)) {
  1834. spin_lock(&BTRFS_I(inode)->lock);
  1835. BTRFS_I(inode)->new_delalloc_bytes += state->end + 1 -
  1836. state->start;
  1837. spin_unlock(&BTRFS_I(inode)->lock);
  1838. }
  1839. }
  1840. /*
  1841. * Once a range is no longer delalloc this function ensures that proper
  1842. * accounting happens.
  1843. */
  1844. void btrfs_clear_delalloc_extent(struct inode *vfs_inode,
  1845. struct extent_state *state, unsigned *bits)
  1846. {
  1847. struct btrfs_inode *inode = BTRFS_I(vfs_inode);
  1848. struct btrfs_fs_info *fs_info = btrfs_sb(vfs_inode->i_sb);
  1849. u64 len = state->end + 1 - state->start;
  1850. u32 num_extents = count_max_extents(len);
  1851. if ((state->state & EXTENT_DEFRAG) && (*bits & EXTENT_DEFRAG)) {
  1852. spin_lock(&inode->lock);
  1853. inode->defrag_bytes -= len;
  1854. spin_unlock(&inode->lock);
  1855. }
  1856. /*
  1857. * set_bit and clear bit hooks normally require _irqsave/restore
  1858. * but in this case, we are only testing for the DELALLOC
  1859. * bit, which is only set or cleared with irqs on
  1860. */
  1861. if ((state->state & EXTENT_DELALLOC) && (*bits & EXTENT_DELALLOC)) {
  1862. struct btrfs_root *root = inode->root;
  1863. bool do_list = !btrfs_is_free_space_inode(inode);
  1864. spin_lock(&inode->lock);
  1865. btrfs_mod_outstanding_extents(inode, -num_extents);
  1866. spin_unlock(&inode->lock);
  1867. /*
  1868. * We don't reserve metadata space for space cache inodes so we
  1869. * don't need to call delalloc_release_metadata if there is an
  1870. * error.
  1871. */
  1872. if (*bits & EXTENT_CLEAR_META_RESV &&
  1873. root != fs_info->tree_root)
  1874. btrfs_delalloc_release_metadata(inode, len, false);
  1875. /* For sanity tests. */
  1876. if (btrfs_is_testing(fs_info))
  1877. return;
  1878. if (root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID &&
  1879. do_list && !(state->state & EXTENT_NORESERVE) &&
  1880. (*bits & EXTENT_CLEAR_DATA_RESV))
  1881. btrfs_free_reserved_data_space_noquota(fs_info, len);
  1882. percpu_counter_add_batch(&fs_info->delalloc_bytes, -len,
  1883. fs_info->delalloc_batch);
  1884. spin_lock(&inode->lock);
  1885. inode->delalloc_bytes -= len;
  1886. if (do_list && inode->delalloc_bytes == 0 &&
  1887. test_bit(BTRFS_INODE_IN_DELALLOC_LIST,
  1888. &inode->runtime_flags))
  1889. btrfs_del_delalloc_inode(root, inode);
  1890. spin_unlock(&inode->lock);
  1891. }
  1892. if ((state->state & EXTENT_DELALLOC_NEW) &&
  1893. (*bits & EXTENT_DELALLOC_NEW)) {
  1894. spin_lock(&inode->lock);
  1895. ASSERT(inode->new_delalloc_bytes >= len);
  1896. inode->new_delalloc_bytes -= len;
  1897. spin_unlock(&inode->lock);
  1898. }
  1899. }
  1900. /*
  1901. * btrfs_bio_fits_in_stripe - Checks whether the size of the given bio will fit
  1902. * in a chunk's stripe. This function ensures that bios do not span a
  1903. * stripe/chunk
  1904. *
  1905. * @page - The page we are about to add to the bio
  1906. * @size - size we want to add to the bio
  1907. * @bio - bio we want to ensure is smaller than a stripe
  1908. * @bio_flags - flags of the bio
  1909. *
  1910. * return 1 if page cannot be added to the bio
  1911. * return 0 if page can be added to the bio
  1912. * return error otherwise
  1913. */
  1914. int btrfs_bio_fits_in_stripe(struct page *page, size_t size, struct bio *bio,
  1915. unsigned long bio_flags)
  1916. {
  1917. struct inode *inode = page->mapping->host;
  1918. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1919. u64 logical = (u64)bio->bi_iter.bi_sector << 9;
  1920. u64 length = 0;
  1921. u64 map_length;
  1922. int ret;
  1923. struct btrfs_io_geometry geom;
  1924. if (bio_flags & EXTENT_BIO_COMPRESSED)
  1925. return 0;
  1926. length = bio->bi_iter.bi_size;
  1927. map_length = length;
  1928. ret = btrfs_get_io_geometry(fs_info, btrfs_op(bio), logical, map_length,
  1929. &geom);
  1930. if (ret < 0)
  1931. return ret;
  1932. if (geom.len < length + size)
  1933. return 1;
  1934. return 0;
  1935. }
  1936. /*
  1937. * in order to insert checksums into the metadata in large chunks,
  1938. * we wait until bio submission time. All the pages in the bio are
  1939. * checksummed and sums are attached onto the ordered extent record.
  1940. *
  1941. * At IO completion time the cums attached on the ordered extent record
  1942. * are inserted into the btree
  1943. */
  1944. static blk_status_t btrfs_submit_bio_start(void *private_data, struct bio *bio,
  1945. u64 bio_offset)
  1946. {
  1947. struct inode *inode = private_data;
  1948. return btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0);
  1949. }
  1950. /*
  1951. * extent_io.c submission hook. This does the right thing for csum calculation
  1952. * on write, or reading the csums from the tree before a read.
  1953. *
  1954. * Rules about async/sync submit,
  1955. * a) read: sync submit
  1956. *
  1957. * b) write without checksum: sync submit
  1958. *
  1959. * c) write with checksum:
  1960. * c-1) if bio is issued by fsync: sync submit
  1961. * (sync_writers != 0)
  1962. *
  1963. * c-2) if root is reloc root: sync submit
  1964. * (only in case of buffered IO)
  1965. *
  1966. * c-3) otherwise: async submit
  1967. */
  1968. blk_status_t btrfs_submit_data_bio(struct inode *inode, struct bio *bio,
  1969. int mirror_num, unsigned long bio_flags)
  1970. {
  1971. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  1972. struct btrfs_root *root = BTRFS_I(inode)->root;
  1973. enum btrfs_wq_endio_type metadata = BTRFS_WQ_ENDIO_DATA;
  1974. blk_status_t ret = 0;
  1975. int skip_sum;
  1976. int async = !atomic_read(&BTRFS_I(inode)->sync_writers);
  1977. skip_sum = BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM;
  1978. if (btrfs_is_free_space_inode(BTRFS_I(inode)))
  1979. metadata = BTRFS_WQ_ENDIO_FREE_SPACE;
  1980. if (bio_op(bio) != REQ_OP_WRITE) {
  1981. ret = btrfs_bio_wq_end_io(fs_info, bio, metadata);
  1982. if (ret)
  1983. goto out;
  1984. if (bio_flags & EXTENT_BIO_COMPRESSED) {
  1985. ret = btrfs_submit_compressed_read(inode, bio,
  1986. mirror_num,
  1987. bio_flags);
  1988. goto out;
  1989. } else if (!skip_sum) {
  1990. ret = btrfs_lookup_bio_sums(inode, bio, (u64)-1, NULL);
  1991. if (ret)
  1992. goto out;
  1993. }
  1994. goto mapit;
  1995. } else if (async && !skip_sum) {
  1996. /* csum items have already been cloned */
  1997. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID)
  1998. goto mapit;
  1999. /* we're doing a write, do the async checksumming */
  2000. ret = btrfs_wq_submit_bio(fs_info, bio, mirror_num, bio_flags,
  2001. 0, inode, btrfs_submit_bio_start);
  2002. goto out;
  2003. } else if (!skip_sum) {
  2004. ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, 0, 0);
  2005. if (ret)
  2006. goto out;
  2007. }
  2008. mapit:
  2009. ret = btrfs_map_bio(fs_info, bio, mirror_num);
  2010. out:
  2011. if (ret) {
  2012. bio->bi_status = ret;
  2013. bio_endio(bio);
  2014. }
  2015. return ret;
  2016. }
  2017. /*
  2018. * given a list of ordered sums record them in the inode. This happens
  2019. * at IO completion time based on sums calculated at bio submission time.
  2020. */
  2021. static int add_pending_csums(struct btrfs_trans_handle *trans,
  2022. struct list_head *list)
  2023. {
  2024. struct btrfs_ordered_sum *sum;
  2025. int ret;
  2026. list_for_each_entry(sum, list, list) {
  2027. trans->adding_csums = true;
  2028. ret = btrfs_csum_file_blocks(trans, trans->fs_info->csum_root, sum);
  2029. trans->adding_csums = false;
  2030. if (ret)
  2031. return ret;
  2032. }
  2033. return 0;
  2034. }
  2035. static int btrfs_find_new_delalloc_bytes(struct btrfs_inode *inode,
  2036. const u64 start,
  2037. const u64 len,
  2038. struct extent_state **cached_state)
  2039. {
  2040. u64 search_start = start;
  2041. const u64 end = start + len - 1;
  2042. while (search_start < end) {
  2043. const u64 search_len = end - search_start + 1;
  2044. struct extent_map *em;
  2045. u64 em_len;
  2046. int ret = 0;
  2047. em = btrfs_get_extent(inode, NULL, 0, search_start, search_len);
  2048. if (IS_ERR(em))
  2049. return PTR_ERR(em);
  2050. if (em->block_start != EXTENT_MAP_HOLE)
  2051. goto next;
  2052. em_len = em->len;
  2053. if (em->start < search_start)
  2054. em_len -= search_start - em->start;
  2055. if (em_len > search_len)
  2056. em_len = search_len;
  2057. ret = set_extent_bit(&inode->io_tree, search_start,
  2058. search_start + em_len - 1,
  2059. EXTENT_DELALLOC_NEW,
  2060. NULL, cached_state, GFP_NOFS);
  2061. next:
  2062. search_start = extent_map_end(em);
  2063. free_extent_map(em);
  2064. if (ret)
  2065. return ret;
  2066. }
  2067. return 0;
  2068. }
  2069. int btrfs_set_extent_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
  2070. unsigned int extra_bits,
  2071. struct extent_state **cached_state)
  2072. {
  2073. WARN_ON(PAGE_ALIGNED(end));
  2074. if (start >= i_size_read(&inode->vfs_inode) &&
  2075. !(inode->flags & BTRFS_INODE_PREALLOC)) {
  2076. /*
  2077. * There can't be any extents following eof in this case so just
  2078. * set the delalloc new bit for the range directly.
  2079. */
  2080. extra_bits |= EXTENT_DELALLOC_NEW;
  2081. } else {
  2082. int ret;
  2083. ret = btrfs_find_new_delalloc_bytes(inode, start,
  2084. end + 1 - start,
  2085. cached_state);
  2086. if (ret)
  2087. return ret;
  2088. }
  2089. return set_extent_delalloc(&inode->io_tree, start, end, extra_bits,
  2090. cached_state);
  2091. }
  2092. /* see btrfs_writepage_start_hook for details on why this is required */
  2093. struct btrfs_writepage_fixup {
  2094. struct page *page;
  2095. struct inode *inode;
  2096. struct btrfs_work work;
  2097. };
  2098. static void btrfs_writepage_fixup_worker(struct btrfs_work *work)
  2099. {
  2100. struct btrfs_writepage_fixup *fixup;
  2101. struct btrfs_ordered_extent *ordered;
  2102. struct extent_state *cached_state = NULL;
  2103. struct extent_changeset *data_reserved = NULL;
  2104. struct page *page;
  2105. struct btrfs_inode *inode;
  2106. u64 page_start;
  2107. u64 page_end;
  2108. int ret = 0;
  2109. bool free_delalloc_space = true;
  2110. fixup = container_of(work, struct btrfs_writepage_fixup, work);
  2111. page = fixup->page;
  2112. inode = BTRFS_I(fixup->inode);
  2113. page_start = page_offset(page);
  2114. page_end = page_offset(page) + PAGE_SIZE - 1;
  2115. /*
  2116. * This is similar to page_mkwrite, we need to reserve the space before
  2117. * we take the page lock.
  2118. */
  2119. ret = btrfs_delalloc_reserve_space(inode, &data_reserved, page_start,
  2120. PAGE_SIZE);
  2121. again:
  2122. lock_page(page);
  2123. /*
  2124. * Before we queued this fixup, we took a reference on the page.
  2125. * page->mapping may go NULL, but it shouldn't be moved to a different
  2126. * address space.
  2127. */
  2128. if (!page->mapping || !PageDirty(page) || !PageChecked(page)) {
  2129. /*
  2130. * Unfortunately this is a little tricky, either
  2131. *
  2132. * 1) We got here and our page had already been dealt with and
  2133. * we reserved our space, thus ret == 0, so we need to just
  2134. * drop our space reservation and bail. This can happen the
  2135. * first time we come into the fixup worker, or could happen
  2136. * while waiting for the ordered extent.
  2137. * 2) Our page was already dealt with, but we happened to get an
  2138. * ENOSPC above from the btrfs_delalloc_reserve_space. In
  2139. * this case we obviously don't have anything to release, but
  2140. * because the page was already dealt with we don't want to
  2141. * mark the page with an error, so make sure we're resetting
  2142. * ret to 0. This is why we have this check _before_ the ret
  2143. * check, because we do not want to have a surprise ENOSPC
  2144. * when the page was already properly dealt with.
  2145. */
  2146. if (!ret) {
  2147. btrfs_delalloc_release_extents(inode, PAGE_SIZE);
  2148. btrfs_delalloc_release_space(inode, data_reserved,
  2149. page_start, PAGE_SIZE,
  2150. true);
  2151. }
  2152. ret = 0;
  2153. goto out_page;
  2154. }
  2155. /*
  2156. * We can't mess with the page state unless it is locked, so now that
  2157. * it is locked bail if we failed to make our space reservation.
  2158. */
  2159. if (ret)
  2160. goto out_page;
  2161. lock_extent_bits(&inode->io_tree, page_start, page_end, &cached_state);
  2162. /* already ordered? We're done */
  2163. if (PagePrivate2(page))
  2164. goto out_reserved;
  2165. ordered = btrfs_lookup_ordered_range(inode, page_start, PAGE_SIZE);
  2166. if (ordered) {
  2167. unlock_extent_cached(&inode->io_tree, page_start, page_end,
  2168. &cached_state);
  2169. unlock_page(page);
  2170. btrfs_start_ordered_extent(ordered, 1);
  2171. btrfs_put_ordered_extent(ordered);
  2172. goto again;
  2173. }
  2174. ret = btrfs_set_extent_delalloc(inode, page_start, page_end, 0,
  2175. &cached_state);
  2176. if (ret)
  2177. goto out_reserved;
  2178. /*
  2179. * Everything went as planned, we're now the owner of a dirty page with
  2180. * delayed allocation bits set and space reserved for our COW
  2181. * destination.
  2182. *
  2183. * The page was dirty when we started, nothing should have cleaned it.
  2184. */
  2185. BUG_ON(!PageDirty(page));
  2186. free_delalloc_space = false;
  2187. out_reserved:
  2188. btrfs_delalloc_release_extents(inode, PAGE_SIZE);
  2189. if (free_delalloc_space)
  2190. btrfs_delalloc_release_space(inode, data_reserved, page_start,
  2191. PAGE_SIZE, true);
  2192. unlock_extent_cached(&inode->io_tree, page_start, page_end,
  2193. &cached_state);
  2194. out_page:
  2195. if (ret) {
  2196. /*
  2197. * We hit ENOSPC or other errors. Update the mapping and page
  2198. * to reflect the errors and clean the page.
  2199. */
  2200. mapping_set_error(page->mapping, ret);
  2201. end_extent_writepage(page, ret, page_start, page_end);
  2202. clear_page_dirty_for_io(page);
  2203. SetPageError(page);
  2204. }
  2205. ClearPageChecked(page);
  2206. unlock_page(page);
  2207. put_page(page);
  2208. kfree(fixup);
  2209. extent_changeset_free(data_reserved);
  2210. /*
  2211. * As a precaution, do a delayed iput in case it would be the last iput
  2212. * that could need flushing space. Recursing back to fixup worker would
  2213. * deadlock.
  2214. */
  2215. btrfs_add_delayed_iput(&inode->vfs_inode);
  2216. }
  2217. /*
  2218. * There are a few paths in the higher layers of the kernel that directly
  2219. * set the page dirty bit without asking the filesystem if it is a
  2220. * good idea. This causes problems because we want to make sure COW
  2221. * properly happens and the data=ordered rules are followed.
  2222. *
  2223. * In our case any range that doesn't have the ORDERED bit set
  2224. * hasn't been properly setup for IO. We kick off an async process
  2225. * to fix it up. The async helper will wait for ordered extents, set
  2226. * the delalloc bit and make it safe to write the page.
  2227. */
  2228. int btrfs_writepage_cow_fixup(struct page *page, u64 start, u64 end)
  2229. {
  2230. struct inode *inode = page->mapping->host;
  2231. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2232. struct btrfs_writepage_fixup *fixup;
  2233. /* this page is properly in the ordered list */
  2234. if (TestClearPagePrivate2(page))
  2235. return 0;
  2236. /*
  2237. * PageChecked is set below when we create a fixup worker for this page,
  2238. * don't try to create another one if we're already PageChecked()
  2239. *
  2240. * The extent_io writepage code will redirty the page if we send back
  2241. * EAGAIN.
  2242. */
  2243. if (PageChecked(page))
  2244. return -EAGAIN;
  2245. fixup = kzalloc(sizeof(*fixup), GFP_NOFS);
  2246. if (!fixup)
  2247. return -EAGAIN;
  2248. /*
  2249. * We are already holding a reference to this inode from
  2250. * write_cache_pages. We need to hold it because the space reservation
  2251. * takes place outside of the page lock, and we can't trust
  2252. * page->mapping outside of the page lock.
  2253. */
  2254. ihold(inode);
  2255. SetPageChecked(page);
  2256. get_page(page);
  2257. btrfs_init_work(&fixup->work, btrfs_writepage_fixup_worker, NULL, NULL);
  2258. fixup->page = page;
  2259. fixup->inode = inode;
  2260. btrfs_queue_work(fs_info->fixup_workers, &fixup->work);
  2261. return -EAGAIN;
  2262. }
  2263. static int insert_reserved_file_extent(struct btrfs_trans_handle *trans,
  2264. struct btrfs_inode *inode, u64 file_pos,
  2265. struct btrfs_file_extent_item *stack_fi,
  2266. u64 qgroup_reserved)
  2267. {
  2268. struct btrfs_root *root = inode->root;
  2269. struct btrfs_path *path;
  2270. struct extent_buffer *leaf;
  2271. struct btrfs_key ins;
  2272. u64 disk_num_bytes = btrfs_stack_file_extent_disk_num_bytes(stack_fi);
  2273. u64 disk_bytenr = btrfs_stack_file_extent_disk_bytenr(stack_fi);
  2274. u64 num_bytes = btrfs_stack_file_extent_num_bytes(stack_fi);
  2275. u64 ram_bytes = btrfs_stack_file_extent_ram_bytes(stack_fi);
  2276. int extent_inserted = 0;
  2277. int ret;
  2278. path = btrfs_alloc_path();
  2279. if (!path)
  2280. return -ENOMEM;
  2281. /*
  2282. * we may be replacing one extent in the tree with another.
  2283. * The new extent is pinned in the extent map, and we don't want
  2284. * to drop it from the cache until it is completely in the btree.
  2285. *
  2286. * So, tell btrfs_drop_extents to leave this extent in the cache.
  2287. * the caller is expected to unpin it and allow it to be merged
  2288. * with the others.
  2289. */
  2290. ret = __btrfs_drop_extents(trans, root, inode, path, file_pos,
  2291. file_pos + num_bytes, NULL, 0,
  2292. 1, sizeof(*stack_fi), &extent_inserted);
  2293. if (ret)
  2294. goto out;
  2295. if (!extent_inserted) {
  2296. ins.objectid = btrfs_ino(inode);
  2297. ins.offset = file_pos;
  2298. ins.type = BTRFS_EXTENT_DATA_KEY;
  2299. path->leave_spinning = 1;
  2300. ret = btrfs_insert_empty_item(trans, root, path, &ins,
  2301. sizeof(*stack_fi));
  2302. if (ret)
  2303. goto out;
  2304. }
  2305. leaf = path->nodes[0];
  2306. btrfs_set_stack_file_extent_generation(stack_fi, trans->transid);
  2307. write_extent_buffer(leaf, stack_fi,
  2308. btrfs_item_ptr_offset(leaf, path->slots[0]),
  2309. sizeof(struct btrfs_file_extent_item));
  2310. btrfs_mark_buffer_dirty(leaf);
  2311. btrfs_release_path(path);
  2312. inode_add_bytes(&inode->vfs_inode, num_bytes);
  2313. ins.objectid = disk_bytenr;
  2314. ins.offset = disk_num_bytes;
  2315. ins.type = BTRFS_EXTENT_ITEM_KEY;
  2316. ret = btrfs_inode_set_file_extent_range(inode, file_pos, ram_bytes);
  2317. if (ret)
  2318. goto out;
  2319. ret = btrfs_alloc_reserved_file_extent(trans, root, btrfs_ino(inode),
  2320. file_pos, qgroup_reserved, &ins);
  2321. out:
  2322. btrfs_free_path(path);
  2323. return ret;
  2324. }
  2325. static void btrfs_release_delalloc_bytes(struct btrfs_fs_info *fs_info,
  2326. u64 start, u64 len)
  2327. {
  2328. struct btrfs_block_group *cache;
  2329. cache = btrfs_lookup_block_group(fs_info, start);
  2330. ASSERT(cache);
  2331. spin_lock(&cache->lock);
  2332. cache->delalloc_bytes -= len;
  2333. spin_unlock(&cache->lock);
  2334. btrfs_put_block_group(cache);
  2335. }
  2336. static int insert_ordered_extent_file_extent(struct btrfs_trans_handle *trans,
  2337. struct btrfs_ordered_extent *oe)
  2338. {
  2339. struct btrfs_file_extent_item stack_fi;
  2340. u64 logical_len;
  2341. memset(&stack_fi, 0, sizeof(stack_fi));
  2342. btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_REG);
  2343. btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, oe->disk_bytenr);
  2344. btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi,
  2345. oe->disk_num_bytes);
  2346. if (test_bit(BTRFS_ORDERED_TRUNCATED, &oe->flags))
  2347. logical_len = oe->truncated_len;
  2348. else
  2349. logical_len = oe->num_bytes;
  2350. btrfs_set_stack_file_extent_num_bytes(&stack_fi, logical_len);
  2351. btrfs_set_stack_file_extent_ram_bytes(&stack_fi, logical_len);
  2352. btrfs_set_stack_file_extent_compression(&stack_fi, oe->compress_type);
  2353. /* Encryption and other encoding is reserved and all 0 */
  2354. return insert_reserved_file_extent(trans, BTRFS_I(oe->inode),
  2355. oe->file_offset, &stack_fi,
  2356. oe->qgroup_rsv);
  2357. }
  2358. /*
  2359. * As ordered data IO finishes, this gets called so we can finish
  2360. * an ordered extent if the range of bytes in the file it covers are
  2361. * fully written.
  2362. */
  2363. static int btrfs_finish_ordered_io(struct btrfs_ordered_extent *ordered_extent)
  2364. {
  2365. struct inode *inode = ordered_extent->inode;
  2366. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2367. struct btrfs_root *root = BTRFS_I(inode)->root;
  2368. struct btrfs_trans_handle *trans = NULL;
  2369. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  2370. struct extent_state *cached_state = NULL;
  2371. u64 start, end;
  2372. int compress_type = 0;
  2373. int ret = 0;
  2374. u64 logical_len = ordered_extent->num_bytes;
  2375. bool freespace_inode;
  2376. bool truncated = false;
  2377. bool range_locked = false;
  2378. bool clear_new_delalloc_bytes = false;
  2379. bool clear_reserved_extent = true;
  2380. unsigned int clear_bits;
  2381. start = ordered_extent->file_offset;
  2382. end = start + ordered_extent->num_bytes - 1;
  2383. if (!test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
  2384. !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags) &&
  2385. !test_bit(BTRFS_ORDERED_DIRECT, &ordered_extent->flags))
  2386. clear_new_delalloc_bytes = true;
  2387. freespace_inode = btrfs_is_free_space_inode(BTRFS_I(inode));
  2388. if (test_bit(BTRFS_ORDERED_IOERR, &ordered_extent->flags)) {
  2389. ret = -EIO;
  2390. goto out;
  2391. }
  2392. btrfs_free_io_failure_record(BTRFS_I(inode), start, end);
  2393. if (test_bit(BTRFS_ORDERED_TRUNCATED, &ordered_extent->flags)) {
  2394. truncated = true;
  2395. logical_len = ordered_extent->truncated_len;
  2396. /* Truncated the entire extent, don't bother adding */
  2397. if (!logical_len)
  2398. goto out;
  2399. }
  2400. if (test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags)) {
  2401. BUG_ON(!list_empty(&ordered_extent->list)); /* Logic error */
  2402. btrfs_inode_safe_disk_i_size_write(inode, 0);
  2403. if (freespace_inode)
  2404. trans = btrfs_join_transaction_spacecache(root);
  2405. else
  2406. trans = btrfs_join_transaction(root);
  2407. if (IS_ERR(trans)) {
  2408. ret = PTR_ERR(trans);
  2409. trans = NULL;
  2410. goto out;
  2411. }
  2412. trans->block_rsv = &BTRFS_I(inode)->block_rsv;
  2413. ret = btrfs_update_inode_fallback(trans, root, inode);
  2414. if (ret) /* -ENOMEM or corruption */
  2415. btrfs_abort_transaction(trans, ret);
  2416. goto out;
  2417. }
  2418. range_locked = true;
  2419. lock_extent_bits(io_tree, start, end, &cached_state);
  2420. if (freespace_inode)
  2421. trans = btrfs_join_transaction_spacecache(root);
  2422. else
  2423. trans = btrfs_join_transaction(root);
  2424. if (IS_ERR(trans)) {
  2425. ret = PTR_ERR(trans);
  2426. trans = NULL;
  2427. goto out;
  2428. }
  2429. trans->block_rsv = &BTRFS_I(inode)->block_rsv;
  2430. if (test_bit(BTRFS_ORDERED_COMPRESSED, &ordered_extent->flags))
  2431. compress_type = ordered_extent->compress_type;
  2432. if (test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
  2433. BUG_ON(compress_type);
  2434. ret = btrfs_mark_extent_written(trans, BTRFS_I(inode),
  2435. ordered_extent->file_offset,
  2436. ordered_extent->file_offset +
  2437. logical_len);
  2438. } else {
  2439. BUG_ON(root == fs_info->tree_root);
  2440. ret = insert_ordered_extent_file_extent(trans, ordered_extent);
  2441. if (!ret) {
  2442. clear_reserved_extent = false;
  2443. btrfs_release_delalloc_bytes(fs_info,
  2444. ordered_extent->disk_bytenr,
  2445. ordered_extent->disk_num_bytes);
  2446. }
  2447. }
  2448. unpin_extent_cache(&BTRFS_I(inode)->extent_tree,
  2449. ordered_extent->file_offset,
  2450. ordered_extent->num_bytes, trans->transid);
  2451. if (ret < 0) {
  2452. btrfs_abort_transaction(trans, ret);
  2453. goto out;
  2454. }
  2455. ret = add_pending_csums(trans, &ordered_extent->list);
  2456. if (ret) {
  2457. btrfs_abort_transaction(trans, ret);
  2458. goto out;
  2459. }
  2460. btrfs_inode_safe_disk_i_size_write(inode, 0);
  2461. ret = btrfs_update_inode_fallback(trans, root, inode);
  2462. if (ret) { /* -ENOMEM or corruption */
  2463. btrfs_abort_transaction(trans, ret);
  2464. goto out;
  2465. }
  2466. ret = 0;
  2467. out:
  2468. clear_bits = EXTENT_DEFRAG;
  2469. if (range_locked)
  2470. clear_bits |= EXTENT_LOCKED;
  2471. if (clear_new_delalloc_bytes)
  2472. clear_bits |= EXTENT_DELALLOC_NEW;
  2473. clear_extent_bit(&BTRFS_I(inode)->io_tree, start, end, clear_bits,
  2474. (clear_bits & EXTENT_LOCKED) ? 1 : 0, 0,
  2475. &cached_state);
  2476. if (trans)
  2477. btrfs_end_transaction(trans);
  2478. if (ret || truncated) {
  2479. u64 unwritten_start = start;
  2480. /*
  2481. * If we failed to finish this ordered extent for any reason we
  2482. * need to make sure BTRFS_ORDERED_IOERR is set on the ordered
  2483. * extent, and mark the inode with the error if it wasn't
  2484. * already set. Any error during writeback would have already
  2485. * set the mapping error, so we need to set it if we're the ones
  2486. * marking this ordered extent as failed.
  2487. */
  2488. if (ret && !test_and_set_bit(BTRFS_ORDERED_IOERR,
  2489. &ordered_extent->flags))
  2490. mapping_set_error(ordered_extent->inode->i_mapping, -EIO);
  2491. if (truncated)
  2492. unwritten_start += logical_len;
  2493. clear_extent_uptodate(io_tree, unwritten_start, end, NULL);
  2494. /* Drop the cache for the part of the extent we didn't write. */
  2495. btrfs_drop_extent_cache(BTRFS_I(inode), unwritten_start, end, 0);
  2496. /*
  2497. * If the ordered extent had an IOERR or something else went
  2498. * wrong we need to return the space for this ordered extent
  2499. * back to the allocator. We only free the extent in the
  2500. * truncated case if we didn't write out the extent at all.
  2501. *
  2502. * If we made it past insert_reserved_file_extent before we
  2503. * errored out then we don't need to do this as the accounting
  2504. * has already been done.
  2505. */
  2506. if ((ret || !logical_len) &&
  2507. clear_reserved_extent &&
  2508. !test_bit(BTRFS_ORDERED_NOCOW, &ordered_extent->flags) &&
  2509. !test_bit(BTRFS_ORDERED_PREALLOC, &ordered_extent->flags)) {
  2510. /*
  2511. * Discard the range before returning it back to the
  2512. * free space pool
  2513. */
  2514. if (ret && btrfs_test_opt(fs_info, DISCARD_SYNC))
  2515. btrfs_discard_extent(fs_info,
  2516. ordered_extent->disk_bytenr,
  2517. ordered_extent->disk_num_bytes,
  2518. NULL);
  2519. btrfs_free_reserved_extent(fs_info,
  2520. ordered_extent->disk_bytenr,
  2521. ordered_extent->disk_num_bytes, 1);
  2522. }
  2523. }
  2524. /*
  2525. * This needs to be done to make sure anybody waiting knows we are done
  2526. * updating everything for this ordered extent.
  2527. */
  2528. btrfs_remove_ordered_extent(BTRFS_I(inode), ordered_extent);
  2529. /* once for us */
  2530. btrfs_put_ordered_extent(ordered_extent);
  2531. /* once for the tree */
  2532. btrfs_put_ordered_extent(ordered_extent);
  2533. return ret;
  2534. }
  2535. static void finish_ordered_fn(struct btrfs_work *work)
  2536. {
  2537. struct btrfs_ordered_extent *ordered_extent;
  2538. ordered_extent = container_of(work, struct btrfs_ordered_extent, work);
  2539. btrfs_finish_ordered_io(ordered_extent);
  2540. }
  2541. void btrfs_writepage_endio_finish_ordered(struct page *page, u64 start,
  2542. u64 end, int uptodate)
  2543. {
  2544. struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
  2545. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  2546. struct btrfs_ordered_extent *ordered_extent = NULL;
  2547. struct btrfs_workqueue *wq;
  2548. trace_btrfs_writepage_end_io_hook(page, start, end, uptodate);
  2549. ClearPagePrivate2(page);
  2550. if (!btrfs_dec_test_ordered_pending(inode, &ordered_extent, start,
  2551. end - start + 1, uptodate))
  2552. return;
  2553. if (btrfs_is_free_space_inode(inode))
  2554. wq = fs_info->endio_freespace_worker;
  2555. else
  2556. wq = fs_info->endio_write_workers;
  2557. btrfs_init_work(&ordered_extent->work, finish_ordered_fn, NULL, NULL);
  2558. btrfs_queue_work(wq, &ordered_extent->work);
  2559. }
  2560. static int check_data_csum(struct inode *inode, struct btrfs_io_bio *io_bio,
  2561. int icsum, struct page *page, int pgoff, u64 start,
  2562. size_t len)
  2563. {
  2564. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2565. SHASH_DESC_ON_STACK(shash, fs_info->csum_shash);
  2566. char *kaddr;
  2567. u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  2568. u8 *csum_expected;
  2569. u8 csum[BTRFS_CSUM_SIZE];
  2570. csum_expected = ((u8 *)io_bio->csum) + icsum * csum_size;
  2571. kaddr = kmap_atomic(page);
  2572. shash->tfm = fs_info->csum_shash;
  2573. crypto_shash_digest(shash, kaddr + pgoff, len, csum);
  2574. if (memcmp(csum, csum_expected, csum_size))
  2575. goto zeroit;
  2576. kunmap_atomic(kaddr);
  2577. return 0;
  2578. zeroit:
  2579. btrfs_print_data_csum_error(BTRFS_I(inode), start, csum, csum_expected,
  2580. io_bio->mirror_num);
  2581. if (io_bio->device)
  2582. btrfs_dev_stat_inc_and_print(io_bio->device,
  2583. BTRFS_DEV_STAT_CORRUPTION_ERRS);
  2584. memset(kaddr + pgoff, 1, len);
  2585. flush_dcache_page(page);
  2586. kunmap_atomic(kaddr);
  2587. return -EIO;
  2588. }
  2589. /*
  2590. * when reads are done, we need to check csums to verify the data is correct
  2591. * if there's a match, we allow the bio to finish. If not, the code in
  2592. * extent_io.c will try to find good copies for us.
  2593. */
  2594. int btrfs_verify_data_csum(struct btrfs_io_bio *io_bio, u64 phy_offset,
  2595. struct page *page, u64 start, u64 end, int mirror)
  2596. {
  2597. size_t offset = start - page_offset(page);
  2598. struct inode *inode = page->mapping->host;
  2599. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  2600. struct btrfs_root *root = BTRFS_I(inode)->root;
  2601. if (PageChecked(page)) {
  2602. ClearPageChecked(page);
  2603. return 0;
  2604. }
  2605. if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
  2606. return 0;
  2607. if (root->root_key.objectid == BTRFS_DATA_RELOC_TREE_OBJECTID &&
  2608. test_range_bit(io_tree, start, end, EXTENT_NODATASUM, 1, NULL)) {
  2609. clear_extent_bits(io_tree, start, end, EXTENT_NODATASUM);
  2610. return 0;
  2611. }
  2612. phy_offset >>= inode->i_sb->s_blocksize_bits;
  2613. return check_data_csum(inode, io_bio, phy_offset, page, offset, start,
  2614. (size_t)(end - start + 1));
  2615. }
  2616. /*
  2617. * btrfs_add_delayed_iput - perform a delayed iput on @inode
  2618. *
  2619. * @inode: The inode we want to perform iput on
  2620. *
  2621. * This function uses the generic vfs_inode::i_count to track whether we should
  2622. * just decrement it (in case it's > 1) or if this is the last iput then link
  2623. * the inode to the delayed iput machinery. Delayed iputs are processed at
  2624. * transaction commit time/superblock commit/cleaner kthread.
  2625. */
  2626. void btrfs_add_delayed_iput(struct inode *inode)
  2627. {
  2628. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2629. struct btrfs_inode *binode = BTRFS_I(inode);
  2630. if (atomic_add_unless(&inode->i_count, -1, 1))
  2631. return;
  2632. atomic_inc(&fs_info->nr_delayed_iputs);
  2633. spin_lock(&fs_info->delayed_iput_lock);
  2634. ASSERT(list_empty(&binode->delayed_iput));
  2635. list_add_tail(&binode->delayed_iput, &fs_info->delayed_iputs);
  2636. spin_unlock(&fs_info->delayed_iput_lock);
  2637. if (!test_bit(BTRFS_FS_CLEANER_RUNNING, &fs_info->flags))
  2638. wake_up_process(fs_info->cleaner_kthread);
  2639. }
  2640. static void run_delayed_iput_locked(struct btrfs_fs_info *fs_info,
  2641. struct btrfs_inode *inode)
  2642. {
  2643. list_del_init(&inode->delayed_iput);
  2644. spin_unlock(&fs_info->delayed_iput_lock);
  2645. iput(&inode->vfs_inode);
  2646. if (atomic_dec_and_test(&fs_info->nr_delayed_iputs))
  2647. wake_up(&fs_info->delayed_iputs_wait);
  2648. spin_lock(&fs_info->delayed_iput_lock);
  2649. }
  2650. static void btrfs_run_delayed_iput(struct btrfs_fs_info *fs_info,
  2651. struct btrfs_inode *inode)
  2652. {
  2653. if (!list_empty(&inode->delayed_iput)) {
  2654. spin_lock(&fs_info->delayed_iput_lock);
  2655. if (!list_empty(&inode->delayed_iput))
  2656. run_delayed_iput_locked(fs_info, inode);
  2657. spin_unlock(&fs_info->delayed_iput_lock);
  2658. }
  2659. }
  2660. void btrfs_run_delayed_iputs(struct btrfs_fs_info *fs_info)
  2661. {
  2662. spin_lock(&fs_info->delayed_iput_lock);
  2663. while (!list_empty(&fs_info->delayed_iputs)) {
  2664. struct btrfs_inode *inode;
  2665. inode = list_first_entry(&fs_info->delayed_iputs,
  2666. struct btrfs_inode, delayed_iput);
  2667. run_delayed_iput_locked(fs_info, inode);
  2668. cond_resched_lock(&fs_info->delayed_iput_lock);
  2669. }
  2670. spin_unlock(&fs_info->delayed_iput_lock);
  2671. }
  2672. /**
  2673. * btrfs_wait_on_delayed_iputs - wait on the delayed iputs to be done running
  2674. * @fs_info - the fs_info for this fs
  2675. * @return - EINTR if we were killed, 0 if nothing's pending
  2676. *
  2677. * This will wait on any delayed iputs that are currently running with KILLABLE
  2678. * set. Once they are all done running we will return, unless we are killed in
  2679. * which case we return EINTR. This helps in user operations like fallocate etc
  2680. * that might get blocked on the iputs.
  2681. */
  2682. int btrfs_wait_on_delayed_iputs(struct btrfs_fs_info *fs_info)
  2683. {
  2684. int ret = wait_event_killable(fs_info->delayed_iputs_wait,
  2685. atomic_read(&fs_info->nr_delayed_iputs) == 0);
  2686. if (ret)
  2687. return -EINTR;
  2688. return 0;
  2689. }
  2690. /*
  2691. * This creates an orphan entry for the given inode in case something goes wrong
  2692. * in the middle of an unlink.
  2693. */
  2694. int btrfs_orphan_add(struct btrfs_trans_handle *trans,
  2695. struct btrfs_inode *inode)
  2696. {
  2697. int ret;
  2698. ret = btrfs_insert_orphan_item(trans, inode->root, btrfs_ino(inode));
  2699. if (ret && ret != -EEXIST) {
  2700. btrfs_abort_transaction(trans, ret);
  2701. return ret;
  2702. }
  2703. return 0;
  2704. }
  2705. /*
  2706. * We have done the delete so we can go ahead and remove the orphan item for
  2707. * this particular inode.
  2708. */
  2709. static int btrfs_orphan_del(struct btrfs_trans_handle *trans,
  2710. struct btrfs_inode *inode)
  2711. {
  2712. return btrfs_del_orphan_item(trans, inode->root, btrfs_ino(inode));
  2713. }
  2714. /*
  2715. * this cleans up any orphans that may be left on the list from the last use
  2716. * of this root.
  2717. */
  2718. int btrfs_orphan_cleanup(struct btrfs_root *root)
  2719. {
  2720. struct btrfs_fs_info *fs_info = root->fs_info;
  2721. struct btrfs_path *path;
  2722. struct extent_buffer *leaf;
  2723. struct btrfs_key key, found_key;
  2724. struct btrfs_trans_handle *trans;
  2725. struct inode *inode;
  2726. u64 last_objectid = 0;
  2727. int ret = 0, nr_unlink = 0;
  2728. if (cmpxchg(&root->orphan_cleanup_state, 0, ORPHAN_CLEANUP_STARTED))
  2729. return 0;
  2730. path = btrfs_alloc_path();
  2731. if (!path) {
  2732. ret = -ENOMEM;
  2733. goto out;
  2734. }
  2735. path->reada = READA_BACK;
  2736. key.objectid = BTRFS_ORPHAN_OBJECTID;
  2737. key.type = BTRFS_ORPHAN_ITEM_KEY;
  2738. key.offset = (u64)-1;
  2739. while (1) {
  2740. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  2741. if (ret < 0)
  2742. goto out;
  2743. /*
  2744. * if ret == 0 means we found what we were searching for, which
  2745. * is weird, but possible, so only screw with path if we didn't
  2746. * find the key and see if we have stuff that matches
  2747. */
  2748. if (ret > 0) {
  2749. ret = 0;
  2750. if (path->slots[0] == 0)
  2751. break;
  2752. path->slots[0]--;
  2753. }
  2754. /* pull out the item */
  2755. leaf = path->nodes[0];
  2756. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  2757. /* make sure the item matches what we want */
  2758. if (found_key.objectid != BTRFS_ORPHAN_OBJECTID)
  2759. break;
  2760. if (found_key.type != BTRFS_ORPHAN_ITEM_KEY)
  2761. break;
  2762. /* release the path since we're done with it */
  2763. btrfs_release_path(path);
  2764. /*
  2765. * this is where we are basically btrfs_lookup, without the
  2766. * crossing root thing. we store the inode number in the
  2767. * offset of the orphan item.
  2768. */
  2769. if (found_key.offset == last_objectid) {
  2770. btrfs_err(fs_info,
  2771. "Error removing orphan entry, stopping orphan cleanup");
  2772. ret = -EINVAL;
  2773. goto out;
  2774. }
  2775. last_objectid = found_key.offset;
  2776. found_key.objectid = found_key.offset;
  2777. found_key.type = BTRFS_INODE_ITEM_KEY;
  2778. found_key.offset = 0;
  2779. inode = btrfs_iget(fs_info->sb, last_objectid, root);
  2780. ret = PTR_ERR_OR_ZERO(inode);
  2781. if (ret && ret != -ENOENT)
  2782. goto out;
  2783. if (ret == -ENOENT && root == fs_info->tree_root) {
  2784. struct btrfs_root *dead_root;
  2785. int is_dead_root = 0;
  2786. /*
  2787. * this is an orphan in the tree root. Currently these
  2788. * could come from 2 sources:
  2789. * a) a snapshot deletion in progress
  2790. * b) a free space cache inode
  2791. * We need to distinguish those two, as the snapshot
  2792. * orphan must not get deleted.
  2793. * find_dead_roots already ran before us, so if this
  2794. * is a snapshot deletion, we should find the root
  2795. * in the fs_roots radix tree.
  2796. */
  2797. spin_lock(&fs_info->fs_roots_radix_lock);
  2798. dead_root = radix_tree_lookup(&fs_info->fs_roots_radix,
  2799. (unsigned long)found_key.objectid);
  2800. if (dead_root && btrfs_root_refs(&dead_root->root_item) == 0)
  2801. is_dead_root = 1;
  2802. spin_unlock(&fs_info->fs_roots_radix_lock);
  2803. if (is_dead_root) {
  2804. /* prevent this orphan from being found again */
  2805. key.offset = found_key.objectid - 1;
  2806. continue;
  2807. }
  2808. }
  2809. /*
  2810. * If we have an inode with links, there are a couple of
  2811. * possibilities. Old kernels (before v3.12) used to create an
  2812. * orphan item for truncate indicating that there were possibly
  2813. * extent items past i_size that needed to be deleted. In v3.12,
  2814. * truncate was changed to update i_size in sync with the extent
  2815. * items, but the (useless) orphan item was still created. Since
  2816. * v4.18, we don't create the orphan item for truncate at all.
  2817. *
  2818. * So, this item could mean that we need to do a truncate, but
  2819. * only if this filesystem was last used on a pre-v3.12 kernel
  2820. * and was not cleanly unmounted. The odds of that are quite
  2821. * slim, and it's a pain to do the truncate now, so just delete
  2822. * the orphan item.
  2823. *
  2824. * It's also possible that this orphan item was supposed to be
  2825. * deleted but wasn't. The inode number may have been reused,
  2826. * but either way, we can delete the orphan item.
  2827. */
  2828. if (ret == -ENOENT || inode->i_nlink) {
  2829. if (!ret)
  2830. iput(inode);
  2831. trans = btrfs_start_transaction(root, 1);
  2832. if (IS_ERR(trans)) {
  2833. ret = PTR_ERR(trans);
  2834. goto out;
  2835. }
  2836. btrfs_debug(fs_info, "auto deleting %Lu",
  2837. found_key.objectid);
  2838. ret = btrfs_del_orphan_item(trans, root,
  2839. found_key.objectid);
  2840. btrfs_end_transaction(trans);
  2841. if (ret)
  2842. goto out;
  2843. continue;
  2844. }
  2845. nr_unlink++;
  2846. /* this will do delete_inode and everything for us */
  2847. iput(inode);
  2848. }
  2849. /* release the path since we're done with it */
  2850. btrfs_release_path(path);
  2851. root->orphan_cleanup_state = ORPHAN_CLEANUP_DONE;
  2852. if (test_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &root->state)) {
  2853. trans = btrfs_join_transaction(root);
  2854. if (!IS_ERR(trans))
  2855. btrfs_end_transaction(trans);
  2856. }
  2857. if (nr_unlink)
  2858. btrfs_debug(fs_info, "unlinked %d orphans", nr_unlink);
  2859. out:
  2860. if (ret)
  2861. btrfs_err(fs_info, "could not do orphan cleanup %d", ret);
  2862. btrfs_free_path(path);
  2863. return ret;
  2864. }
  2865. /*
  2866. * very simple check to peek ahead in the leaf looking for xattrs. If we
  2867. * don't find any xattrs, we know there can't be any acls.
  2868. *
  2869. * slot is the slot the inode is in, objectid is the objectid of the inode
  2870. */
  2871. static noinline int acls_after_inode_item(struct extent_buffer *leaf,
  2872. int slot, u64 objectid,
  2873. int *first_xattr_slot)
  2874. {
  2875. u32 nritems = btrfs_header_nritems(leaf);
  2876. struct btrfs_key found_key;
  2877. static u64 xattr_access = 0;
  2878. static u64 xattr_default = 0;
  2879. int scanned = 0;
  2880. if (!xattr_access) {
  2881. xattr_access = btrfs_name_hash(XATTR_NAME_POSIX_ACL_ACCESS,
  2882. strlen(XATTR_NAME_POSIX_ACL_ACCESS));
  2883. xattr_default = btrfs_name_hash(XATTR_NAME_POSIX_ACL_DEFAULT,
  2884. strlen(XATTR_NAME_POSIX_ACL_DEFAULT));
  2885. }
  2886. slot++;
  2887. *first_xattr_slot = -1;
  2888. while (slot < nritems) {
  2889. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  2890. /* we found a different objectid, there must not be acls */
  2891. if (found_key.objectid != objectid)
  2892. return 0;
  2893. /* we found an xattr, assume we've got an acl */
  2894. if (found_key.type == BTRFS_XATTR_ITEM_KEY) {
  2895. if (*first_xattr_slot == -1)
  2896. *first_xattr_slot = slot;
  2897. if (found_key.offset == xattr_access ||
  2898. found_key.offset == xattr_default)
  2899. return 1;
  2900. }
  2901. /*
  2902. * we found a key greater than an xattr key, there can't
  2903. * be any acls later on
  2904. */
  2905. if (found_key.type > BTRFS_XATTR_ITEM_KEY)
  2906. return 0;
  2907. slot++;
  2908. scanned++;
  2909. /*
  2910. * it goes inode, inode backrefs, xattrs, extents,
  2911. * so if there are a ton of hard links to an inode there can
  2912. * be a lot of backrefs. Don't waste time searching too hard,
  2913. * this is just an optimization
  2914. */
  2915. if (scanned >= 8)
  2916. break;
  2917. }
  2918. /* we hit the end of the leaf before we found an xattr or
  2919. * something larger than an xattr. We have to assume the inode
  2920. * has acls
  2921. */
  2922. if (*first_xattr_slot == -1)
  2923. *first_xattr_slot = slot;
  2924. return 1;
  2925. }
  2926. /*
  2927. * read an inode from the btree into the in-memory inode
  2928. */
  2929. static int btrfs_read_locked_inode(struct inode *inode,
  2930. struct btrfs_path *in_path)
  2931. {
  2932. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  2933. struct btrfs_path *path = in_path;
  2934. struct extent_buffer *leaf;
  2935. struct btrfs_inode_item *inode_item;
  2936. struct btrfs_root *root = BTRFS_I(inode)->root;
  2937. struct btrfs_key location;
  2938. unsigned long ptr;
  2939. int maybe_acls;
  2940. u32 rdev;
  2941. int ret;
  2942. bool filled = false;
  2943. int first_xattr_slot;
  2944. ret = btrfs_fill_inode(inode, &rdev);
  2945. if (!ret)
  2946. filled = true;
  2947. if (!path) {
  2948. path = btrfs_alloc_path();
  2949. if (!path)
  2950. return -ENOMEM;
  2951. }
  2952. memcpy(&location, &BTRFS_I(inode)->location, sizeof(location));
  2953. ret = btrfs_lookup_inode(NULL, root, path, &location, 0);
  2954. if (ret) {
  2955. if (path != in_path)
  2956. btrfs_free_path(path);
  2957. return ret;
  2958. }
  2959. leaf = path->nodes[0];
  2960. if (filled)
  2961. goto cache_index;
  2962. inode_item = btrfs_item_ptr(leaf, path->slots[0],
  2963. struct btrfs_inode_item);
  2964. inode->i_mode = btrfs_inode_mode(leaf, inode_item);
  2965. set_nlink(inode, btrfs_inode_nlink(leaf, inode_item));
  2966. i_uid_write(inode, btrfs_inode_uid(leaf, inode_item));
  2967. i_gid_write(inode, btrfs_inode_gid(leaf, inode_item));
  2968. btrfs_i_size_write(BTRFS_I(inode), btrfs_inode_size(leaf, inode_item));
  2969. btrfs_inode_set_file_extent_range(BTRFS_I(inode), 0,
  2970. round_up(i_size_read(inode), fs_info->sectorsize));
  2971. inode->i_atime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->atime);
  2972. inode->i_atime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->atime);
  2973. inode->i_mtime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->mtime);
  2974. inode->i_mtime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->mtime);
  2975. inode->i_ctime.tv_sec = btrfs_timespec_sec(leaf, &inode_item->ctime);
  2976. inode->i_ctime.tv_nsec = btrfs_timespec_nsec(leaf, &inode_item->ctime);
  2977. BTRFS_I(inode)->i_otime.tv_sec =
  2978. btrfs_timespec_sec(leaf, &inode_item->otime);
  2979. BTRFS_I(inode)->i_otime.tv_nsec =
  2980. btrfs_timespec_nsec(leaf, &inode_item->otime);
  2981. inode_set_bytes(inode, btrfs_inode_nbytes(leaf, inode_item));
  2982. BTRFS_I(inode)->generation = btrfs_inode_generation(leaf, inode_item);
  2983. BTRFS_I(inode)->last_trans = btrfs_inode_transid(leaf, inode_item);
  2984. inode_set_iversion_queried(inode,
  2985. btrfs_inode_sequence(leaf, inode_item));
  2986. inode->i_generation = BTRFS_I(inode)->generation;
  2987. inode->i_rdev = 0;
  2988. rdev = btrfs_inode_rdev(leaf, inode_item);
  2989. BTRFS_I(inode)->index_cnt = (u64)-1;
  2990. BTRFS_I(inode)->flags = btrfs_inode_flags(leaf, inode_item);
  2991. cache_index:
  2992. /*
  2993. * If we were modified in the current generation and evicted from memory
  2994. * and then re-read we need to do a full sync since we don't have any
  2995. * idea about which extents were modified before we were evicted from
  2996. * cache.
  2997. *
  2998. * This is required for both inode re-read from disk and delayed inode
  2999. * in delayed_nodes_tree.
  3000. */
  3001. if (BTRFS_I(inode)->last_trans == fs_info->generation)
  3002. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  3003. &BTRFS_I(inode)->runtime_flags);
  3004. /*
  3005. * We don't persist the id of the transaction where an unlink operation
  3006. * against the inode was last made. So here we assume the inode might
  3007. * have been evicted, and therefore the exact value of last_unlink_trans
  3008. * lost, and set it to last_trans to avoid metadata inconsistencies
  3009. * between the inode and its parent if the inode is fsync'ed and the log
  3010. * replayed. For example, in the scenario:
  3011. *
  3012. * touch mydir/foo
  3013. * ln mydir/foo mydir/bar
  3014. * sync
  3015. * unlink mydir/bar
  3016. * echo 2 > /proc/sys/vm/drop_caches # evicts inode
  3017. * xfs_io -c fsync mydir/foo
  3018. * <power failure>
  3019. * mount fs, triggers fsync log replay
  3020. *
  3021. * We must make sure that when we fsync our inode foo we also log its
  3022. * parent inode, otherwise after log replay the parent still has the
  3023. * dentry with the "bar" name but our inode foo has a link count of 1
  3024. * and doesn't have an inode ref with the name "bar" anymore.
  3025. *
  3026. * Setting last_unlink_trans to last_trans is a pessimistic approach,
  3027. * but it guarantees correctness at the expense of occasional full
  3028. * transaction commits on fsync if our inode is a directory, or if our
  3029. * inode is not a directory, logging its parent unnecessarily.
  3030. */
  3031. BTRFS_I(inode)->last_unlink_trans = BTRFS_I(inode)->last_trans;
  3032. /*
  3033. * Same logic as for last_unlink_trans. We don't persist the generation
  3034. * of the last transaction where this inode was used for a reflink
  3035. * operation, so after eviction and reloading the inode we must be
  3036. * pessimistic and assume the last transaction that modified the inode.
  3037. */
  3038. BTRFS_I(inode)->last_reflink_trans = BTRFS_I(inode)->last_trans;
  3039. path->slots[0]++;
  3040. if (inode->i_nlink != 1 ||
  3041. path->slots[0] >= btrfs_header_nritems(leaf))
  3042. goto cache_acl;
  3043. btrfs_item_key_to_cpu(leaf, &location, path->slots[0]);
  3044. if (location.objectid != btrfs_ino(BTRFS_I(inode)))
  3045. goto cache_acl;
  3046. ptr = btrfs_item_ptr_offset(leaf, path->slots[0]);
  3047. if (location.type == BTRFS_INODE_REF_KEY) {
  3048. struct btrfs_inode_ref *ref;
  3049. ref = (struct btrfs_inode_ref *)ptr;
  3050. BTRFS_I(inode)->dir_index = btrfs_inode_ref_index(leaf, ref);
  3051. } else if (location.type == BTRFS_INODE_EXTREF_KEY) {
  3052. struct btrfs_inode_extref *extref;
  3053. extref = (struct btrfs_inode_extref *)ptr;
  3054. BTRFS_I(inode)->dir_index = btrfs_inode_extref_index(leaf,
  3055. extref);
  3056. }
  3057. cache_acl:
  3058. /*
  3059. * try to precache a NULL acl entry for files that don't have
  3060. * any xattrs or acls
  3061. */
  3062. maybe_acls = acls_after_inode_item(leaf, path->slots[0],
  3063. btrfs_ino(BTRFS_I(inode)), &first_xattr_slot);
  3064. if (first_xattr_slot != -1) {
  3065. path->slots[0] = first_xattr_slot;
  3066. ret = btrfs_load_inode_props(inode, path);
  3067. if (ret)
  3068. btrfs_err(fs_info,
  3069. "error loading props for ino %llu (root %llu): %d",
  3070. btrfs_ino(BTRFS_I(inode)),
  3071. root->root_key.objectid, ret);
  3072. }
  3073. if (path != in_path)
  3074. btrfs_free_path(path);
  3075. if (!maybe_acls)
  3076. cache_no_acl(inode);
  3077. switch (inode->i_mode & S_IFMT) {
  3078. case S_IFREG:
  3079. inode->i_mapping->a_ops = &btrfs_aops;
  3080. inode->i_fop = &btrfs_file_operations;
  3081. inode->i_op = &btrfs_file_inode_operations;
  3082. break;
  3083. case S_IFDIR:
  3084. inode->i_fop = &btrfs_dir_file_operations;
  3085. inode->i_op = &btrfs_dir_inode_operations;
  3086. break;
  3087. case S_IFLNK:
  3088. inode->i_op = &btrfs_symlink_inode_operations;
  3089. inode_nohighmem(inode);
  3090. inode->i_mapping->a_ops = &btrfs_aops;
  3091. break;
  3092. default:
  3093. inode->i_op = &btrfs_special_inode_operations;
  3094. init_special_inode(inode, inode->i_mode, rdev);
  3095. break;
  3096. }
  3097. btrfs_sync_inode_flags_to_i_flags(inode);
  3098. return 0;
  3099. }
  3100. /*
  3101. * given a leaf and an inode, copy the inode fields into the leaf
  3102. */
  3103. static void fill_inode_item(struct btrfs_trans_handle *trans,
  3104. struct extent_buffer *leaf,
  3105. struct btrfs_inode_item *item,
  3106. struct inode *inode)
  3107. {
  3108. struct btrfs_map_token token;
  3109. btrfs_init_map_token(&token, leaf);
  3110. btrfs_set_token_inode_uid(&token, item, i_uid_read(inode));
  3111. btrfs_set_token_inode_gid(&token, item, i_gid_read(inode));
  3112. btrfs_set_token_inode_size(&token, item, BTRFS_I(inode)->disk_i_size);
  3113. btrfs_set_token_inode_mode(&token, item, inode->i_mode);
  3114. btrfs_set_token_inode_nlink(&token, item, inode->i_nlink);
  3115. btrfs_set_token_timespec_sec(&token, &item->atime,
  3116. inode->i_atime.tv_sec);
  3117. btrfs_set_token_timespec_nsec(&token, &item->atime,
  3118. inode->i_atime.tv_nsec);
  3119. btrfs_set_token_timespec_sec(&token, &item->mtime,
  3120. inode->i_mtime.tv_sec);
  3121. btrfs_set_token_timespec_nsec(&token, &item->mtime,
  3122. inode->i_mtime.tv_nsec);
  3123. btrfs_set_token_timespec_sec(&token, &item->ctime,
  3124. inode->i_ctime.tv_sec);
  3125. btrfs_set_token_timespec_nsec(&token, &item->ctime,
  3126. inode->i_ctime.tv_nsec);
  3127. btrfs_set_token_timespec_sec(&token, &item->otime,
  3128. BTRFS_I(inode)->i_otime.tv_sec);
  3129. btrfs_set_token_timespec_nsec(&token, &item->otime,
  3130. BTRFS_I(inode)->i_otime.tv_nsec);
  3131. btrfs_set_token_inode_nbytes(&token, item, inode_get_bytes(inode));
  3132. btrfs_set_token_inode_generation(&token, item,
  3133. BTRFS_I(inode)->generation);
  3134. btrfs_set_token_inode_sequence(&token, item, inode_peek_iversion(inode));
  3135. btrfs_set_token_inode_transid(&token, item, trans->transid);
  3136. btrfs_set_token_inode_rdev(&token, item, inode->i_rdev);
  3137. btrfs_set_token_inode_flags(&token, item, BTRFS_I(inode)->flags);
  3138. btrfs_set_token_inode_block_group(&token, item, 0);
  3139. }
  3140. /*
  3141. * copy everything in the in-memory inode into the btree.
  3142. */
  3143. static noinline int btrfs_update_inode_item(struct btrfs_trans_handle *trans,
  3144. struct btrfs_root *root, struct inode *inode)
  3145. {
  3146. struct btrfs_inode_item *inode_item;
  3147. struct btrfs_path *path;
  3148. struct extent_buffer *leaf;
  3149. int ret;
  3150. path = btrfs_alloc_path();
  3151. if (!path)
  3152. return -ENOMEM;
  3153. path->leave_spinning = 1;
  3154. ret = btrfs_lookup_inode(trans, root, path, &BTRFS_I(inode)->location,
  3155. 1);
  3156. if (ret) {
  3157. if (ret > 0)
  3158. ret = -ENOENT;
  3159. goto failed;
  3160. }
  3161. leaf = path->nodes[0];
  3162. inode_item = btrfs_item_ptr(leaf, path->slots[0],
  3163. struct btrfs_inode_item);
  3164. fill_inode_item(trans, leaf, inode_item, inode);
  3165. btrfs_mark_buffer_dirty(leaf);
  3166. btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
  3167. ret = 0;
  3168. failed:
  3169. btrfs_free_path(path);
  3170. return ret;
  3171. }
  3172. /*
  3173. * copy everything in the in-memory inode into the btree.
  3174. */
  3175. noinline int btrfs_update_inode(struct btrfs_trans_handle *trans,
  3176. struct btrfs_root *root, struct inode *inode)
  3177. {
  3178. struct btrfs_fs_info *fs_info = root->fs_info;
  3179. int ret;
  3180. /*
  3181. * If the inode is a free space inode, we can deadlock during commit
  3182. * if we put it into the delayed code.
  3183. *
  3184. * The data relocation inode should also be directly updated
  3185. * without delay
  3186. */
  3187. if (!btrfs_is_free_space_inode(BTRFS_I(inode))
  3188. && root->root_key.objectid != BTRFS_DATA_RELOC_TREE_OBJECTID
  3189. && !test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags)) {
  3190. btrfs_update_root_times(trans, root);
  3191. ret = btrfs_delayed_update_inode(trans, root, inode);
  3192. if (!ret)
  3193. btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
  3194. return ret;
  3195. }
  3196. return btrfs_update_inode_item(trans, root, inode);
  3197. }
  3198. noinline int btrfs_update_inode_fallback(struct btrfs_trans_handle *trans,
  3199. struct btrfs_root *root,
  3200. struct inode *inode)
  3201. {
  3202. int ret;
  3203. ret = btrfs_update_inode(trans, root, inode);
  3204. if (ret == -ENOSPC)
  3205. return btrfs_update_inode_item(trans, root, inode);
  3206. return ret;
  3207. }
  3208. /*
  3209. * unlink helper that gets used here in inode.c and in the tree logging
  3210. * recovery code. It remove a link in a directory with a given name, and
  3211. * also drops the back refs in the inode to the directory
  3212. */
  3213. static int __btrfs_unlink_inode(struct btrfs_trans_handle *trans,
  3214. struct btrfs_root *root,
  3215. struct btrfs_inode *dir,
  3216. struct btrfs_inode *inode,
  3217. const char *name, int name_len)
  3218. {
  3219. struct btrfs_fs_info *fs_info = root->fs_info;
  3220. struct btrfs_path *path;
  3221. int ret = 0;
  3222. struct btrfs_dir_item *di;
  3223. u64 index;
  3224. u64 ino = btrfs_ino(inode);
  3225. u64 dir_ino = btrfs_ino(dir);
  3226. path = btrfs_alloc_path();
  3227. if (!path) {
  3228. ret = -ENOMEM;
  3229. goto out;
  3230. }
  3231. path->leave_spinning = 1;
  3232. di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
  3233. name, name_len, -1);
  3234. if (IS_ERR_OR_NULL(di)) {
  3235. ret = di ? PTR_ERR(di) : -ENOENT;
  3236. goto err;
  3237. }
  3238. ret = btrfs_delete_one_dir_name(trans, root, path, di);
  3239. if (ret)
  3240. goto err;
  3241. btrfs_release_path(path);
  3242. /*
  3243. * If we don't have dir index, we have to get it by looking up
  3244. * the inode ref, since we get the inode ref, remove it directly,
  3245. * it is unnecessary to do delayed deletion.
  3246. *
  3247. * But if we have dir index, needn't search inode ref to get it.
  3248. * Since the inode ref is close to the inode item, it is better
  3249. * that we delay to delete it, and just do this deletion when
  3250. * we update the inode item.
  3251. */
  3252. if (inode->dir_index) {
  3253. ret = btrfs_delayed_delete_inode_ref(inode);
  3254. if (!ret) {
  3255. index = inode->dir_index;
  3256. goto skip_backref;
  3257. }
  3258. }
  3259. ret = btrfs_del_inode_ref(trans, root, name, name_len, ino,
  3260. dir_ino, &index);
  3261. if (ret) {
  3262. btrfs_info(fs_info,
  3263. "failed to delete reference to %.*s, inode %llu parent %llu",
  3264. name_len, name, ino, dir_ino);
  3265. btrfs_abort_transaction(trans, ret);
  3266. goto err;
  3267. }
  3268. skip_backref:
  3269. ret = btrfs_delete_delayed_dir_index(trans, dir, index);
  3270. if (ret) {
  3271. btrfs_abort_transaction(trans, ret);
  3272. goto err;
  3273. }
  3274. ret = btrfs_del_inode_ref_in_log(trans, root, name, name_len, inode,
  3275. dir_ino);
  3276. if (ret != 0 && ret != -ENOENT) {
  3277. btrfs_abort_transaction(trans, ret);
  3278. goto err;
  3279. }
  3280. ret = btrfs_del_dir_entries_in_log(trans, root, name, name_len, dir,
  3281. index);
  3282. if (ret == -ENOENT)
  3283. ret = 0;
  3284. else if (ret)
  3285. btrfs_abort_transaction(trans, ret);
  3286. /*
  3287. * If we have a pending delayed iput we could end up with the final iput
  3288. * being run in btrfs-cleaner context. If we have enough of these built
  3289. * up we can end up burning a lot of time in btrfs-cleaner without any
  3290. * way to throttle the unlinks. Since we're currently holding a ref on
  3291. * the inode we can run the delayed iput here without any issues as the
  3292. * final iput won't be done until after we drop the ref we're currently
  3293. * holding.
  3294. */
  3295. btrfs_run_delayed_iput(fs_info, inode);
  3296. err:
  3297. btrfs_free_path(path);
  3298. if (ret)
  3299. goto out;
  3300. btrfs_i_size_write(dir, dir->vfs_inode.i_size - name_len * 2);
  3301. inode_inc_iversion(&inode->vfs_inode);
  3302. inode_inc_iversion(&dir->vfs_inode);
  3303. inode->vfs_inode.i_ctime = dir->vfs_inode.i_mtime =
  3304. dir->vfs_inode.i_ctime = current_time(&inode->vfs_inode);
  3305. ret = btrfs_update_inode(trans, root, &dir->vfs_inode);
  3306. out:
  3307. return ret;
  3308. }
  3309. int btrfs_unlink_inode(struct btrfs_trans_handle *trans,
  3310. struct btrfs_root *root,
  3311. struct btrfs_inode *dir, struct btrfs_inode *inode,
  3312. const char *name, int name_len)
  3313. {
  3314. int ret;
  3315. ret = __btrfs_unlink_inode(trans, root, dir, inode, name, name_len);
  3316. if (!ret) {
  3317. drop_nlink(&inode->vfs_inode);
  3318. ret = btrfs_update_inode(trans, root, &inode->vfs_inode);
  3319. }
  3320. return ret;
  3321. }
  3322. /*
  3323. * helper to start transaction for unlink and rmdir.
  3324. *
  3325. * unlink and rmdir are special in btrfs, they do not always free space, so
  3326. * if we cannot make our reservations the normal way try and see if there is
  3327. * plenty of slack room in the global reserve to migrate, otherwise we cannot
  3328. * allow the unlink to occur.
  3329. */
  3330. static struct btrfs_trans_handle *__unlink_start_trans(struct inode *dir)
  3331. {
  3332. struct btrfs_root *root = BTRFS_I(dir)->root;
  3333. /*
  3334. * 1 for the possible orphan item
  3335. * 1 for the dir item
  3336. * 1 for the dir index
  3337. * 1 for the inode ref
  3338. * 1 for the inode
  3339. */
  3340. return btrfs_start_transaction_fallback_global_rsv(root, 5);
  3341. }
  3342. static int btrfs_unlink(struct inode *dir, struct dentry *dentry)
  3343. {
  3344. struct btrfs_root *root = BTRFS_I(dir)->root;
  3345. struct btrfs_trans_handle *trans;
  3346. struct inode *inode = d_inode(dentry);
  3347. int ret;
  3348. trans = __unlink_start_trans(dir);
  3349. if (IS_ERR(trans))
  3350. return PTR_ERR(trans);
  3351. btrfs_record_unlink_dir(trans, BTRFS_I(dir), BTRFS_I(d_inode(dentry)),
  3352. 0);
  3353. ret = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
  3354. BTRFS_I(d_inode(dentry)), dentry->d_name.name,
  3355. dentry->d_name.len);
  3356. if (ret)
  3357. goto out;
  3358. if (inode->i_nlink == 0) {
  3359. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  3360. if (ret)
  3361. goto out;
  3362. }
  3363. out:
  3364. btrfs_end_transaction(trans);
  3365. btrfs_btree_balance_dirty(root->fs_info);
  3366. return ret;
  3367. }
  3368. static int btrfs_unlink_subvol(struct btrfs_trans_handle *trans,
  3369. struct inode *dir, struct dentry *dentry)
  3370. {
  3371. struct btrfs_root *root = BTRFS_I(dir)->root;
  3372. struct btrfs_inode *inode = BTRFS_I(d_inode(dentry));
  3373. struct btrfs_path *path;
  3374. struct extent_buffer *leaf;
  3375. struct btrfs_dir_item *di;
  3376. struct btrfs_key key;
  3377. const char *name = dentry->d_name.name;
  3378. int name_len = dentry->d_name.len;
  3379. u64 index;
  3380. int ret;
  3381. u64 objectid;
  3382. u64 dir_ino = btrfs_ino(BTRFS_I(dir));
  3383. if (btrfs_ino(inode) == BTRFS_FIRST_FREE_OBJECTID) {
  3384. objectid = inode->root->root_key.objectid;
  3385. } else if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
  3386. objectid = inode->location.objectid;
  3387. } else {
  3388. WARN_ON(1);
  3389. return -EINVAL;
  3390. }
  3391. path = btrfs_alloc_path();
  3392. if (!path)
  3393. return -ENOMEM;
  3394. di = btrfs_lookup_dir_item(trans, root, path, dir_ino,
  3395. name, name_len, -1);
  3396. if (IS_ERR_OR_NULL(di)) {
  3397. ret = di ? PTR_ERR(di) : -ENOENT;
  3398. goto out;
  3399. }
  3400. leaf = path->nodes[0];
  3401. btrfs_dir_item_key_to_cpu(leaf, di, &key);
  3402. WARN_ON(key.type != BTRFS_ROOT_ITEM_KEY || key.objectid != objectid);
  3403. ret = btrfs_delete_one_dir_name(trans, root, path, di);
  3404. if (ret) {
  3405. btrfs_abort_transaction(trans, ret);
  3406. goto out;
  3407. }
  3408. btrfs_release_path(path);
  3409. /*
  3410. * This is a placeholder inode for a subvolume we didn't have a
  3411. * reference to at the time of the snapshot creation. In the meantime
  3412. * we could have renamed the real subvol link into our snapshot, so
  3413. * depending on btrfs_del_root_ref to return -ENOENT here is incorret.
  3414. * Instead simply lookup the dir_index_item for this entry so we can
  3415. * remove it. Otherwise we know we have a ref to the root and we can
  3416. * call btrfs_del_root_ref, and it _shouldn't_ fail.
  3417. */
  3418. if (btrfs_ino(inode) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID) {
  3419. di = btrfs_search_dir_index_item(root, path, dir_ino,
  3420. name, name_len);
  3421. if (IS_ERR_OR_NULL(di)) {
  3422. if (!di)
  3423. ret = -ENOENT;
  3424. else
  3425. ret = PTR_ERR(di);
  3426. btrfs_abort_transaction(trans, ret);
  3427. goto out;
  3428. }
  3429. leaf = path->nodes[0];
  3430. btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
  3431. index = key.offset;
  3432. btrfs_release_path(path);
  3433. } else {
  3434. ret = btrfs_del_root_ref(trans, objectid,
  3435. root->root_key.objectid, dir_ino,
  3436. &index, name, name_len);
  3437. if (ret) {
  3438. btrfs_abort_transaction(trans, ret);
  3439. goto out;
  3440. }
  3441. }
  3442. ret = btrfs_delete_delayed_dir_index(trans, BTRFS_I(dir), index);
  3443. if (ret) {
  3444. btrfs_abort_transaction(trans, ret);
  3445. goto out;
  3446. }
  3447. btrfs_i_size_write(BTRFS_I(dir), dir->i_size - name_len * 2);
  3448. inode_inc_iversion(dir);
  3449. dir->i_mtime = dir->i_ctime = current_time(dir);
  3450. ret = btrfs_update_inode_fallback(trans, root, dir);
  3451. if (ret)
  3452. btrfs_abort_transaction(trans, ret);
  3453. out:
  3454. btrfs_free_path(path);
  3455. return ret;
  3456. }
  3457. /*
  3458. * Helper to check if the subvolume references other subvolumes or if it's
  3459. * default.
  3460. */
  3461. static noinline int may_destroy_subvol(struct btrfs_root *root)
  3462. {
  3463. struct btrfs_fs_info *fs_info = root->fs_info;
  3464. struct btrfs_path *path;
  3465. struct btrfs_dir_item *di;
  3466. struct btrfs_key key;
  3467. u64 dir_id;
  3468. int ret;
  3469. path = btrfs_alloc_path();
  3470. if (!path)
  3471. return -ENOMEM;
  3472. /* Make sure this root isn't set as the default subvol */
  3473. dir_id = btrfs_super_root_dir(fs_info->super_copy);
  3474. di = btrfs_lookup_dir_item(NULL, fs_info->tree_root, path,
  3475. dir_id, "default", 7, 0);
  3476. if (di && !IS_ERR(di)) {
  3477. btrfs_dir_item_key_to_cpu(path->nodes[0], di, &key);
  3478. if (key.objectid == root->root_key.objectid) {
  3479. ret = -EPERM;
  3480. btrfs_err(fs_info,
  3481. "deleting default subvolume %llu is not allowed",
  3482. key.objectid);
  3483. goto out;
  3484. }
  3485. btrfs_release_path(path);
  3486. }
  3487. key.objectid = root->root_key.objectid;
  3488. key.type = BTRFS_ROOT_REF_KEY;
  3489. key.offset = (u64)-1;
  3490. ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
  3491. if (ret < 0)
  3492. goto out;
  3493. BUG_ON(ret == 0);
  3494. ret = 0;
  3495. if (path->slots[0] > 0) {
  3496. path->slots[0]--;
  3497. btrfs_item_key_to_cpu(path->nodes[0], &key, path->slots[0]);
  3498. if (key.objectid == root->root_key.objectid &&
  3499. key.type == BTRFS_ROOT_REF_KEY)
  3500. ret = -ENOTEMPTY;
  3501. }
  3502. out:
  3503. btrfs_free_path(path);
  3504. return ret;
  3505. }
  3506. /* Delete all dentries for inodes belonging to the root */
  3507. static void btrfs_prune_dentries(struct btrfs_root *root)
  3508. {
  3509. struct btrfs_fs_info *fs_info = root->fs_info;
  3510. struct rb_node *node;
  3511. struct rb_node *prev;
  3512. struct btrfs_inode *entry;
  3513. struct inode *inode;
  3514. u64 objectid = 0;
  3515. if (!test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  3516. WARN_ON(btrfs_root_refs(&root->root_item) != 0);
  3517. spin_lock(&root->inode_lock);
  3518. again:
  3519. node = root->inode_tree.rb_node;
  3520. prev = NULL;
  3521. while (node) {
  3522. prev = node;
  3523. entry = rb_entry(node, struct btrfs_inode, rb_node);
  3524. if (objectid < btrfs_ino(entry))
  3525. node = node->rb_left;
  3526. else if (objectid > btrfs_ino(entry))
  3527. node = node->rb_right;
  3528. else
  3529. break;
  3530. }
  3531. if (!node) {
  3532. while (prev) {
  3533. entry = rb_entry(prev, struct btrfs_inode, rb_node);
  3534. if (objectid <= btrfs_ino(entry)) {
  3535. node = prev;
  3536. break;
  3537. }
  3538. prev = rb_next(prev);
  3539. }
  3540. }
  3541. while (node) {
  3542. entry = rb_entry(node, struct btrfs_inode, rb_node);
  3543. objectid = btrfs_ino(entry) + 1;
  3544. inode = igrab(&entry->vfs_inode);
  3545. if (inode) {
  3546. spin_unlock(&root->inode_lock);
  3547. if (atomic_read(&inode->i_count) > 1)
  3548. d_prune_aliases(inode);
  3549. /*
  3550. * btrfs_drop_inode will have it removed from the inode
  3551. * cache when its usage count hits zero.
  3552. */
  3553. iput(inode);
  3554. cond_resched();
  3555. spin_lock(&root->inode_lock);
  3556. goto again;
  3557. }
  3558. if (cond_resched_lock(&root->inode_lock))
  3559. goto again;
  3560. node = rb_next(node);
  3561. }
  3562. spin_unlock(&root->inode_lock);
  3563. }
  3564. int btrfs_delete_subvolume(struct inode *dir, struct dentry *dentry)
  3565. {
  3566. struct btrfs_fs_info *fs_info = btrfs_sb(dentry->d_sb);
  3567. struct btrfs_root *root = BTRFS_I(dir)->root;
  3568. struct inode *inode = d_inode(dentry);
  3569. struct btrfs_root *dest = BTRFS_I(inode)->root;
  3570. struct btrfs_trans_handle *trans;
  3571. struct btrfs_block_rsv block_rsv;
  3572. u64 root_flags;
  3573. int ret;
  3574. int err;
  3575. /*
  3576. * Don't allow to delete a subvolume with send in progress. This is
  3577. * inside the inode lock so the error handling that has to drop the bit
  3578. * again is not run concurrently.
  3579. */
  3580. spin_lock(&dest->root_item_lock);
  3581. if (dest->send_in_progress) {
  3582. spin_unlock(&dest->root_item_lock);
  3583. btrfs_warn(fs_info,
  3584. "attempt to delete subvolume %llu during send",
  3585. dest->root_key.objectid);
  3586. return -EPERM;
  3587. }
  3588. if (atomic_read(&dest->nr_swapfiles)) {
  3589. spin_unlock(&dest->root_item_lock);
  3590. btrfs_warn(fs_info,
  3591. "attempt to delete subvolume %llu with active swapfile",
  3592. root->root_key.objectid);
  3593. return -EPERM;
  3594. }
  3595. root_flags = btrfs_root_flags(&dest->root_item);
  3596. btrfs_set_root_flags(&dest->root_item,
  3597. root_flags | BTRFS_ROOT_SUBVOL_DEAD);
  3598. spin_unlock(&dest->root_item_lock);
  3599. down_write(&fs_info->subvol_sem);
  3600. err = may_destroy_subvol(dest);
  3601. if (err)
  3602. goto out_up_write;
  3603. btrfs_init_block_rsv(&block_rsv, BTRFS_BLOCK_RSV_TEMP);
  3604. /*
  3605. * One for dir inode,
  3606. * two for dir entries,
  3607. * two for root ref/backref.
  3608. */
  3609. err = btrfs_subvolume_reserve_metadata(root, &block_rsv, 5, true);
  3610. if (err)
  3611. goto out_up_write;
  3612. trans = btrfs_start_transaction(root, 0);
  3613. if (IS_ERR(trans)) {
  3614. err = PTR_ERR(trans);
  3615. goto out_release;
  3616. }
  3617. trans->block_rsv = &block_rsv;
  3618. trans->bytes_reserved = block_rsv.size;
  3619. btrfs_record_snapshot_destroy(trans, BTRFS_I(dir));
  3620. ret = btrfs_unlink_subvol(trans, dir, dentry);
  3621. if (ret) {
  3622. err = ret;
  3623. btrfs_abort_transaction(trans, ret);
  3624. goto out_end_trans;
  3625. }
  3626. btrfs_record_root_in_trans(trans, dest);
  3627. memset(&dest->root_item.drop_progress, 0,
  3628. sizeof(dest->root_item.drop_progress));
  3629. dest->root_item.drop_level = 0;
  3630. btrfs_set_root_refs(&dest->root_item, 0);
  3631. if (!test_and_set_bit(BTRFS_ROOT_ORPHAN_ITEM_INSERTED, &dest->state)) {
  3632. ret = btrfs_insert_orphan_item(trans,
  3633. fs_info->tree_root,
  3634. dest->root_key.objectid);
  3635. if (ret) {
  3636. btrfs_abort_transaction(trans, ret);
  3637. err = ret;
  3638. goto out_end_trans;
  3639. }
  3640. }
  3641. ret = btrfs_uuid_tree_remove(trans, dest->root_item.uuid,
  3642. BTRFS_UUID_KEY_SUBVOL,
  3643. dest->root_key.objectid);
  3644. if (ret && ret != -ENOENT) {
  3645. btrfs_abort_transaction(trans, ret);
  3646. err = ret;
  3647. goto out_end_trans;
  3648. }
  3649. if (!btrfs_is_empty_uuid(dest->root_item.received_uuid)) {
  3650. ret = btrfs_uuid_tree_remove(trans,
  3651. dest->root_item.received_uuid,
  3652. BTRFS_UUID_KEY_RECEIVED_SUBVOL,
  3653. dest->root_key.objectid);
  3654. if (ret && ret != -ENOENT) {
  3655. btrfs_abort_transaction(trans, ret);
  3656. err = ret;
  3657. goto out_end_trans;
  3658. }
  3659. }
  3660. free_anon_bdev(dest->anon_dev);
  3661. dest->anon_dev = 0;
  3662. out_end_trans:
  3663. trans->block_rsv = NULL;
  3664. trans->bytes_reserved = 0;
  3665. ret = btrfs_end_transaction(trans);
  3666. if (ret && !err)
  3667. err = ret;
  3668. inode->i_flags |= S_DEAD;
  3669. out_release:
  3670. btrfs_subvolume_release_metadata(root, &block_rsv);
  3671. out_up_write:
  3672. up_write(&fs_info->subvol_sem);
  3673. if (err) {
  3674. spin_lock(&dest->root_item_lock);
  3675. root_flags = btrfs_root_flags(&dest->root_item);
  3676. btrfs_set_root_flags(&dest->root_item,
  3677. root_flags & ~BTRFS_ROOT_SUBVOL_DEAD);
  3678. spin_unlock(&dest->root_item_lock);
  3679. } else {
  3680. d_invalidate(dentry);
  3681. btrfs_prune_dentries(dest);
  3682. ASSERT(dest->send_in_progress == 0);
  3683. /* the last ref */
  3684. if (dest->ino_cache_inode) {
  3685. iput(dest->ino_cache_inode);
  3686. dest->ino_cache_inode = NULL;
  3687. }
  3688. }
  3689. return err;
  3690. }
  3691. static int btrfs_rmdir(struct inode *dir, struct dentry *dentry)
  3692. {
  3693. struct inode *inode = d_inode(dentry);
  3694. int err = 0;
  3695. struct btrfs_root *root = BTRFS_I(dir)->root;
  3696. struct btrfs_trans_handle *trans;
  3697. u64 last_unlink_trans;
  3698. if (inode->i_size > BTRFS_EMPTY_DIR_SIZE)
  3699. return -ENOTEMPTY;
  3700. if (btrfs_ino(BTRFS_I(inode)) == BTRFS_FIRST_FREE_OBJECTID)
  3701. return btrfs_delete_subvolume(dir, dentry);
  3702. trans = __unlink_start_trans(dir);
  3703. if (IS_ERR(trans))
  3704. return PTR_ERR(trans);
  3705. if (unlikely(btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
  3706. err = btrfs_unlink_subvol(trans, dir, dentry);
  3707. goto out;
  3708. }
  3709. err = btrfs_orphan_add(trans, BTRFS_I(inode));
  3710. if (err)
  3711. goto out;
  3712. last_unlink_trans = BTRFS_I(inode)->last_unlink_trans;
  3713. /* now the directory is empty */
  3714. err = btrfs_unlink_inode(trans, root, BTRFS_I(dir),
  3715. BTRFS_I(d_inode(dentry)), dentry->d_name.name,
  3716. dentry->d_name.len);
  3717. if (!err) {
  3718. btrfs_i_size_write(BTRFS_I(inode), 0);
  3719. /*
  3720. * Propagate the last_unlink_trans value of the deleted dir to
  3721. * its parent directory. This is to prevent an unrecoverable
  3722. * log tree in the case we do something like this:
  3723. * 1) create dir foo
  3724. * 2) create snapshot under dir foo
  3725. * 3) delete the snapshot
  3726. * 4) rmdir foo
  3727. * 5) mkdir foo
  3728. * 6) fsync foo or some file inside foo
  3729. */
  3730. if (last_unlink_trans >= trans->transid)
  3731. BTRFS_I(dir)->last_unlink_trans = last_unlink_trans;
  3732. }
  3733. out:
  3734. btrfs_end_transaction(trans);
  3735. btrfs_btree_balance_dirty(root->fs_info);
  3736. return err;
  3737. }
  3738. /*
  3739. * Return this if we need to call truncate_block for the last bit of the
  3740. * truncate.
  3741. */
  3742. #define NEED_TRUNCATE_BLOCK 1
  3743. /*
  3744. * this can truncate away extent items, csum items and directory items.
  3745. * It starts at a high offset and removes keys until it can't find
  3746. * any higher than new_size
  3747. *
  3748. * csum items that cross the new i_size are truncated to the new size
  3749. * as well.
  3750. *
  3751. * min_type is the minimum key type to truncate down to. If set to 0, this
  3752. * will kill all the items on this inode, including the INODE_ITEM_KEY.
  3753. */
  3754. int btrfs_truncate_inode_items(struct btrfs_trans_handle *trans,
  3755. struct btrfs_root *root,
  3756. struct inode *inode,
  3757. u64 new_size, u32 min_type)
  3758. {
  3759. struct btrfs_fs_info *fs_info = root->fs_info;
  3760. struct btrfs_path *path;
  3761. struct extent_buffer *leaf;
  3762. struct btrfs_file_extent_item *fi;
  3763. struct btrfs_key key;
  3764. struct btrfs_key found_key;
  3765. u64 extent_start = 0;
  3766. u64 extent_num_bytes = 0;
  3767. u64 extent_offset = 0;
  3768. u64 item_end = 0;
  3769. u64 last_size = new_size;
  3770. u32 found_type = (u8)-1;
  3771. int found_extent;
  3772. int del_item;
  3773. int pending_del_nr = 0;
  3774. int pending_del_slot = 0;
  3775. int extent_type = -1;
  3776. int ret;
  3777. u64 ino = btrfs_ino(BTRFS_I(inode));
  3778. u64 bytes_deleted = 0;
  3779. bool be_nice = false;
  3780. bool should_throttle = false;
  3781. const u64 lock_start = ALIGN_DOWN(new_size, fs_info->sectorsize);
  3782. struct extent_state *cached_state = NULL;
  3783. BUG_ON(new_size > 0 && min_type != BTRFS_EXTENT_DATA_KEY);
  3784. /*
  3785. * For non-free space inodes and non-shareable roots, we want to back
  3786. * off from time to time. This means all inodes in subvolume roots,
  3787. * reloc roots, and data reloc roots.
  3788. */
  3789. if (!btrfs_is_free_space_inode(BTRFS_I(inode)) &&
  3790. test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
  3791. be_nice = true;
  3792. path = btrfs_alloc_path();
  3793. if (!path)
  3794. return -ENOMEM;
  3795. path->reada = READA_BACK;
  3796. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  3797. lock_extent_bits(&BTRFS_I(inode)->io_tree, lock_start, (u64)-1,
  3798. &cached_state);
  3799. /*
  3800. * We want to drop from the next block forward in case this
  3801. * new size is not block aligned since we will be keeping the
  3802. * last block of the extent just the way it is.
  3803. */
  3804. btrfs_drop_extent_cache(BTRFS_I(inode), ALIGN(new_size,
  3805. fs_info->sectorsize),
  3806. (u64)-1, 0);
  3807. }
  3808. /*
  3809. * This function is also used to drop the items in the log tree before
  3810. * we relog the inode, so if root != BTRFS_I(inode)->root, it means
  3811. * it is used to drop the logged items. So we shouldn't kill the delayed
  3812. * items.
  3813. */
  3814. if (min_type == 0 && root == BTRFS_I(inode)->root)
  3815. btrfs_kill_delayed_inode_items(BTRFS_I(inode));
  3816. key.objectid = ino;
  3817. key.offset = (u64)-1;
  3818. key.type = (u8)-1;
  3819. search_again:
  3820. /*
  3821. * with a 16K leaf size and 128MB extents, you can actually queue
  3822. * up a huge file in a single leaf. Most of the time that
  3823. * bytes_deleted is > 0, it will be huge by the time we get here
  3824. */
  3825. if (be_nice && bytes_deleted > SZ_32M &&
  3826. btrfs_should_end_transaction(trans)) {
  3827. ret = -EAGAIN;
  3828. goto out;
  3829. }
  3830. ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
  3831. if (ret < 0)
  3832. goto out;
  3833. if (ret > 0) {
  3834. ret = 0;
  3835. /* there are no items in the tree for us to truncate, we're
  3836. * done
  3837. */
  3838. if (path->slots[0] == 0)
  3839. goto out;
  3840. path->slots[0]--;
  3841. }
  3842. while (1) {
  3843. u64 clear_start = 0, clear_len = 0;
  3844. fi = NULL;
  3845. leaf = path->nodes[0];
  3846. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  3847. found_type = found_key.type;
  3848. if (found_key.objectid != ino)
  3849. break;
  3850. if (found_type < min_type)
  3851. break;
  3852. item_end = found_key.offset;
  3853. if (found_type == BTRFS_EXTENT_DATA_KEY) {
  3854. fi = btrfs_item_ptr(leaf, path->slots[0],
  3855. struct btrfs_file_extent_item);
  3856. extent_type = btrfs_file_extent_type(leaf, fi);
  3857. if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
  3858. item_end +=
  3859. btrfs_file_extent_num_bytes(leaf, fi);
  3860. trace_btrfs_truncate_show_fi_regular(
  3861. BTRFS_I(inode), leaf, fi,
  3862. found_key.offset);
  3863. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  3864. item_end += btrfs_file_extent_ram_bytes(leaf,
  3865. fi);
  3866. trace_btrfs_truncate_show_fi_inline(
  3867. BTRFS_I(inode), leaf, fi, path->slots[0],
  3868. found_key.offset);
  3869. }
  3870. item_end--;
  3871. }
  3872. if (found_type > min_type) {
  3873. del_item = 1;
  3874. } else {
  3875. if (item_end < new_size)
  3876. break;
  3877. if (found_key.offset >= new_size)
  3878. del_item = 1;
  3879. else
  3880. del_item = 0;
  3881. }
  3882. found_extent = 0;
  3883. /* FIXME, shrink the extent if the ref count is only 1 */
  3884. if (found_type != BTRFS_EXTENT_DATA_KEY)
  3885. goto delete;
  3886. if (extent_type != BTRFS_FILE_EXTENT_INLINE) {
  3887. u64 num_dec;
  3888. clear_start = found_key.offset;
  3889. extent_start = btrfs_file_extent_disk_bytenr(leaf, fi);
  3890. if (!del_item) {
  3891. u64 orig_num_bytes =
  3892. btrfs_file_extent_num_bytes(leaf, fi);
  3893. extent_num_bytes = ALIGN(new_size -
  3894. found_key.offset,
  3895. fs_info->sectorsize);
  3896. clear_start = ALIGN(new_size, fs_info->sectorsize);
  3897. btrfs_set_file_extent_num_bytes(leaf, fi,
  3898. extent_num_bytes);
  3899. num_dec = (orig_num_bytes -
  3900. extent_num_bytes);
  3901. if (test_bit(BTRFS_ROOT_SHAREABLE,
  3902. &root->state) &&
  3903. extent_start != 0)
  3904. inode_sub_bytes(inode, num_dec);
  3905. btrfs_mark_buffer_dirty(leaf);
  3906. } else {
  3907. extent_num_bytes =
  3908. btrfs_file_extent_disk_num_bytes(leaf,
  3909. fi);
  3910. extent_offset = found_key.offset -
  3911. btrfs_file_extent_offset(leaf, fi);
  3912. /* FIXME blocksize != 4096 */
  3913. num_dec = btrfs_file_extent_num_bytes(leaf, fi);
  3914. if (extent_start != 0) {
  3915. found_extent = 1;
  3916. if (test_bit(BTRFS_ROOT_SHAREABLE,
  3917. &root->state))
  3918. inode_sub_bytes(inode, num_dec);
  3919. }
  3920. }
  3921. clear_len = num_dec;
  3922. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  3923. /*
  3924. * we can't truncate inline items that have had
  3925. * special encodings
  3926. */
  3927. if (!del_item &&
  3928. btrfs_file_extent_encryption(leaf, fi) == 0 &&
  3929. btrfs_file_extent_other_encoding(leaf, fi) == 0 &&
  3930. btrfs_file_extent_compression(leaf, fi) == 0) {
  3931. u32 size = (u32)(new_size - found_key.offset);
  3932. btrfs_set_file_extent_ram_bytes(leaf, fi, size);
  3933. size = btrfs_file_extent_calc_inline_size(size);
  3934. btrfs_truncate_item(path, size, 1);
  3935. } else if (!del_item) {
  3936. /*
  3937. * We have to bail so the last_size is set to
  3938. * just before this extent.
  3939. */
  3940. ret = NEED_TRUNCATE_BLOCK;
  3941. break;
  3942. } else {
  3943. /*
  3944. * Inline extents are special, we just treat
  3945. * them as a full sector worth in the file
  3946. * extent tree just for simplicity sake.
  3947. */
  3948. clear_len = fs_info->sectorsize;
  3949. }
  3950. if (test_bit(BTRFS_ROOT_SHAREABLE, &root->state))
  3951. inode_sub_bytes(inode, item_end + 1 - new_size);
  3952. }
  3953. delete:
  3954. /*
  3955. * We use btrfs_truncate_inode_items() to clean up log trees for
  3956. * multiple fsyncs, and in this case we don't want to clear the
  3957. * file extent range because it's just the log.
  3958. */
  3959. if (root == BTRFS_I(inode)->root) {
  3960. ret = btrfs_inode_clear_file_extent_range(BTRFS_I(inode),
  3961. clear_start, clear_len);
  3962. if (ret) {
  3963. btrfs_abort_transaction(trans, ret);
  3964. break;
  3965. }
  3966. }
  3967. if (del_item)
  3968. last_size = found_key.offset;
  3969. else
  3970. last_size = new_size;
  3971. if (del_item) {
  3972. if (!pending_del_nr) {
  3973. /* no pending yet, add ourselves */
  3974. pending_del_slot = path->slots[0];
  3975. pending_del_nr = 1;
  3976. } else if (pending_del_nr &&
  3977. path->slots[0] + 1 == pending_del_slot) {
  3978. /* hop on the pending chunk */
  3979. pending_del_nr++;
  3980. pending_del_slot = path->slots[0];
  3981. } else {
  3982. BUG();
  3983. }
  3984. } else {
  3985. break;
  3986. }
  3987. should_throttle = false;
  3988. if (found_extent &&
  3989. root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  3990. struct btrfs_ref ref = { 0 };
  3991. bytes_deleted += extent_num_bytes;
  3992. btrfs_init_generic_ref(&ref, BTRFS_DROP_DELAYED_REF,
  3993. extent_start, extent_num_bytes, 0);
  3994. ref.real_root = root->root_key.objectid;
  3995. btrfs_init_data_ref(&ref, btrfs_header_owner(leaf),
  3996. ino, extent_offset);
  3997. ret = btrfs_free_extent(trans, &ref);
  3998. if (ret) {
  3999. btrfs_abort_transaction(trans, ret);
  4000. break;
  4001. }
  4002. if (be_nice) {
  4003. if (btrfs_should_throttle_delayed_refs(trans))
  4004. should_throttle = true;
  4005. }
  4006. }
  4007. if (found_type == BTRFS_INODE_ITEM_KEY)
  4008. break;
  4009. if (path->slots[0] == 0 ||
  4010. path->slots[0] != pending_del_slot ||
  4011. should_throttle) {
  4012. if (pending_del_nr) {
  4013. ret = btrfs_del_items(trans, root, path,
  4014. pending_del_slot,
  4015. pending_del_nr);
  4016. if (ret) {
  4017. btrfs_abort_transaction(trans, ret);
  4018. break;
  4019. }
  4020. pending_del_nr = 0;
  4021. }
  4022. btrfs_release_path(path);
  4023. /*
  4024. * We can generate a lot of delayed refs, so we need to
  4025. * throttle every once and a while and make sure we're
  4026. * adding enough space to keep up with the work we are
  4027. * generating. Since we hold a transaction here we
  4028. * can't flush, and we don't want to FLUSH_LIMIT because
  4029. * we could have generated too many delayed refs to
  4030. * actually allocate, so just bail if we're short and
  4031. * let the normal reservation dance happen higher up.
  4032. */
  4033. if (should_throttle) {
  4034. ret = btrfs_delayed_refs_rsv_refill(fs_info,
  4035. BTRFS_RESERVE_NO_FLUSH);
  4036. if (ret) {
  4037. ret = -EAGAIN;
  4038. break;
  4039. }
  4040. }
  4041. goto search_again;
  4042. } else {
  4043. path->slots[0]--;
  4044. }
  4045. }
  4046. out:
  4047. if (ret >= 0 && pending_del_nr) {
  4048. int err;
  4049. err = btrfs_del_items(trans, root, path, pending_del_slot,
  4050. pending_del_nr);
  4051. if (err) {
  4052. btrfs_abort_transaction(trans, err);
  4053. ret = err;
  4054. }
  4055. }
  4056. if (root->root_key.objectid != BTRFS_TREE_LOG_OBJECTID) {
  4057. ASSERT(last_size >= new_size);
  4058. if (!ret && last_size > new_size)
  4059. last_size = new_size;
  4060. btrfs_inode_safe_disk_i_size_write(inode, last_size);
  4061. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lock_start,
  4062. (u64)-1, &cached_state);
  4063. }
  4064. btrfs_free_path(path);
  4065. return ret;
  4066. }
  4067. /*
  4068. * btrfs_truncate_block - read, zero a chunk and write a block
  4069. * @inode - inode that we're zeroing
  4070. * @from - the offset to start zeroing
  4071. * @len - the length to zero, 0 to zero the entire range respective to the
  4072. * offset
  4073. * @front - zero up to the offset instead of from the offset on
  4074. *
  4075. * This will find the block for the "from" offset and cow the block and zero the
  4076. * part we want to zero. This is used with truncate and hole punching.
  4077. */
  4078. int btrfs_truncate_block(struct inode *inode, loff_t from, loff_t len,
  4079. int front)
  4080. {
  4081. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4082. struct address_space *mapping = inode->i_mapping;
  4083. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4084. struct btrfs_ordered_extent *ordered;
  4085. struct extent_state *cached_state = NULL;
  4086. struct extent_changeset *data_reserved = NULL;
  4087. char *kaddr;
  4088. bool only_release_metadata = false;
  4089. u32 blocksize = fs_info->sectorsize;
  4090. pgoff_t index = from >> PAGE_SHIFT;
  4091. unsigned offset = from & (blocksize - 1);
  4092. struct page *page;
  4093. gfp_t mask = btrfs_alloc_write_mask(mapping);
  4094. size_t write_bytes = blocksize;
  4095. int ret = 0;
  4096. u64 block_start;
  4097. u64 block_end;
  4098. if (IS_ALIGNED(offset, blocksize) &&
  4099. (!len || IS_ALIGNED(len, blocksize)))
  4100. goto out;
  4101. block_start = round_down(from, blocksize);
  4102. block_end = block_start + blocksize - 1;
  4103. ret = btrfs_check_data_free_space(BTRFS_I(inode), &data_reserved,
  4104. block_start, blocksize);
  4105. if (ret < 0) {
  4106. if (btrfs_check_nocow_lock(BTRFS_I(inode), block_start,
  4107. &write_bytes) > 0) {
  4108. /* For nocow case, no need to reserve data space */
  4109. only_release_metadata = true;
  4110. } else {
  4111. goto out;
  4112. }
  4113. }
  4114. ret = btrfs_delalloc_reserve_metadata(BTRFS_I(inode), blocksize);
  4115. if (ret < 0) {
  4116. if (!only_release_metadata)
  4117. btrfs_free_reserved_data_space(BTRFS_I(inode),
  4118. data_reserved, block_start, blocksize);
  4119. goto out;
  4120. }
  4121. again:
  4122. page = find_or_create_page(mapping, index, mask);
  4123. if (!page) {
  4124. btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved,
  4125. block_start, blocksize, true);
  4126. btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
  4127. ret = -ENOMEM;
  4128. goto out;
  4129. }
  4130. if (!PageUptodate(page)) {
  4131. ret = btrfs_readpage(NULL, page);
  4132. lock_page(page);
  4133. if (page->mapping != mapping) {
  4134. unlock_page(page);
  4135. put_page(page);
  4136. goto again;
  4137. }
  4138. if (!PageUptodate(page)) {
  4139. ret = -EIO;
  4140. goto out_unlock;
  4141. }
  4142. }
  4143. wait_on_page_writeback(page);
  4144. lock_extent_bits(io_tree, block_start, block_end, &cached_state);
  4145. set_page_extent_mapped(page);
  4146. ordered = btrfs_lookup_ordered_extent(BTRFS_I(inode), block_start);
  4147. if (ordered) {
  4148. unlock_extent_cached(io_tree, block_start, block_end,
  4149. &cached_state);
  4150. unlock_page(page);
  4151. put_page(page);
  4152. btrfs_start_ordered_extent(ordered, 1);
  4153. btrfs_put_ordered_extent(ordered);
  4154. goto again;
  4155. }
  4156. clear_extent_bit(&BTRFS_I(inode)->io_tree, block_start, block_end,
  4157. EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG,
  4158. 0, 0, &cached_state);
  4159. ret = btrfs_set_extent_delalloc(BTRFS_I(inode), block_start, block_end, 0,
  4160. &cached_state);
  4161. if (ret) {
  4162. unlock_extent_cached(io_tree, block_start, block_end,
  4163. &cached_state);
  4164. goto out_unlock;
  4165. }
  4166. if (offset != blocksize) {
  4167. if (!len)
  4168. len = blocksize - offset;
  4169. kaddr = kmap(page);
  4170. if (front)
  4171. memset(kaddr + (block_start - page_offset(page)),
  4172. 0, offset);
  4173. else
  4174. memset(kaddr + (block_start - page_offset(page)) + offset,
  4175. 0, len);
  4176. flush_dcache_page(page);
  4177. kunmap(page);
  4178. }
  4179. ClearPageChecked(page);
  4180. set_page_dirty(page);
  4181. unlock_extent_cached(io_tree, block_start, block_end, &cached_state);
  4182. if (only_release_metadata)
  4183. set_extent_bit(&BTRFS_I(inode)->io_tree, block_start,
  4184. block_end, EXTENT_NORESERVE, NULL, NULL,
  4185. GFP_NOFS);
  4186. out_unlock:
  4187. if (ret) {
  4188. if (only_release_metadata)
  4189. btrfs_delalloc_release_metadata(BTRFS_I(inode),
  4190. blocksize, true);
  4191. else
  4192. btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved,
  4193. block_start, blocksize, true);
  4194. }
  4195. btrfs_delalloc_release_extents(BTRFS_I(inode), blocksize);
  4196. unlock_page(page);
  4197. put_page(page);
  4198. out:
  4199. if (only_release_metadata)
  4200. btrfs_check_nocow_unlock(BTRFS_I(inode));
  4201. extent_changeset_free(data_reserved);
  4202. return ret;
  4203. }
  4204. static int maybe_insert_hole(struct btrfs_root *root, struct inode *inode,
  4205. u64 offset, u64 len)
  4206. {
  4207. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4208. struct btrfs_trans_handle *trans;
  4209. int ret;
  4210. /*
  4211. * Still need to make sure the inode looks like it's been updated so
  4212. * that any holes get logged if we fsync.
  4213. */
  4214. if (btrfs_fs_incompat(fs_info, NO_HOLES)) {
  4215. BTRFS_I(inode)->last_trans = fs_info->generation;
  4216. BTRFS_I(inode)->last_sub_trans = root->log_transid;
  4217. BTRFS_I(inode)->last_log_commit = root->last_log_commit;
  4218. return 0;
  4219. }
  4220. /*
  4221. * 1 - for the one we're dropping
  4222. * 1 - for the one we're adding
  4223. * 1 - for updating the inode.
  4224. */
  4225. trans = btrfs_start_transaction(root, 3);
  4226. if (IS_ERR(trans))
  4227. return PTR_ERR(trans);
  4228. ret = btrfs_drop_extents(trans, root, inode, offset, offset + len, 1);
  4229. if (ret) {
  4230. btrfs_abort_transaction(trans, ret);
  4231. btrfs_end_transaction(trans);
  4232. return ret;
  4233. }
  4234. ret = btrfs_insert_file_extent(trans, root, btrfs_ino(BTRFS_I(inode)),
  4235. offset, 0, 0, len, 0, len, 0, 0, 0);
  4236. if (ret)
  4237. btrfs_abort_transaction(trans, ret);
  4238. else
  4239. btrfs_update_inode(trans, root, inode);
  4240. btrfs_end_transaction(trans);
  4241. return ret;
  4242. }
  4243. /*
  4244. * This function puts in dummy file extents for the area we're creating a hole
  4245. * for. So if we are truncating this file to a larger size we need to insert
  4246. * these file extents so that btrfs_get_extent will return a EXTENT_MAP_HOLE for
  4247. * the range between oldsize and size
  4248. */
  4249. int btrfs_cont_expand(struct inode *inode, loff_t oldsize, loff_t size)
  4250. {
  4251. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4252. struct btrfs_root *root = BTRFS_I(inode)->root;
  4253. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4254. struct extent_map *em = NULL;
  4255. struct extent_state *cached_state = NULL;
  4256. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  4257. u64 hole_start = ALIGN(oldsize, fs_info->sectorsize);
  4258. u64 block_end = ALIGN(size, fs_info->sectorsize);
  4259. u64 last_byte;
  4260. u64 cur_offset;
  4261. u64 hole_size;
  4262. int err = 0;
  4263. /*
  4264. * If our size started in the middle of a block we need to zero out the
  4265. * rest of the block before we expand the i_size, otherwise we could
  4266. * expose stale data.
  4267. */
  4268. err = btrfs_truncate_block(inode, oldsize, 0, 0);
  4269. if (err)
  4270. return err;
  4271. if (size <= hole_start)
  4272. return 0;
  4273. btrfs_lock_and_flush_ordered_range(BTRFS_I(inode), hole_start,
  4274. block_end - 1, &cached_state);
  4275. cur_offset = hole_start;
  4276. while (1) {
  4277. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, cur_offset,
  4278. block_end - cur_offset);
  4279. if (IS_ERR(em)) {
  4280. err = PTR_ERR(em);
  4281. em = NULL;
  4282. break;
  4283. }
  4284. last_byte = min(extent_map_end(em), block_end);
  4285. last_byte = ALIGN(last_byte, fs_info->sectorsize);
  4286. hole_size = last_byte - cur_offset;
  4287. if (!test_bit(EXTENT_FLAG_PREALLOC, &em->flags)) {
  4288. struct extent_map *hole_em;
  4289. err = maybe_insert_hole(root, inode, cur_offset,
  4290. hole_size);
  4291. if (err)
  4292. break;
  4293. err = btrfs_inode_set_file_extent_range(BTRFS_I(inode),
  4294. cur_offset, hole_size);
  4295. if (err)
  4296. break;
  4297. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  4298. cur_offset + hole_size - 1, 0);
  4299. hole_em = alloc_extent_map();
  4300. if (!hole_em) {
  4301. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  4302. &BTRFS_I(inode)->runtime_flags);
  4303. goto next;
  4304. }
  4305. hole_em->start = cur_offset;
  4306. hole_em->len = hole_size;
  4307. hole_em->orig_start = cur_offset;
  4308. hole_em->block_start = EXTENT_MAP_HOLE;
  4309. hole_em->block_len = 0;
  4310. hole_em->orig_block_len = 0;
  4311. hole_em->ram_bytes = hole_size;
  4312. hole_em->compress_type = BTRFS_COMPRESS_NONE;
  4313. hole_em->generation = fs_info->generation;
  4314. while (1) {
  4315. write_lock(&em_tree->lock);
  4316. err = add_extent_mapping(em_tree, hole_em, 1);
  4317. write_unlock(&em_tree->lock);
  4318. if (err != -EEXIST)
  4319. break;
  4320. btrfs_drop_extent_cache(BTRFS_I(inode),
  4321. cur_offset,
  4322. cur_offset +
  4323. hole_size - 1, 0);
  4324. }
  4325. free_extent_map(hole_em);
  4326. } else {
  4327. err = btrfs_inode_set_file_extent_range(BTRFS_I(inode),
  4328. cur_offset, hole_size);
  4329. if (err)
  4330. break;
  4331. }
  4332. next:
  4333. free_extent_map(em);
  4334. em = NULL;
  4335. cur_offset = last_byte;
  4336. if (cur_offset >= block_end)
  4337. break;
  4338. }
  4339. free_extent_map(em);
  4340. unlock_extent_cached(io_tree, hole_start, block_end - 1, &cached_state);
  4341. return err;
  4342. }
  4343. static int btrfs_setsize(struct inode *inode, struct iattr *attr)
  4344. {
  4345. struct btrfs_root *root = BTRFS_I(inode)->root;
  4346. struct btrfs_trans_handle *trans;
  4347. loff_t oldsize = i_size_read(inode);
  4348. loff_t newsize = attr->ia_size;
  4349. int mask = attr->ia_valid;
  4350. int ret;
  4351. /*
  4352. * The regular truncate() case without ATTR_CTIME and ATTR_MTIME is a
  4353. * special case where we need to update the times despite not having
  4354. * these flags set. For all other operations the VFS set these flags
  4355. * explicitly if it wants a timestamp update.
  4356. */
  4357. if (newsize != oldsize) {
  4358. inode_inc_iversion(inode);
  4359. if (!(mask & (ATTR_CTIME | ATTR_MTIME)))
  4360. inode->i_ctime = inode->i_mtime =
  4361. current_time(inode);
  4362. }
  4363. if (newsize > oldsize) {
  4364. /*
  4365. * Don't do an expanding truncate while snapshotting is ongoing.
  4366. * This is to ensure the snapshot captures a fully consistent
  4367. * state of this file - if the snapshot captures this expanding
  4368. * truncation, it must capture all writes that happened before
  4369. * this truncation.
  4370. */
  4371. btrfs_drew_write_lock(&root->snapshot_lock);
  4372. ret = btrfs_cont_expand(inode, oldsize, newsize);
  4373. if (ret) {
  4374. btrfs_drew_write_unlock(&root->snapshot_lock);
  4375. return ret;
  4376. }
  4377. trans = btrfs_start_transaction(root, 1);
  4378. if (IS_ERR(trans)) {
  4379. btrfs_drew_write_unlock(&root->snapshot_lock);
  4380. return PTR_ERR(trans);
  4381. }
  4382. i_size_write(inode, newsize);
  4383. btrfs_inode_safe_disk_i_size_write(inode, 0);
  4384. pagecache_isize_extended(inode, oldsize, newsize);
  4385. ret = btrfs_update_inode(trans, root, inode);
  4386. btrfs_drew_write_unlock(&root->snapshot_lock);
  4387. btrfs_end_transaction(trans);
  4388. } else {
  4389. /*
  4390. * We're truncating a file that used to have good data down to
  4391. * zero. Make sure any new writes to the file get on disk
  4392. * on close.
  4393. */
  4394. if (newsize == 0)
  4395. set_bit(BTRFS_INODE_FLUSH_ON_CLOSE,
  4396. &BTRFS_I(inode)->runtime_flags);
  4397. truncate_setsize(inode, newsize);
  4398. inode_dio_wait(inode);
  4399. ret = btrfs_truncate(inode, newsize == oldsize);
  4400. if (ret && inode->i_nlink) {
  4401. int err;
  4402. /*
  4403. * Truncate failed, so fix up the in-memory size. We
  4404. * adjusted disk_i_size down as we removed extents, so
  4405. * wait for disk_i_size to be stable and then update the
  4406. * in-memory size to match.
  4407. */
  4408. err = btrfs_wait_ordered_range(inode, 0, (u64)-1);
  4409. if (err)
  4410. return err;
  4411. i_size_write(inode, BTRFS_I(inode)->disk_i_size);
  4412. }
  4413. }
  4414. return ret;
  4415. }
  4416. static int btrfs_setattr(struct dentry *dentry, struct iattr *attr)
  4417. {
  4418. struct inode *inode = d_inode(dentry);
  4419. struct btrfs_root *root = BTRFS_I(inode)->root;
  4420. int err;
  4421. if (btrfs_root_readonly(root))
  4422. return -EROFS;
  4423. err = setattr_prepare(dentry, attr);
  4424. if (err)
  4425. return err;
  4426. if (S_ISREG(inode->i_mode) && (attr->ia_valid & ATTR_SIZE)) {
  4427. err = btrfs_setsize(inode, attr);
  4428. if (err)
  4429. return err;
  4430. }
  4431. if (attr->ia_valid) {
  4432. setattr_copy(inode, attr);
  4433. inode_inc_iversion(inode);
  4434. err = btrfs_dirty_inode(inode);
  4435. if (!err && attr->ia_valid & ATTR_MODE)
  4436. err = posix_acl_chmod(inode, inode->i_mode);
  4437. }
  4438. return err;
  4439. }
  4440. /*
  4441. * While truncating the inode pages during eviction, we get the VFS calling
  4442. * btrfs_invalidatepage() against each page of the inode. This is slow because
  4443. * the calls to btrfs_invalidatepage() result in a huge amount of calls to
  4444. * lock_extent_bits() and clear_extent_bit(), which keep merging and splitting
  4445. * extent_state structures over and over, wasting lots of time.
  4446. *
  4447. * Therefore if the inode is being evicted, let btrfs_invalidatepage() skip all
  4448. * those expensive operations on a per page basis and do only the ordered io
  4449. * finishing, while we release here the extent_map and extent_state structures,
  4450. * without the excessive merging and splitting.
  4451. */
  4452. static void evict_inode_truncate_pages(struct inode *inode)
  4453. {
  4454. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  4455. struct extent_map_tree *map_tree = &BTRFS_I(inode)->extent_tree;
  4456. struct rb_node *node;
  4457. ASSERT(inode->i_state & I_FREEING);
  4458. truncate_inode_pages_final(&inode->i_data);
  4459. write_lock(&map_tree->lock);
  4460. while (!RB_EMPTY_ROOT(&map_tree->map.rb_root)) {
  4461. struct extent_map *em;
  4462. node = rb_first_cached(&map_tree->map);
  4463. em = rb_entry(node, struct extent_map, rb_node);
  4464. clear_bit(EXTENT_FLAG_PINNED, &em->flags);
  4465. clear_bit(EXTENT_FLAG_LOGGING, &em->flags);
  4466. remove_extent_mapping(map_tree, em);
  4467. free_extent_map(em);
  4468. if (need_resched()) {
  4469. write_unlock(&map_tree->lock);
  4470. cond_resched();
  4471. write_lock(&map_tree->lock);
  4472. }
  4473. }
  4474. write_unlock(&map_tree->lock);
  4475. /*
  4476. * Keep looping until we have no more ranges in the io tree.
  4477. * We can have ongoing bios started by readahead that have
  4478. * their endio callback (extent_io.c:end_bio_extent_readpage)
  4479. * still in progress (unlocked the pages in the bio but did not yet
  4480. * unlocked the ranges in the io tree). Therefore this means some
  4481. * ranges can still be locked and eviction started because before
  4482. * submitting those bios, which are executed by a separate task (work
  4483. * queue kthread), inode references (inode->i_count) were not taken
  4484. * (which would be dropped in the end io callback of each bio).
  4485. * Therefore here we effectively end up waiting for those bios and
  4486. * anyone else holding locked ranges without having bumped the inode's
  4487. * reference count - if we don't do it, when they access the inode's
  4488. * io_tree to unlock a range it may be too late, leading to an
  4489. * use-after-free issue.
  4490. */
  4491. spin_lock(&io_tree->lock);
  4492. while (!RB_EMPTY_ROOT(&io_tree->state)) {
  4493. struct extent_state *state;
  4494. struct extent_state *cached_state = NULL;
  4495. u64 start;
  4496. u64 end;
  4497. unsigned state_flags;
  4498. node = rb_first(&io_tree->state);
  4499. state = rb_entry(node, struct extent_state, rb_node);
  4500. start = state->start;
  4501. end = state->end;
  4502. state_flags = state->state;
  4503. spin_unlock(&io_tree->lock);
  4504. lock_extent_bits(io_tree, start, end, &cached_state);
  4505. /*
  4506. * If still has DELALLOC flag, the extent didn't reach disk,
  4507. * and its reserved space won't be freed by delayed_ref.
  4508. * So we need to free its reserved space here.
  4509. * (Refer to comment in btrfs_invalidatepage, case 2)
  4510. *
  4511. * Note, end is the bytenr of last byte, so we need + 1 here.
  4512. */
  4513. if (state_flags & EXTENT_DELALLOC)
  4514. btrfs_qgroup_free_data(BTRFS_I(inode), NULL, start,
  4515. end - start + 1);
  4516. clear_extent_bit(io_tree, start, end,
  4517. EXTENT_LOCKED | EXTENT_DELALLOC |
  4518. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
  4519. &cached_state);
  4520. cond_resched();
  4521. spin_lock(&io_tree->lock);
  4522. }
  4523. spin_unlock(&io_tree->lock);
  4524. }
  4525. static struct btrfs_trans_handle *evict_refill_and_join(struct btrfs_root *root,
  4526. struct btrfs_block_rsv *rsv)
  4527. {
  4528. struct btrfs_fs_info *fs_info = root->fs_info;
  4529. struct btrfs_block_rsv *global_rsv = &fs_info->global_block_rsv;
  4530. struct btrfs_trans_handle *trans;
  4531. u64 delayed_refs_extra = btrfs_calc_insert_metadata_size(fs_info, 1);
  4532. int ret;
  4533. /*
  4534. * Eviction should be taking place at some place safe because of our
  4535. * delayed iputs. However the normal flushing code will run delayed
  4536. * iputs, so we cannot use FLUSH_ALL otherwise we'll deadlock.
  4537. *
  4538. * We reserve the delayed_refs_extra here again because we can't use
  4539. * btrfs_start_transaction(root, 0) for the same deadlocky reason as
  4540. * above. We reserve our extra bit here because we generate a ton of
  4541. * delayed refs activity by truncating.
  4542. *
  4543. * If we cannot make our reservation we'll attempt to steal from the
  4544. * global reserve, because we really want to be able to free up space.
  4545. */
  4546. ret = btrfs_block_rsv_refill(root, rsv, rsv->size + delayed_refs_extra,
  4547. BTRFS_RESERVE_FLUSH_EVICT);
  4548. if (ret) {
  4549. /*
  4550. * Try to steal from the global reserve if there is space for
  4551. * it.
  4552. */
  4553. if (btrfs_check_space_for_delayed_refs(fs_info) ||
  4554. btrfs_block_rsv_migrate(global_rsv, rsv, rsv->size, 0)) {
  4555. btrfs_warn(fs_info,
  4556. "could not allocate space for delete; will truncate on mount");
  4557. return ERR_PTR(-ENOSPC);
  4558. }
  4559. delayed_refs_extra = 0;
  4560. }
  4561. trans = btrfs_join_transaction(root);
  4562. if (IS_ERR(trans))
  4563. return trans;
  4564. if (delayed_refs_extra) {
  4565. trans->block_rsv = &fs_info->trans_block_rsv;
  4566. trans->bytes_reserved = delayed_refs_extra;
  4567. btrfs_block_rsv_migrate(rsv, trans->block_rsv,
  4568. delayed_refs_extra, 1);
  4569. }
  4570. return trans;
  4571. }
  4572. void btrfs_evict_inode(struct inode *inode)
  4573. {
  4574. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  4575. struct btrfs_trans_handle *trans;
  4576. struct btrfs_root *root = BTRFS_I(inode)->root;
  4577. struct btrfs_block_rsv *rsv;
  4578. int ret;
  4579. trace_btrfs_inode_evict(inode);
  4580. if (!root) {
  4581. clear_inode(inode);
  4582. return;
  4583. }
  4584. evict_inode_truncate_pages(inode);
  4585. if (inode->i_nlink &&
  4586. ((btrfs_root_refs(&root->root_item) != 0 &&
  4587. root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID) ||
  4588. btrfs_is_free_space_inode(BTRFS_I(inode))))
  4589. goto no_delete;
  4590. if (is_bad_inode(inode))
  4591. goto no_delete;
  4592. btrfs_free_io_failure_record(BTRFS_I(inode), 0, (u64)-1);
  4593. if (test_bit(BTRFS_FS_LOG_RECOVERING, &fs_info->flags))
  4594. goto no_delete;
  4595. if (inode->i_nlink > 0) {
  4596. BUG_ON(btrfs_root_refs(&root->root_item) != 0 &&
  4597. root->root_key.objectid != BTRFS_ROOT_TREE_OBJECTID);
  4598. goto no_delete;
  4599. }
  4600. ret = btrfs_commit_inode_delayed_inode(BTRFS_I(inode));
  4601. if (ret)
  4602. goto no_delete;
  4603. rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
  4604. if (!rsv)
  4605. goto no_delete;
  4606. rsv->size = btrfs_calc_metadata_size(fs_info, 1);
  4607. rsv->failfast = 1;
  4608. btrfs_i_size_write(BTRFS_I(inode), 0);
  4609. while (1) {
  4610. trans = evict_refill_and_join(root, rsv);
  4611. if (IS_ERR(trans))
  4612. goto free_rsv;
  4613. trans->block_rsv = rsv;
  4614. ret = btrfs_truncate_inode_items(trans, root, inode, 0, 0);
  4615. trans->block_rsv = &fs_info->trans_block_rsv;
  4616. btrfs_end_transaction(trans);
  4617. btrfs_btree_balance_dirty(fs_info);
  4618. if (ret && ret != -ENOSPC && ret != -EAGAIN)
  4619. goto free_rsv;
  4620. else if (!ret)
  4621. break;
  4622. }
  4623. /*
  4624. * Errors here aren't a big deal, it just means we leave orphan items in
  4625. * the tree. They will be cleaned up on the next mount. If the inode
  4626. * number gets reused, cleanup deletes the orphan item without doing
  4627. * anything, and unlink reuses the existing orphan item.
  4628. *
  4629. * If it turns out that we are dropping too many of these, we might want
  4630. * to add a mechanism for retrying these after a commit.
  4631. */
  4632. trans = evict_refill_and_join(root, rsv);
  4633. if (!IS_ERR(trans)) {
  4634. trans->block_rsv = rsv;
  4635. btrfs_orphan_del(trans, BTRFS_I(inode));
  4636. trans->block_rsv = &fs_info->trans_block_rsv;
  4637. btrfs_end_transaction(trans);
  4638. }
  4639. if (!(root == fs_info->tree_root ||
  4640. root->root_key.objectid == BTRFS_TREE_RELOC_OBJECTID))
  4641. btrfs_return_ino(root, btrfs_ino(BTRFS_I(inode)));
  4642. free_rsv:
  4643. btrfs_free_block_rsv(fs_info, rsv);
  4644. no_delete:
  4645. /*
  4646. * If we didn't successfully delete, the orphan item will still be in
  4647. * the tree and we'll retry on the next mount. Again, we might also want
  4648. * to retry these periodically in the future.
  4649. */
  4650. btrfs_remove_delayed_node(BTRFS_I(inode));
  4651. clear_inode(inode);
  4652. }
  4653. /*
  4654. * Return the key found in the dir entry in the location pointer, fill @type
  4655. * with BTRFS_FT_*, and return 0.
  4656. *
  4657. * If no dir entries were found, returns -ENOENT.
  4658. * If found a corrupted location in dir entry, returns -EUCLEAN.
  4659. */
  4660. static int btrfs_inode_by_name(struct inode *dir, struct dentry *dentry,
  4661. struct btrfs_key *location, u8 *type)
  4662. {
  4663. const char *name = dentry->d_name.name;
  4664. int namelen = dentry->d_name.len;
  4665. struct btrfs_dir_item *di;
  4666. struct btrfs_path *path;
  4667. struct btrfs_root *root = BTRFS_I(dir)->root;
  4668. int ret = 0;
  4669. path = btrfs_alloc_path();
  4670. if (!path)
  4671. return -ENOMEM;
  4672. di = btrfs_lookup_dir_item(NULL, root, path, btrfs_ino(BTRFS_I(dir)),
  4673. name, namelen, 0);
  4674. if (IS_ERR_OR_NULL(di)) {
  4675. ret = di ? PTR_ERR(di) : -ENOENT;
  4676. goto out;
  4677. }
  4678. btrfs_dir_item_key_to_cpu(path->nodes[0], di, location);
  4679. if (location->type != BTRFS_INODE_ITEM_KEY &&
  4680. location->type != BTRFS_ROOT_ITEM_KEY) {
  4681. ret = -EUCLEAN;
  4682. btrfs_warn(root->fs_info,
  4683. "%s gets something invalid in DIR_ITEM (name %s, directory ino %llu, location(%llu %u %llu))",
  4684. __func__, name, btrfs_ino(BTRFS_I(dir)),
  4685. location->objectid, location->type, location->offset);
  4686. }
  4687. if (!ret)
  4688. *type = btrfs_dir_type(path->nodes[0], di);
  4689. out:
  4690. btrfs_free_path(path);
  4691. return ret;
  4692. }
  4693. /*
  4694. * when we hit a tree root in a directory, the btrfs part of the inode
  4695. * needs to be changed to reflect the root directory of the tree root. This
  4696. * is kind of like crossing a mount point.
  4697. */
  4698. static int fixup_tree_root_location(struct btrfs_fs_info *fs_info,
  4699. struct inode *dir,
  4700. struct dentry *dentry,
  4701. struct btrfs_key *location,
  4702. struct btrfs_root **sub_root)
  4703. {
  4704. struct btrfs_path *path;
  4705. struct btrfs_root *new_root;
  4706. struct btrfs_root_ref *ref;
  4707. struct extent_buffer *leaf;
  4708. struct btrfs_key key;
  4709. int ret;
  4710. int err = 0;
  4711. path = btrfs_alloc_path();
  4712. if (!path) {
  4713. err = -ENOMEM;
  4714. goto out;
  4715. }
  4716. err = -ENOENT;
  4717. key.objectid = BTRFS_I(dir)->root->root_key.objectid;
  4718. key.type = BTRFS_ROOT_REF_KEY;
  4719. key.offset = location->objectid;
  4720. ret = btrfs_search_slot(NULL, fs_info->tree_root, &key, path, 0, 0);
  4721. if (ret) {
  4722. if (ret < 0)
  4723. err = ret;
  4724. goto out;
  4725. }
  4726. leaf = path->nodes[0];
  4727. ref = btrfs_item_ptr(leaf, path->slots[0], struct btrfs_root_ref);
  4728. if (btrfs_root_ref_dirid(leaf, ref) != btrfs_ino(BTRFS_I(dir)) ||
  4729. btrfs_root_ref_name_len(leaf, ref) != dentry->d_name.len)
  4730. goto out;
  4731. ret = memcmp_extent_buffer(leaf, dentry->d_name.name,
  4732. (unsigned long)(ref + 1),
  4733. dentry->d_name.len);
  4734. if (ret)
  4735. goto out;
  4736. btrfs_release_path(path);
  4737. new_root = btrfs_get_fs_root(fs_info, location->objectid, true);
  4738. if (IS_ERR(new_root)) {
  4739. err = PTR_ERR(new_root);
  4740. goto out;
  4741. }
  4742. *sub_root = new_root;
  4743. location->objectid = btrfs_root_dirid(&new_root->root_item);
  4744. location->type = BTRFS_INODE_ITEM_KEY;
  4745. location->offset = 0;
  4746. err = 0;
  4747. out:
  4748. btrfs_free_path(path);
  4749. return err;
  4750. }
  4751. static void inode_tree_add(struct inode *inode)
  4752. {
  4753. struct btrfs_root *root = BTRFS_I(inode)->root;
  4754. struct btrfs_inode *entry;
  4755. struct rb_node **p;
  4756. struct rb_node *parent;
  4757. struct rb_node *new = &BTRFS_I(inode)->rb_node;
  4758. u64 ino = btrfs_ino(BTRFS_I(inode));
  4759. if (inode_unhashed(inode))
  4760. return;
  4761. parent = NULL;
  4762. spin_lock(&root->inode_lock);
  4763. p = &root->inode_tree.rb_node;
  4764. while (*p) {
  4765. parent = *p;
  4766. entry = rb_entry(parent, struct btrfs_inode, rb_node);
  4767. if (ino < btrfs_ino(entry))
  4768. p = &parent->rb_left;
  4769. else if (ino > btrfs_ino(entry))
  4770. p = &parent->rb_right;
  4771. else {
  4772. WARN_ON(!(entry->vfs_inode.i_state &
  4773. (I_WILL_FREE | I_FREEING)));
  4774. rb_replace_node(parent, new, &root->inode_tree);
  4775. RB_CLEAR_NODE(parent);
  4776. spin_unlock(&root->inode_lock);
  4777. return;
  4778. }
  4779. }
  4780. rb_link_node(new, parent, p);
  4781. rb_insert_color(new, &root->inode_tree);
  4782. spin_unlock(&root->inode_lock);
  4783. }
  4784. static void inode_tree_del(struct btrfs_inode *inode)
  4785. {
  4786. struct btrfs_root *root = inode->root;
  4787. int empty = 0;
  4788. spin_lock(&root->inode_lock);
  4789. if (!RB_EMPTY_NODE(&inode->rb_node)) {
  4790. rb_erase(&inode->rb_node, &root->inode_tree);
  4791. RB_CLEAR_NODE(&inode->rb_node);
  4792. empty = RB_EMPTY_ROOT(&root->inode_tree);
  4793. }
  4794. spin_unlock(&root->inode_lock);
  4795. if (empty && btrfs_root_refs(&root->root_item) == 0) {
  4796. spin_lock(&root->inode_lock);
  4797. empty = RB_EMPTY_ROOT(&root->inode_tree);
  4798. spin_unlock(&root->inode_lock);
  4799. if (empty)
  4800. btrfs_add_dead_root(root);
  4801. }
  4802. }
  4803. static int btrfs_init_locked_inode(struct inode *inode, void *p)
  4804. {
  4805. struct btrfs_iget_args *args = p;
  4806. inode->i_ino = args->ino;
  4807. BTRFS_I(inode)->location.objectid = args->ino;
  4808. BTRFS_I(inode)->location.type = BTRFS_INODE_ITEM_KEY;
  4809. BTRFS_I(inode)->location.offset = 0;
  4810. BTRFS_I(inode)->root = btrfs_grab_root(args->root);
  4811. BUG_ON(args->root && !BTRFS_I(inode)->root);
  4812. return 0;
  4813. }
  4814. static int btrfs_find_actor(struct inode *inode, void *opaque)
  4815. {
  4816. struct btrfs_iget_args *args = opaque;
  4817. return args->ino == BTRFS_I(inode)->location.objectid &&
  4818. args->root == BTRFS_I(inode)->root;
  4819. }
  4820. static struct inode *btrfs_iget_locked(struct super_block *s, u64 ino,
  4821. struct btrfs_root *root)
  4822. {
  4823. struct inode *inode;
  4824. struct btrfs_iget_args args;
  4825. unsigned long hashval = btrfs_inode_hash(ino, root);
  4826. args.ino = ino;
  4827. args.root = root;
  4828. inode = iget5_locked(s, hashval, btrfs_find_actor,
  4829. btrfs_init_locked_inode,
  4830. (void *)&args);
  4831. return inode;
  4832. }
  4833. /*
  4834. * Get an inode object given its inode number and corresponding root.
  4835. * Path can be preallocated to prevent recursing back to iget through
  4836. * allocator. NULL is also valid but may require an additional allocation
  4837. * later.
  4838. */
  4839. struct inode *btrfs_iget_path(struct super_block *s, u64 ino,
  4840. struct btrfs_root *root, struct btrfs_path *path)
  4841. {
  4842. struct inode *inode;
  4843. inode = btrfs_iget_locked(s, ino, root);
  4844. if (!inode)
  4845. return ERR_PTR(-ENOMEM);
  4846. if (inode->i_state & I_NEW) {
  4847. int ret;
  4848. ret = btrfs_read_locked_inode(inode, path);
  4849. if (!ret) {
  4850. inode_tree_add(inode);
  4851. unlock_new_inode(inode);
  4852. } else {
  4853. iget_failed(inode);
  4854. /*
  4855. * ret > 0 can come from btrfs_search_slot called by
  4856. * btrfs_read_locked_inode, this means the inode item
  4857. * was not found.
  4858. */
  4859. if (ret > 0)
  4860. ret = -ENOENT;
  4861. inode = ERR_PTR(ret);
  4862. }
  4863. }
  4864. return inode;
  4865. }
  4866. struct inode *btrfs_iget(struct super_block *s, u64 ino, struct btrfs_root *root)
  4867. {
  4868. return btrfs_iget_path(s, ino, root, NULL);
  4869. }
  4870. static struct inode *new_simple_dir(struct super_block *s,
  4871. struct btrfs_key *key,
  4872. struct btrfs_root *root)
  4873. {
  4874. struct inode *inode = new_inode(s);
  4875. if (!inode)
  4876. return ERR_PTR(-ENOMEM);
  4877. BTRFS_I(inode)->root = btrfs_grab_root(root);
  4878. memcpy(&BTRFS_I(inode)->location, key, sizeof(*key));
  4879. set_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags);
  4880. inode->i_ino = BTRFS_EMPTY_SUBVOL_DIR_OBJECTID;
  4881. /*
  4882. * We only need lookup, the rest is read-only and there's no inode
  4883. * associated with the dentry
  4884. */
  4885. inode->i_op = &simple_dir_inode_operations;
  4886. inode->i_opflags &= ~IOP_XATTR;
  4887. inode->i_fop = &simple_dir_operations;
  4888. inode->i_mode = S_IFDIR | S_IRUGO | S_IWUSR | S_IXUGO;
  4889. inode->i_mtime = current_time(inode);
  4890. inode->i_atime = inode->i_mtime;
  4891. inode->i_ctime = inode->i_mtime;
  4892. BTRFS_I(inode)->i_otime = inode->i_mtime;
  4893. return inode;
  4894. }
  4895. static inline u8 btrfs_inode_type(struct inode *inode)
  4896. {
  4897. /*
  4898. * Compile-time asserts that generic FT_* types still match
  4899. * BTRFS_FT_* types
  4900. */
  4901. BUILD_BUG_ON(BTRFS_FT_UNKNOWN != FT_UNKNOWN);
  4902. BUILD_BUG_ON(BTRFS_FT_REG_FILE != FT_REG_FILE);
  4903. BUILD_BUG_ON(BTRFS_FT_DIR != FT_DIR);
  4904. BUILD_BUG_ON(BTRFS_FT_CHRDEV != FT_CHRDEV);
  4905. BUILD_BUG_ON(BTRFS_FT_BLKDEV != FT_BLKDEV);
  4906. BUILD_BUG_ON(BTRFS_FT_FIFO != FT_FIFO);
  4907. BUILD_BUG_ON(BTRFS_FT_SOCK != FT_SOCK);
  4908. BUILD_BUG_ON(BTRFS_FT_SYMLINK != FT_SYMLINK);
  4909. return fs_umode_to_ftype(inode->i_mode);
  4910. }
  4911. struct inode *btrfs_lookup_dentry(struct inode *dir, struct dentry *dentry)
  4912. {
  4913. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  4914. struct inode *inode;
  4915. struct btrfs_root *root = BTRFS_I(dir)->root;
  4916. struct btrfs_root *sub_root = root;
  4917. struct btrfs_key location;
  4918. u8 di_type = 0;
  4919. int ret = 0;
  4920. if (dentry->d_name.len > BTRFS_NAME_LEN)
  4921. return ERR_PTR(-ENAMETOOLONG);
  4922. ret = btrfs_inode_by_name(dir, dentry, &location, &di_type);
  4923. if (ret < 0)
  4924. return ERR_PTR(ret);
  4925. if (location.type == BTRFS_INODE_ITEM_KEY) {
  4926. inode = btrfs_iget(dir->i_sb, location.objectid, root);
  4927. if (IS_ERR(inode))
  4928. return inode;
  4929. /* Do extra check against inode mode with di_type */
  4930. if (btrfs_inode_type(inode) != di_type) {
  4931. btrfs_crit(fs_info,
  4932. "inode mode mismatch with dir: inode mode=0%o btrfs type=%u dir type=%u",
  4933. inode->i_mode, btrfs_inode_type(inode),
  4934. di_type);
  4935. iput(inode);
  4936. return ERR_PTR(-EUCLEAN);
  4937. }
  4938. return inode;
  4939. }
  4940. ret = fixup_tree_root_location(fs_info, dir, dentry,
  4941. &location, &sub_root);
  4942. if (ret < 0) {
  4943. if (ret != -ENOENT)
  4944. inode = ERR_PTR(ret);
  4945. else
  4946. inode = new_simple_dir(dir->i_sb, &location, sub_root);
  4947. } else {
  4948. inode = btrfs_iget(dir->i_sb, location.objectid, sub_root);
  4949. }
  4950. if (root != sub_root)
  4951. btrfs_put_root(sub_root);
  4952. if (!IS_ERR(inode) && root != sub_root) {
  4953. down_read(&fs_info->cleanup_work_sem);
  4954. if (!sb_rdonly(inode->i_sb))
  4955. ret = btrfs_orphan_cleanup(sub_root);
  4956. up_read(&fs_info->cleanup_work_sem);
  4957. if (ret) {
  4958. iput(inode);
  4959. inode = ERR_PTR(ret);
  4960. }
  4961. }
  4962. return inode;
  4963. }
  4964. static int btrfs_dentry_delete(const struct dentry *dentry)
  4965. {
  4966. struct btrfs_root *root;
  4967. struct inode *inode = d_inode(dentry);
  4968. if (!inode && !IS_ROOT(dentry))
  4969. inode = d_inode(dentry->d_parent);
  4970. if (inode) {
  4971. root = BTRFS_I(inode)->root;
  4972. if (btrfs_root_refs(&root->root_item) == 0)
  4973. return 1;
  4974. if (btrfs_ino(BTRFS_I(inode)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
  4975. return 1;
  4976. }
  4977. return 0;
  4978. }
  4979. static struct dentry *btrfs_lookup(struct inode *dir, struct dentry *dentry,
  4980. unsigned int flags)
  4981. {
  4982. struct inode *inode = btrfs_lookup_dentry(dir, dentry);
  4983. if (inode == ERR_PTR(-ENOENT))
  4984. inode = NULL;
  4985. return d_splice_alias(inode, dentry);
  4986. }
  4987. /*
  4988. * All this infrastructure exists because dir_emit can fault, and we are holding
  4989. * the tree lock when doing readdir. For now just allocate a buffer and copy
  4990. * our information into that, and then dir_emit from the buffer. This is
  4991. * similar to what NFS does, only we don't keep the buffer around in pagecache
  4992. * because I'm afraid I'll mess that up. Long term we need to make filldir do
  4993. * copy_to_user_inatomic so we don't have to worry about page faulting under the
  4994. * tree lock.
  4995. */
  4996. static int btrfs_opendir(struct inode *inode, struct file *file)
  4997. {
  4998. struct btrfs_file_private *private;
  4999. private = kzalloc(sizeof(struct btrfs_file_private), GFP_KERNEL);
  5000. if (!private)
  5001. return -ENOMEM;
  5002. private->filldir_buf = kzalloc(PAGE_SIZE, GFP_KERNEL);
  5003. if (!private->filldir_buf) {
  5004. kfree(private);
  5005. return -ENOMEM;
  5006. }
  5007. file->private_data = private;
  5008. return 0;
  5009. }
  5010. struct dir_entry {
  5011. u64 ino;
  5012. u64 offset;
  5013. unsigned type;
  5014. int name_len;
  5015. };
  5016. static int btrfs_filldir(void *addr, int entries, struct dir_context *ctx)
  5017. {
  5018. while (entries--) {
  5019. struct dir_entry *entry = addr;
  5020. char *name = (char *)(entry + 1);
  5021. ctx->pos = get_unaligned(&entry->offset);
  5022. if (!dir_emit(ctx, name, get_unaligned(&entry->name_len),
  5023. get_unaligned(&entry->ino),
  5024. get_unaligned(&entry->type)))
  5025. return 1;
  5026. addr += sizeof(struct dir_entry) +
  5027. get_unaligned(&entry->name_len);
  5028. ctx->pos++;
  5029. }
  5030. return 0;
  5031. }
  5032. static int btrfs_real_readdir(struct file *file, struct dir_context *ctx)
  5033. {
  5034. struct inode *inode = file_inode(file);
  5035. struct btrfs_root *root = BTRFS_I(inode)->root;
  5036. struct btrfs_file_private *private = file->private_data;
  5037. struct btrfs_dir_item *di;
  5038. struct btrfs_key key;
  5039. struct btrfs_key found_key;
  5040. struct btrfs_path *path;
  5041. void *addr;
  5042. struct list_head ins_list;
  5043. struct list_head del_list;
  5044. int ret;
  5045. struct extent_buffer *leaf;
  5046. int slot;
  5047. char *name_ptr;
  5048. int name_len;
  5049. int entries = 0;
  5050. int total_len = 0;
  5051. bool put = false;
  5052. struct btrfs_key location;
  5053. if (!dir_emit_dots(file, ctx))
  5054. return 0;
  5055. path = btrfs_alloc_path();
  5056. if (!path)
  5057. return -ENOMEM;
  5058. addr = private->filldir_buf;
  5059. path->reada = READA_FORWARD;
  5060. INIT_LIST_HEAD(&ins_list);
  5061. INIT_LIST_HEAD(&del_list);
  5062. put = btrfs_readdir_get_delayed_items(inode, &ins_list, &del_list);
  5063. again:
  5064. key.type = BTRFS_DIR_INDEX_KEY;
  5065. key.offset = ctx->pos;
  5066. key.objectid = btrfs_ino(BTRFS_I(inode));
  5067. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5068. if (ret < 0)
  5069. goto err;
  5070. while (1) {
  5071. struct dir_entry *entry;
  5072. leaf = path->nodes[0];
  5073. slot = path->slots[0];
  5074. if (slot >= btrfs_header_nritems(leaf)) {
  5075. ret = btrfs_next_leaf(root, path);
  5076. if (ret < 0)
  5077. goto err;
  5078. else if (ret > 0)
  5079. break;
  5080. continue;
  5081. }
  5082. btrfs_item_key_to_cpu(leaf, &found_key, slot);
  5083. if (found_key.objectid != key.objectid)
  5084. break;
  5085. if (found_key.type != BTRFS_DIR_INDEX_KEY)
  5086. break;
  5087. if (found_key.offset < ctx->pos)
  5088. goto next;
  5089. if (btrfs_should_delete_dir_index(&del_list, found_key.offset))
  5090. goto next;
  5091. di = btrfs_item_ptr(leaf, slot, struct btrfs_dir_item);
  5092. name_len = btrfs_dir_name_len(leaf, di);
  5093. if ((total_len + sizeof(struct dir_entry) + name_len) >=
  5094. PAGE_SIZE) {
  5095. btrfs_release_path(path);
  5096. ret = btrfs_filldir(private->filldir_buf, entries, ctx);
  5097. if (ret)
  5098. goto nopos;
  5099. addr = private->filldir_buf;
  5100. entries = 0;
  5101. total_len = 0;
  5102. goto again;
  5103. }
  5104. entry = addr;
  5105. put_unaligned(name_len, &entry->name_len);
  5106. name_ptr = (char *)(entry + 1);
  5107. read_extent_buffer(leaf, name_ptr, (unsigned long)(di + 1),
  5108. name_len);
  5109. put_unaligned(fs_ftype_to_dtype(btrfs_dir_type(leaf, di)),
  5110. &entry->type);
  5111. btrfs_dir_item_key_to_cpu(leaf, di, &location);
  5112. put_unaligned(location.objectid, &entry->ino);
  5113. put_unaligned(found_key.offset, &entry->offset);
  5114. entries++;
  5115. addr += sizeof(struct dir_entry) + name_len;
  5116. total_len += sizeof(struct dir_entry) + name_len;
  5117. next:
  5118. path->slots[0]++;
  5119. }
  5120. btrfs_release_path(path);
  5121. ret = btrfs_filldir(private->filldir_buf, entries, ctx);
  5122. if (ret)
  5123. goto nopos;
  5124. ret = btrfs_readdir_delayed_dir_index(ctx, &ins_list);
  5125. if (ret)
  5126. goto nopos;
  5127. /*
  5128. * Stop new entries from being returned after we return the last
  5129. * entry.
  5130. *
  5131. * New directory entries are assigned a strictly increasing
  5132. * offset. This means that new entries created during readdir
  5133. * are *guaranteed* to be seen in the future by that readdir.
  5134. * This has broken buggy programs which operate on names as
  5135. * they're returned by readdir. Until we re-use freed offsets
  5136. * we have this hack to stop new entries from being returned
  5137. * under the assumption that they'll never reach this huge
  5138. * offset.
  5139. *
  5140. * This is being careful not to overflow 32bit loff_t unless the
  5141. * last entry requires it because doing so has broken 32bit apps
  5142. * in the past.
  5143. */
  5144. if (ctx->pos >= INT_MAX)
  5145. ctx->pos = LLONG_MAX;
  5146. else
  5147. ctx->pos = INT_MAX;
  5148. nopos:
  5149. ret = 0;
  5150. err:
  5151. if (put)
  5152. btrfs_readdir_put_delayed_items(inode, &ins_list, &del_list);
  5153. btrfs_free_path(path);
  5154. return ret;
  5155. }
  5156. /*
  5157. * This is somewhat expensive, updating the tree every time the
  5158. * inode changes. But, it is most likely to find the inode in cache.
  5159. * FIXME, needs more benchmarking...there are no reasons other than performance
  5160. * to keep or drop this code.
  5161. */
  5162. static int btrfs_dirty_inode(struct inode *inode)
  5163. {
  5164. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  5165. struct btrfs_root *root = BTRFS_I(inode)->root;
  5166. struct btrfs_trans_handle *trans;
  5167. int ret;
  5168. if (test_bit(BTRFS_INODE_DUMMY, &BTRFS_I(inode)->runtime_flags))
  5169. return 0;
  5170. trans = btrfs_join_transaction(root);
  5171. if (IS_ERR(trans))
  5172. return PTR_ERR(trans);
  5173. ret = btrfs_update_inode(trans, root, inode);
  5174. if (ret && (ret == -ENOSPC || ret == -EDQUOT)) {
  5175. /* whoops, lets try again with the full transaction */
  5176. btrfs_end_transaction(trans);
  5177. trans = btrfs_start_transaction(root, 1);
  5178. if (IS_ERR(trans))
  5179. return PTR_ERR(trans);
  5180. ret = btrfs_update_inode(trans, root, inode);
  5181. }
  5182. btrfs_end_transaction(trans);
  5183. if (BTRFS_I(inode)->delayed_node)
  5184. btrfs_balance_delayed_items(fs_info);
  5185. return ret;
  5186. }
  5187. /*
  5188. * This is a copy of file_update_time. We need this so we can return error on
  5189. * ENOSPC for updating the inode in the case of file write and mmap writes.
  5190. */
  5191. static int btrfs_update_time(struct inode *inode, struct timespec64 *now,
  5192. int flags)
  5193. {
  5194. struct btrfs_root *root = BTRFS_I(inode)->root;
  5195. bool dirty = flags & ~S_VERSION;
  5196. if (btrfs_root_readonly(root))
  5197. return -EROFS;
  5198. if (flags & S_VERSION)
  5199. dirty |= inode_maybe_inc_iversion(inode, dirty);
  5200. if (flags & S_CTIME)
  5201. inode->i_ctime = *now;
  5202. if (flags & S_MTIME)
  5203. inode->i_mtime = *now;
  5204. if (flags & S_ATIME)
  5205. inode->i_atime = *now;
  5206. return dirty ? btrfs_dirty_inode(inode) : 0;
  5207. }
  5208. /*
  5209. * find the highest existing sequence number in a directory
  5210. * and then set the in-memory index_cnt variable to reflect
  5211. * free sequence numbers
  5212. */
  5213. static int btrfs_set_inode_index_count(struct btrfs_inode *inode)
  5214. {
  5215. struct btrfs_root *root = inode->root;
  5216. struct btrfs_key key, found_key;
  5217. struct btrfs_path *path;
  5218. struct extent_buffer *leaf;
  5219. int ret;
  5220. key.objectid = btrfs_ino(inode);
  5221. key.type = BTRFS_DIR_INDEX_KEY;
  5222. key.offset = (u64)-1;
  5223. path = btrfs_alloc_path();
  5224. if (!path)
  5225. return -ENOMEM;
  5226. ret = btrfs_search_slot(NULL, root, &key, path, 0, 0);
  5227. if (ret < 0)
  5228. goto out;
  5229. /* FIXME: we should be able to handle this */
  5230. if (ret == 0)
  5231. goto out;
  5232. ret = 0;
  5233. /*
  5234. * MAGIC NUMBER EXPLANATION:
  5235. * since we search a directory based on f_pos we have to start at 2
  5236. * since '.' and '..' have f_pos of 0 and 1 respectively, so everybody
  5237. * else has to start at 2
  5238. */
  5239. if (path->slots[0] == 0) {
  5240. inode->index_cnt = 2;
  5241. goto out;
  5242. }
  5243. path->slots[0]--;
  5244. leaf = path->nodes[0];
  5245. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  5246. if (found_key.objectid != btrfs_ino(inode) ||
  5247. found_key.type != BTRFS_DIR_INDEX_KEY) {
  5248. inode->index_cnt = 2;
  5249. goto out;
  5250. }
  5251. inode->index_cnt = found_key.offset + 1;
  5252. out:
  5253. btrfs_free_path(path);
  5254. return ret;
  5255. }
  5256. /*
  5257. * helper to find a free sequence number in a given directory. This current
  5258. * code is very simple, later versions will do smarter things in the btree
  5259. */
  5260. int btrfs_set_inode_index(struct btrfs_inode *dir, u64 *index)
  5261. {
  5262. int ret = 0;
  5263. if (dir->index_cnt == (u64)-1) {
  5264. ret = btrfs_inode_delayed_dir_index_count(dir);
  5265. if (ret) {
  5266. ret = btrfs_set_inode_index_count(dir);
  5267. if (ret)
  5268. return ret;
  5269. }
  5270. }
  5271. *index = dir->index_cnt;
  5272. dir->index_cnt++;
  5273. return ret;
  5274. }
  5275. static int btrfs_insert_inode_locked(struct inode *inode)
  5276. {
  5277. struct btrfs_iget_args args;
  5278. args.ino = BTRFS_I(inode)->location.objectid;
  5279. args.root = BTRFS_I(inode)->root;
  5280. return insert_inode_locked4(inode,
  5281. btrfs_inode_hash(inode->i_ino, BTRFS_I(inode)->root),
  5282. btrfs_find_actor, &args);
  5283. }
  5284. /*
  5285. * Inherit flags from the parent inode.
  5286. *
  5287. * Currently only the compression flags and the cow flags are inherited.
  5288. */
  5289. static void btrfs_inherit_iflags(struct inode *inode, struct inode *dir)
  5290. {
  5291. unsigned int flags;
  5292. if (!dir)
  5293. return;
  5294. flags = BTRFS_I(dir)->flags;
  5295. if (flags & BTRFS_INODE_NOCOMPRESS) {
  5296. BTRFS_I(inode)->flags &= ~BTRFS_INODE_COMPRESS;
  5297. BTRFS_I(inode)->flags |= BTRFS_INODE_NOCOMPRESS;
  5298. } else if (flags & BTRFS_INODE_COMPRESS) {
  5299. BTRFS_I(inode)->flags &= ~BTRFS_INODE_NOCOMPRESS;
  5300. BTRFS_I(inode)->flags |= BTRFS_INODE_COMPRESS;
  5301. }
  5302. if (flags & BTRFS_INODE_NODATACOW) {
  5303. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW;
  5304. if (S_ISREG(inode->i_mode))
  5305. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
  5306. }
  5307. btrfs_sync_inode_flags_to_i_flags(inode);
  5308. }
  5309. static struct inode *btrfs_new_inode(struct btrfs_trans_handle *trans,
  5310. struct btrfs_root *root,
  5311. struct inode *dir,
  5312. const char *name, int name_len,
  5313. u64 ref_objectid, u64 objectid,
  5314. umode_t mode, u64 *index)
  5315. {
  5316. struct btrfs_fs_info *fs_info = root->fs_info;
  5317. struct inode *inode;
  5318. struct btrfs_inode_item *inode_item;
  5319. struct btrfs_key *location;
  5320. struct btrfs_path *path;
  5321. struct btrfs_inode_ref *ref;
  5322. struct btrfs_key key[2];
  5323. u32 sizes[2];
  5324. int nitems = name ? 2 : 1;
  5325. unsigned long ptr;
  5326. unsigned int nofs_flag;
  5327. int ret;
  5328. path = btrfs_alloc_path();
  5329. if (!path)
  5330. return ERR_PTR(-ENOMEM);
  5331. nofs_flag = memalloc_nofs_save();
  5332. inode = new_inode(fs_info->sb);
  5333. memalloc_nofs_restore(nofs_flag);
  5334. if (!inode) {
  5335. btrfs_free_path(path);
  5336. return ERR_PTR(-ENOMEM);
  5337. }
  5338. /*
  5339. * O_TMPFILE, set link count to 0, so that after this point,
  5340. * we fill in an inode item with the correct link count.
  5341. */
  5342. if (!name)
  5343. set_nlink(inode, 0);
  5344. /*
  5345. * we have to initialize this early, so we can reclaim the inode
  5346. * number if we fail afterwards in this function.
  5347. */
  5348. inode->i_ino = objectid;
  5349. if (dir && name) {
  5350. trace_btrfs_inode_request(dir);
  5351. ret = btrfs_set_inode_index(BTRFS_I(dir), index);
  5352. if (ret) {
  5353. btrfs_free_path(path);
  5354. iput(inode);
  5355. return ERR_PTR(ret);
  5356. }
  5357. } else if (dir) {
  5358. *index = 0;
  5359. }
  5360. /*
  5361. * index_cnt is ignored for everything but a dir,
  5362. * btrfs_set_inode_index_count has an explanation for the magic
  5363. * number
  5364. */
  5365. BTRFS_I(inode)->index_cnt = 2;
  5366. BTRFS_I(inode)->dir_index = *index;
  5367. BTRFS_I(inode)->root = btrfs_grab_root(root);
  5368. BTRFS_I(inode)->generation = trans->transid;
  5369. inode->i_generation = BTRFS_I(inode)->generation;
  5370. /*
  5371. * We could have gotten an inode number from somebody who was fsynced
  5372. * and then removed in this same transaction, so let's just set full
  5373. * sync since it will be a full sync anyway and this will blow away the
  5374. * old info in the log.
  5375. */
  5376. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
  5377. key[0].objectid = objectid;
  5378. key[0].type = BTRFS_INODE_ITEM_KEY;
  5379. key[0].offset = 0;
  5380. sizes[0] = sizeof(struct btrfs_inode_item);
  5381. if (name) {
  5382. /*
  5383. * Start new inodes with an inode_ref. This is slightly more
  5384. * efficient for small numbers of hard links since they will
  5385. * be packed into one item. Extended refs will kick in if we
  5386. * add more hard links than can fit in the ref item.
  5387. */
  5388. key[1].objectid = objectid;
  5389. key[1].type = BTRFS_INODE_REF_KEY;
  5390. key[1].offset = ref_objectid;
  5391. sizes[1] = name_len + sizeof(*ref);
  5392. }
  5393. location = &BTRFS_I(inode)->location;
  5394. location->objectid = objectid;
  5395. location->offset = 0;
  5396. location->type = BTRFS_INODE_ITEM_KEY;
  5397. ret = btrfs_insert_inode_locked(inode);
  5398. if (ret < 0) {
  5399. iput(inode);
  5400. goto fail;
  5401. }
  5402. path->leave_spinning = 1;
  5403. ret = btrfs_insert_empty_items(trans, root, path, key, sizes, nitems);
  5404. if (ret != 0)
  5405. goto fail_unlock;
  5406. inode_init_owner(inode, dir, mode);
  5407. inode_set_bytes(inode, 0);
  5408. inode->i_mtime = current_time(inode);
  5409. inode->i_atime = inode->i_mtime;
  5410. inode->i_ctime = inode->i_mtime;
  5411. BTRFS_I(inode)->i_otime = inode->i_mtime;
  5412. inode_item = btrfs_item_ptr(path->nodes[0], path->slots[0],
  5413. struct btrfs_inode_item);
  5414. memzero_extent_buffer(path->nodes[0], (unsigned long)inode_item,
  5415. sizeof(*inode_item));
  5416. fill_inode_item(trans, path->nodes[0], inode_item, inode);
  5417. if (name) {
  5418. ref = btrfs_item_ptr(path->nodes[0], path->slots[0] + 1,
  5419. struct btrfs_inode_ref);
  5420. btrfs_set_inode_ref_name_len(path->nodes[0], ref, name_len);
  5421. btrfs_set_inode_ref_index(path->nodes[0], ref, *index);
  5422. ptr = (unsigned long)(ref + 1);
  5423. write_extent_buffer(path->nodes[0], name, ptr, name_len);
  5424. }
  5425. btrfs_mark_buffer_dirty(path->nodes[0]);
  5426. btrfs_free_path(path);
  5427. btrfs_inherit_iflags(inode, dir);
  5428. if (S_ISREG(mode)) {
  5429. if (btrfs_test_opt(fs_info, NODATASUM))
  5430. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATASUM;
  5431. if (btrfs_test_opt(fs_info, NODATACOW))
  5432. BTRFS_I(inode)->flags |= BTRFS_INODE_NODATACOW |
  5433. BTRFS_INODE_NODATASUM;
  5434. }
  5435. inode_tree_add(inode);
  5436. trace_btrfs_inode_new(inode);
  5437. btrfs_set_inode_last_trans(trans, BTRFS_I(inode));
  5438. btrfs_update_root_times(trans, root);
  5439. ret = btrfs_inode_inherit_props(trans, inode, dir);
  5440. if (ret)
  5441. btrfs_err(fs_info,
  5442. "error inheriting props for ino %llu (root %llu): %d",
  5443. btrfs_ino(BTRFS_I(inode)), root->root_key.objectid, ret);
  5444. return inode;
  5445. fail_unlock:
  5446. discard_new_inode(inode);
  5447. fail:
  5448. if (dir && name)
  5449. BTRFS_I(dir)->index_cnt--;
  5450. btrfs_free_path(path);
  5451. return ERR_PTR(ret);
  5452. }
  5453. /*
  5454. * utility function to add 'inode' into 'parent_inode' with
  5455. * a give name and a given sequence number.
  5456. * if 'add_backref' is true, also insert a backref from the
  5457. * inode to the parent directory.
  5458. */
  5459. int btrfs_add_link(struct btrfs_trans_handle *trans,
  5460. struct btrfs_inode *parent_inode, struct btrfs_inode *inode,
  5461. const char *name, int name_len, int add_backref, u64 index)
  5462. {
  5463. int ret = 0;
  5464. struct btrfs_key key;
  5465. struct btrfs_root *root = parent_inode->root;
  5466. u64 ino = btrfs_ino(inode);
  5467. u64 parent_ino = btrfs_ino(parent_inode);
  5468. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5469. memcpy(&key, &inode->root->root_key, sizeof(key));
  5470. } else {
  5471. key.objectid = ino;
  5472. key.type = BTRFS_INODE_ITEM_KEY;
  5473. key.offset = 0;
  5474. }
  5475. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5476. ret = btrfs_add_root_ref(trans, key.objectid,
  5477. root->root_key.objectid, parent_ino,
  5478. index, name, name_len);
  5479. } else if (add_backref) {
  5480. ret = btrfs_insert_inode_ref(trans, root, name, name_len, ino,
  5481. parent_ino, index);
  5482. }
  5483. /* Nothing to clean up yet */
  5484. if (ret)
  5485. return ret;
  5486. ret = btrfs_insert_dir_item(trans, name, name_len, parent_inode, &key,
  5487. btrfs_inode_type(&inode->vfs_inode), index);
  5488. if (ret == -EEXIST || ret == -EOVERFLOW)
  5489. goto fail_dir_item;
  5490. else if (ret) {
  5491. btrfs_abort_transaction(trans, ret);
  5492. return ret;
  5493. }
  5494. btrfs_i_size_write(parent_inode, parent_inode->vfs_inode.i_size +
  5495. name_len * 2);
  5496. inode_inc_iversion(&parent_inode->vfs_inode);
  5497. /*
  5498. * If we are replaying a log tree, we do not want to update the mtime
  5499. * and ctime of the parent directory with the current time, since the
  5500. * log replay procedure is responsible for setting them to their correct
  5501. * values (the ones it had when the fsync was done).
  5502. */
  5503. if (!test_bit(BTRFS_FS_LOG_RECOVERING, &root->fs_info->flags)) {
  5504. struct timespec64 now = current_time(&parent_inode->vfs_inode);
  5505. parent_inode->vfs_inode.i_mtime = now;
  5506. parent_inode->vfs_inode.i_ctime = now;
  5507. }
  5508. ret = btrfs_update_inode(trans, root, &parent_inode->vfs_inode);
  5509. if (ret)
  5510. btrfs_abort_transaction(trans, ret);
  5511. return ret;
  5512. fail_dir_item:
  5513. if (unlikely(ino == BTRFS_FIRST_FREE_OBJECTID)) {
  5514. u64 local_index;
  5515. int err;
  5516. err = btrfs_del_root_ref(trans, key.objectid,
  5517. root->root_key.objectid, parent_ino,
  5518. &local_index, name, name_len);
  5519. if (err)
  5520. btrfs_abort_transaction(trans, err);
  5521. } else if (add_backref) {
  5522. u64 local_index;
  5523. int err;
  5524. err = btrfs_del_inode_ref(trans, root, name, name_len,
  5525. ino, parent_ino, &local_index);
  5526. if (err)
  5527. btrfs_abort_transaction(trans, err);
  5528. }
  5529. /* Return the original error code */
  5530. return ret;
  5531. }
  5532. static int btrfs_add_nondir(struct btrfs_trans_handle *trans,
  5533. struct btrfs_inode *dir, struct dentry *dentry,
  5534. struct btrfs_inode *inode, int backref, u64 index)
  5535. {
  5536. int err = btrfs_add_link(trans, dir, inode,
  5537. dentry->d_name.name, dentry->d_name.len,
  5538. backref, index);
  5539. if (err > 0)
  5540. err = -EEXIST;
  5541. return err;
  5542. }
  5543. static int btrfs_mknod(struct inode *dir, struct dentry *dentry,
  5544. umode_t mode, dev_t rdev)
  5545. {
  5546. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5547. struct btrfs_trans_handle *trans;
  5548. struct btrfs_root *root = BTRFS_I(dir)->root;
  5549. struct inode *inode = NULL;
  5550. int err;
  5551. u64 objectid;
  5552. u64 index = 0;
  5553. /*
  5554. * 2 for inode item and ref
  5555. * 2 for dir items
  5556. * 1 for xattr if selinux is on
  5557. */
  5558. trans = btrfs_start_transaction(root, 5);
  5559. if (IS_ERR(trans))
  5560. return PTR_ERR(trans);
  5561. err = btrfs_find_free_ino(root, &objectid);
  5562. if (err)
  5563. goto out_unlock;
  5564. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5565. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5566. mode, &index);
  5567. if (IS_ERR(inode)) {
  5568. err = PTR_ERR(inode);
  5569. inode = NULL;
  5570. goto out_unlock;
  5571. }
  5572. /*
  5573. * If the active LSM wants to access the inode during
  5574. * d_instantiate it needs these. Smack checks to see
  5575. * if the filesystem supports xattrs by looking at the
  5576. * ops vector.
  5577. */
  5578. inode->i_op = &btrfs_special_inode_operations;
  5579. init_special_inode(inode, inode->i_mode, rdev);
  5580. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5581. if (err)
  5582. goto out_unlock;
  5583. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5584. 0, index);
  5585. if (err)
  5586. goto out_unlock;
  5587. btrfs_update_inode(trans, root, inode);
  5588. d_instantiate_new(dentry, inode);
  5589. out_unlock:
  5590. btrfs_end_transaction(trans);
  5591. btrfs_btree_balance_dirty(fs_info);
  5592. if (err && inode) {
  5593. inode_dec_link_count(inode);
  5594. discard_new_inode(inode);
  5595. }
  5596. return err;
  5597. }
  5598. static int btrfs_create(struct inode *dir, struct dentry *dentry,
  5599. umode_t mode, bool excl)
  5600. {
  5601. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5602. struct btrfs_trans_handle *trans;
  5603. struct btrfs_root *root = BTRFS_I(dir)->root;
  5604. struct inode *inode = NULL;
  5605. int err;
  5606. u64 objectid;
  5607. u64 index = 0;
  5608. /*
  5609. * 2 for inode item and ref
  5610. * 2 for dir items
  5611. * 1 for xattr if selinux is on
  5612. */
  5613. trans = btrfs_start_transaction(root, 5);
  5614. if (IS_ERR(trans))
  5615. return PTR_ERR(trans);
  5616. err = btrfs_find_free_ino(root, &objectid);
  5617. if (err)
  5618. goto out_unlock;
  5619. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5620. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5621. mode, &index);
  5622. if (IS_ERR(inode)) {
  5623. err = PTR_ERR(inode);
  5624. inode = NULL;
  5625. goto out_unlock;
  5626. }
  5627. /*
  5628. * If the active LSM wants to access the inode during
  5629. * d_instantiate it needs these. Smack checks to see
  5630. * if the filesystem supports xattrs by looking at the
  5631. * ops vector.
  5632. */
  5633. inode->i_fop = &btrfs_file_operations;
  5634. inode->i_op = &btrfs_file_inode_operations;
  5635. inode->i_mapping->a_ops = &btrfs_aops;
  5636. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5637. if (err)
  5638. goto out_unlock;
  5639. err = btrfs_update_inode(trans, root, inode);
  5640. if (err)
  5641. goto out_unlock;
  5642. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5643. 0, index);
  5644. if (err)
  5645. goto out_unlock;
  5646. d_instantiate_new(dentry, inode);
  5647. out_unlock:
  5648. btrfs_end_transaction(trans);
  5649. if (err && inode) {
  5650. inode_dec_link_count(inode);
  5651. discard_new_inode(inode);
  5652. }
  5653. btrfs_btree_balance_dirty(fs_info);
  5654. return err;
  5655. }
  5656. static int btrfs_link(struct dentry *old_dentry, struct inode *dir,
  5657. struct dentry *dentry)
  5658. {
  5659. struct btrfs_trans_handle *trans = NULL;
  5660. struct btrfs_root *root = BTRFS_I(dir)->root;
  5661. struct inode *inode = d_inode(old_dentry);
  5662. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  5663. u64 index;
  5664. int err;
  5665. int drop_inode = 0;
  5666. /* do not allow sys_link's with other subvols of the same device */
  5667. if (root->root_key.objectid != BTRFS_I(inode)->root->root_key.objectid)
  5668. return -EXDEV;
  5669. if (inode->i_nlink >= BTRFS_LINK_MAX)
  5670. return -EMLINK;
  5671. err = btrfs_set_inode_index(BTRFS_I(dir), &index);
  5672. if (err)
  5673. goto fail;
  5674. /*
  5675. * 2 items for inode and inode ref
  5676. * 2 items for dir items
  5677. * 1 item for parent inode
  5678. * 1 item for orphan item deletion if O_TMPFILE
  5679. */
  5680. trans = btrfs_start_transaction(root, inode->i_nlink ? 5 : 6);
  5681. if (IS_ERR(trans)) {
  5682. err = PTR_ERR(trans);
  5683. trans = NULL;
  5684. goto fail;
  5685. }
  5686. /* There are several dir indexes for this inode, clear the cache. */
  5687. BTRFS_I(inode)->dir_index = 0ULL;
  5688. inc_nlink(inode);
  5689. inode_inc_iversion(inode);
  5690. inode->i_ctime = current_time(inode);
  5691. ihold(inode);
  5692. set_bit(BTRFS_INODE_COPY_EVERYTHING, &BTRFS_I(inode)->runtime_flags);
  5693. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry, BTRFS_I(inode),
  5694. 1, index);
  5695. if (err) {
  5696. drop_inode = 1;
  5697. } else {
  5698. struct dentry *parent = dentry->d_parent;
  5699. err = btrfs_update_inode(trans, root, inode);
  5700. if (err)
  5701. goto fail;
  5702. if (inode->i_nlink == 1) {
  5703. /*
  5704. * If new hard link count is 1, it's a file created
  5705. * with open(2) O_TMPFILE flag.
  5706. */
  5707. err = btrfs_orphan_del(trans, BTRFS_I(inode));
  5708. if (err)
  5709. goto fail;
  5710. }
  5711. d_instantiate(dentry, inode);
  5712. btrfs_log_new_name(trans, BTRFS_I(inode), NULL, parent);
  5713. }
  5714. fail:
  5715. if (trans)
  5716. btrfs_end_transaction(trans);
  5717. if (drop_inode) {
  5718. inode_dec_link_count(inode);
  5719. iput(inode);
  5720. }
  5721. btrfs_btree_balance_dirty(fs_info);
  5722. return err;
  5723. }
  5724. static int btrfs_mkdir(struct inode *dir, struct dentry *dentry, umode_t mode)
  5725. {
  5726. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  5727. struct inode *inode = NULL;
  5728. struct btrfs_trans_handle *trans;
  5729. struct btrfs_root *root = BTRFS_I(dir)->root;
  5730. int err = 0;
  5731. u64 objectid = 0;
  5732. u64 index = 0;
  5733. /*
  5734. * 2 items for inode and ref
  5735. * 2 items for dir items
  5736. * 1 for xattr if selinux is on
  5737. */
  5738. trans = btrfs_start_transaction(root, 5);
  5739. if (IS_ERR(trans))
  5740. return PTR_ERR(trans);
  5741. err = btrfs_find_free_ino(root, &objectid);
  5742. if (err)
  5743. goto out_fail;
  5744. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  5745. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)), objectid,
  5746. S_IFDIR | mode, &index);
  5747. if (IS_ERR(inode)) {
  5748. err = PTR_ERR(inode);
  5749. inode = NULL;
  5750. goto out_fail;
  5751. }
  5752. /* these must be set before we unlock the inode */
  5753. inode->i_op = &btrfs_dir_inode_operations;
  5754. inode->i_fop = &btrfs_dir_file_operations;
  5755. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  5756. if (err)
  5757. goto out_fail;
  5758. btrfs_i_size_write(BTRFS_I(inode), 0);
  5759. err = btrfs_update_inode(trans, root, inode);
  5760. if (err)
  5761. goto out_fail;
  5762. err = btrfs_add_link(trans, BTRFS_I(dir), BTRFS_I(inode),
  5763. dentry->d_name.name,
  5764. dentry->d_name.len, 0, index);
  5765. if (err)
  5766. goto out_fail;
  5767. d_instantiate_new(dentry, inode);
  5768. out_fail:
  5769. btrfs_end_transaction(trans);
  5770. if (err && inode) {
  5771. inode_dec_link_count(inode);
  5772. discard_new_inode(inode);
  5773. }
  5774. btrfs_btree_balance_dirty(fs_info);
  5775. return err;
  5776. }
  5777. static noinline int uncompress_inline(struct btrfs_path *path,
  5778. struct page *page,
  5779. size_t pg_offset, u64 extent_offset,
  5780. struct btrfs_file_extent_item *item)
  5781. {
  5782. int ret;
  5783. struct extent_buffer *leaf = path->nodes[0];
  5784. char *tmp;
  5785. size_t max_size;
  5786. unsigned long inline_size;
  5787. unsigned long ptr;
  5788. int compress_type;
  5789. WARN_ON(pg_offset != 0);
  5790. compress_type = btrfs_file_extent_compression(leaf, item);
  5791. max_size = btrfs_file_extent_ram_bytes(leaf, item);
  5792. inline_size = btrfs_file_extent_inline_item_len(leaf,
  5793. btrfs_item_nr(path->slots[0]));
  5794. tmp = kmalloc(inline_size, GFP_NOFS);
  5795. if (!tmp)
  5796. return -ENOMEM;
  5797. ptr = btrfs_file_extent_inline_start(item);
  5798. read_extent_buffer(leaf, tmp, ptr, inline_size);
  5799. max_size = min_t(unsigned long, PAGE_SIZE, max_size);
  5800. ret = btrfs_decompress(compress_type, tmp, page,
  5801. extent_offset, inline_size, max_size);
  5802. /*
  5803. * decompression code contains a memset to fill in any space between the end
  5804. * of the uncompressed data and the end of max_size in case the decompressed
  5805. * data ends up shorter than ram_bytes. That doesn't cover the hole between
  5806. * the end of an inline extent and the beginning of the next block, so we
  5807. * cover that region here.
  5808. */
  5809. if (max_size + pg_offset < PAGE_SIZE) {
  5810. char *map = kmap(page);
  5811. memset(map + pg_offset + max_size, 0, PAGE_SIZE - max_size - pg_offset);
  5812. kunmap(page);
  5813. }
  5814. kfree(tmp);
  5815. return ret;
  5816. }
  5817. /**
  5818. * btrfs_get_extent - Lookup the first extent overlapping a range in a file.
  5819. * @inode: file to search in
  5820. * @page: page to read extent data into if the extent is inline
  5821. * @pg_offset: offset into @page to copy to
  5822. * @start: file offset
  5823. * @len: length of range starting at @start
  5824. *
  5825. * This returns the first &struct extent_map which overlaps with the given
  5826. * range, reading it from the B-tree and caching it if necessary. Note that
  5827. * there may be more extents which overlap the given range after the returned
  5828. * extent_map.
  5829. *
  5830. * If @page is not NULL and the extent is inline, this also reads the extent
  5831. * data directly into the page and marks the extent up to date in the io_tree.
  5832. *
  5833. * Return: ERR_PTR on error, non-NULL extent_map on success.
  5834. */
  5835. struct extent_map *btrfs_get_extent(struct btrfs_inode *inode,
  5836. struct page *page, size_t pg_offset,
  5837. u64 start, u64 len)
  5838. {
  5839. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  5840. int ret = 0;
  5841. u64 extent_start = 0;
  5842. u64 extent_end = 0;
  5843. u64 objectid = btrfs_ino(inode);
  5844. int extent_type = -1;
  5845. struct btrfs_path *path = NULL;
  5846. struct btrfs_root *root = inode->root;
  5847. struct btrfs_file_extent_item *item;
  5848. struct extent_buffer *leaf;
  5849. struct btrfs_key found_key;
  5850. struct extent_map *em = NULL;
  5851. struct extent_map_tree *em_tree = &inode->extent_tree;
  5852. struct extent_io_tree *io_tree = &inode->io_tree;
  5853. read_lock(&em_tree->lock);
  5854. em = lookup_extent_mapping(em_tree, start, len);
  5855. read_unlock(&em_tree->lock);
  5856. if (em) {
  5857. if (em->start > start || em->start + em->len <= start)
  5858. free_extent_map(em);
  5859. else if (em->block_start == EXTENT_MAP_INLINE && page)
  5860. free_extent_map(em);
  5861. else
  5862. goto out;
  5863. }
  5864. em = alloc_extent_map();
  5865. if (!em) {
  5866. ret = -ENOMEM;
  5867. goto out;
  5868. }
  5869. em->start = EXTENT_MAP_HOLE;
  5870. em->orig_start = EXTENT_MAP_HOLE;
  5871. em->len = (u64)-1;
  5872. em->block_len = (u64)-1;
  5873. path = btrfs_alloc_path();
  5874. if (!path) {
  5875. ret = -ENOMEM;
  5876. goto out;
  5877. }
  5878. /* Chances are we'll be called again, so go ahead and do readahead */
  5879. path->reada = READA_FORWARD;
  5880. /*
  5881. * Unless we're going to uncompress the inline extent, no sleep would
  5882. * happen.
  5883. */
  5884. path->leave_spinning = 1;
  5885. path->recurse = btrfs_is_free_space_inode(inode);
  5886. ret = btrfs_lookup_file_extent(NULL, root, path, objectid, start, 0);
  5887. if (ret < 0) {
  5888. goto out;
  5889. } else if (ret > 0) {
  5890. if (path->slots[0] == 0)
  5891. goto not_found;
  5892. path->slots[0]--;
  5893. ret = 0;
  5894. }
  5895. leaf = path->nodes[0];
  5896. item = btrfs_item_ptr(leaf, path->slots[0],
  5897. struct btrfs_file_extent_item);
  5898. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  5899. if (found_key.objectid != objectid ||
  5900. found_key.type != BTRFS_EXTENT_DATA_KEY) {
  5901. /*
  5902. * If we backup past the first extent we want to move forward
  5903. * and see if there is an extent in front of us, otherwise we'll
  5904. * say there is a hole for our whole search range which can
  5905. * cause problems.
  5906. */
  5907. extent_end = start;
  5908. goto next;
  5909. }
  5910. extent_type = btrfs_file_extent_type(leaf, item);
  5911. extent_start = found_key.offset;
  5912. extent_end = btrfs_file_extent_end(path);
  5913. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  5914. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  5915. /* Only regular file could have regular/prealloc extent */
  5916. if (!S_ISREG(inode->vfs_inode.i_mode)) {
  5917. ret = -EUCLEAN;
  5918. btrfs_crit(fs_info,
  5919. "regular/prealloc extent found for non-regular inode %llu",
  5920. btrfs_ino(inode));
  5921. goto out;
  5922. }
  5923. trace_btrfs_get_extent_show_fi_regular(inode, leaf, item,
  5924. extent_start);
  5925. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  5926. trace_btrfs_get_extent_show_fi_inline(inode, leaf, item,
  5927. path->slots[0],
  5928. extent_start);
  5929. }
  5930. next:
  5931. if (start >= extent_end) {
  5932. path->slots[0]++;
  5933. if (path->slots[0] >= btrfs_header_nritems(leaf)) {
  5934. ret = btrfs_next_leaf(root, path);
  5935. if (ret < 0)
  5936. goto out;
  5937. else if (ret > 0)
  5938. goto not_found;
  5939. leaf = path->nodes[0];
  5940. }
  5941. btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
  5942. if (found_key.objectid != objectid ||
  5943. found_key.type != BTRFS_EXTENT_DATA_KEY)
  5944. goto not_found;
  5945. if (start + len <= found_key.offset)
  5946. goto not_found;
  5947. if (start > found_key.offset)
  5948. goto next;
  5949. /* New extent overlaps with existing one */
  5950. em->start = start;
  5951. em->orig_start = start;
  5952. em->len = found_key.offset - start;
  5953. em->block_start = EXTENT_MAP_HOLE;
  5954. goto insert;
  5955. }
  5956. btrfs_extent_item_to_extent_map(inode, path, item, !page, em);
  5957. if (extent_type == BTRFS_FILE_EXTENT_REG ||
  5958. extent_type == BTRFS_FILE_EXTENT_PREALLOC) {
  5959. goto insert;
  5960. } else if (extent_type == BTRFS_FILE_EXTENT_INLINE) {
  5961. unsigned long ptr;
  5962. char *map;
  5963. size_t size;
  5964. size_t extent_offset;
  5965. size_t copy_size;
  5966. if (!page)
  5967. goto out;
  5968. size = btrfs_file_extent_ram_bytes(leaf, item);
  5969. extent_offset = page_offset(page) + pg_offset - extent_start;
  5970. copy_size = min_t(u64, PAGE_SIZE - pg_offset,
  5971. size - extent_offset);
  5972. em->start = extent_start + extent_offset;
  5973. em->len = ALIGN(copy_size, fs_info->sectorsize);
  5974. em->orig_block_len = em->len;
  5975. em->orig_start = em->start;
  5976. ptr = btrfs_file_extent_inline_start(item) + extent_offset;
  5977. btrfs_set_path_blocking(path);
  5978. if (!PageUptodate(page)) {
  5979. if (btrfs_file_extent_compression(leaf, item) !=
  5980. BTRFS_COMPRESS_NONE) {
  5981. ret = uncompress_inline(path, page, pg_offset,
  5982. extent_offset, item);
  5983. if (ret)
  5984. goto out;
  5985. } else {
  5986. map = kmap(page);
  5987. read_extent_buffer(leaf, map + pg_offset, ptr,
  5988. copy_size);
  5989. if (pg_offset + copy_size < PAGE_SIZE) {
  5990. memset(map + pg_offset + copy_size, 0,
  5991. PAGE_SIZE - pg_offset -
  5992. copy_size);
  5993. }
  5994. kunmap(page);
  5995. }
  5996. flush_dcache_page(page);
  5997. }
  5998. set_extent_uptodate(io_tree, em->start,
  5999. extent_map_end(em) - 1, NULL, GFP_NOFS);
  6000. goto insert;
  6001. }
  6002. not_found:
  6003. em->start = start;
  6004. em->orig_start = start;
  6005. em->len = len;
  6006. em->block_start = EXTENT_MAP_HOLE;
  6007. insert:
  6008. ret = 0;
  6009. btrfs_release_path(path);
  6010. if (em->start > start || extent_map_end(em) <= start) {
  6011. btrfs_err(fs_info,
  6012. "bad extent! em: [%llu %llu] passed [%llu %llu]",
  6013. em->start, em->len, start, len);
  6014. ret = -EIO;
  6015. goto out;
  6016. }
  6017. write_lock(&em_tree->lock);
  6018. ret = btrfs_add_extent_mapping(fs_info, em_tree, &em, start, len);
  6019. write_unlock(&em_tree->lock);
  6020. out:
  6021. btrfs_free_path(path);
  6022. trace_btrfs_get_extent(root, inode, em);
  6023. if (ret) {
  6024. free_extent_map(em);
  6025. return ERR_PTR(ret);
  6026. }
  6027. return em;
  6028. }
  6029. struct extent_map *btrfs_get_extent_fiemap(struct btrfs_inode *inode,
  6030. u64 start, u64 len)
  6031. {
  6032. struct extent_map *em;
  6033. struct extent_map *hole_em = NULL;
  6034. u64 delalloc_start = start;
  6035. u64 end;
  6036. u64 delalloc_len;
  6037. u64 delalloc_end;
  6038. int err = 0;
  6039. em = btrfs_get_extent(inode, NULL, 0, start, len);
  6040. if (IS_ERR(em))
  6041. return em;
  6042. /*
  6043. * If our em maps to:
  6044. * - a hole or
  6045. * - a pre-alloc extent,
  6046. * there might actually be delalloc bytes behind it.
  6047. */
  6048. if (em->block_start != EXTENT_MAP_HOLE &&
  6049. !test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  6050. return em;
  6051. else
  6052. hole_em = em;
  6053. /* check to see if we've wrapped (len == -1 or similar) */
  6054. end = start + len;
  6055. if (end < start)
  6056. end = (u64)-1;
  6057. else
  6058. end -= 1;
  6059. em = NULL;
  6060. /* ok, we didn't find anything, lets look for delalloc */
  6061. delalloc_len = count_range_bits(&inode->io_tree, &delalloc_start,
  6062. end, len, EXTENT_DELALLOC, 1);
  6063. delalloc_end = delalloc_start + delalloc_len;
  6064. if (delalloc_end < delalloc_start)
  6065. delalloc_end = (u64)-1;
  6066. /*
  6067. * We didn't find anything useful, return the original results from
  6068. * get_extent()
  6069. */
  6070. if (delalloc_start > end || delalloc_end <= start) {
  6071. em = hole_em;
  6072. hole_em = NULL;
  6073. goto out;
  6074. }
  6075. /*
  6076. * Adjust the delalloc_start to make sure it doesn't go backwards from
  6077. * the start they passed in
  6078. */
  6079. delalloc_start = max(start, delalloc_start);
  6080. delalloc_len = delalloc_end - delalloc_start;
  6081. if (delalloc_len > 0) {
  6082. u64 hole_start;
  6083. u64 hole_len;
  6084. const u64 hole_end = extent_map_end(hole_em);
  6085. em = alloc_extent_map();
  6086. if (!em) {
  6087. err = -ENOMEM;
  6088. goto out;
  6089. }
  6090. ASSERT(hole_em);
  6091. /*
  6092. * When btrfs_get_extent can't find anything it returns one
  6093. * huge hole
  6094. *
  6095. * Make sure what it found really fits our range, and adjust to
  6096. * make sure it is based on the start from the caller
  6097. */
  6098. if (hole_end <= start || hole_em->start > end) {
  6099. free_extent_map(hole_em);
  6100. hole_em = NULL;
  6101. } else {
  6102. hole_start = max(hole_em->start, start);
  6103. hole_len = hole_end - hole_start;
  6104. }
  6105. if (hole_em && delalloc_start > hole_start) {
  6106. /*
  6107. * Our hole starts before our delalloc, so we have to
  6108. * return just the parts of the hole that go until the
  6109. * delalloc starts
  6110. */
  6111. em->len = min(hole_len, delalloc_start - hole_start);
  6112. em->start = hole_start;
  6113. em->orig_start = hole_start;
  6114. /*
  6115. * Don't adjust block start at all, it is fixed at
  6116. * EXTENT_MAP_HOLE
  6117. */
  6118. em->block_start = hole_em->block_start;
  6119. em->block_len = hole_len;
  6120. if (test_bit(EXTENT_FLAG_PREALLOC, &hole_em->flags))
  6121. set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
  6122. } else {
  6123. /*
  6124. * Hole is out of passed range or it starts after
  6125. * delalloc range
  6126. */
  6127. em->start = delalloc_start;
  6128. em->len = delalloc_len;
  6129. em->orig_start = delalloc_start;
  6130. em->block_start = EXTENT_MAP_DELALLOC;
  6131. em->block_len = delalloc_len;
  6132. }
  6133. } else {
  6134. return hole_em;
  6135. }
  6136. out:
  6137. free_extent_map(hole_em);
  6138. if (err) {
  6139. free_extent_map(em);
  6140. return ERR_PTR(err);
  6141. }
  6142. return em;
  6143. }
  6144. static struct extent_map *btrfs_create_dio_extent(struct btrfs_inode *inode,
  6145. const u64 start,
  6146. const u64 len,
  6147. const u64 orig_start,
  6148. const u64 block_start,
  6149. const u64 block_len,
  6150. const u64 orig_block_len,
  6151. const u64 ram_bytes,
  6152. const int type)
  6153. {
  6154. struct extent_map *em = NULL;
  6155. int ret;
  6156. if (type != BTRFS_ORDERED_NOCOW) {
  6157. em = create_io_em(inode, start, len, orig_start, block_start,
  6158. block_len, orig_block_len, ram_bytes,
  6159. BTRFS_COMPRESS_NONE, /* compress_type */
  6160. type);
  6161. if (IS_ERR(em))
  6162. goto out;
  6163. }
  6164. ret = btrfs_add_ordered_extent_dio(inode, start, block_start, len,
  6165. block_len, type);
  6166. if (ret) {
  6167. if (em) {
  6168. free_extent_map(em);
  6169. btrfs_drop_extent_cache(inode, start, start + len - 1, 0);
  6170. }
  6171. em = ERR_PTR(ret);
  6172. }
  6173. out:
  6174. return em;
  6175. }
  6176. static struct extent_map *btrfs_new_extent_direct(struct btrfs_inode *inode,
  6177. u64 start, u64 len)
  6178. {
  6179. struct btrfs_root *root = inode->root;
  6180. struct btrfs_fs_info *fs_info = root->fs_info;
  6181. struct extent_map *em;
  6182. struct btrfs_key ins;
  6183. u64 alloc_hint;
  6184. int ret;
  6185. alloc_hint = get_extent_allocation_hint(inode, start, len);
  6186. ret = btrfs_reserve_extent(root, len, len, fs_info->sectorsize,
  6187. 0, alloc_hint, &ins, 1, 1);
  6188. if (ret)
  6189. return ERR_PTR(ret);
  6190. em = btrfs_create_dio_extent(inode, start, ins.offset, start,
  6191. ins.objectid, ins.offset, ins.offset,
  6192. ins.offset, BTRFS_ORDERED_REGULAR);
  6193. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  6194. if (IS_ERR(em))
  6195. btrfs_free_reserved_extent(fs_info, ins.objectid, ins.offset,
  6196. 1);
  6197. return em;
  6198. }
  6199. /*
  6200. * Check if we can do nocow write into the range [@offset, @offset + @len)
  6201. *
  6202. * @offset: File offset
  6203. * @len: The length to write, will be updated to the nocow writeable
  6204. * range
  6205. * @orig_start: (optional) Return the original file offset of the file extent
  6206. * @orig_len: (optional) Return the original on-disk length of the file extent
  6207. * @ram_bytes: (optional) Return the ram_bytes of the file extent
  6208. * @strict: if true, omit optimizations that might force us into unnecessary
  6209. * cow. e.g., don't trust generation number.
  6210. *
  6211. * This function will flush ordered extents in the range to ensure proper
  6212. * nocow checks for (nowait == false) case.
  6213. *
  6214. * Return:
  6215. * >0 and update @len if we can do nocow write
  6216. * 0 if we can't do nocow write
  6217. * <0 if error happened
  6218. *
  6219. * NOTE: This only checks the file extents, caller is responsible to wait for
  6220. * any ordered extents.
  6221. */
  6222. noinline int can_nocow_extent(struct inode *inode, u64 offset, u64 *len,
  6223. u64 *orig_start, u64 *orig_block_len,
  6224. u64 *ram_bytes, bool strict)
  6225. {
  6226. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6227. struct btrfs_path *path;
  6228. int ret;
  6229. struct extent_buffer *leaf;
  6230. struct btrfs_root *root = BTRFS_I(inode)->root;
  6231. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  6232. struct btrfs_file_extent_item *fi;
  6233. struct btrfs_key key;
  6234. u64 disk_bytenr;
  6235. u64 backref_offset;
  6236. u64 extent_end;
  6237. u64 num_bytes;
  6238. int slot;
  6239. int found_type;
  6240. bool nocow = (BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW);
  6241. path = btrfs_alloc_path();
  6242. if (!path)
  6243. return -ENOMEM;
  6244. ret = btrfs_lookup_file_extent(NULL, root, path,
  6245. btrfs_ino(BTRFS_I(inode)), offset, 0);
  6246. if (ret < 0)
  6247. goto out;
  6248. slot = path->slots[0];
  6249. if (ret == 1) {
  6250. if (slot == 0) {
  6251. /* can't find the item, must cow */
  6252. ret = 0;
  6253. goto out;
  6254. }
  6255. slot--;
  6256. }
  6257. ret = 0;
  6258. leaf = path->nodes[0];
  6259. btrfs_item_key_to_cpu(leaf, &key, slot);
  6260. if (key.objectid != btrfs_ino(BTRFS_I(inode)) ||
  6261. key.type != BTRFS_EXTENT_DATA_KEY) {
  6262. /* not our file or wrong item type, must cow */
  6263. goto out;
  6264. }
  6265. if (key.offset > offset) {
  6266. /* Wrong offset, must cow */
  6267. goto out;
  6268. }
  6269. fi = btrfs_item_ptr(leaf, slot, struct btrfs_file_extent_item);
  6270. found_type = btrfs_file_extent_type(leaf, fi);
  6271. if (found_type != BTRFS_FILE_EXTENT_REG &&
  6272. found_type != BTRFS_FILE_EXTENT_PREALLOC) {
  6273. /* not a regular extent, must cow */
  6274. goto out;
  6275. }
  6276. if (!nocow && found_type == BTRFS_FILE_EXTENT_REG)
  6277. goto out;
  6278. extent_end = key.offset + btrfs_file_extent_num_bytes(leaf, fi);
  6279. if (extent_end <= offset)
  6280. goto out;
  6281. disk_bytenr = btrfs_file_extent_disk_bytenr(leaf, fi);
  6282. if (disk_bytenr == 0)
  6283. goto out;
  6284. if (btrfs_file_extent_compression(leaf, fi) ||
  6285. btrfs_file_extent_encryption(leaf, fi) ||
  6286. btrfs_file_extent_other_encoding(leaf, fi))
  6287. goto out;
  6288. /*
  6289. * Do the same check as in btrfs_cross_ref_exist but without the
  6290. * unnecessary search.
  6291. */
  6292. if (!strict &&
  6293. (btrfs_file_extent_generation(leaf, fi) <=
  6294. btrfs_root_last_snapshot(&root->root_item)))
  6295. goto out;
  6296. backref_offset = btrfs_file_extent_offset(leaf, fi);
  6297. if (orig_start) {
  6298. *orig_start = key.offset - backref_offset;
  6299. *orig_block_len = btrfs_file_extent_disk_num_bytes(leaf, fi);
  6300. *ram_bytes = btrfs_file_extent_ram_bytes(leaf, fi);
  6301. }
  6302. if (btrfs_extent_readonly(fs_info, disk_bytenr))
  6303. goto out;
  6304. num_bytes = min(offset + *len, extent_end) - offset;
  6305. if (!nocow && found_type == BTRFS_FILE_EXTENT_PREALLOC) {
  6306. u64 range_end;
  6307. range_end = round_up(offset + num_bytes,
  6308. root->fs_info->sectorsize) - 1;
  6309. ret = test_range_bit(io_tree, offset, range_end,
  6310. EXTENT_DELALLOC, 0, NULL);
  6311. if (ret) {
  6312. ret = -EAGAIN;
  6313. goto out;
  6314. }
  6315. }
  6316. btrfs_release_path(path);
  6317. /*
  6318. * look for other files referencing this extent, if we
  6319. * find any we must cow
  6320. */
  6321. ret = btrfs_cross_ref_exist(root, btrfs_ino(BTRFS_I(inode)),
  6322. key.offset - backref_offset, disk_bytenr,
  6323. strict);
  6324. if (ret) {
  6325. ret = 0;
  6326. goto out;
  6327. }
  6328. /*
  6329. * adjust disk_bytenr and num_bytes to cover just the bytes
  6330. * in this extent we are about to write. If there
  6331. * are any csums in that range we have to cow in order
  6332. * to keep the csums correct
  6333. */
  6334. disk_bytenr += backref_offset;
  6335. disk_bytenr += offset - key.offset;
  6336. if (csum_exist_in_range(fs_info, disk_bytenr, num_bytes))
  6337. goto out;
  6338. /*
  6339. * all of the above have passed, it is safe to overwrite this extent
  6340. * without cow
  6341. */
  6342. *len = num_bytes;
  6343. ret = 1;
  6344. out:
  6345. btrfs_free_path(path);
  6346. return ret;
  6347. }
  6348. static int lock_extent_direct(struct inode *inode, u64 lockstart, u64 lockend,
  6349. struct extent_state **cached_state, bool writing)
  6350. {
  6351. struct btrfs_ordered_extent *ordered;
  6352. int ret = 0;
  6353. while (1) {
  6354. lock_extent_bits(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6355. cached_state);
  6356. /*
  6357. * We're concerned with the entire range that we're going to be
  6358. * doing DIO to, so we need to make sure there's no ordered
  6359. * extents in this range.
  6360. */
  6361. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), lockstart,
  6362. lockend - lockstart + 1);
  6363. /*
  6364. * We need to make sure there are no buffered pages in this
  6365. * range either, we could have raced between the invalidate in
  6366. * generic_file_direct_write and locking the extent. The
  6367. * invalidate needs to happen so that reads after a write do not
  6368. * get stale data.
  6369. */
  6370. if (!ordered &&
  6371. (!writing || !filemap_range_has_page(inode->i_mapping,
  6372. lockstart, lockend)))
  6373. break;
  6374. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6375. cached_state);
  6376. if (ordered) {
  6377. /*
  6378. * If we are doing a DIO read and the ordered extent we
  6379. * found is for a buffered write, we can not wait for it
  6380. * to complete and retry, because if we do so we can
  6381. * deadlock with concurrent buffered writes on page
  6382. * locks. This happens only if our DIO read covers more
  6383. * than one extent map, if at this point has already
  6384. * created an ordered extent for a previous extent map
  6385. * and locked its range in the inode's io tree, and a
  6386. * concurrent write against that previous extent map's
  6387. * range and this range started (we unlock the ranges
  6388. * in the io tree only when the bios complete and
  6389. * buffered writes always lock pages before attempting
  6390. * to lock range in the io tree).
  6391. */
  6392. if (writing ||
  6393. test_bit(BTRFS_ORDERED_DIRECT, &ordered->flags))
  6394. btrfs_start_ordered_extent(ordered, 1);
  6395. else
  6396. ret = -ENOTBLK;
  6397. btrfs_put_ordered_extent(ordered);
  6398. } else {
  6399. /*
  6400. * We could trigger writeback for this range (and wait
  6401. * for it to complete) and then invalidate the pages for
  6402. * this range (through invalidate_inode_pages2_range()),
  6403. * but that can lead us to a deadlock with a concurrent
  6404. * call to readahead (a buffered read or a defrag call
  6405. * triggered a readahead) on a page lock due to an
  6406. * ordered dio extent we created before but did not have
  6407. * yet a corresponding bio submitted (whence it can not
  6408. * complete), which makes readahead wait for that
  6409. * ordered extent to complete while holding a lock on
  6410. * that page.
  6411. */
  6412. ret = -ENOTBLK;
  6413. }
  6414. if (ret)
  6415. break;
  6416. cond_resched();
  6417. }
  6418. return ret;
  6419. }
  6420. /* The callers of this must take lock_extent() */
  6421. static struct extent_map *create_io_em(struct btrfs_inode *inode, u64 start,
  6422. u64 len, u64 orig_start, u64 block_start,
  6423. u64 block_len, u64 orig_block_len,
  6424. u64 ram_bytes, int compress_type,
  6425. int type)
  6426. {
  6427. struct extent_map_tree *em_tree;
  6428. struct extent_map *em;
  6429. int ret;
  6430. ASSERT(type == BTRFS_ORDERED_PREALLOC ||
  6431. type == BTRFS_ORDERED_COMPRESSED ||
  6432. type == BTRFS_ORDERED_NOCOW ||
  6433. type == BTRFS_ORDERED_REGULAR);
  6434. em_tree = &inode->extent_tree;
  6435. em = alloc_extent_map();
  6436. if (!em)
  6437. return ERR_PTR(-ENOMEM);
  6438. em->start = start;
  6439. em->orig_start = orig_start;
  6440. em->len = len;
  6441. em->block_len = block_len;
  6442. em->block_start = block_start;
  6443. em->orig_block_len = orig_block_len;
  6444. em->ram_bytes = ram_bytes;
  6445. em->generation = -1;
  6446. set_bit(EXTENT_FLAG_PINNED, &em->flags);
  6447. if (type == BTRFS_ORDERED_PREALLOC) {
  6448. set_bit(EXTENT_FLAG_FILLING, &em->flags);
  6449. } else if (type == BTRFS_ORDERED_COMPRESSED) {
  6450. set_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
  6451. em->compress_type = compress_type;
  6452. }
  6453. do {
  6454. btrfs_drop_extent_cache(inode, em->start,
  6455. em->start + em->len - 1, 0);
  6456. write_lock(&em_tree->lock);
  6457. ret = add_extent_mapping(em_tree, em, 1);
  6458. write_unlock(&em_tree->lock);
  6459. /*
  6460. * The caller has taken lock_extent(), who could race with us
  6461. * to add em?
  6462. */
  6463. } while (ret == -EEXIST);
  6464. if (ret) {
  6465. free_extent_map(em);
  6466. return ERR_PTR(ret);
  6467. }
  6468. /* em got 2 refs now, callers needs to do free_extent_map once. */
  6469. return em;
  6470. }
  6471. static int btrfs_get_blocks_direct_write(struct extent_map **map,
  6472. struct inode *inode,
  6473. struct btrfs_dio_data *dio_data,
  6474. u64 start, u64 len)
  6475. {
  6476. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6477. struct extent_map *em = *map;
  6478. int ret = 0;
  6479. /*
  6480. * We don't allocate a new extent in the following cases
  6481. *
  6482. * 1) The inode is marked as NODATACOW. In this case we'll just use the
  6483. * existing extent.
  6484. * 2) The extent is marked as PREALLOC. We're good to go here and can
  6485. * just use the extent.
  6486. *
  6487. */
  6488. if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) ||
  6489. ((BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW) &&
  6490. em->block_start != EXTENT_MAP_HOLE)) {
  6491. int type;
  6492. u64 block_start, orig_start, orig_block_len, ram_bytes;
  6493. if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
  6494. type = BTRFS_ORDERED_PREALLOC;
  6495. else
  6496. type = BTRFS_ORDERED_NOCOW;
  6497. len = min(len, em->len - (start - em->start));
  6498. block_start = em->block_start + (start - em->start);
  6499. if (can_nocow_extent(inode, start, &len, &orig_start,
  6500. &orig_block_len, &ram_bytes, false) == 1 &&
  6501. btrfs_inc_nocow_writers(fs_info, block_start)) {
  6502. struct extent_map *em2;
  6503. em2 = btrfs_create_dio_extent(BTRFS_I(inode), start, len,
  6504. orig_start, block_start,
  6505. len, orig_block_len,
  6506. ram_bytes, type);
  6507. btrfs_dec_nocow_writers(fs_info, block_start);
  6508. if (type == BTRFS_ORDERED_PREALLOC) {
  6509. free_extent_map(em);
  6510. *map = em = em2;
  6511. }
  6512. if (em2 && IS_ERR(em2)) {
  6513. ret = PTR_ERR(em2);
  6514. goto out;
  6515. }
  6516. /*
  6517. * For inode marked NODATACOW or extent marked PREALLOC,
  6518. * use the existing or preallocated extent, so does not
  6519. * need to adjust btrfs_space_info's bytes_may_use.
  6520. */
  6521. btrfs_free_reserved_data_space_noquota(fs_info, len);
  6522. goto skip_cow;
  6523. }
  6524. }
  6525. /* this will cow the extent */
  6526. free_extent_map(em);
  6527. *map = em = btrfs_new_extent_direct(BTRFS_I(inode), start, len);
  6528. if (IS_ERR(em)) {
  6529. ret = PTR_ERR(em);
  6530. goto out;
  6531. }
  6532. len = min(len, em->len - (start - em->start));
  6533. skip_cow:
  6534. /*
  6535. * Need to update the i_size under the extent lock so buffered
  6536. * readers will get the updated i_size when we unlock.
  6537. */
  6538. if (start + len > i_size_read(inode))
  6539. i_size_write(inode, start + len);
  6540. dio_data->reserve -= len;
  6541. out:
  6542. return ret;
  6543. }
  6544. static int btrfs_dio_iomap_begin(struct inode *inode, loff_t start,
  6545. loff_t length, unsigned int flags, struct iomap *iomap,
  6546. struct iomap *srcmap)
  6547. {
  6548. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6549. struct extent_map *em;
  6550. struct extent_state *cached_state = NULL;
  6551. struct btrfs_dio_data *dio_data = NULL;
  6552. u64 lockstart, lockend;
  6553. const bool write = !!(flags & IOMAP_WRITE);
  6554. int ret = 0;
  6555. u64 len = length;
  6556. bool unlock_extents = false;
  6557. bool sync = (current->journal_info == BTRFS_DIO_SYNC_STUB);
  6558. /*
  6559. * We used current->journal_info here to see if we were sync, but
  6560. * there's a lot of tests in the enospc machinery to not do flushing if
  6561. * we have a journal_info set, so we need to clear this out and re-set
  6562. * it in iomap_end.
  6563. */
  6564. ASSERT(current->journal_info == NULL ||
  6565. current->journal_info == BTRFS_DIO_SYNC_STUB);
  6566. current->journal_info = NULL;
  6567. if (!write)
  6568. len = min_t(u64, len, fs_info->sectorsize);
  6569. lockstart = start;
  6570. lockend = start + len - 1;
  6571. /*
  6572. * The generic stuff only does filemap_write_and_wait_range, which
  6573. * isn't enough if we've written compressed pages to this area, so we
  6574. * need to flush the dirty pages again to make absolutely sure that any
  6575. * outstanding dirty pages are on disk.
  6576. */
  6577. if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  6578. &BTRFS_I(inode)->runtime_flags)) {
  6579. ret = filemap_fdatawrite_range(inode->i_mapping, start,
  6580. start + length - 1);
  6581. if (ret)
  6582. return ret;
  6583. }
  6584. dio_data = kzalloc(sizeof(*dio_data), GFP_NOFS);
  6585. if (!dio_data)
  6586. return -ENOMEM;
  6587. dio_data->sync = sync;
  6588. dio_data->length = length;
  6589. if (write) {
  6590. dio_data->reserve = round_up(length, fs_info->sectorsize);
  6591. ret = btrfs_delalloc_reserve_space(BTRFS_I(inode),
  6592. &dio_data->data_reserved,
  6593. start, dio_data->reserve);
  6594. if (ret) {
  6595. extent_changeset_free(dio_data->data_reserved);
  6596. kfree(dio_data);
  6597. return ret;
  6598. }
  6599. }
  6600. iomap->private = dio_data;
  6601. /*
  6602. * If this errors out it's because we couldn't invalidate pagecache for
  6603. * this range and we need to fallback to buffered.
  6604. */
  6605. if (lock_extent_direct(inode, lockstart, lockend, &cached_state, write)) {
  6606. ret = -ENOTBLK;
  6607. goto err;
  6608. }
  6609. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len);
  6610. if (IS_ERR(em)) {
  6611. ret = PTR_ERR(em);
  6612. goto unlock_err;
  6613. }
  6614. /*
  6615. * Ok for INLINE and COMPRESSED extents we need to fallback on buffered
  6616. * io. INLINE is special, and we could probably kludge it in here, but
  6617. * it's still buffered so for safety lets just fall back to the generic
  6618. * buffered path.
  6619. *
  6620. * For COMPRESSED we _have_ to read the entire extent in so we can
  6621. * decompress it, so there will be buffering required no matter what we
  6622. * do, so go ahead and fallback to buffered.
  6623. *
  6624. * We return -ENOTBLK because that's what makes DIO go ahead and go back
  6625. * to buffered IO. Don't blame me, this is the price we pay for using
  6626. * the generic code.
  6627. */
  6628. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) ||
  6629. em->block_start == EXTENT_MAP_INLINE) {
  6630. free_extent_map(em);
  6631. ret = -ENOTBLK;
  6632. goto unlock_err;
  6633. }
  6634. len = min(len, em->len - (start - em->start));
  6635. if (write) {
  6636. ret = btrfs_get_blocks_direct_write(&em, inode, dio_data,
  6637. start, len);
  6638. if (ret < 0)
  6639. goto unlock_err;
  6640. unlock_extents = true;
  6641. /* Recalc len in case the new em is smaller than requested */
  6642. len = min(len, em->len - (start - em->start));
  6643. } else {
  6644. /*
  6645. * We need to unlock only the end area that we aren't using.
  6646. * The rest is going to be unlocked by the endio routine.
  6647. */
  6648. lockstart = start + len;
  6649. if (lockstart < lockend)
  6650. unlock_extents = true;
  6651. }
  6652. if (unlock_extents)
  6653. unlock_extent_cached(&BTRFS_I(inode)->io_tree,
  6654. lockstart, lockend, &cached_state);
  6655. else
  6656. free_extent_state(cached_state);
  6657. /*
  6658. * Translate extent map information to iomap.
  6659. * We trim the extents (and move the addr) even though iomap code does
  6660. * that, since we have locked only the parts we are performing I/O in.
  6661. */
  6662. if ((em->block_start == EXTENT_MAP_HOLE) ||
  6663. (test_bit(EXTENT_FLAG_PREALLOC, &em->flags) && !write)) {
  6664. iomap->addr = IOMAP_NULL_ADDR;
  6665. iomap->type = IOMAP_HOLE;
  6666. } else {
  6667. iomap->addr = em->block_start + (start - em->start);
  6668. iomap->type = IOMAP_MAPPED;
  6669. }
  6670. iomap->offset = start;
  6671. iomap->bdev = fs_info->fs_devices->latest_bdev;
  6672. iomap->length = len;
  6673. free_extent_map(em);
  6674. return 0;
  6675. unlock_err:
  6676. unlock_extent_cached(&BTRFS_I(inode)->io_tree, lockstart, lockend,
  6677. &cached_state);
  6678. err:
  6679. if (dio_data) {
  6680. btrfs_delalloc_release_space(BTRFS_I(inode),
  6681. dio_data->data_reserved, start,
  6682. dio_data->reserve, true);
  6683. btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->reserve);
  6684. extent_changeset_free(dio_data->data_reserved);
  6685. kfree(dio_data);
  6686. }
  6687. return ret;
  6688. }
  6689. static int btrfs_dio_iomap_end(struct inode *inode, loff_t pos, loff_t length,
  6690. ssize_t written, unsigned int flags, struct iomap *iomap)
  6691. {
  6692. int ret = 0;
  6693. struct btrfs_dio_data *dio_data = iomap->private;
  6694. size_t submitted = dio_data->submitted;
  6695. const bool write = !!(flags & IOMAP_WRITE);
  6696. if (!write && (iomap->type == IOMAP_HOLE)) {
  6697. /* If reading from a hole, unlock and return */
  6698. unlock_extent(&BTRFS_I(inode)->io_tree, pos, pos + length - 1);
  6699. goto out;
  6700. }
  6701. if (submitted < length) {
  6702. pos += submitted;
  6703. length -= submitted;
  6704. if (write)
  6705. __endio_write_update_ordered(BTRFS_I(inode), pos,
  6706. length, false);
  6707. else
  6708. unlock_extent(&BTRFS_I(inode)->io_tree, pos,
  6709. pos + length - 1);
  6710. ret = -ENOTBLK;
  6711. }
  6712. if (write) {
  6713. if (dio_data->reserve)
  6714. btrfs_delalloc_release_space(BTRFS_I(inode),
  6715. dio_data->data_reserved, pos,
  6716. dio_data->reserve, true);
  6717. btrfs_delalloc_release_extents(BTRFS_I(inode), dio_data->length);
  6718. extent_changeset_free(dio_data->data_reserved);
  6719. }
  6720. out:
  6721. /*
  6722. * We're all done, we can re-set the current->journal_info now safely
  6723. * for our endio.
  6724. */
  6725. if (dio_data->sync) {
  6726. ASSERT(current->journal_info == NULL);
  6727. current->journal_info = BTRFS_DIO_SYNC_STUB;
  6728. }
  6729. kfree(dio_data);
  6730. iomap->private = NULL;
  6731. return ret;
  6732. }
  6733. static void btrfs_dio_private_put(struct btrfs_dio_private *dip)
  6734. {
  6735. /*
  6736. * This implies a barrier so that stores to dio_bio->bi_status before
  6737. * this and loads of dio_bio->bi_status after this are fully ordered.
  6738. */
  6739. if (!refcount_dec_and_test(&dip->refs))
  6740. return;
  6741. if (bio_op(dip->dio_bio) == REQ_OP_WRITE) {
  6742. __endio_write_update_ordered(BTRFS_I(dip->inode),
  6743. dip->logical_offset,
  6744. dip->bytes,
  6745. !dip->dio_bio->bi_status);
  6746. } else {
  6747. unlock_extent(&BTRFS_I(dip->inode)->io_tree,
  6748. dip->logical_offset,
  6749. dip->logical_offset + dip->bytes - 1);
  6750. }
  6751. bio_endio(dip->dio_bio);
  6752. kfree(dip);
  6753. }
  6754. static blk_status_t submit_dio_repair_bio(struct inode *inode, struct bio *bio,
  6755. int mirror_num,
  6756. unsigned long bio_flags)
  6757. {
  6758. struct btrfs_dio_private *dip = bio->bi_private;
  6759. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6760. blk_status_t ret;
  6761. BUG_ON(bio_op(bio) == REQ_OP_WRITE);
  6762. ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
  6763. if (ret)
  6764. return ret;
  6765. refcount_inc(&dip->refs);
  6766. ret = btrfs_map_bio(fs_info, bio, mirror_num);
  6767. if (ret)
  6768. refcount_dec(&dip->refs);
  6769. return ret;
  6770. }
  6771. static blk_status_t btrfs_check_read_dio_bio(struct inode *inode,
  6772. struct btrfs_io_bio *io_bio,
  6773. const bool uptodate)
  6774. {
  6775. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  6776. const u32 sectorsize = fs_info->sectorsize;
  6777. struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
  6778. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  6779. const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
  6780. struct bio_vec bvec;
  6781. struct bvec_iter iter;
  6782. u64 start = io_bio->logical;
  6783. int icsum = 0;
  6784. blk_status_t err = BLK_STS_OK;
  6785. __bio_for_each_segment(bvec, &io_bio->bio, iter, io_bio->iter) {
  6786. unsigned int i, nr_sectors, pgoff;
  6787. nr_sectors = BTRFS_BYTES_TO_BLKS(fs_info, bvec.bv_len);
  6788. pgoff = bvec.bv_offset;
  6789. for (i = 0; i < nr_sectors; i++) {
  6790. ASSERT(pgoff < PAGE_SIZE);
  6791. if (uptodate &&
  6792. (!csum || !check_data_csum(inode, io_bio, icsum,
  6793. bvec.bv_page, pgoff,
  6794. start, sectorsize))) {
  6795. clean_io_failure(fs_info, failure_tree, io_tree,
  6796. start, bvec.bv_page,
  6797. btrfs_ino(BTRFS_I(inode)),
  6798. pgoff);
  6799. } else {
  6800. blk_status_t status;
  6801. status = btrfs_submit_read_repair(inode,
  6802. &io_bio->bio,
  6803. start - io_bio->logical,
  6804. bvec.bv_page, pgoff,
  6805. start,
  6806. start + sectorsize - 1,
  6807. io_bio->mirror_num,
  6808. submit_dio_repair_bio);
  6809. if (status)
  6810. err = status;
  6811. }
  6812. start += sectorsize;
  6813. icsum++;
  6814. pgoff += sectorsize;
  6815. }
  6816. }
  6817. return err;
  6818. }
  6819. static void __endio_write_update_ordered(struct btrfs_inode *inode,
  6820. const u64 offset, const u64 bytes,
  6821. const bool uptodate)
  6822. {
  6823. struct btrfs_fs_info *fs_info = inode->root->fs_info;
  6824. struct btrfs_ordered_extent *ordered = NULL;
  6825. struct btrfs_workqueue *wq;
  6826. u64 ordered_offset = offset;
  6827. u64 ordered_bytes = bytes;
  6828. u64 last_offset;
  6829. if (btrfs_is_free_space_inode(inode))
  6830. wq = fs_info->endio_freespace_worker;
  6831. else
  6832. wq = fs_info->endio_write_workers;
  6833. while (ordered_offset < offset + bytes) {
  6834. last_offset = ordered_offset;
  6835. if (btrfs_dec_test_first_ordered_pending(inode, &ordered,
  6836. &ordered_offset,
  6837. ordered_bytes,
  6838. uptodate)) {
  6839. btrfs_init_work(&ordered->work, finish_ordered_fn, NULL,
  6840. NULL);
  6841. btrfs_queue_work(wq, &ordered->work);
  6842. }
  6843. /*
  6844. * If btrfs_dec_test_ordered_pending does not find any ordered
  6845. * extent in the range, we can exit.
  6846. */
  6847. if (ordered_offset == last_offset)
  6848. return;
  6849. /*
  6850. * Our bio might span multiple ordered extents. In this case
  6851. * we keep going until we have accounted the whole dio.
  6852. */
  6853. if (ordered_offset < offset + bytes) {
  6854. ordered_bytes = offset + bytes - ordered_offset;
  6855. ordered = NULL;
  6856. }
  6857. }
  6858. }
  6859. static blk_status_t btrfs_submit_bio_start_direct_io(void *private_data,
  6860. struct bio *bio, u64 offset)
  6861. {
  6862. struct inode *inode = private_data;
  6863. return btrfs_csum_one_bio(BTRFS_I(inode), bio, offset, 1);
  6864. }
  6865. static void btrfs_end_dio_bio(struct bio *bio)
  6866. {
  6867. struct btrfs_dio_private *dip = bio->bi_private;
  6868. blk_status_t err = bio->bi_status;
  6869. if (err)
  6870. btrfs_warn(BTRFS_I(dip->inode)->root->fs_info,
  6871. "direct IO failed ino %llu rw %d,%u sector %#Lx len %u err no %d",
  6872. btrfs_ino(BTRFS_I(dip->inode)), bio_op(bio),
  6873. bio->bi_opf,
  6874. (unsigned long long)bio->bi_iter.bi_sector,
  6875. bio->bi_iter.bi_size, err);
  6876. if (bio_op(bio) == REQ_OP_READ) {
  6877. err = btrfs_check_read_dio_bio(dip->inode, btrfs_io_bio(bio),
  6878. !err);
  6879. }
  6880. if (err)
  6881. dip->dio_bio->bi_status = err;
  6882. bio_put(bio);
  6883. btrfs_dio_private_put(dip);
  6884. }
  6885. static inline blk_status_t btrfs_submit_dio_bio(struct bio *bio,
  6886. struct inode *inode, u64 file_offset, int async_submit)
  6887. {
  6888. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6889. struct btrfs_dio_private *dip = bio->bi_private;
  6890. bool write = bio_op(bio) == REQ_OP_WRITE;
  6891. blk_status_t ret;
  6892. /* Check btrfs_submit_bio_hook() for rules about async submit. */
  6893. if (async_submit)
  6894. async_submit = !atomic_read(&BTRFS_I(inode)->sync_writers);
  6895. if (!write) {
  6896. ret = btrfs_bio_wq_end_io(fs_info, bio, BTRFS_WQ_ENDIO_DATA);
  6897. if (ret)
  6898. goto err;
  6899. }
  6900. if (BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)
  6901. goto map;
  6902. if (write && async_submit) {
  6903. ret = btrfs_wq_submit_bio(fs_info, bio, 0, 0,
  6904. file_offset, inode,
  6905. btrfs_submit_bio_start_direct_io);
  6906. goto err;
  6907. } else if (write) {
  6908. /*
  6909. * If we aren't doing async submit, calculate the csum of the
  6910. * bio now.
  6911. */
  6912. ret = btrfs_csum_one_bio(BTRFS_I(inode), bio, file_offset, 1);
  6913. if (ret)
  6914. goto err;
  6915. } else {
  6916. u64 csum_offset;
  6917. csum_offset = file_offset - dip->logical_offset;
  6918. csum_offset >>= inode->i_sb->s_blocksize_bits;
  6919. csum_offset *= btrfs_super_csum_size(fs_info->super_copy);
  6920. btrfs_io_bio(bio)->csum = dip->csums + csum_offset;
  6921. }
  6922. map:
  6923. ret = btrfs_map_bio(fs_info, bio, 0);
  6924. err:
  6925. return ret;
  6926. }
  6927. /*
  6928. * If this succeeds, the btrfs_dio_private is responsible for cleaning up locked
  6929. * or ordered extents whether or not we submit any bios.
  6930. */
  6931. static struct btrfs_dio_private *btrfs_create_dio_private(struct bio *dio_bio,
  6932. struct inode *inode,
  6933. loff_t file_offset)
  6934. {
  6935. const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
  6936. const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
  6937. size_t dip_size;
  6938. struct btrfs_dio_private *dip;
  6939. dip_size = sizeof(*dip);
  6940. if (!write && csum) {
  6941. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6942. const u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
  6943. size_t nblocks;
  6944. nblocks = dio_bio->bi_iter.bi_size >> inode->i_sb->s_blocksize_bits;
  6945. dip_size += csum_size * nblocks;
  6946. }
  6947. dip = kzalloc(dip_size, GFP_NOFS);
  6948. if (!dip)
  6949. return NULL;
  6950. dip->inode = inode;
  6951. dip->logical_offset = file_offset;
  6952. dip->bytes = dio_bio->bi_iter.bi_size;
  6953. dip->disk_bytenr = (u64)dio_bio->bi_iter.bi_sector << 9;
  6954. dip->dio_bio = dio_bio;
  6955. refcount_set(&dip->refs, 1);
  6956. return dip;
  6957. }
  6958. static blk_qc_t btrfs_submit_direct(struct inode *inode, struct iomap *iomap,
  6959. struct bio *dio_bio, loff_t file_offset)
  6960. {
  6961. const bool write = (bio_op(dio_bio) == REQ_OP_WRITE);
  6962. const bool csum = !(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM);
  6963. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  6964. const bool raid56 = (btrfs_data_alloc_profile(fs_info) &
  6965. BTRFS_BLOCK_GROUP_RAID56_MASK);
  6966. struct btrfs_dio_private *dip;
  6967. struct bio *bio;
  6968. u64 start_sector;
  6969. int async_submit = 0;
  6970. u64 submit_len;
  6971. int clone_offset = 0;
  6972. int clone_len;
  6973. int ret;
  6974. blk_status_t status;
  6975. struct btrfs_io_geometry geom;
  6976. struct btrfs_dio_data *dio_data = iomap->private;
  6977. dip = btrfs_create_dio_private(dio_bio, inode, file_offset);
  6978. if (!dip) {
  6979. if (!write) {
  6980. unlock_extent(&BTRFS_I(inode)->io_tree, file_offset,
  6981. file_offset + dio_bio->bi_iter.bi_size - 1);
  6982. }
  6983. dio_bio->bi_status = BLK_STS_RESOURCE;
  6984. bio_endio(dio_bio);
  6985. return BLK_QC_T_NONE;
  6986. }
  6987. if (!write && csum) {
  6988. /*
  6989. * Load the csums up front to reduce csum tree searches and
  6990. * contention when submitting bios.
  6991. */
  6992. status = btrfs_lookup_bio_sums(inode, dio_bio, file_offset,
  6993. dip->csums);
  6994. if (status != BLK_STS_OK)
  6995. goto out_err;
  6996. }
  6997. start_sector = dio_bio->bi_iter.bi_sector;
  6998. submit_len = dio_bio->bi_iter.bi_size;
  6999. do {
  7000. ret = btrfs_get_io_geometry(fs_info, btrfs_op(dio_bio),
  7001. start_sector << 9, submit_len,
  7002. &geom);
  7003. if (ret) {
  7004. status = errno_to_blk_status(ret);
  7005. goto out_err;
  7006. }
  7007. ASSERT(geom.len <= INT_MAX);
  7008. clone_len = min_t(int, submit_len, geom.len);
  7009. /*
  7010. * This will never fail as it's passing GPF_NOFS and
  7011. * the allocation is backed by btrfs_bioset.
  7012. */
  7013. bio = btrfs_bio_clone_partial(dio_bio, clone_offset, clone_len);
  7014. bio->bi_private = dip;
  7015. bio->bi_end_io = btrfs_end_dio_bio;
  7016. btrfs_io_bio(bio)->logical = file_offset;
  7017. ASSERT(submit_len >= clone_len);
  7018. submit_len -= clone_len;
  7019. /*
  7020. * Increase the count before we submit the bio so we know
  7021. * the end IO handler won't happen before we increase the
  7022. * count. Otherwise, the dip might get freed before we're
  7023. * done setting it up.
  7024. *
  7025. * We transfer the initial reference to the last bio, so we
  7026. * don't need to increment the reference count for the last one.
  7027. */
  7028. if (submit_len > 0) {
  7029. refcount_inc(&dip->refs);
  7030. /*
  7031. * If we are submitting more than one bio, submit them
  7032. * all asynchronously. The exception is RAID 5 or 6, as
  7033. * asynchronous checksums make it difficult to collect
  7034. * full stripe writes.
  7035. */
  7036. if (!raid56)
  7037. async_submit = 1;
  7038. }
  7039. status = btrfs_submit_dio_bio(bio, inode, file_offset,
  7040. async_submit);
  7041. if (status) {
  7042. bio_put(bio);
  7043. if (submit_len > 0)
  7044. refcount_dec(&dip->refs);
  7045. goto out_err;
  7046. }
  7047. dio_data->submitted += clone_len;
  7048. clone_offset += clone_len;
  7049. start_sector += clone_len >> 9;
  7050. file_offset += clone_len;
  7051. } while (submit_len > 0);
  7052. return BLK_QC_T_NONE;
  7053. out_err:
  7054. dip->dio_bio->bi_status = status;
  7055. btrfs_dio_private_put(dip);
  7056. return BLK_QC_T_NONE;
  7057. }
  7058. static ssize_t check_direct_IO(struct btrfs_fs_info *fs_info,
  7059. const struct iov_iter *iter, loff_t offset)
  7060. {
  7061. int seg;
  7062. int i;
  7063. unsigned int blocksize_mask = fs_info->sectorsize - 1;
  7064. ssize_t retval = -EINVAL;
  7065. if (offset & blocksize_mask)
  7066. goto out;
  7067. if (iov_iter_alignment(iter) & blocksize_mask)
  7068. goto out;
  7069. /* If this is a write we don't need to check anymore */
  7070. if (iov_iter_rw(iter) != READ || !iter_is_iovec(iter))
  7071. return 0;
  7072. /*
  7073. * Check to make sure we don't have duplicate iov_base's in this
  7074. * iovec, if so return EINVAL, otherwise we'll get csum errors
  7075. * when reading back.
  7076. */
  7077. for (seg = 0; seg < iter->nr_segs; seg++) {
  7078. for (i = seg + 1; i < iter->nr_segs; i++) {
  7079. if (iter->iov[seg].iov_base == iter->iov[i].iov_base)
  7080. goto out;
  7081. }
  7082. }
  7083. retval = 0;
  7084. out:
  7085. return retval;
  7086. }
  7087. static inline int btrfs_maybe_fsync_end_io(struct kiocb *iocb, ssize_t size,
  7088. int error, unsigned flags)
  7089. {
  7090. /*
  7091. * Now if we're still in the context of our submitter we know we can't
  7092. * safely run generic_write_sync(), so clear our flag here so that the
  7093. * caller knows to follow up with a sync.
  7094. */
  7095. if (current->journal_info == BTRFS_DIO_SYNC_STUB) {
  7096. current->journal_info = NULL;
  7097. return error;
  7098. }
  7099. if (error)
  7100. return error;
  7101. if (size) {
  7102. iocb->ki_flags |= IOCB_DSYNC;
  7103. return generic_write_sync(iocb, size);
  7104. }
  7105. return 0;
  7106. }
  7107. static const struct iomap_ops btrfs_dio_iomap_ops = {
  7108. .iomap_begin = btrfs_dio_iomap_begin,
  7109. .iomap_end = btrfs_dio_iomap_end,
  7110. };
  7111. static const struct iomap_dio_ops btrfs_dio_ops = {
  7112. .submit_io = btrfs_submit_direct,
  7113. };
  7114. static const struct iomap_dio_ops btrfs_sync_dops = {
  7115. .submit_io = btrfs_submit_direct,
  7116. .end_io = btrfs_maybe_fsync_end_io,
  7117. };
  7118. ssize_t btrfs_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  7119. {
  7120. struct file *file = iocb->ki_filp;
  7121. struct inode *inode = file->f_mapping->host;
  7122. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7123. struct extent_changeset *data_reserved = NULL;
  7124. loff_t offset = iocb->ki_pos;
  7125. size_t count = 0;
  7126. bool relock = false;
  7127. ssize_t ret;
  7128. if (check_direct_IO(fs_info, iter, offset)) {
  7129. ASSERT(current->journal_info == NULL ||
  7130. current->journal_info == BTRFS_DIO_SYNC_STUB);
  7131. current->journal_info = NULL;
  7132. return 0;
  7133. }
  7134. count = iov_iter_count(iter);
  7135. if (iov_iter_rw(iter) == WRITE) {
  7136. /*
  7137. * If the write DIO is beyond the EOF, we need update
  7138. * the isize, but it is protected by i_mutex. So we can
  7139. * not unlock the i_mutex at this case.
  7140. */
  7141. if (offset + count <= inode->i_size) {
  7142. inode_unlock(inode);
  7143. relock = true;
  7144. }
  7145. down_read(&BTRFS_I(inode)->dio_sem);
  7146. }
  7147. /*
  7148. * We have are actually a sync iocb, so we need our fancy endio to know
  7149. * if we need to sync.
  7150. */
  7151. if (current->journal_info)
  7152. ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops,
  7153. &btrfs_sync_dops, is_sync_kiocb(iocb));
  7154. else
  7155. ret = iomap_dio_rw(iocb, iter, &btrfs_dio_iomap_ops,
  7156. &btrfs_dio_ops, is_sync_kiocb(iocb));
  7157. if (ret == -ENOTBLK)
  7158. ret = 0;
  7159. if (iov_iter_rw(iter) == WRITE)
  7160. up_read(&BTRFS_I(inode)->dio_sem);
  7161. if (relock)
  7162. inode_lock(inode);
  7163. extent_changeset_free(data_reserved);
  7164. return ret;
  7165. }
  7166. static int btrfs_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
  7167. u64 start, u64 len)
  7168. {
  7169. int ret;
  7170. ret = fiemap_prep(inode, fieinfo, start, &len, 0);
  7171. if (ret)
  7172. return ret;
  7173. return extent_fiemap(BTRFS_I(inode), fieinfo, start, len);
  7174. }
  7175. int btrfs_readpage(struct file *file, struct page *page)
  7176. {
  7177. struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
  7178. u64 start = page_offset(page);
  7179. u64 end = start + PAGE_SIZE - 1;
  7180. unsigned long bio_flags = 0;
  7181. struct bio *bio = NULL;
  7182. int ret;
  7183. btrfs_lock_and_flush_ordered_range(inode, start, end, NULL);
  7184. ret = btrfs_do_readpage(page, NULL, &bio, &bio_flags, 0, NULL);
  7185. if (bio)
  7186. ret = submit_one_bio(bio, 0, bio_flags);
  7187. return ret;
  7188. }
  7189. static int btrfs_writepage(struct page *page, struct writeback_control *wbc)
  7190. {
  7191. struct inode *inode = page->mapping->host;
  7192. int ret;
  7193. if (current->flags & PF_MEMALLOC) {
  7194. redirty_page_for_writepage(wbc, page);
  7195. unlock_page(page);
  7196. return 0;
  7197. }
  7198. /*
  7199. * If we are under memory pressure we will call this directly from the
  7200. * VM, we need to make sure we have the inode referenced for the ordered
  7201. * extent. If not just return like we didn't do anything.
  7202. */
  7203. if (!igrab(inode)) {
  7204. redirty_page_for_writepage(wbc, page);
  7205. return AOP_WRITEPAGE_ACTIVATE;
  7206. }
  7207. ret = extent_write_full_page(page, wbc);
  7208. btrfs_add_delayed_iput(inode);
  7209. return ret;
  7210. }
  7211. static int btrfs_writepages(struct address_space *mapping,
  7212. struct writeback_control *wbc)
  7213. {
  7214. return extent_writepages(mapping, wbc);
  7215. }
  7216. static void btrfs_readahead(struct readahead_control *rac)
  7217. {
  7218. extent_readahead(rac);
  7219. }
  7220. static int __btrfs_releasepage(struct page *page, gfp_t gfp_flags)
  7221. {
  7222. int ret = try_release_extent_mapping(page, gfp_flags);
  7223. if (ret == 1)
  7224. detach_page_private(page);
  7225. return ret;
  7226. }
  7227. static int btrfs_releasepage(struct page *page, gfp_t gfp_flags)
  7228. {
  7229. if (PageWriteback(page) || PageDirty(page))
  7230. return 0;
  7231. return __btrfs_releasepage(page, gfp_flags);
  7232. }
  7233. #ifdef CONFIG_MIGRATION
  7234. static int btrfs_migratepage(struct address_space *mapping,
  7235. struct page *newpage, struct page *page,
  7236. enum migrate_mode mode)
  7237. {
  7238. int ret;
  7239. ret = migrate_page_move_mapping(mapping, newpage, page, 0);
  7240. if (ret != MIGRATEPAGE_SUCCESS)
  7241. return ret;
  7242. if (page_has_private(page))
  7243. attach_page_private(newpage, detach_page_private(page));
  7244. if (PagePrivate2(page)) {
  7245. ClearPagePrivate2(page);
  7246. SetPagePrivate2(newpage);
  7247. }
  7248. if (mode != MIGRATE_SYNC_NO_COPY)
  7249. migrate_page_copy(newpage, page);
  7250. else
  7251. migrate_page_states(newpage, page);
  7252. return MIGRATEPAGE_SUCCESS;
  7253. }
  7254. #endif
  7255. static void btrfs_invalidatepage(struct page *page, unsigned int offset,
  7256. unsigned int length)
  7257. {
  7258. struct btrfs_inode *inode = BTRFS_I(page->mapping->host);
  7259. struct extent_io_tree *tree = &inode->io_tree;
  7260. struct btrfs_ordered_extent *ordered;
  7261. struct extent_state *cached_state = NULL;
  7262. u64 page_start = page_offset(page);
  7263. u64 page_end = page_start + PAGE_SIZE - 1;
  7264. u64 start;
  7265. u64 end;
  7266. int inode_evicting = inode->vfs_inode.i_state & I_FREEING;
  7267. /*
  7268. * we have the page locked, so new writeback can't start,
  7269. * and the dirty bit won't be cleared while we are here.
  7270. *
  7271. * Wait for IO on this page so that we can safely clear
  7272. * the PagePrivate2 bit and do ordered accounting
  7273. */
  7274. wait_on_page_writeback(page);
  7275. /*
  7276. * For subpage case, we have call sites like
  7277. * btrfs_punch_hole_lock_range() which passes range not aligned to
  7278. * sectorsize.
  7279. * If the range doesn't cover the full page, we don't need to and
  7280. * shouldn't clear page extent mapped, as page->private can still
  7281. * record subpage dirty bits for other part of the range.
  7282. *
  7283. * For cases that can invalidate the full even the range doesn't
  7284. * cover the full page, like invalidating the last page, we're
  7285. * still safe to wait for ordered extent to finish.
  7286. */
  7287. if (!(offset == 0 && length == PAGE_SIZE)) {
  7288. btrfs_releasepage(page, GFP_NOFS);
  7289. return;
  7290. }
  7291. if (!inode_evicting)
  7292. lock_extent_bits(tree, page_start, page_end, &cached_state);
  7293. start = page_start;
  7294. again:
  7295. ordered = btrfs_lookup_ordered_range(inode, start, page_end - start + 1);
  7296. if (ordered) {
  7297. end = min(page_end,
  7298. ordered->file_offset + ordered->num_bytes - 1);
  7299. /*
  7300. * IO on this page will never be started, so we need
  7301. * to account for any ordered extents now
  7302. */
  7303. if (!inode_evicting)
  7304. clear_extent_bit(tree, start, end,
  7305. EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
  7306. EXTENT_LOCKED | EXTENT_DO_ACCOUNTING |
  7307. EXTENT_DEFRAG, 1, 0, &cached_state);
  7308. /*
  7309. * whoever cleared the private bit is responsible
  7310. * for the finish_ordered_io
  7311. */
  7312. if (TestClearPagePrivate2(page)) {
  7313. struct btrfs_ordered_inode_tree *tree;
  7314. u64 new_len;
  7315. tree = &inode->ordered_tree;
  7316. spin_lock_irq(&tree->lock);
  7317. set_bit(BTRFS_ORDERED_TRUNCATED, &ordered->flags);
  7318. new_len = start - ordered->file_offset;
  7319. if (new_len < ordered->truncated_len)
  7320. ordered->truncated_len = new_len;
  7321. spin_unlock_irq(&tree->lock);
  7322. if (btrfs_dec_test_ordered_pending(inode, &ordered,
  7323. start,
  7324. end - start + 1, 1))
  7325. btrfs_finish_ordered_io(ordered);
  7326. }
  7327. btrfs_put_ordered_extent(ordered);
  7328. if (!inode_evicting) {
  7329. cached_state = NULL;
  7330. lock_extent_bits(tree, start, end,
  7331. &cached_state);
  7332. }
  7333. start = end + 1;
  7334. if (start < page_end)
  7335. goto again;
  7336. }
  7337. /*
  7338. * Qgroup reserved space handler
  7339. * Page here will be either
  7340. * 1) Already written to disk or ordered extent already submitted
  7341. * Then its QGROUP_RESERVED bit in io_tree is already cleaned.
  7342. * Qgroup will be handled by its qgroup_record then.
  7343. * btrfs_qgroup_free_data() call will do nothing here.
  7344. *
  7345. * 2) Not written to disk yet
  7346. * Then btrfs_qgroup_free_data() call will clear the QGROUP_RESERVED
  7347. * bit of its io_tree, and free the qgroup reserved data space.
  7348. * Since the IO will never happen for this page.
  7349. */
  7350. btrfs_qgroup_free_data(inode, NULL, page_start, PAGE_SIZE);
  7351. if (!inode_evicting) {
  7352. clear_extent_bit(tree, page_start, page_end, EXTENT_LOCKED |
  7353. EXTENT_DELALLOC | EXTENT_DELALLOC_NEW |
  7354. EXTENT_DO_ACCOUNTING | EXTENT_DEFRAG, 1, 1,
  7355. &cached_state);
  7356. __btrfs_releasepage(page, GFP_NOFS);
  7357. }
  7358. ClearPageChecked(page);
  7359. detach_page_private(page);
  7360. }
  7361. /*
  7362. * btrfs_page_mkwrite() is not allowed to change the file size as it gets
  7363. * called from a page fault handler when a page is first dirtied. Hence we must
  7364. * be careful to check for EOF conditions here. We set the page up correctly
  7365. * for a written page which means we get ENOSPC checking when writing into
  7366. * holes and correct delalloc and unwritten extent mapping on filesystems that
  7367. * support these features.
  7368. *
  7369. * We are not allowed to take the i_mutex here so we have to play games to
  7370. * protect against truncate races as the page could now be beyond EOF. Because
  7371. * truncate_setsize() writes the inode size before removing pages, once we have
  7372. * the page lock we can determine safely if the page is beyond EOF. If it is not
  7373. * beyond EOF, then the page is guaranteed safe against truncation until we
  7374. * unlock the page.
  7375. */
  7376. vm_fault_t btrfs_page_mkwrite(struct vm_fault *vmf)
  7377. {
  7378. struct page *page = vmf->page;
  7379. struct inode *inode = file_inode(vmf->vma->vm_file);
  7380. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7381. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  7382. struct btrfs_ordered_extent *ordered;
  7383. struct extent_state *cached_state = NULL;
  7384. struct extent_changeset *data_reserved = NULL;
  7385. char *kaddr;
  7386. unsigned long zero_start;
  7387. loff_t size;
  7388. vm_fault_t ret;
  7389. int ret2;
  7390. int reserved = 0;
  7391. u64 reserved_space;
  7392. u64 page_start;
  7393. u64 page_end;
  7394. u64 end;
  7395. reserved_space = PAGE_SIZE;
  7396. sb_start_pagefault(inode->i_sb);
  7397. page_start = page_offset(page);
  7398. page_end = page_start + PAGE_SIZE - 1;
  7399. end = page_end;
  7400. /*
  7401. * Reserving delalloc space after obtaining the page lock can lead to
  7402. * deadlock. For example, if a dirty page is locked by this function
  7403. * and the call to btrfs_delalloc_reserve_space() ends up triggering
  7404. * dirty page write out, then the btrfs_writepage() function could
  7405. * end up waiting indefinitely to get a lock on the page currently
  7406. * being processed by btrfs_page_mkwrite() function.
  7407. */
  7408. ret2 = btrfs_delalloc_reserve_space(BTRFS_I(inode), &data_reserved,
  7409. page_start, reserved_space);
  7410. if (!ret2) {
  7411. ret2 = file_update_time(vmf->vma->vm_file);
  7412. reserved = 1;
  7413. }
  7414. if (ret2) {
  7415. ret = vmf_error(ret2);
  7416. if (reserved)
  7417. goto out;
  7418. goto out_noreserve;
  7419. }
  7420. ret = VM_FAULT_NOPAGE; /* make the VM retry the fault */
  7421. again:
  7422. lock_page(page);
  7423. size = i_size_read(inode);
  7424. if ((page->mapping != inode->i_mapping) ||
  7425. (page_start >= size)) {
  7426. /* page got truncated out from underneath us */
  7427. goto out_unlock;
  7428. }
  7429. wait_on_page_writeback(page);
  7430. lock_extent_bits(io_tree, page_start, page_end, &cached_state);
  7431. set_page_extent_mapped(page);
  7432. /*
  7433. * we can't set the delalloc bits if there are pending ordered
  7434. * extents. Drop our locks and wait for them to finish
  7435. */
  7436. ordered = btrfs_lookup_ordered_range(BTRFS_I(inode), page_start,
  7437. PAGE_SIZE);
  7438. if (ordered) {
  7439. unlock_extent_cached(io_tree, page_start, page_end,
  7440. &cached_state);
  7441. unlock_page(page);
  7442. btrfs_start_ordered_extent(ordered, 1);
  7443. btrfs_put_ordered_extent(ordered);
  7444. goto again;
  7445. }
  7446. if (page->index == ((size - 1) >> PAGE_SHIFT)) {
  7447. reserved_space = round_up(size - page_start,
  7448. fs_info->sectorsize);
  7449. if (reserved_space < PAGE_SIZE) {
  7450. end = page_start + reserved_space - 1;
  7451. btrfs_delalloc_release_space(BTRFS_I(inode),
  7452. data_reserved, page_start,
  7453. PAGE_SIZE - reserved_space, true);
  7454. }
  7455. }
  7456. /*
  7457. * page_mkwrite gets called when the page is firstly dirtied after it's
  7458. * faulted in, but write(2) could also dirty a page and set delalloc
  7459. * bits, thus in this case for space account reason, we still need to
  7460. * clear any delalloc bits within this page range since we have to
  7461. * reserve data&meta space before lock_page() (see above comments).
  7462. */
  7463. clear_extent_bit(&BTRFS_I(inode)->io_tree, page_start, end,
  7464. EXTENT_DELALLOC | EXTENT_DO_ACCOUNTING |
  7465. EXTENT_DEFRAG, 0, 0, &cached_state);
  7466. ret2 = btrfs_set_extent_delalloc(BTRFS_I(inode), page_start, end, 0,
  7467. &cached_state);
  7468. if (ret2) {
  7469. unlock_extent_cached(io_tree, page_start, page_end,
  7470. &cached_state);
  7471. ret = VM_FAULT_SIGBUS;
  7472. goto out_unlock;
  7473. }
  7474. /* page is wholly or partially inside EOF */
  7475. if (page_start + PAGE_SIZE > size)
  7476. zero_start = offset_in_page(size);
  7477. else
  7478. zero_start = PAGE_SIZE;
  7479. if (zero_start != PAGE_SIZE) {
  7480. kaddr = kmap(page);
  7481. memset(kaddr + zero_start, 0, PAGE_SIZE - zero_start);
  7482. flush_dcache_page(page);
  7483. kunmap(page);
  7484. }
  7485. ClearPageChecked(page);
  7486. set_page_dirty(page);
  7487. SetPageUptodate(page);
  7488. btrfs_set_inode_last_sub_trans(BTRFS_I(inode));
  7489. unlock_extent_cached(io_tree, page_start, page_end, &cached_state);
  7490. btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
  7491. sb_end_pagefault(inode->i_sb);
  7492. extent_changeset_free(data_reserved);
  7493. return VM_FAULT_LOCKED;
  7494. out_unlock:
  7495. unlock_page(page);
  7496. out:
  7497. btrfs_delalloc_release_extents(BTRFS_I(inode), PAGE_SIZE);
  7498. btrfs_delalloc_release_space(BTRFS_I(inode), data_reserved, page_start,
  7499. reserved_space, (ret != 0));
  7500. out_noreserve:
  7501. sb_end_pagefault(inode->i_sb);
  7502. extent_changeset_free(data_reserved);
  7503. return ret;
  7504. }
  7505. static int btrfs_truncate(struct inode *inode, bool skip_writeback)
  7506. {
  7507. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  7508. struct btrfs_root *root = BTRFS_I(inode)->root;
  7509. struct btrfs_block_rsv *rsv;
  7510. int ret;
  7511. struct btrfs_trans_handle *trans;
  7512. u64 mask = fs_info->sectorsize - 1;
  7513. u64 min_size = btrfs_calc_metadata_size(fs_info, 1);
  7514. if (!skip_writeback) {
  7515. ret = btrfs_wait_ordered_range(inode, inode->i_size & (~mask),
  7516. (u64)-1);
  7517. if (ret)
  7518. return ret;
  7519. }
  7520. /*
  7521. * Yes ladies and gentlemen, this is indeed ugly. We have a couple of
  7522. * things going on here:
  7523. *
  7524. * 1) We need to reserve space to update our inode.
  7525. *
  7526. * 2) We need to have something to cache all the space that is going to
  7527. * be free'd up by the truncate operation, but also have some slack
  7528. * space reserved in case it uses space during the truncate (thank you
  7529. * very much snapshotting).
  7530. *
  7531. * And we need these to be separate. The fact is we can use a lot of
  7532. * space doing the truncate, and we have no earthly idea how much space
  7533. * we will use, so we need the truncate reservation to be separate so it
  7534. * doesn't end up using space reserved for updating the inode. We also
  7535. * need to be able to stop the transaction and start a new one, which
  7536. * means we need to be able to update the inode several times, and we
  7537. * have no idea of knowing how many times that will be, so we can't just
  7538. * reserve 1 item for the entirety of the operation, so that has to be
  7539. * done separately as well.
  7540. *
  7541. * So that leaves us with
  7542. *
  7543. * 1) rsv - for the truncate reservation, which we will steal from the
  7544. * transaction reservation.
  7545. * 2) fs_info->trans_block_rsv - this will have 1 items worth left for
  7546. * updating the inode.
  7547. */
  7548. rsv = btrfs_alloc_block_rsv(fs_info, BTRFS_BLOCK_RSV_TEMP);
  7549. if (!rsv)
  7550. return -ENOMEM;
  7551. rsv->size = min_size;
  7552. rsv->failfast = 1;
  7553. /*
  7554. * 1 for the truncate slack space
  7555. * 1 for updating the inode.
  7556. */
  7557. trans = btrfs_start_transaction(root, 2);
  7558. if (IS_ERR(trans)) {
  7559. ret = PTR_ERR(trans);
  7560. goto out;
  7561. }
  7562. /* Migrate the slack space for the truncate to our reserve */
  7563. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv, rsv,
  7564. min_size, false);
  7565. BUG_ON(ret);
  7566. /*
  7567. * So if we truncate and then write and fsync we normally would just
  7568. * write the extents that changed, which is a problem if we need to
  7569. * first truncate that entire inode. So set this flag so we write out
  7570. * all of the extents in the inode to the sync log so we're completely
  7571. * safe.
  7572. */
  7573. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC, &BTRFS_I(inode)->runtime_flags);
  7574. trans->block_rsv = rsv;
  7575. while (1) {
  7576. ret = btrfs_truncate_inode_items(trans, root, inode,
  7577. inode->i_size,
  7578. BTRFS_EXTENT_DATA_KEY);
  7579. trans->block_rsv = &fs_info->trans_block_rsv;
  7580. if (ret != -ENOSPC && ret != -EAGAIN)
  7581. break;
  7582. ret = btrfs_update_inode(trans, root, inode);
  7583. if (ret)
  7584. break;
  7585. btrfs_end_transaction(trans);
  7586. btrfs_btree_balance_dirty(fs_info);
  7587. trans = btrfs_start_transaction(root, 2);
  7588. if (IS_ERR(trans)) {
  7589. ret = PTR_ERR(trans);
  7590. trans = NULL;
  7591. break;
  7592. }
  7593. btrfs_block_rsv_release(fs_info, rsv, -1, NULL);
  7594. ret = btrfs_block_rsv_migrate(&fs_info->trans_block_rsv,
  7595. rsv, min_size, false);
  7596. BUG_ON(ret); /* shouldn't happen */
  7597. trans->block_rsv = rsv;
  7598. }
  7599. /*
  7600. * We can't call btrfs_truncate_block inside a trans handle as we could
  7601. * deadlock with freeze, if we got NEED_TRUNCATE_BLOCK then we know
  7602. * we've truncated everything except the last little bit, and can do
  7603. * btrfs_truncate_block and then update the disk_i_size.
  7604. */
  7605. if (ret == NEED_TRUNCATE_BLOCK) {
  7606. btrfs_end_transaction(trans);
  7607. btrfs_btree_balance_dirty(fs_info);
  7608. ret = btrfs_truncate_block(inode, inode->i_size, 0, 0);
  7609. if (ret)
  7610. goto out;
  7611. trans = btrfs_start_transaction(root, 1);
  7612. if (IS_ERR(trans)) {
  7613. ret = PTR_ERR(trans);
  7614. goto out;
  7615. }
  7616. btrfs_inode_safe_disk_i_size_write(inode, 0);
  7617. }
  7618. if (trans) {
  7619. int ret2;
  7620. trans->block_rsv = &fs_info->trans_block_rsv;
  7621. ret2 = btrfs_update_inode(trans, root, inode);
  7622. if (ret2 && !ret)
  7623. ret = ret2;
  7624. ret2 = btrfs_end_transaction(trans);
  7625. if (ret2 && !ret)
  7626. ret = ret2;
  7627. btrfs_btree_balance_dirty(fs_info);
  7628. }
  7629. out:
  7630. btrfs_free_block_rsv(fs_info, rsv);
  7631. return ret;
  7632. }
  7633. /*
  7634. * create a new subvolume directory/inode (helper for the ioctl).
  7635. */
  7636. int btrfs_create_subvol_root(struct btrfs_trans_handle *trans,
  7637. struct btrfs_root *new_root,
  7638. struct btrfs_root *parent_root,
  7639. u64 new_dirid)
  7640. {
  7641. struct inode *inode;
  7642. int err;
  7643. u64 index = 0;
  7644. inode = btrfs_new_inode(trans, new_root, NULL, "..", 2,
  7645. new_dirid, new_dirid,
  7646. S_IFDIR | (~current_umask() & S_IRWXUGO),
  7647. &index);
  7648. if (IS_ERR(inode))
  7649. return PTR_ERR(inode);
  7650. inode->i_op = &btrfs_dir_inode_operations;
  7651. inode->i_fop = &btrfs_dir_file_operations;
  7652. set_nlink(inode, 1);
  7653. btrfs_i_size_write(BTRFS_I(inode), 0);
  7654. unlock_new_inode(inode);
  7655. err = btrfs_subvol_inherit_props(trans, new_root, parent_root);
  7656. if (err)
  7657. btrfs_err(new_root->fs_info,
  7658. "error inheriting subvolume %llu properties: %d",
  7659. new_root->root_key.objectid, err);
  7660. err = btrfs_update_inode(trans, new_root, inode);
  7661. iput(inode);
  7662. return err;
  7663. }
  7664. struct inode *btrfs_alloc_inode(struct super_block *sb)
  7665. {
  7666. struct btrfs_fs_info *fs_info = btrfs_sb(sb);
  7667. struct btrfs_inode *ei;
  7668. struct inode *inode;
  7669. ei = kmem_cache_alloc(btrfs_inode_cachep, GFP_KERNEL);
  7670. if (!ei)
  7671. return NULL;
  7672. ei->root = NULL;
  7673. ei->generation = 0;
  7674. ei->last_trans = 0;
  7675. ei->last_sub_trans = 0;
  7676. ei->logged_trans = 0;
  7677. ei->delalloc_bytes = 0;
  7678. ei->new_delalloc_bytes = 0;
  7679. ei->defrag_bytes = 0;
  7680. ei->disk_i_size = 0;
  7681. ei->flags = 0;
  7682. ei->csum_bytes = 0;
  7683. ei->index_cnt = (u64)-1;
  7684. ei->dir_index = 0;
  7685. ei->last_unlink_trans = 0;
  7686. ei->last_reflink_trans = 0;
  7687. ei->last_log_commit = 0;
  7688. spin_lock_init(&ei->lock);
  7689. ei->outstanding_extents = 0;
  7690. if (sb->s_magic != BTRFS_TEST_MAGIC)
  7691. btrfs_init_metadata_block_rsv(fs_info, &ei->block_rsv,
  7692. BTRFS_BLOCK_RSV_DELALLOC);
  7693. ei->runtime_flags = 0;
  7694. ei->prop_compress = BTRFS_COMPRESS_NONE;
  7695. ei->defrag_compress = BTRFS_COMPRESS_NONE;
  7696. ei->delayed_node = NULL;
  7697. ei->i_otime.tv_sec = 0;
  7698. ei->i_otime.tv_nsec = 0;
  7699. inode = &ei->vfs_inode;
  7700. extent_map_tree_init(&ei->extent_tree);
  7701. extent_io_tree_init(fs_info, &ei->io_tree, IO_TREE_INODE_IO, inode);
  7702. extent_io_tree_init(fs_info, &ei->io_failure_tree,
  7703. IO_TREE_INODE_IO_FAILURE, inode);
  7704. extent_io_tree_init(fs_info, &ei->file_extent_tree,
  7705. IO_TREE_INODE_FILE_EXTENT, inode);
  7706. ei->io_tree.track_uptodate = true;
  7707. ei->io_failure_tree.track_uptodate = true;
  7708. atomic_set(&ei->sync_writers, 0);
  7709. mutex_init(&ei->log_mutex);
  7710. btrfs_ordered_inode_tree_init(&ei->ordered_tree);
  7711. INIT_LIST_HEAD(&ei->delalloc_inodes);
  7712. INIT_LIST_HEAD(&ei->delayed_iput);
  7713. RB_CLEAR_NODE(&ei->rb_node);
  7714. init_rwsem(&ei->dio_sem);
  7715. return inode;
  7716. }
  7717. #ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
  7718. void btrfs_test_destroy_inode(struct inode *inode)
  7719. {
  7720. btrfs_drop_extent_cache(BTRFS_I(inode), 0, (u64)-1, 0);
  7721. kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
  7722. }
  7723. #endif
  7724. void btrfs_free_inode(struct inode *inode)
  7725. {
  7726. kmem_cache_free(btrfs_inode_cachep, BTRFS_I(inode));
  7727. }
  7728. void btrfs_destroy_inode(struct inode *vfs_inode)
  7729. {
  7730. struct btrfs_ordered_extent *ordered;
  7731. struct btrfs_inode *inode = BTRFS_I(vfs_inode);
  7732. struct btrfs_root *root = inode->root;
  7733. WARN_ON(!hlist_empty(&vfs_inode->i_dentry));
  7734. WARN_ON(vfs_inode->i_data.nrpages);
  7735. WARN_ON(inode->block_rsv.reserved);
  7736. WARN_ON(inode->block_rsv.size);
  7737. WARN_ON(inode->outstanding_extents);
  7738. WARN_ON(inode->delalloc_bytes);
  7739. WARN_ON(inode->new_delalloc_bytes);
  7740. WARN_ON(inode->csum_bytes);
  7741. WARN_ON(inode->defrag_bytes);
  7742. /*
  7743. * This can happen where we create an inode, but somebody else also
  7744. * created the same inode and we need to destroy the one we already
  7745. * created.
  7746. */
  7747. if (!root)
  7748. return;
  7749. while (1) {
  7750. ordered = btrfs_lookup_first_ordered_extent(inode, (u64)-1);
  7751. if (!ordered)
  7752. break;
  7753. else {
  7754. btrfs_err(root->fs_info,
  7755. "found ordered extent %llu %llu on inode cleanup",
  7756. ordered->file_offset, ordered->num_bytes);
  7757. btrfs_remove_ordered_extent(inode, ordered);
  7758. btrfs_put_ordered_extent(ordered);
  7759. btrfs_put_ordered_extent(ordered);
  7760. }
  7761. }
  7762. btrfs_qgroup_check_reserved_leak(inode);
  7763. inode_tree_del(inode);
  7764. btrfs_drop_extent_cache(inode, 0, (u64)-1, 0);
  7765. btrfs_inode_clear_file_extent_range(inode, 0, (u64)-1);
  7766. btrfs_put_root(inode->root);
  7767. }
  7768. int btrfs_drop_inode(struct inode *inode)
  7769. {
  7770. struct btrfs_root *root = BTRFS_I(inode)->root;
  7771. if (root == NULL)
  7772. return 1;
  7773. /* the snap/subvol tree is on deleting */
  7774. if (btrfs_root_refs(&root->root_item) == 0)
  7775. return 1;
  7776. else
  7777. return generic_drop_inode(inode);
  7778. }
  7779. static void init_once(void *foo)
  7780. {
  7781. struct btrfs_inode *ei = (struct btrfs_inode *) foo;
  7782. inode_init_once(&ei->vfs_inode);
  7783. }
  7784. void __cold btrfs_destroy_cachep(void)
  7785. {
  7786. /*
  7787. * Make sure all delayed rcu free inodes are flushed before we
  7788. * destroy cache.
  7789. */
  7790. rcu_barrier();
  7791. kmem_cache_destroy(btrfs_inode_cachep);
  7792. kmem_cache_destroy(btrfs_trans_handle_cachep);
  7793. kmem_cache_destroy(btrfs_path_cachep);
  7794. kmem_cache_destroy(btrfs_free_space_cachep);
  7795. kmem_cache_destroy(btrfs_free_space_bitmap_cachep);
  7796. }
  7797. int __init btrfs_init_cachep(void)
  7798. {
  7799. btrfs_inode_cachep = kmem_cache_create("btrfs_inode",
  7800. sizeof(struct btrfs_inode), 0,
  7801. SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD | SLAB_ACCOUNT,
  7802. init_once);
  7803. if (!btrfs_inode_cachep)
  7804. goto fail;
  7805. btrfs_trans_handle_cachep = kmem_cache_create("btrfs_trans_handle",
  7806. sizeof(struct btrfs_trans_handle), 0,
  7807. SLAB_TEMPORARY | SLAB_MEM_SPREAD, NULL);
  7808. if (!btrfs_trans_handle_cachep)
  7809. goto fail;
  7810. btrfs_path_cachep = kmem_cache_create("btrfs_path",
  7811. sizeof(struct btrfs_path), 0,
  7812. SLAB_MEM_SPREAD, NULL);
  7813. if (!btrfs_path_cachep)
  7814. goto fail;
  7815. btrfs_free_space_cachep = kmem_cache_create("btrfs_free_space",
  7816. sizeof(struct btrfs_free_space), 0,
  7817. SLAB_MEM_SPREAD, NULL);
  7818. if (!btrfs_free_space_cachep)
  7819. goto fail;
  7820. btrfs_free_space_bitmap_cachep = kmem_cache_create("btrfs_free_space_bitmap",
  7821. PAGE_SIZE, PAGE_SIZE,
  7822. SLAB_MEM_SPREAD, NULL);
  7823. if (!btrfs_free_space_bitmap_cachep)
  7824. goto fail;
  7825. return 0;
  7826. fail:
  7827. btrfs_destroy_cachep();
  7828. return -ENOMEM;
  7829. }
  7830. static int btrfs_getattr(const struct path *path, struct kstat *stat,
  7831. u32 request_mask, unsigned int flags)
  7832. {
  7833. u64 delalloc_bytes;
  7834. struct inode *inode = d_inode(path->dentry);
  7835. u32 blocksize = inode->i_sb->s_blocksize;
  7836. u32 bi_flags = BTRFS_I(inode)->flags;
  7837. stat->result_mask |= STATX_BTIME;
  7838. stat->btime.tv_sec = BTRFS_I(inode)->i_otime.tv_sec;
  7839. stat->btime.tv_nsec = BTRFS_I(inode)->i_otime.tv_nsec;
  7840. if (bi_flags & BTRFS_INODE_APPEND)
  7841. stat->attributes |= STATX_ATTR_APPEND;
  7842. if (bi_flags & BTRFS_INODE_COMPRESS)
  7843. stat->attributes |= STATX_ATTR_COMPRESSED;
  7844. if (bi_flags & BTRFS_INODE_IMMUTABLE)
  7845. stat->attributes |= STATX_ATTR_IMMUTABLE;
  7846. if (bi_flags & BTRFS_INODE_NODUMP)
  7847. stat->attributes |= STATX_ATTR_NODUMP;
  7848. stat->attributes_mask |= (STATX_ATTR_APPEND |
  7849. STATX_ATTR_COMPRESSED |
  7850. STATX_ATTR_IMMUTABLE |
  7851. STATX_ATTR_NODUMP);
  7852. generic_fillattr(inode, stat);
  7853. stat->dev = BTRFS_I(inode)->root->anon_dev;
  7854. spin_lock(&BTRFS_I(inode)->lock);
  7855. delalloc_bytes = BTRFS_I(inode)->new_delalloc_bytes;
  7856. spin_unlock(&BTRFS_I(inode)->lock);
  7857. stat->blocks = (ALIGN(inode_get_bytes(inode), blocksize) +
  7858. ALIGN(delalloc_bytes, blocksize)) >> 9;
  7859. return 0;
  7860. }
  7861. static int btrfs_rename_exchange(struct inode *old_dir,
  7862. struct dentry *old_dentry,
  7863. struct inode *new_dir,
  7864. struct dentry *new_dentry)
  7865. {
  7866. struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
  7867. struct btrfs_trans_handle *trans;
  7868. struct btrfs_root *root = BTRFS_I(old_dir)->root;
  7869. struct btrfs_root *dest = BTRFS_I(new_dir)->root;
  7870. struct inode *new_inode = new_dentry->d_inode;
  7871. struct inode *old_inode = old_dentry->d_inode;
  7872. struct timespec64 ctime = current_time(old_inode);
  7873. u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
  7874. u64 new_ino = btrfs_ino(BTRFS_I(new_inode));
  7875. u64 old_idx = 0;
  7876. u64 new_idx = 0;
  7877. int ret;
  7878. int ret2;
  7879. bool root_log_pinned = false;
  7880. bool dest_log_pinned = false;
  7881. bool need_abort = false;
  7882. /*
  7883. * For non-subvolumes allow exchange only within one subvolume, in the
  7884. * same inode namespace. Two subvolumes (represented as directory) can
  7885. * be exchanged as they're a logical link and have a fixed inode number.
  7886. */
  7887. if (root != dest &&
  7888. (old_ino != BTRFS_FIRST_FREE_OBJECTID ||
  7889. new_ino != BTRFS_FIRST_FREE_OBJECTID))
  7890. return -EXDEV;
  7891. /* close the race window with snapshot create/destroy ioctl */
  7892. if (old_ino == BTRFS_FIRST_FREE_OBJECTID ||
  7893. new_ino == BTRFS_FIRST_FREE_OBJECTID)
  7894. down_read(&fs_info->subvol_sem);
  7895. /*
  7896. * We want to reserve the absolute worst case amount of items. So if
  7897. * both inodes are subvols and we need to unlink them then that would
  7898. * require 4 item modifications, but if they are both normal inodes it
  7899. * would require 5 item modifications, so we'll assume their normal
  7900. * inodes. So 5 * 2 is 10, plus 2 for the new links, so 12 total items
  7901. * should cover the worst case number of items we'll modify.
  7902. */
  7903. trans = btrfs_start_transaction(root, 12);
  7904. if (IS_ERR(trans)) {
  7905. ret = PTR_ERR(trans);
  7906. goto out_notrans;
  7907. }
  7908. if (dest != root)
  7909. btrfs_record_root_in_trans(trans, dest);
  7910. /*
  7911. * We need to find a free sequence number both in the source and
  7912. * in the destination directory for the exchange.
  7913. */
  7914. ret = btrfs_set_inode_index(BTRFS_I(new_dir), &old_idx);
  7915. if (ret)
  7916. goto out_fail;
  7917. ret = btrfs_set_inode_index(BTRFS_I(old_dir), &new_idx);
  7918. if (ret)
  7919. goto out_fail;
  7920. BTRFS_I(old_inode)->dir_index = 0ULL;
  7921. BTRFS_I(new_inode)->dir_index = 0ULL;
  7922. /* Reference for the source. */
  7923. if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
  7924. /* force full log commit if subvolume involved. */
  7925. btrfs_set_log_full_commit(trans);
  7926. } else {
  7927. btrfs_pin_log_trans(root);
  7928. root_log_pinned = true;
  7929. ret = btrfs_insert_inode_ref(trans, dest,
  7930. new_dentry->d_name.name,
  7931. new_dentry->d_name.len,
  7932. old_ino,
  7933. btrfs_ino(BTRFS_I(new_dir)),
  7934. old_idx);
  7935. if (ret)
  7936. goto out_fail;
  7937. need_abort = true;
  7938. }
  7939. /* And now for the dest. */
  7940. if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
  7941. /* force full log commit if subvolume involved. */
  7942. btrfs_set_log_full_commit(trans);
  7943. } else {
  7944. btrfs_pin_log_trans(dest);
  7945. dest_log_pinned = true;
  7946. ret = btrfs_insert_inode_ref(trans, root,
  7947. old_dentry->d_name.name,
  7948. old_dentry->d_name.len,
  7949. new_ino,
  7950. btrfs_ino(BTRFS_I(old_dir)),
  7951. new_idx);
  7952. if (ret) {
  7953. if (need_abort)
  7954. btrfs_abort_transaction(trans, ret);
  7955. goto out_fail;
  7956. }
  7957. }
  7958. /* Update inode version and ctime/mtime. */
  7959. inode_inc_iversion(old_dir);
  7960. inode_inc_iversion(new_dir);
  7961. inode_inc_iversion(old_inode);
  7962. inode_inc_iversion(new_inode);
  7963. old_dir->i_ctime = old_dir->i_mtime = ctime;
  7964. new_dir->i_ctime = new_dir->i_mtime = ctime;
  7965. old_inode->i_ctime = ctime;
  7966. new_inode->i_ctime = ctime;
  7967. if (old_dentry->d_parent != new_dentry->d_parent) {
  7968. btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
  7969. BTRFS_I(old_inode), 1);
  7970. btrfs_record_unlink_dir(trans, BTRFS_I(new_dir),
  7971. BTRFS_I(new_inode), 1);
  7972. }
  7973. /* src is a subvolume */
  7974. if (old_ino == BTRFS_FIRST_FREE_OBJECTID) {
  7975. ret = btrfs_unlink_subvol(trans, old_dir, old_dentry);
  7976. } else { /* src is an inode */
  7977. ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
  7978. BTRFS_I(old_dentry->d_inode),
  7979. old_dentry->d_name.name,
  7980. old_dentry->d_name.len);
  7981. if (!ret)
  7982. ret = btrfs_update_inode(trans, root, old_inode);
  7983. }
  7984. if (ret) {
  7985. btrfs_abort_transaction(trans, ret);
  7986. goto out_fail;
  7987. }
  7988. /* dest is a subvolume */
  7989. if (new_ino == BTRFS_FIRST_FREE_OBJECTID) {
  7990. ret = btrfs_unlink_subvol(trans, new_dir, new_dentry);
  7991. } else { /* dest is an inode */
  7992. ret = __btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
  7993. BTRFS_I(new_dentry->d_inode),
  7994. new_dentry->d_name.name,
  7995. new_dentry->d_name.len);
  7996. if (!ret)
  7997. ret = btrfs_update_inode(trans, dest, new_inode);
  7998. }
  7999. if (ret) {
  8000. btrfs_abort_transaction(trans, ret);
  8001. goto out_fail;
  8002. }
  8003. ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
  8004. new_dentry->d_name.name,
  8005. new_dentry->d_name.len, 0, old_idx);
  8006. if (ret) {
  8007. btrfs_abort_transaction(trans, ret);
  8008. goto out_fail;
  8009. }
  8010. ret = btrfs_add_link(trans, BTRFS_I(old_dir), BTRFS_I(new_inode),
  8011. old_dentry->d_name.name,
  8012. old_dentry->d_name.len, 0, new_idx);
  8013. if (ret) {
  8014. btrfs_abort_transaction(trans, ret);
  8015. goto out_fail;
  8016. }
  8017. if (old_inode->i_nlink == 1)
  8018. BTRFS_I(old_inode)->dir_index = old_idx;
  8019. if (new_inode->i_nlink == 1)
  8020. BTRFS_I(new_inode)->dir_index = new_idx;
  8021. if (root_log_pinned) {
  8022. btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
  8023. new_dentry->d_parent);
  8024. btrfs_end_log_trans(root);
  8025. root_log_pinned = false;
  8026. }
  8027. if (dest_log_pinned) {
  8028. btrfs_log_new_name(trans, BTRFS_I(new_inode), BTRFS_I(new_dir),
  8029. old_dentry->d_parent);
  8030. btrfs_end_log_trans(dest);
  8031. dest_log_pinned = false;
  8032. }
  8033. out_fail:
  8034. /*
  8035. * If we have pinned a log and an error happened, we unpin tasks
  8036. * trying to sync the log and force them to fallback to a transaction
  8037. * commit if the log currently contains any of the inodes involved in
  8038. * this rename operation (to ensure we do not persist a log with an
  8039. * inconsistent state for any of these inodes or leading to any
  8040. * inconsistencies when replayed). If the transaction was aborted, the
  8041. * abortion reason is propagated to userspace when attempting to commit
  8042. * the transaction. If the log does not contain any of these inodes, we
  8043. * allow the tasks to sync it.
  8044. */
  8045. if (ret && (root_log_pinned || dest_log_pinned)) {
  8046. if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
  8047. btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
  8048. btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
  8049. (new_inode &&
  8050. btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
  8051. btrfs_set_log_full_commit(trans);
  8052. if (root_log_pinned) {
  8053. btrfs_end_log_trans(root);
  8054. root_log_pinned = false;
  8055. }
  8056. if (dest_log_pinned) {
  8057. btrfs_end_log_trans(dest);
  8058. dest_log_pinned = false;
  8059. }
  8060. }
  8061. ret2 = btrfs_end_transaction(trans);
  8062. ret = ret ? ret : ret2;
  8063. out_notrans:
  8064. if (new_ino == BTRFS_FIRST_FREE_OBJECTID ||
  8065. old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8066. up_read(&fs_info->subvol_sem);
  8067. return ret;
  8068. }
  8069. static int btrfs_whiteout_for_rename(struct btrfs_trans_handle *trans,
  8070. struct btrfs_root *root,
  8071. struct inode *dir,
  8072. struct dentry *dentry)
  8073. {
  8074. int ret;
  8075. struct inode *inode;
  8076. u64 objectid;
  8077. u64 index;
  8078. ret = btrfs_find_free_ino(root, &objectid);
  8079. if (ret)
  8080. return ret;
  8081. inode = btrfs_new_inode(trans, root, dir,
  8082. dentry->d_name.name,
  8083. dentry->d_name.len,
  8084. btrfs_ino(BTRFS_I(dir)),
  8085. objectid,
  8086. S_IFCHR | WHITEOUT_MODE,
  8087. &index);
  8088. if (IS_ERR(inode)) {
  8089. ret = PTR_ERR(inode);
  8090. return ret;
  8091. }
  8092. inode->i_op = &btrfs_special_inode_operations;
  8093. init_special_inode(inode, inode->i_mode,
  8094. WHITEOUT_DEV);
  8095. ret = btrfs_init_inode_security(trans, inode, dir,
  8096. &dentry->d_name);
  8097. if (ret)
  8098. goto out;
  8099. ret = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
  8100. BTRFS_I(inode), 0, index);
  8101. if (ret)
  8102. goto out;
  8103. ret = btrfs_update_inode(trans, root, inode);
  8104. out:
  8105. unlock_new_inode(inode);
  8106. if (ret)
  8107. inode_dec_link_count(inode);
  8108. iput(inode);
  8109. return ret;
  8110. }
  8111. static int btrfs_rename(struct inode *old_dir, struct dentry *old_dentry,
  8112. struct inode *new_dir, struct dentry *new_dentry,
  8113. unsigned int flags)
  8114. {
  8115. struct btrfs_fs_info *fs_info = btrfs_sb(old_dir->i_sb);
  8116. struct btrfs_trans_handle *trans;
  8117. unsigned int trans_num_items;
  8118. struct btrfs_root *root = BTRFS_I(old_dir)->root;
  8119. struct btrfs_root *dest = BTRFS_I(new_dir)->root;
  8120. struct inode *new_inode = d_inode(new_dentry);
  8121. struct inode *old_inode = d_inode(old_dentry);
  8122. u64 index = 0;
  8123. int ret;
  8124. int ret2;
  8125. u64 old_ino = btrfs_ino(BTRFS_I(old_inode));
  8126. bool log_pinned = false;
  8127. if (btrfs_ino(BTRFS_I(new_dir)) == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)
  8128. return -EPERM;
  8129. /* we only allow rename subvolume link between subvolumes */
  8130. if (old_ino != BTRFS_FIRST_FREE_OBJECTID && root != dest)
  8131. return -EXDEV;
  8132. if (old_ino == BTRFS_EMPTY_SUBVOL_DIR_OBJECTID ||
  8133. (new_inode && btrfs_ino(BTRFS_I(new_inode)) == BTRFS_FIRST_FREE_OBJECTID))
  8134. return -ENOTEMPTY;
  8135. if (S_ISDIR(old_inode->i_mode) && new_inode &&
  8136. new_inode->i_size > BTRFS_EMPTY_DIR_SIZE)
  8137. return -ENOTEMPTY;
  8138. /* check for collisions, even if the name isn't there */
  8139. ret = btrfs_check_dir_item_collision(dest, new_dir->i_ino,
  8140. new_dentry->d_name.name,
  8141. new_dentry->d_name.len);
  8142. if (ret) {
  8143. if (ret == -EEXIST) {
  8144. /* we shouldn't get
  8145. * eexist without a new_inode */
  8146. if (WARN_ON(!new_inode)) {
  8147. return ret;
  8148. }
  8149. } else {
  8150. /* maybe -EOVERFLOW */
  8151. return ret;
  8152. }
  8153. }
  8154. ret = 0;
  8155. /*
  8156. * we're using rename to replace one file with another. Start IO on it
  8157. * now so we don't add too much work to the end of the transaction
  8158. */
  8159. if (new_inode && S_ISREG(old_inode->i_mode) && new_inode->i_size)
  8160. filemap_flush(old_inode->i_mapping);
  8161. /* close the racy window with snapshot create/destroy ioctl */
  8162. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8163. down_read(&fs_info->subvol_sem);
  8164. /*
  8165. * We want to reserve the absolute worst case amount of items. So if
  8166. * both inodes are subvols and we need to unlink them then that would
  8167. * require 4 item modifications, but if they are both normal inodes it
  8168. * would require 5 item modifications, so we'll assume they are normal
  8169. * inodes. So 5 * 2 is 10, plus 1 for the new link, so 11 total items
  8170. * should cover the worst case number of items we'll modify.
  8171. * If our rename has the whiteout flag, we need more 5 units for the
  8172. * new inode (1 inode item, 1 inode ref, 2 dir items and 1 xattr item
  8173. * when selinux is enabled).
  8174. */
  8175. trans_num_items = 11;
  8176. if (flags & RENAME_WHITEOUT)
  8177. trans_num_items += 5;
  8178. trans = btrfs_start_transaction(root, trans_num_items);
  8179. if (IS_ERR(trans)) {
  8180. ret = PTR_ERR(trans);
  8181. goto out_notrans;
  8182. }
  8183. if (dest != root)
  8184. btrfs_record_root_in_trans(trans, dest);
  8185. ret = btrfs_set_inode_index(BTRFS_I(new_dir), &index);
  8186. if (ret)
  8187. goto out_fail;
  8188. BTRFS_I(old_inode)->dir_index = 0ULL;
  8189. if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
  8190. /* force full log commit if subvolume involved. */
  8191. btrfs_set_log_full_commit(trans);
  8192. } else {
  8193. btrfs_pin_log_trans(root);
  8194. log_pinned = true;
  8195. ret = btrfs_insert_inode_ref(trans, dest,
  8196. new_dentry->d_name.name,
  8197. new_dentry->d_name.len,
  8198. old_ino,
  8199. btrfs_ino(BTRFS_I(new_dir)), index);
  8200. if (ret)
  8201. goto out_fail;
  8202. }
  8203. inode_inc_iversion(old_dir);
  8204. inode_inc_iversion(new_dir);
  8205. inode_inc_iversion(old_inode);
  8206. old_dir->i_ctime = old_dir->i_mtime =
  8207. new_dir->i_ctime = new_dir->i_mtime =
  8208. old_inode->i_ctime = current_time(old_dir);
  8209. if (old_dentry->d_parent != new_dentry->d_parent)
  8210. btrfs_record_unlink_dir(trans, BTRFS_I(old_dir),
  8211. BTRFS_I(old_inode), 1);
  8212. if (unlikely(old_ino == BTRFS_FIRST_FREE_OBJECTID)) {
  8213. ret = btrfs_unlink_subvol(trans, old_dir, old_dentry);
  8214. } else {
  8215. ret = __btrfs_unlink_inode(trans, root, BTRFS_I(old_dir),
  8216. BTRFS_I(d_inode(old_dentry)),
  8217. old_dentry->d_name.name,
  8218. old_dentry->d_name.len);
  8219. if (!ret)
  8220. ret = btrfs_update_inode(trans, root, old_inode);
  8221. }
  8222. if (ret) {
  8223. btrfs_abort_transaction(trans, ret);
  8224. goto out_fail;
  8225. }
  8226. if (new_inode) {
  8227. inode_inc_iversion(new_inode);
  8228. new_inode->i_ctime = current_time(new_inode);
  8229. if (unlikely(btrfs_ino(BTRFS_I(new_inode)) ==
  8230. BTRFS_EMPTY_SUBVOL_DIR_OBJECTID)) {
  8231. ret = btrfs_unlink_subvol(trans, new_dir, new_dentry);
  8232. BUG_ON(new_inode->i_nlink == 0);
  8233. } else {
  8234. ret = btrfs_unlink_inode(trans, dest, BTRFS_I(new_dir),
  8235. BTRFS_I(d_inode(new_dentry)),
  8236. new_dentry->d_name.name,
  8237. new_dentry->d_name.len);
  8238. }
  8239. if (!ret && new_inode->i_nlink == 0)
  8240. ret = btrfs_orphan_add(trans,
  8241. BTRFS_I(d_inode(new_dentry)));
  8242. if (ret) {
  8243. btrfs_abort_transaction(trans, ret);
  8244. goto out_fail;
  8245. }
  8246. }
  8247. ret = btrfs_add_link(trans, BTRFS_I(new_dir), BTRFS_I(old_inode),
  8248. new_dentry->d_name.name,
  8249. new_dentry->d_name.len, 0, index);
  8250. if (ret) {
  8251. btrfs_abort_transaction(trans, ret);
  8252. goto out_fail;
  8253. }
  8254. if (old_inode->i_nlink == 1)
  8255. BTRFS_I(old_inode)->dir_index = index;
  8256. if (log_pinned) {
  8257. btrfs_log_new_name(trans, BTRFS_I(old_inode), BTRFS_I(old_dir),
  8258. new_dentry->d_parent);
  8259. btrfs_end_log_trans(root);
  8260. log_pinned = false;
  8261. }
  8262. if (flags & RENAME_WHITEOUT) {
  8263. ret = btrfs_whiteout_for_rename(trans, root, old_dir,
  8264. old_dentry);
  8265. if (ret) {
  8266. btrfs_abort_transaction(trans, ret);
  8267. goto out_fail;
  8268. }
  8269. }
  8270. out_fail:
  8271. /*
  8272. * If we have pinned the log and an error happened, we unpin tasks
  8273. * trying to sync the log and force them to fallback to a transaction
  8274. * commit if the log currently contains any of the inodes involved in
  8275. * this rename operation (to ensure we do not persist a log with an
  8276. * inconsistent state for any of these inodes or leading to any
  8277. * inconsistencies when replayed). If the transaction was aborted, the
  8278. * abortion reason is propagated to userspace when attempting to commit
  8279. * the transaction. If the log does not contain any of these inodes, we
  8280. * allow the tasks to sync it.
  8281. */
  8282. if (ret && log_pinned) {
  8283. if (btrfs_inode_in_log(BTRFS_I(old_dir), fs_info->generation) ||
  8284. btrfs_inode_in_log(BTRFS_I(new_dir), fs_info->generation) ||
  8285. btrfs_inode_in_log(BTRFS_I(old_inode), fs_info->generation) ||
  8286. (new_inode &&
  8287. btrfs_inode_in_log(BTRFS_I(new_inode), fs_info->generation)))
  8288. btrfs_set_log_full_commit(trans);
  8289. btrfs_end_log_trans(root);
  8290. log_pinned = false;
  8291. }
  8292. ret2 = btrfs_end_transaction(trans);
  8293. ret = ret ? ret : ret2;
  8294. out_notrans:
  8295. if (old_ino == BTRFS_FIRST_FREE_OBJECTID)
  8296. up_read(&fs_info->subvol_sem);
  8297. return ret;
  8298. }
  8299. static int btrfs_rename2(struct inode *old_dir, struct dentry *old_dentry,
  8300. struct inode *new_dir, struct dentry *new_dentry,
  8301. unsigned int flags)
  8302. {
  8303. if (flags & ~(RENAME_NOREPLACE | RENAME_EXCHANGE | RENAME_WHITEOUT))
  8304. return -EINVAL;
  8305. if (flags & RENAME_EXCHANGE)
  8306. return btrfs_rename_exchange(old_dir, old_dentry, new_dir,
  8307. new_dentry);
  8308. return btrfs_rename(old_dir, old_dentry, new_dir, new_dentry, flags);
  8309. }
  8310. struct btrfs_delalloc_work {
  8311. struct inode *inode;
  8312. struct completion completion;
  8313. struct list_head list;
  8314. struct btrfs_work work;
  8315. };
  8316. static void btrfs_run_delalloc_work(struct btrfs_work *work)
  8317. {
  8318. struct btrfs_delalloc_work *delalloc_work;
  8319. struct inode *inode;
  8320. delalloc_work = container_of(work, struct btrfs_delalloc_work,
  8321. work);
  8322. inode = delalloc_work->inode;
  8323. filemap_flush(inode->i_mapping);
  8324. if (test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  8325. &BTRFS_I(inode)->runtime_flags))
  8326. filemap_flush(inode->i_mapping);
  8327. iput(inode);
  8328. complete(&delalloc_work->completion);
  8329. }
  8330. static struct btrfs_delalloc_work *btrfs_alloc_delalloc_work(struct inode *inode)
  8331. {
  8332. struct btrfs_delalloc_work *work;
  8333. work = kmalloc(sizeof(*work), GFP_NOFS);
  8334. if (!work)
  8335. return NULL;
  8336. init_completion(&work->completion);
  8337. INIT_LIST_HEAD(&work->list);
  8338. work->inode = inode;
  8339. btrfs_init_work(&work->work, btrfs_run_delalloc_work, NULL, NULL);
  8340. return work;
  8341. }
  8342. /*
  8343. * some fairly slow code that needs optimization. This walks the list
  8344. * of all the inodes with pending delalloc and forces them to disk.
  8345. */
  8346. static int start_delalloc_inodes(struct btrfs_root *root,
  8347. struct writeback_control *wbc, bool snapshot,
  8348. bool in_reclaim_context)
  8349. {
  8350. struct btrfs_inode *binode;
  8351. struct inode *inode;
  8352. struct btrfs_delalloc_work *work, *next;
  8353. struct list_head works;
  8354. struct list_head splice;
  8355. int ret = 0;
  8356. bool full_flush = wbc->nr_to_write == LONG_MAX;
  8357. INIT_LIST_HEAD(&works);
  8358. INIT_LIST_HEAD(&splice);
  8359. mutex_lock(&root->delalloc_mutex);
  8360. spin_lock(&root->delalloc_lock);
  8361. list_splice_init(&root->delalloc_inodes, &splice);
  8362. while (!list_empty(&splice)) {
  8363. binode = list_entry(splice.next, struct btrfs_inode,
  8364. delalloc_inodes);
  8365. list_move_tail(&binode->delalloc_inodes,
  8366. &root->delalloc_inodes);
  8367. if (in_reclaim_context &&
  8368. test_bit(BTRFS_INODE_NO_DELALLOC_FLUSH, &binode->runtime_flags))
  8369. continue;
  8370. inode = igrab(&binode->vfs_inode);
  8371. if (!inode) {
  8372. cond_resched_lock(&root->delalloc_lock);
  8373. continue;
  8374. }
  8375. spin_unlock(&root->delalloc_lock);
  8376. if (snapshot)
  8377. set_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
  8378. &binode->runtime_flags);
  8379. if (full_flush) {
  8380. work = btrfs_alloc_delalloc_work(inode);
  8381. if (!work) {
  8382. iput(inode);
  8383. ret = -ENOMEM;
  8384. goto out;
  8385. }
  8386. list_add_tail(&work->list, &works);
  8387. btrfs_queue_work(root->fs_info->flush_workers,
  8388. &work->work);
  8389. } else {
  8390. ret = sync_inode(inode, wbc);
  8391. if (!ret &&
  8392. test_bit(BTRFS_INODE_HAS_ASYNC_EXTENT,
  8393. &BTRFS_I(inode)->runtime_flags))
  8394. ret = sync_inode(inode, wbc);
  8395. btrfs_add_delayed_iput(inode);
  8396. if (ret || wbc->nr_to_write <= 0)
  8397. goto out;
  8398. }
  8399. cond_resched();
  8400. spin_lock(&root->delalloc_lock);
  8401. }
  8402. spin_unlock(&root->delalloc_lock);
  8403. out:
  8404. list_for_each_entry_safe(work, next, &works, list) {
  8405. list_del_init(&work->list);
  8406. wait_for_completion(&work->completion);
  8407. kfree(work);
  8408. }
  8409. if (!list_empty(&splice)) {
  8410. spin_lock(&root->delalloc_lock);
  8411. list_splice_tail(&splice, &root->delalloc_inodes);
  8412. spin_unlock(&root->delalloc_lock);
  8413. }
  8414. mutex_unlock(&root->delalloc_mutex);
  8415. return ret;
  8416. }
  8417. int btrfs_start_delalloc_snapshot(struct btrfs_root *root)
  8418. {
  8419. struct writeback_control wbc = {
  8420. .nr_to_write = LONG_MAX,
  8421. .sync_mode = WB_SYNC_NONE,
  8422. .range_start = 0,
  8423. .range_end = LLONG_MAX,
  8424. };
  8425. struct btrfs_fs_info *fs_info = root->fs_info;
  8426. if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  8427. return -EROFS;
  8428. return start_delalloc_inodes(root, &wbc, true, false);
  8429. }
  8430. int btrfs_start_delalloc_roots(struct btrfs_fs_info *fs_info, u64 nr,
  8431. bool in_reclaim_context)
  8432. {
  8433. struct writeback_control wbc = {
  8434. .nr_to_write = (nr == U64_MAX) ? LONG_MAX : (unsigned long)nr,
  8435. .sync_mode = WB_SYNC_NONE,
  8436. .range_start = 0,
  8437. .range_end = LLONG_MAX,
  8438. };
  8439. struct btrfs_root *root;
  8440. struct list_head splice;
  8441. int ret;
  8442. if (test_bit(BTRFS_FS_STATE_ERROR, &fs_info->fs_state))
  8443. return -EROFS;
  8444. INIT_LIST_HEAD(&splice);
  8445. mutex_lock(&fs_info->delalloc_root_mutex);
  8446. spin_lock(&fs_info->delalloc_root_lock);
  8447. list_splice_init(&fs_info->delalloc_roots, &splice);
  8448. while (!list_empty(&splice) && nr) {
  8449. /*
  8450. * Reset nr_to_write here so we know that we're doing a full
  8451. * flush.
  8452. */
  8453. if (nr == U64_MAX)
  8454. wbc.nr_to_write = LONG_MAX;
  8455. root = list_first_entry(&splice, struct btrfs_root,
  8456. delalloc_root);
  8457. root = btrfs_grab_root(root);
  8458. BUG_ON(!root);
  8459. list_move_tail(&root->delalloc_root,
  8460. &fs_info->delalloc_roots);
  8461. spin_unlock(&fs_info->delalloc_root_lock);
  8462. ret = start_delalloc_inodes(root, &wbc, false, in_reclaim_context);
  8463. btrfs_put_root(root);
  8464. if (ret < 0 || wbc.nr_to_write <= 0)
  8465. goto out;
  8466. spin_lock(&fs_info->delalloc_root_lock);
  8467. }
  8468. spin_unlock(&fs_info->delalloc_root_lock);
  8469. ret = 0;
  8470. out:
  8471. if (!list_empty(&splice)) {
  8472. spin_lock(&fs_info->delalloc_root_lock);
  8473. list_splice_tail(&splice, &fs_info->delalloc_roots);
  8474. spin_unlock(&fs_info->delalloc_root_lock);
  8475. }
  8476. mutex_unlock(&fs_info->delalloc_root_mutex);
  8477. return ret;
  8478. }
  8479. static int btrfs_symlink(struct inode *dir, struct dentry *dentry,
  8480. const char *symname)
  8481. {
  8482. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  8483. struct btrfs_trans_handle *trans;
  8484. struct btrfs_root *root = BTRFS_I(dir)->root;
  8485. struct btrfs_path *path;
  8486. struct btrfs_key key;
  8487. struct inode *inode = NULL;
  8488. int err;
  8489. u64 objectid;
  8490. u64 index = 0;
  8491. int name_len;
  8492. int datasize;
  8493. unsigned long ptr;
  8494. struct btrfs_file_extent_item *ei;
  8495. struct extent_buffer *leaf;
  8496. name_len = strlen(symname);
  8497. if (name_len > BTRFS_MAX_INLINE_DATA_SIZE(fs_info))
  8498. return -ENAMETOOLONG;
  8499. /*
  8500. * 2 items for inode item and ref
  8501. * 2 items for dir items
  8502. * 1 item for updating parent inode item
  8503. * 1 item for the inline extent item
  8504. * 1 item for xattr if selinux is on
  8505. */
  8506. trans = btrfs_start_transaction(root, 7);
  8507. if (IS_ERR(trans))
  8508. return PTR_ERR(trans);
  8509. err = btrfs_find_free_ino(root, &objectid);
  8510. if (err)
  8511. goto out_unlock;
  8512. inode = btrfs_new_inode(trans, root, dir, dentry->d_name.name,
  8513. dentry->d_name.len, btrfs_ino(BTRFS_I(dir)),
  8514. objectid, S_IFLNK|S_IRWXUGO, &index);
  8515. if (IS_ERR(inode)) {
  8516. err = PTR_ERR(inode);
  8517. inode = NULL;
  8518. goto out_unlock;
  8519. }
  8520. /*
  8521. * If the active LSM wants to access the inode during
  8522. * d_instantiate it needs these. Smack checks to see
  8523. * if the filesystem supports xattrs by looking at the
  8524. * ops vector.
  8525. */
  8526. inode->i_fop = &btrfs_file_operations;
  8527. inode->i_op = &btrfs_file_inode_operations;
  8528. inode->i_mapping->a_ops = &btrfs_aops;
  8529. err = btrfs_init_inode_security(trans, inode, dir, &dentry->d_name);
  8530. if (err)
  8531. goto out_unlock;
  8532. path = btrfs_alloc_path();
  8533. if (!path) {
  8534. err = -ENOMEM;
  8535. goto out_unlock;
  8536. }
  8537. key.objectid = btrfs_ino(BTRFS_I(inode));
  8538. key.offset = 0;
  8539. key.type = BTRFS_EXTENT_DATA_KEY;
  8540. datasize = btrfs_file_extent_calc_inline_size(name_len);
  8541. err = btrfs_insert_empty_item(trans, root, path, &key,
  8542. datasize);
  8543. if (err) {
  8544. btrfs_free_path(path);
  8545. goto out_unlock;
  8546. }
  8547. leaf = path->nodes[0];
  8548. ei = btrfs_item_ptr(leaf, path->slots[0],
  8549. struct btrfs_file_extent_item);
  8550. btrfs_set_file_extent_generation(leaf, ei, trans->transid);
  8551. btrfs_set_file_extent_type(leaf, ei,
  8552. BTRFS_FILE_EXTENT_INLINE);
  8553. btrfs_set_file_extent_encryption(leaf, ei, 0);
  8554. btrfs_set_file_extent_compression(leaf, ei, 0);
  8555. btrfs_set_file_extent_other_encoding(leaf, ei, 0);
  8556. btrfs_set_file_extent_ram_bytes(leaf, ei, name_len);
  8557. ptr = btrfs_file_extent_inline_start(ei);
  8558. write_extent_buffer(leaf, symname, ptr, name_len);
  8559. btrfs_mark_buffer_dirty(leaf);
  8560. btrfs_free_path(path);
  8561. inode->i_op = &btrfs_symlink_inode_operations;
  8562. inode_nohighmem(inode);
  8563. inode_set_bytes(inode, name_len);
  8564. btrfs_i_size_write(BTRFS_I(inode), name_len);
  8565. err = btrfs_update_inode(trans, root, inode);
  8566. /*
  8567. * Last step, add directory indexes for our symlink inode. This is the
  8568. * last step to avoid extra cleanup of these indexes if an error happens
  8569. * elsewhere above.
  8570. */
  8571. if (!err)
  8572. err = btrfs_add_nondir(trans, BTRFS_I(dir), dentry,
  8573. BTRFS_I(inode), 0, index);
  8574. if (err)
  8575. goto out_unlock;
  8576. d_instantiate_new(dentry, inode);
  8577. out_unlock:
  8578. btrfs_end_transaction(trans);
  8579. if (err && inode) {
  8580. inode_dec_link_count(inode);
  8581. discard_new_inode(inode);
  8582. }
  8583. btrfs_btree_balance_dirty(fs_info);
  8584. return err;
  8585. }
  8586. static struct btrfs_trans_handle *insert_prealloc_file_extent(
  8587. struct btrfs_trans_handle *trans_in,
  8588. struct inode *inode, struct btrfs_key *ins,
  8589. u64 file_offset)
  8590. {
  8591. struct btrfs_file_extent_item stack_fi;
  8592. struct btrfs_replace_extent_info extent_info;
  8593. struct btrfs_trans_handle *trans = trans_in;
  8594. struct btrfs_path *path;
  8595. u64 start = ins->objectid;
  8596. u64 len = ins->offset;
  8597. int ret;
  8598. memset(&stack_fi, 0, sizeof(stack_fi));
  8599. btrfs_set_stack_file_extent_type(&stack_fi, BTRFS_FILE_EXTENT_PREALLOC);
  8600. btrfs_set_stack_file_extent_disk_bytenr(&stack_fi, start);
  8601. btrfs_set_stack_file_extent_disk_num_bytes(&stack_fi, len);
  8602. btrfs_set_stack_file_extent_num_bytes(&stack_fi, len);
  8603. btrfs_set_stack_file_extent_ram_bytes(&stack_fi, len);
  8604. btrfs_set_stack_file_extent_compression(&stack_fi, BTRFS_COMPRESS_NONE);
  8605. /* Encryption and other encoding is reserved and all 0 */
  8606. ret = btrfs_qgroup_release_data(BTRFS_I(inode), file_offset, len);
  8607. if (ret < 0)
  8608. return ERR_PTR(ret);
  8609. if (trans) {
  8610. ret = insert_reserved_file_extent(trans, BTRFS_I(inode),
  8611. file_offset, &stack_fi, ret);
  8612. if (ret)
  8613. return ERR_PTR(ret);
  8614. return trans;
  8615. }
  8616. extent_info.disk_offset = start;
  8617. extent_info.disk_len = len;
  8618. extent_info.data_offset = 0;
  8619. extent_info.data_len = len;
  8620. extent_info.file_offset = file_offset;
  8621. extent_info.extent_buf = (char *)&stack_fi;
  8622. extent_info.is_new_extent = true;
  8623. extent_info.qgroup_reserved = ret;
  8624. extent_info.insertions = 0;
  8625. path = btrfs_alloc_path();
  8626. if (!path)
  8627. return ERR_PTR(-ENOMEM);
  8628. ret = btrfs_replace_file_extents(inode, path, file_offset,
  8629. file_offset + len - 1, &extent_info,
  8630. &trans);
  8631. btrfs_free_path(path);
  8632. if (ret)
  8633. return ERR_PTR(ret);
  8634. return trans;
  8635. }
  8636. static int __btrfs_prealloc_file_range(struct inode *inode, int mode,
  8637. u64 start, u64 num_bytes, u64 min_size,
  8638. loff_t actual_len, u64 *alloc_hint,
  8639. struct btrfs_trans_handle *trans)
  8640. {
  8641. struct btrfs_fs_info *fs_info = btrfs_sb(inode->i_sb);
  8642. struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
  8643. struct extent_map *em;
  8644. struct btrfs_root *root = BTRFS_I(inode)->root;
  8645. struct btrfs_key ins;
  8646. u64 cur_offset = start;
  8647. u64 clear_offset = start;
  8648. u64 i_size;
  8649. u64 cur_bytes;
  8650. u64 last_alloc = (u64)-1;
  8651. int ret = 0;
  8652. bool own_trans = true;
  8653. u64 end = start + num_bytes - 1;
  8654. if (trans)
  8655. own_trans = false;
  8656. while (num_bytes > 0) {
  8657. cur_bytes = min_t(u64, num_bytes, SZ_256M);
  8658. cur_bytes = max(cur_bytes, min_size);
  8659. /*
  8660. * If we are severely fragmented we could end up with really
  8661. * small allocations, so if the allocator is returning small
  8662. * chunks lets make its job easier by only searching for those
  8663. * sized chunks.
  8664. */
  8665. cur_bytes = min(cur_bytes, last_alloc);
  8666. ret = btrfs_reserve_extent(root, cur_bytes, cur_bytes,
  8667. min_size, 0, *alloc_hint, &ins, 1, 0);
  8668. if (ret)
  8669. break;
  8670. /*
  8671. * We've reserved this space, and thus converted it from
  8672. * ->bytes_may_use to ->bytes_reserved. Any error that happens
  8673. * from here on out we will only need to clear our reservation
  8674. * for the remaining unreserved area, so advance our
  8675. * clear_offset by our extent size.
  8676. */
  8677. clear_offset += ins.offset;
  8678. last_alloc = ins.offset;
  8679. trans = insert_prealloc_file_extent(trans, inode, &ins, cur_offset);
  8680. /*
  8681. * Now that we inserted the prealloc extent we can finally
  8682. * decrement the number of reservations in the block group.
  8683. * If we did it before, we could race with relocation and have
  8684. * relocation miss the reserved extent, making it fail later.
  8685. */
  8686. btrfs_dec_block_group_reservations(fs_info, ins.objectid);
  8687. if (IS_ERR(trans)) {
  8688. ret = PTR_ERR(trans);
  8689. btrfs_free_reserved_extent(fs_info, ins.objectid,
  8690. ins.offset, 0);
  8691. break;
  8692. }
  8693. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  8694. cur_offset + ins.offset -1, 0);
  8695. em = alloc_extent_map();
  8696. if (!em) {
  8697. set_bit(BTRFS_INODE_NEEDS_FULL_SYNC,
  8698. &BTRFS_I(inode)->runtime_flags);
  8699. goto next;
  8700. }
  8701. em->start = cur_offset;
  8702. em->orig_start = cur_offset;
  8703. em->len = ins.offset;
  8704. em->block_start = ins.objectid;
  8705. em->block_len = ins.offset;
  8706. em->orig_block_len = ins.offset;
  8707. em->ram_bytes = ins.offset;
  8708. set_bit(EXTENT_FLAG_PREALLOC, &em->flags);
  8709. em->generation = trans->transid;
  8710. while (1) {
  8711. write_lock(&em_tree->lock);
  8712. ret = add_extent_mapping(em_tree, em, 1);
  8713. write_unlock(&em_tree->lock);
  8714. if (ret != -EEXIST)
  8715. break;
  8716. btrfs_drop_extent_cache(BTRFS_I(inode), cur_offset,
  8717. cur_offset + ins.offset - 1,
  8718. 0);
  8719. }
  8720. free_extent_map(em);
  8721. next:
  8722. num_bytes -= ins.offset;
  8723. cur_offset += ins.offset;
  8724. *alloc_hint = ins.objectid + ins.offset;
  8725. inode_inc_iversion(inode);
  8726. inode->i_ctime = current_time(inode);
  8727. BTRFS_I(inode)->flags |= BTRFS_INODE_PREALLOC;
  8728. if (!(mode & FALLOC_FL_KEEP_SIZE) &&
  8729. (actual_len > inode->i_size) &&
  8730. (cur_offset > inode->i_size)) {
  8731. if (cur_offset > actual_len)
  8732. i_size = actual_len;
  8733. else
  8734. i_size = cur_offset;
  8735. i_size_write(inode, i_size);
  8736. btrfs_inode_safe_disk_i_size_write(inode, 0);
  8737. }
  8738. ret = btrfs_update_inode(trans, root, inode);
  8739. if (ret) {
  8740. btrfs_abort_transaction(trans, ret);
  8741. if (own_trans)
  8742. btrfs_end_transaction(trans);
  8743. break;
  8744. }
  8745. if (own_trans) {
  8746. btrfs_end_transaction(trans);
  8747. trans = NULL;
  8748. }
  8749. }
  8750. if (clear_offset < end)
  8751. btrfs_free_reserved_data_space(BTRFS_I(inode), NULL, clear_offset,
  8752. end - clear_offset + 1);
  8753. return ret;
  8754. }
  8755. int btrfs_prealloc_file_range(struct inode *inode, int mode,
  8756. u64 start, u64 num_bytes, u64 min_size,
  8757. loff_t actual_len, u64 *alloc_hint)
  8758. {
  8759. return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
  8760. min_size, actual_len, alloc_hint,
  8761. NULL);
  8762. }
  8763. int btrfs_prealloc_file_range_trans(struct inode *inode,
  8764. struct btrfs_trans_handle *trans, int mode,
  8765. u64 start, u64 num_bytes, u64 min_size,
  8766. loff_t actual_len, u64 *alloc_hint)
  8767. {
  8768. return __btrfs_prealloc_file_range(inode, mode, start, num_bytes,
  8769. min_size, actual_len, alloc_hint, trans);
  8770. }
  8771. static int btrfs_set_page_dirty(struct page *page)
  8772. {
  8773. return __set_page_dirty_nobuffers(page);
  8774. }
  8775. static int btrfs_permission(struct inode *inode, int mask)
  8776. {
  8777. struct btrfs_root *root = BTRFS_I(inode)->root;
  8778. umode_t mode = inode->i_mode;
  8779. if (mask & MAY_WRITE &&
  8780. (S_ISREG(mode) || S_ISDIR(mode) || S_ISLNK(mode))) {
  8781. if (btrfs_root_readonly(root))
  8782. return -EROFS;
  8783. if (BTRFS_I(inode)->flags & BTRFS_INODE_READONLY)
  8784. return -EACCES;
  8785. }
  8786. return generic_permission(inode, mask);
  8787. }
  8788. static int btrfs_tmpfile(struct inode *dir, struct dentry *dentry, umode_t mode)
  8789. {
  8790. struct btrfs_fs_info *fs_info = btrfs_sb(dir->i_sb);
  8791. struct btrfs_trans_handle *trans;
  8792. struct btrfs_root *root = BTRFS_I(dir)->root;
  8793. struct inode *inode = NULL;
  8794. u64 objectid;
  8795. u64 index;
  8796. int ret = 0;
  8797. /*
  8798. * 5 units required for adding orphan entry
  8799. */
  8800. trans = btrfs_start_transaction(root, 5);
  8801. if (IS_ERR(trans))
  8802. return PTR_ERR(trans);
  8803. ret = btrfs_find_free_ino(root, &objectid);
  8804. if (ret)
  8805. goto out;
  8806. inode = btrfs_new_inode(trans, root, dir, NULL, 0,
  8807. btrfs_ino(BTRFS_I(dir)), objectid, mode, &index);
  8808. if (IS_ERR(inode)) {
  8809. ret = PTR_ERR(inode);
  8810. inode = NULL;
  8811. goto out;
  8812. }
  8813. inode->i_fop = &btrfs_file_operations;
  8814. inode->i_op = &btrfs_file_inode_operations;
  8815. inode->i_mapping->a_ops = &btrfs_aops;
  8816. ret = btrfs_init_inode_security(trans, inode, dir, NULL);
  8817. if (ret)
  8818. goto out;
  8819. ret = btrfs_update_inode(trans, root, inode);
  8820. if (ret)
  8821. goto out;
  8822. ret = btrfs_orphan_add(trans, BTRFS_I(inode));
  8823. if (ret)
  8824. goto out;
  8825. /*
  8826. * We set number of links to 0 in btrfs_new_inode(), and here we set
  8827. * it to 1 because d_tmpfile() will issue a warning if the count is 0,
  8828. * through:
  8829. *
  8830. * d_tmpfile() -> inode_dec_link_count() -> drop_nlink()
  8831. */
  8832. set_nlink(inode, 1);
  8833. d_tmpfile(dentry, inode);
  8834. unlock_new_inode(inode);
  8835. mark_inode_dirty(inode);
  8836. out:
  8837. btrfs_end_transaction(trans);
  8838. if (ret && inode)
  8839. discard_new_inode(inode);
  8840. btrfs_btree_balance_dirty(fs_info);
  8841. return ret;
  8842. }
  8843. void btrfs_set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
  8844. {
  8845. struct inode *inode = tree->private_data;
  8846. unsigned long index = start >> PAGE_SHIFT;
  8847. unsigned long end_index = end >> PAGE_SHIFT;
  8848. struct page *page;
  8849. while (index <= end_index) {
  8850. page = find_get_page(inode->i_mapping, index);
  8851. ASSERT(page); /* Pages should be in the extent_io_tree */
  8852. set_page_writeback(page);
  8853. put_page(page);
  8854. index++;
  8855. }
  8856. }
  8857. #ifdef CONFIG_SWAP
  8858. /*
  8859. * Add an entry indicating a block group or device which is pinned by a
  8860. * swapfile. Returns 0 on success, 1 if there is already an entry for it, or a
  8861. * negative errno on failure.
  8862. */
  8863. static int btrfs_add_swapfile_pin(struct inode *inode, void *ptr,
  8864. bool is_block_group)
  8865. {
  8866. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  8867. struct btrfs_swapfile_pin *sp, *entry;
  8868. struct rb_node **p;
  8869. struct rb_node *parent = NULL;
  8870. sp = kmalloc(sizeof(*sp), GFP_NOFS);
  8871. if (!sp)
  8872. return -ENOMEM;
  8873. sp->ptr = ptr;
  8874. sp->inode = inode;
  8875. sp->is_block_group = is_block_group;
  8876. sp->bg_extent_count = 1;
  8877. spin_lock(&fs_info->swapfile_pins_lock);
  8878. p = &fs_info->swapfile_pins.rb_node;
  8879. while (*p) {
  8880. parent = *p;
  8881. entry = rb_entry(parent, struct btrfs_swapfile_pin, node);
  8882. if (sp->ptr < entry->ptr ||
  8883. (sp->ptr == entry->ptr && sp->inode < entry->inode)) {
  8884. p = &(*p)->rb_left;
  8885. } else if (sp->ptr > entry->ptr ||
  8886. (sp->ptr == entry->ptr && sp->inode > entry->inode)) {
  8887. p = &(*p)->rb_right;
  8888. } else {
  8889. if (is_block_group)
  8890. entry->bg_extent_count++;
  8891. spin_unlock(&fs_info->swapfile_pins_lock);
  8892. kfree(sp);
  8893. return 1;
  8894. }
  8895. }
  8896. rb_link_node(&sp->node, parent, p);
  8897. rb_insert_color(&sp->node, &fs_info->swapfile_pins);
  8898. spin_unlock(&fs_info->swapfile_pins_lock);
  8899. return 0;
  8900. }
  8901. /* Free all of the entries pinned by this swapfile. */
  8902. static void btrfs_free_swapfile_pins(struct inode *inode)
  8903. {
  8904. struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
  8905. struct btrfs_swapfile_pin *sp;
  8906. struct rb_node *node, *next;
  8907. spin_lock(&fs_info->swapfile_pins_lock);
  8908. node = rb_first(&fs_info->swapfile_pins);
  8909. while (node) {
  8910. next = rb_next(node);
  8911. sp = rb_entry(node, struct btrfs_swapfile_pin, node);
  8912. if (sp->inode == inode) {
  8913. rb_erase(&sp->node, &fs_info->swapfile_pins);
  8914. if (sp->is_block_group) {
  8915. btrfs_dec_block_group_swap_extents(sp->ptr,
  8916. sp->bg_extent_count);
  8917. btrfs_put_block_group(sp->ptr);
  8918. }
  8919. kfree(sp);
  8920. }
  8921. node = next;
  8922. }
  8923. spin_unlock(&fs_info->swapfile_pins_lock);
  8924. }
  8925. struct btrfs_swap_info {
  8926. u64 start;
  8927. u64 block_start;
  8928. u64 block_len;
  8929. u64 lowest_ppage;
  8930. u64 highest_ppage;
  8931. unsigned long nr_pages;
  8932. int nr_extents;
  8933. };
  8934. static int btrfs_add_swap_extent(struct swap_info_struct *sis,
  8935. struct btrfs_swap_info *bsi)
  8936. {
  8937. unsigned long nr_pages;
  8938. unsigned long max_pages;
  8939. u64 first_ppage, first_ppage_reported, next_ppage;
  8940. int ret;
  8941. /*
  8942. * Our swapfile may have had its size extended after the swap header was
  8943. * written. In that case activating the swapfile should not go beyond
  8944. * the max size set in the swap header.
  8945. */
  8946. if (bsi->nr_pages >= sis->max)
  8947. return 0;
  8948. max_pages = sis->max - bsi->nr_pages;
  8949. first_ppage = ALIGN(bsi->block_start, PAGE_SIZE) >> PAGE_SHIFT;
  8950. next_ppage = ALIGN_DOWN(bsi->block_start + bsi->block_len,
  8951. PAGE_SIZE) >> PAGE_SHIFT;
  8952. if (first_ppage >= next_ppage)
  8953. return 0;
  8954. nr_pages = next_ppage - first_ppage;
  8955. nr_pages = min(nr_pages, max_pages);
  8956. first_ppage_reported = first_ppage;
  8957. if (bsi->start == 0)
  8958. first_ppage_reported++;
  8959. if (bsi->lowest_ppage > first_ppage_reported)
  8960. bsi->lowest_ppage = first_ppage_reported;
  8961. if (bsi->highest_ppage < (next_ppage - 1))
  8962. bsi->highest_ppage = next_ppage - 1;
  8963. ret = add_swap_extent(sis, bsi->nr_pages, nr_pages, first_ppage);
  8964. if (ret < 0)
  8965. return ret;
  8966. bsi->nr_extents += ret;
  8967. bsi->nr_pages += nr_pages;
  8968. return 0;
  8969. }
  8970. static void btrfs_swap_deactivate(struct file *file)
  8971. {
  8972. struct inode *inode = file_inode(file);
  8973. btrfs_free_swapfile_pins(inode);
  8974. atomic_dec(&BTRFS_I(inode)->root->nr_swapfiles);
  8975. }
  8976. static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
  8977. sector_t *span)
  8978. {
  8979. struct inode *inode = file_inode(file);
  8980. struct btrfs_root *root = BTRFS_I(inode)->root;
  8981. struct btrfs_fs_info *fs_info = root->fs_info;
  8982. struct extent_io_tree *io_tree = &BTRFS_I(inode)->io_tree;
  8983. struct extent_state *cached_state = NULL;
  8984. struct extent_map *em = NULL;
  8985. struct btrfs_device *device = NULL;
  8986. struct btrfs_swap_info bsi = {
  8987. .lowest_ppage = (sector_t)-1ULL,
  8988. };
  8989. int ret = 0;
  8990. u64 isize;
  8991. u64 start;
  8992. /*
  8993. * If the swap file was just created, make sure delalloc is done. If the
  8994. * file changes again after this, the user is doing something stupid and
  8995. * we don't really care.
  8996. */
  8997. ret = btrfs_wait_ordered_range(inode, 0, (u64)-1);
  8998. if (ret)
  8999. return ret;
  9000. /*
  9001. * The inode is locked, so these flags won't change after we check them.
  9002. */
  9003. if (BTRFS_I(inode)->flags & BTRFS_INODE_COMPRESS) {
  9004. btrfs_warn(fs_info, "swapfile must not be compressed");
  9005. return -EINVAL;
  9006. }
  9007. if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATACOW)) {
  9008. btrfs_warn(fs_info, "swapfile must not be copy-on-write");
  9009. return -EINVAL;
  9010. }
  9011. if (!(BTRFS_I(inode)->flags & BTRFS_INODE_NODATASUM)) {
  9012. btrfs_warn(fs_info, "swapfile must not be checksummed");
  9013. return -EINVAL;
  9014. }
  9015. /*
  9016. * Balance or device remove/replace/resize can move stuff around from
  9017. * under us. The exclop protection makes sure they aren't running/won't
  9018. * run concurrently while we are mapping the swap extents, and
  9019. * fs_info->swapfile_pins prevents them from running while the swap
  9020. * file is active and moving the extents. Note that this also prevents
  9021. * a concurrent device add which isn't actually necessary, but it's not
  9022. * really worth the trouble to allow it.
  9023. */
  9024. if (!btrfs_exclop_start(fs_info, BTRFS_EXCLOP_SWAP_ACTIVATE)) {
  9025. btrfs_warn(fs_info,
  9026. "cannot activate swapfile while exclusive operation is running");
  9027. return -EBUSY;
  9028. }
  9029. /*
  9030. * Prevent snapshot creation while we are activating the swap file.
  9031. * We do not want to race with snapshot creation. If snapshot creation
  9032. * already started before we bumped nr_swapfiles from 0 to 1 and
  9033. * completes before the first write into the swap file after it is
  9034. * activated, than that write would fallback to COW.
  9035. */
  9036. if (!btrfs_drew_try_write_lock(&root->snapshot_lock)) {
  9037. btrfs_exclop_finish(fs_info);
  9038. btrfs_warn(fs_info,
  9039. "cannot activate swapfile because snapshot creation is in progress");
  9040. return -EINVAL;
  9041. }
  9042. /*
  9043. * Snapshots can create extents which require COW even if NODATACOW is
  9044. * set. We use this counter to prevent snapshots. We must increment it
  9045. * before walking the extents because we don't want a concurrent
  9046. * snapshot to run after we've already checked the extents.
  9047. *
  9048. * It is possible that subvolume is marked for deletion but still not
  9049. * removed yet. To prevent this race, we check the root status before
  9050. * activating the swapfile.
  9051. */
  9052. spin_lock(&root->root_item_lock);
  9053. if (btrfs_root_dead(root)) {
  9054. spin_unlock(&root->root_item_lock);
  9055. btrfs_exclop_finish(fs_info);
  9056. btrfs_warn(fs_info,
  9057. "cannot activate swapfile because subvolume %llu is being deleted",
  9058. root->root_key.objectid);
  9059. return -EPERM;
  9060. }
  9061. atomic_inc(&root->nr_swapfiles);
  9062. spin_unlock(&root->root_item_lock);
  9063. isize = ALIGN_DOWN(inode->i_size, fs_info->sectorsize);
  9064. lock_extent_bits(io_tree, 0, isize - 1, &cached_state);
  9065. start = 0;
  9066. while (start < isize) {
  9067. u64 logical_block_start, physical_block_start;
  9068. struct btrfs_block_group *bg;
  9069. u64 len = isize - start;
  9070. em = btrfs_get_extent(BTRFS_I(inode), NULL, 0, start, len);
  9071. if (IS_ERR(em)) {
  9072. ret = PTR_ERR(em);
  9073. goto out;
  9074. }
  9075. if (em->block_start == EXTENT_MAP_HOLE) {
  9076. btrfs_warn(fs_info, "swapfile must not have holes");
  9077. ret = -EINVAL;
  9078. goto out;
  9079. }
  9080. if (em->block_start == EXTENT_MAP_INLINE) {
  9081. /*
  9082. * It's unlikely we'll ever actually find ourselves
  9083. * here, as a file small enough to fit inline won't be
  9084. * big enough to store more than the swap header, but in
  9085. * case something changes in the future, let's catch it
  9086. * here rather than later.
  9087. */
  9088. btrfs_warn(fs_info, "swapfile must not be inline");
  9089. ret = -EINVAL;
  9090. goto out;
  9091. }
  9092. if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
  9093. btrfs_warn(fs_info, "swapfile must not be compressed");
  9094. ret = -EINVAL;
  9095. goto out;
  9096. }
  9097. logical_block_start = em->block_start + (start - em->start);
  9098. len = min(len, em->len - (start - em->start));
  9099. free_extent_map(em);
  9100. em = NULL;
  9101. ret = can_nocow_extent(inode, start, &len, NULL, NULL, NULL, true);
  9102. if (ret < 0) {
  9103. goto out;
  9104. } else if (ret) {
  9105. ret = 0;
  9106. } else {
  9107. btrfs_warn(fs_info,
  9108. "swapfile must not be copy-on-write");
  9109. ret = -EINVAL;
  9110. goto out;
  9111. }
  9112. em = btrfs_get_chunk_map(fs_info, logical_block_start, len);
  9113. if (IS_ERR(em)) {
  9114. ret = PTR_ERR(em);
  9115. goto out;
  9116. }
  9117. if (em->map_lookup->type & BTRFS_BLOCK_GROUP_PROFILE_MASK) {
  9118. btrfs_warn(fs_info,
  9119. "swapfile must have single data profile");
  9120. ret = -EINVAL;
  9121. goto out;
  9122. }
  9123. if (device == NULL) {
  9124. device = em->map_lookup->stripes[0].dev;
  9125. ret = btrfs_add_swapfile_pin(inode, device, false);
  9126. if (ret == 1)
  9127. ret = 0;
  9128. else if (ret)
  9129. goto out;
  9130. } else if (device != em->map_lookup->stripes[0].dev) {
  9131. btrfs_warn(fs_info, "swapfile must be on one device");
  9132. ret = -EINVAL;
  9133. goto out;
  9134. }
  9135. physical_block_start = (em->map_lookup->stripes[0].physical +
  9136. (logical_block_start - em->start));
  9137. len = min(len, em->len - (logical_block_start - em->start));
  9138. free_extent_map(em);
  9139. em = NULL;
  9140. bg = btrfs_lookup_block_group(fs_info, logical_block_start);
  9141. if (!bg) {
  9142. btrfs_warn(fs_info,
  9143. "could not find block group containing swapfile");
  9144. ret = -EINVAL;
  9145. goto out;
  9146. }
  9147. if (!btrfs_inc_block_group_swap_extents(bg)) {
  9148. btrfs_warn(fs_info,
  9149. "block group for swapfile at %llu is read-only%s",
  9150. bg->start,
  9151. atomic_read(&fs_info->scrubs_running) ?
  9152. " (scrub running)" : "");
  9153. btrfs_put_block_group(bg);
  9154. ret = -EINVAL;
  9155. goto out;
  9156. }
  9157. ret = btrfs_add_swapfile_pin(inode, bg, true);
  9158. if (ret) {
  9159. btrfs_put_block_group(bg);
  9160. if (ret == 1)
  9161. ret = 0;
  9162. else
  9163. goto out;
  9164. }
  9165. if (bsi.block_len &&
  9166. bsi.block_start + bsi.block_len == physical_block_start) {
  9167. bsi.block_len += len;
  9168. } else {
  9169. if (bsi.block_len) {
  9170. ret = btrfs_add_swap_extent(sis, &bsi);
  9171. if (ret)
  9172. goto out;
  9173. }
  9174. bsi.start = start;
  9175. bsi.block_start = physical_block_start;
  9176. bsi.block_len = len;
  9177. }
  9178. start += len;
  9179. }
  9180. if (bsi.block_len)
  9181. ret = btrfs_add_swap_extent(sis, &bsi);
  9182. out:
  9183. if (!IS_ERR_OR_NULL(em))
  9184. free_extent_map(em);
  9185. unlock_extent_cached(io_tree, 0, isize - 1, &cached_state);
  9186. if (ret)
  9187. btrfs_swap_deactivate(file);
  9188. btrfs_drew_write_unlock(&root->snapshot_lock);
  9189. btrfs_exclop_finish(fs_info);
  9190. if (ret)
  9191. return ret;
  9192. if (device)
  9193. sis->bdev = device->bdev;
  9194. *span = bsi.highest_ppage - bsi.lowest_ppage + 1;
  9195. sis->max = bsi.nr_pages;
  9196. sis->pages = bsi.nr_pages - 1;
  9197. sis->highest_bit = bsi.nr_pages - 1;
  9198. return bsi.nr_extents;
  9199. }
  9200. #else
  9201. static void btrfs_swap_deactivate(struct file *file)
  9202. {
  9203. }
  9204. static int btrfs_swap_activate(struct swap_info_struct *sis, struct file *file,
  9205. sector_t *span)
  9206. {
  9207. return -EOPNOTSUPP;
  9208. }
  9209. #endif
  9210. static const struct inode_operations btrfs_dir_inode_operations = {
  9211. .getattr = btrfs_getattr,
  9212. .lookup = btrfs_lookup,
  9213. .create = btrfs_create,
  9214. .unlink = btrfs_unlink,
  9215. .link = btrfs_link,
  9216. .mkdir = btrfs_mkdir,
  9217. .rmdir = btrfs_rmdir,
  9218. .rename = btrfs_rename2,
  9219. .symlink = btrfs_symlink,
  9220. .setattr = btrfs_setattr,
  9221. .mknod = btrfs_mknod,
  9222. .listxattr = btrfs_listxattr,
  9223. .permission = btrfs_permission,
  9224. .get_acl = btrfs_get_acl,
  9225. .set_acl = btrfs_set_acl,
  9226. .update_time = btrfs_update_time,
  9227. .tmpfile = btrfs_tmpfile,
  9228. };
  9229. static const struct file_operations btrfs_dir_file_operations = {
  9230. .llseek = generic_file_llseek,
  9231. .read = generic_read_dir,
  9232. .iterate_shared = btrfs_real_readdir,
  9233. .open = btrfs_opendir,
  9234. .unlocked_ioctl = btrfs_ioctl,
  9235. #ifdef CONFIG_COMPAT
  9236. .compat_ioctl = btrfs_compat_ioctl,
  9237. #endif
  9238. .release = btrfs_release_file,
  9239. .fsync = btrfs_sync_file,
  9240. };
  9241. /*
  9242. * btrfs doesn't support the bmap operation because swapfiles
  9243. * use bmap to make a mapping of extents in the file. They assume
  9244. * these extents won't change over the life of the file and they
  9245. * use the bmap result to do IO directly to the drive.
  9246. *
  9247. * the btrfs bmap call would return logical addresses that aren't
  9248. * suitable for IO and they also will change frequently as COW
  9249. * operations happen. So, swapfile + btrfs == corruption.
  9250. *
  9251. * For now we're avoiding this by dropping bmap.
  9252. */
  9253. static const struct address_space_operations btrfs_aops = {
  9254. .readpage = btrfs_readpage,
  9255. .writepage = btrfs_writepage,
  9256. .writepages = btrfs_writepages,
  9257. .readahead = btrfs_readahead,
  9258. .direct_IO = noop_direct_IO,
  9259. .invalidatepage = btrfs_invalidatepage,
  9260. .releasepage = btrfs_releasepage,
  9261. #ifdef CONFIG_MIGRATION
  9262. .migratepage = btrfs_migratepage,
  9263. #endif
  9264. .set_page_dirty = btrfs_set_page_dirty,
  9265. .error_remove_page = generic_error_remove_page,
  9266. .swap_activate = btrfs_swap_activate,
  9267. .swap_deactivate = btrfs_swap_deactivate,
  9268. };
  9269. static const struct inode_operations btrfs_file_inode_operations = {
  9270. .getattr = btrfs_getattr,
  9271. .setattr = btrfs_setattr,
  9272. .listxattr = btrfs_listxattr,
  9273. .permission = btrfs_permission,
  9274. .fiemap = btrfs_fiemap,
  9275. .get_acl = btrfs_get_acl,
  9276. .set_acl = btrfs_set_acl,
  9277. .update_time = btrfs_update_time,
  9278. };
  9279. static const struct inode_operations btrfs_special_inode_operations = {
  9280. .getattr = btrfs_getattr,
  9281. .setattr = btrfs_setattr,
  9282. .permission = btrfs_permission,
  9283. .listxattr = btrfs_listxattr,
  9284. .get_acl = btrfs_get_acl,
  9285. .set_acl = btrfs_set_acl,
  9286. .update_time = btrfs_update_time,
  9287. };
  9288. static const struct inode_operations btrfs_symlink_inode_operations = {
  9289. .get_link = page_get_link,
  9290. .getattr = btrfs_getattr,
  9291. .setattr = btrfs_setattr,
  9292. .permission = btrfs_permission,
  9293. .listxattr = btrfs_listxattr,
  9294. .update_time = btrfs_update_time,
  9295. };
  9296. const struct dentry_operations btrfs_dentry_operations = {
  9297. .d_delete = btrfs_dentry_delete,
  9298. };