segment.c 137 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229323032313232323332343235323632373238323932403241324232433244324532463247324832493250325132523253325432553256325732583259326032613262326332643265326632673268326932703271327232733274327532763277327832793280328132823283328432853286328732883289329032913292329332943295329632973298329933003301330233033304330533063307330833093310331133123313331433153316331733183319332033213322332333243325332633273328332933303331333233333334333533363337333833393340334133423343334433453346334733483349335033513352335333543355335633573358335933603361336233633364336533663367336833693370337133723373337433753376337733783379338033813382338333843385338633873388338933903391339233933394339533963397339833993400340134023403340434053406340734083409341034113412341334143415341634173418341934203421342234233424342534263427342834293430343134323433343434353436343734383439344034413442344334443445344634473448344934503451345234533454345534563457345834593460346134623463346434653466346734683469347034713472347334743475347634773478347934803481348234833484348534863487348834893490349134923493349434953496349734983499350035013502350335043505350635073508350935103511351235133514351535163517351835193520352135223523352435253526352735283529353035313532353335343535353635373538353935403541354235433544354535463547354835493550355135523553355435553556355735583559356035613562356335643565356635673568356935703571357235733574357535763577357835793580358135823583358435853586358735883589359035913592359335943595359635973598359936003601360236033604360536063607360836093610361136123613361436153616361736183619362036213622362336243625362636273628362936303631363236333634363536363637363836393640364136423643364436453646364736483649365036513652365336543655365636573658365936603661366236633664366536663667366836693670367136723673367436753676367736783679368036813682368336843685368636873688368936903691369236933694369536963697369836993700370137023703370437053706370737083709371037113712371337143715371637173718371937203721372237233724372537263727372837293730373137323733373437353736373737383739374037413742374337443745374637473748374937503751375237533754375537563757375837593760376137623763376437653766376737683769377037713772377337743775377637773778377937803781378237833784378537863787378837893790379137923793379437953796379737983799380038013802380338043805380638073808380938103811381238133814381538163817381838193820382138223823382438253826382738283829383038313832383338343835383638373838383938403841384238433844384538463847384838493850385138523853385438553856385738583859386038613862386338643865386638673868386938703871387238733874387538763877387838793880388138823883388438853886388738883889389038913892389338943895389638973898389939003901390239033904390539063907390839093910391139123913391439153916391739183919392039213922392339243925392639273928392939303931393239333934393539363937393839393940394139423943394439453946394739483949395039513952395339543955395639573958395939603961396239633964396539663967396839693970397139723973397439753976397739783979398039813982398339843985398639873988398939903991399239933994399539963997399839994000400140024003400440054006400740084009401040114012401340144015401640174018401940204021402240234024402540264027402840294030403140324033403440354036403740384039404040414042404340444045404640474048404940504051405240534054405540564057405840594060406140624063406440654066406740684069407040714072407340744075407640774078407940804081408240834084408540864087408840894090409140924093409440954096409740984099410041014102410341044105410641074108410941104111411241134114411541164117411841194120412141224123412441254126412741284129413041314132413341344135413641374138413941404141414241434144414541464147414841494150415141524153415441554156415741584159416041614162416341644165416641674168416941704171417241734174417541764177417841794180418141824183418441854186418741884189419041914192419341944195419641974198419942004201420242034204420542064207420842094210421142124213421442154216421742184219422042214222422342244225422642274228422942304231423242334234423542364237423842394240424142424243424442454246424742484249425042514252425342544255425642574258425942604261426242634264426542664267426842694270427142724273427442754276427742784279428042814282428342844285428642874288428942904291429242934294429542964297429842994300430143024303430443054306430743084309431043114312431343144315431643174318431943204321432243234324432543264327432843294330433143324333433443354336433743384339434043414342434343444345434643474348434943504351435243534354435543564357435843594360436143624363436443654366436743684369437043714372437343744375437643774378437943804381438243834384438543864387438843894390439143924393439443954396439743984399440044014402440344044405440644074408440944104411441244134414441544164417441844194420442144224423442444254426442744284429443044314432443344344435443644374438443944404441444244434444444544464447444844494450445144524453445444554456445744584459446044614462446344644465446644674468446944704471447244734474447544764477447844794480448144824483448444854486448744884489449044914492449344944495449644974498449945004501450245034504450545064507450845094510451145124513451445154516451745184519452045214522452345244525452645274528452945304531453245334534453545364537453845394540454145424543454445454546454745484549455045514552455345544555455645574558455945604561456245634564456545664567456845694570457145724573457445754576457745784579458045814582458345844585458645874588458945904591459245934594459545964597459845994600460146024603460446054606460746084609461046114612461346144615461646174618461946204621462246234624462546264627462846294630463146324633463446354636463746384639464046414642464346444645464646474648464946504651465246534654465546564657465846594660466146624663466446654666466746684669467046714672467346744675467646774678467946804681468246834684468546864687468846894690469146924693469446954696469746984699470047014702470347044705470647074708470947104711471247134714471547164717471847194720472147224723472447254726472747284729473047314732473347344735473647374738473947404741474247434744474547464747474847494750475147524753475447554756475747584759476047614762476347644765476647674768476947704771477247734774477547764777477847794780478147824783478447854786478747884789479047914792479347944795479647974798479948004801480248034804480548064807480848094810481148124813481448154816481748184819482048214822482348244825482648274828482948304831483248334834483548364837483848394840484148424843484448454846484748484849485048514852485348544855485648574858485948604861486248634864486548664867486848694870487148724873487448754876487748784879488048814882488348844885488648874888488948904891489248934894489548964897489848994900490149024903490449054906490749084909491049114912491349144915491649174918491949204921492249234924492549264927492849294930493149324933493449354936493749384939494049414942494349444945494649474948494949504951495249534954495549564957495849594960496149624963496449654966496749684969497049714972497349744975497649774978497949804981498249834984498549864987498849894990499149924993499449954996499749984999500050015002500350045005500650075008500950105011501250135014501550165017501850195020502150225023502450255026502750285029503050315032503350345035503650375038503950405041504250435044504550465047504850495050505150525053505450555056505750585059506050615062506350645065506650675068506950705071507250735074507550765077507850795080508150825083508450855086508750885089509050915092509350945095509650975098509951005101510251035104510551065107510851095110511151125113511451155116511751185119512051215122512351245125512651275128512951305131513251335134513551365137513851395140514151425143514451455146514751485149515051515152515351545155515651575158515951605161516251635164516551665167516851695170517151725173517451755176517751785179518051815182518351845185518651875188518951905191519251935194519551965197519851995200520152025203520452055206520752085209521052115212521352145215521652175218521952205221522252235224522552265227522852295230523152325233523452355236523752385239524052415242524352445245524652475248524952505251525252535254525552565257525852595260526152625263526452655266526752685269527052715272527352745275527652775278527952805281528252835284528552865287528852895290529152925293529452955296529752985299530053015302530353045305530653075308530953105311531253135314531553165317531853195320532153225323532453255326532753285329533053315332533353345335533653375338533953405341534253435344534553465347534853495350535153525353535453555356535753585359536053615362
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * fs/f2fs/segment.c
  4. *
  5. * Copyright (c) 2012 Samsung Electronics Co., Ltd.
  6. * http://www.samsung.com/
  7. */
  8. #include <linux/fs.h>
  9. #include <linux/f2fs_fs.h>
  10. #include <linux/bio.h>
  11. #include <linux/blkdev.h>
  12. #include <linux/prefetch.h>
  13. #include <linux/kthread.h>
  14. #include <linux/swap.h>
  15. #include <linux/timer.h>
  16. #include <linux/freezer.h>
  17. #include <linux/sched/signal.h>
  18. #include "f2fs.h"
  19. #include "segment.h"
  20. #include "node.h"
  21. #include "gc.h"
  22. #include <trace/events/f2fs.h>
  23. #define __reverse_ffz(x) __reverse_ffs(~(x))
  24. static struct kmem_cache *discard_entry_slab;
  25. static struct kmem_cache *discard_cmd_slab;
  26. static struct kmem_cache *sit_entry_set_slab;
  27. static struct kmem_cache *inmem_entry_slab;
  28. static unsigned long __reverse_ulong(unsigned char *str)
  29. {
  30. unsigned long tmp = 0;
  31. int shift = 24, idx = 0;
  32. #if BITS_PER_LONG == 64
  33. shift = 56;
  34. #endif
  35. while (shift >= 0) {
  36. tmp |= (unsigned long)str[idx++] << shift;
  37. shift -= BITS_PER_BYTE;
  38. }
  39. return tmp;
  40. }
  41. /*
  42. * __reverse_ffs is copied from include/asm-generic/bitops/__ffs.h since
  43. * MSB and LSB are reversed in a byte by f2fs_set_bit.
  44. */
  45. static inline unsigned long __reverse_ffs(unsigned long word)
  46. {
  47. int num = 0;
  48. #if BITS_PER_LONG == 64
  49. if ((word & 0xffffffff00000000UL) == 0)
  50. num += 32;
  51. else
  52. word >>= 32;
  53. #endif
  54. if ((word & 0xffff0000) == 0)
  55. num += 16;
  56. else
  57. word >>= 16;
  58. if ((word & 0xff00) == 0)
  59. num += 8;
  60. else
  61. word >>= 8;
  62. if ((word & 0xf0) == 0)
  63. num += 4;
  64. else
  65. word >>= 4;
  66. if ((word & 0xc) == 0)
  67. num += 2;
  68. else
  69. word >>= 2;
  70. if ((word & 0x2) == 0)
  71. num += 1;
  72. return num;
  73. }
  74. /*
  75. * __find_rev_next(_zero)_bit is copied from lib/find_next_bit.c because
  76. * f2fs_set_bit makes MSB and LSB reversed in a byte.
  77. * @size must be integral times of unsigned long.
  78. * Example:
  79. * MSB <--> LSB
  80. * f2fs_set_bit(0, bitmap) => 1000 0000
  81. * f2fs_set_bit(7, bitmap) => 0000 0001
  82. */
  83. static unsigned long __find_rev_next_bit(const unsigned long *addr,
  84. unsigned long size, unsigned long offset)
  85. {
  86. const unsigned long *p = addr + BIT_WORD(offset);
  87. unsigned long result = size;
  88. unsigned long tmp;
  89. if (offset >= size)
  90. return size;
  91. size -= (offset & ~(BITS_PER_LONG - 1));
  92. offset %= BITS_PER_LONG;
  93. while (1) {
  94. if (*p == 0)
  95. goto pass;
  96. tmp = __reverse_ulong((unsigned char *)p);
  97. tmp &= ~0UL >> offset;
  98. if (size < BITS_PER_LONG)
  99. tmp &= (~0UL << (BITS_PER_LONG - size));
  100. if (tmp)
  101. goto found;
  102. pass:
  103. if (size <= BITS_PER_LONG)
  104. break;
  105. size -= BITS_PER_LONG;
  106. offset = 0;
  107. p++;
  108. }
  109. return result;
  110. found:
  111. return result - size + __reverse_ffs(tmp);
  112. }
  113. static unsigned long __find_rev_next_zero_bit(const unsigned long *addr,
  114. unsigned long size, unsigned long offset)
  115. {
  116. const unsigned long *p = addr + BIT_WORD(offset);
  117. unsigned long result = size;
  118. unsigned long tmp;
  119. if (offset >= size)
  120. return size;
  121. size -= (offset & ~(BITS_PER_LONG - 1));
  122. offset %= BITS_PER_LONG;
  123. while (1) {
  124. if (*p == ~0UL)
  125. goto pass;
  126. tmp = __reverse_ulong((unsigned char *)p);
  127. if (offset)
  128. tmp |= ~0UL << (BITS_PER_LONG - offset);
  129. if (size < BITS_PER_LONG)
  130. tmp |= ~0UL >> size;
  131. if (tmp != ~0UL)
  132. goto found;
  133. pass:
  134. if (size <= BITS_PER_LONG)
  135. break;
  136. size -= BITS_PER_LONG;
  137. offset = 0;
  138. p++;
  139. }
  140. return result;
  141. found:
  142. return result - size + __reverse_ffz(tmp);
  143. }
  144. bool f2fs_need_SSR(struct f2fs_sb_info *sbi)
  145. {
  146. int node_secs = get_blocktype_secs(sbi, F2FS_DIRTY_NODES);
  147. int dent_secs = get_blocktype_secs(sbi, F2FS_DIRTY_DENTS);
  148. int imeta_secs = get_blocktype_secs(sbi, F2FS_DIRTY_IMETA);
  149. if (f2fs_lfs_mode(sbi))
  150. return false;
  151. if (sbi->gc_mode == GC_URGENT_HIGH)
  152. return true;
  153. if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
  154. return true;
  155. return free_sections(sbi) <= (node_secs + 2 * dent_secs + imeta_secs +
  156. SM_I(sbi)->min_ssr_sections + reserved_sections(sbi));
  157. }
  158. void f2fs_register_inmem_page(struct inode *inode, struct page *page)
  159. {
  160. struct inmem_pages *new;
  161. set_page_private_atomic(page);
  162. new = f2fs_kmem_cache_alloc(inmem_entry_slab, GFP_NOFS);
  163. /* add atomic page indices to the list */
  164. new->page = page;
  165. INIT_LIST_HEAD(&new->list);
  166. /* increase reference count with clean state */
  167. get_page(page);
  168. mutex_lock(&F2FS_I(inode)->inmem_lock);
  169. list_add_tail(&new->list, &F2FS_I(inode)->inmem_pages);
  170. inc_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
  171. mutex_unlock(&F2FS_I(inode)->inmem_lock);
  172. trace_f2fs_register_inmem_page(page, INMEM);
  173. }
  174. static int __revoke_inmem_pages(struct inode *inode,
  175. struct list_head *head, bool drop, bool recover,
  176. bool trylock)
  177. {
  178. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  179. struct inmem_pages *cur, *tmp;
  180. int err = 0;
  181. list_for_each_entry_safe(cur, tmp, head, list) {
  182. struct page *page = cur->page;
  183. if (drop)
  184. trace_f2fs_commit_inmem_page(page, INMEM_DROP);
  185. if (trylock) {
  186. /*
  187. * to avoid deadlock in between page lock and
  188. * inmem_lock.
  189. */
  190. if (!trylock_page(page))
  191. continue;
  192. } else {
  193. lock_page(page);
  194. }
  195. f2fs_wait_on_page_writeback(page, DATA, true, true);
  196. if (recover) {
  197. struct dnode_of_data dn;
  198. struct node_info ni;
  199. trace_f2fs_commit_inmem_page(page, INMEM_REVOKE);
  200. retry:
  201. set_new_dnode(&dn, inode, NULL, NULL, 0);
  202. err = f2fs_get_dnode_of_data(&dn, page->index,
  203. LOOKUP_NODE);
  204. if (err) {
  205. if (err == -ENOMEM) {
  206. congestion_wait(BLK_RW_ASYNC,
  207. DEFAULT_IO_TIMEOUT);
  208. cond_resched();
  209. goto retry;
  210. }
  211. err = -EAGAIN;
  212. goto next;
  213. }
  214. err = f2fs_get_node_info(sbi, dn.nid, &ni, false);
  215. if (err) {
  216. f2fs_put_dnode(&dn);
  217. return err;
  218. }
  219. if (cur->old_addr == NEW_ADDR) {
  220. f2fs_invalidate_blocks(sbi, dn.data_blkaddr);
  221. f2fs_update_data_blkaddr(&dn, NEW_ADDR);
  222. } else
  223. f2fs_replace_block(sbi, &dn, dn.data_blkaddr,
  224. cur->old_addr, ni.version, true, true);
  225. f2fs_put_dnode(&dn);
  226. }
  227. next:
  228. /* we don't need to invalidate this in the sccessful status */
  229. if (drop || recover) {
  230. ClearPageUptodate(page);
  231. clear_page_private_gcing(page);
  232. }
  233. detach_page_private(page);
  234. set_page_private(page, 0);
  235. f2fs_put_page(page, 1);
  236. list_del(&cur->list);
  237. kmem_cache_free(inmem_entry_slab, cur);
  238. dec_page_count(F2FS_I_SB(inode), F2FS_INMEM_PAGES);
  239. }
  240. return err;
  241. }
  242. void f2fs_drop_inmem_pages_all(struct f2fs_sb_info *sbi, bool gc_failure)
  243. {
  244. struct list_head *head = &sbi->inode_list[ATOMIC_FILE];
  245. struct inode *inode;
  246. struct f2fs_inode_info *fi;
  247. unsigned int count = sbi->atomic_files;
  248. unsigned int looped = 0;
  249. next:
  250. spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
  251. if (list_empty(head)) {
  252. spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
  253. return;
  254. }
  255. fi = list_first_entry(head, struct f2fs_inode_info, inmem_ilist);
  256. inode = igrab(&fi->vfs_inode);
  257. if (inode)
  258. list_move_tail(&fi->inmem_ilist, head);
  259. spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
  260. if (inode) {
  261. if (gc_failure) {
  262. if (!fi->i_gc_failures[GC_FAILURE_ATOMIC])
  263. goto skip;
  264. }
  265. set_inode_flag(inode, FI_ATOMIC_REVOKE_REQUEST);
  266. f2fs_drop_inmem_pages(inode);
  267. skip:
  268. iput(inode);
  269. }
  270. congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
  271. cond_resched();
  272. if (gc_failure) {
  273. if (++looped >= count)
  274. return;
  275. }
  276. goto next;
  277. }
  278. void f2fs_drop_inmem_pages(struct inode *inode)
  279. {
  280. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  281. struct f2fs_inode_info *fi = F2FS_I(inode);
  282. do {
  283. mutex_lock(&fi->inmem_lock);
  284. if (list_empty(&fi->inmem_pages)) {
  285. fi->i_gc_failures[GC_FAILURE_ATOMIC] = 0;
  286. spin_lock(&sbi->inode_lock[ATOMIC_FILE]);
  287. if (!list_empty(&fi->inmem_ilist))
  288. list_del_init(&fi->inmem_ilist);
  289. if (f2fs_is_atomic_file(inode)) {
  290. clear_inode_flag(inode, FI_ATOMIC_FILE);
  291. sbi->atomic_files--;
  292. }
  293. spin_unlock(&sbi->inode_lock[ATOMIC_FILE]);
  294. mutex_unlock(&fi->inmem_lock);
  295. break;
  296. }
  297. __revoke_inmem_pages(inode, &fi->inmem_pages,
  298. true, false, true);
  299. mutex_unlock(&fi->inmem_lock);
  300. } while (1);
  301. }
  302. void f2fs_drop_inmem_page(struct inode *inode, struct page *page)
  303. {
  304. struct f2fs_inode_info *fi = F2FS_I(inode);
  305. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  306. struct list_head *head = &fi->inmem_pages;
  307. struct inmem_pages *cur = NULL;
  308. f2fs_bug_on(sbi, !page_private_atomic(page));
  309. mutex_lock(&fi->inmem_lock);
  310. list_for_each_entry(cur, head, list) {
  311. if (cur->page == page)
  312. break;
  313. }
  314. f2fs_bug_on(sbi, list_empty(head) || cur->page != page);
  315. list_del(&cur->list);
  316. mutex_unlock(&fi->inmem_lock);
  317. dec_page_count(sbi, F2FS_INMEM_PAGES);
  318. kmem_cache_free(inmem_entry_slab, cur);
  319. ClearPageUptodate(page);
  320. clear_page_private_atomic(page);
  321. f2fs_put_page(page, 0);
  322. detach_page_private(page);
  323. set_page_private(page, 0);
  324. trace_f2fs_commit_inmem_page(page, INMEM_INVALIDATE);
  325. }
  326. static int __f2fs_commit_inmem_pages(struct inode *inode)
  327. {
  328. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  329. struct f2fs_inode_info *fi = F2FS_I(inode);
  330. struct inmem_pages *cur, *tmp;
  331. struct f2fs_io_info fio = {
  332. .sbi = sbi,
  333. .ino = inode->i_ino,
  334. .type = DATA,
  335. .op = REQ_OP_WRITE,
  336. .op_flags = REQ_SYNC | REQ_PRIO,
  337. .io_type = FS_DATA_IO,
  338. };
  339. struct list_head revoke_list;
  340. bool submit_bio = false;
  341. int err = 0;
  342. INIT_LIST_HEAD(&revoke_list);
  343. list_for_each_entry_safe(cur, tmp, &fi->inmem_pages, list) {
  344. struct page *page = cur->page;
  345. lock_page(page);
  346. if (page->mapping == inode->i_mapping) {
  347. trace_f2fs_commit_inmem_page(page, INMEM);
  348. f2fs_wait_on_page_writeback(page, DATA, true, true);
  349. set_page_dirty(page);
  350. if (clear_page_dirty_for_io(page)) {
  351. inode_dec_dirty_pages(inode);
  352. f2fs_remove_dirty_inode(inode);
  353. }
  354. retry:
  355. fio.page = page;
  356. fio.old_blkaddr = NULL_ADDR;
  357. fio.encrypted_page = NULL;
  358. fio.need_lock = LOCK_DONE;
  359. err = f2fs_do_write_data_page(&fio);
  360. if (err) {
  361. if (err == -ENOMEM) {
  362. congestion_wait(BLK_RW_ASYNC,
  363. DEFAULT_IO_TIMEOUT);
  364. cond_resched();
  365. goto retry;
  366. }
  367. unlock_page(page);
  368. break;
  369. }
  370. /* record old blkaddr for revoking */
  371. cur->old_addr = fio.old_blkaddr;
  372. submit_bio = true;
  373. }
  374. unlock_page(page);
  375. list_move_tail(&cur->list, &revoke_list);
  376. }
  377. if (submit_bio)
  378. f2fs_submit_merged_write_cond(sbi, inode, NULL, 0, DATA);
  379. if (err) {
  380. /*
  381. * try to revoke all committed pages, but still we could fail
  382. * due to no memory or other reason, if that happened, EAGAIN
  383. * will be returned, which means in such case, transaction is
  384. * already not integrity, caller should use journal to do the
  385. * recovery or rewrite & commit last transaction. For other
  386. * error number, revoking was done by filesystem itself.
  387. */
  388. err = __revoke_inmem_pages(inode, &revoke_list,
  389. false, true, false);
  390. /* drop all uncommitted pages */
  391. __revoke_inmem_pages(inode, &fi->inmem_pages,
  392. true, false, false);
  393. } else {
  394. __revoke_inmem_pages(inode, &revoke_list,
  395. false, false, false);
  396. }
  397. return err;
  398. }
  399. int f2fs_commit_inmem_pages(struct inode *inode)
  400. {
  401. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  402. struct f2fs_inode_info *fi = F2FS_I(inode);
  403. int err;
  404. f2fs_balance_fs(sbi, true);
  405. f2fs_down_write(&fi->i_gc_rwsem[WRITE]);
  406. f2fs_lock_op(sbi);
  407. set_inode_flag(inode, FI_ATOMIC_COMMIT);
  408. mutex_lock(&fi->inmem_lock);
  409. err = __f2fs_commit_inmem_pages(inode);
  410. mutex_unlock(&fi->inmem_lock);
  411. clear_inode_flag(inode, FI_ATOMIC_COMMIT);
  412. f2fs_unlock_op(sbi);
  413. f2fs_up_write(&fi->i_gc_rwsem[WRITE]);
  414. return err;
  415. }
  416. /*
  417. * This function balances dirty node and dentry pages.
  418. * In addition, it controls garbage collection.
  419. */
  420. void f2fs_balance_fs(struct f2fs_sb_info *sbi, bool need)
  421. {
  422. if (time_to_inject(sbi, FAULT_CHECKPOINT)) {
  423. f2fs_show_injection_info(sbi, FAULT_CHECKPOINT);
  424. f2fs_stop_checkpoint(sbi, false);
  425. }
  426. /* balance_fs_bg is able to be pending */
  427. if (need && excess_cached_nats(sbi))
  428. f2fs_balance_fs_bg(sbi, false);
  429. if (!f2fs_is_checkpoint_ready(sbi))
  430. return;
  431. /*
  432. * We should do GC or end up with checkpoint, if there are so many dirty
  433. * dir/node pages without enough free segments.
  434. */
  435. if (has_not_enough_free_secs(sbi, 0, 0)) {
  436. if (test_opt(sbi, GC_MERGE) && sbi->gc_thread &&
  437. sbi->gc_thread->f2fs_gc_task) {
  438. DEFINE_WAIT(wait);
  439. prepare_to_wait(&sbi->gc_thread->fggc_wq, &wait,
  440. TASK_UNINTERRUPTIBLE);
  441. wake_up(&sbi->gc_thread->gc_wait_queue_head);
  442. io_schedule();
  443. finish_wait(&sbi->gc_thread->fggc_wq, &wait);
  444. } else {
  445. f2fs_down_write(&sbi->gc_lock);
  446. f2fs_gc(sbi, false, false, false, NULL_SEGNO);
  447. }
  448. }
  449. }
  450. void f2fs_balance_fs_bg(struct f2fs_sb_info *sbi, bool from_bg)
  451. {
  452. if (unlikely(is_sbi_flag_set(sbi, SBI_POR_DOING)))
  453. return;
  454. /* try to shrink extent cache when there is no enough memory */
  455. if (!f2fs_available_free_memory(sbi, EXTENT_CACHE))
  456. f2fs_shrink_extent_tree(sbi, EXTENT_CACHE_SHRINK_NUMBER);
  457. /* check the # of cached NAT entries */
  458. if (!f2fs_available_free_memory(sbi, NAT_ENTRIES))
  459. f2fs_try_to_free_nats(sbi, NAT_ENTRY_PER_BLOCK);
  460. if (!f2fs_available_free_memory(sbi, FREE_NIDS))
  461. f2fs_try_to_free_nids(sbi, MAX_FREE_NIDS);
  462. else
  463. f2fs_build_free_nids(sbi, false, false);
  464. if (excess_dirty_nats(sbi) || excess_dirty_nodes(sbi) ||
  465. excess_prefree_segs(sbi))
  466. goto do_sync;
  467. /* there is background inflight IO or foreground operation recently */
  468. if (is_inflight_io(sbi, REQ_TIME) ||
  469. (!f2fs_time_over(sbi, REQ_TIME) && f2fs_rwsem_is_locked(&sbi->cp_rwsem)))
  470. return;
  471. /* exceed periodical checkpoint timeout threshold */
  472. if (f2fs_time_over(sbi, CP_TIME))
  473. goto do_sync;
  474. /* checkpoint is the only way to shrink partial cached entries */
  475. if (f2fs_available_free_memory(sbi, NAT_ENTRIES) ||
  476. f2fs_available_free_memory(sbi, INO_ENTRIES))
  477. return;
  478. do_sync:
  479. if (test_opt(sbi, DATA_FLUSH) && from_bg) {
  480. struct blk_plug plug;
  481. mutex_lock(&sbi->flush_lock);
  482. blk_start_plug(&plug);
  483. f2fs_sync_dirty_inodes(sbi, FILE_INODE);
  484. blk_finish_plug(&plug);
  485. mutex_unlock(&sbi->flush_lock);
  486. }
  487. f2fs_sync_fs(sbi->sb, true);
  488. stat_inc_bg_cp_count(sbi->stat_info);
  489. }
  490. static int __submit_flush_wait(struct f2fs_sb_info *sbi,
  491. struct block_device *bdev)
  492. {
  493. int ret = blkdev_issue_flush(bdev, GFP_NOFS);
  494. trace_f2fs_issue_flush(bdev, test_opt(sbi, NOBARRIER),
  495. test_opt(sbi, FLUSH_MERGE), ret);
  496. return ret;
  497. }
  498. static int submit_flush_wait(struct f2fs_sb_info *sbi, nid_t ino)
  499. {
  500. int ret = 0;
  501. int i;
  502. if (!f2fs_is_multi_device(sbi))
  503. return __submit_flush_wait(sbi, sbi->sb->s_bdev);
  504. for (i = 0; i < sbi->s_ndevs; i++) {
  505. if (!f2fs_is_dirty_device(sbi, ino, i, FLUSH_INO))
  506. continue;
  507. ret = __submit_flush_wait(sbi, FDEV(i).bdev);
  508. if (ret)
  509. break;
  510. }
  511. return ret;
  512. }
  513. static int issue_flush_thread(void *data)
  514. {
  515. struct f2fs_sb_info *sbi = data;
  516. struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
  517. wait_queue_head_t *q = &fcc->flush_wait_queue;
  518. repeat:
  519. if (kthread_should_stop())
  520. return 0;
  521. if (!llist_empty(&fcc->issue_list)) {
  522. struct flush_cmd *cmd, *next;
  523. int ret;
  524. fcc->dispatch_list = llist_del_all(&fcc->issue_list);
  525. fcc->dispatch_list = llist_reverse_order(fcc->dispatch_list);
  526. cmd = llist_entry(fcc->dispatch_list, struct flush_cmd, llnode);
  527. ret = submit_flush_wait(sbi, cmd->ino);
  528. atomic_inc(&fcc->issued_flush);
  529. llist_for_each_entry_safe(cmd, next,
  530. fcc->dispatch_list, llnode) {
  531. cmd->ret = ret;
  532. complete(&cmd->wait);
  533. }
  534. fcc->dispatch_list = NULL;
  535. }
  536. wait_event_interruptible(*q,
  537. kthread_should_stop() || !llist_empty(&fcc->issue_list));
  538. goto repeat;
  539. }
  540. int f2fs_issue_flush(struct f2fs_sb_info *sbi, nid_t ino)
  541. {
  542. struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
  543. struct flush_cmd cmd;
  544. int ret;
  545. if (test_opt(sbi, NOBARRIER))
  546. return 0;
  547. if (!test_opt(sbi, FLUSH_MERGE)) {
  548. atomic_inc(&fcc->queued_flush);
  549. ret = submit_flush_wait(sbi, ino);
  550. atomic_dec(&fcc->queued_flush);
  551. atomic_inc(&fcc->issued_flush);
  552. return ret;
  553. }
  554. if (atomic_inc_return(&fcc->queued_flush) == 1 ||
  555. f2fs_is_multi_device(sbi)) {
  556. ret = submit_flush_wait(sbi, ino);
  557. atomic_dec(&fcc->queued_flush);
  558. atomic_inc(&fcc->issued_flush);
  559. return ret;
  560. }
  561. cmd.ino = ino;
  562. init_completion(&cmd.wait);
  563. llist_add(&cmd.llnode, &fcc->issue_list);
  564. /*
  565. * update issue_list before we wake up issue_flush thread, this
  566. * smp_mb() pairs with another barrier in ___wait_event(), see
  567. * more details in comments of waitqueue_active().
  568. */
  569. smp_mb();
  570. if (waitqueue_active(&fcc->flush_wait_queue))
  571. wake_up(&fcc->flush_wait_queue);
  572. if (fcc->f2fs_issue_flush) {
  573. wait_for_completion(&cmd.wait);
  574. atomic_dec(&fcc->queued_flush);
  575. } else {
  576. struct llist_node *list;
  577. list = llist_del_all(&fcc->issue_list);
  578. if (!list) {
  579. wait_for_completion(&cmd.wait);
  580. atomic_dec(&fcc->queued_flush);
  581. } else {
  582. struct flush_cmd *tmp, *next;
  583. ret = submit_flush_wait(sbi, ino);
  584. llist_for_each_entry_safe(tmp, next, list, llnode) {
  585. if (tmp == &cmd) {
  586. cmd.ret = ret;
  587. atomic_dec(&fcc->queued_flush);
  588. continue;
  589. }
  590. tmp->ret = ret;
  591. complete(&tmp->wait);
  592. }
  593. }
  594. }
  595. return cmd.ret;
  596. }
  597. int f2fs_create_flush_cmd_control(struct f2fs_sb_info *sbi)
  598. {
  599. dev_t dev = sbi->sb->s_bdev->bd_dev;
  600. struct flush_cmd_control *fcc;
  601. int err = 0;
  602. if (SM_I(sbi)->fcc_info) {
  603. fcc = SM_I(sbi)->fcc_info;
  604. if (fcc->f2fs_issue_flush)
  605. return err;
  606. goto init_thread;
  607. }
  608. fcc = f2fs_kzalloc(sbi, sizeof(struct flush_cmd_control), GFP_KERNEL);
  609. if (!fcc)
  610. return -ENOMEM;
  611. atomic_set(&fcc->issued_flush, 0);
  612. atomic_set(&fcc->queued_flush, 0);
  613. init_waitqueue_head(&fcc->flush_wait_queue);
  614. init_llist_head(&fcc->issue_list);
  615. SM_I(sbi)->fcc_info = fcc;
  616. if (!test_opt(sbi, FLUSH_MERGE))
  617. return err;
  618. init_thread:
  619. fcc->f2fs_issue_flush = kthread_run(issue_flush_thread, sbi,
  620. "f2fs_flush-%u:%u", MAJOR(dev), MINOR(dev));
  621. if (IS_ERR(fcc->f2fs_issue_flush)) {
  622. err = PTR_ERR(fcc->f2fs_issue_flush);
  623. kfree(fcc);
  624. SM_I(sbi)->fcc_info = NULL;
  625. return err;
  626. }
  627. return err;
  628. }
  629. void f2fs_destroy_flush_cmd_control(struct f2fs_sb_info *sbi, bool free)
  630. {
  631. struct flush_cmd_control *fcc = SM_I(sbi)->fcc_info;
  632. if (fcc && fcc->f2fs_issue_flush) {
  633. struct task_struct *flush_thread = fcc->f2fs_issue_flush;
  634. fcc->f2fs_issue_flush = NULL;
  635. kthread_stop(flush_thread);
  636. }
  637. if (free) {
  638. kfree(fcc);
  639. SM_I(sbi)->fcc_info = NULL;
  640. }
  641. }
  642. int f2fs_flush_device_cache(struct f2fs_sb_info *sbi)
  643. {
  644. int ret = 0, i;
  645. if (!f2fs_is_multi_device(sbi))
  646. return 0;
  647. if (test_opt(sbi, NOBARRIER))
  648. return 0;
  649. for (i = 1; i < sbi->s_ndevs; i++) {
  650. if (!f2fs_test_bit(i, (char *)&sbi->dirty_device))
  651. continue;
  652. ret = __submit_flush_wait(sbi, FDEV(i).bdev);
  653. if (ret)
  654. break;
  655. spin_lock(&sbi->dev_lock);
  656. f2fs_clear_bit(i, (char *)&sbi->dirty_device);
  657. spin_unlock(&sbi->dev_lock);
  658. }
  659. return ret;
  660. }
  661. static void __locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
  662. enum dirty_type dirty_type)
  663. {
  664. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  665. /* need not be added */
  666. if (IS_CURSEG(sbi, segno))
  667. return;
  668. if (!test_and_set_bit(segno, dirty_i->dirty_segmap[dirty_type]))
  669. dirty_i->nr_dirty[dirty_type]++;
  670. if (dirty_type == DIRTY) {
  671. struct seg_entry *sentry = get_seg_entry(sbi, segno);
  672. enum dirty_type t = sentry->type;
  673. if (unlikely(t >= DIRTY)) {
  674. f2fs_bug_on(sbi, 1);
  675. return;
  676. }
  677. if (!test_and_set_bit(segno, dirty_i->dirty_segmap[t]))
  678. dirty_i->nr_dirty[t]++;
  679. if (__is_large_section(sbi)) {
  680. unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
  681. block_t valid_blocks =
  682. get_valid_blocks(sbi, segno, true);
  683. f2fs_bug_on(sbi, unlikely(!valid_blocks ||
  684. valid_blocks == BLKS_PER_SEC(sbi)));
  685. if (!IS_CURSEC(sbi, secno))
  686. set_bit(secno, dirty_i->dirty_secmap);
  687. }
  688. }
  689. }
  690. static void __remove_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno,
  691. enum dirty_type dirty_type)
  692. {
  693. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  694. block_t valid_blocks;
  695. if (test_and_clear_bit(segno, dirty_i->dirty_segmap[dirty_type]))
  696. dirty_i->nr_dirty[dirty_type]--;
  697. if (dirty_type == DIRTY) {
  698. struct seg_entry *sentry = get_seg_entry(sbi, segno);
  699. enum dirty_type t = sentry->type;
  700. if (test_and_clear_bit(segno, dirty_i->dirty_segmap[t]))
  701. dirty_i->nr_dirty[t]--;
  702. valid_blocks = get_valid_blocks(sbi, segno, true);
  703. if (valid_blocks == 0) {
  704. clear_bit(GET_SEC_FROM_SEG(sbi, segno),
  705. dirty_i->victim_secmap);
  706. #ifdef CONFIG_F2FS_CHECK_FS
  707. clear_bit(segno, SIT_I(sbi)->invalid_segmap);
  708. #endif
  709. }
  710. if (__is_large_section(sbi)) {
  711. unsigned int secno = GET_SEC_FROM_SEG(sbi, segno);
  712. if (!valid_blocks ||
  713. valid_blocks == BLKS_PER_SEC(sbi)) {
  714. clear_bit(secno, dirty_i->dirty_secmap);
  715. return;
  716. }
  717. if (!IS_CURSEC(sbi, secno))
  718. set_bit(secno, dirty_i->dirty_secmap);
  719. }
  720. }
  721. }
  722. /*
  723. * Should not occur error such as -ENOMEM.
  724. * Adding dirty entry into seglist is not critical operation.
  725. * If a given segment is one of current working segments, it won't be added.
  726. */
  727. static void locate_dirty_segment(struct f2fs_sb_info *sbi, unsigned int segno)
  728. {
  729. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  730. unsigned short valid_blocks, ckpt_valid_blocks;
  731. unsigned int usable_blocks;
  732. if (segno == NULL_SEGNO || IS_CURSEG(sbi, segno))
  733. return;
  734. usable_blocks = f2fs_usable_blks_in_seg(sbi, segno);
  735. mutex_lock(&dirty_i->seglist_lock);
  736. valid_blocks = get_valid_blocks(sbi, segno, false);
  737. ckpt_valid_blocks = get_ckpt_valid_blocks(sbi, segno, false);
  738. if (valid_blocks == 0 && (!is_sbi_flag_set(sbi, SBI_CP_DISABLED) ||
  739. ckpt_valid_blocks == usable_blocks)) {
  740. __locate_dirty_segment(sbi, segno, PRE);
  741. __remove_dirty_segment(sbi, segno, DIRTY);
  742. } else if (valid_blocks < usable_blocks) {
  743. __locate_dirty_segment(sbi, segno, DIRTY);
  744. } else {
  745. /* Recovery routine with SSR needs this */
  746. __remove_dirty_segment(sbi, segno, DIRTY);
  747. }
  748. mutex_unlock(&dirty_i->seglist_lock);
  749. }
  750. /* This moves currently empty dirty blocks to prefree. Must hold seglist_lock */
  751. void f2fs_dirty_to_prefree(struct f2fs_sb_info *sbi)
  752. {
  753. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  754. unsigned int segno;
  755. mutex_lock(&dirty_i->seglist_lock);
  756. for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
  757. if (get_valid_blocks(sbi, segno, false))
  758. continue;
  759. if (IS_CURSEG(sbi, segno))
  760. continue;
  761. __locate_dirty_segment(sbi, segno, PRE);
  762. __remove_dirty_segment(sbi, segno, DIRTY);
  763. }
  764. mutex_unlock(&dirty_i->seglist_lock);
  765. }
  766. block_t f2fs_get_unusable_blocks(struct f2fs_sb_info *sbi)
  767. {
  768. int ovp_hole_segs =
  769. (overprovision_segments(sbi) - reserved_segments(sbi));
  770. block_t ovp_holes = ovp_hole_segs << sbi->log_blocks_per_seg;
  771. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  772. block_t holes[2] = {0, 0}; /* DATA and NODE */
  773. block_t unusable;
  774. struct seg_entry *se;
  775. unsigned int segno;
  776. mutex_lock(&dirty_i->seglist_lock);
  777. for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
  778. se = get_seg_entry(sbi, segno);
  779. if (IS_NODESEG(se->type))
  780. holes[NODE] += f2fs_usable_blks_in_seg(sbi, segno) -
  781. se->valid_blocks;
  782. else
  783. holes[DATA] += f2fs_usable_blks_in_seg(sbi, segno) -
  784. se->valid_blocks;
  785. }
  786. mutex_unlock(&dirty_i->seglist_lock);
  787. unusable = holes[DATA] > holes[NODE] ? holes[DATA] : holes[NODE];
  788. if (unusable > ovp_holes)
  789. return unusable - ovp_holes;
  790. return 0;
  791. }
  792. int f2fs_disable_cp_again(struct f2fs_sb_info *sbi, block_t unusable)
  793. {
  794. int ovp_hole_segs =
  795. (overprovision_segments(sbi) - reserved_segments(sbi));
  796. if (unusable > F2FS_OPTION(sbi).unusable_cap)
  797. return -EAGAIN;
  798. if (is_sbi_flag_set(sbi, SBI_CP_DISABLED_QUICK) &&
  799. dirty_segments(sbi) > ovp_hole_segs)
  800. return -EAGAIN;
  801. return 0;
  802. }
  803. /* This is only used by SBI_CP_DISABLED */
  804. static unsigned int get_free_segment(struct f2fs_sb_info *sbi)
  805. {
  806. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  807. unsigned int segno = 0;
  808. mutex_lock(&dirty_i->seglist_lock);
  809. for_each_set_bit(segno, dirty_i->dirty_segmap[DIRTY], MAIN_SEGS(sbi)) {
  810. if (get_valid_blocks(sbi, segno, false))
  811. continue;
  812. if (get_ckpt_valid_blocks(sbi, segno, false))
  813. continue;
  814. mutex_unlock(&dirty_i->seglist_lock);
  815. return segno;
  816. }
  817. mutex_unlock(&dirty_i->seglist_lock);
  818. return NULL_SEGNO;
  819. }
  820. static struct discard_cmd *__create_discard_cmd(struct f2fs_sb_info *sbi,
  821. struct block_device *bdev, block_t lstart,
  822. block_t start, block_t len)
  823. {
  824. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  825. struct list_head *pend_list;
  826. struct discard_cmd *dc;
  827. f2fs_bug_on(sbi, !len);
  828. pend_list = &dcc->pend_list[plist_idx(len)];
  829. dc = f2fs_kmem_cache_alloc(discard_cmd_slab, GFP_NOFS);
  830. INIT_LIST_HEAD(&dc->list);
  831. dc->bdev = bdev;
  832. dc->lstart = lstart;
  833. dc->start = start;
  834. dc->len = len;
  835. dc->ref = 0;
  836. dc->state = D_PREP;
  837. dc->queued = 0;
  838. dc->error = 0;
  839. init_completion(&dc->wait);
  840. list_add_tail(&dc->list, pend_list);
  841. spin_lock_init(&dc->lock);
  842. dc->bio_ref = 0;
  843. atomic_inc(&dcc->discard_cmd_cnt);
  844. dcc->undiscard_blks += len;
  845. return dc;
  846. }
  847. static struct discard_cmd *__attach_discard_cmd(struct f2fs_sb_info *sbi,
  848. struct block_device *bdev, block_t lstart,
  849. block_t start, block_t len,
  850. struct rb_node *parent, struct rb_node **p,
  851. bool leftmost)
  852. {
  853. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  854. struct discard_cmd *dc;
  855. dc = __create_discard_cmd(sbi, bdev, lstart, start, len);
  856. rb_link_node(&dc->rb_node, parent, p);
  857. rb_insert_color_cached(&dc->rb_node, &dcc->root, leftmost);
  858. return dc;
  859. }
  860. static void __detach_discard_cmd(struct discard_cmd_control *dcc,
  861. struct discard_cmd *dc)
  862. {
  863. if (dc->state == D_DONE)
  864. atomic_sub(dc->queued, &dcc->queued_discard);
  865. list_del(&dc->list);
  866. rb_erase_cached(&dc->rb_node, &dcc->root);
  867. dcc->undiscard_blks -= dc->len;
  868. kmem_cache_free(discard_cmd_slab, dc);
  869. atomic_dec(&dcc->discard_cmd_cnt);
  870. }
  871. static void __remove_discard_cmd(struct f2fs_sb_info *sbi,
  872. struct discard_cmd *dc)
  873. {
  874. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  875. unsigned long flags;
  876. trace_f2fs_remove_discard(dc->bdev, dc->start, dc->len);
  877. spin_lock_irqsave(&dc->lock, flags);
  878. if (dc->bio_ref) {
  879. spin_unlock_irqrestore(&dc->lock, flags);
  880. return;
  881. }
  882. spin_unlock_irqrestore(&dc->lock, flags);
  883. f2fs_bug_on(sbi, dc->ref);
  884. if (dc->error == -EOPNOTSUPP)
  885. dc->error = 0;
  886. if (dc->error)
  887. printk_ratelimited(
  888. "%sF2FS-fs (%s): Issue discard(%u, %u, %u) failed, ret: %d",
  889. KERN_INFO, sbi->sb->s_id,
  890. dc->lstart, dc->start, dc->len, dc->error);
  891. __detach_discard_cmd(dcc, dc);
  892. }
  893. static void f2fs_submit_discard_endio(struct bio *bio)
  894. {
  895. struct discard_cmd *dc = (struct discard_cmd *)bio->bi_private;
  896. unsigned long flags;
  897. spin_lock_irqsave(&dc->lock, flags);
  898. if (!dc->error)
  899. dc->error = blk_status_to_errno(bio->bi_status);
  900. dc->bio_ref--;
  901. if (!dc->bio_ref && dc->state == D_SUBMIT) {
  902. dc->state = D_DONE;
  903. complete_all(&dc->wait);
  904. }
  905. spin_unlock_irqrestore(&dc->lock, flags);
  906. bio_put(bio);
  907. }
  908. static void __check_sit_bitmap(struct f2fs_sb_info *sbi,
  909. block_t start, block_t end)
  910. {
  911. #ifdef CONFIG_F2FS_CHECK_FS
  912. struct seg_entry *sentry;
  913. unsigned int segno;
  914. block_t blk = start;
  915. unsigned long offset, size, max_blocks = sbi->blocks_per_seg;
  916. unsigned long *map;
  917. while (blk < end) {
  918. segno = GET_SEGNO(sbi, blk);
  919. sentry = get_seg_entry(sbi, segno);
  920. offset = GET_BLKOFF_FROM_SEG0(sbi, blk);
  921. if (end < START_BLOCK(sbi, segno + 1))
  922. size = GET_BLKOFF_FROM_SEG0(sbi, end);
  923. else
  924. size = max_blocks;
  925. map = (unsigned long *)(sentry->cur_valid_map);
  926. offset = __find_rev_next_bit(map, size, offset);
  927. f2fs_bug_on(sbi, offset != size);
  928. blk = START_BLOCK(sbi, segno + 1);
  929. }
  930. #endif
  931. }
  932. static void __init_discard_policy(struct f2fs_sb_info *sbi,
  933. struct discard_policy *dpolicy,
  934. int discard_type, unsigned int granularity)
  935. {
  936. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  937. /* common policy */
  938. dpolicy->type = discard_type;
  939. dpolicy->sync = true;
  940. dpolicy->ordered = false;
  941. dpolicy->granularity = granularity;
  942. dpolicy->max_requests = DEF_MAX_DISCARD_REQUEST;
  943. dpolicy->io_aware_gran = MAX_PLIST_NUM;
  944. dpolicy->timeout = false;
  945. if (discard_type == DPOLICY_BG) {
  946. dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
  947. dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
  948. dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
  949. dpolicy->io_aware = true;
  950. dpolicy->sync = false;
  951. dpolicy->ordered = true;
  952. if (utilization(sbi) > DEF_DISCARD_URGENT_UTIL) {
  953. dpolicy->granularity = 1;
  954. if (atomic_read(&dcc->discard_cmd_cnt))
  955. dpolicy->max_interval =
  956. DEF_MIN_DISCARD_ISSUE_TIME;
  957. }
  958. } else if (discard_type == DPOLICY_FORCE) {
  959. dpolicy->min_interval = DEF_MIN_DISCARD_ISSUE_TIME;
  960. dpolicy->mid_interval = DEF_MID_DISCARD_ISSUE_TIME;
  961. dpolicy->max_interval = DEF_MAX_DISCARD_ISSUE_TIME;
  962. dpolicy->io_aware = false;
  963. } else if (discard_type == DPOLICY_FSTRIM) {
  964. dpolicy->io_aware = false;
  965. } else if (discard_type == DPOLICY_UMOUNT) {
  966. dpolicy->io_aware = false;
  967. /* we need to issue all to keep CP_TRIMMED_FLAG */
  968. dpolicy->granularity = 1;
  969. dpolicy->timeout = true;
  970. }
  971. }
  972. static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
  973. struct block_device *bdev, block_t lstart,
  974. block_t start, block_t len);
  975. /* this function is copied from blkdev_issue_discard from block/blk-lib.c */
  976. static int __submit_discard_cmd(struct f2fs_sb_info *sbi,
  977. struct discard_policy *dpolicy,
  978. struct discard_cmd *dc,
  979. unsigned int *issued)
  980. {
  981. struct block_device *bdev = dc->bdev;
  982. struct request_queue *q = bdev_get_queue(bdev);
  983. unsigned int max_discard_blocks =
  984. SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
  985. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  986. struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
  987. &(dcc->fstrim_list) : &(dcc->wait_list);
  988. int flag = dpolicy->sync ? REQ_SYNC : 0;
  989. block_t lstart, start, len, total_len;
  990. int err = 0;
  991. if (dc->state != D_PREP)
  992. return 0;
  993. if (is_sbi_flag_set(sbi, SBI_NEED_FSCK))
  994. return 0;
  995. trace_f2fs_issue_discard(bdev, dc->start, dc->len);
  996. lstart = dc->lstart;
  997. start = dc->start;
  998. len = dc->len;
  999. total_len = len;
  1000. dc->len = 0;
  1001. while (total_len && *issued < dpolicy->max_requests && !err) {
  1002. struct bio *bio = NULL;
  1003. unsigned long flags;
  1004. bool last = true;
  1005. if (len > max_discard_blocks) {
  1006. len = max_discard_blocks;
  1007. last = false;
  1008. }
  1009. (*issued)++;
  1010. if (*issued == dpolicy->max_requests)
  1011. last = true;
  1012. dc->len += len;
  1013. if (time_to_inject(sbi, FAULT_DISCARD)) {
  1014. f2fs_show_injection_info(sbi, FAULT_DISCARD);
  1015. err = -EIO;
  1016. goto submit;
  1017. }
  1018. err = __blkdev_issue_discard(bdev,
  1019. SECTOR_FROM_BLOCK(start),
  1020. SECTOR_FROM_BLOCK(len),
  1021. GFP_NOFS, 0, &bio);
  1022. submit:
  1023. if (err) {
  1024. spin_lock_irqsave(&dc->lock, flags);
  1025. if (dc->state == D_PARTIAL)
  1026. dc->state = D_SUBMIT;
  1027. spin_unlock_irqrestore(&dc->lock, flags);
  1028. break;
  1029. }
  1030. f2fs_bug_on(sbi, !bio);
  1031. /*
  1032. * should keep before submission to avoid D_DONE
  1033. * right away
  1034. */
  1035. spin_lock_irqsave(&dc->lock, flags);
  1036. if (last)
  1037. dc->state = D_SUBMIT;
  1038. else
  1039. dc->state = D_PARTIAL;
  1040. dc->bio_ref++;
  1041. spin_unlock_irqrestore(&dc->lock, flags);
  1042. atomic_inc(&dcc->queued_discard);
  1043. dc->queued++;
  1044. list_move_tail(&dc->list, wait_list);
  1045. /* sanity check on discard range */
  1046. __check_sit_bitmap(sbi, lstart, lstart + len);
  1047. bio->bi_private = dc;
  1048. bio->bi_end_io = f2fs_submit_discard_endio;
  1049. bio->bi_opf |= flag;
  1050. submit_bio(bio);
  1051. atomic_inc(&dcc->issued_discard);
  1052. f2fs_update_iostat(sbi, FS_DISCARD, 1);
  1053. lstart += len;
  1054. start += len;
  1055. total_len -= len;
  1056. len = total_len;
  1057. }
  1058. if (!err && len) {
  1059. dcc->undiscard_blks -= len;
  1060. __update_discard_tree_range(sbi, bdev, lstart, start, len);
  1061. }
  1062. return err;
  1063. }
  1064. static void __insert_discard_tree(struct f2fs_sb_info *sbi,
  1065. struct block_device *bdev, block_t lstart,
  1066. block_t start, block_t len,
  1067. struct rb_node **insert_p,
  1068. struct rb_node *insert_parent)
  1069. {
  1070. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1071. struct rb_node **p;
  1072. struct rb_node *parent = NULL;
  1073. bool leftmost = true;
  1074. if (insert_p && insert_parent) {
  1075. parent = insert_parent;
  1076. p = insert_p;
  1077. goto do_insert;
  1078. }
  1079. p = f2fs_lookup_rb_tree_for_insert(sbi, &dcc->root, &parent,
  1080. lstart, &leftmost);
  1081. do_insert:
  1082. __attach_discard_cmd(sbi, bdev, lstart, start, len, parent,
  1083. p, leftmost);
  1084. }
  1085. static void __relocate_discard_cmd(struct discard_cmd_control *dcc,
  1086. struct discard_cmd *dc)
  1087. {
  1088. list_move_tail(&dc->list, &dcc->pend_list[plist_idx(dc->len)]);
  1089. }
  1090. static void __punch_discard_cmd(struct f2fs_sb_info *sbi,
  1091. struct discard_cmd *dc, block_t blkaddr)
  1092. {
  1093. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1094. struct discard_info di = dc->di;
  1095. bool modified = false;
  1096. if (dc->state == D_DONE || dc->len == 1) {
  1097. __remove_discard_cmd(sbi, dc);
  1098. return;
  1099. }
  1100. dcc->undiscard_blks -= di.len;
  1101. if (blkaddr > di.lstart) {
  1102. dc->len = blkaddr - dc->lstart;
  1103. dcc->undiscard_blks += dc->len;
  1104. __relocate_discard_cmd(dcc, dc);
  1105. modified = true;
  1106. }
  1107. if (blkaddr < di.lstart + di.len - 1) {
  1108. if (modified) {
  1109. __insert_discard_tree(sbi, dc->bdev, blkaddr + 1,
  1110. di.start + blkaddr + 1 - di.lstart,
  1111. di.lstart + di.len - 1 - blkaddr,
  1112. NULL, NULL);
  1113. } else {
  1114. dc->lstart++;
  1115. dc->len--;
  1116. dc->start++;
  1117. dcc->undiscard_blks += dc->len;
  1118. __relocate_discard_cmd(dcc, dc);
  1119. }
  1120. }
  1121. }
  1122. static void __update_discard_tree_range(struct f2fs_sb_info *sbi,
  1123. struct block_device *bdev, block_t lstart,
  1124. block_t start, block_t len)
  1125. {
  1126. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1127. struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
  1128. struct discard_cmd *dc;
  1129. struct discard_info di = {0};
  1130. struct rb_node **insert_p = NULL, *insert_parent = NULL;
  1131. struct request_queue *q = bdev_get_queue(bdev);
  1132. unsigned int max_discard_blocks =
  1133. SECTOR_TO_BLOCK(q->limits.max_discard_sectors);
  1134. block_t end = lstart + len;
  1135. dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
  1136. NULL, lstart,
  1137. (struct rb_entry **)&prev_dc,
  1138. (struct rb_entry **)&next_dc,
  1139. &insert_p, &insert_parent, true, NULL);
  1140. if (dc)
  1141. prev_dc = dc;
  1142. if (!prev_dc) {
  1143. di.lstart = lstart;
  1144. di.len = next_dc ? next_dc->lstart - lstart : len;
  1145. di.len = min(di.len, len);
  1146. di.start = start;
  1147. }
  1148. while (1) {
  1149. struct rb_node *node;
  1150. bool merged = false;
  1151. struct discard_cmd *tdc = NULL;
  1152. if (prev_dc) {
  1153. di.lstart = prev_dc->lstart + prev_dc->len;
  1154. if (di.lstart < lstart)
  1155. di.lstart = lstart;
  1156. if (di.lstart >= end)
  1157. break;
  1158. if (!next_dc || next_dc->lstart > end)
  1159. di.len = end - di.lstart;
  1160. else
  1161. di.len = next_dc->lstart - di.lstart;
  1162. di.start = start + di.lstart - lstart;
  1163. }
  1164. if (!di.len)
  1165. goto next;
  1166. if (prev_dc && prev_dc->state == D_PREP &&
  1167. prev_dc->bdev == bdev &&
  1168. __is_discard_back_mergeable(&di, &prev_dc->di,
  1169. max_discard_blocks)) {
  1170. prev_dc->di.len += di.len;
  1171. dcc->undiscard_blks += di.len;
  1172. __relocate_discard_cmd(dcc, prev_dc);
  1173. di = prev_dc->di;
  1174. tdc = prev_dc;
  1175. merged = true;
  1176. }
  1177. if (next_dc && next_dc->state == D_PREP &&
  1178. next_dc->bdev == bdev &&
  1179. __is_discard_front_mergeable(&di, &next_dc->di,
  1180. max_discard_blocks)) {
  1181. next_dc->di.lstart = di.lstart;
  1182. next_dc->di.len += di.len;
  1183. next_dc->di.start = di.start;
  1184. dcc->undiscard_blks += di.len;
  1185. __relocate_discard_cmd(dcc, next_dc);
  1186. if (tdc)
  1187. __remove_discard_cmd(sbi, tdc);
  1188. merged = true;
  1189. }
  1190. if (!merged) {
  1191. __insert_discard_tree(sbi, bdev, di.lstart, di.start,
  1192. di.len, NULL, NULL);
  1193. }
  1194. next:
  1195. prev_dc = next_dc;
  1196. if (!prev_dc)
  1197. break;
  1198. node = rb_next(&prev_dc->rb_node);
  1199. next_dc = rb_entry_safe(node, struct discard_cmd, rb_node);
  1200. }
  1201. }
  1202. static int __queue_discard_cmd(struct f2fs_sb_info *sbi,
  1203. struct block_device *bdev, block_t blkstart, block_t blklen)
  1204. {
  1205. block_t lblkstart = blkstart;
  1206. if (!f2fs_bdev_support_discard(bdev))
  1207. return 0;
  1208. trace_f2fs_queue_discard(bdev, blkstart, blklen);
  1209. if (f2fs_is_multi_device(sbi)) {
  1210. int devi = f2fs_target_device_index(sbi, blkstart);
  1211. blkstart -= FDEV(devi).start_blk;
  1212. }
  1213. mutex_lock(&SM_I(sbi)->dcc_info->cmd_lock);
  1214. __update_discard_tree_range(sbi, bdev, lblkstart, blkstart, blklen);
  1215. mutex_unlock(&SM_I(sbi)->dcc_info->cmd_lock);
  1216. return 0;
  1217. }
  1218. static unsigned int __issue_discard_cmd_orderly(struct f2fs_sb_info *sbi,
  1219. struct discard_policy *dpolicy)
  1220. {
  1221. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1222. struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
  1223. struct rb_node **insert_p = NULL, *insert_parent = NULL;
  1224. struct discard_cmd *dc;
  1225. struct blk_plug plug;
  1226. unsigned int pos = dcc->next_pos;
  1227. unsigned int issued = 0;
  1228. bool io_interrupted = false;
  1229. mutex_lock(&dcc->cmd_lock);
  1230. dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
  1231. NULL, pos,
  1232. (struct rb_entry **)&prev_dc,
  1233. (struct rb_entry **)&next_dc,
  1234. &insert_p, &insert_parent, true, NULL);
  1235. if (!dc)
  1236. dc = next_dc;
  1237. blk_start_plug(&plug);
  1238. while (dc) {
  1239. struct rb_node *node;
  1240. int err = 0;
  1241. if (dc->state != D_PREP)
  1242. goto next;
  1243. if (dpolicy->io_aware && !is_idle(sbi, DISCARD_TIME)) {
  1244. io_interrupted = true;
  1245. break;
  1246. }
  1247. dcc->next_pos = dc->lstart + dc->len;
  1248. err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
  1249. if (issued >= dpolicy->max_requests)
  1250. break;
  1251. next:
  1252. node = rb_next(&dc->rb_node);
  1253. if (err)
  1254. __remove_discard_cmd(sbi, dc);
  1255. dc = rb_entry_safe(node, struct discard_cmd, rb_node);
  1256. }
  1257. blk_finish_plug(&plug);
  1258. if (!dc)
  1259. dcc->next_pos = 0;
  1260. mutex_unlock(&dcc->cmd_lock);
  1261. if (!issued && io_interrupted)
  1262. issued = -1;
  1263. return issued;
  1264. }
  1265. static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
  1266. struct discard_policy *dpolicy);
  1267. static int __issue_discard_cmd(struct f2fs_sb_info *sbi,
  1268. struct discard_policy *dpolicy)
  1269. {
  1270. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1271. struct list_head *pend_list;
  1272. struct discard_cmd *dc, *tmp;
  1273. struct blk_plug plug;
  1274. int i, issued;
  1275. bool io_interrupted = false;
  1276. if (dpolicy->timeout)
  1277. f2fs_update_time(sbi, UMOUNT_DISCARD_TIMEOUT);
  1278. retry:
  1279. issued = 0;
  1280. for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
  1281. if (dpolicy->timeout &&
  1282. f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
  1283. break;
  1284. if (i + 1 < dpolicy->granularity)
  1285. break;
  1286. if (i < DEFAULT_DISCARD_GRANULARITY && dpolicy->ordered)
  1287. return __issue_discard_cmd_orderly(sbi, dpolicy);
  1288. pend_list = &dcc->pend_list[i];
  1289. mutex_lock(&dcc->cmd_lock);
  1290. if (list_empty(pend_list))
  1291. goto next;
  1292. if (unlikely(dcc->rbtree_check))
  1293. f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
  1294. &dcc->root, false));
  1295. blk_start_plug(&plug);
  1296. list_for_each_entry_safe(dc, tmp, pend_list, list) {
  1297. f2fs_bug_on(sbi, dc->state != D_PREP);
  1298. if (dpolicy->timeout &&
  1299. f2fs_time_over(sbi, UMOUNT_DISCARD_TIMEOUT))
  1300. break;
  1301. if (dpolicy->io_aware && i < dpolicy->io_aware_gran &&
  1302. !is_idle(sbi, DISCARD_TIME)) {
  1303. io_interrupted = true;
  1304. break;
  1305. }
  1306. __submit_discard_cmd(sbi, dpolicy, dc, &issued);
  1307. if (issued >= dpolicy->max_requests)
  1308. break;
  1309. }
  1310. blk_finish_plug(&plug);
  1311. next:
  1312. mutex_unlock(&dcc->cmd_lock);
  1313. if (issued >= dpolicy->max_requests || io_interrupted)
  1314. break;
  1315. }
  1316. if (dpolicy->type == DPOLICY_UMOUNT && issued) {
  1317. __wait_all_discard_cmd(sbi, dpolicy);
  1318. goto retry;
  1319. }
  1320. if (!issued && io_interrupted)
  1321. issued = -1;
  1322. return issued;
  1323. }
  1324. static bool __drop_discard_cmd(struct f2fs_sb_info *sbi)
  1325. {
  1326. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1327. struct list_head *pend_list;
  1328. struct discard_cmd *dc, *tmp;
  1329. int i;
  1330. bool dropped = false;
  1331. mutex_lock(&dcc->cmd_lock);
  1332. for (i = MAX_PLIST_NUM - 1; i >= 0; i--) {
  1333. pend_list = &dcc->pend_list[i];
  1334. list_for_each_entry_safe(dc, tmp, pend_list, list) {
  1335. f2fs_bug_on(sbi, dc->state != D_PREP);
  1336. __remove_discard_cmd(sbi, dc);
  1337. dropped = true;
  1338. }
  1339. }
  1340. mutex_unlock(&dcc->cmd_lock);
  1341. return dropped;
  1342. }
  1343. void f2fs_drop_discard_cmd(struct f2fs_sb_info *sbi)
  1344. {
  1345. __drop_discard_cmd(sbi);
  1346. }
  1347. static unsigned int __wait_one_discard_bio(struct f2fs_sb_info *sbi,
  1348. struct discard_cmd *dc)
  1349. {
  1350. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1351. unsigned int len = 0;
  1352. wait_for_completion_io(&dc->wait);
  1353. mutex_lock(&dcc->cmd_lock);
  1354. f2fs_bug_on(sbi, dc->state != D_DONE);
  1355. dc->ref--;
  1356. if (!dc->ref) {
  1357. if (!dc->error)
  1358. len = dc->len;
  1359. __remove_discard_cmd(sbi, dc);
  1360. }
  1361. mutex_unlock(&dcc->cmd_lock);
  1362. return len;
  1363. }
  1364. static unsigned int __wait_discard_cmd_range(struct f2fs_sb_info *sbi,
  1365. struct discard_policy *dpolicy,
  1366. block_t start, block_t end)
  1367. {
  1368. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1369. struct list_head *wait_list = (dpolicy->type == DPOLICY_FSTRIM) ?
  1370. &(dcc->fstrim_list) : &(dcc->wait_list);
  1371. struct discard_cmd *dc, *tmp;
  1372. bool need_wait;
  1373. unsigned int trimmed = 0;
  1374. next:
  1375. need_wait = false;
  1376. mutex_lock(&dcc->cmd_lock);
  1377. list_for_each_entry_safe(dc, tmp, wait_list, list) {
  1378. if (dc->lstart + dc->len <= start || end <= dc->lstart)
  1379. continue;
  1380. if (dc->len < dpolicy->granularity)
  1381. continue;
  1382. if (dc->state == D_DONE && !dc->ref) {
  1383. wait_for_completion_io(&dc->wait);
  1384. if (!dc->error)
  1385. trimmed += dc->len;
  1386. __remove_discard_cmd(sbi, dc);
  1387. } else {
  1388. dc->ref++;
  1389. need_wait = true;
  1390. break;
  1391. }
  1392. }
  1393. mutex_unlock(&dcc->cmd_lock);
  1394. if (need_wait) {
  1395. trimmed += __wait_one_discard_bio(sbi, dc);
  1396. goto next;
  1397. }
  1398. return trimmed;
  1399. }
  1400. static unsigned int __wait_all_discard_cmd(struct f2fs_sb_info *sbi,
  1401. struct discard_policy *dpolicy)
  1402. {
  1403. struct discard_policy dp;
  1404. unsigned int discard_blks;
  1405. if (dpolicy)
  1406. return __wait_discard_cmd_range(sbi, dpolicy, 0, UINT_MAX);
  1407. /* wait all */
  1408. __init_discard_policy(sbi, &dp, DPOLICY_FSTRIM, 1);
  1409. discard_blks = __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
  1410. __init_discard_policy(sbi, &dp, DPOLICY_UMOUNT, 1);
  1411. discard_blks += __wait_discard_cmd_range(sbi, &dp, 0, UINT_MAX);
  1412. return discard_blks;
  1413. }
  1414. /* This should be covered by global mutex, &sit_i->sentry_lock */
  1415. static void f2fs_wait_discard_bio(struct f2fs_sb_info *sbi, block_t blkaddr)
  1416. {
  1417. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1418. struct discard_cmd *dc;
  1419. bool need_wait = false;
  1420. mutex_lock(&dcc->cmd_lock);
  1421. dc = (struct discard_cmd *)f2fs_lookup_rb_tree(&dcc->root,
  1422. NULL, blkaddr);
  1423. if (dc) {
  1424. if (dc->state == D_PREP) {
  1425. __punch_discard_cmd(sbi, dc, blkaddr);
  1426. } else {
  1427. dc->ref++;
  1428. need_wait = true;
  1429. }
  1430. }
  1431. mutex_unlock(&dcc->cmd_lock);
  1432. if (need_wait)
  1433. __wait_one_discard_bio(sbi, dc);
  1434. }
  1435. void f2fs_stop_discard_thread(struct f2fs_sb_info *sbi)
  1436. {
  1437. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1438. if (dcc && dcc->f2fs_issue_discard) {
  1439. struct task_struct *discard_thread = dcc->f2fs_issue_discard;
  1440. dcc->f2fs_issue_discard = NULL;
  1441. kthread_stop(discard_thread);
  1442. }
  1443. }
  1444. /* This comes from f2fs_put_super */
  1445. bool f2fs_issue_discard_timeout(struct f2fs_sb_info *sbi)
  1446. {
  1447. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1448. struct discard_policy dpolicy;
  1449. bool dropped;
  1450. __init_discard_policy(sbi, &dpolicy, DPOLICY_UMOUNT,
  1451. dcc->discard_granularity);
  1452. __issue_discard_cmd(sbi, &dpolicy);
  1453. dropped = __drop_discard_cmd(sbi);
  1454. /* just to make sure there is no pending discard commands */
  1455. __wait_all_discard_cmd(sbi, NULL);
  1456. f2fs_bug_on(sbi, atomic_read(&dcc->discard_cmd_cnt));
  1457. return dropped;
  1458. }
  1459. static int issue_discard_thread(void *data)
  1460. {
  1461. struct f2fs_sb_info *sbi = data;
  1462. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1463. wait_queue_head_t *q = &dcc->discard_wait_queue;
  1464. struct discard_policy dpolicy;
  1465. unsigned int wait_ms = DEF_MIN_DISCARD_ISSUE_TIME;
  1466. int issued;
  1467. set_freezable();
  1468. do {
  1469. if (sbi->gc_mode == GC_URGENT_HIGH ||
  1470. !f2fs_available_free_memory(sbi, DISCARD_CACHE))
  1471. __init_discard_policy(sbi, &dpolicy, DPOLICY_FORCE, 1);
  1472. else
  1473. __init_discard_policy(sbi, &dpolicy, DPOLICY_BG,
  1474. dcc->discard_granularity);
  1475. if (!atomic_read(&dcc->discard_cmd_cnt))
  1476. wait_ms = dpolicy.max_interval;
  1477. wait_event_interruptible_timeout(*q,
  1478. kthread_should_stop() || freezing(current) ||
  1479. dcc->discard_wake,
  1480. msecs_to_jiffies(wait_ms));
  1481. if (dcc->discard_wake)
  1482. dcc->discard_wake = 0;
  1483. /* clean up pending candidates before going to sleep */
  1484. if (atomic_read(&dcc->queued_discard))
  1485. __wait_all_discard_cmd(sbi, NULL);
  1486. if (try_to_freeze())
  1487. continue;
  1488. if (f2fs_readonly(sbi->sb))
  1489. continue;
  1490. if (kthread_should_stop())
  1491. return 0;
  1492. if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
  1493. wait_ms = dpolicy.max_interval;
  1494. continue;
  1495. }
  1496. if (!atomic_read(&dcc->discard_cmd_cnt))
  1497. continue;
  1498. sb_start_intwrite(sbi->sb);
  1499. issued = __issue_discard_cmd(sbi, &dpolicy);
  1500. if (issued > 0) {
  1501. __wait_all_discard_cmd(sbi, &dpolicy);
  1502. wait_ms = dpolicy.min_interval;
  1503. } else if (issued == -1) {
  1504. wait_ms = f2fs_time_to_wait(sbi, DISCARD_TIME);
  1505. if (!wait_ms)
  1506. wait_ms = dpolicy.mid_interval;
  1507. } else {
  1508. wait_ms = dpolicy.max_interval;
  1509. }
  1510. sb_end_intwrite(sbi->sb);
  1511. } while (!kthread_should_stop());
  1512. return 0;
  1513. }
  1514. #ifdef CONFIG_BLK_DEV_ZONED
  1515. static int __f2fs_issue_discard_zone(struct f2fs_sb_info *sbi,
  1516. struct block_device *bdev, block_t blkstart, block_t blklen)
  1517. {
  1518. sector_t sector, nr_sects;
  1519. block_t lblkstart = blkstart;
  1520. int devi = 0;
  1521. if (f2fs_is_multi_device(sbi)) {
  1522. devi = f2fs_target_device_index(sbi, blkstart);
  1523. if (blkstart < FDEV(devi).start_blk ||
  1524. blkstart > FDEV(devi).end_blk) {
  1525. f2fs_err(sbi, "Invalid block %x", blkstart);
  1526. return -EIO;
  1527. }
  1528. blkstart -= FDEV(devi).start_blk;
  1529. }
  1530. /* For sequential zones, reset the zone write pointer */
  1531. if (f2fs_blkz_is_seq(sbi, devi, blkstart)) {
  1532. sector = SECTOR_FROM_BLOCK(blkstart);
  1533. nr_sects = SECTOR_FROM_BLOCK(blklen);
  1534. if (sector & (bdev_zone_sectors(bdev) - 1) ||
  1535. nr_sects != bdev_zone_sectors(bdev)) {
  1536. f2fs_err(sbi, "(%d) %s: Unaligned zone reset attempted (block %x + %x)",
  1537. devi, sbi->s_ndevs ? FDEV(devi).path : "",
  1538. blkstart, blklen);
  1539. return -EIO;
  1540. }
  1541. trace_f2fs_issue_reset_zone(bdev, blkstart);
  1542. return blkdev_zone_mgmt(bdev, REQ_OP_ZONE_RESET,
  1543. sector, nr_sects, GFP_NOFS);
  1544. }
  1545. /* For conventional zones, use regular discard if supported */
  1546. return __queue_discard_cmd(sbi, bdev, lblkstart, blklen);
  1547. }
  1548. #endif
  1549. static int __issue_discard_async(struct f2fs_sb_info *sbi,
  1550. struct block_device *bdev, block_t blkstart, block_t blklen)
  1551. {
  1552. #ifdef CONFIG_BLK_DEV_ZONED
  1553. if (f2fs_sb_has_blkzoned(sbi) && bdev_is_zoned(bdev))
  1554. return __f2fs_issue_discard_zone(sbi, bdev, blkstart, blklen);
  1555. #endif
  1556. return __queue_discard_cmd(sbi, bdev, blkstart, blklen);
  1557. }
  1558. static int f2fs_issue_discard(struct f2fs_sb_info *sbi,
  1559. block_t blkstart, block_t blklen)
  1560. {
  1561. sector_t start = blkstart, len = 0;
  1562. struct block_device *bdev;
  1563. struct seg_entry *se;
  1564. unsigned int offset;
  1565. block_t i;
  1566. int err = 0;
  1567. bdev = f2fs_target_device(sbi, blkstart, NULL);
  1568. for (i = blkstart; i < blkstart + blklen; i++, len++) {
  1569. if (i != start) {
  1570. struct block_device *bdev2 =
  1571. f2fs_target_device(sbi, i, NULL);
  1572. if (bdev2 != bdev) {
  1573. err = __issue_discard_async(sbi, bdev,
  1574. start, len);
  1575. if (err)
  1576. return err;
  1577. bdev = bdev2;
  1578. start = i;
  1579. len = 0;
  1580. }
  1581. }
  1582. se = get_seg_entry(sbi, GET_SEGNO(sbi, i));
  1583. offset = GET_BLKOFF_FROM_SEG0(sbi, i);
  1584. if (!f2fs_test_and_set_bit(offset, se->discard_map))
  1585. sbi->discard_blks--;
  1586. }
  1587. if (len)
  1588. err = __issue_discard_async(sbi, bdev, start, len);
  1589. return err;
  1590. }
  1591. static bool add_discard_addrs(struct f2fs_sb_info *sbi, struct cp_control *cpc,
  1592. bool check_only)
  1593. {
  1594. int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
  1595. int max_blocks = sbi->blocks_per_seg;
  1596. struct seg_entry *se = get_seg_entry(sbi, cpc->trim_start);
  1597. unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
  1598. unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
  1599. unsigned long *discard_map = (unsigned long *)se->discard_map;
  1600. unsigned long *dmap = SIT_I(sbi)->tmp_map;
  1601. unsigned int start = 0, end = -1;
  1602. bool force = (cpc->reason & CP_DISCARD);
  1603. struct discard_entry *de = NULL;
  1604. struct list_head *head = &SM_I(sbi)->dcc_info->entry_list;
  1605. int i;
  1606. if (se->valid_blocks == max_blocks || !f2fs_hw_support_discard(sbi))
  1607. return false;
  1608. if (!force) {
  1609. if (!f2fs_realtime_discard_enable(sbi) || !se->valid_blocks ||
  1610. SM_I(sbi)->dcc_info->nr_discards >=
  1611. SM_I(sbi)->dcc_info->max_discards)
  1612. return false;
  1613. }
  1614. /* SIT_VBLOCK_MAP_SIZE should be multiple of sizeof(unsigned long) */
  1615. for (i = 0; i < entries; i++)
  1616. dmap[i] = force ? ~ckpt_map[i] & ~discard_map[i] :
  1617. (cur_map[i] ^ ckpt_map[i]) & ckpt_map[i];
  1618. while (force || SM_I(sbi)->dcc_info->nr_discards <=
  1619. SM_I(sbi)->dcc_info->max_discards) {
  1620. start = __find_rev_next_bit(dmap, max_blocks, end + 1);
  1621. if (start >= max_blocks)
  1622. break;
  1623. end = __find_rev_next_zero_bit(dmap, max_blocks, start + 1);
  1624. if (force && start && end != max_blocks
  1625. && (end - start) < cpc->trim_minlen)
  1626. continue;
  1627. if (check_only)
  1628. return true;
  1629. if (!de) {
  1630. de = f2fs_kmem_cache_alloc(discard_entry_slab,
  1631. GFP_F2FS_ZERO);
  1632. de->start_blkaddr = START_BLOCK(sbi, cpc->trim_start);
  1633. list_add_tail(&de->list, head);
  1634. }
  1635. for (i = start; i < end; i++)
  1636. __set_bit_le(i, (void *)de->discard_map);
  1637. SM_I(sbi)->dcc_info->nr_discards += end - start;
  1638. }
  1639. return false;
  1640. }
  1641. static void release_discard_addr(struct discard_entry *entry)
  1642. {
  1643. list_del(&entry->list);
  1644. kmem_cache_free(discard_entry_slab, entry);
  1645. }
  1646. void f2fs_release_discard_addrs(struct f2fs_sb_info *sbi)
  1647. {
  1648. struct list_head *head = &(SM_I(sbi)->dcc_info->entry_list);
  1649. struct discard_entry *entry, *this;
  1650. /* drop caches */
  1651. list_for_each_entry_safe(entry, this, head, list)
  1652. release_discard_addr(entry);
  1653. }
  1654. /*
  1655. * Should call f2fs_clear_prefree_segments after checkpoint is done.
  1656. */
  1657. static void set_prefree_as_free_segments(struct f2fs_sb_info *sbi)
  1658. {
  1659. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  1660. unsigned int segno;
  1661. mutex_lock(&dirty_i->seglist_lock);
  1662. for_each_set_bit(segno, dirty_i->dirty_segmap[PRE], MAIN_SEGS(sbi))
  1663. __set_test_and_free(sbi, segno, false);
  1664. mutex_unlock(&dirty_i->seglist_lock);
  1665. }
  1666. void f2fs_clear_prefree_segments(struct f2fs_sb_info *sbi,
  1667. struct cp_control *cpc)
  1668. {
  1669. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1670. struct list_head *head = &dcc->entry_list;
  1671. struct discard_entry *entry, *this;
  1672. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  1673. unsigned long *prefree_map = dirty_i->dirty_segmap[PRE];
  1674. unsigned int start = 0, end = -1;
  1675. unsigned int secno, start_segno;
  1676. bool force = (cpc->reason & CP_DISCARD);
  1677. bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
  1678. mutex_lock(&dirty_i->seglist_lock);
  1679. while (1) {
  1680. int i;
  1681. if (need_align && end != -1)
  1682. end--;
  1683. start = find_next_bit(prefree_map, MAIN_SEGS(sbi), end + 1);
  1684. if (start >= MAIN_SEGS(sbi))
  1685. break;
  1686. end = find_next_zero_bit(prefree_map, MAIN_SEGS(sbi),
  1687. start + 1);
  1688. if (need_align) {
  1689. start = rounddown(start, sbi->segs_per_sec);
  1690. end = roundup(end, sbi->segs_per_sec);
  1691. }
  1692. for (i = start; i < end; i++) {
  1693. if (test_and_clear_bit(i, prefree_map))
  1694. dirty_i->nr_dirty[PRE]--;
  1695. }
  1696. if (!f2fs_realtime_discard_enable(sbi))
  1697. continue;
  1698. if (force && start >= cpc->trim_start &&
  1699. (end - 1) <= cpc->trim_end)
  1700. continue;
  1701. if (!f2fs_lfs_mode(sbi) || !__is_large_section(sbi)) {
  1702. f2fs_issue_discard(sbi, START_BLOCK(sbi, start),
  1703. (end - start) << sbi->log_blocks_per_seg);
  1704. continue;
  1705. }
  1706. next:
  1707. secno = GET_SEC_FROM_SEG(sbi, start);
  1708. start_segno = GET_SEG_FROM_SEC(sbi, secno);
  1709. if (!IS_CURSEC(sbi, secno) &&
  1710. !get_valid_blocks(sbi, start, true))
  1711. f2fs_issue_discard(sbi, START_BLOCK(sbi, start_segno),
  1712. sbi->segs_per_sec << sbi->log_blocks_per_seg);
  1713. start = start_segno + sbi->segs_per_sec;
  1714. if (start < end)
  1715. goto next;
  1716. else
  1717. end = start - 1;
  1718. }
  1719. mutex_unlock(&dirty_i->seglist_lock);
  1720. /* send small discards */
  1721. list_for_each_entry_safe(entry, this, head, list) {
  1722. unsigned int cur_pos = 0, next_pos, len, total_len = 0;
  1723. bool is_valid = test_bit_le(0, entry->discard_map);
  1724. find_next:
  1725. if (is_valid) {
  1726. next_pos = find_next_zero_bit_le(entry->discard_map,
  1727. sbi->blocks_per_seg, cur_pos);
  1728. len = next_pos - cur_pos;
  1729. if (f2fs_sb_has_blkzoned(sbi) ||
  1730. (force && len < cpc->trim_minlen))
  1731. goto skip;
  1732. f2fs_issue_discard(sbi, entry->start_blkaddr + cur_pos,
  1733. len);
  1734. total_len += len;
  1735. } else {
  1736. next_pos = find_next_bit_le(entry->discard_map,
  1737. sbi->blocks_per_seg, cur_pos);
  1738. }
  1739. skip:
  1740. cur_pos = next_pos;
  1741. is_valid = !is_valid;
  1742. if (cur_pos < sbi->blocks_per_seg)
  1743. goto find_next;
  1744. release_discard_addr(entry);
  1745. dcc->nr_discards -= total_len;
  1746. }
  1747. wake_up_discard_thread(sbi, false);
  1748. }
  1749. static int create_discard_cmd_control(struct f2fs_sb_info *sbi)
  1750. {
  1751. dev_t dev = sbi->sb->s_bdev->bd_dev;
  1752. struct discard_cmd_control *dcc;
  1753. int err = 0, i;
  1754. if (SM_I(sbi)->dcc_info) {
  1755. dcc = SM_I(sbi)->dcc_info;
  1756. goto init_thread;
  1757. }
  1758. dcc = f2fs_kzalloc(sbi, sizeof(struct discard_cmd_control), GFP_KERNEL);
  1759. if (!dcc)
  1760. return -ENOMEM;
  1761. dcc->discard_granularity = DEFAULT_DISCARD_GRANULARITY;
  1762. INIT_LIST_HEAD(&dcc->entry_list);
  1763. for (i = 0; i < MAX_PLIST_NUM; i++)
  1764. INIT_LIST_HEAD(&dcc->pend_list[i]);
  1765. INIT_LIST_HEAD(&dcc->wait_list);
  1766. INIT_LIST_HEAD(&dcc->fstrim_list);
  1767. mutex_init(&dcc->cmd_lock);
  1768. atomic_set(&dcc->issued_discard, 0);
  1769. atomic_set(&dcc->queued_discard, 0);
  1770. atomic_set(&dcc->discard_cmd_cnt, 0);
  1771. dcc->nr_discards = 0;
  1772. dcc->max_discards = MAIN_SEGS(sbi) << sbi->log_blocks_per_seg;
  1773. dcc->undiscard_blks = 0;
  1774. dcc->next_pos = 0;
  1775. dcc->root = RB_ROOT_CACHED;
  1776. dcc->rbtree_check = false;
  1777. init_waitqueue_head(&dcc->discard_wait_queue);
  1778. SM_I(sbi)->dcc_info = dcc;
  1779. init_thread:
  1780. dcc->f2fs_issue_discard = kthread_run(issue_discard_thread, sbi,
  1781. "f2fs_discard-%u:%u", MAJOR(dev), MINOR(dev));
  1782. if (IS_ERR(dcc->f2fs_issue_discard)) {
  1783. err = PTR_ERR(dcc->f2fs_issue_discard);
  1784. kfree(dcc);
  1785. SM_I(sbi)->dcc_info = NULL;
  1786. return err;
  1787. }
  1788. return err;
  1789. }
  1790. static void destroy_discard_cmd_control(struct f2fs_sb_info *sbi)
  1791. {
  1792. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  1793. if (!dcc)
  1794. return;
  1795. f2fs_stop_discard_thread(sbi);
  1796. /*
  1797. * Recovery can cache discard commands, so in error path of
  1798. * fill_super(), it needs to give a chance to handle them.
  1799. */
  1800. if (unlikely(atomic_read(&dcc->discard_cmd_cnt)))
  1801. f2fs_issue_discard_timeout(sbi);
  1802. kfree(dcc);
  1803. SM_I(sbi)->dcc_info = NULL;
  1804. }
  1805. static bool __mark_sit_entry_dirty(struct f2fs_sb_info *sbi, unsigned int segno)
  1806. {
  1807. struct sit_info *sit_i = SIT_I(sbi);
  1808. if (!__test_and_set_bit(segno, sit_i->dirty_sentries_bitmap)) {
  1809. sit_i->dirty_sentries++;
  1810. return false;
  1811. }
  1812. return true;
  1813. }
  1814. static void __set_sit_entry_type(struct f2fs_sb_info *sbi, int type,
  1815. unsigned int segno, int modified)
  1816. {
  1817. struct seg_entry *se = get_seg_entry(sbi, segno);
  1818. se->type = type;
  1819. if (modified)
  1820. __mark_sit_entry_dirty(sbi, segno);
  1821. }
  1822. static inline unsigned long long get_segment_mtime(struct f2fs_sb_info *sbi,
  1823. block_t blkaddr)
  1824. {
  1825. unsigned int segno = GET_SEGNO(sbi, blkaddr);
  1826. if (segno == NULL_SEGNO)
  1827. return 0;
  1828. return get_seg_entry(sbi, segno)->mtime;
  1829. }
  1830. static void update_segment_mtime(struct f2fs_sb_info *sbi, block_t blkaddr,
  1831. unsigned long long old_mtime)
  1832. {
  1833. struct seg_entry *se;
  1834. unsigned int segno = GET_SEGNO(sbi, blkaddr);
  1835. unsigned long long ctime = get_mtime(sbi, false);
  1836. unsigned long long mtime = old_mtime ? old_mtime : ctime;
  1837. if (segno == NULL_SEGNO)
  1838. return;
  1839. se = get_seg_entry(sbi, segno);
  1840. if (!se->mtime)
  1841. se->mtime = mtime;
  1842. else
  1843. se->mtime = div_u64(se->mtime * se->valid_blocks + mtime,
  1844. se->valid_blocks + 1);
  1845. if (ctime > SIT_I(sbi)->max_mtime)
  1846. SIT_I(sbi)->max_mtime = ctime;
  1847. }
  1848. static void update_sit_entry(struct f2fs_sb_info *sbi, block_t blkaddr, int del)
  1849. {
  1850. struct seg_entry *se;
  1851. unsigned int segno, offset;
  1852. long int new_vblocks;
  1853. bool exist;
  1854. #ifdef CONFIG_F2FS_CHECK_FS
  1855. bool mir_exist;
  1856. #endif
  1857. segno = GET_SEGNO(sbi, blkaddr);
  1858. se = get_seg_entry(sbi, segno);
  1859. new_vblocks = se->valid_blocks + del;
  1860. offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
  1861. f2fs_bug_on(sbi, (new_vblocks < 0 ||
  1862. (new_vblocks > f2fs_usable_blks_in_seg(sbi, segno))));
  1863. se->valid_blocks = new_vblocks;
  1864. /* Update valid block bitmap */
  1865. if (del > 0) {
  1866. exist = f2fs_test_and_set_bit(offset, se->cur_valid_map);
  1867. #ifdef CONFIG_F2FS_CHECK_FS
  1868. mir_exist = f2fs_test_and_set_bit(offset,
  1869. se->cur_valid_map_mir);
  1870. if (unlikely(exist != mir_exist)) {
  1871. f2fs_err(sbi, "Inconsistent error when setting bitmap, blk:%u, old bit:%d",
  1872. blkaddr, exist);
  1873. f2fs_bug_on(sbi, 1);
  1874. }
  1875. #endif
  1876. if (unlikely(exist)) {
  1877. f2fs_err(sbi, "Bitmap was wrongly set, blk:%u",
  1878. blkaddr);
  1879. f2fs_bug_on(sbi, 1);
  1880. se->valid_blocks--;
  1881. del = 0;
  1882. }
  1883. if (!f2fs_test_and_set_bit(offset, se->discard_map))
  1884. sbi->discard_blks--;
  1885. /*
  1886. * SSR should never reuse block which is checkpointed
  1887. * or newly invalidated.
  1888. */
  1889. if (!is_sbi_flag_set(sbi, SBI_CP_DISABLED)) {
  1890. if (!f2fs_test_and_set_bit(offset, se->ckpt_valid_map))
  1891. se->ckpt_valid_blocks++;
  1892. }
  1893. } else {
  1894. exist = f2fs_test_and_clear_bit(offset, se->cur_valid_map);
  1895. #ifdef CONFIG_F2FS_CHECK_FS
  1896. mir_exist = f2fs_test_and_clear_bit(offset,
  1897. se->cur_valid_map_mir);
  1898. if (unlikely(exist != mir_exist)) {
  1899. f2fs_err(sbi, "Inconsistent error when clearing bitmap, blk:%u, old bit:%d",
  1900. blkaddr, exist);
  1901. f2fs_bug_on(sbi, 1);
  1902. }
  1903. #endif
  1904. if (unlikely(!exist)) {
  1905. f2fs_err(sbi, "Bitmap was wrongly cleared, blk:%u",
  1906. blkaddr);
  1907. f2fs_bug_on(sbi, 1);
  1908. se->valid_blocks++;
  1909. del = 0;
  1910. } else if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
  1911. /*
  1912. * If checkpoints are off, we must not reuse data that
  1913. * was used in the previous checkpoint. If it was used
  1914. * before, we must track that to know how much space we
  1915. * really have.
  1916. */
  1917. if (f2fs_test_bit(offset, se->ckpt_valid_map)) {
  1918. spin_lock(&sbi->stat_lock);
  1919. sbi->unusable_block_count++;
  1920. spin_unlock(&sbi->stat_lock);
  1921. }
  1922. }
  1923. if (f2fs_test_and_clear_bit(offset, se->discard_map))
  1924. sbi->discard_blks++;
  1925. }
  1926. if (!f2fs_test_bit(offset, se->ckpt_valid_map))
  1927. se->ckpt_valid_blocks += del;
  1928. __mark_sit_entry_dirty(sbi, segno);
  1929. /* update total number of valid blocks to be written in ckpt area */
  1930. SIT_I(sbi)->written_valid_blocks += del;
  1931. if (__is_large_section(sbi))
  1932. get_sec_entry(sbi, segno)->valid_blocks += del;
  1933. }
  1934. void f2fs_invalidate_blocks(struct f2fs_sb_info *sbi, block_t addr)
  1935. {
  1936. unsigned int segno = GET_SEGNO(sbi, addr);
  1937. struct sit_info *sit_i = SIT_I(sbi);
  1938. f2fs_bug_on(sbi, addr == NULL_ADDR);
  1939. if (addr == NEW_ADDR || addr == COMPRESS_ADDR)
  1940. return;
  1941. invalidate_mapping_pages(META_MAPPING(sbi), addr, addr);
  1942. f2fs_invalidate_compress_page(sbi, addr);
  1943. /* add it into sit main buffer */
  1944. down_write(&sit_i->sentry_lock);
  1945. update_segment_mtime(sbi, addr, 0);
  1946. update_sit_entry(sbi, addr, -1);
  1947. /* add it into dirty seglist */
  1948. locate_dirty_segment(sbi, segno);
  1949. up_write(&sit_i->sentry_lock);
  1950. }
  1951. bool f2fs_is_checkpointed_data(struct f2fs_sb_info *sbi, block_t blkaddr)
  1952. {
  1953. struct sit_info *sit_i = SIT_I(sbi);
  1954. unsigned int segno, offset;
  1955. struct seg_entry *se;
  1956. bool is_cp = false;
  1957. if (!__is_valid_data_blkaddr(blkaddr))
  1958. return true;
  1959. down_read(&sit_i->sentry_lock);
  1960. segno = GET_SEGNO(sbi, blkaddr);
  1961. se = get_seg_entry(sbi, segno);
  1962. offset = GET_BLKOFF_FROM_SEG0(sbi, blkaddr);
  1963. if (f2fs_test_bit(offset, se->ckpt_valid_map))
  1964. is_cp = true;
  1965. up_read(&sit_i->sentry_lock);
  1966. return is_cp;
  1967. }
  1968. /*
  1969. * This function should be resided under the curseg_mutex lock
  1970. */
  1971. static void __add_sum_entry(struct f2fs_sb_info *sbi, int type,
  1972. struct f2fs_summary *sum)
  1973. {
  1974. struct curseg_info *curseg = CURSEG_I(sbi, type);
  1975. void *addr = curseg->sum_blk;
  1976. addr += curseg->next_blkoff * sizeof(struct f2fs_summary);
  1977. memcpy(addr, sum, sizeof(struct f2fs_summary));
  1978. }
  1979. /*
  1980. * Calculate the number of current summary pages for writing
  1981. */
  1982. int f2fs_npages_for_summary_flush(struct f2fs_sb_info *sbi, bool for_ra)
  1983. {
  1984. int valid_sum_count = 0;
  1985. int i, sum_in_page;
  1986. for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
  1987. if (sbi->ckpt->alloc_type[i] == SSR)
  1988. valid_sum_count += sbi->blocks_per_seg;
  1989. else {
  1990. if (for_ra)
  1991. valid_sum_count += le16_to_cpu(
  1992. F2FS_CKPT(sbi)->cur_data_blkoff[i]);
  1993. else
  1994. valid_sum_count += curseg_blkoff(sbi, i);
  1995. }
  1996. }
  1997. sum_in_page = (PAGE_SIZE - 2 * SUM_JOURNAL_SIZE -
  1998. SUM_FOOTER_SIZE) / SUMMARY_SIZE;
  1999. if (valid_sum_count <= sum_in_page)
  2000. return 1;
  2001. else if ((valid_sum_count - sum_in_page) <=
  2002. (PAGE_SIZE - SUM_FOOTER_SIZE) / SUMMARY_SIZE)
  2003. return 2;
  2004. return 3;
  2005. }
  2006. /*
  2007. * Caller should put this summary page
  2008. */
  2009. struct page *f2fs_get_sum_page(struct f2fs_sb_info *sbi, unsigned int segno)
  2010. {
  2011. if (unlikely(f2fs_cp_error(sbi)))
  2012. return ERR_PTR(-EIO);
  2013. return f2fs_get_meta_page_retry(sbi, GET_SUM_BLOCK(sbi, segno));
  2014. }
  2015. void f2fs_update_meta_page(struct f2fs_sb_info *sbi,
  2016. void *src, block_t blk_addr)
  2017. {
  2018. struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
  2019. memcpy(page_address(page), src, PAGE_SIZE);
  2020. set_page_dirty(page);
  2021. f2fs_put_page(page, 1);
  2022. }
  2023. static void write_sum_page(struct f2fs_sb_info *sbi,
  2024. struct f2fs_summary_block *sum_blk, block_t blk_addr)
  2025. {
  2026. f2fs_update_meta_page(sbi, (void *)sum_blk, blk_addr);
  2027. }
  2028. static void write_current_sum_page(struct f2fs_sb_info *sbi,
  2029. int type, block_t blk_addr)
  2030. {
  2031. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2032. struct page *page = f2fs_grab_meta_page(sbi, blk_addr);
  2033. struct f2fs_summary_block *src = curseg->sum_blk;
  2034. struct f2fs_summary_block *dst;
  2035. dst = (struct f2fs_summary_block *)page_address(page);
  2036. memset(dst, 0, PAGE_SIZE);
  2037. mutex_lock(&curseg->curseg_mutex);
  2038. down_read(&curseg->journal_rwsem);
  2039. memcpy(&dst->journal, curseg->journal, SUM_JOURNAL_SIZE);
  2040. up_read(&curseg->journal_rwsem);
  2041. memcpy(dst->entries, src->entries, SUM_ENTRY_SIZE);
  2042. memcpy(&dst->footer, &src->footer, SUM_FOOTER_SIZE);
  2043. mutex_unlock(&curseg->curseg_mutex);
  2044. set_page_dirty(page);
  2045. f2fs_put_page(page, 1);
  2046. }
  2047. static int is_next_segment_free(struct f2fs_sb_info *sbi,
  2048. struct curseg_info *curseg, int type)
  2049. {
  2050. unsigned int segno = curseg->segno + 1;
  2051. struct free_segmap_info *free_i = FREE_I(sbi);
  2052. if (segno < MAIN_SEGS(sbi) && segno % sbi->segs_per_sec)
  2053. return !test_bit(segno, free_i->free_segmap);
  2054. return 0;
  2055. }
  2056. /*
  2057. * Find a new segment from the free segments bitmap to right order
  2058. * This function should be returned with success, otherwise BUG
  2059. */
  2060. static void get_new_segment(struct f2fs_sb_info *sbi,
  2061. unsigned int *newseg, bool new_sec, int dir)
  2062. {
  2063. struct free_segmap_info *free_i = FREE_I(sbi);
  2064. unsigned int segno, secno, zoneno;
  2065. unsigned int total_zones = MAIN_SECS(sbi) / sbi->secs_per_zone;
  2066. unsigned int hint = GET_SEC_FROM_SEG(sbi, *newseg);
  2067. unsigned int old_zoneno = GET_ZONE_FROM_SEG(sbi, *newseg);
  2068. unsigned int left_start = hint;
  2069. bool init = true;
  2070. int go_left = 0;
  2071. int i;
  2072. spin_lock(&free_i->segmap_lock);
  2073. if (!new_sec && ((*newseg + 1) % sbi->segs_per_sec)) {
  2074. segno = find_next_zero_bit(free_i->free_segmap,
  2075. GET_SEG_FROM_SEC(sbi, hint + 1), *newseg + 1);
  2076. if (segno < GET_SEG_FROM_SEC(sbi, hint + 1))
  2077. goto got_it;
  2078. }
  2079. find_other_zone:
  2080. secno = find_next_zero_bit(free_i->free_secmap, MAIN_SECS(sbi), hint);
  2081. if (secno >= MAIN_SECS(sbi)) {
  2082. if (dir == ALLOC_RIGHT) {
  2083. secno = find_next_zero_bit(free_i->free_secmap,
  2084. MAIN_SECS(sbi), 0);
  2085. f2fs_bug_on(sbi, secno >= MAIN_SECS(sbi));
  2086. } else {
  2087. go_left = 1;
  2088. left_start = hint - 1;
  2089. }
  2090. }
  2091. if (go_left == 0)
  2092. goto skip_left;
  2093. while (test_bit(left_start, free_i->free_secmap)) {
  2094. if (left_start > 0) {
  2095. left_start--;
  2096. continue;
  2097. }
  2098. left_start = find_next_zero_bit(free_i->free_secmap,
  2099. MAIN_SECS(sbi), 0);
  2100. f2fs_bug_on(sbi, left_start >= MAIN_SECS(sbi));
  2101. break;
  2102. }
  2103. secno = left_start;
  2104. skip_left:
  2105. segno = GET_SEG_FROM_SEC(sbi, secno);
  2106. zoneno = GET_ZONE_FROM_SEC(sbi, secno);
  2107. /* give up on finding another zone */
  2108. if (!init)
  2109. goto got_it;
  2110. if (sbi->secs_per_zone == 1)
  2111. goto got_it;
  2112. if (zoneno == old_zoneno)
  2113. goto got_it;
  2114. if (dir == ALLOC_LEFT) {
  2115. if (!go_left && zoneno + 1 >= total_zones)
  2116. goto got_it;
  2117. if (go_left && zoneno == 0)
  2118. goto got_it;
  2119. }
  2120. for (i = 0; i < NR_CURSEG_TYPE; i++)
  2121. if (CURSEG_I(sbi, i)->zone == zoneno)
  2122. break;
  2123. if (i < NR_CURSEG_TYPE) {
  2124. /* zone is in user, try another */
  2125. if (go_left)
  2126. hint = zoneno * sbi->secs_per_zone - 1;
  2127. else if (zoneno + 1 >= total_zones)
  2128. hint = 0;
  2129. else
  2130. hint = (zoneno + 1) * sbi->secs_per_zone;
  2131. init = false;
  2132. goto find_other_zone;
  2133. }
  2134. got_it:
  2135. /* set it as dirty segment in free segmap */
  2136. f2fs_bug_on(sbi, test_bit(segno, free_i->free_segmap));
  2137. __set_inuse(sbi, segno);
  2138. *newseg = segno;
  2139. spin_unlock(&free_i->segmap_lock);
  2140. }
  2141. static void reset_curseg(struct f2fs_sb_info *sbi, int type, int modified)
  2142. {
  2143. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2144. struct summary_footer *sum_footer;
  2145. unsigned short seg_type = curseg->seg_type;
  2146. curseg->inited = true;
  2147. curseg->segno = curseg->next_segno;
  2148. curseg->zone = GET_ZONE_FROM_SEG(sbi, curseg->segno);
  2149. curseg->next_blkoff = 0;
  2150. curseg->next_segno = NULL_SEGNO;
  2151. sum_footer = &(curseg->sum_blk->footer);
  2152. memset(sum_footer, 0, sizeof(struct summary_footer));
  2153. sanity_check_seg_type(sbi, seg_type);
  2154. if (IS_DATASEG(seg_type))
  2155. SET_SUM_TYPE(sum_footer, SUM_TYPE_DATA);
  2156. if (IS_NODESEG(seg_type))
  2157. SET_SUM_TYPE(sum_footer, SUM_TYPE_NODE);
  2158. __set_sit_entry_type(sbi, seg_type, curseg->segno, modified);
  2159. }
  2160. static unsigned int __get_next_segno(struct f2fs_sb_info *sbi, int type)
  2161. {
  2162. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2163. unsigned short seg_type = curseg->seg_type;
  2164. sanity_check_seg_type(sbi, seg_type);
  2165. /* if segs_per_sec is large than 1, we need to keep original policy. */
  2166. if (__is_large_section(sbi))
  2167. return curseg->segno;
  2168. /* inmem log may not locate on any segment after mount */
  2169. if (!curseg->inited)
  2170. return 0;
  2171. if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
  2172. return 0;
  2173. if (test_opt(sbi, NOHEAP) &&
  2174. (seg_type == CURSEG_HOT_DATA || IS_NODESEG(seg_type)))
  2175. return 0;
  2176. if (SIT_I(sbi)->last_victim[ALLOC_NEXT])
  2177. return SIT_I(sbi)->last_victim[ALLOC_NEXT];
  2178. /* find segments from 0 to reuse freed segments */
  2179. if (F2FS_OPTION(sbi).alloc_mode == ALLOC_MODE_REUSE)
  2180. return 0;
  2181. return curseg->segno;
  2182. }
  2183. /*
  2184. * Allocate a current working segment.
  2185. * This function always allocates a free segment in LFS manner.
  2186. */
  2187. static void new_curseg(struct f2fs_sb_info *sbi, int type, bool new_sec)
  2188. {
  2189. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2190. unsigned short seg_type = curseg->seg_type;
  2191. unsigned int segno = curseg->segno;
  2192. int dir = ALLOC_LEFT;
  2193. if (curseg->inited)
  2194. write_sum_page(sbi, curseg->sum_blk,
  2195. GET_SUM_BLOCK(sbi, segno));
  2196. if (seg_type == CURSEG_WARM_DATA || seg_type == CURSEG_COLD_DATA)
  2197. dir = ALLOC_RIGHT;
  2198. if (test_opt(sbi, NOHEAP))
  2199. dir = ALLOC_RIGHT;
  2200. segno = __get_next_segno(sbi, type);
  2201. get_new_segment(sbi, &segno, new_sec, dir);
  2202. curseg->next_segno = segno;
  2203. reset_curseg(sbi, type, 1);
  2204. curseg->alloc_type = LFS;
  2205. }
  2206. static int __next_free_blkoff(struct f2fs_sb_info *sbi,
  2207. int segno, block_t start)
  2208. {
  2209. struct seg_entry *se = get_seg_entry(sbi, segno);
  2210. int entries = SIT_VBLOCK_MAP_SIZE / sizeof(unsigned long);
  2211. unsigned long *target_map = SIT_I(sbi)->tmp_map;
  2212. unsigned long *ckpt_map = (unsigned long *)se->ckpt_valid_map;
  2213. unsigned long *cur_map = (unsigned long *)se->cur_valid_map;
  2214. int i;
  2215. for (i = 0; i < entries; i++)
  2216. target_map[i] = ckpt_map[i] | cur_map[i];
  2217. return __find_rev_next_zero_bit(target_map, sbi->blocks_per_seg, start);
  2218. }
  2219. /*
  2220. * If a segment is written by LFS manner, next block offset is just obtained
  2221. * by increasing the current block offset. However, if a segment is written by
  2222. * SSR manner, next block offset obtained by calling __next_free_blkoff
  2223. */
  2224. static void __refresh_next_blkoff(struct f2fs_sb_info *sbi,
  2225. struct curseg_info *seg)
  2226. {
  2227. if (seg->alloc_type == SSR)
  2228. seg->next_blkoff =
  2229. __next_free_blkoff(sbi, seg->segno,
  2230. seg->next_blkoff + 1);
  2231. else
  2232. seg->next_blkoff++;
  2233. }
  2234. bool f2fs_segment_has_free_slot(struct f2fs_sb_info *sbi, int segno)
  2235. {
  2236. return __next_free_blkoff(sbi, segno, 0) < sbi->blocks_per_seg;
  2237. }
  2238. /*
  2239. * This function always allocates a used segment(from dirty seglist) by SSR
  2240. * manner, so it should recover the existing segment information of valid blocks
  2241. */
  2242. static void change_curseg(struct f2fs_sb_info *sbi, int type, bool flush)
  2243. {
  2244. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  2245. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2246. unsigned int new_segno = curseg->next_segno;
  2247. struct f2fs_summary_block *sum_node;
  2248. struct page *sum_page;
  2249. if (flush)
  2250. write_sum_page(sbi, curseg->sum_blk,
  2251. GET_SUM_BLOCK(sbi, curseg->segno));
  2252. __set_test_and_inuse(sbi, new_segno);
  2253. mutex_lock(&dirty_i->seglist_lock);
  2254. __remove_dirty_segment(sbi, new_segno, PRE);
  2255. __remove_dirty_segment(sbi, new_segno, DIRTY);
  2256. mutex_unlock(&dirty_i->seglist_lock);
  2257. reset_curseg(sbi, type, 1);
  2258. curseg->alloc_type = SSR;
  2259. curseg->next_blkoff = __next_free_blkoff(sbi, curseg->segno, 0);
  2260. sum_page = f2fs_get_sum_page(sbi, new_segno);
  2261. if (IS_ERR(sum_page)) {
  2262. /* GC won't be able to use stale summary pages by cp_error */
  2263. memset(curseg->sum_blk, 0, SUM_ENTRY_SIZE);
  2264. return;
  2265. }
  2266. sum_node = (struct f2fs_summary_block *)page_address(sum_page);
  2267. memcpy(curseg->sum_blk, sum_node, SUM_ENTRY_SIZE);
  2268. f2fs_put_page(sum_page, 1);
  2269. }
  2270. static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
  2271. int alloc_mode, unsigned long long age);
  2272. static void get_atssr_segment(struct f2fs_sb_info *sbi, int type,
  2273. int target_type, int alloc_mode,
  2274. unsigned long long age)
  2275. {
  2276. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2277. curseg->seg_type = target_type;
  2278. if (get_ssr_segment(sbi, type, alloc_mode, age)) {
  2279. struct seg_entry *se = get_seg_entry(sbi, curseg->next_segno);
  2280. curseg->seg_type = se->type;
  2281. change_curseg(sbi, type, true);
  2282. } else {
  2283. /* allocate cold segment by default */
  2284. curseg->seg_type = CURSEG_COLD_DATA;
  2285. new_curseg(sbi, type, true);
  2286. }
  2287. stat_inc_seg_type(sbi, curseg);
  2288. }
  2289. static void __f2fs_init_atgc_curseg(struct f2fs_sb_info *sbi)
  2290. {
  2291. struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_ALL_DATA_ATGC);
  2292. if (!sbi->am.atgc_enabled)
  2293. return;
  2294. f2fs_down_read(&SM_I(sbi)->curseg_lock);
  2295. mutex_lock(&curseg->curseg_mutex);
  2296. down_write(&SIT_I(sbi)->sentry_lock);
  2297. get_atssr_segment(sbi, CURSEG_ALL_DATA_ATGC, CURSEG_COLD_DATA, SSR, 0);
  2298. up_write(&SIT_I(sbi)->sentry_lock);
  2299. mutex_unlock(&curseg->curseg_mutex);
  2300. f2fs_up_read(&SM_I(sbi)->curseg_lock);
  2301. }
  2302. void f2fs_init_inmem_curseg(struct f2fs_sb_info *sbi)
  2303. {
  2304. __f2fs_init_atgc_curseg(sbi);
  2305. }
  2306. static void __f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi, int type)
  2307. {
  2308. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2309. mutex_lock(&curseg->curseg_mutex);
  2310. if (!curseg->inited)
  2311. goto out;
  2312. if (get_valid_blocks(sbi, curseg->segno, false)) {
  2313. write_sum_page(sbi, curseg->sum_blk,
  2314. GET_SUM_BLOCK(sbi, curseg->segno));
  2315. } else {
  2316. mutex_lock(&DIRTY_I(sbi)->seglist_lock);
  2317. __set_test_and_free(sbi, curseg->segno, true);
  2318. mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
  2319. }
  2320. out:
  2321. mutex_unlock(&curseg->curseg_mutex);
  2322. }
  2323. void f2fs_save_inmem_curseg(struct f2fs_sb_info *sbi)
  2324. {
  2325. __f2fs_save_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
  2326. if (sbi->am.atgc_enabled)
  2327. __f2fs_save_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
  2328. }
  2329. static void __f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi, int type)
  2330. {
  2331. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2332. mutex_lock(&curseg->curseg_mutex);
  2333. if (!curseg->inited)
  2334. goto out;
  2335. if (get_valid_blocks(sbi, curseg->segno, false))
  2336. goto out;
  2337. mutex_lock(&DIRTY_I(sbi)->seglist_lock);
  2338. __set_test_and_inuse(sbi, curseg->segno);
  2339. mutex_unlock(&DIRTY_I(sbi)->seglist_lock);
  2340. out:
  2341. mutex_unlock(&curseg->curseg_mutex);
  2342. }
  2343. void f2fs_restore_inmem_curseg(struct f2fs_sb_info *sbi)
  2344. {
  2345. __f2fs_restore_inmem_curseg(sbi, CURSEG_COLD_DATA_PINNED);
  2346. if (sbi->am.atgc_enabled)
  2347. __f2fs_restore_inmem_curseg(sbi, CURSEG_ALL_DATA_ATGC);
  2348. }
  2349. static int get_ssr_segment(struct f2fs_sb_info *sbi, int type,
  2350. int alloc_mode, unsigned long long age)
  2351. {
  2352. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2353. const struct victim_selection *v_ops = DIRTY_I(sbi)->v_ops;
  2354. unsigned segno = NULL_SEGNO;
  2355. unsigned short seg_type = curseg->seg_type;
  2356. int i, cnt;
  2357. bool reversed = false;
  2358. sanity_check_seg_type(sbi, seg_type);
  2359. /* f2fs_need_SSR() already forces to do this */
  2360. if (!v_ops->get_victim(sbi, &segno, BG_GC, seg_type, alloc_mode, age)) {
  2361. curseg->next_segno = segno;
  2362. return 1;
  2363. }
  2364. /* For node segments, let's do SSR more intensively */
  2365. if (IS_NODESEG(seg_type)) {
  2366. if (seg_type >= CURSEG_WARM_NODE) {
  2367. reversed = true;
  2368. i = CURSEG_COLD_NODE;
  2369. } else {
  2370. i = CURSEG_HOT_NODE;
  2371. }
  2372. cnt = NR_CURSEG_NODE_TYPE;
  2373. } else {
  2374. if (seg_type >= CURSEG_WARM_DATA) {
  2375. reversed = true;
  2376. i = CURSEG_COLD_DATA;
  2377. } else {
  2378. i = CURSEG_HOT_DATA;
  2379. }
  2380. cnt = NR_CURSEG_DATA_TYPE;
  2381. }
  2382. for (; cnt-- > 0; reversed ? i-- : i++) {
  2383. if (i == seg_type)
  2384. continue;
  2385. if (!v_ops->get_victim(sbi, &segno, BG_GC, i, alloc_mode, age)) {
  2386. curseg->next_segno = segno;
  2387. return 1;
  2388. }
  2389. }
  2390. /* find valid_blocks=0 in dirty list */
  2391. if (unlikely(is_sbi_flag_set(sbi, SBI_CP_DISABLED))) {
  2392. segno = get_free_segment(sbi);
  2393. if (segno != NULL_SEGNO) {
  2394. curseg->next_segno = segno;
  2395. return 1;
  2396. }
  2397. }
  2398. return 0;
  2399. }
  2400. /*
  2401. * flush out current segment and replace it with new segment
  2402. * This function should be returned with success, otherwise BUG
  2403. */
  2404. static void allocate_segment_by_default(struct f2fs_sb_info *sbi,
  2405. int type, bool force)
  2406. {
  2407. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2408. if (force)
  2409. new_curseg(sbi, type, true);
  2410. else if (!is_set_ckpt_flags(sbi, CP_CRC_RECOVERY_FLAG) &&
  2411. curseg->seg_type == CURSEG_WARM_NODE)
  2412. new_curseg(sbi, type, false);
  2413. else if (curseg->alloc_type == LFS &&
  2414. is_next_segment_free(sbi, curseg, type) &&
  2415. likely(!is_sbi_flag_set(sbi, SBI_CP_DISABLED)))
  2416. new_curseg(sbi, type, false);
  2417. else if (f2fs_need_SSR(sbi) &&
  2418. get_ssr_segment(sbi, type, SSR, 0))
  2419. change_curseg(sbi, type, true);
  2420. else
  2421. new_curseg(sbi, type, false);
  2422. stat_inc_seg_type(sbi, curseg);
  2423. }
  2424. void f2fs_allocate_segment_for_resize(struct f2fs_sb_info *sbi, int type,
  2425. unsigned int start, unsigned int end)
  2426. {
  2427. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2428. unsigned int segno;
  2429. f2fs_down_read(&SM_I(sbi)->curseg_lock);
  2430. mutex_lock(&curseg->curseg_mutex);
  2431. down_write(&SIT_I(sbi)->sentry_lock);
  2432. segno = CURSEG_I(sbi, type)->segno;
  2433. if (segno < start || segno > end)
  2434. goto unlock;
  2435. if (f2fs_need_SSR(sbi) && get_ssr_segment(sbi, type, SSR, 0))
  2436. change_curseg(sbi, type, true);
  2437. else
  2438. new_curseg(sbi, type, true);
  2439. stat_inc_seg_type(sbi, curseg);
  2440. locate_dirty_segment(sbi, segno);
  2441. unlock:
  2442. up_write(&SIT_I(sbi)->sentry_lock);
  2443. if (segno != curseg->segno)
  2444. f2fs_notice(sbi, "For resize: curseg of type %d: %u ==> %u",
  2445. type, segno, curseg->segno);
  2446. mutex_unlock(&curseg->curseg_mutex);
  2447. f2fs_up_read(&SM_I(sbi)->curseg_lock);
  2448. }
  2449. static void __allocate_new_segment(struct f2fs_sb_info *sbi, int type,
  2450. bool new_sec, bool force)
  2451. {
  2452. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2453. unsigned int old_segno;
  2454. if (!curseg->inited)
  2455. goto alloc;
  2456. if (force || curseg->next_blkoff ||
  2457. get_valid_blocks(sbi, curseg->segno, new_sec))
  2458. goto alloc;
  2459. if (!get_ckpt_valid_blocks(sbi, curseg->segno, new_sec))
  2460. return;
  2461. alloc:
  2462. old_segno = curseg->segno;
  2463. SIT_I(sbi)->s_ops->allocate_segment(sbi, type, true);
  2464. locate_dirty_segment(sbi, old_segno);
  2465. }
  2466. static void __allocate_new_section(struct f2fs_sb_info *sbi,
  2467. int type, bool force)
  2468. {
  2469. __allocate_new_segment(sbi, type, true, force);
  2470. }
  2471. void f2fs_allocate_new_section(struct f2fs_sb_info *sbi, int type, bool force)
  2472. {
  2473. f2fs_down_read(&SM_I(sbi)->curseg_lock);
  2474. down_write(&SIT_I(sbi)->sentry_lock);
  2475. __allocate_new_section(sbi, type, force);
  2476. up_write(&SIT_I(sbi)->sentry_lock);
  2477. f2fs_up_read(&SM_I(sbi)->curseg_lock);
  2478. }
  2479. void f2fs_allocate_new_segments(struct f2fs_sb_info *sbi)
  2480. {
  2481. int i;
  2482. f2fs_down_read(&SM_I(sbi)->curseg_lock);
  2483. down_write(&SIT_I(sbi)->sentry_lock);
  2484. for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++)
  2485. __allocate_new_segment(sbi, i, false, false);
  2486. up_write(&SIT_I(sbi)->sentry_lock);
  2487. f2fs_up_read(&SM_I(sbi)->curseg_lock);
  2488. }
  2489. static const struct segment_allocation default_salloc_ops = {
  2490. .allocate_segment = allocate_segment_by_default,
  2491. };
  2492. bool f2fs_exist_trim_candidates(struct f2fs_sb_info *sbi,
  2493. struct cp_control *cpc)
  2494. {
  2495. __u64 trim_start = cpc->trim_start;
  2496. bool has_candidate = false;
  2497. down_write(&SIT_I(sbi)->sentry_lock);
  2498. for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++) {
  2499. if (add_discard_addrs(sbi, cpc, true)) {
  2500. has_candidate = true;
  2501. break;
  2502. }
  2503. }
  2504. up_write(&SIT_I(sbi)->sentry_lock);
  2505. cpc->trim_start = trim_start;
  2506. return has_candidate;
  2507. }
  2508. static unsigned int __issue_discard_cmd_range(struct f2fs_sb_info *sbi,
  2509. struct discard_policy *dpolicy,
  2510. unsigned int start, unsigned int end)
  2511. {
  2512. struct discard_cmd_control *dcc = SM_I(sbi)->dcc_info;
  2513. struct discard_cmd *prev_dc = NULL, *next_dc = NULL;
  2514. struct rb_node **insert_p = NULL, *insert_parent = NULL;
  2515. struct discard_cmd *dc;
  2516. struct blk_plug plug;
  2517. int issued;
  2518. unsigned int trimmed = 0;
  2519. next:
  2520. issued = 0;
  2521. mutex_lock(&dcc->cmd_lock);
  2522. if (unlikely(dcc->rbtree_check))
  2523. f2fs_bug_on(sbi, !f2fs_check_rb_tree_consistence(sbi,
  2524. &dcc->root, false));
  2525. dc = (struct discard_cmd *)f2fs_lookup_rb_tree_ret(&dcc->root,
  2526. NULL, start,
  2527. (struct rb_entry **)&prev_dc,
  2528. (struct rb_entry **)&next_dc,
  2529. &insert_p, &insert_parent, true, NULL);
  2530. if (!dc)
  2531. dc = next_dc;
  2532. blk_start_plug(&plug);
  2533. while (dc && dc->lstart <= end) {
  2534. struct rb_node *node;
  2535. int err = 0;
  2536. if (dc->len < dpolicy->granularity)
  2537. goto skip;
  2538. if (dc->state != D_PREP) {
  2539. list_move_tail(&dc->list, &dcc->fstrim_list);
  2540. goto skip;
  2541. }
  2542. err = __submit_discard_cmd(sbi, dpolicy, dc, &issued);
  2543. if (issued >= dpolicy->max_requests) {
  2544. start = dc->lstart + dc->len;
  2545. if (err)
  2546. __remove_discard_cmd(sbi, dc);
  2547. blk_finish_plug(&plug);
  2548. mutex_unlock(&dcc->cmd_lock);
  2549. trimmed += __wait_all_discard_cmd(sbi, NULL);
  2550. congestion_wait(BLK_RW_ASYNC, DEFAULT_IO_TIMEOUT);
  2551. goto next;
  2552. }
  2553. skip:
  2554. node = rb_next(&dc->rb_node);
  2555. if (err)
  2556. __remove_discard_cmd(sbi, dc);
  2557. dc = rb_entry_safe(node, struct discard_cmd, rb_node);
  2558. if (fatal_signal_pending(current))
  2559. break;
  2560. }
  2561. blk_finish_plug(&plug);
  2562. mutex_unlock(&dcc->cmd_lock);
  2563. return trimmed;
  2564. }
  2565. int f2fs_trim_fs(struct f2fs_sb_info *sbi, struct fstrim_range *range)
  2566. {
  2567. __u64 start = F2FS_BYTES_TO_BLK(range->start);
  2568. __u64 end = start + F2FS_BYTES_TO_BLK(range->len) - 1;
  2569. unsigned int start_segno, end_segno;
  2570. block_t start_block, end_block;
  2571. struct cp_control cpc;
  2572. struct discard_policy dpolicy;
  2573. unsigned long long trimmed = 0;
  2574. int err = 0;
  2575. bool need_align = f2fs_lfs_mode(sbi) && __is_large_section(sbi);
  2576. if (start >= MAX_BLKADDR(sbi) || range->len < sbi->blocksize)
  2577. return -EINVAL;
  2578. if (end < MAIN_BLKADDR(sbi))
  2579. goto out;
  2580. if (is_sbi_flag_set(sbi, SBI_NEED_FSCK)) {
  2581. f2fs_warn(sbi, "Found FS corruption, run fsck to fix.");
  2582. return -EFSCORRUPTED;
  2583. }
  2584. /* start/end segment number in main_area */
  2585. start_segno = (start <= MAIN_BLKADDR(sbi)) ? 0 : GET_SEGNO(sbi, start);
  2586. end_segno = (end >= MAX_BLKADDR(sbi)) ? MAIN_SEGS(sbi) - 1 :
  2587. GET_SEGNO(sbi, end);
  2588. if (need_align) {
  2589. start_segno = rounddown(start_segno, sbi->segs_per_sec);
  2590. end_segno = roundup(end_segno + 1, sbi->segs_per_sec) - 1;
  2591. }
  2592. cpc.reason = CP_DISCARD;
  2593. cpc.trim_minlen = max_t(__u64, 1, F2FS_BYTES_TO_BLK(range->minlen));
  2594. cpc.trim_start = start_segno;
  2595. cpc.trim_end = end_segno;
  2596. if (sbi->discard_blks == 0)
  2597. goto out;
  2598. f2fs_down_write(&sbi->gc_lock);
  2599. err = f2fs_write_checkpoint(sbi, &cpc);
  2600. f2fs_up_write(&sbi->gc_lock);
  2601. if (err)
  2602. goto out;
  2603. /*
  2604. * We filed discard candidates, but actually we don't need to wait for
  2605. * all of them, since they'll be issued in idle time along with runtime
  2606. * discard option. User configuration looks like using runtime discard
  2607. * or periodic fstrim instead of it.
  2608. */
  2609. if (f2fs_realtime_discard_enable(sbi))
  2610. goto out;
  2611. start_block = START_BLOCK(sbi, start_segno);
  2612. end_block = START_BLOCK(sbi, end_segno + 1);
  2613. __init_discard_policy(sbi, &dpolicy, DPOLICY_FSTRIM, cpc.trim_minlen);
  2614. trimmed = __issue_discard_cmd_range(sbi, &dpolicy,
  2615. start_block, end_block);
  2616. trimmed += __wait_discard_cmd_range(sbi, &dpolicy,
  2617. start_block, end_block);
  2618. out:
  2619. if (!err)
  2620. range->len = F2FS_BLK_TO_BYTES(trimmed);
  2621. return err;
  2622. }
  2623. static bool __has_curseg_space(struct f2fs_sb_info *sbi,
  2624. struct curseg_info *curseg)
  2625. {
  2626. return curseg->next_blkoff < f2fs_usable_blks_in_seg(sbi,
  2627. curseg->segno);
  2628. }
  2629. int f2fs_rw_hint_to_seg_type(enum rw_hint hint)
  2630. {
  2631. switch (hint) {
  2632. case WRITE_LIFE_SHORT:
  2633. return CURSEG_HOT_DATA;
  2634. case WRITE_LIFE_EXTREME:
  2635. return CURSEG_COLD_DATA;
  2636. default:
  2637. return CURSEG_WARM_DATA;
  2638. }
  2639. }
  2640. /* This returns write hints for each segment type. This hints will be
  2641. * passed down to block layer. There are mapping tables which depend on
  2642. * the mount option 'whint_mode'.
  2643. *
  2644. * 1) whint_mode=off. F2FS only passes down WRITE_LIFE_NOT_SET.
  2645. *
  2646. * 2) whint_mode=user-based. F2FS tries to pass down hints given by users.
  2647. *
  2648. * User F2FS Block
  2649. * ---- ---- -----
  2650. * META WRITE_LIFE_NOT_SET
  2651. * HOT_NODE "
  2652. * WARM_NODE "
  2653. * COLD_NODE "
  2654. * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
  2655. * extension list " "
  2656. *
  2657. * -- buffered io
  2658. * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
  2659. * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
  2660. * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
  2661. * WRITE_LIFE_NONE " "
  2662. * WRITE_LIFE_MEDIUM " "
  2663. * WRITE_LIFE_LONG " "
  2664. *
  2665. * -- direct io
  2666. * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
  2667. * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
  2668. * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
  2669. * WRITE_LIFE_NONE " WRITE_LIFE_NONE
  2670. * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
  2671. * WRITE_LIFE_LONG " WRITE_LIFE_LONG
  2672. *
  2673. * 3) whint_mode=fs-based. F2FS passes down hints with its policy.
  2674. *
  2675. * User F2FS Block
  2676. * ---- ---- -----
  2677. * META WRITE_LIFE_MEDIUM;
  2678. * HOT_NODE WRITE_LIFE_NOT_SET
  2679. * WARM_NODE "
  2680. * COLD_NODE WRITE_LIFE_NONE
  2681. * ioctl(COLD) COLD_DATA WRITE_LIFE_EXTREME
  2682. * extension list " "
  2683. *
  2684. * -- buffered io
  2685. * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
  2686. * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
  2687. * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_LONG
  2688. * WRITE_LIFE_NONE " "
  2689. * WRITE_LIFE_MEDIUM " "
  2690. * WRITE_LIFE_LONG " "
  2691. *
  2692. * -- direct io
  2693. * WRITE_LIFE_EXTREME COLD_DATA WRITE_LIFE_EXTREME
  2694. * WRITE_LIFE_SHORT HOT_DATA WRITE_LIFE_SHORT
  2695. * WRITE_LIFE_NOT_SET WARM_DATA WRITE_LIFE_NOT_SET
  2696. * WRITE_LIFE_NONE " WRITE_LIFE_NONE
  2697. * WRITE_LIFE_MEDIUM " WRITE_LIFE_MEDIUM
  2698. * WRITE_LIFE_LONG " WRITE_LIFE_LONG
  2699. */
  2700. enum rw_hint f2fs_io_type_to_rw_hint(struct f2fs_sb_info *sbi,
  2701. enum page_type type, enum temp_type temp)
  2702. {
  2703. if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_USER) {
  2704. if (type == DATA) {
  2705. if (temp == WARM)
  2706. return WRITE_LIFE_NOT_SET;
  2707. else if (temp == HOT)
  2708. return WRITE_LIFE_SHORT;
  2709. else if (temp == COLD)
  2710. return WRITE_LIFE_EXTREME;
  2711. } else {
  2712. return WRITE_LIFE_NOT_SET;
  2713. }
  2714. } else if (F2FS_OPTION(sbi).whint_mode == WHINT_MODE_FS) {
  2715. if (type == DATA) {
  2716. if (temp == WARM)
  2717. return WRITE_LIFE_LONG;
  2718. else if (temp == HOT)
  2719. return WRITE_LIFE_SHORT;
  2720. else if (temp == COLD)
  2721. return WRITE_LIFE_EXTREME;
  2722. } else if (type == NODE) {
  2723. if (temp == WARM || temp == HOT)
  2724. return WRITE_LIFE_NOT_SET;
  2725. else if (temp == COLD)
  2726. return WRITE_LIFE_NONE;
  2727. } else if (type == META) {
  2728. return WRITE_LIFE_MEDIUM;
  2729. }
  2730. }
  2731. return WRITE_LIFE_NOT_SET;
  2732. }
  2733. static int __get_segment_type_2(struct f2fs_io_info *fio)
  2734. {
  2735. if (fio->type == DATA)
  2736. return CURSEG_HOT_DATA;
  2737. else
  2738. return CURSEG_HOT_NODE;
  2739. }
  2740. static int __get_segment_type_4(struct f2fs_io_info *fio)
  2741. {
  2742. if (fio->type == DATA) {
  2743. struct inode *inode = fio->page->mapping->host;
  2744. if (S_ISDIR(inode->i_mode))
  2745. return CURSEG_HOT_DATA;
  2746. else
  2747. return CURSEG_COLD_DATA;
  2748. } else {
  2749. if (IS_DNODE(fio->page) && is_cold_node(fio->page))
  2750. return CURSEG_WARM_NODE;
  2751. else
  2752. return CURSEG_COLD_NODE;
  2753. }
  2754. }
  2755. static int __get_segment_type_6(struct f2fs_io_info *fio)
  2756. {
  2757. if (fio->type == DATA) {
  2758. struct inode *inode = fio->page->mapping->host;
  2759. if (is_inode_flag_set(inode, FI_ALIGNED_WRITE))
  2760. return CURSEG_COLD_DATA_PINNED;
  2761. if (page_private_gcing(fio->page)) {
  2762. if (fio->sbi->am.atgc_enabled &&
  2763. (fio->io_type == FS_DATA_IO) &&
  2764. (fio->sbi->gc_mode != GC_URGENT_HIGH))
  2765. return CURSEG_ALL_DATA_ATGC;
  2766. else
  2767. return CURSEG_COLD_DATA;
  2768. }
  2769. if (file_is_cold(inode) || f2fs_need_compress_data(inode))
  2770. return CURSEG_COLD_DATA;
  2771. if (file_is_hot(inode) ||
  2772. is_inode_flag_set(inode, FI_HOT_DATA) ||
  2773. f2fs_is_atomic_file(inode) ||
  2774. f2fs_is_volatile_file(inode))
  2775. return CURSEG_HOT_DATA;
  2776. return f2fs_rw_hint_to_seg_type(inode->i_write_hint);
  2777. } else {
  2778. if (IS_DNODE(fio->page))
  2779. return is_cold_node(fio->page) ? CURSEG_WARM_NODE :
  2780. CURSEG_HOT_NODE;
  2781. return CURSEG_COLD_NODE;
  2782. }
  2783. }
  2784. static int __get_segment_type(struct f2fs_io_info *fio)
  2785. {
  2786. int type = 0;
  2787. switch (F2FS_OPTION(fio->sbi).active_logs) {
  2788. case 2:
  2789. type = __get_segment_type_2(fio);
  2790. break;
  2791. case 4:
  2792. type = __get_segment_type_4(fio);
  2793. break;
  2794. case 6:
  2795. type = __get_segment_type_6(fio);
  2796. break;
  2797. default:
  2798. f2fs_bug_on(fio->sbi, true);
  2799. }
  2800. if (IS_HOT(type))
  2801. fio->temp = HOT;
  2802. else if (IS_WARM(type))
  2803. fio->temp = WARM;
  2804. else
  2805. fio->temp = COLD;
  2806. return type;
  2807. }
  2808. void f2fs_allocate_data_block(struct f2fs_sb_info *sbi, struct page *page,
  2809. block_t old_blkaddr, block_t *new_blkaddr,
  2810. struct f2fs_summary *sum, int type,
  2811. struct f2fs_io_info *fio)
  2812. {
  2813. struct sit_info *sit_i = SIT_I(sbi);
  2814. struct curseg_info *curseg = CURSEG_I(sbi, type);
  2815. unsigned long long old_mtime;
  2816. bool from_gc = (type == CURSEG_ALL_DATA_ATGC);
  2817. struct seg_entry *se = NULL;
  2818. f2fs_down_read(&SM_I(sbi)->curseg_lock);
  2819. mutex_lock(&curseg->curseg_mutex);
  2820. down_write(&sit_i->sentry_lock);
  2821. if (from_gc) {
  2822. f2fs_bug_on(sbi, GET_SEGNO(sbi, old_blkaddr) == NULL_SEGNO);
  2823. se = get_seg_entry(sbi, GET_SEGNO(sbi, old_blkaddr));
  2824. sanity_check_seg_type(sbi, se->type);
  2825. f2fs_bug_on(sbi, IS_NODESEG(se->type));
  2826. }
  2827. *new_blkaddr = NEXT_FREE_BLKADDR(sbi, curseg);
  2828. f2fs_bug_on(sbi, curseg->next_blkoff >= sbi->blocks_per_seg);
  2829. f2fs_wait_discard_bio(sbi, *new_blkaddr);
  2830. /*
  2831. * __add_sum_entry should be resided under the curseg_mutex
  2832. * because, this function updates a summary entry in the
  2833. * current summary block.
  2834. */
  2835. __add_sum_entry(sbi, type, sum);
  2836. __refresh_next_blkoff(sbi, curseg);
  2837. stat_inc_block_count(sbi, curseg);
  2838. if (from_gc) {
  2839. old_mtime = get_segment_mtime(sbi, old_blkaddr);
  2840. } else {
  2841. update_segment_mtime(sbi, old_blkaddr, 0);
  2842. old_mtime = 0;
  2843. }
  2844. update_segment_mtime(sbi, *new_blkaddr, old_mtime);
  2845. /*
  2846. * SIT information should be updated before segment allocation,
  2847. * since SSR needs latest valid block information.
  2848. */
  2849. update_sit_entry(sbi, *new_blkaddr, 1);
  2850. if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO)
  2851. update_sit_entry(sbi, old_blkaddr, -1);
  2852. if (!__has_curseg_space(sbi, curseg)) {
  2853. if (from_gc)
  2854. get_atssr_segment(sbi, type, se->type,
  2855. AT_SSR, se->mtime);
  2856. else
  2857. sit_i->s_ops->allocate_segment(sbi, type, false);
  2858. }
  2859. /*
  2860. * segment dirty status should be updated after segment allocation,
  2861. * so we just need to update status only one time after previous
  2862. * segment being closed.
  2863. */
  2864. locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
  2865. locate_dirty_segment(sbi, GET_SEGNO(sbi, *new_blkaddr));
  2866. up_write(&sit_i->sentry_lock);
  2867. if (page && IS_NODESEG(type)) {
  2868. fill_node_footer_blkaddr(page, NEXT_FREE_BLKADDR(sbi, curseg));
  2869. f2fs_inode_chksum_set(sbi, page);
  2870. }
  2871. if (fio) {
  2872. struct f2fs_bio_info *io;
  2873. if (F2FS_IO_ALIGNED(sbi))
  2874. fio->retry = false;
  2875. INIT_LIST_HEAD(&fio->list);
  2876. fio->in_list = true;
  2877. io = sbi->write_io[fio->type] + fio->temp;
  2878. spin_lock(&io->io_lock);
  2879. list_add_tail(&fio->list, &io->io_list);
  2880. spin_unlock(&io->io_lock);
  2881. }
  2882. mutex_unlock(&curseg->curseg_mutex);
  2883. f2fs_up_read(&SM_I(sbi)->curseg_lock);
  2884. }
  2885. static void update_device_state(struct f2fs_io_info *fio)
  2886. {
  2887. struct f2fs_sb_info *sbi = fio->sbi;
  2888. unsigned int devidx;
  2889. if (!f2fs_is_multi_device(sbi))
  2890. return;
  2891. devidx = f2fs_target_device_index(sbi, fio->new_blkaddr);
  2892. /* update device state for fsync */
  2893. f2fs_set_dirty_device(sbi, fio->ino, devidx, FLUSH_INO);
  2894. /* update device state for checkpoint */
  2895. if (!f2fs_test_bit(devidx, (char *)&sbi->dirty_device)) {
  2896. spin_lock(&sbi->dev_lock);
  2897. f2fs_set_bit(devidx, (char *)&sbi->dirty_device);
  2898. spin_unlock(&sbi->dev_lock);
  2899. }
  2900. }
  2901. static void do_write_page(struct f2fs_summary *sum, struct f2fs_io_info *fio)
  2902. {
  2903. int type = __get_segment_type(fio);
  2904. bool keep_order = (f2fs_lfs_mode(fio->sbi) && type == CURSEG_COLD_DATA);
  2905. if (keep_order)
  2906. f2fs_down_read(&fio->sbi->io_order_lock);
  2907. reallocate:
  2908. f2fs_allocate_data_block(fio->sbi, fio->page, fio->old_blkaddr,
  2909. &fio->new_blkaddr, sum, type, fio);
  2910. if (GET_SEGNO(fio->sbi, fio->old_blkaddr) != NULL_SEGNO) {
  2911. invalidate_mapping_pages(META_MAPPING(fio->sbi),
  2912. fio->old_blkaddr, fio->old_blkaddr);
  2913. f2fs_invalidate_compress_page(fio->sbi, fio->old_blkaddr);
  2914. }
  2915. /* writeout dirty page into bdev */
  2916. f2fs_submit_page_write(fio);
  2917. if (fio->retry) {
  2918. fio->old_blkaddr = fio->new_blkaddr;
  2919. goto reallocate;
  2920. }
  2921. update_device_state(fio);
  2922. if (keep_order)
  2923. f2fs_up_read(&fio->sbi->io_order_lock);
  2924. }
  2925. void f2fs_do_write_meta_page(struct f2fs_sb_info *sbi, struct page *page,
  2926. enum iostat_type io_type)
  2927. {
  2928. struct f2fs_io_info fio = {
  2929. .sbi = sbi,
  2930. .type = META,
  2931. .temp = HOT,
  2932. .op = REQ_OP_WRITE,
  2933. .op_flags = REQ_SYNC | REQ_META | REQ_PRIO,
  2934. .old_blkaddr = page->index,
  2935. .new_blkaddr = page->index,
  2936. .page = page,
  2937. .encrypted_page = NULL,
  2938. .in_list = false,
  2939. };
  2940. if (unlikely(page->index >= MAIN_BLKADDR(sbi)))
  2941. fio.op_flags &= ~REQ_META;
  2942. set_page_writeback(page);
  2943. ClearPageError(page);
  2944. f2fs_submit_page_write(&fio);
  2945. stat_inc_meta_count(sbi, page->index);
  2946. f2fs_update_iostat(sbi, io_type, F2FS_BLKSIZE);
  2947. }
  2948. void f2fs_do_write_node_page(unsigned int nid, struct f2fs_io_info *fio)
  2949. {
  2950. struct f2fs_summary sum;
  2951. set_summary(&sum, nid, 0, 0);
  2952. do_write_page(&sum, fio);
  2953. f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
  2954. }
  2955. void f2fs_outplace_write_data(struct dnode_of_data *dn,
  2956. struct f2fs_io_info *fio)
  2957. {
  2958. struct f2fs_sb_info *sbi = fio->sbi;
  2959. struct f2fs_summary sum;
  2960. f2fs_bug_on(sbi, dn->data_blkaddr == NULL_ADDR);
  2961. set_summary(&sum, dn->nid, dn->ofs_in_node, fio->version);
  2962. do_write_page(&sum, fio);
  2963. f2fs_update_data_blkaddr(dn, fio->new_blkaddr);
  2964. f2fs_update_iostat(sbi, fio->io_type, F2FS_BLKSIZE);
  2965. }
  2966. int f2fs_inplace_write_data(struct f2fs_io_info *fio)
  2967. {
  2968. int err;
  2969. struct f2fs_sb_info *sbi = fio->sbi;
  2970. unsigned int segno;
  2971. fio->new_blkaddr = fio->old_blkaddr;
  2972. /* i/o temperature is needed for passing down write hints */
  2973. __get_segment_type(fio);
  2974. segno = GET_SEGNO(sbi, fio->new_blkaddr);
  2975. if (!IS_DATASEG(get_seg_entry(sbi, segno)->type)) {
  2976. set_sbi_flag(sbi, SBI_NEED_FSCK);
  2977. f2fs_warn(sbi, "%s: incorrect segment(%u) type, run fsck to fix.",
  2978. __func__, segno);
  2979. err = -EFSCORRUPTED;
  2980. goto drop_bio;
  2981. }
  2982. if (f2fs_cp_error(sbi)) {
  2983. err = -EIO;
  2984. goto drop_bio;
  2985. }
  2986. stat_inc_inplace_blocks(fio->sbi);
  2987. if (fio->bio && !(SM_I(sbi)->ipu_policy & (1 << F2FS_IPU_NOCACHE)))
  2988. err = f2fs_merge_page_bio(fio);
  2989. else
  2990. err = f2fs_submit_page_bio(fio);
  2991. if (!err) {
  2992. update_device_state(fio);
  2993. f2fs_update_iostat(fio->sbi, fio->io_type, F2FS_BLKSIZE);
  2994. }
  2995. return err;
  2996. drop_bio:
  2997. if (fio->bio && *(fio->bio)) {
  2998. struct bio *bio = *(fio->bio);
  2999. bio->bi_status = BLK_STS_IOERR;
  3000. bio_endio(bio);
  3001. *(fio->bio) = NULL;
  3002. }
  3003. return err;
  3004. }
  3005. static inline int __f2fs_get_curseg(struct f2fs_sb_info *sbi,
  3006. unsigned int segno)
  3007. {
  3008. int i;
  3009. for (i = CURSEG_HOT_DATA; i < NO_CHECK_TYPE; i++) {
  3010. if (CURSEG_I(sbi, i)->segno == segno)
  3011. break;
  3012. }
  3013. return i;
  3014. }
  3015. void f2fs_do_replace_block(struct f2fs_sb_info *sbi, struct f2fs_summary *sum,
  3016. block_t old_blkaddr, block_t new_blkaddr,
  3017. bool recover_curseg, bool recover_newaddr,
  3018. bool from_gc)
  3019. {
  3020. struct sit_info *sit_i = SIT_I(sbi);
  3021. struct curseg_info *curseg;
  3022. unsigned int segno, old_cursegno;
  3023. struct seg_entry *se;
  3024. int type;
  3025. unsigned short old_blkoff;
  3026. unsigned char old_alloc_type;
  3027. segno = GET_SEGNO(sbi, new_blkaddr);
  3028. se = get_seg_entry(sbi, segno);
  3029. type = se->type;
  3030. f2fs_down_write(&SM_I(sbi)->curseg_lock);
  3031. if (!recover_curseg) {
  3032. /* for recovery flow */
  3033. if (se->valid_blocks == 0 && !IS_CURSEG(sbi, segno)) {
  3034. if (old_blkaddr == NULL_ADDR)
  3035. type = CURSEG_COLD_DATA;
  3036. else
  3037. type = CURSEG_WARM_DATA;
  3038. }
  3039. } else {
  3040. if (IS_CURSEG(sbi, segno)) {
  3041. /* se->type is volatile as SSR allocation */
  3042. type = __f2fs_get_curseg(sbi, segno);
  3043. f2fs_bug_on(sbi, type == NO_CHECK_TYPE);
  3044. } else {
  3045. type = CURSEG_WARM_DATA;
  3046. }
  3047. }
  3048. f2fs_bug_on(sbi, !IS_DATASEG(type));
  3049. curseg = CURSEG_I(sbi, type);
  3050. mutex_lock(&curseg->curseg_mutex);
  3051. down_write(&sit_i->sentry_lock);
  3052. old_cursegno = curseg->segno;
  3053. old_blkoff = curseg->next_blkoff;
  3054. old_alloc_type = curseg->alloc_type;
  3055. /* change the current segment */
  3056. if (segno != curseg->segno) {
  3057. curseg->next_segno = segno;
  3058. change_curseg(sbi, type, true);
  3059. }
  3060. curseg->next_blkoff = GET_BLKOFF_FROM_SEG0(sbi, new_blkaddr);
  3061. __add_sum_entry(sbi, type, sum);
  3062. if (!recover_curseg || recover_newaddr) {
  3063. if (!from_gc)
  3064. update_segment_mtime(sbi, new_blkaddr, 0);
  3065. update_sit_entry(sbi, new_blkaddr, 1);
  3066. }
  3067. if (GET_SEGNO(sbi, old_blkaddr) != NULL_SEGNO) {
  3068. invalidate_mapping_pages(META_MAPPING(sbi),
  3069. old_blkaddr, old_blkaddr);
  3070. f2fs_invalidate_compress_page(sbi, old_blkaddr);
  3071. if (!from_gc)
  3072. update_segment_mtime(sbi, old_blkaddr, 0);
  3073. update_sit_entry(sbi, old_blkaddr, -1);
  3074. }
  3075. locate_dirty_segment(sbi, GET_SEGNO(sbi, old_blkaddr));
  3076. locate_dirty_segment(sbi, GET_SEGNO(sbi, new_blkaddr));
  3077. locate_dirty_segment(sbi, old_cursegno);
  3078. if (recover_curseg) {
  3079. if (old_cursegno != curseg->segno) {
  3080. curseg->next_segno = old_cursegno;
  3081. change_curseg(sbi, type, true);
  3082. }
  3083. curseg->next_blkoff = old_blkoff;
  3084. curseg->alloc_type = old_alloc_type;
  3085. }
  3086. up_write(&sit_i->sentry_lock);
  3087. mutex_unlock(&curseg->curseg_mutex);
  3088. f2fs_up_write(&SM_I(sbi)->curseg_lock);
  3089. }
  3090. void f2fs_replace_block(struct f2fs_sb_info *sbi, struct dnode_of_data *dn,
  3091. block_t old_addr, block_t new_addr,
  3092. unsigned char version, bool recover_curseg,
  3093. bool recover_newaddr)
  3094. {
  3095. struct f2fs_summary sum;
  3096. set_summary(&sum, dn->nid, dn->ofs_in_node, version);
  3097. f2fs_do_replace_block(sbi, &sum, old_addr, new_addr,
  3098. recover_curseg, recover_newaddr, false);
  3099. f2fs_update_data_blkaddr(dn, new_addr);
  3100. }
  3101. void f2fs_wait_on_page_writeback(struct page *page,
  3102. enum page_type type, bool ordered, bool locked)
  3103. {
  3104. if (PageWriteback(page)) {
  3105. struct f2fs_sb_info *sbi = F2FS_P_SB(page);
  3106. /* submit cached LFS IO */
  3107. f2fs_submit_merged_write_cond(sbi, NULL, page, 0, type);
  3108. /* sbumit cached IPU IO */
  3109. f2fs_submit_merged_ipu_write(sbi, NULL, page);
  3110. if (ordered) {
  3111. wait_on_page_writeback(page);
  3112. f2fs_bug_on(sbi, locked && PageWriteback(page));
  3113. } else {
  3114. wait_for_stable_page(page);
  3115. }
  3116. }
  3117. }
  3118. void f2fs_wait_on_block_writeback(struct inode *inode, block_t blkaddr)
  3119. {
  3120. struct f2fs_sb_info *sbi = F2FS_I_SB(inode);
  3121. struct page *cpage;
  3122. if (!f2fs_post_read_required(inode))
  3123. return;
  3124. if (!__is_valid_data_blkaddr(blkaddr))
  3125. return;
  3126. cpage = find_lock_page(META_MAPPING(sbi), blkaddr);
  3127. if (cpage) {
  3128. f2fs_wait_on_page_writeback(cpage, DATA, true, true);
  3129. f2fs_put_page(cpage, 1);
  3130. }
  3131. }
  3132. void f2fs_wait_on_block_writeback_range(struct inode *inode, block_t blkaddr,
  3133. block_t len)
  3134. {
  3135. block_t i;
  3136. for (i = 0; i < len; i++)
  3137. f2fs_wait_on_block_writeback(inode, blkaddr + i);
  3138. }
  3139. static int read_compacted_summaries(struct f2fs_sb_info *sbi)
  3140. {
  3141. struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
  3142. struct curseg_info *seg_i;
  3143. unsigned char *kaddr;
  3144. struct page *page;
  3145. block_t start;
  3146. int i, j, offset;
  3147. start = start_sum_block(sbi);
  3148. page = f2fs_get_meta_page(sbi, start++);
  3149. if (IS_ERR(page))
  3150. return PTR_ERR(page);
  3151. kaddr = (unsigned char *)page_address(page);
  3152. /* Step 1: restore nat cache */
  3153. seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
  3154. memcpy(seg_i->journal, kaddr, SUM_JOURNAL_SIZE);
  3155. /* Step 2: restore sit cache */
  3156. seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
  3157. memcpy(seg_i->journal, kaddr + SUM_JOURNAL_SIZE, SUM_JOURNAL_SIZE);
  3158. offset = 2 * SUM_JOURNAL_SIZE;
  3159. /* Step 3: restore summary entries */
  3160. for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
  3161. unsigned short blk_off;
  3162. unsigned int segno;
  3163. seg_i = CURSEG_I(sbi, i);
  3164. segno = le32_to_cpu(ckpt->cur_data_segno[i]);
  3165. blk_off = le16_to_cpu(ckpt->cur_data_blkoff[i]);
  3166. seg_i->next_segno = segno;
  3167. reset_curseg(sbi, i, 0);
  3168. seg_i->alloc_type = ckpt->alloc_type[i];
  3169. seg_i->next_blkoff = blk_off;
  3170. if (seg_i->alloc_type == SSR)
  3171. blk_off = sbi->blocks_per_seg;
  3172. for (j = 0; j < blk_off; j++) {
  3173. struct f2fs_summary *s;
  3174. s = (struct f2fs_summary *)(kaddr + offset);
  3175. seg_i->sum_blk->entries[j] = *s;
  3176. offset += SUMMARY_SIZE;
  3177. if (offset + SUMMARY_SIZE <= PAGE_SIZE -
  3178. SUM_FOOTER_SIZE)
  3179. continue;
  3180. f2fs_put_page(page, 1);
  3181. page = NULL;
  3182. page = f2fs_get_meta_page(sbi, start++);
  3183. if (IS_ERR(page))
  3184. return PTR_ERR(page);
  3185. kaddr = (unsigned char *)page_address(page);
  3186. offset = 0;
  3187. }
  3188. }
  3189. f2fs_put_page(page, 1);
  3190. return 0;
  3191. }
  3192. static int read_normal_summaries(struct f2fs_sb_info *sbi, int type)
  3193. {
  3194. struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
  3195. struct f2fs_summary_block *sum;
  3196. struct curseg_info *curseg;
  3197. struct page *new;
  3198. unsigned short blk_off;
  3199. unsigned int segno = 0;
  3200. block_t blk_addr = 0;
  3201. int err = 0;
  3202. /* get segment number and block addr */
  3203. if (IS_DATASEG(type)) {
  3204. segno = le32_to_cpu(ckpt->cur_data_segno[type]);
  3205. blk_off = le16_to_cpu(ckpt->cur_data_blkoff[type -
  3206. CURSEG_HOT_DATA]);
  3207. if (__exist_node_summaries(sbi))
  3208. blk_addr = sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type);
  3209. else
  3210. blk_addr = sum_blk_addr(sbi, NR_CURSEG_DATA_TYPE, type);
  3211. } else {
  3212. segno = le32_to_cpu(ckpt->cur_node_segno[type -
  3213. CURSEG_HOT_NODE]);
  3214. blk_off = le16_to_cpu(ckpt->cur_node_blkoff[type -
  3215. CURSEG_HOT_NODE]);
  3216. if (__exist_node_summaries(sbi))
  3217. blk_addr = sum_blk_addr(sbi, NR_CURSEG_NODE_TYPE,
  3218. type - CURSEG_HOT_NODE);
  3219. else
  3220. blk_addr = GET_SUM_BLOCK(sbi, segno);
  3221. }
  3222. new = f2fs_get_meta_page(sbi, blk_addr);
  3223. if (IS_ERR(new))
  3224. return PTR_ERR(new);
  3225. sum = (struct f2fs_summary_block *)page_address(new);
  3226. if (IS_NODESEG(type)) {
  3227. if (__exist_node_summaries(sbi)) {
  3228. struct f2fs_summary *ns = &sum->entries[0];
  3229. int i;
  3230. for (i = 0; i < sbi->blocks_per_seg; i++, ns++) {
  3231. ns->version = 0;
  3232. ns->ofs_in_node = 0;
  3233. }
  3234. } else {
  3235. err = f2fs_restore_node_summary(sbi, segno, sum);
  3236. if (err)
  3237. goto out;
  3238. }
  3239. }
  3240. /* set uncompleted segment to curseg */
  3241. curseg = CURSEG_I(sbi, type);
  3242. mutex_lock(&curseg->curseg_mutex);
  3243. /* update journal info */
  3244. down_write(&curseg->journal_rwsem);
  3245. memcpy(curseg->journal, &sum->journal, SUM_JOURNAL_SIZE);
  3246. up_write(&curseg->journal_rwsem);
  3247. memcpy(curseg->sum_blk->entries, sum->entries, SUM_ENTRY_SIZE);
  3248. memcpy(&curseg->sum_blk->footer, &sum->footer, SUM_FOOTER_SIZE);
  3249. curseg->next_segno = segno;
  3250. reset_curseg(sbi, type, 0);
  3251. curseg->alloc_type = ckpt->alloc_type[type];
  3252. curseg->next_blkoff = blk_off;
  3253. mutex_unlock(&curseg->curseg_mutex);
  3254. out:
  3255. f2fs_put_page(new, 1);
  3256. return err;
  3257. }
  3258. static int restore_curseg_summaries(struct f2fs_sb_info *sbi)
  3259. {
  3260. struct f2fs_journal *sit_j = CURSEG_I(sbi, CURSEG_COLD_DATA)->journal;
  3261. struct f2fs_journal *nat_j = CURSEG_I(sbi, CURSEG_HOT_DATA)->journal;
  3262. int type = CURSEG_HOT_DATA;
  3263. int err;
  3264. if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG)) {
  3265. int npages = f2fs_npages_for_summary_flush(sbi, true);
  3266. if (npages >= 2)
  3267. f2fs_ra_meta_pages(sbi, start_sum_block(sbi), npages,
  3268. META_CP, true);
  3269. /* restore for compacted data summary */
  3270. err = read_compacted_summaries(sbi);
  3271. if (err)
  3272. return err;
  3273. type = CURSEG_HOT_NODE;
  3274. }
  3275. if (__exist_node_summaries(sbi))
  3276. f2fs_ra_meta_pages(sbi,
  3277. sum_blk_addr(sbi, NR_CURSEG_PERSIST_TYPE, type),
  3278. NR_CURSEG_PERSIST_TYPE - type, META_CP, true);
  3279. for (; type <= CURSEG_COLD_NODE; type++) {
  3280. err = read_normal_summaries(sbi, type);
  3281. if (err)
  3282. return err;
  3283. }
  3284. /* sanity check for summary blocks */
  3285. if (nats_in_cursum(nat_j) > NAT_JOURNAL_ENTRIES ||
  3286. sits_in_cursum(sit_j) > SIT_JOURNAL_ENTRIES) {
  3287. f2fs_err(sbi, "invalid journal entries nats %u sits %u",
  3288. nats_in_cursum(nat_j), sits_in_cursum(sit_j));
  3289. return -EINVAL;
  3290. }
  3291. return 0;
  3292. }
  3293. static void write_compacted_summaries(struct f2fs_sb_info *sbi, block_t blkaddr)
  3294. {
  3295. struct page *page;
  3296. unsigned char *kaddr;
  3297. struct f2fs_summary *summary;
  3298. struct curseg_info *seg_i;
  3299. int written_size = 0;
  3300. int i, j;
  3301. page = f2fs_grab_meta_page(sbi, blkaddr++);
  3302. kaddr = (unsigned char *)page_address(page);
  3303. memset(kaddr, 0, PAGE_SIZE);
  3304. /* Step 1: write nat cache */
  3305. seg_i = CURSEG_I(sbi, CURSEG_HOT_DATA);
  3306. memcpy(kaddr, seg_i->journal, SUM_JOURNAL_SIZE);
  3307. written_size += SUM_JOURNAL_SIZE;
  3308. /* Step 2: write sit cache */
  3309. seg_i = CURSEG_I(sbi, CURSEG_COLD_DATA);
  3310. memcpy(kaddr + written_size, seg_i->journal, SUM_JOURNAL_SIZE);
  3311. written_size += SUM_JOURNAL_SIZE;
  3312. /* Step 3: write summary entries */
  3313. for (i = CURSEG_HOT_DATA; i <= CURSEG_COLD_DATA; i++) {
  3314. unsigned short blkoff;
  3315. seg_i = CURSEG_I(sbi, i);
  3316. if (sbi->ckpt->alloc_type[i] == SSR)
  3317. blkoff = sbi->blocks_per_seg;
  3318. else
  3319. blkoff = curseg_blkoff(sbi, i);
  3320. for (j = 0; j < blkoff; j++) {
  3321. if (!page) {
  3322. page = f2fs_grab_meta_page(sbi, blkaddr++);
  3323. kaddr = (unsigned char *)page_address(page);
  3324. memset(kaddr, 0, PAGE_SIZE);
  3325. written_size = 0;
  3326. }
  3327. summary = (struct f2fs_summary *)(kaddr + written_size);
  3328. *summary = seg_i->sum_blk->entries[j];
  3329. written_size += SUMMARY_SIZE;
  3330. if (written_size + SUMMARY_SIZE <= PAGE_SIZE -
  3331. SUM_FOOTER_SIZE)
  3332. continue;
  3333. set_page_dirty(page);
  3334. f2fs_put_page(page, 1);
  3335. page = NULL;
  3336. }
  3337. }
  3338. if (page) {
  3339. set_page_dirty(page);
  3340. f2fs_put_page(page, 1);
  3341. }
  3342. }
  3343. static void write_normal_summaries(struct f2fs_sb_info *sbi,
  3344. block_t blkaddr, int type)
  3345. {
  3346. int i, end;
  3347. if (IS_DATASEG(type))
  3348. end = type + NR_CURSEG_DATA_TYPE;
  3349. else
  3350. end = type + NR_CURSEG_NODE_TYPE;
  3351. for (i = type; i < end; i++)
  3352. write_current_sum_page(sbi, i, blkaddr + (i - type));
  3353. }
  3354. void f2fs_write_data_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
  3355. {
  3356. if (is_set_ckpt_flags(sbi, CP_COMPACT_SUM_FLAG))
  3357. write_compacted_summaries(sbi, start_blk);
  3358. else
  3359. write_normal_summaries(sbi, start_blk, CURSEG_HOT_DATA);
  3360. }
  3361. void f2fs_write_node_summaries(struct f2fs_sb_info *sbi, block_t start_blk)
  3362. {
  3363. write_normal_summaries(sbi, start_blk, CURSEG_HOT_NODE);
  3364. }
  3365. int f2fs_lookup_journal_in_cursum(struct f2fs_journal *journal, int type,
  3366. unsigned int val, int alloc)
  3367. {
  3368. int i;
  3369. if (type == NAT_JOURNAL) {
  3370. for (i = 0; i < nats_in_cursum(journal); i++) {
  3371. if (le32_to_cpu(nid_in_journal(journal, i)) == val)
  3372. return i;
  3373. }
  3374. if (alloc && __has_cursum_space(journal, 1, NAT_JOURNAL))
  3375. return update_nats_in_cursum(journal, 1);
  3376. } else if (type == SIT_JOURNAL) {
  3377. for (i = 0; i < sits_in_cursum(journal); i++)
  3378. if (le32_to_cpu(segno_in_journal(journal, i)) == val)
  3379. return i;
  3380. if (alloc && __has_cursum_space(journal, 1, SIT_JOURNAL))
  3381. return update_sits_in_cursum(journal, 1);
  3382. }
  3383. return -1;
  3384. }
  3385. static struct page *get_current_sit_page(struct f2fs_sb_info *sbi,
  3386. unsigned int segno)
  3387. {
  3388. return f2fs_get_meta_page(sbi, current_sit_addr(sbi, segno));
  3389. }
  3390. static struct page *get_next_sit_page(struct f2fs_sb_info *sbi,
  3391. unsigned int start)
  3392. {
  3393. struct sit_info *sit_i = SIT_I(sbi);
  3394. struct page *page;
  3395. pgoff_t src_off, dst_off;
  3396. src_off = current_sit_addr(sbi, start);
  3397. dst_off = next_sit_addr(sbi, src_off);
  3398. page = f2fs_grab_meta_page(sbi, dst_off);
  3399. seg_info_to_sit_page(sbi, page, start);
  3400. set_page_dirty(page);
  3401. set_to_next_sit(sit_i, start);
  3402. return page;
  3403. }
  3404. static struct sit_entry_set *grab_sit_entry_set(void)
  3405. {
  3406. struct sit_entry_set *ses =
  3407. f2fs_kmem_cache_alloc(sit_entry_set_slab, GFP_NOFS);
  3408. ses->entry_cnt = 0;
  3409. INIT_LIST_HEAD(&ses->set_list);
  3410. return ses;
  3411. }
  3412. static void release_sit_entry_set(struct sit_entry_set *ses)
  3413. {
  3414. list_del(&ses->set_list);
  3415. kmem_cache_free(sit_entry_set_slab, ses);
  3416. }
  3417. static void adjust_sit_entry_set(struct sit_entry_set *ses,
  3418. struct list_head *head)
  3419. {
  3420. struct sit_entry_set *next = ses;
  3421. if (list_is_last(&ses->set_list, head))
  3422. return;
  3423. list_for_each_entry_continue(next, head, set_list)
  3424. if (ses->entry_cnt <= next->entry_cnt)
  3425. break;
  3426. list_move_tail(&ses->set_list, &next->set_list);
  3427. }
  3428. static void add_sit_entry(unsigned int segno, struct list_head *head)
  3429. {
  3430. struct sit_entry_set *ses;
  3431. unsigned int start_segno = START_SEGNO(segno);
  3432. list_for_each_entry(ses, head, set_list) {
  3433. if (ses->start_segno == start_segno) {
  3434. ses->entry_cnt++;
  3435. adjust_sit_entry_set(ses, head);
  3436. return;
  3437. }
  3438. }
  3439. ses = grab_sit_entry_set();
  3440. ses->start_segno = start_segno;
  3441. ses->entry_cnt++;
  3442. list_add(&ses->set_list, head);
  3443. }
  3444. static void add_sits_in_set(struct f2fs_sb_info *sbi)
  3445. {
  3446. struct f2fs_sm_info *sm_info = SM_I(sbi);
  3447. struct list_head *set_list = &sm_info->sit_entry_set;
  3448. unsigned long *bitmap = SIT_I(sbi)->dirty_sentries_bitmap;
  3449. unsigned int segno;
  3450. for_each_set_bit(segno, bitmap, MAIN_SEGS(sbi))
  3451. add_sit_entry(segno, set_list);
  3452. }
  3453. static void remove_sits_in_journal(struct f2fs_sb_info *sbi)
  3454. {
  3455. struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
  3456. struct f2fs_journal *journal = curseg->journal;
  3457. int i;
  3458. down_write(&curseg->journal_rwsem);
  3459. for (i = 0; i < sits_in_cursum(journal); i++) {
  3460. unsigned int segno;
  3461. bool dirtied;
  3462. segno = le32_to_cpu(segno_in_journal(journal, i));
  3463. dirtied = __mark_sit_entry_dirty(sbi, segno);
  3464. if (!dirtied)
  3465. add_sit_entry(segno, &SM_I(sbi)->sit_entry_set);
  3466. }
  3467. update_sits_in_cursum(journal, -i);
  3468. up_write(&curseg->journal_rwsem);
  3469. }
  3470. /*
  3471. * CP calls this function, which flushes SIT entries including sit_journal,
  3472. * and moves prefree segs to free segs.
  3473. */
  3474. void f2fs_flush_sit_entries(struct f2fs_sb_info *sbi, struct cp_control *cpc)
  3475. {
  3476. struct sit_info *sit_i = SIT_I(sbi);
  3477. unsigned long *bitmap = sit_i->dirty_sentries_bitmap;
  3478. struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
  3479. struct f2fs_journal *journal = curseg->journal;
  3480. struct sit_entry_set *ses, *tmp;
  3481. struct list_head *head = &SM_I(sbi)->sit_entry_set;
  3482. bool to_journal = !is_sbi_flag_set(sbi, SBI_IS_RESIZEFS);
  3483. struct seg_entry *se;
  3484. down_write(&sit_i->sentry_lock);
  3485. if (!sit_i->dirty_sentries)
  3486. goto out;
  3487. /*
  3488. * add and account sit entries of dirty bitmap in sit entry
  3489. * set temporarily
  3490. */
  3491. add_sits_in_set(sbi);
  3492. /*
  3493. * if there are no enough space in journal to store dirty sit
  3494. * entries, remove all entries from journal and add and account
  3495. * them in sit entry set.
  3496. */
  3497. if (!__has_cursum_space(journal, sit_i->dirty_sentries, SIT_JOURNAL) ||
  3498. !to_journal)
  3499. remove_sits_in_journal(sbi);
  3500. /*
  3501. * there are two steps to flush sit entries:
  3502. * #1, flush sit entries to journal in current cold data summary block.
  3503. * #2, flush sit entries to sit page.
  3504. */
  3505. list_for_each_entry_safe(ses, tmp, head, set_list) {
  3506. struct page *page = NULL;
  3507. struct f2fs_sit_block *raw_sit = NULL;
  3508. unsigned int start_segno = ses->start_segno;
  3509. unsigned int end = min(start_segno + SIT_ENTRY_PER_BLOCK,
  3510. (unsigned long)MAIN_SEGS(sbi));
  3511. unsigned int segno = start_segno;
  3512. if (to_journal &&
  3513. !__has_cursum_space(journal, ses->entry_cnt, SIT_JOURNAL))
  3514. to_journal = false;
  3515. if (to_journal) {
  3516. down_write(&curseg->journal_rwsem);
  3517. } else {
  3518. page = get_next_sit_page(sbi, start_segno);
  3519. raw_sit = page_address(page);
  3520. }
  3521. /* flush dirty sit entries in region of current sit set */
  3522. for_each_set_bit_from(segno, bitmap, end) {
  3523. int offset, sit_offset;
  3524. se = get_seg_entry(sbi, segno);
  3525. #ifdef CONFIG_F2FS_CHECK_FS
  3526. if (memcmp(se->cur_valid_map, se->cur_valid_map_mir,
  3527. SIT_VBLOCK_MAP_SIZE))
  3528. f2fs_bug_on(sbi, 1);
  3529. #endif
  3530. /* add discard candidates */
  3531. if (!(cpc->reason & CP_DISCARD)) {
  3532. cpc->trim_start = segno;
  3533. add_discard_addrs(sbi, cpc, false);
  3534. }
  3535. if (to_journal) {
  3536. offset = f2fs_lookup_journal_in_cursum(journal,
  3537. SIT_JOURNAL, segno, 1);
  3538. f2fs_bug_on(sbi, offset < 0);
  3539. segno_in_journal(journal, offset) =
  3540. cpu_to_le32(segno);
  3541. seg_info_to_raw_sit(se,
  3542. &sit_in_journal(journal, offset));
  3543. check_block_count(sbi, segno,
  3544. &sit_in_journal(journal, offset));
  3545. } else {
  3546. sit_offset = SIT_ENTRY_OFFSET(sit_i, segno);
  3547. seg_info_to_raw_sit(se,
  3548. &raw_sit->entries[sit_offset]);
  3549. check_block_count(sbi, segno,
  3550. &raw_sit->entries[sit_offset]);
  3551. }
  3552. __clear_bit(segno, bitmap);
  3553. sit_i->dirty_sentries--;
  3554. ses->entry_cnt--;
  3555. }
  3556. if (to_journal)
  3557. up_write(&curseg->journal_rwsem);
  3558. else
  3559. f2fs_put_page(page, 1);
  3560. f2fs_bug_on(sbi, ses->entry_cnt);
  3561. release_sit_entry_set(ses);
  3562. }
  3563. f2fs_bug_on(sbi, !list_empty(head));
  3564. f2fs_bug_on(sbi, sit_i->dirty_sentries);
  3565. out:
  3566. if (cpc->reason & CP_DISCARD) {
  3567. __u64 trim_start = cpc->trim_start;
  3568. for (; cpc->trim_start <= cpc->trim_end; cpc->trim_start++)
  3569. add_discard_addrs(sbi, cpc, false);
  3570. cpc->trim_start = trim_start;
  3571. }
  3572. up_write(&sit_i->sentry_lock);
  3573. set_prefree_as_free_segments(sbi);
  3574. }
  3575. static int build_sit_info(struct f2fs_sb_info *sbi)
  3576. {
  3577. struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
  3578. struct sit_info *sit_i;
  3579. unsigned int sit_segs, start;
  3580. char *src_bitmap, *bitmap;
  3581. unsigned int bitmap_size, main_bitmap_size, sit_bitmap_size;
  3582. /* allocate memory for SIT information */
  3583. sit_i = f2fs_kzalloc(sbi, sizeof(struct sit_info), GFP_KERNEL);
  3584. if (!sit_i)
  3585. return -ENOMEM;
  3586. SM_I(sbi)->sit_info = sit_i;
  3587. sit_i->sentries =
  3588. f2fs_kvzalloc(sbi, array_size(sizeof(struct seg_entry),
  3589. MAIN_SEGS(sbi)),
  3590. GFP_KERNEL);
  3591. if (!sit_i->sentries)
  3592. return -ENOMEM;
  3593. main_bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
  3594. sit_i->dirty_sentries_bitmap = f2fs_kvzalloc(sbi, main_bitmap_size,
  3595. GFP_KERNEL);
  3596. if (!sit_i->dirty_sentries_bitmap)
  3597. return -ENOMEM;
  3598. #ifdef CONFIG_F2FS_CHECK_FS
  3599. bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 4;
  3600. #else
  3601. bitmap_size = MAIN_SEGS(sbi) * SIT_VBLOCK_MAP_SIZE * 3;
  3602. #endif
  3603. sit_i->bitmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
  3604. if (!sit_i->bitmap)
  3605. return -ENOMEM;
  3606. bitmap = sit_i->bitmap;
  3607. for (start = 0; start < MAIN_SEGS(sbi); start++) {
  3608. sit_i->sentries[start].cur_valid_map = bitmap;
  3609. bitmap += SIT_VBLOCK_MAP_SIZE;
  3610. sit_i->sentries[start].ckpt_valid_map = bitmap;
  3611. bitmap += SIT_VBLOCK_MAP_SIZE;
  3612. #ifdef CONFIG_F2FS_CHECK_FS
  3613. sit_i->sentries[start].cur_valid_map_mir = bitmap;
  3614. bitmap += SIT_VBLOCK_MAP_SIZE;
  3615. #endif
  3616. sit_i->sentries[start].discard_map = bitmap;
  3617. bitmap += SIT_VBLOCK_MAP_SIZE;
  3618. }
  3619. sit_i->tmp_map = f2fs_kzalloc(sbi, SIT_VBLOCK_MAP_SIZE, GFP_KERNEL);
  3620. if (!sit_i->tmp_map)
  3621. return -ENOMEM;
  3622. if (__is_large_section(sbi)) {
  3623. sit_i->sec_entries =
  3624. f2fs_kvzalloc(sbi, array_size(sizeof(struct sec_entry),
  3625. MAIN_SECS(sbi)),
  3626. GFP_KERNEL);
  3627. if (!sit_i->sec_entries)
  3628. return -ENOMEM;
  3629. }
  3630. /* get information related with SIT */
  3631. sit_segs = le32_to_cpu(raw_super->segment_count_sit) >> 1;
  3632. /* setup SIT bitmap from ckeckpoint pack */
  3633. sit_bitmap_size = __bitmap_size(sbi, SIT_BITMAP);
  3634. src_bitmap = __bitmap_ptr(sbi, SIT_BITMAP);
  3635. sit_i->sit_bitmap = kmemdup(src_bitmap, sit_bitmap_size, GFP_KERNEL);
  3636. if (!sit_i->sit_bitmap)
  3637. return -ENOMEM;
  3638. #ifdef CONFIG_F2FS_CHECK_FS
  3639. sit_i->sit_bitmap_mir = kmemdup(src_bitmap,
  3640. sit_bitmap_size, GFP_KERNEL);
  3641. if (!sit_i->sit_bitmap_mir)
  3642. return -ENOMEM;
  3643. sit_i->invalid_segmap = f2fs_kvzalloc(sbi,
  3644. main_bitmap_size, GFP_KERNEL);
  3645. if (!sit_i->invalid_segmap)
  3646. return -ENOMEM;
  3647. #endif
  3648. /* init SIT information */
  3649. sit_i->s_ops = &default_salloc_ops;
  3650. sit_i->sit_base_addr = le32_to_cpu(raw_super->sit_blkaddr);
  3651. sit_i->sit_blocks = sit_segs << sbi->log_blocks_per_seg;
  3652. sit_i->written_valid_blocks = 0;
  3653. sit_i->bitmap_size = sit_bitmap_size;
  3654. sit_i->dirty_sentries = 0;
  3655. sit_i->sents_per_block = SIT_ENTRY_PER_BLOCK;
  3656. sit_i->elapsed_time = le64_to_cpu(sbi->ckpt->elapsed_time);
  3657. sit_i->mounted_time = ktime_get_boottime_seconds();
  3658. init_rwsem(&sit_i->sentry_lock);
  3659. return 0;
  3660. }
  3661. static int build_free_segmap(struct f2fs_sb_info *sbi)
  3662. {
  3663. struct free_segmap_info *free_i;
  3664. unsigned int bitmap_size, sec_bitmap_size;
  3665. /* allocate memory for free segmap information */
  3666. free_i = f2fs_kzalloc(sbi, sizeof(struct free_segmap_info), GFP_KERNEL);
  3667. if (!free_i)
  3668. return -ENOMEM;
  3669. SM_I(sbi)->free_info = free_i;
  3670. bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
  3671. free_i->free_segmap = f2fs_kvmalloc(sbi, bitmap_size, GFP_KERNEL);
  3672. if (!free_i->free_segmap)
  3673. return -ENOMEM;
  3674. sec_bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
  3675. free_i->free_secmap = f2fs_kvmalloc(sbi, sec_bitmap_size, GFP_KERNEL);
  3676. if (!free_i->free_secmap)
  3677. return -ENOMEM;
  3678. /* set all segments as dirty temporarily */
  3679. memset(free_i->free_segmap, 0xff, bitmap_size);
  3680. memset(free_i->free_secmap, 0xff, sec_bitmap_size);
  3681. /* init free segmap information */
  3682. free_i->start_segno = GET_SEGNO_FROM_SEG0(sbi, MAIN_BLKADDR(sbi));
  3683. free_i->free_segments = 0;
  3684. free_i->free_sections = 0;
  3685. spin_lock_init(&free_i->segmap_lock);
  3686. return 0;
  3687. }
  3688. static int build_curseg(struct f2fs_sb_info *sbi)
  3689. {
  3690. struct curseg_info *array;
  3691. int i;
  3692. array = f2fs_kzalloc(sbi, array_size(NR_CURSEG_TYPE,
  3693. sizeof(*array)), GFP_KERNEL);
  3694. if (!array)
  3695. return -ENOMEM;
  3696. SM_I(sbi)->curseg_array = array;
  3697. for (i = 0; i < NO_CHECK_TYPE; i++) {
  3698. mutex_init(&array[i].curseg_mutex);
  3699. array[i].sum_blk = f2fs_kzalloc(sbi, PAGE_SIZE, GFP_KERNEL);
  3700. if (!array[i].sum_blk)
  3701. return -ENOMEM;
  3702. init_rwsem(&array[i].journal_rwsem);
  3703. array[i].journal = f2fs_kzalloc(sbi,
  3704. sizeof(struct f2fs_journal), GFP_KERNEL);
  3705. if (!array[i].journal)
  3706. return -ENOMEM;
  3707. if (i < NR_PERSISTENT_LOG)
  3708. array[i].seg_type = CURSEG_HOT_DATA + i;
  3709. else if (i == CURSEG_COLD_DATA_PINNED)
  3710. array[i].seg_type = CURSEG_COLD_DATA;
  3711. else if (i == CURSEG_ALL_DATA_ATGC)
  3712. array[i].seg_type = CURSEG_COLD_DATA;
  3713. array[i].segno = NULL_SEGNO;
  3714. array[i].next_blkoff = 0;
  3715. array[i].inited = false;
  3716. }
  3717. return restore_curseg_summaries(sbi);
  3718. }
  3719. static int build_sit_entries(struct f2fs_sb_info *sbi)
  3720. {
  3721. struct sit_info *sit_i = SIT_I(sbi);
  3722. struct curseg_info *curseg = CURSEG_I(sbi, CURSEG_COLD_DATA);
  3723. struct f2fs_journal *journal = curseg->journal;
  3724. struct seg_entry *se;
  3725. struct f2fs_sit_entry sit;
  3726. int sit_blk_cnt = SIT_BLK_CNT(sbi);
  3727. unsigned int i, start, end;
  3728. unsigned int readed, start_blk = 0;
  3729. int err = 0;
  3730. block_t total_node_blocks = 0;
  3731. do {
  3732. readed = f2fs_ra_meta_pages(sbi, start_blk, BIO_MAX_PAGES,
  3733. META_SIT, true);
  3734. start = start_blk * sit_i->sents_per_block;
  3735. end = (start_blk + readed) * sit_i->sents_per_block;
  3736. for (; start < end && start < MAIN_SEGS(sbi); start++) {
  3737. struct f2fs_sit_block *sit_blk;
  3738. struct page *page;
  3739. se = &sit_i->sentries[start];
  3740. page = get_current_sit_page(sbi, start);
  3741. if (IS_ERR(page))
  3742. return PTR_ERR(page);
  3743. sit_blk = (struct f2fs_sit_block *)page_address(page);
  3744. sit = sit_blk->entries[SIT_ENTRY_OFFSET(sit_i, start)];
  3745. f2fs_put_page(page, 1);
  3746. err = check_block_count(sbi, start, &sit);
  3747. if (err)
  3748. return err;
  3749. seg_info_from_raw_sit(se, &sit);
  3750. if (IS_NODESEG(se->type))
  3751. total_node_blocks += se->valid_blocks;
  3752. /* build discard map only one time */
  3753. if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
  3754. memset(se->discard_map, 0xff,
  3755. SIT_VBLOCK_MAP_SIZE);
  3756. } else {
  3757. memcpy(se->discard_map,
  3758. se->cur_valid_map,
  3759. SIT_VBLOCK_MAP_SIZE);
  3760. sbi->discard_blks +=
  3761. sbi->blocks_per_seg -
  3762. se->valid_blocks;
  3763. }
  3764. if (__is_large_section(sbi))
  3765. get_sec_entry(sbi, start)->valid_blocks +=
  3766. se->valid_blocks;
  3767. }
  3768. start_blk += readed;
  3769. } while (start_blk < sit_blk_cnt);
  3770. down_read(&curseg->journal_rwsem);
  3771. for (i = 0; i < sits_in_cursum(journal); i++) {
  3772. unsigned int old_valid_blocks;
  3773. start = le32_to_cpu(segno_in_journal(journal, i));
  3774. if (start >= MAIN_SEGS(sbi)) {
  3775. f2fs_err(sbi, "Wrong journal entry on segno %u",
  3776. start);
  3777. err = -EFSCORRUPTED;
  3778. break;
  3779. }
  3780. se = &sit_i->sentries[start];
  3781. sit = sit_in_journal(journal, i);
  3782. old_valid_blocks = se->valid_blocks;
  3783. if (IS_NODESEG(se->type))
  3784. total_node_blocks -= old_valid_blocks;
  3785. err = check_block_count(sbi, start, &sit);
  3786. if (err)
  3787. break;
  3788. seg_info_from_raw_sit(se, &sit);
  3789. if (IS_NODESEG(se->type))
  3790. total_node_blocks += se->valid_blocks;
  3791. if (is_set_ckpt_flags(sbi, CP_TRIMMED_FLAG)) {
  3792. memset(se->discard_map, 0xff, SIT_VBLOCK_MAP_SIZE);
  3793. } else {
  3794. memcpy(se->discard_map, se->cur_valid_map,
  3795. SIT_VBLOCK_MAP_SIZE);
  3796. sbi->discard_blks += old_valid_blocks;
  3797. sbi->discard_blks -= se->valid_blocks;
  3798. }
  3799. if (__is_large_section(sbi)) {
  3800. get_sec_entry(sbi, start)->valid_blocks +=
  3801. se->valid_blocks;
  3802. get_sec_entry(sbi, start)->valid_blocks -=
  3803. old_valid_blocks;
  3804. }
  3805. }
  3806. up_read(&curseg->journal_rwsem);
  3807. if (!err && total_node_blocks != valid_node_count(sbi)) {
  3808. f2fs_err(sbi, "SIT is corrupted node# %u vs %u",
  3809. total_node_blocks, valid_node_count(sbi));
  3810. err = -EFSCORRUPTED;
  3811. }
  3812. return err;
  3813. }
  3814. static void init_free_segmap(struct f2fs_sb_info *sbi)
  3815. {
  3816. unsigned int start;
  3817. int type;
  3818. struct seg_entry *sentry;
  3819. for (start = 0; start < MAIN_SEGS(sbi); start++) {
  3820. if (f2fs_usable_blks_in_seg(sbi, start) == 0)
  3821. continue;
  3822. sentry = get_seg_entry(sbi, start);
  3823. if (!sentry->valid_blocks)
  3824. __set_free(sbi, start);
  3825. else
  3826. SIT_I(sbi)->written_valid_blocks +=
  3827. sentry->valid_blocks;
  3828. }
  3829. /* set use the current segments */
  3830. for (type = CURSEG_HOT_DATA; type <= CURSEG_COLD_NODE; type++) {
  3831. struct curseg_info *curseg_t = CURSEG_I(sbi, type);
  3832. __set_test_and_inuse(sbi, curseg_t->segno);
  3833. }
  3834. }
  3835. static void init_dirty_segmap(struct f2fs_sb_info *sbi)
  3836. {
  3837. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  3838. struct free_segmap_info *free_i = FREE_I(sbi);
  3839. unsigned int segno = 0, offset = 0, secno;
  3840. block_t valid_blocks, usable_blks_in_seg;
  3841. block_t blks_per_sec = BLKS_PER_SEC(sbi);
  3842. while (1) {
  3843. /* find dirty segment based on free segmap */
  3844. segno = find_next_inuse(free_i, MAIN_SEGS(sbi), offset);
  3845. if (segno >= MAIN_SEGS(sbi))
  3846. break;
  3847. offset = segno + 1;
  3848. valid_blocks = get_valid_blocks(sbi, segno, false);
  3849. usable_blks_in_seg = f2fs_usable_blks_in_seg(sbi, segno);
  3850. if (valid_blocks == usable_blks_in_seg || !valid_blocks)
  3851. continue;
  3852. if (valid_blocks > usable_blks_in_seg) {
  3853. f2fs_bug_on(sbi, 1);
  3854. continue;
  3855. }
  3856. mutex_lock(&dirty_i->seglist_lock);
  3857. __locate_dirty_segment(sbi, segno, DIRTY);
  3858. mutex_unlock(&dirty_i->seglist_lock);
  3859. }
  3860. if (!__is_large_section(sbi))
  3861. return;
  3862. mutex_lock(&dirty_i->seglist_lock);
  3863. for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
  3864. valid_blocks = get_valid_blocks(sbi, segno, true);
  3865. secno = GET_SEC_FROM_SEG(sbi, segno);
  3866. if (!valid_blocks || valid_blocks == blks_per_sec)
  3867. continue;
  3868. if (IS_CURSEC(sbi, secno))
  3869. continue;
  3870. set_bit(secno, dirty_i->dirty_secmap);
  3871. }
  3872. mutex_unlock(&dirty_i->seglist_lock);
  3873. }
  3874. static int init_victim_secmap(struct f2fs_sb_info *sbi)
  3875. {
  3876. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  3877. unsigned int bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
  3878. dirty_i->victim_secmap = f2fs_kvzalloc(sbi, bitmap_size, GFP_KERNEL);
  3879. if (!dirty_i->victim_secmap)
  3880. return -ENOMEM;
  3881. return 0;
  3882. }
  3883. static int build_dirty_segmap(struct f2fs_sb_info *sbi)
  3884. {
  3885. struct dirty_seglist_info *dirty_i;
  3886. unsigned int bitmap_size, i;
  3887. /* allocate memory for dirty segments list information */
  3888. dirty_i = f2fs_kzalloc(sbi, sizeof(struct dirty_seglist_info),
  3889. GFP_KERNEL);
  3890. if (!dirty_i)
  3891. return -ENOMEM;
  3892. SM_I(sbi)->dirty_info = dirty_i;
  3893. mutex_init(&dirty_i->seglist_lock);
  3894. bitmap_size = f2fs_bitmap_size(MAIN_SEGS(sbi));
  3895. for (i = 0; i < NR_DIRTY_TYPE; i++) {
  3896. dirty_i->dirty_segmap[i] = f2fs_kvzalloc(sbi, bitmap_size,
  3897. GFP_KERNEL);
  3898. if (!dirty_i->dirty_segmap[i])
  3899. return -ENOMEM;
  3900. }
  3901. if (__is_large_section(sbi)) {
  3902. bitmap_size = f2fs_bitmap_size(MAIN_SECS(sbi));
  3903. dirty_i->dirty_secmap = f2fs_kvzalloc(sbi,
  3904. bitmap_size, GFP_KERNEL);
  3905. if (!dirty_i->dirty_secmap)
  3906. return -ENOMEM;
  3907. }
  3908. init_dirty_segmap(sbi);
  3909. return init_victim_secmap(sbi);
  3910. }
  3911. static int sanity_check_curseg(struct f2fs_sb_info *sbi)
  3912. {
  3913. int i;
  3914. /*
  3915. * In LFS/SSR curseg, .next_blkoff should point to an unused blkaddr;
  3916. * In LFS curseg, all blkaddr after .next_blkoff should be unused.
  3917. */
  3918. for (i = 0; i < NR_PERSISTENT_LOG; i++) {
  3919. struct curseg_info *curseg = CURSEG_I(sbi, i);
  3920. struct seg_entry *se = get_seg_entry(sbi, curseg->segno);
  3921. unsigned int blkofs = curseg->next_blkoff;
  3922. if (f2fs_sb_has_readonly(sbi) &&
  3923. i != CURSEG_HOT_DATA && i != CURSEG_HOT_NODE)
  3924. continue;
  3925. sanity_check_seg_type(sbi, curseg->seg_type);
  3926. if (curseg->alloc_type != LFS && curseg->alloc_type != SSR) {
  3927. f2fs_err(sbi,
  3928. "Current segment has invalid alloc_type:%d",
  3929. curseg->alloc_type);
  3930. return -EFSCORRUPTED;
  3931. }
  3932. if (f2fs_test_bit(blkofs, se->cur_valid_map))
  3933. goto out;
  3934. if (curseg->alloc_type == SSR)
  3935. continue;
  3936. for (blkofs += 1; blkofs < sbi->blocks_per_seg; blkofs++) {
  3937. if (!f2fs_test_bit(blkofs, se->cur_valid_map))
  3938. continue;
  3939. out:
  3940. f2fs_err(sbi,
  3941. "Current segment's next free block offset is inconsistent with bitmap, logtype:%u, segno:%u, type:%u, next_blkoff:%u, blkofs:%u",
  3942. i, curseg->segno, curseg->alloc_type,
  3943. curseg->next_blkoff, blkofs);
  3944. return -EFSCORRUPTED;
  3945. }
  3946. }
  3947. return 0;
  3948. }
  3949. #ifdef CONFIG_BLK_DEV_ZONED
  3950. static int check_zone_write_pointer(struct f2fs_sb_info *sbi,
  3951. struct f2fs_dev_info *fdev,
  3952. struct blk_zone *zone)
  3953. {
  3954. unsigned int wp_segno, wp_blkoff, zone_secno, zone_segno, segno;
  3955. block_t zone_block, wp_block, last_valid_block;
  3956. unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
  3957. int i, s, b, ret;
  3958. struct seg_entry *se;
  3959. if (zone->type != BLK_ZONE_TYPE_SEQWRITE_REQ)
  3960. return 0;
  3961. wp_block = fdev->start_blk + (zone->wp >> log_sectors_per_block);
  3962. wp_segno = GET_SEGNO(sbi, wp_block);
  3963. wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
  3964. zone_block = fdev->start_blk + (zone->start >> log_sectors_per_block);
  3965. zone_segno = GET_SEGNO(sbi, zone_block);
  3966. zone_secno = GET_SEC_FROM_SEG(sbi, zone_segno);
  3967. if (zone_segno >= MAIN_SEGS(sbi))
  3968. return 0;
  3969. /*
  3970. * Skip check of zones cursegs point to, since
  3971. * fix_curseg_write_pointer() checks them.
  3972. */
  3973. for (i = 0; i < NO_CHECK_TYPE; i++)
  3974. if (zone_secno == GET_SEC_FROM_SEG(sbi,
  3975. CURSEG_I(sbi, i)->segno))
  3976. return 0;
  3977. /*
  3978. * Get last valid block of the zone.
  3979. */
  3980. last_valid_block = zone_block - 1;
  3981. for (s = sbi->segs_per_sec - 1; s >= 0; s--) {
  3982. segno = zone_segno + s;
  3983. se = get_seg_entry(sbi, segno);
  3984. for (b = sbi->blocks_per_seg - 1; b >= 0; b--)
  3985. if (f2fs_test_bit(b, se->cur_valid_map)) {
  3986. last_valid_block = START_BLOCK(sbi, segno) + b;
  3987. break;
  3988. }
  3989. if (last_valid_block >= zone_block)
  3990. break;
  3991. }
  3992. /*
  3993. * If last valid block is beyond the write pointer, report the
  3994. * inconsistency. This inconsistency does not cause write error
  3995. * because the zone will not be selected for write operation until
  3996. * it get discarded. Just report it.
  3997. */
  3998. if (last_valid_block >= wp_block) {
  3999. f2fs_notice(sbi, "Valid block beyond write pointer: "
  4000. "valid block[0x%x,0x%x] wp[0x%x,0x%x]",
  4001. GET_SEGNO(sbi, last_valid_block),
  4002. GET_BLKOFF_FROM_SEG0(sbi, last_valid_block),
  4003. wp_segno, wp_blkoff);
  4004. return 0;
  4005. }
  4006. /*
  4007. * If there is no valid block in the zone and if write pointer is
  4008. * not at zone start, reset the write pointer.
  4009. */
  4010. if (last_valid_block + 1 == zone_block && zone->wp != zone->start) {
  4011. f2fs_notice(sbi,
  4012. "Zone without valid block has non-zero write "
  4013. "pointer. Reset the write pointer: wp[0x%x,0x%x]",
  4014. wp_segno, wp_blkoff);
  4015. ret = __f2fs_issue_discard_zone(sbi, fdev->bdev, zone_block,
  4016. zone->len >> log_sectors_per_block);
  4017. if (ret) {
  4018. f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
  4019. fdev->path, ret);
  4020. return ret;
  4021. }
  4022. }
  4023. return 0;
  4024. }
  4025. static struct f2fs_dev_info *get_target_zoned_dev(struct f2fs_sb_info *sbi,
  4026. block_t zone_blkaddr)
  4027. {
  4028. int i;
  4029. for (i = 0; i < sbi->s_ndevs; i++) {
  4030. if (!bdev_is_zoned(FDEV(i).bdev))
  4031. continue;
  4032. if (sbi->s_ndevs == 1 || (FDEV(i).start_blk <= zone_blkaddr &&
  4033. zone_blkaddr <= FDEV(i).end_blk))
  4034. return &FDEV(i);
  4035. }
  4036. return NULL;
  4037. }
  4038. static int report_one_zone_cb(struct blk_zone *zone, unsigned int idx,
  4039. void *data)
  4040. {
  4041. memcpy(data, zone, sizeof(struct blk_zone));
  4042. return 0;
  4043. }
  4044. static int fix_curseg_write_pointer(struct f2fs_sb_info *sbi, int type)
  4045. {
  4046. struct curseg_info *cs = CURSEG_I(sbi, type);
  4047. struct f2fs_dev_info *zbd;
  4048. struct blk_zone zone;
  4049. unsigned int cs_section, wp_segno, wp_blkoff, wp_sector_off;
  4050. block_t cs_zone_block, wp_block;
  4051. unsigned int log_sectors_per_block = sbi->log_blocksize - SECTOR_SHIFT;
  4052. sector_t zone_sector;
  4053. int err;
  4054. cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
  4055. cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
  4056. zbd = get_target_zoned_dev(sbi, cs_zone_block);
  4057. if (!zbd)
  4058. return 0;
  4059. /* report zone for the sector the curseg points to */
  4060. zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
  4061. << log_sectors_per_block;
  4062. err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
  4063. report_one_zone_cb, &zone);
  4064. if (err != 1) {
  4065. f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
  4066. zbd->path, err);
  4067. return err;
  4068. }
  4069. if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
  4070. return 0;
  4071. wp_block = zbd->start_blk + (zone.wp >> log_sectors_per_block);
  4072. wp_segno = GET_SEGNO(sbi, wp_block);
  4073. wp_blkoff = wp_block - START_BLOCK(sbi, wp_segno);
  4074. wp_sector_off = zone.wp & GENMASK(log_sectors_per_block - 1, 0);
  4075. if (cs->segno == wp_segno && cs->next_blkoff == wp_blkoff &&
  4076. wp_sector_off == 0)
  4077. return 0;
  4078. f2fs_notice(sbi, "Unaligned curseg[%d] with write pointer: "
  4079. "curseg[0x%x,0x%x] wp[0x%x,0x%x]",
  4080. type, cs->segno, cs->next_blkoff, wp_segno, wp_blkoff);
  4081. f2fs_notice(sbi, "Assign new section to curseg[%d]: "
  4082. "curseg[0x%x,0x%x]", type, cs->segno, cs->next_blkoff);
  4083. f2fs_allocate_new_section(sbi, type, true);
  4084. /* check consistency of the zone curseg pointed to */
  4085. if (check_zone_write_pointer(sbi, zbd, &zone))
  4086. return -EIO;
  4087. /* check newly assigned zone */
  4088. cs_section = GET_SEC_FROM_SEG(sbi, cs->segno);
  4089. cs_zone_block = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, cs_section));
  4090. zbd = get_target_zoned_dev(sbi, cs_zone_block);
  4091. if (!zbd)
  4092. return 0;
  4093. zone_sector = (sector_t)(cs_zone_block - zbd->start_blk)
  4094. << log_sectors_per_block;
  4095. err = blkdev_report_zones(zbd->bdev, zone_sector, 1,
  4096. report_one_zone_cb, &zone);
  4097. if (err != 1) {
  4098. f2fs_err(sbi, "Report zone failed: %s errno=(%d)",
  4099. zbd->path, err);
  4100. return err;
  4101. }
  4102. if (zone.type != BLK_ZONE_TYPE_SEQWRITE_REQ)
  4103. return 0;
  4104. if (zone.wp != zone.start) {
  4105. f2fs_notice(sbi,
  4106. "New zone for curseg[%d] is not yet discarded. "
  4107. "Reset the zone: curseg[0x%x,0x%x]",
  4108. type, cs->segno, cs->next_blkoff);
  4109. err = __f2fs_issue_discard_zone(sbi, zbd->bdev,
  4110. zone_sector >> log_sectors_per_block,
  4111. zone.len >> log_sectors_per_block);
  4112. if (err) {
  4113. f2fs_err(sbi, "Discard zone failed: %s (errno=%d)",
  4114. zbd->path, err);
  4115. return err;
  4116. }
  4117. }
  4118. return 0;
  4119. }
  4120. int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
  4121. {
  4122. int i, ret;
  4123. for (i = 0; i < NR_PERSISTENT_LOG; i++) {
  4124. ret = fix_curseg_write_pointer(sbi, i);
  4125. if (ret)
  4126. return ret;
  4127. }
  4128. return 0;
  4129. }
  4130. struct check_zone_write_pointer_args {
  4131. struct f2fs_sb_info *sbi;
  4132. struct f2fs_dev_info *fdev;
  4133. };
  4134. static int check_zone_write_pointer_cb(struct blk_zone *zone, unsigned int idx,
  4135. void *data)
  4136. {
  4137. struct check_zone_write_pointer_args *args;
  4138. args = (struct check_zone_write_pointer_args *)data;
  4139. return check_zone_write_pointer(args->sbi, args->fdev, zone);
  4140. }
  4141. int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
  4142. {
  4143. int i, ret;
  4144. struct check_zone_write_pointer_args args;
  4145. for (i = 0; i < sbi->s_ndevs; i++) {
  4146. if (!bdev_is_zoned(FDEV(i).bdev))
  4147. continue;
  4148. args.sbi = sbi;
  4149. args.fdev = &FDEV(i);
  4150. ret = blkdev_report_zones(FDEV(i).bdev, 0, BLK_ALL_ZONES,
  4151. check_zone_write_pointer_cb, &args);
  4152. if (ret < 0)
  4153. return ret;
  4154. }
  4155. return 0;
  4156. }
  4157. static bool is_conv_zone(struct f2fs_sb_info *sbi, unsigned int zone_idx,
  4158. unsigned int dev_idx)
  4159. {
  4160. if (!bdev_is_zoned(FDEV(dev_idx).bdev))
  4161. return true;
  4162. return !test_bit(zone_idx, FDEV(dev_idx).blkz_seq);
  4163. }
  4164. /* Return the zone index in the given device */
  4165. static unsigned int get_zone_idx(struct f2fs_sb_info *sbi, unsigned int secno,
  4166. int dev_idx)
  4167. {
  4168. block_t sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
  4169. return (sec_start_blkaddr - FDEV(dev_idx).start_blk) >>
  4170. sbi->log_blocks_per_blkz;
  4171. }
  4172. /*
  4173. * Return the usable segments in a section based on the zone's
  4174. * corresponding zone capacity. Zone is equal to a section.
  4175. */
  4176. static inline unsigned int f2fs_usable_zone_segs_in_sec(
  4177. struct f2fs_sb_info *sbi, unsigned int segno)
  4178. {
  4179. unsigned int dev_idx, zone_idx, unusable_segs_in_sec;
  4180. dev_idx = f2fs_target_device_index(sbi, START_BLOCK(sbi, segno));
  4181. zone_idx = get_zone_idx(sbi, GET_SEC_FROM_SEG(sbi, segno), dev_idx);
  4182. /* Conventional zone's capacity is always equal to zone size */
  4183. if (is_conv_zone(sbi, zone_idx, dev_idx))
  4184. return sbi->segs_per_sec;
  4185. /*
  4186. * If the zone_capacity_blocks array is NULL, then zone capacity
  4187. * is equal to the zone size for all zones
  4188. */
  4189. if (!FDEV(dev_idx).zone_capacity_blocks)
  4190. return sbi->segs_per_sec;
  4191. /* Get the segment count beyond zone capacity block */
  4192. unusable_segs_in_sec = (sbi->blocks_per_blkz -
  4193. FDEV(dev_idx).zone_capacity_blocks[zone_idx]) >>
  4194. sbi->log_blocks_per_seg;
  4195. return sbi->segs_per_sec - unusable_segs_in_sec;
  4196. }
  4197. /*
  4198. * Return the number of usable blocks in a segment. The number of blocks
  4199. * returned is always equal to the number of blocks in a segment for
  4200. * segments fully contained within a sequential zone capacity or a
  4201. * conventional zone. For segments partially contained in a sequential
  4202. * zone capacity, the number of usable blocks up to the zone capacity
  4203. * is returned. 0 is returned in all other cases.
  4204. */
  4205. static inline unsigned int f2fs_usable_zone_blks_in_seg(
  4206. struct f2fs_sb_info *sbi, unsigned int segno)
  4207. {
  4208. block_t seg_start, sec_start_blkaddr, sec_cap_blkaddr;
  4209. unsigned int zone_idx, dev_idx, secno;
  4210. secno = GET_SEC_FROM_SEG(sbi, segno);
  4211. seg_start = START_BLOCK(sbi, segno);
  4212. dev_idx = f2fs_target_device_index(sbi, seg_start);
  4213. zone_idx = get_zone_idx(sbi, secno, dev_idx);
  4214. /*
  4215. * Conventional zone's capacity is always equal to zone size,
  4216. * so, blocks per segment is unchanged.
  4217. */
  4218. if (is_conv_zone(sbi, zone_idx, dev_idx))
  4219. return sbi->blocks_per_seg;
  4220. if (!FDEV(dev_idx).zone_capacity_blocks)
  4221. return sbi->blocks_per_seg;
  4222. sec_start_blkaddr = START_BLOCK(sbi, GET_SEG_FROM_SEC(sbi, secno));
  4223. sec_cap_blkaddr = sec_start_blkaddr +
  4224. FDEV(dev_idx).zone_capacity_blocks[zone_idx];
  4225. /*
  4226. * If segment starts before zone capacity and spans beyond
  4227. * zone capacity, then usable blocks are from seg start to
  4228. * zone capacity. If the segment starts after the zone capacity,
  4229. * then there are no usable blocks.
  4230. */
  4231. if (seg_start >= sec_cap_blkaddr)
  4232. return 0;
  4233. if (seg_start + sbi->blocks_per_seg > sec_cap_blkaddr)
  4234. return sec_cap_blkaddr - seg_start;
  4235. return sbi->blocks_per_seg;
  4236. }
  4237. #else
  4238. int f2fs_fix_curseg_write_pointer(struct f2fs_sb_info *sbi)
  4239. {
  4240. return 0;
  4241. }
  4242. int f2fs_check_write_pointer(struct f2fs_sb_info *sbi)
  4243. {
  4244. return 0;
  4245. }
  4246. static inline unsigned int f2fs_usable_zone_blks_in_seg(struct f2fs_sb_info *sbi,
  4247. unsigned int segno)
  4248. {
  4249. return 0;
  4250. }
  4251. static inline unsigned int f2fs_usable_zone_segs_in_sec(struct f2fs_sb_info *sbi,
  4252. unsigned int segno)
  4253. {
  4254. return 0;
  4255. }
  4256. #endif
  4257. unsigned int f2fs_usable_blks_in_seg(struct f2fs_sb_info *sbi,
  4258. unsigned int segno)
  4259. {
  4260. if (f2fs_sb_has_blkzoned(sbi))
  4261. return f2fs_usable_zone_blks_in_seg(sbi, segno);
  4262. return sbi->blocks_per_seg;
  4263. }
  4264. unsigned int f2fs_usable_segs_in_sec(struct f2fs_sb_info *sbi,
  4265. unsigned int segno)
  4266. {
  4267. if (f2fs_sb_has_blkzoned(sbi))
  4268. return f2fs_usable_zone_segs_in_sec(sbi, segno);
  4269. return sbi->segs_per_sec;
  4270. }
  4271. /*
  4272. * Update min, max modified time for cost-benefit GC algorithm
  4273. */
  4274. static void init_min_max_mtime(struct f2fs_sb_info *sbi)
  4275. {
  4276. struct sit_info *sit_i = SIT_I(sbi);
  4277. unsigned int segno;
  4278. down_write(&sit_i->sentry_lock);
  4279. sit_i->min_mtime = ULLONG_MAX;
  4280. for (segno = 0; segno < MAIN_SEGS(sbi); segno += sbi->segs_per_sec) {
  4281. unsigned int i;
  4282. unsigned long long mtime = 0;
  4283. for (i = 0; i < sbi->segs_per_sec; i++)
  4284. mtime += get_seg_entry(sbi, segno + i)->mtime;
  4285. mtime = div_u64(mtime, sbi->segs_per_sec);
  4286. if (sit_i->min_mtime > mtime)
  4287. sit_i->min_mtime = mtime;
  4288. }
  4289. sit_i->max_mtime = get_mtime(sbi, false);
  4290. sit_i->dirty_max_mtime = 0;
  4291. up_write(&sit_i->sentry_lock);
  4292. }
  4293. int f2fs_build_segment_manager(struct f2fs_sb_info *sbi)
  4294. {
  4295. struct f2fs_super_block *raw_super = F2FS_RAW_SUPER(sbi);
  4296. struct f2fs_checkpoint *ckpt = F2FS_CKPT(sbi);
  4297. struct f2fs_sm_info *sm_info;
  4298. int err;
  4299. sm_info = f2fs_kzalloc(sbi, sizeof(struct f2fs_sm_info), GFP_KERNEL);
  4300. if (!sm_info)
  4301. return -ENOMEM;
  4302. /* init sm info */
  4303. sbi->sm_info = sm_info;
  4304. sm_info->seg0_blkaddr = le32_to_cpu(raw_super->segment0_blkaddr);
  4305. sm_info->main_blkaddr = le32_to_cpu(raw_super->main_blkaddr);
  4306. sm_info->segment_count = le32_to_cpu(raw_super->segment_count);
  4307. sm_info->reserved_segments = le32_to_cpu(ckpt->rsvd_segment_count);
  4308. sm_info->ovp_segments = le32_to_cpu(ckpt->overprov_segment_count);
  4309. sm_info->main_segments = le32_to_cpu(raw_super->segment_count_main);
  4310. sm_info->ssa_blkaddr = le32_to_cpu(raw_super->ssa_blkaddr);
  4311. sm_info->rec_prefree_segments = sm_info->main_segments *
  4312. DEF_RECLAIM_PREFREE_SEGMENTS / 100;
  4313. if (sm_info->rec_prefree_segments > DEF_MAX_RECLAIM_PREFREE_SEGMENTS)
  4314. sm_info->rec_prefree_segments = DEF_MAX_RECLAIM_PREFREE_SEGMENTS;
  4315. if (!f2fs_lfs_mode(sbi))
  4316. sm_info->ipu_policy = 1 << F2FS_IPU_FSYNC;
  4317. sm_info->min_ipu_util = DEF_MIN_IPU_UTIL;
  4318. sm_info->min_fsync_blocks = DEF_MIN_FSYNC_BLOCKS;
  4319. sm_info->min_seq_blocks = sbi->blocks_per_seg * sbi->segs_per_sec;
  4320. sm_info->min_hot_blocks = DEF_MIN_HOT_BLOCKS;
  4321. sm_info->min_ssr_sections = reserved_sections(sbi);
  4322. INIT_LIST_HEAD(&sm_info->sit_entry_set);
  4323. init_f2fs_rwsem(&sm_info->curseg_lock);
  4324. if (!f2fs_readonly(sbi->sb)) {
  4325. err = f2fs_create_flush_cmd_control(sbi);
  4326. if (err)
  4327. return err;
  4328. }
  4329. err = create_discard_cmd_control(sbi);
  4330. if (err)
  4331. return err;
  4332. err = build_sit_info(sbi);
  4333. if (err)
  4334. return err;
  4335. err = build_free_segmap(sbi);
  4336. if (err)
  4337. return err;
  4338. err = build_curseg(sbi);
  4339. if (err)
  4340. return err;
  4341. /* reinit free segmap based on SIT */
  4342. err = build_sit_entries(sbi);
  4343. if (err)
  4344. return err;
  4345. init_free_segmap(sbi);
  4346. err = build_dirty_segmap(sbi);
  4347. if (err)
  4348. return err;
  4349. err = sanity_check_curseg(sbi);
  4350. if (err)
  4351. return err;
  4352. init_min_max_mtime(sbi);
  4353. return 0;
  4354. }
  4355. static void discard_dirty_segmap(struct f2fs_sb_info *sbi,
  4356. enum dirty_type dirty_type)
  4357. {
  4358. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  4359. mutex_lock(&dirty_i->seglist_lock);
  4360. kvfree(dirty_i->dirty_segmap[dirty_type]);
  4361. dirty_i->nr_dirty[dirty_type] = 0;
  4362. mutex_unlock(&dirty_i->seglist_lock);
  4363. }
  4364. static void destroy_victim_secmap(struct f2fs_sb_info *sbi)
  4365. {
  4366. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  4367. kvfree(dirty_i->victim_secmap);
  4368. }
  4369. static void destroy_dirty_segmap(struct f2fs_sb_info *sbi)
  4370. {
  4371. struct dirty_seglist_info *dirty_i = DIRTY_I(sbi);
  4372. int i;
  4373. if (!dirty_i)
  4374. return;
  4375. /* discard pre-free/dirty segments list */
  4376. for (i = 0; i < NR_DIRTY_TYPE; i++)
  4377. discard_dirty_segmap(sbi, i);
  4378. if (__is_large_section(sbi)) {
  4379. mutex_lock(&dirty_i->seglist_lock);
  4380. kvfree(dirty_i->dirty_secmap);
  4381. mutex_unlock(&dirty_i->seglist_lock);
  4382. }
  4383. destroy_victim_secmap(sbi);
  4384. SM_I(sbi)->dirty_info = NULL;
  4385. kfree(dirty_i);
  4386. }
  4387. static void destroy_curseg(struct f2fs_sb_info *sbi)
  4388. {
  4389. struct curseg_info *array = SM_I(sbi)->curseg_array;
  4390. int i;
  4391. if (!array)
  4392. return;
  4393. SM_I(sbi)->curseg_array = NULL;
  4394. for (i = 0; i < NR_CURSEG_TYPE; i++) {
  4395. kfree(array[i].sum_blk);
  4396. kfree(array[i].journal);
  4397. }
  4398. kfree(array);
  4399. }
  4400. static void destroy_free_segmap(struct f2fs_sb_info *sbi)
  4401. {
  4402. struct free_segmap_info *free_i = SM_I(sbi)->free_info;
  4403. if (!free_i)
  4404. return;
  4405. SM_I(sbi)->free_info = NULL;
  4406. kvfree(free_i->free_segmap);
  4407. kvfree(free_i->free_secmap);
  4408. kfree(free_i);
  4409. }
  4410. static void destroy_sit_info(struct f2fs_sb_info *sbi)
  4411. {
  4412. struct sit_info *sit_i = SIT_I(sbi);
  4413. if (!sit_i)
  4414. return;
  4415. if (sit_i->sentries)
  4416. kvfree(sit_i->bitmap);
  4417. kfree(sit_i->tmp_map);
  4418. kvfree(sit_i->sentries);
  4419. kvfree(sit_i->sec_entries);
  4420. kvfree(sit_i->dirty_sentries_bitmap);
  4421. SM_I(sbi)->sit_info = NULL;
  4422. kvfree(sit_i->sit_bitmap);
  4423. #ifdef CONFIG_F2FS_CHECK_FS
  4424. kvfree(sit_i->sit_bitmap_mir);
  4425. kvfree(sit_i->invalid_segmap);
  4426. #endif
  4427. kfree(sit_i);
  4428. }
  4429. void f2fs_destroy_segment_manager(struct f2fs_sb_info *sbi)
  4430. {
  4431. struct f2fs_sm_info *sm_info = SM_I(sbi);
  4432. if (!sm_info)
  4433. return;
  4434. f2fs_destroy_flush_cmd_control(sbi, true);
  4435. destroy_discard_cmd_control(sbi);
  4436. destroy_dirty_segmap(sbi);
  4437. destroy_curseg(sbi);
  4438. destroy_free_segmap(sbi);
  4439. destroy_sit_info(sbi);
  4440. sbi->sm_info = NULL;
  4441. kfree(sm_info);
  4442. }
  4443. int __init f2fs_create_segment_manager_caches(void)
  4444. {
  4445. discard_entry_slab = f2fs_kmem_cache_create("f2fs_discard_entry",
  4446. sizeof(struct discard_entry));
  4447. if (!discard_entry_slab)
  4448. goto fail;
  4449. discard_cmd_slab = f2fs_kmem_cache_create("f2fs_discard_cmd",
  4450. sizeof(struct discard_cmd));
  4451. if (!discard_cmd_slab)
  4452. goto destroy_discard_entry;
  4453. sit_entry_set_slab = f2fs_kmem_cache_create("f2fs_sit_entry_set",
  4454. sizeof(struct sit_entry_set));
  4455. if (!sit_entry_set_slab)
  4456. goto destroy_discard_cmd;
  4457. inmem_entry_slab = f2fs_kmem_cache_create("f2fs_inmem_page_entry",
  4458. sizeof(struct inmem_pages));
  4459. if (!inmem_entry_slab)
  4460. goto destroy_sit_entry_set;
  4461. return 0;
  4462. destroy_sit_entry_set:
  4463. kmem_cache_destroy(sit_entry_set_slab);
  4464. destroy_discard_cmd:
  4465. kmem_cache_destroy(discard_cmd_slab);
  4466. destroy_discard_entry:
  4467. kmem_cache_destroy(discard_entry_slab);
  4468. fail:
  4469. return -ENOMEM;
  4470. }
  4471. void f2fs_destroy_segment_manager_caches(void)
  4472. {
  4473. kmem_cache_destroy(sit_entry_set_slab);
  4474. kmem_cache_destroy(discard_cmd_slab);
  4475. kmem_cache_destroy(discard_entry_slab);
  4476. kmem_cache_destroy(inmem_entry_slab);
  4477. }