dm-zoned-metadata.c 72 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2017 Western Digital Corporation or its affiliates.
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include "dm-zoned.h"
  8. #include <linux/module.h>
  9. #include <linux/crc32.h>
  10. #include <linux/sched/mm.h>
  11. #define DM_MSG_PREFIX "zoned metadata"
  12. /*
  13. * Metadata version.
  14. */
  15. #define DMZ_META_VER 2
  16. /*
  17. * On-disk super block magic.
  18. */
  19. #define DMZ_MAGIC ((((unsigned int)('D')) << 24) | \
  20. (((unsigned int)('Z')) << 16) | \
  21. (((unsigned int)('B')) << 8) | \
  22. ((unsigned int)('D')))
  23. /*
  24. * On disk super block.
  25. * This uses only 512 B but uses on disk a full 4KB block. This block is
  26. * followed on disk by the mapping table of chunks to zones and the bitmap
  27. * blocks indicating zone block validity.
  28. * The overall resulting metadata format is:
  29. * (1) Super block (1 block)
  30. * (2) Chunk mapping table (nr_map_blocks)
  31. * (3) Bitmap blocks (nr_bitmap_blocks)
  32. * All metadata blocks are stored in conventional zones, starting from
  33. * the first conventional zone found on disk.
  34. */
  35. struct dmz_super {
  36. /* Magic number */
  37. __le32 magic; /* 4 */
  38. /* Metadata version number */
  39. __le32 version; /* 8 */
  40. /* Generation number */
  41. __le64 gen; /* 16 */
  42. /* This block number */
  43. __le64 sb_block; /* 24 */
  44. /* The number of metadata blocks, including this super block */
  45. __le32 nr_meta_blocks; /* 28 */
  46. /* The number of sequential zones reserved for reclaim */
  47. __le32 nr_reserved_seq; /* 32 */
  48. /* The number of entries in the mapping table */
  49. __le32 nr_chunks; /* 36 */
  50. /* The number of blocks used for the chunk mapping table */
  51. __le32 nr_map_blocks; /* 40 */
  52. /* The number of blocks used for the block bitmaps */
  53. __le32 nr_bitmap_blocks; /* 44 */
  54. /* Checksum */
  55. __le32 crc; /* 48 */
  56. /* DM-Zoned label */
  57. u8 dmz_label[32]; /* 80 */
  58. /* DM-Zoned UUID */
  59. u8 dmz_uuid[16]; /* 96 */
  60. /* Device UUID */
  61. u8 dev_uuid[16]; /* 112 */
  62. /* Padding to full 512B sector */
  63. u8 reserved[400]; /* 512 */
  64. };
  65. /*
  66. * Chunk mapping entry: entries are indexed by chunk number
  67. * and give the zone ID (dzone_id) mapping the chunk on disk.
  68. * This zone may be sequential or random. If it is a sequential
  69. * zone, a second zone (bzone_id) used as a write buffer may
  70. * also be specified. This second zone will always be a randomly
  71. * writeable zone.
  72. */
  73. struct dmz_map {
  74. __le32 dzone_id;
  75. __le32 bzone_id;
  76. };
  77. /*
  78. * Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
  79. */
  80. #define DMZ_MAP_ENTRIES (DMZ_BLOCK_SIZE / sizeof(struct dmz_map))
  81. #define DMZ_MAP_ENTRIES_SHIFT (ilog2(DMZ_MAP_ENTRIES))
  82. #define DMZ_MAP_ENTRIES_MASK (DMZ_MAP_ENTRIES - 1)
  83. #define DMZ_MAP_UNMAPPED UINT_MAX
  84. /*
  85. * Meta data block descriptor (for cached metadata blocks).
  86. */
  87. struct dmz_mblock {
  88. struct rb_node node;
  89. struct list_head link;
  90. sector_t no;
  91. unsigned int ref;
  92. unsigned long state;
  93. struct page *page;
  94. void *data;
  95. };
  96. /*
  97. * Metadata block state flags.
  98. */
  99. enum {
  100. DMZ_META_DIRTY,
  101. DMZ_META_READING,
  102. DMZ_META_WRITING,
  103. DMZ_META_ERROR,
  104. };
  105. /*
  106. * Super block information (one per metadata set).
  107. */
  108. struct dmz_sb {
  109. sector_t block;
  110. struct dmz_dev *dev;
  111. struct dmz_mblock *mblk;
  112. struct dmz_super *sb;
  113. struct dm_zone *zone;
  114. };
  115. /*
  116. * In-memory metadata.
  117. */
  118. struct dmz_metadata {
  119. struct dmz_dev *dev;
  120. unsigned int nr_devs;
  121. char devname[BDEVNAME_SIZE];
  122. char label[BDEVNAME_SIZE];
  123. uuid_t uuid;
  124. sector_t zone_bitmap_size;
  125. unsigned int zone_nr_bitmap_blocks;
  126. unsigned int zone_bits_per_mblk;
  127. sector_t zone_nr_blocks;
  128. sector_t zone_nr_blocks_shift;
  129. sector_t zone_nr_sectors;
  130. sector_t zone_nr_sectors_shift;
  131. unsigned int nr_bitmap_blocks;
  132. unsigned int nr_map_blocks;
  133. unsigned int nr_zones;
  134. unsigned int nr_useable_zones;
  135. unsigned int nr_meta_blocks;
  136. unsigned int nr_meta_zones;
  137. unsigned int nr_data_zones;
  138. unsigned int nr_cache_zones;
  139. unsigned int nr_rnd_zones;
  140. unsigned int nr_reserved_seq;
  141. unsigned int nr_chunks;
  142. /* Zone information array */
  143. struct xarray zones;
  144. struct dmz_sb sb[2];
  145. unsigned int mblk_primary;
  146. unsigned int sb_version;
  147. u64 sb_gen;
  148. unsigned int min_nr_mblks;
  149. unsigned int max_nr_mblks;
  150. atomic_t nr_mblks;
  151. struct rw_semaphore mblk_sem;
  152. struct mutex mblk_flush_lock;
  153. spinlock_t mblk_lock;
  154. struct rb_root mblk_rbtree;
  155. struct list_head mblk_lru_list;
  156. struct list_head mblk_dirty_list;
  157. struct shrinker mblk_shrinker;
  158. /* Zone allocation management */
  159. struct mutex map_lock;
  160. struct dmz_mblock **map_mblk;
  161. unsigned int nr_cache;
  162. atomic_t unmap_nr_cache;
  163. struct list_head unmap_cache_list;
  164. struct list_head map_cache_list;
  165. atomic_t nr_reserved_seq_zones;
  166. struct list_head reserved_seq_zones_list;
  167. wait_queue_head_t free_wq;
  168. };
  169. #define dmz_zmd_info(zmd, format, args...) \
  170. DMINFO("(%s): " format, (zmd)->label, ## args)
  171. #define dmz_zmd_err(zmd, format, args...) \
  172. DMERR("(%s): " format, (zmd)->label, ## args)
  173. #define dmz_zmd_warn(zmd, format, args...) \
  174. DMWARN("(%s): " format, (zmd)->label, ## args)
  175. #define dmz_zmd_debug(zmd, format, args...) \
  176. DMDEBUG("(%s): " format, (zmd)->label, ## args)
  177. /*
  178. * Various accessors
  179. */
  180. static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)
  181. {
  182. if (WARN_ON(!zone))
  183. return 0;
  184. return zone->id - zone->dev->zone_offset;
  185. }
  186. sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
  187. {
  188. unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
  189. return (sector_t)zone_id << zmd->zone_nr_sectors_shift;
  190. }
  191. sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
  192. {
  193. unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
  194. return (sector_t)zone_id << zmd->zone_nr_blocks_shift;
  195. }
  196. unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)
  197. {
  198. return zmd->zone_nr_blocks;
  199. }
  200. unsigned int dmz_zone_nr_blocks_shift(struct dmz_metadata *zmd)
  201. {
  202. return zmd->zone_nr_blocks_shift;
  203. }
  204. unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)
  205. {
  206. return zmd->zone_nr_sectors;
  207. }
  208. unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)
  209. {
  210. return zmd->zone_nr_sectors_shift;
  211. }
  212. unsigned int dmz_nr_zones(struct dmz_metadata *zmd)
  213. {
  214. return zmd->nr_zones;
  215. }
  216. unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
  217. {
  218. return zmd->nr_chunks;
  219. }
  220. unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)
  221. {
  222. return zmd->dev[idx].nr_rnd;
  223. }
  224. unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)
  225. {
  226. return atomic_read(&zmd->dev[idx].unmap_nr_rnd);
  227. }
  228. unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)
  229. {
  230. return zmd->nr_cache;
  231. }
  232. unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)
  233. {
  234. return atomic_read(&zmd->unmap_nr_cache);
  235. }
  236. unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)
  237. {
  238. return zmd->dev[idx].nr_seq;
  239. }
  240. unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)
  241. {
  242. return atomic_read(&zmd->dev[idx].unmap_nr_seq);
  243. }
  244. static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
  245. {
  246. return xa_load(&zmd->zones, zone_id);
  247. }
  248. static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,
  249. unsigned int zone_id, struct dmz_dev *dev)
  250. {
  251. struct dm_zone *zone = kzalloc(sizeof(struct dm_zone), GFP_KERNEL);
  252. if (!zone)
  253. return ERR_PTR(-ENOMEM);
  254. if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {
  255. kfree(zone);
  256. return ERR_PTR(-EBUSY);
  257. }
  258. INIT_LIST_HEAD(&zone->link);
  259. atomic_set(&zone->refcount, 0);
  260. zone->id = zone_id;
  261. zone->chunk = DMZ_MAP_UNMAPPED;
  262. zone->dev = dev;
  263. return zone;
  264. }
  265. const char *dmz_metadata_label(struct dmz_metadata *zmd)
  266. {
  267. return (const char *)zmd->label;
  268. }
  269. bool dmz_check_dev(struct dmz_metadata *zmd)
  270. {
  271. unsigned int i;
  272. for (i = 0; i < zmd->nr_devs; i++) {
  273. if (!dmz_check_bdev(&zmd->dev[i]))
  274. return false;
  275. }
  276. return true;
  277. }
  278. bool dmz_dev_is_dying(struct dmz_metadata *zmd)
  279. {
  280. unsigned int i;
  281. for (i = 0; i < zmd->nr_devs; i++) {
  282. if (dmz_bdev_is_dying(&zmd->dev[i]))
  283. return true;
  284. }
  285. return false;
  286. }
  287. /*
  288. * Lock/unlock mapping table.
  289. * The map lock also protects all the zone lists.
  290. */
  291. void dmz_lock_map(struct dmz_metadata *zmd)
  292. {
  293. mutex_lock(&zmd->map_lock);
  294. }
  295. void dmz_unlock_map(struct dmz_metadata *zmd)
  296. {
  297. mutex_unlock(&zmd->map_lock);
  298. }
  299. /*
  300. * Lock/unlock metadata access. This is a "read" lock on a semaphore
  301. * that prevents metadata flush from running while metadata are being
  302. * modified. The actual metadata write mutual exclusion is achieved with
  303. * the map lock and zone state management (active and reclaim state are
  304. * mutually exclusive).
  305. */
  306. void dmz_lock_metadata(struct dmz_metadata *zmd)
  307. {
  308. down_read(&zmd->mblk_sem);
  309. }
  310. void dmz_unlock_metadata(struct dmz_metadata *zmd)
  311. {
  312. up_read(&zmd->mblk_sem);
  313. }
  314. /*
  315. * Lock/unlock flush: prevent concurrent executions
  316. * of dmz_flush_metadata as well as metadata modification in reclaim
  317. * while flush is being executed.
  318. */
  319. void dmz_lock_flush(struct dmz_metadata *zmd)
  320. {
  321. mutex_lock(&zmd->mblk_flush_lock);
  322. }
  323. void dmz_unlock_flush(struct dmz_metadata *zmd)
  324. {
  325. mutex_unlock(&zmd->mblk_flush_lock);
  326. }
  327. /*
  328. * Allocate a metadata block.
  329. */
  330. static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,
  331. sector_t mblk_no)
  332. {
  333. struct dmz_mblock *mblk = NULL;
  334. /* See if we can reuse cached blocks */
  335. if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {
  336. spin_lock(&zmd->mblk_lock);
  337. mblk = list_first_entry_or_null(&zmd->mblk_lru_list,
  338. struct dmz_mblock, link);
  339. if (mblk) {
  340. list_del_init(&mblk->link);
  341. rb_erase(&mblk->node, &zmd->mblk_rbtree);
  342. mblk->no = mblk_no;
  343. }
  344. spin_unlock(&zmd->mblk_lock);
  345. if (mblk)
  346. return mblk;
  347. }
  348. /* Allocate a new block */
  349. mblk = kmalloc(sizeof(struct dmz_mblock), GFP_NOIO);
  350. if (!mblk)
  351. return NULL;
  352. mblk->page = alloc_page(GFP_NOIO);
  353. if (!mblk->page) {
  354. kfree(mblk);
  355. return NULL;
  356. }
  357. RB_CLEAR_NODE(&mblk->node);
  358. INIT_LIST_HEAD(&mblk->link);
  359. mblk->ref = 0;
  360. mblk->state = 0;
  361. mblk->no = mblk_no;
  362. mblk->data = page_address(mblk->page);
  363. atomic_inc(&zmd->nr_mblks);
  364. return mblk;
  365. }
  366. /*
  367. * Free a metadata block.
  368. */
  369. static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
  370. {
  371. __free_pages(mblk->page, 0);
  372. kfree(mblk);
  373. atomic_dec(&zmd->nr_mblks);
  374. }
  375. /*
  376. * Insert a metadata block in the rbtree.
  377. */
  378. static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
  379. {
  380. struct rb_root *root = &zmd->mblk_rbtree;
  381. struct rb_node **new = &(root->rb_node), *parent = NULL;
  382. struct dmz_mblock *b;
  383. /* Figure out where to put the new node */
  384. while (*new) {
  385. b = container_of(*new, struct dmz_mblock, node);
  386. parent = *new;
  387. new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right);
  388. }
  389. /* Add new node and rebalance tree */
  390. rb_link_node(&mblk->node, parent, new);
  391. rb_insert_color(&mblk->node, root);
  392. }
  393. /*
  394. * Lookup a metadata block in the rbtree. If the block is found, increment
  395. * its reference count.
  396. */
  397. static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,
  398. sector_t mblk_no)
  399. {
  400. struct rb_root *root = &zmd->mblk_rbtree;
  401. struct rb_node *node = root->rb_node;
  402. struct dmz_mblock *mblk;
  403. while (node) {
  404. mblk = container_of(node, struct dmz_mblock, node);
  405. if (mblk->no == mblk_no) {
  406. /*
  407. * If this is the first reference to the block,
  408. * remove it from the LRU list.
  409. */
  410. mblk->ref++;
  411. if (mblk->ref == 1 &&
  412. !test_bit(DMZ_META_DIRTY, &mblk->state))
  413. list_del_init(&mblk->link);
  414. return mblk;
  415. }
  416. node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right;
  417. }
  418. return NULL;
  419. }
  420. /*
  421. * Metadata block BIO end callback.
  422. */
  423. static void dmz_mblock_bio_end_io(struct bio *bio)
  424. {
  425. struct dmz_mblock *mblk = bio->bi_private;
  426. int flag;
  427. if (bio->bi_status)
  428. set_bit(DMZ_META_ERROR, &mblk->state);
  429. if (bio_op(bio) == REQ_OP_WRITE)
  430. flag = DMZ_META_WRITING;
  431. else
  432. flag = DMZ_META_READING;
  433. clear_bit_unlock(flag, &mblk->state);
  434. smp_mb__after_atomic();
  435. wake_up_bit(&mblk->state, flag);
  436. bio_put(bio);
  437. }
  438. /*
  439. * Read an uncached metadata block from disk and add it to the cache.
  440. */
  441. static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
  442. sector_t mblk_no)
  443. {
  444. struct dmz_mblock *mblk, *m;
  445. sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
  446. struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
  447. struct bio *bio;
  448. if (dmz_bdev_is_dying(dev))
  449. return ERR_PTR(-EIO);
  450. /* Get a new block and a BIO to read it */
  451. mblk = dmz_alloc_mblock(zmd, mblk_no);
  452. if (!mblk)
  453. return ERR_PTR(-ENOMEM);
  454. bio = bio_alloc(GFP_NOIO, 1);
  455. if (!bio) {
  456. dmz_free_mblock(zmd, mblk);
  457. return ERR_PTR(-ENOMEM);
  458. }
  459. spin_lock(&zmd->mblk_lock);
  460. /*
  461. * Make sure that another context did not start reading
  462. * the block already.
  463. */
  464. m = dmz_get_mblock_fast(zmd, mblk_no);
  465. if (m) {
  466. spin_unlock(&zmd->mblk_lock);
  467. dmz_free_mblock(zmd, mblk);
  468. bio_put(bio);
  469. return m;
  470. }
  471. mblk->ref++;
  472. set_bit(DMZ_META_READING, &mblk->state);
  473. dmz_insert_mblock(zmd, mblk);
  474. spin_unlock(&zmd->mblk_lock);
  475. /* Submit read BIO */
  476. bio->bi_iter.bi_sector = dmz_blk2sect(block);
  477. bio_set_dev(bio, dev->bdev);
  478. bio->bi_private = mblk;
  479. bio->bi_end_io = dmz_mblock_bio_end_io;
  480. bio_set_op_attrs(bio, REQ_OP_READ, REQ_META | REQ_PRIO);
  481. bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
  482. submit_bio(bio);
  483. return mblk;
  484. }
  485. /*
  486. * Free metadata blocks.
  487. */
  488. static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,
  489. unsigned long limit)
  490. {
  491. struct dmz_mblock *mblk;
  492. unsigned long count = 0;
  493. if (!zmd->max_nr_mblks)
  494. return 0;
  495. while (!list_empty(&zmd->mblk_lru_list) &&
  496. atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&
  497. count < limit) {
  498. mblk = list_first_entry(&zmd->mblk_lru_list,
  499. struct dmz_mblock, link);
  500. list_del_init(&mblk->link);
  501. rb_erase(&mblk->node, &zmd->mblk_rbtree);
  502. dmz_free_mblock(zmd, mblk);
  503. count++;
  504. }
  505. return count;
  506. }
  507. /*
  508. * For mblock shrinker: get the number of unused metadata blocks in the cache.
  509. */
  510. static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink,
  511. struct shrink_control *sc)
  512. {
  513. struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
  514. return atomic_read(&zmd->nr_mblks);
  515. }
  516. /*
  517. * For mblock shrinker: scan unused metadata blocks and shrink the cache.
  518. */
  519. static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink,
  520. struct shrink_control *sc)
  521. {
  522. struct dmz_metadata *zmd = container_of(shrink, struct dmz_metadata, mblk_shrinker);
  523. unsigned long count;
  524. spin_lock(&zmd->mblk_lock);
  525. count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);
  526. spin_unlock(&zmd->mblk_lock);
  527. return count ? count : SHRINK_STOP;
  528. }
  529. /*
  530. * Release a metadata block.
  531. */
  532. static void dmz_release_mblock(struct dmz_metadata *zmd,
  533. struct dmz_mblock *mblk)
  534. {
  535. if (!mblk)
  536. return;
  537. spin_lock(&zmd->mblk_lock);
  538. mblk->ref--;
  539. if (mblk->ref == 0) {
  540. if (test_bit(DMZ_META_ERROR, &mblk->state)) {
  541. rb_erase(&mblk->node, &zmd->mblk_rbtree);
  542. dmz_free_mblock(zmd, mblk);
  543. } else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) {
  544. list_add_tail(&mblk->link, &zmd->mblk_lru_list);
  545. dmz_shrink_mblock_cache(zmd, 1);
  546. }
  547. }
  548. spin_unlock(&zmd->mblk_lock);
  549. }
  550. /*
  551. * Get a metadata block from the rbtree. If the block
  552. * is not present, read it from disk.
  553. */
  554. static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
  555. sector_t mblk_no)
  556. {
  557. struct dmz_mblock *mblk;
  558. struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
  559. /* Check rbtree */
  560. spin_lock(&zmd->mblk_lock);
  561. mblk = dmz_get_mblock_fast(zmd, mblk_no);
  562. spin_unlock(&zmd->mblk_lock);
  563. if (!mblk) {
  564. /* Cache miss: read the block from disk */
  565. mblk = dmz_get_mblock_slow(zmd, mblk_no);
  566. if (IS_ERR(mblk))
  567. return mblk;
  568. }
  569. /* Wait for on-going read I/O and check for error */
  570. wait_on_bit_io(&mblk->state, DMZ_META_READING,
  571. TASK_UNINTERRUPTIBLE);
  572. if (test_bit(DMZ_META_ERROR, &mblk->state)) {
  573. dmz_release_mblock(zmd, mblk);
  574. dmz_check_bdev(dev);
  575. return ERR_PTR(-EIO);
  576. }
  577. return mblk;
  578. }
  579. /*
  580. * Mark a metadata block dirty.
  581. */
  582. static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
  583. {
  584. spin_lock(&zmd->mblk_lock);
  585. if (!test_and_set_bit(DMZ_META_DIRTY, &mblk->state))
  586. list_add_tail(&mblk->link, &zmd->mblk_dirty_list);
  587. spin_unlock(&zmd->mblk_lock);
  588. }
  589. /*
  590. * Issue a metadata block write BIO.
  591. */
  592. static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
  593. unsigned int set)
  594. {
  595. struct dmz_dev *dev = zmd->sb[set].dev;
  596. sector_t block = zmd->sb[set].block + mblk->no;
  597. struct bio *bio;
  598. if (dmz_bdev_is_dying(dev))
  599. return -EIO;
  600. bio = bio_alloc(GFP_NOIO, 1);
  601. if (!bio) {
  602. set_bit(DMZ_META_ERROR, &mblk->state);
  603. return -ENOMEM;
  604. }
  605. set_bit(DMZ_META_WRITING, &mblk->state);
  606. bio->bi_iter.bi_sector = dmz_blk2sect(block);
  607. bio_set_dev(bio, dev->bdev);
  608. bio->bi_private = mblk;
  609. bio->bi_end_io = dmz_mblock_bio_end_io;
  610. bio_set_op_attrs(bio, REQ_OP_WRITE, REQ_META | REQ_PRIO);
  611. bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
  612. submit_bio(bio);
  613. return 0;
  614. }
  615. /*
  616. * Read/write a metadata block.
  617. */
  618. static int dmz_rdwr_block(struct dmz_dev *dev, int op,
  619. sector_t block, struct page *page)
  620. {
  621. struct bio *bio;
  622. int ret;
  623. if (WARN_ON(!dev))
  624. return -EIO;
  625. if (dmz_bdev_is_dying(dev))
  626. return -EIO;
  627. bio = bio_alloc(GFP_NOIO, 1);
  628. if (!bio)
  629. return -ENOMEM;
  630. bio->bi_iter.bi_sector = dmz_blk2sect(block);
  631. bio_set_dev(bio, dev->bdev);
  632. bio_set_op_attrs(bio, op, REQ_SYNC | REQ_META | REQ_PRIO);
  633. bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
  634. ret = submit_bio_wait(bio);
  635. bio_put(bio);
  636. if (ret)
  637. dmz_check_bdev(dev);
  638. return ret;
  639. }
  640. /*
  641. * Write super block of the specified metadata set.
  642. */
  643. static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
  644. {
  645. struct dmz_mblock *mblk = zmd->sb[set].mblk;
  646. struct dmz_super *sb = zmd->sb[set].sb;
  647. struct dmz_dev *dev = zmd->sb[set].dev;
  648. sector_t sb_block;
  649. u64 sb_gen = zmd->sb_gen + 1;
  650. int ret;
  651. sb->magic = cpu_to_le32(DMZ_MAGIC);
  652. sb->version = cpu_to_le32(zmd->sb_version);
  653. if (zmd->sb_version > 1) {
  654. BUILD_BUG_ON(UUID_SIZE != 16);
  655. export_uuid(sb->dmz_uuid, &zmd->uuid);
  656. memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);
  657. export_uuid(sb->dev_uuid, &dev->uuid);
  658. }
  659. sb->gen = cpu_to_le64(sb_gen);
  660. /*
  661. * The metadata always references the absolute block address,
  662. * ie relative to the entire block range, not the per-device
  663. * block address.
  664. */
  665. sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;
  666. sb->sb_block = cpu_to_le64(sb_block);
  667. sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
  668. sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
  669. sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
  670. sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);
  671. sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);
  672. sb->crc = 0;
  673. sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
  674. ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
  675. mblk->page);
  676. if (ret == 0)
  677. ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
  678. return ret;
  679. }
  680. /*
  681. * Write dirty metadata blocks to the specified set.
  682. */
  683. static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
  684. struct list_head *write_list,
  685. unsigned int set)
  686. {
  687. struct dmz_mblock *mblk;
  688. struct dmz_dev *dev = zmd->sb[set].dev;
  689. struct blk_plug plug;
  690. int ret = 0, nr_mblks_submitted = 0;
  691. /* Issue writes */
  692. blk_start_plug(&plug);
  693. list_for_each_entry(mblk, write_list, link) {
  694. ret = dmz_write_mblock(zmd, mblk, set);
  695. if (ret)
  696. break;
  697. nr_mblks_submitted++;
  698. }
  699. blk_finish_plug(&plug);
  700. /* Wait for completion */
  701. list_for_each_entry(mblk, write_list, link) {
  702. if (!nr_mblks_submitted)
  703. break;
  704. wait_on_bit_io(&mblk->state, DMZ_META_WRITING,
  705. TASK_UNINTERRUPTIBLE);
  706. if (test_bit(DMZ_META_ERROR, &mblk->state)) {
  707. clear_bit(DMZ_META_ERROR, &mblk->state);
  708. dmz_check_bdev(dev);
  709. ret = -EIO;
  710. }
  711. nr_mblks_submitted--;
  712. }
  713. /* Flush drive cache (this will also sync data) */
  714. if (ret == 0)
  715. ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
  716. return ret;
  717. }
  718. /*
  719. * Log dirty metadata blocks.
  720. */
  721. static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,
  722. struct list_head *write_list)
  723. {
  724. unsigned int log_set = zmd->mblk_primary ^ 0x1;
  725. int ret;
  726. /* Write dirty blocks to the log */
  727. ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);
  728. if (ret)
  729. return ret;
  730. /*
  731. * No error so far: now validate the log by updating the
  732. * log index super block generation.
  733. */
  734. ret = dmz_write_sb(zmd, log_set);
  735. if (ret)
  736. return ret;
  737. return 0;
  738. }
  739. /*
  740. * Flush dirty metadata blocks.
  741. */
  742. int dmz_flush_metadata(struct dmz_metadata *zmd)
  743. {
  744. struct dmz_mblock *mblk;
  745. struct list_head write_list;
  746. struct dmz_dev *dev;
  747. int ret;
  748. if (WARN_ON(!zmd))
  749. return 0;
  750. INIT_LIST_HEAD(&write_list);
  751. /*
  752. * Make sure that metadata blocks are stable before logging: take
  753. * the write lock on the metadata semaphore to prevent target BIOs
  754. * from modifying metadata.
  755. */
  756. down_write(&zmd->mblk_sem);
  757. dev = zmd->sb[zmd->mblk_primary].dev;
  758. /*
  759. * This is called from the target flush work and reclaim work.
  760. * Concurrent execution is not allowed.
  761. */
  762. dmz_lock_flush(zmd);
  763. if (dmz_bdev_is_dying(dev)) {
  764. ret = -EIO;
  765. goto out;
  766. }
  767. /* Get dirty blocks */
  768. spin_lock(&zmd->mblk_lock);
  769. list_splice_init(&zmd->mblk_dirty_list, &write_list);
  770. spin_unlock(&zmd->mblk_lock);
  771. /* If there are no dirty metadata blocks, just flush the device cache */
  772. if (list_empty(&write_list)) {
  773. ret = blkdev_issue_flush(dev->bdev, GFP_NOIO);
  774. goto err;
  775. }
  776. /*
  777. * The primary metadata set is still clean. Keep it this way until
  778. * all updates are successful in the secondary set. That is, use
  779. * the secondary set as a log.
  780. */
  781. ret = dmz_log_dirty_mblocks(zmd, &write_list);
  782. if (ret)
  783. goto err;
  784. /*
  785. * The log is on disk. It is now safe to update in place
  786. * in the primary metadata set.
  787. */
  788. ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);
  789. if (ret)
  790. goto err;
  791. ret = dmz_write_sb(zmd, zmd->mblk_primary);
  792. if (ret)
  793. goto err;
  794. while (!list_empty(&write_list)) {
  795. mblk = list_first_entry(&write_list, struct dmz_mblock, link);
  796. list_del_init(&mblk->link);
  797. spin_lock(&zmd->mblk_lock);
  798. clear_bit(DMZ_META_DIRTY, &mblk->state);
  799. if (mblk->ref == 0)
  800. list_add_tail(&mblk->link, &zmd->mblk_lru_list);
  801. spin_unlock(&zmd->mblk_lock);
  802. }
  803. zmd->sb_gen++;
  804. out:
  805. dmz_unlock_flush(zmd);
  806. up_write(&zmd->mblk_sem);
  807. return ret;
  808. err:
  809. if (!list_empty(&write_list)) {
  810. spin_lock(&zmd->mblk_lock);
  811. list_splice(&write_list, &zmd->mblk_dirty_list);
  812. spin_unlock(&zmd->mblk_lock);
  813. }
  814. if (!dmz_check_bdev(dev))
  815. ret = -EIO;
  816. goto out;
  817. }
  818. /*
  819. * Check super block.
  820. */
  821. static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,
  822. bool tertiary)
  823. {
  824. struct dmz_super *sb = dsb->sb;
  825. struct dmz_dev *dev = dsb->dev;
  826. unsigned int nr_meta_zones, nr_data_zones;
  827. u32 crc, stored_crc;
  828. u64 gen, sb_block;
  829. if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
  830. dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
  831. DMZ_MAGIC, le32_to_cpu(sb->magic));
  832. return -ENXIO;
  833. }
  834. zmd->sb_version = le32_to_cpu(sb->version);
  835. if (zmd->sb_version > DMZ_META_VER) {
  836. dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
  837. DMZ_META_VER, zmd->sb_version);
  838. return -EINVAL;
  839. }
  840. if (zmd->sb_version < 2 && tertiary) {
  841. dmz_dev_err(dev, "Tertiary superblocks are not supported");
  842. return -EINVAL;
  843. }
  844. gen = le64_to_cpu(sb->gen);
  845. stored_crc = le32_to_cpu(sb->crc);
  846. sb->crc = 0;
  847. crc = crc32_le(gen, (unsigned char *)sb, DMZ_BLOCK_SIZE);
  848. if (crc != stored_crc) {
  849. dmz_dev_err(dev, "Invalid checksum (needed 0x%08x, got 0x%08x)",
  850. crc, stored_crc);
  851. return -ENXIO;
  852. }
  853. sb_block = le64_to_cpu(sb->sb_block);
  854. if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift ) {
  855. dmz_dev_err(dev, "Invalid superblock position "
  856. "(is %llu expected %llu)",
  857. sb_block,
  858. (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);
  859. return -EINVAL;
  860. }
  861. if (zmd->sb_version > 1) {
  862. uuid_t sb_uuid;
  863. import_uuid(&sb_uuid, sb->dmz_uuid);
  864. if (uuid_is_null(&sb_uuid)) {
  865. dmz_dev_err(dev, "NULL DM-Zoned uuid");
  866. return -ENXIO;
  867. } else if (uuid_is_null(&zmd->uuid)) {
  868. uuid_copy(&zmd->uuid, &sb_uuid);
  869. } else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {
  870. dmz_dev_err(dev, "mismatching DM-Zoned uuid, "
  871. "is %pUl expected %pUl",
  872. &sb_uuid, &zmd->uuid);
  873. return -ENXIO;
  874. }
  875. if (!strlen(zmd->label))
  876. memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);
  877. else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {
  878. dmz_dev_err(dev, "mismatching DM-Zoned label, "
  879. "is %s expected %s",
  880. sb->dmz_label, zmd->label);
  881. return -ENXIO;
  882. }
  883. import_uuid(&dev->uuid, sb->dev_uuid);
  884. if (uuid_is_null(&dev->uuid)) {
  885. dmz_dev_err(dev, "NULL device uuid");
  886. return -ENXIO;
  887. }
  888. if (tertiary) {
  889. /*
  890. * Generation number should be 0, but it doesn't
  891. * really matter if it isn't.
  892. */
  893. if (gen != 0)
  894. dmz_dev_warn(dev, "Invalid generation %llu",
  895. gen);
  896. return 0;
  897. }
  898. }
  899. nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)
  900. >> zmd->zone_nr_blocks_shift;
  901. if (!nr_meta_zones ||
  902. (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||
  903. (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {
  904. dmz_dev_err(dev, "Invalid number of metadata blocks");
  905. return -ENXIO;
  906. }
  907. if (!le32_to_cpu(sb->nr_reserved_seq) ||
  908. le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {
  909. dmz_dev_err(dev, "Invalid number of reserved sequential zones");
  910. return -ENXIO;
  911. }
  912. nr_data_zones = zmd->nr_useable_zones -
  913. (nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq));
  914. if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) {
  915. dmz_dev_err(dev, "Invalid number of chunks %u / %u",
  916. le32_to_cpu(sb->nr_chunks), nr_data_zones);
  917. return -ENXIO;
  918. }
  919. /* OK */
  920. zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);
  921. zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);
  922. zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);
  923. zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);
  924. zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);
  925. zmd->nr_meta_zones = nr_meta_zones;
  926. zmd->nr_data_zones = nr_data_zones;
  927. return 0;
  928. }
  929. /*
  930. * Read the first or second super block from disk.
  931. */
  932. static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
  933. {
  934. dmz_zmd_debug(zmd, "read superblock set %d dev %s block %llu",
  935. set, sb->dev->name, sb->block);
  936. return dmz_rdwr_block(sb->dev, REQ_OP_READ,
  937. sb->block, sb->mblk->page);
  938. }
  939. /*
  940. * Determine the position of the secondary super blocks on disk.
  941. * This is used only if a corruption of the primary super block
  942. * is detected.
  943. */
  944. static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
  945. {
  946. unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
  947. struct dmz_mblock *mblk;
  948. unsigned int zone_id = zmd->sb[0].zone->id;
  949. int i;
  950. /* Allocate a block */
  951. mblk = dmz_alloc_mblock(zmd, 0);
  952. if (!mblk)
  953. return -ENOMEM;
  954. zmd->sb[1].mblk = mblk;
  955. zmd->sb[1].sb = mblk->data;
  956. /* Bad first super block: search for the second one */
  957. zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
  958. zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);
  959. zmd->sb[1].dev = zmd->sb[0].dev;
  960. for (i = 1; i < zmd->nr_rnd_zones; i++) {
  961. if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)
  962. break;
  963. if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
  964. return 0;
  965. zmd->sb[1].block += zone_nr_blocks;
  966. zmd->sb[1].zone = dmz_get(zmd, zone_id + i);
  967. }
  968. dmz_free_mblock(zmd, mblk);
  969. zmd->sb[1].mblk = NULL;
  970. zmd->sb[1].zone = NULL;
  971. zmd->sb[1].dev = NULL;
  972. return -EIO;
  973. }
  974. /*
  975. * Read a super block from disk.
  976. */
  977. static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
  978. {
  979. struct dmz_mblock *mblk;
  980. int ret;
  981. /* Allocate a block */
  982. mblk = dmz_alloc_mblock(zmd, 0);
  983. if (!mblk)
  984. return -ENOMEM;
  985. sb->mblk = mblk;
  986. sb->sb = mblk->data;
  987. /* Read super block */
  988. ret = dmz_read_sb(zmd, sb, set);
  989. if (ret) {
  990. dmz_free_mblock(zmd, mblk);
  991. sb->mblk = NULL;
  992. return ret;
  993. }
  994. return 0;
  995. }
  996. /*
  997. * Recover a metadata set.
  998. */
  999. static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
  1000. {
  1001. unsigned int src_set = dst_set ^ 0x1;
  1002. struct page *page;
  1003. int i, ret;
  1004. dmz_dev_warn(zmd->sb[dst_set].dev,
  1005. "Metadata set %u invalid: recovering", dst_set);
  1006. if (dst_set == 0)
  1007. zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
  1008. else
  1009. zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
  1010. page = alloc_page(GFP_NOIO);
  1011. if (!page)
  1012. return -ENOMEM;
  1013. /* Copy metadata blocks */
  1014. for (i = 1; i < zmd->nr_meta_blocks; i++) {
  1015. ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,
  1016. zmd->sb[src_set].block + i, page);
  1017. if (ret)
  1018. goto out;
  1019. ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,
  1020. zmd->sb[dst_set].block + i, page);
  1021. if (ret)
  1022. goto out;
  1023. }
  1024. /* Finalize with the super block */
  1025. if (!zmd->sb[dst_set].mblk) {
  1026. zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);
  1027. if (!zmd->sb[dst_set].mblk) {
  1028. ret = -ENOMEM;
  1029. goto out;
  1030. }
  1031. zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;
  1032. }
  1033. ret = dmz_write_sb(zmd, dst_set);
  1034. out:
  1035. __free_pages(page, 0);
  1036. return ret;
  1037. }
  1038. /*
  1039. * Get super block from disk.
  1040. */
  1041. static int dmz_load_sb(struct dmz_metadata *zmd)
  1042. {
  1043. bool sb_good[2] = {false, false};
  1044. u64 sb_gen[2] = {0, 0};
  1045. int ret;
  1046. if (!zmd->sb[0].zone) {
  1047. dmz_zmd_err(zmd, "Primary super block zone not set");
  1048. return -ENXIO;
  1049. }
  1050. /* Read and check the primary super block */
  1051. zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
  1052. zmd->sb[0].dev = zmd->sb[0].zone->dev;
  1053. ret = dmz_get_sb(zmd, &zmd->sb[0], 0);
  1054. if (ret) {
  1055. dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");
  1056. return ret;
  1057. }
  1058. ret = dmz_check_sb(zmd, &zmd->sb[0], false);
  1059. /* Read and check secondary super block */
  1060. if (ret == 0) {
  1061. sb_good[0] = true;
  1062. if (!zmd->sb[1].zone) {
  1063. unsigned int zone_id =
  1064. zmd->sb[0].zone->id + zmd->nr_meta_zones;
  1065. zmd->sb[1].zone = dmz_get(zmd, zone_id);
  1066. }
  1067. zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
  1068. zmd->sb[1].dev = zmd->sb[0].dev;
  1069. ret = dmz_get_sb(zmd, &zmd->sb[1], 1);
  1070. } else
  1071. ret = dmz_lookup_secondary_sb(zmd);
  1072. if (ret) {
  1073. dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");
  1074. return ret;
  1075. }
  1076. ret = dmz_check_sb(zmd, &zmd->sb[1], false);
  1077. if (ret == 0)
  1078. sb_good[1] = true;
  1079. /* Use highest generation sb first */
  1080. if (!sb_good[0] && !sb_good[1]) {
  1081. dmz_zmd_err(zmd, "No valid super block found");
  1082. return -EIO;
  1083. }
  1084. if (sb_good[0])
  1085. sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
  1086. else {
  1087. ret = dmz_recover_mblocks(zmd, 0);
  1088. if (ret) {
  1089. dmz_dev_err(zmd->sb[0].dev,
  1090. "Recovery of superblock 0 failed");
  1091. return -EIO;
  1092. }
  1093. }
  1094. if (sb_good[1])
  1095. sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
  1096. else {
  1097. ret = dmz_recover_mblocks(zmd, 1);
  1098. if (ret) {
  1099. dmz_dev_err(zmd->sb[1].dev,
  1100. "Recovery of superblock 1 failed");
  1101. return -EIO;
  1102. }
  1103. }
  1104. if (sb_gen[0] >= sb_gen[1]) {
  1105. zmd->sb_gen = sb_gen[0];
  1106. zmd->mblk_primary = 0;
  1107. } else {
  1108. zmd->sb_gen = sb_gen[1];
  1109. zmd->mblk_primary = 1;
  1110. }
  1111. dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,
  1112. "Using super block %u (gen %llu)",
  1113. zmd->mblk_primary, zmd->sb_gen);
  1114. if (zmd->sb_version > 1) {
  1115. int i;
  1116. struct dmz_sb *sb;
  1117. sb = kzalloc(sizeof(struct dmz_sb), GFP_KERNEL);
  1118. if (!sb)
  1119. return -ENOMEM;
  1120. for (i = 1; i < zmd->nr_devs; i++) {
  1121. sb->block = 0;
  1122. sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);
  1123. sb->dev = &zmd->dev[i];
  1124. if (!dmz_is_meta(sb->zone)) {
  1125. dmz_dev_err(sb->dev,
  1126. "Tertiary super block zone %u not marked as metadata zone",
  1127. sb->zone->id);
  1128. ret = -EINVAL;
  1129. goto out_kfree;
  1130. }
  1131. ret = dmz_get_sb(zmd, sb, i + 1);
  1132. if (ret) {
  1133. dmz_dev_err(sb->dev,
  1134. "Read tertiary super block failed");
  1135. dmz_free_mblock(zmd, sb->mblk);
  1136. goto out_kfree;
  1137. }
  1138. ret = dmz_check_sb(zmd, sb, true);
  1139. dmz_free_mblock(zmd, sb->mblk);
  1140. if (ret == -EINVAL)
  1141. goto out_kfree;
  1142. }
  1143. out_kfree:
  1144. kfree(sb);
  1145. }
  1146. return ret;
  1147. }
  1148. /*
  1149. * Initialize a zone descriptor.
  1150. */
  1151. static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data)
  1152. {
  1153. struct dmz_dev *dev = data;
  1154. struct dmz_metadata *zmd = dev->metadata;
  1155. int idx = num + dev->zone_offset;
  1156. struct dm_zone *zone;
  1157. zone = dmz_insert(zmd, idx, dev);
  1158. if (IS_ERR(zone))
  1159. return PTR_ERR(zone);
  1160. if (blkz->len != zmd->zone_nr_sectors) {
  1161. if (zmd->sb_version > 1) {
  1162. /* Ignore the eventual runt (smaller) zone */
  1163. set_bit(DMZ_OFFLINE, &zone->flags);
  1164. return 0;
  1165. } else if (blkz->start + blkz->len == dev->capacity)
  1166. return 0;
  1167. return -ENXIO;
  1168. }
  1169. /*
  1170. * Devices that have zones with a capacity smaller than the zone size
  1171. * (e.g. NVMe zoned namespaces) are not supported.
  1172. */
  1173. if (blkz->capacity != blkz->len)
  1174. return -ENXIO;
  1175. switch (blkz->type) {
  1176. case BLK_ZONE_TYPE_CONVENTIONAL:
  1177. set_bit(DMZ_RND, &zone->flags);
  1178. break;
  1179. case BLK_ZONE_TYPE_SEQWRITE_REQ:
  1180. case BLK_ZONE_TYPE_SEQWRITE_PREF:
  1181. set_bit(DMZ_SEQ, &zone->flags);
  1182. break;
  1183. default:
  1184. return -ENXIO;
  1185. }
  1186. if (dmz_is_rnd(zone))
  1187. zone->wp_block = 0;
  1188. else
  1189. zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
  1190. if (blkz->cond == BLK_ZONE_COND_OFFLINE)
  1191. set_bit(DMZ_OFFLINE, &zone->flags);
  1192. else if (blkz->cond == BLK_ZONE_COND_READONLY)
  1193. set_bit(DMZ_READ_ONLY, &zone->flags);
  1194. else {
  1195. zmd->nr_useable_zones++;
  1196. if (dmz_is_rnd(zone)) {
  1197. zmd->nr_rnd_zones++;
  1198. if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {
  1199. /* Primary super block zone */
  1200. zmd->sb[0].zone = zone;
  1201. }
  1202. }
  1203. if (zmd->nr_devs > 1 && num == 0) {
  1204. /*
  1205. * Tertiary superblock zones are always at the
  1206. * start of the zoned devices, so mark them
  1207. * as metadata zone.
  1208. */
  1209. set_bit(DMZ_META, &zone->flags);
  1210. }
  1211. }
  1212. return 0;
  1213. }
  1214. static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)
  1215. {
  1216. int idx;
  1217. sector_t zone_offset = 0;
  1218. for(idx = 0; idx < dev->nr_zones; idx++) {
  1219. struct dm_zone *zone;
  1220. zone = dmz_insert(zmd, idx, dev);
  1221. if (IS_ERR(zone))
  1222. return PTR_ERR(zone);
  1223. set_bit(DMZ_CACHE, &zone->flags);
  1224. zone->wp_block = 0;
  1225. zmd->nr_cache_zones++;
  1226. zmd->nr_useable_zones++;
  1227. if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {
  1228. /* Disable runt zone */
  1229. set_bit(DMZ_OFFLINE, &zone->flags);
  1230. break;
  1231. }
  1232. zone_offset += zmd->zone_nr_sectors;
  1233. }
  1234. return 0;
  1235. }
  1236. /*
  1237. * Free zones descriptors.
  1238. */
  1239. static void dmz_drop_zones(struct dmz_metadata *zmd)
  1240. {
  1241. int idx;
  1242. for(idx = 0; idx < zmd->nr_zones; idx++) {
  1243. struct dm_zone *zone = xa_load(&zmd->zones, idx);
  1244. kfree(zone);
  1245. xa_erase(&zmd->zones, idx);
  1246. }
  1247. xa_destroy(&zmd->zones);
  1248. }
  1249. /*
  1250. * Allocate and initialize zone descriptors using the zone
  1251. * information from disk.
  1252. */
  1253. static int dmz_init_zones(struct dmz_metadata *zmd)
  1254. {
  1255. int i, ret;
  1256. struct dmz_dev *zoned_dev = &zmd->dev[0];
  1257. /* Init */
  1258. zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;
  1259. zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);
  1260. zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);
  1261. zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);
  1262. zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;
  1263. zmd->zone_nr_bitmap_blocks =
  1264. max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
  1265. zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,
  1266. DMZ_BLOCK_SIZE_BITS);
  1267. /* Allocate zone array */
  1268. zmd->nr_zones = 0;
  1269. for (i = 0; i < zmd->nr_devs; i++) {
  1270. struct dmz_dev *dev = &zmd->dev[i];
  1271. dev->metadata = zmd;
  1272. zmd->nr_zones += dev->nr_zones;
  1273. atomic_set(&dev->unmap_nr_rnd, 0);
  1274. INIT_LIST_HEAD(&dev->unmap_rnd_list);
  1275. INIT_LIST_HEAD(&dev->map_rnd_list);
  1276. atomic_set(&dev->unmap_nr_seq, 0);
  1277. INIT_LIST_HEAD(&dev->unmap_seq_list);
  1278. INIT_LIST_HEAD(&dev->map_seq_list);
  1279. }
  1280. if (!zmd->nr_zones) {
  1281. DMERR("(%s): No zones found", zmd->devname);
  1282. return -ENXIO;
  1283. }
  1284. xa_init(&zmd->zones);
  1285. DMDEBUG("(%s): Using %zu B for zone information",
  1286. zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);
  1287. if (zmd->nr_devs > 1) {
  1288. ret = dmz_emulate_zones(zmd, &zmd->dev[0]);
  1289. if (ret < 0) {
  1290. DMDEBUG("(%s): Failed to emulate zones, error %d",
  1291. zmd->devname, ret);
  1292. dmz_drop_zones(zmd);
  1293. return ret;
  1294. }
  1295. /*
  1296. * Primary superblock zone is always at zone 0 when multiple
  1297. * drives are present.
  1298. */
  1299. zmd->sb[0].zone = dmz_get(zmd, 0);
  1300. for (i = 1; i < zmd->nr_devs; i++) {
  1301. zoned_dev = &zmd->dev[i];
  1302. ret = blkdev_report_zones(zoned_dev->bdev, 0,
  1303. BLK_ALL_ZONES,
  1304. dmz_init_zone, zoned_dev);
  1305. if (ret < 0) {
  1306. DMDEBUG("(%s): Failed to report zones, error %d",
  1307. zmd->devname, ret);
  1308. dmz_drop_zones(zmd);
  1309. return ret;
  1310. }
  1311. }
  1312. return 0;
  1313. }
  1314. /*
  1315. * Get zone information and initialize zone descriptors. At the same
  1316. * time, determine where the super block should be: first block of the
  1317. * first randomly writable zone.
  1318. */
  1319. ret = blkdev_report_zones(zoned_dev->bdev, 0, BLK_ALL_ZONES,
  1320. dmz_init_zone, zoned_dev);
  1321. if (ret < 0) {
  1322. DMDEBUG("(%s): Failed to report zones, error %d",
  1323. zmd->devname, ret);
  1324. dmz_drop_zones(zmd);
  1325. return ret;
  1326. }
  1327. return 0;
  1328. }
  1329. static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
  1330. void *data)
  1331. {
  1332. struct dm_zone *zone = data;
  1333. clear_bit(DMZ_OFFLINE, &zone->flags);
  1334. clear_bit(DMZ_READ_ONLY, &zone->flags);
  1335. if (blkz->cond == BLK_ZONE_COND_OFFLINE)
  1336. set_bit(DMZ_OFFLINE, &zone->flags);
  1337. else if (blkz->cond == BLK_ZONE_COND_READONLY)
  1338. set_bit(DMZ_READ_ONLY, &zone->flags);
  1339. if (dmz_is_seq(zone))
  1340. zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
  1341. else
  1342. zone->wp_block = 0;
  1343. return 0;
  1344. }
  1345. /*
  1346. * Update a zone information.
  1347. */
  1348. static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  1349. {
  1350. struct dmz_dev *dev = zone->dev;
  1351. unsigned int noio_flag;
  1352. int ret;
  1353. if (dev->flags & DMZ_BDEV_REGULAR)
  1354. return 0;
  1355. /*
  1356. * Get zone information from disk. Since blkdev_report_zones() uses
  1357. * GFP_KERNEL by default for memory allocations, set the per-task
  1358. * PF_MEMALLOC_NOIO flag so that all allocations are done as if
  1359. * GFP_NOIO was specified.
  1360. */
  1361. noio_flag = memalloc_noio_save();
  1362. ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,
  1363. dmz_update_zone_cb, zone);
  1364. memalloc_noio_restore(noio_flag);
  1365. if (ret == 0)
  1366. ret = -EIO;
  1367. if (ret < 0) {
  1368. dmz_dev_err(dev, "Get zone %u report failed",
  1369. zone->id);
  1370. dmz_check_bdev(dev);
  1371. return ret;
  1372. }
  1373. return 0;
  1374. }
  1375. /*
  1376. * Check a zone write pointer position when the zone is marked
  1377. * with the sequential write error flag.
  1378. */
  1379. static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
  1380. struct dm_zone *zone)
  1381. {
  1382. struct dmz_dev *dev = zone->dev;
  1383. unsigned int wp = 0;
  1384. int ret;
  1385. wp = zone->wp_block;
  1386. ret = dmz_update_zone(zmd, zone);
  1387. if (ret)
  1388. return ret;
  1389. dmz_dev_warn(dev, "Processing zone %u write error (zone wp %u/%u)",
  1390. zone->id, zone->wp_block, wp);
  1391. if (zone->wp_block < wp) {
  1392. dmz_invalidate_blocks(zmd, zone, zone->wp_block,
  1393. wp - zone->wp_block);
  1394. }
  1395. return 0;
  1396. }
  1397. /*
  1398. * Reset a zone write pointer.
  1399. */
  1400. static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  1401. {
  1402. int ret;
  1403. /*
  1404. * Ignore offline zones, read only zones,
  1405. * and conventional zones.
  1406. */
  1407. if (dmz_is_offline(zone) ||
  1408. dmz_is_readonly(zone) ||
  1409. dmz_is_rnd(zone))
  1410. return 0;
  1411. if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
  1412. struct dmz_dev *dev = zone->dev;
  1413. ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
  1414. dmz_start_sect(zmd, zone),
  1415. zmd->zone_nr_sectors, GFP_NOIO);
  1416. if (ret) {
  1417. dmz_dev_err(dev, "Reset zone %u failed %d",
  1418. zone->id, ret);
  1419. return ret;
  1420. }
  1421. }
  1422. /* Clear write error bit and rewind write pointer position */
  1423. clear_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
  1424. zone->wp_block = 0;
  1425. return 0;
  1426. }
  1427. static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
  1428. /*
  1429. * Initialize chunk mapping.
  1430. */
  1431. static int dmz_load_mapping(struct dmz_metadata *zmd)
  1432. {
  1433. struct dm_zone *dzone, *bzone;
  1434. struct dmz_mblock *dmap_mblk = NULL;
  1435. struct dmz_map *dmap;
  1436. unsigned int i = 0, e = 0, chunk = 0;
  1437. unsigned int dzone_id;
  1438. unsigned int bzone_id;
  1439. /* Metadata block array for the chunk mapping table */
  1440. zmd->map_mblk = kcalloc(zmd->nr_map_blocks,
  1441. sizeof(struct dmz_mblk *), GFP_KERNEL);
  1442. if (!zmd->map_mblk)
  1443. return -ENOMEM;
  1444. /* Get chunk mapping table blocks and initialize zone mapping */
  1445. while (chunk < zmd->nr_chunks) {
  1446. if (!dmap_mblk) {
  1447. /* Get mapping block */
  1448. dmap_mblk = dmz_get_mblock(zmd, i + 1);
  1449. if (IS_ERR(dmap_mblk))
  1450. return PTR_ERR(dmap_mblk);
  1451. zmd->map_mblk[i] = dmap_mblk;
  1452. dmap = (struct dmz_map *) dmap_mblk->data;
  1453. i++;
  1454. e = 0;
  1455. }
  1456. /* Check data zone */
  1457. dzone_id = le32_to_cpu(dmap[e].dzone_id);
  1458. if (dzone_id == DMZ_MAP_UNMAPPED)
  1459. goto next;
  1460. if (dzone_id >= zmd->nr_zones) {
  1461. dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",
  1462. chunk, dzone_id);
  1463. return -EIO;
  1464. }
  1465. dzone = dmz_get(zmd, dzone_id);
  1466. if (!dzone) {
  1467. dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",
  1468. chunk, dzone_id);
  1469. return -EIO;
  1470. }
  1471. set_bit(DMZ_DATA, &dzone->flags);
  1472. dzone->chunk = chunk;
  1473. dmz_get_zone_weight(zmd, dzone);
  1474. if (dmz_is_cache(dzone))
  1475. list_add_tail(&dzone->link, &zmd->map_cache_list);
  1476. else if (dmz_is_rnd(dzone))
  1477. list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
  1478. else
  1479. list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
  1480. /* Check buffer zone */
  1481. bzone_id = le32_to_cpu(dmap[e].bzone_id);
  1482. if (bzone_id == DMZ_MAP_UNMAPPED)
  1483. goto next;
  1484. if (bzone_id >= zmd->nr_zones) {
  1485. dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",
  1486. chunk, bzone_id);
  1487. return -EIO;
  1488. }
  1489. bzone = dmz_get(zmd, bzone_id);
  1490. if (!bzone) {
  1491. dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",
  1492. chunk, bzone_id);
  1493. return -EIO;
  1494. }
  1495. if (!dmz_is_rnd(bzone) && !dmz_is_cache(bzone)) {
  1496. dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",
  1497. chunk, bzone_id);
  1498. return -EIO;
  1499. }
  1500. set_bit(DMZ_DATA, &bzone->flags);
  1501. set_bit(DMZ_BUF, &bzone->flags);
  1502. bzone->chunk = chunk;
  1503. bzone->bzone = dzone;
  1504. dzone->bzone = bzone;
  1505. dmz_get_zone_weight(zmd, bzone);
  1506. if (dmz_is_cache(bzone))
  1507. list_add_tail(&bzone->link, &zmd->map_cache_list);
  1508. else
  1509. list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
  1510. next:
  1511. chunk++;
  1512. e++;
  1513. if (e >= DMZ_MAP_ENTRIES)
  1514. dmap_mblk = NULL;
  1515. }
  1516. /*
  1517. * At this point, only meta zones and mapped data zones were
  1518. * fully initialized. All remaining zones are unmapped data
  1519. * zones. Finish initializing those here.
  1520. */
  1521. for (i = 0; i < zmd->nr_zones; i++) {
  1522. dzone = dmz_get(zmd, i);
  1523. if (!dzone)
  1524. continue;
  1525. if (dmz_is_meta(dzone))
  1526. continue;
  1527. if (dmz_is_offline(dzone))
  1528. continue;
  1529. if (dmz_is_cache(dzone))
  1530. zmd->nr_cache++;
  1531. else if (dmz_is_rnd(dzone))
  1532. dzone->dev->nr_rnd++;
  1533. else
  1534. dzone->dev->nr_seq++;
  1535. if (dmz_is_data(dzone)) {
  1536. /* Already initialized */
  1537. continue;
  1538. }
  1539. /* Unmapped data zone */
  1540. set_bit(DMZ_DATA, &dzone->flags);
  1541. dzone->chunk = DMZ_MAP_UNMAPPED;
  1542. if (dmz_is_cache(dzone)) {
  1543. list_add_tail(&dzone->link, &zmd->unmap_cache_list);
  1544. atomic_inc(&zmd->unmap_nr_cache);
  1545. } else if (dmz_is_rnd(dzone)) {
  1546. list_add_tail(&dzone->link,
  1547. &dzone->dev->unmap_rnd_list);
  1548. atomic_inc(&dzone->dev->unmap_nr_rnd);
  1549. } else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
  1550. list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);
  1551. set_bit(DMZ_RESERVED, &dzone->flags);
  1552. atomic_inc(&zmd->nr_reserved_seq_zones);
  1553. dzone->dev->nr_seq--;
  1554. } else {
  1555. list_add_tail(&dzone->link,
  1556. &dzone->dev->unmap_seq_list);
  1557. atomic_inc(&dzone->dev->unmap_nr_seq);
  1558. }
  1559. }
  1560. return 0;
  1561. }
  1562. /*
  1563. * Set a data chunk mapping.
  1564. */
  1565. static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
  1566. unsigned int dzone_id, unsigned int bzone_id)
  1567. {
  1568. struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
  1569. struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
  1570. int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
  1571. dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
  1572. dmap[map_idx].bzone_id = cpu_to_le32(bzone_id);
  1573. dmz_dirty_mblock(zmd, dmap_mblk);
  1574. }
  1575. /*
  1576. * The list of mapped zones is maintained in LRU order.
  1577. * This rotates a zone at the end of its map list.
  1578. */
  1579. static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  1580. {
  1581. if (list_empty(&zone->link))
  1582. return;
  1583. list_del_init(&zone->link);
  1584. if (dmz_is_seq(zone)) {
  1585. /* LRU rotate sequential zone */
  1586. list_add_tail(&zone->link, &zone->dev->map_seq_list);
  1587. } else if (dmz_is_cache(zone)) {
  1588. /* LRU rotate cache zone */
  1589. list_add_tail(&zone->link, &zmd->map_cache_list);
  1590. } else {
  1591. /* LRU rotate random zone */
  1592. list_add_tail(&zone->link, &zone->dev->map_rnd_list);
  1593. }
  1594. }
  1595. /*
  1596. * The list of mapped random zones is maintained
  1597. * in LRU order. This rotates a zone at the end of the list.
  1598. */
  1599. static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  1600. {
  1601. __dmz_lru_zone(zmd, zone);
  1602. if (zone->bzone)
  1603. __dmz_lru_zone(zmd, zone->bzone);
  1604. }
  1605. /*
  1606. * Wait for any zone to be freed.
  1607. */
  1608. static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)
  1609. {
  1610. DEFINE_WAIT(wait);
  1611. prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);
  1612. dmz_unlock_map(zmd);
  1613. dmz_unlock_metadata(zmd);
  1614. io_schedule_timeout(HZ);
  1615. dmz_lock_metadata(zmd);
  1616. dmz_lock_map(zmd);
  1617. finish_wait(&zmd->free_wq, &wait);
  1618. }
  1619. /*
  1620. * Lock a zone for reclaim (set the zone RECLAIM bit).
  1621. * Returns false if the zone cannot be locked or if it is already locked
  1622. * and 1 otherwise.
  1623. */
  1624. int dmz_lock_zone_reclaim(struct dm_zone *zone)
  1625. {
  1626. /* Active zones cannot be reclaimed */
  1627. if (dmz_is_active(zone))
  1628. return 0;
  1629. return !test_and_set_bit(DMZ_RECLAIM, &zone->flags);
  1630. }
  1631. /*
  1632. * Clear a zone reclaim flag.
  1633. */
  1634. void dmz_unlock_zone_reclaim(struct dm_zone *zone)
  1635. {
  1636. WARN_ON(dmz_is_active(zone));
  1637. WARN_ON(!dmz_in_reclaim(zone));
  1638. clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
  1639. smp_mb__after_atomic();
  1640. wake_up_bit(&zone->flags, DMZ_RECLAIM);
  1641. }
  1642. /*
  1643. * Wait for a zone reclaim to complete.
  1644. */
  1645. static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
  1646. {
  1647. dmz_unlock_map(zmd);
  1648. dmz_unlock_metadata(zmd);
  1649. set_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
  1650. wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
  1651. clear_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
  1652. dmz_lock_metadata(zmd);
  1653. dmz_lock_map(zmd);
  1654. }
  1655. /*
  1656. * Select a cache or random write zone for reclaim.
  1657. */
  1658. static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,
  1659. unsigned int idx, bool idle)
  1660. {
  1661. struct dm_zone *dzone = NULL;
  1662. struct dm_zone *zone, *maxw_z = NULL;
  1663. struct list_head *zone_list;
  1664. /* If we have cache zones select from the cache zone list */
  1665. if (zmd->nr_cache) {
  1666. zone_list = &zmd->map_cache_list;
  1667. /* Try to relaim random zones, too, when idle */
  1668. if (idle && list_empty(zone_list))
  1669. zone_list = &zmd->dev[idx].map_rnd_list;
  1670. } else
  1671. zone_list = &zmd->dev[idx].map_rnd_list;
  1672. /*
  1673. * Find the buffer zone with the heaviest weight or the first (oldest)
  1674. * data zone that can be reclaimed.
  1675. */
  1676. list_for_each_entry(zone, zone_list, link) {
  1677. if (dmz_is_buf(zone)) {
  1678. dzone = zone->bzone;
  1679. if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
  1680. continue;
  1681. if (!maxw_z || maxw_z->weight < dzone->weight)
  1682. maxw_z = dzone;
  1683. } else {
  1684. dzone = zone;
  1685. if (dmz_lock_zone_reclaim(dzone))
  1686. return dzone;
  1687. }
  1688. }
  1689. if (maxw_z && dmz_lock_zone_reclaim(maxw_z))
  1690. return maxw_z;
  1691. /*
  1692. * If we come here, none of the zones inspected could be locked for
  1693. * reclaim. Try again, being more aggressive, that is, find the
  1694. * first zone that can be reclaimed regardless of its weitght.
  1695. */
  1696. list_for_each_entry(zone, zone_list, link) {
  1697. if (dmz_is_buf(zone)) {
  1698. dzone = zone->bzone;
  1699. if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
  1700. continue;
  1701. } else
  1702. dzone = zone;
  1703. if (dmz_lock_zone_reclaim(dzone))
  1704. return dzone;
  1705. }
  1706. return NULL;
  1707. }
  1708. /*
  1709. * Select a buffered sequential zone for reclaim.
  1710. */
  1711. static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,
  1712. unsigned int idx)
  1713. {
  1714. struct dm_zone *zone;
  1715. list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {
  1716. if (!zone->bzone)
  1717. continue;
  1718. if (dmz_lock_zone_reclaim(zone))
  1719. return zone;
  1720. }
  1721. return NULL;
  1722. }
  1723. /*
  1724. * Select a zone for reclaim.
  1725. */
  1726. struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
  1727. unsigned int dev_idx, bool idle)
  1728. {
  1729. struct dm_zone *zone = NULL;
  1730. /*
  1731. * Search for a zone candidate to reclaim: 2 cases are possible.
  1732. * (1) There is no free sequential zones. Then a random data zone
  1733. * cannot be reclaimed. So choose a sequential zone to reclaim so
  1734. * that afterward a random zone can be reclaimed.
  1735. * (2) At least one free sequential zone is available, then choose
  1736. * the oldest random zone (data or buffer) that can be locked.
  1737. */
  1738. dmz_lock_map(zmd);
  1739. if (list_empty(&zmd->reserved_seq_zones_list))
  1740. zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);
  1741. if (!zone)
  1742. zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);
  1743. dmz_unlock_map(zmd);
  1744. return zone;
  1745. }
  1746. /*
  1747. * Get the zone mapping a chunk, if the chunk is mapped already.
  1748. * If no mapping exist and the operation is WRITE, a zone is
  1749. * allocated and used to map the chunk.
  1750. * The zone returned will be set to the active state.
  1751. */
  1752. struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk, int op)
  1753. {
  1754. struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
  1755. struct dmz_map *dmap = (struct dmz_map *) dmap_mblk->data;
  1756. int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
  1757. unsigned int dzone_id;
  1758. struct dm_zone *dzone = NULL;
  1759. int ret = 0;
  1760. int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
  1761. dmz_lock_map(zmd);
  1762. again:
  1763. /* Get the chunk mapping */
  1764. dzone_id = le32_to_cpu(dmap[dmap_idx].dzone_id);
  1765. if (dzone_id == DMZ_MAP_UNMAPPED) {
  1766. /*
  1767. * Read or discard in unmapped chunks are fine. But for
  1768. * writes, we need a mapping, so get one.
  1769. */
  1770. if (op != REQ_OP_WRITE)
  1771. goto out;
  1772. /* Allocate a random zone */
  1773. dzone = dmz_alloc_zone(zmd, 0, alloc_flags);
  1774. if (!dzone) {
  1775. if (dmz_dev_is_dying(zmd)) {
  1776. dzone = ERR_PTR(-EIO);
  1777. goto out;
  1778. }
  1779. dmz_wait_for_free_zones(zmd);
  1780. goto again;
  1781. }
  1782. dmz_map_zone(zmd, dzone, chunk);
  1783. } else {
  1784. /* The chunk is already mapped: get the mapping zone */
  1785. dzone = dmz_get(zmd, dzone_id);
  1786. if (!dzone) {
  1787. dzone = ERR_PTR(-EIO);
  1788. goto out;
  1789. }
  1790. if (dzone->chunk != chunk) {
  1791. dzone = ERR_PTR(-EIO);
  1792. goto out;
  1793. }
  1794. /* Repair write pointer if the sequential dzone has error */
  1795. if (dmz_seq_write_err(dzone)) {
  1796. ret = dmz_handle_seq_write_err(zmd, dzone);
  1797. if (ret) {
  1798. dzone = ERR_PTR(-EIO);
  1799. goto out;
  1800. }
  1801. clear_bit(DMZ_SEQ_WRITE_ERR, &dzone->flags);
  1802. }
  1803. }
  1804. /*
  1805. * If the zone is being reclaimed, the chunk mapping may change
  1806. * to a different zone. So wait for reclaim and retry. Otherwise,
  1807. * activate the zone (this will prevent reclaim from touching it).
  1808. */
  1809. if (dmz_in_reclaim(dzone)) {
  1810. dmz_wait_for_reclaim(zmd, dzone);
  1811. goto again;
  1812. }
  1813. dmz_activate_zone(dzone);
  1814. dmz_lru_zone(zmd, dzone);
  1815. out:
  1816. dmz_unlock_map(zmd);
  1817. return dzone;
  1818. }
  1819. /*
  1820. * Write and discard change the block validity of data zones and their buffer
  1821. * zones. Check here that valid blocks are still present. If all blocks are
  1822. * invalid, the zones can be unmapped on the fly without waiting for reclaim
  1823. * to do it.
  1824. */
  1825. void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)
  1826. {
  1827. struct dm_zone *bzone;
  1828. dmz_lock_map(zmd);
  1829. bzone = dzone->bzone;
  1830. if (bzone) {
  1831. if (dmz_weight(bzone))
  1832. dmz_lru_zone(zmd, bzone);
  1833. else {
  1834. /* Empty buffer zone: reclaim it */
  1835. dmz_unmap_zone(zmd, bzone);
  1836. dmz_free_zone(zmd, bzone);
  1837. bzone = NULL;
  1838. }
  1839. }
  1840. /* Deactivate the data zone */
  1841. dmz_deactivate_zone(dzone);
  1842. if (dmz_is_active(dzone) || bzone || dmz_weight(dzone))
  1843. dmz_lru_zone(zmd, dzone);
  1844. else {
  1845. /* Unbuffered inactive empty data zone: reclaim it */
  1846. dmz_unmap_zone(zmd, dzone);
  1847. dmz_free_zone(zmd, dzone);
  1848. }
  1849. dmz_unlock_map(zmd);
  1850. }
  1851. /*
  1852. * Allocate and map a random zone to buffer a chunk
  1853. * already mapped to a sequential zone.
  1854. */
  1855. struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
  1856. struct dm_zone *dzone)
  1857. {
  1858. struct dm_zone *bzone;
  1859. int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
  1860. dmz_lock_map(zmd);
  1861. again:
  1862. bzone = dzone->bzone;
  1863. if (bzone)
  1864. goto out;
  1865. /* Allocate a random zone */
  1866. bzone = dmz_alloc_zone(zmd, 0, alloc_flags);
  1867. if (!bzone) {
  1868. if (dmz_dev_is_dying(zmd)) {
  1869. bzone = ERR_PTR(-EIO);
  1870. goto out;
  1871. }
  1872. dmz_wait_for_free_zones(zmd);
  1873. goto again;
  1874. }
  1875. /* Update the chunk mapping */
  1876. dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);
  1877. set_bit(DMZ_BUF, &bzone->flags);
  1878. bzone->chunk = dzone->chunk;
  1879. bzone->bzone = dzone;
  1880. dzone->bzone = bzone;
  1881. if (dmz_is_cache(bzone))
  1882. list_add_tail(&bzone->link, &zmd->map_cache_list);
  1883. else
  1884. list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
  1885. out:
  1886. dmz_unlock_map(zmd);
  1887. return bzone;
  1888. }
  1889. /*
  1890. * Get an unmapped (free) zone.
  1891. * This must be called with the mapping lock held.
  1892. */
  1893. struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,
  1894. unsigned long flags)
  1895. {
  1896. struct list_head *list;
  1897. struct dm_zone *zone;
  1898. int i;
  1899. /* Schedule reclaim to ensure free zones are available */
  1900. if (!(flags & DMZ_ALLOC_RECLAIM)) {
  1901. for (i = 0; i < zmd->nr_devs; i++)
  1902. dmz_schedule_reclaim(zmd->dev[i].reclaim);
  1903. }
  1904. i = 0;
  1905. again:
  1906. if (flags & DMZ_ALLOC_CACHE)
  1907. list = &zmd->unmap_cache_list;
  1908. else if (flags & DMZ_ALLOC_RND)
  1909. list = &zmd->dev[dev_idx].unmap_rnd_list;
  1910. else
  1911. list = &zmd->dev[dev_idx].unmap_seq_list;
  1912. if (list_empty(list)) {
  1913. /*
  1914. * No free zone: return NULL if this is for not reclaim.
  1915. */
  1916. if (!(flags & DMZ_ALLOC_RECLAIM))
  1917. return NULL;
  1918. /*
  1919. * Try to allocate from other devices
  1920. */
  1921. if (i < zmd->nr_devs) {
  1922. dev_idx = (dev_idx + 1) % zmd->nr_devs;
  1923. i++;
  1924. goto again;
  1925. }
  1926. /*
  1927. * Fallback to the reserved sequential zones
  1928. */
  1929. zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,
  1930. struct dm_zone, link);
  1931. if (zone) {
  1932. list_del_init(&zone->link);
  1933. atomic_dec(&zmd->nr_reserved_seq_zones);
  1934. }
  1935. return zone;
  1936. }
  1937. zone = list_first_entry(list, struct dm_zone, link);
  1938. list_del_init(&zone->link);
  1939. if (dmz_is_cache(zone))
  1940. atomic_dec(&zmd->unmap_nr_cache);
  1941. else if (dmz_is_rnd(zone))
  1942. atomic_dec(&zone->dev->unmap_nr_rnd);
  1943. else
  1944. atomic_dec(&zone->dev->unmap_nr_seq);
  1945. if (dmz_is_offline(zone)) {
  1946. dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);
  1947. zone = NULL;
  1948. goto again;
  1949. }
  1950. if (dmz_is_meta(zone)) {
  1951. dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);
  1952. zone = NULL;
  1953. goto again;
  1954. }
  1955. return zone;
  1956. }
  1957. /*
  1958. * Free a zone.
  1959. * This must be called with the mapping lock held.
  1960. */
  1961. void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  1962. {
  1963. /* If this is a sequential zone, reset it */
  1964. if (dmz_is_seq(zone))
  1965. dmz_reset_zone(zmd, zone);
  1966. /* Return the zone to its type unmap list */
  1967. if (dmz_is_cache(zone)) {
  1968. list_add_tail(&zone->link, &zmd->unmap_cache_list);
  1969. atomic_inc(&zmd->unmap_nr_cache);
  1970. } else if (dmz_is_rnd(zone)) {
  1971. list_add_tail(&zone->link, &zone->dev->unmap_rnd_list);
  1972. atomic_inc(&zone->dev->unmap_nr_rnd);
  1973. } else if (dmz_is_reserved(zone)) {
  1974. list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);
  1975. atomic_inc(&zmd->nr_reserved_seq_zones);
  1976. } else {
  1977. list_add_tail(&zone->link, &zone->dev->unmap_seq_list);
  1978. atomic_inc(&zone->dev->unmap_nr_seq);
  1979. }
  1980. wake_up_all(&zmd->free_wq);
  1981. }
  1982. /*
  1983. * Map a chunk to a zone.
  1984. * This must be called with the mapping lock held.
  1985. */
  1986. void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
  1987. unsigned int chunk)
  1988. {
  1989. /* Set the chunk mapping */
  1990. dmz_set_chunk_mapping(zmd, chunk, dzone->id,
  1991. DMZ_MAP_UNMAPPED);
  1992. dzone->chunk = chunk;
  1993. if (dmz_is_cache(dzone))
  1994. list_add_tail(&dzone->link, &zmd->map_cache_list);
  1995. else if (dmz_is_rnd(dzone))
  1996. list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
  1997. else
  1998. list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
  1999. }
  2000. /*
  2001. * Unmap a zone.
  2002. * This must be called with the mapping lock held.
  2003. */
  2004. void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
  2005. {
  2006. unsigned int chunk = zone->chunk;
  2007. unsigned int dzone_id;
  2008. if (chunk == DMZ_MAP_UNMAPPED) {
  2009. /* Already unmapped */
  2010. return;
  2011. }
  2012. if (test_and_clear_bit(DMZ_BUF, &zone->flags)) {
  2013. /*
  2014. * Unmapping the chunk buffer zone: clear only
  2015. * the chunk buffer mapping
  2016. */
  2017. dzone_id = zone->bzone->id;
  2018. zone->bzone->bzone = NULL;
  2019. zone->bzone = NULL;
  2020. } else {
  2021. /*
  2022. * Unmapping the chunk data zone: the zone must
  2023. * not be buffered.
  2024. */
  2025. if (WARN_ON(zone->bzone)) {
  2026. zone->bzone->bzone = NULL;
  2027. zone->bzone = NULL;
  2028. }
  2029. dzone_id = DMZ_MAP_UNMAPPED;
  2030. }
  2031. dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);
  2032. zone->chunk = DMZ_MAP_UNMAPPED;
  2033. list_del_init(&zone->link);
  2034. }
  2035. /*
  2036. * Set @nr_bits bits in @bitmap starting from @bit.
  2037. * Return the number of bits changed from 0 to 1.
  2038. */
  2039. static unsigned int dmz_set_bits(unsigned long *bitmap,
  2040. unsigned int bit, unsigned int nr_bits)
  2041. {
  2042. unsigned long *addr;
  2043. unsigned int end = bit + nr_bits;
  2044. unsigned int n = 0;
  2045. while (bit < end) {
  2046. if (((bit & (BITS_PER_LONG - 1)) == 0) &&
  2047. ((end - bit) >= BITS_PER_LONG)) {
  2048. /* Try to set the whole word at once */
  2049. addr = bitmap + BIT_WORD(bit);
  2050. if (*addr == 0) {
  2051. *addr = ULONG_MAX;
  2052. n += BITS_PER_LONG;
  2053. bit += BITS_PER_LONG;
  2054. continue;
  2055. }
  2056. }
  2057. if (!test_and_set_bit(bit, bitmap))
  2058. n++;
  2059. bit++;
  2060. }
  2061. return n;
  2062. }
  2063. /*
  2064. * Get the bitmap block storing the bit for chunk_block in zone.
  2065. */
  2066. static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
  2067. struct dm_zone *zone,
  2068. sector_t chunk_block)
  2069. {
  2070. sector_t bitmap_block = 1 + zmd->nr_map_blocks +
  2071. (sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +
  2072. (chunk_block >> DMZ_BLOCK_SHIFT_BITS);
  2073. return dmz_get_mblock(zmd, bitmap_block);
  2074. }
  2075. /*
  2076. * Copy the valid blocks bitmap of from_zone to the bitmap of to_zone.
  2077. */
  2078. int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
  2079. struct dm_zone *to_zone)
  2080. {
  2081. struct dmz_mblock *from_mblk, *to_mblk;
  2082. sector_t chunk_block = 0;
  2083. /* Get the zones bitmap blocks */
  2084. while (chunk_block < zmd->zone_nr_blocks) {
  2085. from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);
  2086. if (IS_ERR(from_mblk))
  2087. return PTR_ERR(from_mblk);
  2088. to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);
  2089. if (IS_ERR(to_mblk)) {
  2090. dmz_release_mblock(zmd, from_mblk);
  2091. return PTR_ERR(to_mblk);
  2092. }
  2093. memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE);
  2094. dmz_dirty_mblock(zmd, to_mblk);
  2095. dmz_release_mblock(zmd, to_mblk);
  2096. dmz_release_mblock(zmd, from_mblk);
  2097. chunk_block += zmd->zone_bits_per_mblk;
  2098. }
  2099. to_zone->weight = from_zone->weight;
  2100. return 0;
  2101. }
  2102. /*
  2103. * Merge the valid blocks bitmap of from_zone into the bitmap of to_zone,
  2104. * starting from chunk_block.
  2105. */
  2106. int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
  2107. struct dm_zone *to_zone, sector_t chunk_block)
  2108. {
  2109. unsigned int nr_blocks;
  2110. int ret;
  2111. /* Get the zones bitmap blocks */
  2112. while (chunk_block < zmd->zone_nr_blocks) {
  2113. /* Get a valid region from the source zone */
  2114. ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);
  2115. if (ret <= 0)
  2116. return ret;
  2117. nr_blocks = ret;
  2118. ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);
  2119. if (ret)
  2120. return ret;
  2121. chunk_block += nr_blocks;
  2122. }
  2123. return 0;
  2124. }
  2125. /*
  2126. * Validate all the blocks in the range [block..block+nr_blocks-1].
  2127. */
  2128. int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
  2129. sector_t chunk_block, unsigned int nr_blocks)
  2130. {
  2131. unsigned int count, bit, nr_bits;
  2132. unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
  2133. struct dmz_mblock *mblk;
  2134. unsigned int n = 0;
  2135. dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",
  2136. zone->id, (unsigned long long)chunk_block,
  2137. nr_blocks);
  2138. WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
  2139. while (nr_blocks) {
  2140. /* Get bitmap block */
  2141. mblk = dmz_get_bitmap(zmd, zone, chunk_block);
  2142. if (IS_ERR(mblk))
  2143. return PTR_ERR(mblk);
  2144. /* Set bits */
  2145. bit = chunk_block & DMZ_BLOCK_MASK_BITS;
  2146. nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
  2147. count = dmz_set_bits((unsigned long *)mblk->data, bit, nr_bits);
  2148. if (count) {
  2149. dmz_dirty_mblock(zmd, mblk);
  2150. n += count;
  2151. }
  2152. dmz_release_mblock(zmd, mblk);
  2153. nr_blocks -= nr_bits;
  2154. chunk_block += nr_bits;
  2155. }
  2156. if (likely(zone->weight + n <= zone_nr_blocks))
  2157. zone->weight += n;
  2158. else {
  2159. dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",
  2160. zone->id, zone->weight,
  2161. zone_nr_blocks - n);
  2162. zone->weight = zone_nr_blocks;
  2163. }
  2164. return 0;
  2165. }
  2166. /*
  2167. * Clear nr_bits bits in bitmap starting from bit.
  2168. * Return the number of bits cleared.
  2169. */
  2170. static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits)
  2171. {
  2172. unsigned long *addr;
  2173. int end = bit + nr_bits;
  2174. int n = 0;
  2175. while (bit < end) {
  2176. if (((bit & (BITS_PER_LONG - 1)) == 0) &&
  2177. ((end - bit) >= BITS_PER_LONG)) {
  2178. /* Try to clear whole word at once */
  2179. addr = bitmap + BIT_WORD(bit);
  2180. if (*addr == ULONG_MAX) {
  2181. *addr = 0;
  2182. n += BITS_PER_LONG;
  2183. bit += BITS_PER_LONG;
  2184. continue;
  2185. }
  2186. }
  2187. if (test_and_clear_bit(bit, bitmap))
  2188. n++;
  2189. bit++;
  2190. }
  2191. return n;
  2192. }
  2193. /*
  2194. * Invalidate all the blocks in the range [block..block+nr_blocks-1].
  2195. */
  2196. int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
  2197. sector_t chunk_block, unsigned int nr_blocks)
  2198. {
  2199. unsigned int count, bit, nr_bits;
  2200. struct dmz_mblock *mblk;
  2201. unsigned int n = 0;
  2202. dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",
  2203. zone->id, (u64)chunk_block, nr_blocks);
  2204. WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
  2205. while (nr_blocks) {
  2206. /* Get bitmap block */
  2207. mblk = dmz_get_bitmap(zmd, zone, chunk_block);
  2208. if (IS_ERR(mblk))
  2209. return PTR_ERR(mblk);
  2210. /* Clear bits */
  2211. bit = chunk_block & DMZ_BLOCK_MASK_BITS;
  2212. nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
  2213. count = dmz_clear_bits((unsigned long *)mblk->data,
  2214. bit, nr_bits);
  2215. if (count) {
  2216. dmz_dirty_mblock(zmd, mblk);
  2217. n += count;
  2218. }
  2219. dmz_release_mblock(zmd, mblk);
  2220. nr_blocks -= nr_bits;
  2221. chunk_block += nr_bits;
  2222. }
  2223. if (zone->weight >= n)
  2224. zone->weight -= n;
  2225. else {
  2226. dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",
  2227. zone->id, zone->weight, n);
  2228. zone->weight = 0;
  2229. }
  2230. return 0;
  2231. }
  2232. /*
  2233. * Get a block bit value.
  2234. */
  2235. static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
  2236. sector_t chunk_block)
  2237. {
  2238. struct dmz_mblock *mblk;
  2239. int ret;
  2240. WARN_ON(chunk_block >= zmd->zone_nr_blocks);
  2241. /* Get bitmap block */
  2242. mblk = dmz_get_bitmap(zmd, zone, chunk_block);
  2243. if (IS_ERR(mblk))
  2244. return PTR_ERR(mblk);
  2245. /* Get offset */
  2246. ret = test_bit(chunk_block & DMZ_BLOCK_MASK_BITS,
  2247. (unsigned long *) mblk->data) != 0;
  2248. dmz_release_mblock(zmd, mblk);
  2249. return ret;
  2250. }
  2251. /*
  2252. * Return the number of blocks from chunk_block to the first block with a bit
  2253. * value specified by set. Search at most nr_blocks blocks from chunk_block.
  2254. */
  2255. static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
  2256. sector_t chunk_block, unsigned int nr_blocks,
  2257. int set)
  2258. {
  2259. struct dmz_mblock *mblk;
  2260. unsigned int bit, set_bit, nr_bits;
  2261. unsigned int zone_bits = zmd->zone_bits_per_mblk;
  2262. unsigned long *bitmap;
  2263. int n = 0;
  2264. WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
  2265. while (nr_blocks) {
  2266. /* Get bitmap block */
  2267. mblk = dmz_get_bitmap(zmd, zone, chunk_block);
  2268. if (IS_ERR(mblk))
  2269. return PTR_ERR(mblk);
  2270. /* Get offset */
  2271. bitmap = (unsigned long *) mblk->data;
  2272. bit = chunk_block & DMZ_BLOCK_MASK_BITS;
  2273. nr_bits = min(nr_blocks, zone_bits - bit);
  2274. if (set)
  2275. set_bit = find_next_bit(bitmap, zone_bits, bit);
  2276. else
  2277. set_bit = find_next_zero_bit(bitmap, zone_bits, bit);
  2278. dmz_release_mblock(zmd, mblk);
  2279. n += set_bit - bit;
  2280. if (set_bit < zone_bits)
  2281. break;
  2282. nr_blocks -= nr_bits;
  2283. chunk_block += nr_bits;
  2284. }
  2285. return n;
  2286. }
  2287. /*
  2288. * Test if chunk_block is valid. If it is, the number of consecutive
  2289. * valid blocks from chunk_block will be returned.
  2290. */
  2291. int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
  2292. sector_t chunk_block)
  2293. {
  2294. int valid;
  2295. valid = dmz_test_block(zmd, zone, chunk_block);
  2296. if (valid <= 0)
  2297. return valid;
  2298. /* The block is valid: get the number of valid blocks from block */
  2299. return dmz_to_next_set_block(zmd, zone, chunk_block,
  2300. zmd->zone_nr_blocks - chunk_block, 0);
  2301. }
  2302. /*
  2303. * Find the first valid block from @chunk_block in @zone.
  2304. * If such a block is found, its number is returned using
  2305. * @chunk_block and the total number of valid blocks from @chunk_block
  2306. * is returned.
  2307. */
  2308. int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
  2309. sector_t *chunk_block)
  2310. {
  2311. sector_t start_block = *chunk_block;
  2312. int ret;
  2313. ret = dmz_to_next_set_block(zmd, zone, start_block,
  2314. zmd->zone_nr_blocks - start_block, 1);
  2315. if (ret < 0)
  2316. return ret;
  2317. start_block += ret;
  2318. *chunk_block = start_block;
  2319. return dmz_to_next_set_block(zmd, zone, start_block,
  2320. zmd->zone_nr_blocks - start_block, 0);
  2321. }
  2322. /*
  2323. * Count the number of bits set starting from bit up to bit + nr_bits - 1.
  2324. */
  2325. static int dmz_count_bits(void *bitmap, int bit, int nr_bits)
  2326. {
  2327. unsigned long *addr;
  2328. int end = bit + nr_bits;
  2329. int n = 0;
  2330. while (bit < end) {
  2331. if (((bit & (BITS_PER_LONG - 1)) == 0) &&
  2332. ((end - bit) >= BITS_PER_LONG)) {
  2333. addr = (unsigned long *)bitmap + BIT_WORD(bit);
  2334. if (*addr == ULONG_MAX) {
  2335. n += BITS_PER_LONG;
  2336. bit += BITS_PER_LONG;
  2337. continue;
  2338. }
  2339. }
  2340. if (test_bit(bit, bitmap))
  2341. n++;
  2342. bit++;
  2343. }
  2344. return n;
  2345. }
  2346. /*
  2347. * Get a zone weight.
  2348. */
  2349. static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
  2350. {
  2351. struct dmz_mblock *mblk;
  2352. sector_t chunk_block = 0;
  2353. unsigned int bit, nr_bits;
  2354. unsigned int nr_blocks = zmd->zone_nr_blocks;
  2355. void *bitmap;
  2356. int n = 0;
  2357. while (nr_blocks) {
  2358. /* Get bitmap block */
  2359. mblk = dmz_get_bitmap(zmd, zone, chunk_block);
  2360. if (IS_ERR(mblk)) {
  2361. n = 0;
  2362. break;
  2363. }
  2364. /* Count bits in this block */
  2365. bitmap = mblk->data;
  2366. bit = chunk_block & DMZ_BLOCK_MASK_BITS;
  2367. nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
  2368. n += dmz_count_bits(bitmap, bit, nr_bits);
  2369. dmz_release_mblock(zmd, mblk);
  2370. nr_blocks -= nr_bits;
  2371. chunk_block += nr_bits;
  2372. }
  2373. zone->weight = n;
  2374. }
  2375. /*
  2376. * Cleanup the zoned metadata resources.
  2377. */
  2378. static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
  2379. {
  2380. struct rb_root *root;
  2381. struct dmz_mblock *mblk, *next;
  2382. int i;
  2383. /* Release zone mapping resources */
  2384. if (zmd->map_mblk) {
  2385. for (i = 0; i < zmd->nr_map_blocks; i++)
  2386. dmz_release_mblock(zmd, zmd->map_mblk[i]);
  2387. kfree(zmd->map_mblk);
  2388. zmd->map_mblk = NULL;
  2389. }
  2390. /* Release super blocks */
  2391. for (i = 0; i < 2; i++) {
  2392. if (zmd->sb[i].mblk) {
  2393. dmz_free_mblock(zmd, zmd->sb[i].mblk);
  2394. zmd->sb[i].mblk = NULL;
  2395. }
  2396. }
  2397. /* Free cached blocks */
  2398. while (!list_empty(&zmd->mblk_dirty_list)) {
  2399. mblk = list_first_entry(&zmd->mblk_dirty_list,
  2400. struct dmz_mblock, link);
  2401. dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",
  2402. (u64)mblk->no, mblk->ref);
  2403. list_del_init(&mblk->link);
  2404. rb_erase(&mblk->node, &zmd->mblk_rbtree);
  2405. dmz_free_mblock(zmd, mblk);
  2406. }
  2407. while (!list_empty(&zmd->mblk_lru_list)) {
  2408. mblk = list_first_entry(&zmd->mblk_lru_list,
  2409. struct dmz_mblock, link);
  2410. list_del_init(&mblk->link);
  2411. rb_erase(&mblk->node, &zmd->mblk_rbtree);
  2412. dmz_free_mblock(zmd, mblk);
  2413. }
  2414. /* Sanity checks: the mblock rbtree should now be empty */
  2415. root = &zmd->mblk_rbtree;
  2416. rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
  2417. dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",
  2418. (u64)mblk->no, mblk->ref);
  2419. mblk->ref = 0;
  2420. dmz_free_mblock(zmd, mblk);
  2421. }
  2422. /* Free the zone descriptors */
  2423. dmz_drop_zones(zmd);
  2424. mutex_destroy(&zmd->mblk_flush_lock);
  2425. mutex_destroy(&zmd->map_lock);
  2426. }
  2427. static void dmz_print_dev(struct dmz_metadata *zmd, int num)
  2428. {
  2429. struct dmz_dev *dev = &zmd->dev[num];
  2430. if (bdev_zoned_model(dev->bdev) == BLK_ZONED_NONE)
  2431. dmz_dev_info(dev, "Regular block device");
  2432. else
  2433. dmz_dev_info(dev, "Host-%s zoned block device",
  2434. bdev_zoned_model(dev->bdev) == BLK_ZONED_HA ?
  2435. "aware" : "managed");
  2436. if (zmd->sb_version > 1) {
  2437. sector_t sector_offset =
  2438. dev->zone_offset << zmd->zone_nr_sectors_shift;
  2439. dmz_dev_info(dev, " %llu 512-byte logical sectors (offset %llu)",
  2440. (u64)dev->capacity, (u64)sector_offset);
  2441. dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors (offset %llu)",
  2442. dev->nr_zones, (u64)zmd->zone_nr_sectors,
  2443. (u64)dev->zone_offset);
  2444. } else {
  2445. dmz_dev_info(dev, " %llu 512-byte logical sectors",
  2446. (u64)dev->capacity);
  2447. dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors",
  2448. dev->nr_zones, (u64)zmd->zone_nr_sectors);
  2449. }
  2450. }
  2451. /*
  2452. * Initialize the zoned metadata.
  2453. */
  2454. int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
  2455. struct dmz_metadata **metadata,
  2456. const char *devname)
  2457. {
  2458. struct dmz_metadata *zmd;
  2459. unsigned int i;
  2460. struct dm_zone *zone;
  2461. int ret;
  2462. zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);
  2463. if (!zmd)
  2464. return -ENOMEM;
  2465. strcpy(zmd->devname, devname);
  2466. zmd->dev = dev;
  2467. zmd->nr_devs = num_dev;
  2468. zmd->mblk_rbtree = RB_ROOT;
  2469. init_rwsem(&zmd->mblk_sem);
  2470. mutex_init(&zmd->mblk_flush_lock);
  2471. spin_lock_init(&zmd->mblk_lock);
  2472. INIT_LIST_HEAD(&zmd->mblk_lru_list);
  2473. INIT_LIST_HEAD(&zmd->mblk_dirty_list);
  2474. mutex_init(&zmd->map_lock);
  2475. atomic_set(&zmd->unmap_nr_cache, 0);
  2476. INIT_LIST_HEAD(&zmd->unmap_cache_list);
  2477. INIT_LIST_HEAD(&zmd->map_cache_list);
  2478. atomic_set(&zmd->nr_reserved_seq_zones, 0);
  2479. INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);
  2480. init_waitqueue_head(&zmd->free_wq);
  2481. /* Initialize zone descriptors */
  2482. ret = dmz_init_zones(zmd);
  2483. if (ret)
  2484. goto err;
  2485. /* Get super block */
  2486. ret = dmz_load_sb(zmd);
  2487. if (ret)
  2488. goto err;
  2489. /* Set metadata zones starting from sb_zone */
  2490. for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
  2491. zone = dmz_get(zmd, zmd->sb[0].zone->id + i);
  2492. if (!zone) {
  2493. dmz_zmd_err(zmd,
  2494. "metadata zone %u not present", i);
  2495. ret = -ENXIO;
  2496. goto err;
  2497. }
  2498. if (!dmz_is_rnd(zone) && !dmz_is_cache(zone)) {
  2499. dmz_zmd_err(zmd,
  2500. "metadata zone %d is not random", i);
  2501. ret = -ENXIO;
  2502. goto err;
  2503. }
  2504. set_bit(DMZ_META, &zone->flags);
  2505. }
  2506. /* Load mapping table */
  2507. ret = dmz_load_mapping(zmd);
  2508. if (ret)
  2509. goto err;
  2510. /*
  2511. * Cache size boundaries: allow at least 2 super blocks, the chunk map
  2512. * blocks and enough blocks to be able to cache the bitmap blocks of
  2513. * up to 16 zones when idle (min_nr_mblks). Otherwise, if busy, allow
  2514. * the cache to add 512 more metadata blocks.
  2515. */
  2516. zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;
  2517. zmd->max_nr_mblks = zmd->min_nr_mblks + 512;
  2518. zmd->mblk_shrinker.count_objects = dmz_mblock_shrinker_count;
  2519. zmd->mblk_shrinker.scan_objects = dmz_mblock_shrinker_scan;
  2520. zmd->mblk_shrinker.seeks = DEFAULT_SEEKS;
  2521. /* Metadata cache shrinker */
  2522. ret = register_shrinker(&zmd->mblk_shrinker);
  2523. if (ret) {
  2524. dmz_zmd_err(zmd, "Register metadata cache shrinker failed");
  2525. goto err;
  2526. }
  2527. dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);
  2528. for (i = 0; i < zmd->nr_devs; i++)
  2529. dmz_print_dev(zmd, i);
  2530. dmz_zmd_info(zmd, " %u zones of %llu 512-byte logical sectors",
  2531. zmd->nr_zones, (u64)zmd->zone_nr_sectors);
  2532. dmz_zmd_debug(zmd, " %u metadata zones",
  2533. zmd->nr_meta_zones * 2);
  2534. dmz_zmd_debug(zmd, " %u data zones for %u chunks",
  2535. zmd->nr_data_zones, zmd->nr_chunks);
  2536. dmz_zmd_debug(zmd, " %u cache zones (%u unmapped)",
  2537. zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));
  2538. for (i = 0; i < zmd->nr_devs; i++) {
  2539. dmz_zmd_debug(zmd, " %u random zones (%u unmapped)",
  2540. dmz_nr_rnd_zones(zmd, i),
  2541. dmz_nr_unmap_rnd_zones(zmd, i));
  2542. dmz_zmd_debug(zmd, " %u sequential zones (%u unmapped)",
  2543. dmz_nr_seq_zones(zmd, i),
  2544. dmz_nr_unmap_seq_zones(zmd, i));
  2545. }
  2546. dmz_zmd_debug(zmd, " %u reserved sequential data zones",
  2547. zmd->nr_reserved_seq);
  2548. dmz_zmd_debug(zmd, "Format:");
  2549. dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",
  2550. zmd->nr_meta_blocks, zmd->max_nr_mblks);
  2551. dmz_zmd_debug(zmd, " %u data zone mapping blocks",
  2552. zmd->nr_map_blocks);
  2553. dmz_zmd_debug(zmd, " %u bitmap blocks",
  2554. zmd->nr_bitmap_blocks);
  2555. *metadata = zmd;
  2556. return 0;
  2557. err:
  2558. dmz_cleanup_metadata(zmd);
  2559. kfree(zmd);
  2560. *metadata = NULL;
  2561. return ret;
  2562. }
  2563. /*
  2564. * Cleanup the zoned metadata resources.
  2565. */
  2566. void dmz_dtr_metadata(struct dmz_metadata *zmd)
  2567. {
  2568. unregister_shrinker(&zmd->mblk_shrinker);
  2569. dmz_cleanup_metadata(zmd);
  2570. kfree(zmd);
  2571. }
  2572. /*
  2573. * Check zone information on resume.
  2574. */
  2575. int dmz_resume_metadata(struct dmz_metadata *zmd)
  2576. {
  2577. struct dm_zone *zone;
  2578. sector_t wp_block;
  2579. unsigned int i;
  2580. int ret;
  2581. /* Check zones */
  2582. for (i = 0; i < zmd->nr_zones; i++) {
  2583. zone = dmz_get(zmd, i);
  2584. if (!zone) {
  2585. dmz_zmd_err(zmd, "Unable to get zone %u", i);
  2586. return -EIO;
  2587. }
  2588. wp_block = zone->wp_block;
  2589. ret = dmz_update_zone(zmd, zone);
  2590. if (ret) {
  2591. dmz_zmd_err(zmd, "Broken zone %u", i);
  2592. return ret;
  2593. }
  2594. if (dmz_is_offline(zone)) {
  2595. dmz_zmd_warn(zmd, "Zone %u is offline", i);
  2596. continue;
  2597. }
  2598. /* Check write pointer */
  2599. if (!dmz_is_seq(zone))
  2600. zone->wp_block = 0;
  2601. else if (zone->wp_block != wp_block) {
  2602. dmz_zmd_err(zmd, "Zone %u: Invalid wp (%llu / %llu)",
  2603. i, (u64)zone->wp_block, (u64)wp_block);
  2604. zone->wp_block = wp_block;
  2605. dmz_invalidate_blocks(zmd, zone, zone->wp_block,
  2606. zmd->zone_nr_blocks - zone->wp_block);
  2607. }
  2608. }
  2609. return 0;
  2610. }