dm-clone-target.c 55 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved.
  4. */
  5. #include <linux/mm.h>
  6. #include <linux/bio.h>
  7. #include <linux/err.h>
  8. #include <linux/hash.h>
  9. #include <linux/list.h>
  10. #include <linux/log2.h>
  11. #include <linux/init.h>
  12. #include <linux/slab.h>
  13. #include <linux/wait.h>
  14. #include <linux/dm-io.h>
  15. #include <linux/mutex.h>
  16. #include <linux/atomic.h>
  17. #include <linux/bitops.h>
  18. #include <linux/blkdev.h>
  19. #include <linux/kdev_t.h>
  20. #include <linux/kernel.h>
  21. #include <linux/module.h>
  22. #include <linux/jiffies.h>
  23. #include <linux/mempool.h>
  24. #include <linux/spinlock.h>
  25. #include <linux/blk_types.h>
  26. #include <linux/dm-kcopyd.h>
  27. #include <linux/workqueue.h>
  28. #include <linux/backing-dev.h>
  29. #include <linux/device-mapper.h>
  30. #include "dm.h"
  31. #include "dm-clone-metadata.h"
  32. #define DM_MSG_PREFIX "clone"
  33. /*
  34. * Minimum and maximum allowed region sizes
  35. */
  36. #define MIN_REGION_SIZE (1 << 3) /* 4KB */
  37. #define MAX_REGION_SIZE (1 << 21) /* 1GB */
  38. #define MIN_HYDRATIONS 256 /* Size of hydration mempool */
  39. #define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */
  40. #define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */
  41. #define COMMIT_PERIOD HZ /* 1 sec */
  42. /*
  43. * Hydration hash table size: 1 << HASH_TABLE_BITS
  44. */
  45. #define HASH_TABLE_BITS 15
  46. DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle,
  47. "A percentage of time allocated for hydrating regions");
  48. /* Slab cache for struct dm_clone_region_hydration */
  49. static struct kmem_cache *_hydration_cache;
  50. /* dm-clone metadata modes */
  51. enum clone_metadata_mode {
  52. CM_WRITE, /* metadata may be changed */
  53. CM_READ_ONLY, /* metadata may not be changed */
  54. CM_FAIL, /* all metadata I/O fails */
  55. };
  56. struct hash_table_bucket;
  57. struct clone {
  58. struct dm_target *ti;
  59. struct dm_dev *metadata_dev;
  60. struct dm_dev *dest_dev;
  61. struct dm_dev *source_dev;
  62. unsigned long nr_regions;
  63. sector_t region_size;
  64. unsigned int region_shift;
  65. /*
  66. * A metadata commit and the actions taken in case it fails should run
  67. * as a single atomic step.
  68. */
  69. struct mutex commit_lock;
  70. struct dm_clone_metadata *cmd;
  71. /*
  72. * bio used to flush the destination device, before committing the
  73. * metadata.
  74. */
  75. struct bio flush_bio;
  76. /* Region hydration hash table */
  77. struct hash_table_bucket *ht;
  78. atomic_t ios_in_flight;
  79. wait_queue_head_t hydration_stopped;
  80. mempool_t hydration_pool;
  81. unsigned long last_commit_jiffies;
  82. /*
  83. * We defer incoming WRITE bios for regions that are not hydrated,
  84. * until after these regions have been hydrated.
  85. *
  86. * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the
  87. * metadata have been committed.
  88. */
  89. spinlock_t lock;
  90. struct bio_list deferred_bios;
  91. struct bio_list deferred_discard_bios;
  92. struct bio_list deferred_flush_bios;
  93. struct bio_list deferred_flush_completions;
  94. /* Maximum number of regions being copied during background hydration. */
  95. unsigned int hydration_threshold;
  96. /* Number of regions to batch together during background hydration. */
  97. unsigned int hydration_batch_size;
  98. /* Which region to hydrate next */
  99. unsigned long hydration_offset;
  100. atomic_t hydrations_in_flight;
  101. /*
  102. * Save a copy of the table line rather than reconstructing it for the
  103. * status.
  104. */
  105. unsigned int nr_ctr_args;
  106. const char **ctr_args;
  107. struct workqueue_struct *wq;
  108. struct work_struct worker;
  109. struct delayed_work waker;
  110. struct dm_kcopyd_client *kcopyd_client;
  111. enum clone_metadata_mode mode;
  112. unsigned long flags;
  113. };
  114. /*
  115. * dm-clone flags
  116. */
  117. #define DM_CLONE_DISCARD_PASSDOWN 0
  118. #define DM_CLONE_HYDRATION_ENABLED 1
  119. #define DM_CLONE_HYDRATION_SUSPENDED 2
  120. /*---------------------------------------------------------------------------*/
  121. /*
  122. * Metadata failure handling.
  123. */
  124. static enum clone_metadata_mode get_clone_mode(struct clone *clone)
  125. {
  126. return READ_ONCE(clone->mode);
  127. }
  128. static const char *clone_device_name(struct clone *clone)
  129. {
  130. return dm_table_device_name(clone->ti->table);
  131. }
  132. static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode)
  133. {
  134. const char *descs[] = {
  135. "read-write",
  136. "read-only",
  137. "fail"
  138. };
  139. enum clone_metadata_mode old_mode = get_clone_mode(clone);
  140. /* Never move out of fail mode */
  141. if (old_mode == CM_FAIL)
  142. new_mode = CM_FAIL;
  143. switch (new_mode) {
  144. case CM_FAIL:
  145. case CM_READ_ONLY:
  146. dm_clone_metadata_set_read_only(clone->cmd);
  147. break;
  148. case CM_WRITE:
  149. dm_clone_metadata_set_read_write(clone->cmd);
  150. break;
  151. }
  152. WRITE_ONCE(clone->mode, new_mode);
  153. if (new_mode != old_mode) {
  154. dm_table_event(clone->ti->table);
  155. DMINFO("%s: Switching to %s mode", clone_device_name(clone),
  156. descs[(int)new_mode]);
  157. }
  158. }
  159. static void __abort_transaction(struct clone *clone)
  160. {
  161. const char *dev_name = clone_device_name(clone);
  162. if (get_clone_mode(clone) >= CM_READ_ONLY)
  163. return;
  164. DMERR("%s: Aborting current metadata transaction", dev_name);
  165. if (dm_clone_metadata_abort(clone->cmd)) {
  166. DMERR("%s: Failed to abort metadata transaction", dev_name);
  167. __set_clone_mode(clone, CM_FAIL);
  168. }
  169. }
  170. static void __reload_in_core_bitset(struct clone *clone)
  171. {
  172. const char *dev_name = clone_device_name(clone);
  173. if (get_clone_mode(clone) == CM_FAIL)
  174. return;
  175. /* Reload the on-disk bitset */
  176. DMINFO("%s: Reloading on-disk bitmap", dev_name);
  177. if (dm_clone_reload_in_core_bitset(clone->cmd)) {
  178. DMERR("%s: Failed to reload on-disk bitmap", dev_name);
  179. __set_clone_mode(clone, CM_FAIL);
  180. }
  181. }
  182. static void __metadata_operation_failed(struct clone *clone, const char *op, int r)
  183. {
  184. DMERR("%s: Metadata operation `%s' failed: error = %d",
  185. clone_device_name(clone), op, r);
  186. __abort_transaction(clone);
  187. __set_clone_mode(clone, CM_READ_ONLY);
  188. /*
  189. * dm_clone_reload_in_core_bitset() may run concurrently with either
  190. * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but
  191. * it's safe as we have already set the metadata to read-only mode.
  192. */
  193. __reload_in_core_bitset(clone);
  194. }
  195. /*---------------------------------------------------------------------------*/
  196. /* Wake up anyone waiting for region hydrations to stop */
  197. static inline void wakeup_hydration_waiters(struct clone *clone)
  198. {
  199. wake_up_all(&clone->hydration_stopped);
  200. }
  201. static inline void wake_worker(struct clone *clone)
  202. {
  203. queue_work(clone->wq, &clone->worker);
  204. }
  205. /*---------------------------------------------------------------------------*/
  206. /*
  207. * bio helper functions.
  208. */
  209. static inline void remap_to_source(struct clone *clone, struct bio *bio)
  210. {
  211. bio_set_dev(bio, clone->source_dev->bdev);
  212. }
  213. static inline void remap_to_dest(struct clone *clone, struct bio *bio)
  214. {
  215. bio_set_dev(bio, clone->dest_dev->bdev);
  216. }
  217. static bool bio_triggers_commit(struct clone *clone, struct bio *bio)
  218. {
  219. return op_is_flush(bio->bi_opf) &&
  220. dm_clone_changed_this_transaction(clone->cmd);
  221. }
  222. /* Get the address of the region in sectors */
  223. static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr)
  224. {
  225. return ((sector_t)region_nr << clone->region_shift);
  226. }
  227. /* Get the region number of the bio */
  228. static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio)
  229. {
  230. return (bio->bi_iter.bi_sector >> clone->region_shift);
  231. }
  232. /* Get the region range covered by the bio */
  233. static void bio_region_range(struct clone *clone, struct bio *bio,
  234. unsigned long *rs, unsigned long *nr_regions)
  235. {
  236. unsigned long end;
  237. *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size);
  238. end = bio_end_sector(bio) >> clone->region_shift;
  239. if (*rs >= end)
  240. *nr_regions = 0;
  241. else
  242. *nr_regions = end - *rs;
  243. }
  244. /* Check whether a bio overwrites a region */
  245. static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio)
  246. {
  247. return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size);
  248. }
  249. static void fail_bios(struct bio_list *bios, blk_status_t status)
  250. {
  251. struct bio *bio;
  252. while ((bio = bio_list_pop(bios))) {
  253. bio->bi_status = status;
  254. bio_endio(bio);
  255. }
  256. }
  257. static void submit_bios(struct bio_list *bios)
  258. {
  259. struct bio *bio;
  260. struct blk_plug plug;
  261. blk_start_plug(&plug);
  262. while ((bio = bio_list_pop(bios)))
  263. submit_bio_noacct(bio);
  264. blk_finish_plug(&plug);
  265. }
  266. /*
  267. * Submit bio to the underlying device.
  268. *
  269. * If the bio triggers a commit, delay it, until after the metadata have been
  270. * committed.
  271. *
  272. * NOTE: The bio remapping must be performed by the caller.
  273. */
  274. static void issue_bio(struct clone *clone, struct bio *bio)
  275. {
  276. if (!bio_triggers_commit(clone, bio)) {
  277. submit_bio_noacct(bio);
  278. return;
  279. }
  280. /*
  281. * If the metadata mode is RO or FAIL we won't be able to commit the
  282. * metadata, so we complete the bio with an error.
  283. */
  284. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
  285. bio_io_error(bio);
  286. return;
  287. }
  288. /*
  289. * Batch together any bios that trigger commits and then issue a single
  290. * commit for them in process_deferred_flush_bios().
  291. */
  292. spin_lock_irq(&clone->lock);
  293. bio_list_add(&clone->deferred_flush_bios, bio);
  294. spin_unlock_irq(&clone->lock);
  295. wake_worker(clone);
  296. }
  297. /*
  298. * Remap bio to the destination device and submit it.
  299. *
  300. * If the bio triggers a commit, delay it, until after the metadata have been
  301. * committed.
  302. */
  303. static void remap_and_issue(struct clone *clone, struct bio *bio)
  304. {
  305. remap_to_dest(clone, bio);
  306. issue_bio(clone, bio);
  307. }
  308. /*
  309. * Issue bios that have been deferred until after their region has finished
  310. * hydrating.
  311. *
  312. * We delegate the bio submission to the worker thread, so this is safe to call
  313. * from interrupt context.
  314. */
  315. static void issue_deferred_bios(struct clone *clone, struct bio_list *bios)
  316. {
  317. struct bio *bio;
  318. unsigned long flags;
  319. struct bio_list flush_bios = BIO_EMPTY_LIST;
  320. struct bio_list normal_bios = BIO_EMPTY_LIST;
  321. if (bio_list_empty(bios))
  322. return;
  323. while ((bio = bio_list_pop(bios))) {
  324. if (bio_triggers_commit(clone, bio))
  325. bio_list_add(&flush_bios, bio);
  326. else
  327. bio_list_add(&normal_bios, bio);
  328. }
  329. spin_lock_irqsave(&clone->lock, flags);
  330. bio_list_merge(&clone->deferred_bios, &normal_bios);
  331. bio_list_merge(&clone->deferred_flush_bios, &flush_bios);
  332. spin_unlock_irqrestore(&clone->lock, flags);
  333. wake_worker(clone);
  334. }
  335. static void complete_overwrite_bio(struct clone *clone, struct bio *bio)
  336. {
  337. unsigned long flags;
  338. /*
  339. * If the bio has the REQ_FUA flag set we must commit the metadata
  340. * before signaling its completion.
  341. *
  342. * complete_overwrite_bio() is only called by hydration_complete(),
  343. * after having successfully updated the metadata. This means we don't
  344. * need to call dm_clone_changed_this_transaction() to check if the
  345. * metadata has changed and thus we can avoid taking the metadata spin
  346. * lock.
  347. */
  348. if (!(bio->bi_opf & REQ_FUA)) {
  349. bio_endio(bio);
  350. return;
  351. }
  352. /*
  353. * If the metadata mode is RO or FAIL we won't be able to commit the
  354. * metadata, so we complete the bio with an error.
  355. */
  356. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
  357. bio_io_error(bio);
  358. return;
  359. }
  360. /*
  361. * Batch together any bios that trigger commits and then issue a single
  362. * commit for them in process_deferred_flush_bios().
  363. */
  364. spin_lock_irqsave(&clone->lock, flags);
  365. bio_list_add(&clone->deferred_flush_completions, bio);
  366. spin_unlock_irqrestore(&clone->lock, flags);
  367. wake_worker(clone);
  368. }
  369. static void trim_bio(struct bio *bio, sector_t sector, unsigned int len)
  370. {
  371. bio->bi_iter.bi_sector = sector;
  372. bio->bi_iter.bi_size = to_bytes(len);
  373. }
  374. static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success)
  375. {
  376. unsigned long rs, nr_regions;
  377. /*
  378. * If the destination device supports discards, remap and trim the
  379. * discard bio and pass it down. Otherwise complete the bio
  380. * immediately.
  381. */
  382. if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) {
  383. remap_to_dest(clone, bio);
  384. bio_region_range(clone, bio, &rs, &nr_regions);
  385. trim_bio(bio, region_to_sector(clone, rs),
  386. nr_regions << clone->region_shift);
  387. submit_bio_noacct(bio);
  388. } else
  389. bio_endio(bio);
  390. }
  391. static void process_discard_bio(struct clone *clone, struct bio *bio)
  392. {
  393. unsigned long rs, nr_regions;
  394. bio_region_range(clone, bio, &rs, &nr_regions);
  395. if (!nr_regions) {
  396. bio_endio(bio);
  397. return;
  398. }
  399. if (WARN_ON(rs >= clone->nr_regions || (rs + nr_regions) < rs ||
  400. (rs + nr_regions) > clone->nr_regions)) {
  401. DMERR("%s: Invalid range (%lu + %lu, total regions %lu) for discard (%llu + %u)",
  402. clone_device_name(clone), rs, nr_regions,
  403. clone->nr_regions,
  404. (unsigned long long)bio->bi_iter.bi_sector,
  405. bio_sectors(bio));
  406. bio_endio(bio);
  407. return;
  408. }
  409. /*
  410. * The covered regions are already hydrated so we just need to pass
  411. * down the discard.
  412. */
  413. if (dm_clone_is_range_hydrated(clone->cmd, rs, nr_regions)) {
  414. complete_discard_bio(clone, bio, true);
  415. return;
  416. }
  417. /*
  418. * If the metadata mode is RO or FAIL we won't be able to update the
  419. * metadata for the regions covered by the discard so we just ignore
  420. * it.
  421. */
  422. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
  423. bio_endio(bio);
  424. return;
  425. }
  426. /*
  427. * Defer discard processing.
  428. */
  429. spin_lock_irq(&clone->lock);
  430. bio_list_add(&clone->deferred_discard_bios, bio);
  431. spin_unlock_irq(&clone->lock);
  432. wake_worker(clone);
  433. }
  434. /*---------------------------------------------------------------------------*/
  435. /*
  436. * dm-clone region hydrations.
  437. */
  438. struct dm_clone_region_hydration {
  439. struct clone *clone;
  440. unsigned long region_nr;
  441. struct bio *overwrite_bio;
  442. bio_end_io_t *overwrite_bio_end_io;
  443. struct bio_list deferred_bios;
  444. blk_status_t status;
  445. /* Used by hydration batching */
  446. struct list_head list;
  447. /* Used by hydration hash table */
  448. struct hlist_node h;
  449. };
  450. /*
  451. * Hydration hash table implementation.
  452. *
  453. * Ideally we would like to use list_bl, which uses bit spin locks and employs
  454. * the least significant bit of the list head to lock the corresponding bucket,
  455. * reducing the memory overhead for the locks. But, currently, list_bl and bit
  456. * spin locks don't support IRQ safe versions. Since we have to take the lock
  457. * in both process and interrupt context, we must fall back to using regular
  458. * spin locks; one per hash table bucket.
  459. */
  460. struct hash_table_bucket {
  461. struct hlist_head head;
  462. /* Spinlock protecting the bucket */
  463. spinlock_t lock;
  464. };
  465. #define bucket_lock_irqsave(bucket, flags) \
  466. spin_lock_irqsave(&(bucket)->lock, flags)
  467. #define bucket_unlock_irqrestore(bucket, flags) \
  468. spin_unlock_irqrestore(&(bucket)->lock, flags)
  469. #define bucket_lock_irq(bucket) \
  470. spin_lock_irq(&(bucket)->lock)
  471. #define bucket_unlock_irq(bucket) \
  472. spin_unlock_irq(&(bucket)->lock)
  473. static int hash_table_init(struct clone *clone)
  474. {
  475. unsigned int i, sz;
  476. struct hash_table_bucket *bucket;
  477. sz = 1 << HASH_TABLE_BITS;
  478. clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL);
  479. if (!clone->ht)
  480. return -ENOMEM;
  481. for (i = 0; i < sz; i++) {
  482. bucket = clone->ht + i;
  483. INIT_HLIST_HEAD(&bucket->head);
  484. spin_lock_init(&bucket->lock);
  485. }
  486. return 0;
  487. }
  488. static void hash_table_exit(struct clone *clone)
  489. {
  490. kvfree(clone->ht);
  491. }
  492. static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone,
  493. unsigned long region_nr)
  494. {
  495. return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)];
  496. }
  497. /*
  498. * Search hash table for a hydration with hd->region_nr == region_nr
  499. *
  500. * NOTE: Must be called with the bucket lock held
  501. */
  502. static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket,
  503. unsigned long region_nr)
  504. {
  505. struct dm_clone_region_hydration *hd;
  506. hlist_for_each_entry(hd, &bucket->head, h) {
  507. if (hd->region_nr == region_nr)
  508. return hd;
  509. }
  510. return NULL;
  511. }
  512. /*
  513. * Insert a hydration into the hash table.
  514. *
  515. * NOTE: Must be called with the bucket lock held.
  516. */
  517. static inline void __insert_region_hydration(struct hash_table_bucket *bucket,
  518. struct dm_clone_region_hydration *hd)
  519. {
  520. hlist_add_head(&hd->h, &bucket->head);
  521. }
  522. /*
  523. * This function inserts a hydration into the hash table, unless someone else
  524. * managed to insert a hydration for the same region first. In the latter case
  525. * it returns the existing hydration descriptor for this region.
  526. *
  527. * NOTE: Must be called with the hydration hash table lock held.
  528. */
  529. static struct dm_clone_region_hydration *
  530. __find_or_insert_region_hydration(struct hash_table_bucket *bucket,
  531. struct dm_clone_region_hydration *hd)
  532. {
  533. struct dm_clone_region_hydration *hd2;
  534. hd2 = __hash_find(bucket, hd->region_nr);
  535. if (hd2)
  536. return hd2;
  537. __insert_region_hydration(bucket, hd);
  538. return hd;
  539. }
  540. /*---------------------------------------------------------------------------*/
  541. /* Allocate a hydration */
  542. static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone)
  543. {
  544. struct dm_clone_region_hydration *hd;
  545. /*
  546. * Allocate a hydration from the hydration mempool.
  547. * This might block but it can't fail.
  548. */
  549. hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);
  550. hd->clone = clone;
  551. return hd;
  552. }
  553. static inline void free_hydration(struct dm_clone_region_hydration *hd)
  554. {
  555. mempool_free(hd, &hd->clone->hydration_pool);
  556. }
  557. /* Initialize a hydration */
  558. static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)
  559. {
  560. hd->region_nr = region_nr;
  561. hd->overwrite_bio = NULL;
  562. bio_list_init(&hd->deferred_bios);
  563. hd->status = 0;
  564. INIT_LIST_HEAD(&hd->list);
  565. INIT_HLIST_NODE(&hd->h);
  566. }
  567. /*---------------------------------------------------------------------------*/
  568. /*
  569. * Update dm-clone's metadata after a region has finished hydrating and remove
  570. * hydration from the hash table.
  571. */
  572. static int hydration_update_metadata(struct dm_clone_region_hydration *hd)
  573. {
  574. int r = 0;
  575. unsigned long flags;
  576. struct hash_table_bucket *bucket;
  577. struct clone *clone = hd->clone;
  578. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
  579. r = -EPERM;
  580. /* Update the metadata */
  581. if (likely(!r) && hd->status == BLK_STS_OK)
  582. r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);
  583. bucket = get_hash_table_bucket(clone, hd->region_nr);
  584. /* Remove hydration from hash table */
  585. bucket_lock_irqsave(bucket, flags);
  586. hlist_del(&hd->h);
  587. bucket_unlock_irqrestore(bucket, flags);
  588. return r;
  589. }
  590. /*
  591. * Complete a region's hydration:
  592. *
  593. * 1. Update dm-clone's metadata.
  594. * 2. Remove hydration from hash table.
  595. * 3. Complete overwrite bio.
  596. * 4. Issue deferred bios.
  597. * 5. If this was the last hydration, wake up anyone waiting for
  598. * hydrations to finish.
  599. */
  600. static void hydration_complete(struct dm_clone_region_hydration *hd)
  601. {
  602. int r;
  603. blk_status_t status;
  604. struct clone *clone = hd->clone;
  605. r = hydration_update_metadata(hd);
  606. if (hd->status == BLK_STS_OK && likely(!r)) {
  607. if (hd->overwrite_bio)
  608. complete_overwrite_bio(clone, hd->overwrite_bio);
  609. issue_deferred_bios(clone, &hd->deferred_bios);
  610. } else {
  611. status = r ? BLK_STS_IOERR : hd->status;
  612. if (hd->overwrite_bio)
  613. bio_list_add(&hd->deferred_bios, hd->overwrite_bio);
  614. fail_bios(&hd->deferred_bios, status);
  615. }
  616. free_hydration(hd);
  617. if (atomic_dec_and_test(&clone->hydrations_in_flight))
  618. wakeup_hydration_waiters(clone);
  619. }
  620. static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context)
  621. {
  622. blk_status_t status;
  623. struct dm_clone_region_hydration *tmp, *hd = context;
  624. struct clone *clone = hd->clone;
  625. LIST_HEAD(batched_hydrations);
  626. if (read_err || write_err) {
  627. DMERR_LIMIT("%s: hydration failed", clone_device_name(clone));
  628. status = BLK_STS_IOERR;
  629. } else {
  630. status = BLK_STS_OK;
  631. }
  632. list_splice_tail(&hd->list, &batched_hydrations);
  633. hd->status = status;
  634. hydration_complete(hd);
  635. /* Complete batched hydrations */
  636. list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {
  637. hd->status = status;
  638. hydration_complete(hd);
  639. }
  640. /* Continue background hydration, if there is no I/O in-flight */
  641. if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
  642. !atomic_read(&clone->ios_in_flight))
  643. wake_worker(clone);
  644. }
  645. static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)
  646. {
  647. unsigned long region_start, region_end;
  648. sector_t tail_size, region_size, total_size;
  649. struct dm_io_region from, to;
  650. struct clone *clone = hd->clone;
  651. if (WARN_ON(!nr_regions))
  652. return;
  653. region_size = clone->region_size;
  654. region_start = hd->region_nr;
  655. region_end = region_start + nr_regions - 1;
  656. total_size = region_to_sector(clone, nr_regions - 1);
  657. if (region_end == clone->nr_regions - 1) {
  658. /*
  659. * The last region of the target might be smaller than
  660. * region_size.
  661. */
  662. tail_size = clone->ti->len & (region_size - 1);
  663. if (!tail_size)
  664. tail_size = region_size;
  665. } else {
  666. tail_size = region_size;
  667. }
  668. total_size += tail_size;
  669. from.bdev = clone->source_dev->bdev;
  670. from.sector = region_to_sector(clone, region_start);
  671. from.count = total_size;
  672. to.bdev = clone->dest_dev->bdev;
  673. to.sector = from.sector;
  674. to.count = from.count;
  675. /* Issue copy */
  676. atomic_add(nr_regions, &clone->hydrations_in_flight);
  677. dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0,
  678. hydration_kcopyd_callback, hd);
  679. }
  680. static void overwrite_endio(struct bio *bio)
  681. {
  682. struct dm_clone_region_hydration *hd = bio->bi_private;
  683. bio->bi_end_io = hd->overwrite_bio_end_io;
  684. hd->status = bio->bi_status;
  685. hydration_complete(hd);
  686. }
  687. static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)
  688. {
  689. /*
  690. * We don't need to save and restore bio->bi_private because device
  691. * mapper core generates a new bio for us to use, with clean
  692. * bi_private.
  693. */
  694. hd->overwrite_bio = bio;
  695. hd->overwrite_bio_end_io = bio->bi_end_io;
  696. bio->bi_end_io = overwrite_endio;
  697. bio->bi_private = hd;
  698. atomic_inc(&hd->clone->hydrations_in_flight);
  699. submit_bio_noacct(bio);
  700. }
  701. /*
  702. * Hydrate bio's region.
  703. *
  704. * This function starts the hydration of the bio's region and puts the bio in
  705. * the list of deferred bios for this region. In case, by the time this
  706. * function is called, the region has finished hydrating it's submitted to the
  707. * destination device.
  708. *
  709. * NOTE: The bio remapping must be performed by the caller.
  710. */
  711. static void hydrate_bio_region(struct clone *clone, struct bio *bio)
  712. {
  713. unsigned long region_nr;
  714. struct hash_table_bucket *bucket;
  715. struct dm_clone_region_hydration *hd, *hd2;
  716. region_nr = bio_to_region(clone, bio);
  717. bucket = get_hash_table_bucket(clone, region_nr);
  718. bucket_lock_irq(bucket);
  719. hd = __hash_find(bucket, region_nr);
  720. if (hd) {
  721. /* Someone else is hydrating the region */
  722. bio_list_add(&hd->deferred_bios, bio);
  723. bucket_unlock_irq(bucket);
  724. return;
  725. }
  726. if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
  727. /* The region has been hydrated */
  728. bucket_unlock_irq(bucket);
  729. issue_bio(clone, bio);
  730. return;
  731. }
  732. /*
  733. * We must allocate a hydration descriptor and start the hydration of
  734. * the corresponding region.
  735. */
  736. bucket_unlock_irq(bucket);
  737. hd = alloc_hydration(clone);
  738. hydration_init(hd, region_nr);
  739. bucket_lock_irq(bucket);
  740. /* Check if the region has been hydrated in the meantime. */
  741. if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
  742. bucket_unlock_irq(bucket);
  743. free_hydration(hd);
  744. issue_bio(clone, bio);
  745. return;
  746. }
  747. hd2 = __find_or_insert_region_hydration(bucket, hd);
  748. if (hd2 != hd) {
  749. /* Someone else started the region's hydration. */
  750. bio_list_add(&hd2->deferred_bios, bio);
  751. bucket_unlock_irq(bucket);
  752. free_hydration(hd);
  753. return;
  754. }
  755. /*
  756. * If the metadata mode is RO or FAIL then there is no point starting a
  757. * hydration, since we will not be able to update the metadata when the
  758. * hydration finishes.
  759. */
  760. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
  761. hlist_del(&hd->h);
  762. bucket_unlock_irq(bucket);
  763. free_hydration(hd);
  764. bio_io_error(bio);
  765. return;
  766. }
  767. /*
  768. * Start region hydration.
  769. *
  770. * If a bio overwrites a region, i.e., its size is equal to the
  771. * region's size, then we don't need to copy the region from the source
  772. * to the destination device.
  773. */
  774. if (is_overwrite_bio(clone, bio)) {
  775. bucket_unlock_irq(bucket);
  776. hydration_overwrite(hd, bio);
  777. } else {
  778. bio_list_add(&hd->deferred_bios, bio);
  779. bucket_unlock_irq(bucket);
  780. hydration_copy(hd, 1);
  781. }
  782. }
  783. /*---------------------------------------------------------------------------*/
  784. /*
  785. * Background hydrations.
  786. */
  787. /*
  788. * Batch region hydrations.
  789. *
  790. * To better utilize device bandwidth we batch together the hydration of
  791. * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which
  792. * is good for small, random write performance (because of the overwriting of
  793. * un-hydrated regions) and at the same time issue big copy requests to kcopyd
  794. * to achieve high hydration bandwidth.
  795. */
  796. struct batch_info {
  797. struct dm_clone_region_hydration *head;
  798. unsigned int nr_batched_regions;
  799. };
  800. static void __batch_hydration(struct batch_info *batch,
  801. struct dm_clone_region_hydration *hd)
  802. {
  803. struct clone *clone = hd->clone;
  804. unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size);
  805. if (batch->head) {
  806. /* Try to extend the current batch */
  807. if (batch->nr_batched_regions < max_batch_size &&
  808. (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {
  809. list_add_tail(&hd->list, &batch->head->list);
  810. batch->nr_batched_regions++;
  811. hd = NULL;
  812. }
  813. /* Check if we should issue the current batch */
  814. if (batch->nr_batched_regions >= max_batch_size || hd) {
  815. hydration_copy(batch->head, batch->nr_batched_regions);
  816. batch->head = NULL;
  817. batch->nr_batched_regions = 0;
  818. }
  819. }
  820. if (!hd)
  821. return;
  822. /* We treat max batch sizes of zero and one equivalently */
  823. if (max_batch_size <= 1) {
  824. hydration_copy(hd, 1);
  825. return;
  826. }
  827. /* Start a new batch */
  828. BUG_ON(!list_empty(&hd->list));
  829. batch->head = hd;
  830. batch->nr_batched_regions = 1;
  831. }
  832. static unsigned long __start_next_hydration(struct clone *clone,
  833. unsigned long offset,
  834. struct batch_info *batch)
  835. {
  836. struct hash_table_bucket *bucket;
  837. struct dm_clone_region_hydration *hd;
  838. unsigned long nr_regions = clone->nr_regions;
  839. hd = alloc_hydration(clone);
  840. /* Try to find a region to hydrate. */
  841. do {
  842. offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset);
  843. if (offset == nr_regions)
  844. break;
  845. bucket = get_hash_table_bucket(clone, offset);
  846. bucket_lock_irq(bucket);
  847. if (!dm_clone_is_region_hydrated(clone->cmd, offset) &&
  848. !__hash_find(bucket, offset)) {
  849. hydration_init(hd, offset);
  850. __insert_region_hydration(bucket, hd);
  851. bucket_unlock_irq(bucket);
  852. /* Batch hydration */
  853. __batch_hydration(batch, hd);
  854. return (offset + 1);
  855. }
  856. bucket_unlock_irq(bucket);
  857. } while (++offset < nr_regions);
  858. if (hd)
  859. free_hydration(hd);
  860. return offset;
  861. }
  862. /*
  863. * This function searches for regions that still reside in the source device
  864. * and starts their hydration.
  865. */
  866. static void do_hydration(struct clone *clone)
  867. {
  868. unsigned int current_volume;
  869. unsigned long offset, nr_regions = clone->nr_regions;
  870. struct batch_info batch = {
  871. .head = NULL,
  872. .nr_batched_regions = 0,
  873. };
  874. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
  875. return;
  876. if (dm_clone_is_hydration_done(clone->cmd))
  877. return;
  878. /*
  879. * Avoid race with device suspension.
  880. */
  881. atomic_inc(&clone->hydrations_in_flight);
  882. /*
  883. * Make sure atomic_inc() is ordered before test_bit(), otherwise we
  884. * might race with clone_postsuspend() and start a region hydration
  885. * after the target has been suspended.
  886. *
  887. * This is paired with the smp_mb__after_atomic() in
  888. * clone_postsuspend().
  889. */
  890. smp_mb__after_atomic();
  891. offset = clone->hydration_offset;
  892. while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) &&
  893. !atomic_read(&clone->ios_in_flight) &&
  894. test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
  895. offset < nr_regions) {
  896. current_volume = atomic_read(&clone->hydrations_in_flight);
  897. current_volume += batch.nr_batched_regions;
  898. if (current_volume > READ_ONCE(clone->hydration_threshold))
  899. break;
  900. offset = __start_next_hydration(clone, offset, &batch);
  901. }
  902. if (batch.head)
  903. hydration_copy(batch.head, batch.nr_batched_regions);
  904. if (offset >= nr_regions)
  905. offset = 0;
  906. clone->hydration_offset = offset;
  907. if (atomic_dec_and_test(&clone->hydrations_in_flight))
  908. wakeup_hydration_waiters(clone);
  909. }
  910. /*---------------------------------------------------------------------------*/
  911. static bool need_commit_due_to_time(struct clone *clone)
  912. {
  913. return !time_in_range(jiffies, clone->last_commit_jiffies,
  914. clone->last_commit_jiffies + COMMIT_PERIOD);
  915. }
  916. /*
  917. * A non-zero return indicates read-only or fail mode.
  918. */
  919. static int commit_metadata(struct clone *clone, bool *dest_dev_flushed)
  920. {
  921. int r = 0;
  922. if (dest_dev_flushed)
  923. *dest_dev_flushed = false;
  924. mutex_lock(&clone->commit_lock);
  925. if (!dm_clone_changed_this_transaction(clone->cmd))
  926. goto out;
  927. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
  928. r = -EPERM;
  929. goto out;
  930. }
  931. r = dm_clone_metadata_pre_commit(clone->cmd);
  932. if (unlikely(r)) {
  933. __metadata_operation_failed(clone, "dm_clone_metadata_pre_commit", r);
  934. goto out;
  935. }
  936. bio_reset(&clone->flush_bio);
  937. bio_set_dev(&clone->flush_bio, clone->dest_dev->bdev);
  938. clone->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
  939. r = submit_bio_wait(&clone->flush_bio);
  940. if (unlikely(r)) {
  941. __metadata_operation_failed(clone, "flush destination device", r);
  942. goto out;
  943. }
  944. if (dest_dev_flushed)
  945. *dest_dev_flushed = true;
  946. r = dm_clone_metadata_commit(clone->cmd);
  947. if (unlikely(r)) {
  948. __metadata_operation_failed(clone, "dm_clone_metadata_commit", r);
  949. goto out;
  950. }
  951. if (dm_clone_is_hydration_done(clone->cmd))
  952. dm_table_event(clone->ti->table);
  953. out:
  954. mutex_unlock(&clone->commit_lock);
  955. return r;
  956. }
  957. static void process_deferred_discards(struct clone *clone)
  958. {
  959. int r = -EPERM;
  960. struct bio *bio;
  961. struct blk_plug plug;
  962. unsigned long rs, nr_regions;
  963. struct bio_list discards = BIO_EMPTY_LIST;
  964. spin_lock_irq(&clone->lock);
  965. bio_list_merge(&discards, &clone->deferred_discard_bios);
  966. bio_list_init(&clone->deferred_discard_bios);
  967. spin_unlock_irq(&clone->lock);
  968. if (bio_list_empty(&discards))
  969. return;
  970. if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
  971. goto out;
  972. /* Update the metadata */
  973. bio_list_for_each(bio, &discards) {
  974. bio_region_range(clone, bio, &rs, &nr_regions);
  975. /*
  976. * A discard request might cover regions that have been already
  977. * hydrated. There is no need to update the metadata for these
  978. * regions.
  979. */
  980. r = dm_clone_cond_set_range(clone->cmd, rs, nr_regions);
  981. if (unlikely(r))
  982. break;
  983. }
  984. out:
  985. blk_start_plug(&plug);
  986. while ((bio = bio_list_pop(&discards)))
  987. complete_discard_bio(clone, bio, r == 0);
  988. blk_finish_plug(&plug);
  989. }
  990. static void process_deferred_bios(struct clone *clone)
  991. {
  992. struct bio_list bios = BIO_EMPTY_LIST;
  993. spin_lock_irq(&clone->lock);
  994. bio_list_merge(&bios, &clone->deferred_bios);
  995. bio_list_init(&clone->deferred_bios);
  996. spin_unlock_irq(&clone->lock);
  997. if (bio_list_empty(&bios))
  998. return;
  999. submit_bios(&bios);
  1000. }
  1001. static void process_deferred_flush_bios(struct clone *clone)
  1002. {
  1003. struct bio *bio;
  1004. bool dest_dev_flushed;
  1005. struct bio_list bios = BIO_EMPTY_LIST;
  1006. struct bio_list bio_completions = BIO_EMPTY_LIST;
  1007. /*
  1008. * If there are any deferred flush bios, we must commit the metadata
  1009. * before issuing them or signaling their completion.
  1010. */
  1011. spin_lock_irq(&clone->lock);
  1012. bio_list_merge(&bios, &clone->deferred_flush_bios);
  1013. bio_list_init(&clone->deferred_flush_bios);
  1014. bio_list_merge(&bio_completions, &clone->deferred_flush_completions);
  1015. bio_list_init(&clone->deferred_flush_completions);
  1016. spin_unlock_irq(&clone->lock);
  1017. if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
  1018. !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone)))
  1019. return;
  1020. if (commit_metadata(clone, &dest_dev_flushed)) {
  1021. bio_list_merge(&bios, &bio_completions);
  1022. while ((bio = bio_list_pop(&bios)))
  1023. bio_io_error(bio);
  1024. return;
  1025. }
  1026. clone->last_commit_jiffies = jiffies;
  1027. while ((bio = bio_list_pop(&bio_completions)))
  1028. bio_endio(bio);
  1029. while ((bio = bio_list_pop(&bios))) {
  1030. if ((bio->bi_opf & REQ_PREFLUSH) && dest_dev_flushed) {
  1031. /* We just flushed the destination device as part of
  1032. * the metadata commit, so there is no reason to send
  1033. * another flush.
  1034. */
  1035. bio_endio(bio);
  1036. } else {
  1037. submit_bio_noacct(bio);
  1038. }
  1039. }
  1040. }
  1041. static void do_worker(struct work_struct *work)
  1042. {
  1043. struct clone *clone = container_of(work, typeof(*clone), worker);
  1044. process_deferred_bios(clone);
  1045. process_deferred_discards(clone);
  1046. /*
  1047. * process_deferred_flush_bios():
  1048. *
  1049. * - Commit metadata
  1050. *
  1051. * - Process deferred REQ_FUA completions
  1052. *
  1053. * - Process deferred REQ_PREFLUSH bios
  1054. */
  1055. process_deferred_flush_bios(clone);
  1056. /* Background hydration */
  1057. do_hydration(clone);
  1058. }
  1059. /*
  1060. * Commit periodically so that not too much unwritten data builds up.
  1061. *
  1062. * Also, restart background hydration, if it has been stopped by in-flight I/O.
  1063. */
  1064. static void do_waker(struct work_struct *work)
  1065. {
  1066. struct clone *clone = container_of(to_delayed_work(work), struct clone, waker);
  1067. wake_worker(clone);
  1068. queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD);
  1069. }
  1070. /*---------------------------------------------------------------------------*/
  1071. /*
  1072. * Target methods
  1073. */
  1074. static int clone_map(struct dm_target *ti, struct bio *bio)
  1075. {
  1076. struct clone *clone = ti->private;
  1077. unsigned long region_nr;
  1078. atomic_inc(&clone->ios_in_flight);
  1079. if (unlikely(get_clone_mode(clone) == CM_FAIL))
  1080. return DM_MAPIO_KILL;
  1081. /*
  1082. * REQ_PREFLUSH bios carry no data:
  1083. *
  1084. * - Commit metadata, if changed
  1085. *
  1086. * - Pass down to destination device
  1087. */
  1088. if (bio->bi_opf & REQ_PREFLUSH) {
  1089. remap_and_issue(clone, bio);
  1090. return DM_MAPIO_SUBMITTED;
  1091. }
  1092. bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
  1093. /*
  1094. * dm-clone interprets discards and performs a fast hydration of the
  1095. * discarded regions, i.e., we skip the copy from the source device and
  1096. * just mark the regions as hydrated.
  1097. */
  1098. if (bio_op(bio) == REQ_OP_DISCARD) {
  1099. process_discard_bio(clone, bio);
  1100. return DM_MAPIO_SUBMITTED;
  1101. }
  1102. /*
  1103. * If the bio's region is hydrated, redirect it to the destination
  1104. * device.
  1105. *
  1106. * If the region is not hydrated and the bio is a READ, redirect it to
  1107. * the source device.
  1108. *
  1109. * Else, defer WRITE bio until after its region has been hydrated and
  1110. * start the region's hydration immediately.
  1111. */
  1112. region_nr = bio_to_region(clone, bio);
  1113. if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
  1114. remap_and_issue(clone, bio);
  1115. return DM_MAPIO_SUBMITTED;
  1116. } else if (bio_data_dir(bio) == READ) {
  1117. remap_to_source(clone, bio);
  1118. return DM_MAPIO_REMAPPED;
  1119. }
  1120. remap_to_dest(clone, bio);
  1121. hydrate_bio_region(clone, bio);
  1122. return DM_MAPIO_SUBMITTED;
  1123. }
  1124. static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error)
  1125. {
  1126. struct clone *clone = ti->private;
  1127. atomic_dec(&clone->ios_in_flight);
  1128. return DM_ENDIO_DONE;
  1129. }
  1130. static void emit_flags(struct clone *clone, char *result, unsigned int maxlen,
  1131. ssize_t *sz_ptr)
  1132. {
  1133. ssize_t sz = *sz_ptr;
  1134. unsigned int count;
  1135. count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
  1136. count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
  1137. DMEMIT("%u ", count);
  1138. if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
  1139. DMEMIT("no_hydration ");
  1140. if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
  1141. DMEMIT("no_discard_passdown ");
  1142. *sz_ptr = sz;
  1143. }
  1144. static void emit_core_args(struct clone *clone, char *result,
  1145. unsigned int maxlen, ssize_t *sz_ptr)
  1146. {
  1147. ssize_t sz = *sz_ptr;
  1148. unsigned int count = 4;
  1149. DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count,
  1150. READ_ONCE(clone->hydration_threshold),
  1151. READ_ONCE(clone->hydration_batch_size));
  1152. *sz_ptr = sz;
  1153. }
  1154. /*
  1155. * Status format:
  1156. *
  1157. * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
  1158. * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions>
  1159. * <#features> <features>* <#core args> <core args>* <clone metadata mode>
  1160. */
  1161. static void clone_status(struct dm_target *ti, status_type_t type,
  1162. unsigned int status_flags, char *result,
  1163. unsigned int maxlen)
  1164. {
  1165. int r;
  1166. unsigned int i;
  1167. ssize_t sz = 0;
  1168. dm_block_t nr_free_metadata_blocks = 0;
  1169. dm_block_t nr_metadata_blocks = 0;
  1170. char buf[BDEVNAME_SIZE];
  1171. struct clone *clone = ti->private;
  1172. switch (type) {
  1173. case STATUSTYPE_INFO:
  1174. if (get_clone_mode(clone) == CM_FAIL) {
  1175. DMEMIT("Fail");
  1176. break;
  1177. }
  1178. /* Commit to ensure statistics aren't out-of-date */
  1179. if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
  1180. (void) commit_metadata(clone, NULL);
  1181. r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks);
  1182. if (r) {
  1183. DMERR("%s: dm_clone_get_free_metadata_block_count returned %d",
  1184. clone_device_name(clone), r);
  1185. goto error;
  1186. }
  1187. r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks);
  1188. if (r) {
  1189. DMERR("%s: dm_clone_get_metadata_dev_size returned %d",
  1190. clone_device_name(clone), r);
  1191. goto error;
  1192. }
  1193. DMEMIT("%u %llu/%llu %llu %u/%lu %u ",
  1194. DM_CLONE_METADATA_BLOCK_SIZE,
  1195. (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks),
  1196. (unsigned long long)nr_metadata_blocks,
  1197. (unsigned long long)clone->region_size,
  1198. dm_clone_nr_of_hydrated_regions(clone->cmd),
  1199. clone->nr_regions,
  1200. atomic_read(&clone->hydrations_in_flight));
  1201. emit_flags(clone, result, maxlen, &sz);
  1202. emit_core_args(clone, result, maxlen, &sz);
  1203. switch (get_clone_mode(clone)) {
  1204. case CM_WRITE:
  1205. DMEMIT("rw");
  1206. break;
  1207. case CM_READ_ONLY:
  1208. DMEMIT("ro");
  1209. break;
  1210. case CM_FAIL:
  1211. DMEMIT("Fail");
  1212. }
  1213. break;
  1214. case STATUSTYPE_TABLE:
  1215. format_dev_t(buf, clone->metadata_dev->bdev->bd_dev);
  1216. DMEMIT("%s ", buf);
  1217. format_dev_t(buf, clone->dest_dev->bdev->bd_dev);
  1218. DMEMIT("%s ", buf);
  1219. format_dev_t(buf, clone->source_dev->bdev->bd_dev);
  1220. DMEMIT("%s", buf);
  1221. for (i = 0; i < clone->nr_ctr_args; i++)
  1222. DMEMIT(" %s", clone->ctr_args[i]);
  1223. }
  1224. return;
  1225. error:
  1226. DMEMIT("Error");
  1227. }
  1228. static sector_t get_dev_size(struct dm_dev *dev)
  1229. {
  1230. return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
  1231. }
  1232. /*---------------------------------------------------------------------------*/
  1233. /*
  1234. * Construct a clone device mapping:
  1235. *
  1236. * clone <metadata dev> <destination dev> <source dev> <region size>
  1237. * [<#feature args> [<feature arg>]* [<#core args> [key value]*]]
  1238. *
  1239. * metadata dev: Fast device holding the persistent metadata
  1240. * destination dev: The destination device, which will become a clone of the
  1241. * source device
  1242. * source dev: The read-only source device that gets cloned
  1243. * region size: dm-clone unit size in sectors
  1244. *
  1245. * #feature args: Number of feature arguments passed
  1246. * feature args: E.g. no_hydration, no_discard_passdown
  1247. *
  1248. * #core arguments: An even number of core arguments
  1249. * core arguments: Key/value pairs for tuning the core
  1250. * E.g. 'hydration_threshold 256'
  1251. */
  1252. static int parse_feature_args(struct dm_arg_set *as, struct clone *clone)
  1253. {
  1254. int r;
  1255. unsigned int argc;
  1256. const char *arg_name;
  1257. struct dm_target *ti = clone->ti;
  1258. const struct dm_arg args = {
  1259. .min = 0,
  1260. .max = 2,
  1261. .error = "Invalid number of feature arguments"
  1262. };
  1263. /* No feature arguments supplied */
  1264. if (!as->argc)
  1265. return 0;
  1266. r = dm_read_arg_group(&args, as, &argc, &ti->error);
  1267. if (r)
  1268. return r;
  1269. while (argc) {
  1270. arg_name = dm_shift_arg(as);
  1271. argc--;
  1272. if (!strcasecmp(arg_name, "no_hydration")) {
  1273. __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
  1274. } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
  1275. __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
  1276. } else {
  1277. ti->error = "Invalid feature argument";
  1278. return -EINVAL;
  1279. }
  1280. }
  1281. return 0;
  1282. }
  1283. static int parse_core_args(struct dm_arg_set *as, struct clone *clone)
  1284. {
  1285. int r;
  1286. unsigned int argc;
  1287. unsigned int value;
  1288. const char *arg_name;
  1289. struct dm_target *ti = clone->ti;
  1290. const struct dm_arg args = {
  1291. .min = 0,
  1292. .max = 4,
  1293. .error = "Invalid number of core arguments"
  1294. };
  1295. /* Initialize core arguments */
  1296. clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE;
  1297. clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD;
  1298. /* No core arguments supplied */
  1299. if (!as->argc)
  1300. return 0;
  1301. r = dm_read_arg_group(&args, as, &argc, &ti->error);
  1302. if (r)
  1303. return r;
  1304. if (argc & 1) {
  1305. ti->error = "Number of core arguments must be even";
  1306. return -EINVAL;
  1307. }
  1308. while (argc) {
  1309. arg_name = dm_shift_arg(as);
  1310. argc -= 2;
  1311. if (!strcasecmp(arg_name, "hydration_threshold")) {
  1312. if (kstrtouint(dm_shift_arg(as), 10, &value)) {
  1313. ti->error = "Invalid value for argument `hydration_threshold'";
  1314. return -EINVAL;
  1315. }
  1316. clone->hydration_threshold = value;
  1317. } else if (!strcasecmp(arg_name, "hydration_batch_size")) {
  1318. if (kstrtouint(dm_shift_arg(as), 10, &value)) {
  1319. ti->error = "Invalid value for argument `hydration_batch_size'";
  1320. return -EINVAL;
  1321. }
  1322. clone->hydration_batch_size = value;
  1323. } else {
  1324. ti->error = "Invalid core argument";
  1325. return -EINVAL;
  1326. }
  1327. }
  1328. return 0;
  1329. }
  1330. static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error)
  1331. {
  1332. int r;
  1333. unsigned int region_size;
  1334. struct dm_arg arg;
  1335. arg.min = MIN_REGION_SIZE;
  1336. arg.max = MAX_REGION_SIZE;
  1337. arg.error = "Invalid region size";
  1338. r = dm_read_arg(&arg, as, &region_size, error);
  1339. if (r)
  1340. return r;
  1341. /* Check region size is a power of 2 */
  1342. if (!is_power_of_2(region_size)) {
  1343. *error = "Region size is not a power of 2";
  1344. return -EINVAL;
  1345. }
  1346. /* Validate the region size against the device logical block size */
  1347. if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) ||
  1348. region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) {
  1349. *error = "Region size is not a multiple of device logical block size";
  1350. return -EINVAL;
  1351. }
  1352. clone->region_size = region_size;
  1353. return 0;
  1354. }
  1355. static int validate_nr_regions(unsigned long n, char **error)
  1356. {
  1357. /*
  1358. * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us
  1359. * further to 2^31 regions.
  1360. */
  1361. if (n > (1UL << 31)) {
  1362. *error = "Too many regions. Consider increasing the region size";
  1363. return -EINVAL;
  1364. }
  1365. return 0;
  1366. }
  1367. static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error)
  1368. {
  1369. int r;
  1370. sector_t metadata_dev_size;
  1371. char b[BDEVNAME_SIZE];
  1372. r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
  1373. &clone->metadata_dev);
  1374. if (r) {
  1375. *error = "Error opening metadata device";
  1376. return r;
  1377. }
  1378. metadata_dev_size = get_dev_size(clone->metadata_dev);
  1379. if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING)
  1380. DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
  1381. bdevname(clone->metadata_dev->bdev, b), DM_CLONE_METADATA_MAX_SECTORS);
  1382. return 0;
  1383. }
  1384. static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error)
  1385. {
  1386. int r;
  1387. sector_t dest_dev_size;
  1388. r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
  1389. &clone->dest_dev);
  1390. if (r) {
  1391. *error = "Error opening destination device";
  1392. return r;
  1393. }
  1394. dest_dev_size = get_dev_size(clone->dest_dev);
  1395. if (dest_dev_size < clone->ti->len) {
  1396. dm_put_device(clone->ti, clone->dest_dev);
  1397. *error = "Device size larger than destination device";
  1398. return -EINVAL;
  1399. }
  1400. return 0;
  1401. }
  1402. static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error)
  1403. {
  1404. int r;
  1405. sector_t source_dev_size;
  1406. r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ,
  1407. &clone->source_dev);
  1408. if (r) {
  1409. *error = "Error opening source device";
  1410. return r;
  1411. }
  1412. source_dev_size = get_dev_size(clone->source_dev);
  1413. if (source_dev_size < clone->ti->len) {
  1414. dm_put_device(clone->ti, clone->source_dev);
  1415. *error = "Device size larger than source device";
  1416. return -EINVAL;
  1417. }
  1418. return 0;
  1419. }
  1420. static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error)
  1421. {
  1422. unsigned int i;
  1423. const char **copy;
  1424. copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
  1425. if (!copy)
  1426. goto error;
  1427. for (i = 0; i < argc; i++) {
  1428. copy[i] = kstrdup(argv[i], GFP_KERNEL);
  1429. if (!copy[i]) {
  1430. while (i--)
  1431. kfree(copy[i]);
  1432. kfree(copy);
  1433. goto error;
  1434. }
  1435. }
  1436. clone->nr_ctr_args = argc;
  1437. clone->ctr_args = copy;
  1438. return 0;
  1439. error:
  1440. *error = "Failed to allocate memory for table line";
  1441. return -ENOMEM;
  1442. }
  1443. static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv)
  1444. {
  1445. int r;
  1446. sector_t nr_regions;
  1447. struct clone *clone;
  1448. struct dm_arg_set as;
  1449. if (argc < 4) {
  1450. ti->error = "Invalid number of arguments";
  1451. return -EINVAL;
  1452. }
  1453. as.argc = argc;
  1454. as.argv = argv;
  1455. clone = kzalloc(sizeof(*clone), GFP_KERNEL);
  1456. if (!clone) {
  1457. ti->error = "Failed to allocate clone structure";
  1458. return -ENOMEM;
  1459. }
  1460. clone->ti = ti;
  1461. /* Initialize dm-clone flags */
  1462. __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
  1463. __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
  1464. __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
  1465. r = parse_metadata_dev(clone, &as, &ti->error);
  1466. if (r)
  1467. goto out_with_clone;
  1468. r = parse_dest_dev(clone, &as, &ti->error);
  1469. if (r)
  1470. goto out_with_meta_dev;
  1471. r = parse_source_dev(clone, &as, &ti->error);
  1472. if (r)
  1473. goto out_with_dest_dev;
  1474. r = parse_region_size(clone, &as, &ti->error);
  1475. if (r)
  1476. goto out_with_source_dev;
  1477. clone->region_shift = __ffs(clone->region_size);
  1478. nr_regions = dm_sector_div_up(ti->len, clone->region_size);
  1479. /* Check for overflow */
  1480. if (nr_regions != (unsigned long)nr_regions) {
  1481. ti->error = "Too many regions. Consider increasing the region size";
  1482. r = -EOVERFLOW;
  1483. goto out_with_source_dev;
  1484. }
  1485. clone->nr_regions = nr_regions;
  1486. r = validate_nr_regions(clone->nr_regions, &ti->error);
  1487. if (r)
  1488. goto out_with_source_dev;
  1489. r = dm_set_target_max_io_len(ti, clone->region_size);
  1490. if (r) {
  1491. ti->error = "Failed to set max io len";
  1492. goto out_with_source_dev;
  1493. }
  1494. r = parse_feature_args(&as, clone);
  1495. if (r)
  1496. goto out_with_source_dev;
  1497. r = parse_core_args(&as, clone);
  1498. if (r)
  1499. goto out_with_source_dev;
  1500. /* Load metadata */
  1501. clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len,
  1502. clone->region_size);
  1503. if (IS_ERR(clone->cmd)) {
  1504. ti->error = "Failed to load metadata";
  1505. r = PTR_ERR(clone->cmd);
  1506. goto out_with_source_dev;
  1507. }
  1508. __set_clone_mode(clone, CM_WRITE);
  1509. if (get_clone_mode(clone) != CM_WRITE) {
  1510. ti->error = "Unable to get write access to metadata, please check/repair metadata";
  1511. r = -EPERM;
  1512. goto out_with_metadata;
  1513. }
  1514. clone->last_commit_jiffies = jiffies;
  1515. /* Allocate hydration hash table */
  1516. r = hash_table_init(clone);
  1517. if (r) {
  1518. ti->error = "Failed to allocate hydration hash table";
  1519. goto out_with_metadata;
  1520. }
  1521. atomic_set(&clone->ios_in_flight, 0);
  1522. init_waitqueue_head(&clone->hydration_stopped);
  1523. spin_lock_init(&clone->lock);
  1524. bio_list_init(&clone->deferred_bios);
  1525. bio_list_init(&clone->deferred_discard_bios);
  1526. bio_list_init(&clone->deferred_flush_bios);
  1527. bio_list_init(&clone->deferred_flush_completions);
  1528. clone->hydration_offset = 0;
  1529. atomic_set(&clone->hydrations_in_flight, 0);
  1530. bio_init(&clone->flush_bio, NULL, 0);
  1531. clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
  1532. if (!clone->wq) {
  1533. ti->error = "Failed to allocate workqueue";
  1534. r = -ENOMEM;
  1535. goto out_with_ht;
  1536. }
  1537. INIT_WORK(&clone->worker, do_worker);
  1538. INIT_DELAYED_WORK(&clone->waker, do_waker);
  1539. clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
  1540. if (IS_ERR(clone->kcopyd_client)) {
  1541. r = PTR_ERR(clone->kcopyd_client);
  1542. goto out_with_wq;
  1543. }
  1544. r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS,
  1545. _hydration_cache);
  1546. if (r) {
  1547. ti->error = "Failed to create dm_clone_region_hydration memory pool";
  1548. goto out_with_kcopyd;
  1549. }
  1550. /* Save a copy of the table line */
  1551. r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error);
  1552. if (r)
  1553. goto out_with_mempool;
  1554. mutex_init(&clone->commit_lock);
  1555. /* Enable flushes */
  1556. ti->num_flush_bios = 1;
  1557. ti->flush_supported = true;
  1558. /* Enable discards */
  1559. ti->discards_supported = true;
  1560. ti->num_discard_bios = 1;
  1561. ti->private = clone;
  1562. return 0;
  1563. out_with_mempool:
  1564. mempool_exit(&clone->hydration_pool);
  1565. out_with_kcopyd:
  1566. dm_kcopyd_client_destroy(clone->kcopyd_client);
  1567. out_with_wq:
  1568. destroy_workqueue(clone->wq);
  1569. out_with_ht:
  1570. hash_table_exit(clone);
  1571. out_with_metadata:
  1572. dm_clone_metadata_close(clone->cmd);
  1573. out_with_source_dev:
  1574. dm_put_device(ti, clone->source_dev);
  1575. out_with_dest_dev:
  1576. dm_put_device(ti, clone->dest_dev);
  1577. out_with_meta_dev:
  1578. dm_put_device(ti, clone->metadata_dev);
  1579. out_with_clone:
  1580. kfree(clone);
  1581. return r;
  1582. }
  1583. static void clone_dtr(struct dm_target *ti)
  1584. {
  1585. unsigned int i;
  1586. struct clone *clone = ti->private;
  1587. mutex_destroy(&clone->commit_lock);
  1588. bio_uninit(&clone->flush_bio);
  1589. for (i = 0; i < clone->nr_ctr_args; i++)
  1590. kfree(clone->ctr_args[i]);
  1591. kfree(clone->ctr_args);
  1592. mempool_exit(&clone->hydration_pool);
  1593. dm_kcopyd_client_destroy(clone->kcopyd_client);
  1594. destroy_workqueue(clone->wq);
  1595. hash_table_exit(clone);
  1596. dm_clone_metadata_close(clone->cmd);
  1597. dm_put_device(ti, clone->source_dev);
  1598. dm_put_device(ti, clone->dest_dev);
  1599. dm_put_device(ti, clone->metadata_dev);
  1600. kfree(clone);
  1601. }
  1602. /*---------------------------------------------------------------------------*/
  1603. static void clone_postsuspend(struct dm_target *ti)
  1604. {
  1605. struct clone *clone = ti->private;
  1606. /*
  1607. * To successfully suspend the device:
  1608. *
  1609. * - We cancel the delayed work for periodic commits and wait for
  1610. * it to finish.
  1611. *
  1612. * - We stop the background hydration, i.e. we prevent new region
  1613. * hydrations from starting.
  1614. *
  1615. * - We wait for any in-flight hydrations to finish.
  1616. *
  1617. * - We flush the workqueue.
  1618. *
  1619. * - We commit the metadata.
  1620. */
  1621. cancel_delayed_work_sync(&clone->waker);
  1622. set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
  1623. /*
  1624. * Make sure set_bit() is ordered before atomic_read(), otherwise we
  1625. * might race with do_hydration() and miss some started region
  1626. * hydrations.
  1627. *
  1628. * This is paired with smp_mb__after_atomic() in do_hydration().
  1629. */
  1630. smp_mb__after_atomic();
  1631. wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight));
  1632. flush_workqueue(clone->wq);
  1633. (void) commit_metadata(clone, NULL);
  1634. }
  1635. static void clone_resume(struct dm_target *ti)
  1636. {
  1637. struct clone *clone = ti->private;
  1638. clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
  1639. do_waker(&clone->waker.work);
  1640. }
  1641. static bool bdev_supports_discards(struct block_device *bdev)
  1642. {
  1643. struct request_queue *q = bdev_get_queue(bdev);
  1644. return (q && blk_queue_discard(q));
  1645. }
  1646. /*
  1647. * If discard_passdown was enabled verify that the destination device supports
  1648. * discards. Disable discard_passdown if not.
  1649. */
  1650. static void disable_passdown_if_not_supported(struct clone *clone)
  1651. {
  1652. struct block_device *dest_dev = clone->dest_dev->bdev;
  1653. struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits;
  1654. const char *reason = NULL;
  1655. char buf[BDEVNAME_SIZE];
  1656. if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
  1657. return;
  1658. if (!bdev_supports_discards(dest_dev))
  1659. reason = "discard unsupported";
  1660. else if (dest_limits->max_discard_sectors < clone->region_size)
  1661. reason = "max discard sectors smaller than a region";
  1662. if (reason) {
  1663. DMWARN("Destination device (%s) %s: Disabling discard passdown.",
  1664. bdevname(dest_dev, buf), reason);
  1665. clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
  1666. }
  1667. }
  1668. static void set_discard_limits(struct clone *clone, struct queue_limits *limits)
  1669. {
  1670. struct block_device *dest_bdev = clone->dest_dev->bdev;
  1671. struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits;
  1672. if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) {
  1673. /* No passdown is done so we set our own virtual limits */
  1674. limits->discard_granularity = clone->region_size << SECTOR_SHIFT;
  1675. limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size);
  1676. return;
  1677. }
  1678. /*
  1679. * clone_iterate_devices() is stacking both the source and destination
  1680. * device limits but discards aren't passed to the source device, so
  1681. * inherit destination's limits.
  1682. */
  1683. limits->max_discard_sectors = dest_limits->max_discard_sectors;
  1684. limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors;
  1685. limits->discard_granularity = dest_limits->discard_granularity;
  1686. limits->discard_alignment = dest_limits->discard_alignment;
  1687. limits->discard_misaligned = dest_limits->discard_misaligned;
  1688. limits->max_discard_segments = dest_limits->max_discard_segments;
  1689. }
  1690. static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1691. {
  1692. struct clone *clone = ti->private;
  1693. u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
  1694. /*
  1695. * If the system-determined stacked limits are compatible with
  1696. * dm-clone's region size (io_opt is a factor) do not override them.
  1697. */
  1698. if (io_opt_sectors < clone->region_size ||
  1699. do_div(io_opt_sectors, clone->region_size)) {
  1700. blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT);
  1701. blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT);
  1702. }
  1703. disable_passdown_if_not_supported(clone);
  1704. set_discard_limits(clone, limits);
  1705. }
  1706. static int clone_iterate_devices(struct dm_target *ti,
  1707. iterate_devices_callout_fn fn, void *data)
  1708. {
  1709. int ret;
  1710. struct clone *clone = ti->private;
  1711. struct dm_dev *dest_dev = clone->dest_dev;
  1712. struct dm_dev *source_dev = clone->source_dev;
  1713. ret = fn(ti, source_dev, 0, ti->len, data);
  1714. if (!ret)
  1715. ret = fn(ti, dest_dev, 0, ti->len, data);
  1716. return ret;
  1717. }
  1718. /*
  1719. * dm-clone message functions.
  1720. */
  1721. static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions)
  1722. {
  1723. WRITE_ONCE(clone->hydration_threshold, nr_regions);
  1724. /*
  1725. * If user space sets hydration_threshold to zero then the hydration
  1726. * will stop. If at a later time the hydration_threshold is increased
  1727. * we must restart the hydration process by waking up the worker.
  1728. */
  1729. wake_worker(clone);
  1730. }
  1731. static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions)
  1732. {
  1733. WRITE_ONCE(clone->hydration_batch_size, nr_regions);
  1734. }
  1735. static void enable_hydration(struct clone *clone)
  1736. {
  1737. if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
  1738. wake_worker(clone);
  1739. }
  1740. static void disable_hydration(struct clone *clone)
  1741. {
  1742. clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
  1743. }
  1744. static int clone_message(struct dm_target *ti, unsigned int argc, char **argv,
  1745. char *result, unsigned int maxlen)
  1746. {
  1747. struct clone *clone = ti->private;
  1748. unsigned int value;
  1749. if (!argc)
  1750. return -EINVAL;
  1751. if (!strcasecmp(argv[0], "enable_hydration")) {
  1752. enable_hydration(clone);
  1753. return 0;
  1754. }
  1755. if (!strcasecmp(argv[0], "disable_hydration")) {
  1756. disable_hydration(clone);
  1757. return 0;
  1758. }
  1759. if (argc != 2)
  1760. return -EINVAL;
  1761. if (!strcasecmp(argv[0], "hydration_threshold")) {
  1762. if (kstrtouint(argv[1], 10, &value))
  1763. return -EINVAL;
  1764. set_hydration_threshold(clone, value);
  1765. return 0;
  1766. }
  1767. if (!strcasecmp(argv[0], "hydration_batch_size")) {
  1768. if (kstrtouint(argv[1], 10, &value))
  1769. return -EINVAL;
  1770. set_hydration_batch_size(clone, value);
  1771. return 0;
  1772. }
  1773. DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]);
  1774. return -EINVAL;
  1775. }
  1776. static struct target_type clone_target = {
  1777. .name = "clone",
  1778. .version = {1, 0, 0},
  1779. .module = THIS_MODULE,
  1780. .ctr = clone_ctr,
  1781. .dtr = clone_dtr,
  1782. .map = clone_map,
  1783. .end_io = clone_endio,
  1784. .postsuspend = clone_postsuspend,
  1785. .resume = clone_resume,
  1786. .status = clone_status,
  1787. .message = clone_message,
  1788. .io_hints = clone_io_hints,
  1789. .iterate_devices = clone_iterate_devices,
  1790. };
  1791. /*---------------------------------------------------------------------------*/
  1792. /* Module functions */
  1793. static int __init dm_clone_init(void)
  1794. {
  1795. int r;
  1796. _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0);
  1797. if (!_hydration_cache)
  1798. return -ENOMEM;
  1799. r = dm_register_target(&clone_target);
  1800. if (r < 0) {
  1801. DMERR("Failed to register clone target");
  1802. return r;
  1803. }
  1804. return 0;
  1805. }
  1806. static void __exit dm_clone_exit(void)
  1807. {
  1808. dm_unregister_target(&clone_target);
  1809. kmem_cache_destroy(_hydration_cache);
  1810. _hydration_cache = NULL;
  1811. }
  1812. /* Module hooks */
  1813. module_init(dm_clone_init);
  1814. module_exit(dm_clone_exit);
  1815. MODULE_DESCRIPTION(DM_NAME " clone target");
  1816. MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>");
  1817. MODULE_LICENSE("GPL");