dm-writecache.c 65 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Copyright (C) 2018 Red Hat. All rights reserved.
  4. *
  5. * This file is released under the GPL.
  6. */
  7. #include <linux/device-mapper.h>
  8. #include <linux/module.h>
  9. #include <linux/init.h>
  10. #include <linux/vmalloc.h>
  11. #include <linux/kthread.h>
  12. #include <linux/dm-io.h>
  13. #include <linux/dm-kcopyd.h>
  14. #include <linux/dax.h>
  15. #include <linux/pfn_t.h>
  16. #include <linux/libnvdimm.h>
  17. #define DM_MSG_PREFIX "writecache"
  18. #define HIGH_WATERMARK 50
  19. #define LOW_WATERMARK 45
  20. #define MAX_WRITEBACK_JOBS 0
  21. #define ENDIO_LATENCY 16
  22. #define WRITEBACK_LATENCY 64
  23. #define AUTOCOMMIT_BLOCKS_SSD 65536
  24. #define AUTOCOMMIT_BLOCKS_PMEM 64
  25. #define AUTOCOMMIT_MSEC 1000
  26. #define MAX_AGE_DIV 16
  27. #define MAX_AGE_UNSPECIFIED -1UL
  28. #define BITMAP_GRANULARITY 65536
  29. #if BITMAP_GRANULARITY < PAGE_SIZE
  30. #undef BITMAP_GRANULARITY
  31. #define BITMAP_GRANULARITY PAGE_SIZE
  32. #endif
  33. #if IS_ENABLED(CONFIG_ARCH_HAS_PMEM_API) && IS_ENABLED(CONFIG_DAX_DRIVER)
  34. #define DM_WRITECACHE_HAS_PMEM
  35. #endif
  36. #ifdef DM_WRITECACHE_HAS_PMEM
  37. #define pmem_assign(dest, src) \
  38. do { \
  39. typeof(dest) uniq = (src); \
  40. memcpy_flushcache(&(dest), &uniq, sizeof(dest)); \
  41. } while (0)
  42. #else
  43. #define pmem_assign(dest, src) ((dest) = (src))
  44. #endif
  45. #if IS_ENABLED(CONFIG_ARCH_HAS_COPY_MC) && defined(DM_WRITECACHE_HAS_PMEM)
  46. #define DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  47. #endif
  48. #define MEMORY_SUPERBLOCK_MAGIC 0x23489321
  49. #define MEMORY_SUPERBLOCK_VERSION 1
  50. struct wc_memory_entry {
  51. __le64 original_sector;
  52. __le64 seq_count;
  53. };
  54. struct wc_memory_superblock {
  55. union {
  56. struct {
  57. __le32 magic;
  58. __le32 version;
  59. __le32 block_size;
  60. __le32 pad;
  61. __le64 n_blocks;
  62. __le64 seq_count;
  63. };
  64. __le64 padding[8];
  65. };
  66. struct wc_memory_entry entries[0];
  67. };
  68. struct wc_entry {
  69. struct rb_node rb_node;
  70. struct list_head lru;
  71. unsigned short wc_list_contiguous;
  72. bool write_in_progress
  73. #if BITS_PER_LONG == 64
  74. :1
  75. #endif
  76. ;
  77. unsigned long index
  78. #if BITS_PER_LONG == 64
  79. :47
  80. #endif
  81. ;
  82. unsigned long age;
  83. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  84. uint64_t original_sector;
  85. uint64_t seq_count;
  86. #endif
  87. };
  88. #ifdef DM_WRITECACHE_HAS_PMEM
  89. #define WC_MODE_PMEM(wc) ((wc)->pmem_mode)
  90. #define WC_MODE_FUA(wc) ((wc)->writeback_fua)
  91. #else
  92. #define WC_MODE_PMEM(wc) false
  93. #define WC_MODE_FUA(wc) false
  94. #endif
  95. #define WC_MODE_SORT_FREELIST(wc) (!WC_MODE_PMEM(wc))
  96. struct dm_writecache {
  97. struct mutex lock;
  98. struct list_head lru;
  99. union {
  100. struct list_head freelist;
  101. struct {
  102. struct rb_root freetree;
  103. struct wc_entry *current_free;
  104. };
  105. };
  106. struct rb_root tree;
  107. size_t freelist_size;
  108. size_t writeback_size;
  109. size_t freelist_high_watermark;
  110. size_t freelist_low_watermark;
  111. unsigned long max_age;
  112. unsigned uncommitted_blocks;
  113. unsigned autocommit_blocks;
  114. unsigned max_writeback_jobs;
  115. int error;
  116. unsigned long autocommit_jiffies;
  117. struct timer_list autocommit_timer;
  118. struct wait_queue_head freelist_wait;
  119. struct timer_list max_age_timer;
  120. atomic_t bio_in_progress[2];
  121. struct wait_queue_head bio_in_progress_wait[2];
  122. struct dm_target *ti;
  123. struct dm_dev *dev;
  124. struct dm_dev *ssd_dev;
  125. sector_t start_sector;
  126. void *memory_map;
  127. uint64_t memory_map_size;
  128. size_t metadata_sectors;
  129. size_t n_blocks;
  130. uint64_t seq_count;
  131. sector_t data_device_sectors;
  132. void *block_start;
  133. struct wc_entry *entries;
  134. unsigned block_size;
  135. unsigned char block_size_bits;
  136. bool pmem_mode:1;
  137. bool writeback_fua:1;
  138. bool overwrote_committed:1;
  139. bool memory_vmapped:1;
  140. bool start_sector_set:1;
  141. bool high_wm_percent_set:1;
  142. bool low_wm_percent_set:1;
  143. bool max_writeback_jobs_set:1;
  144. bool autocommit_blocks_set:1;
  145. bool autocommit_time_set:1;
  146. bool max_age_set:1;
  147. bool writeback_fua_set:1;
  148. bool flush_on_suspend:1;
  149. bool cleaner:1;
  150. bool cleaner_set:1;
  151. unsigned high_wm_percent_value;
  152. unsigned low_wm_percent_value;
  153. unsigned autocommit_time_value;
  154. unsigned max_age_value;
  155. unsigned writeback_all;
  156. struct workqueue_struct *writeback_wq;
  157. struct work_struct writeback_work;
  158. struct work_struct flush_work;
  159. struct dm_io_client *dm_io;
  160. raw_spinlock_t endio_list_lock;
  161. struct list_head endio_list;
  162. struct task_struct *endio_thread;
  163. struct task_struct *flush_thread;
  164. struct bio_list flush_list;
  165. struct dm_kcopyd_client *dm_kcopyd;
  166. unsigned long *dirty_bitmap;
  167. unsigned dirty_bitmap_size;
  168. struct bio_set bio_set;
  169. mempool_t copy_pool;
  170. };
  171. #define WB_LIST_INLINE 16
  172. struct writeback_struct {
  173. struct list_head endio_entry;
  174. struct dm_writecache *wc;
  175. struct wc_entry **wc_list;
  176. unsigned wc_list_n;
  177. struct wc_entry *wc_list_inline[WB_LIST_INLINE];
  178. struct bio bio;
  179. };
  180. struct copy_struct {
  181. struct list_head endio_entry;
  182. struct dm_writecache *wc;
  183. struct wc_entry *e;
  184. unsigned n_entries;
  185. int error;
  186. };
  187. DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(dm_writecache_throttle,
  188. "A percentage of time allocated for data copying");
  189. static void wc_lock(struct dm_writecache *wc)
  190. {
  191. mutex_lock(&wc->lock);
  192. }
  193. static void wc_unlock(struct dm_writecache *wc)
  194. {
  195. mutex_unlock(&wc->lock);
  196. }
  197. #ifdef DM_WRITECACHE_HAS_PMEM
  198. static int persistent_memory_claim(struct dm_writecache *wc)
  199. {
  200. int r;
  201. loff_t s;
  202. long p, da;
  203. pfn_t pfn;
  204. int id;
  205. struct page **pages;
  206. sector_t offset;
  207. wc->memory_vmapped = false;
  208. s = wc->memory_map_size;
  209. p = s >> PAGE_SHIFT;
  210. if (!p) {
  211. r = -EINVAL;
  212. goto err1;
  213. }
  214. if (p != s >> PAGE_SHIFT) {
  215. r = -EOVERFLOW;
  216. goto err1;
  217. }
  218. offset = get_start_sect(wc->ssd_dev->bdev);
  219. if (offset & (PAGE_SIZE / 512 - 1)) {
  220. r = -EINVAL;
  221. goto err1;
  222. }
  223. offset >>= PAGE_SHIFT - 9;
  224. id = dax_read_lock();
  225. da = dax_direct_access(wc->ssd_dev->dax_dev, offset, p, &wc->memory_map, &pfn);
  226. if (da < 0) {
  227. wc->memory_map = NULL;
  228. r = da;
  229. goto err2;
  230. }
  231. if (!pfn_t_has_page(pfn)) {
  232. wc->memory_map = NULL;
  233. r = -EOPNOTSUPP;
  234. goto err2;
  235. }
  236. if (da != p) {
  237. long i;
  238. wc->memory_map = NULL;
  239. pages = kvmalloc_array(p, sizeof(struct page *), GFP_KERNEL);
  240. if (!pages) {
  241. r = -ENOMEM;
  242. goto err2;
  243. }
  244. i = 0;
  245. do {
  246. long daa;
  247. daa = dax_direct_access(wc->ssd_dev->dax_dev, offset + i, p - i,
  248. NULL, &pfn);
  249. if (daa <= 0) {
  250. r = daa ? daa : -EINVAL;
  251. goto err3;
  252. }
  253. if (!pfn_t_has_page(pfn)) {
  254. r = -EOPNOTSUPP;
  255. goto err3;
  256. }
  257. while (daa-- && i < p) {
  258. pages[i++] = pfn_t_to_page(pfn);
  259. pfn.val++;
  260. if (!(i & 15))
  261. cond_resched();
  262. }
  263. } while (i < p);
  264. wc->memory_map = vmap(pages, p, VM_MAP, PAGE_KERNEL);
  265. if (!wc->memory_map) {
  266. r = -ENOMEM;
  267. goto err3;
  268. }
  269. kvfree(pages);
  270. wc->memory_vmapped = true;
  271. }
  272. dax_read_unlock(id);
  273. wc->memory_map += (size_t)wc->start_sector << SECTOR_SHIFT;
  274. wc->memory_map_size -= (size_t)wc->start_sector << SECTOR_SHIFT;
  275. return 0;
  276. err3:
  277. kvfree(pages);
  278. err2:
  279. dax_read_unlock(id);
  280. err1:
  281. return r;
  282. }
  283. #else
  284. static int persistent_memory_claim(struct dm_writecache *wc)
  285. {
  286. return -EOPNOTSUPP;
  287. }
  288. #endif
  289. static void persistent_memory_release(struct dm_writecache *wc)
  290. {
  291. if (wc->memory_vmapped)
  292. vunmap(wc->memory_map - ((size_t)wc->start_sector << SECTOR_SHIFT));
  293. }
  294. static struct page *persistent_memory_page(void *addr)
  295. {
  296. if (is_vmalloc_addr(addr))
  297. return vmalloc_to_page(addr);
  298. else
  299. return virt_to_page(addr);
  300. }
  301. static unsigned persistent_memory_page_offset(void *addr)
  302. {
  303. return (unsigned long)addr & (PAGE_SIZE - 1);
  304. }
  305. static void persistent_memory_flush_cache(void *ptr, size_t size)
  306. {
  307. if (is_vmalloc_addr(ptr))
  308. flush_kernel_vmap_range(ptr, size);
  309. }
  310. static void persistent_memory_invalidate_cache(void *ptr, size_t size)
  311. {
  312. if (is_vmalloc_addr(ptr))
  313. invalidate_kernel_vmap_range(ptr, size);
  314. }
  315. static struct wc_memory_superblock *sb(struct dm_writecache *wc)
  316. {
  317. return wc->memory_map;
  318. }
  319. static struct wc_memory_entry *memory_entry(struct dm_writecache *wc, struct wc_entry *e)
  320. {
  321. return &sb(wc)->entries[e->index];
  322. }
  323. static void *memory_data(struct dm_writecache *wc, struct wc_entry *e)
  324. {
  325. return (char *)wc->block_start + (e->index << wc->block_size_bits);
  326. }
  327. static sector_t cache_sector(struct dm_writecache *wc, struct wc_entry *e)
  328. {
  329. return wc->start_sector + wc->metadata_sectors +
  330. ((sector_t)e->index << (wc->block_size_bits - SECTOR_SHIFT));
  331. }
  332. static uint64_t read_original_sector(struct dm_writecache *wc, struct wc_entry *e)
  333. {
  334. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  335. return e->original_sector;
  336. #else
  337. return le64_to_cpu(memory_entry(wc, e)->original_sector);
  338. #endif
  339. }
  340. static uint64_t read_seq_count(struct dm_writecache *wc, struct wc_entry *e)
  341. {
  342. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  343. return e->seq_count;
  344. #else
  345. return le64_to_cpu(memory_entry(wc, e)->seq_count);
  346. #endif
  347. }
  348. static void clear_seq_count(struct dm_writecache *wc, struct wc_entry *e)
  349. {
  350. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  351. e->seq_count = -1;
  352. #endif
  353. pmem_assign(memory_entry(wc, e)->seq_count, cpu_to_le64(-1));
  354. }
  355. static void write_original_sector_seq_count(struct dm_writecache *wc, struct wc_entry *e,
  356. uint64_t original_sector, uint64_t seq_count)
  357. {
  358. struct wc_memory_entry me;
  359. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  360. e->original_sector = original_sector;
  361. e->seq_count = seq_count;
  362. #endif
  363. me.original_sector = cpu_to_le64(original_sector);
  364. me.seq_count = cpu_to_le64(seq_count);
  365. pmem_assign(*memory_entry(wc, e), me);
  366. }
  367. #define writecache_error(wc, err, msg, arg...) \
  368. do { \
  369. if (!cmpxchg(&(wc)->error, 0, err)) \
  370. DMERR(msg, ##arg); \
  371. wake_up(&(wc)->freelist_wait); \
  372. } while (0)
  373. #define writecache_has_error(wc) (unlikely(READ_ONCE((wc)->error)))
  374. static void writecache_flush_all_metadata(struct dm_writecache *wc)
  375. {
  376. if (!WC_MODE_PMEM(wc))
  377. memset(wc->dirty_bitmap, -1, wc->dirty_bitmap_size);
  378. }
  379. static void writecache_flush_region(struct dm_writecache *wc, void *ptr, size_t size)
  380. {
  381. if (!WC_MODE_PMEM(wc))
  382. __set_bit(((char *)ptr - (char *)wc->memory_map) / BITMAP_GRANULARITY,
  383. wc->dirty_bitmap);
  384. }
  385. static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev);
  386. struct io_notify {
  387. struct dm_writecache *wc;
  388. struct completion c;
  389. atomic_t count;
  390. };
  391. static void writecache_notify_io(unsigned long error, void *context)
  392. {
  393. struct io_notify *endio = context;
  394. if (unlikely(error != 0))
  395. writecache_error(endio->wc, -EIO, "error writing metadata");
  396. BUG_ON(atomic_read(&endio->count) <= 0);
  397. if (atomic_dec_and_test(&endio->count))
  398. complete(&endio->c);
  399. }
  400. static void writecache_wait_for_ios(struct dm_writecache *wc, int direction)
  401. {
  402. wait_event(wc->bio_in_progress_wait[direction],
  403. !atomic_read(&wc->bio_in_progress[direction]));
  404. }
  405. static void ssd_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
  406. {
  407. struct dm_io_region region;
  408. struct dm_io_request req;
  409. struct io_notify endio = {
  410. wc,
  411. COMPLETION_INITIALIZER_ONSTACK(endio.c),
  412. ATOMIC_INIT(1),
  413. };
  414. unsigned bitmap_bits = wc->dirty_bitmap_size * 8;
  415. unsigned i = 0;
  416. while (1) {
  417. unsigned j;
  418. i = find_next_bit(wc->dirty_bitmap, bitmap_bits, i);
  419. if (unlikely(i == bitmap_bits))
  420. break;
  421. j = find_next_zero_bit(wc->dirty_bitmap, bitmap_bits, i);
  422. region.bdev = wc->ssd_dev->bdev;
  423. region.sector = (sector_t)i * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
  424. region.count = (sector_t)(j - i) * (BITMAP_GRANULARITY >> SECTOR_SHIFT);
  425. if (unlikely(region.sector >= wc->metadata_sectors))
  426. break;
  427. if (unlikely(region.sector + region.count > wc->metadata_sectors))
  428. region.count = wc->metadata_sectors - region.sector;
  429. region.sector += wc->start_sector;
  430. atomic_inc(&endio.count);
  431. req.bi_op = REQ_OP_WRITE;
  432. req.bi_op_flags = REQ_SYNC;
  433. req.mem.type = DM_IO_VMA;
  434. req.mem.ptr.vma = (char *)wc->memory_map + (size_t)i * BITMAP_GRANULARITY;
  435. req.client = wc->dm_io;
  436. req.notify.fn = writecache_notify_io;
  437. req.notify.context = &endio;
  438. /* writing via async dm-io (implied by notify.fn above) won't return an error */
  439. (void) dm_io(&req, 1, &region, NULL);
  440. i = j;
  441. }
  442. writecache_notify_io(0, &endio);
  443. wait_for_completion_io(&endio.c);
  444. if (wait_for_ios)
  445. writecache_wait_for_ios(wc, WRITE);
  446. writecache_disk_flush(wc, wc->ssd_dev);
  447. memset(wc->dirty_bitmap, 0, wc->dirty_bitmap_size);
  448. }
  449. static void ssd_commit_superblock(struct dm_writecache *wc)
  450. {
  451. int r;
  452. struct dm_io_region region;
  453. struct dm_io_request req;
  454. region.bdev = wc->ssd_dev->bdev;
  455. region.sector = 0;
  456. region.count = max(4096U, wc->block_size) >> SECTOR_SHIFT;
  457. if (unlikely(region.sector + region.count > wc->metadata_sectors))
  458. region.count = wc->metadata_sectors - region.sector;
  459. region.sector += wc->start_sector;
  460. req.bi_op = REQ_OP_WRITE;
  461. req.bi_op_flags = REQ_SYNC | REQ_FUA;
  462. req.mem.type = DM_IO_VMA;
  463. req.mem.ptr.vma = (char *)wc->memory_map;
  464. req.client = wc->dm_io;
  465. req.notify.fn = NULL;
  466. req.notify.context = NULL;
  467. r = dm_io(&req, 1, &region, NULL);
  468. if (unlikely(r))
  469. writecache_error(wc, r, "error writing superblock");
  470. }
  471. static void writecache_commit_flushed(struct dm_writecache *wc, bool wait_for_ios)
  472. {
  473. if (WC_MODE_PMEM(wc))
  474. pmem_wmb();
  475. else
  476. ssd_commit_flushed(wc, wait_for_ios);
  477. }
  478. static void writecache_disk_flush(struct dm_writecache *wc, struct dm_dev *dev)
  479. {
  480. int r;
  481. struct dm_io_region region;
  482. struct dm_io_request req;
  483. region.bdev = dev->bdev;
  484. region.sector = 0;
  485. region.count = 0;
  486. req.bi_op = REQ_OP_WRITE;
  487. req.bi_op_flags = REQ_PREFLUSH;
  488. req.mem.type = DM_IO_KMEM;
  489. req.mem.ptr.addr = NULL;
  490. req.client = wc->dm_io;
  491. req.notify.fn = NULL;
  492. r = dm_io(&req, 1, &region, NULL);
  493. if (unlikely(r))
  494. writecache_error(wc, r, "error flushing metadata: %d", r);
  495. }
  496. #define WFE_RETURN_FOLLOWING 1
  497. #define WFE_LOWEST_SEQ 2
  498. static struct wc_entry *writecache_find_entry(struct dm_writecache *wc,
  499. uint64_t block, int flags)
  500. {
  501. struct wc_entry *e;
  502. struct rb_node *node = wc->tree.rb_node;
  503. if (unlikely(!node))
  504. return NULL;
  505. while (1) {
  506. e = container_of(node, struct wc_entry, rb_node);
  507. if (read_original_sector(wc, e) == block)
  508. break;
  509. node = (read_original_sector(wc, e) >= block ?
  510. e->rb_node.rb_left : e->rb_node.rb_right);
  511. if (unlikely(!node)) {
  512. if (!(flags & WFE_RETURN_FOLLOWING))
  513. return NULL;
  514. if (read_original_sector(wc, e) >= block) {
  515. return e;
  516. } else {
  517. node = rb_next(&e->rb_node);
  518. if (unlikely(!node))
  519. return NULL;
  520. e = container_of(node, struct wc_entry, rb_node);
  521. return e;
  522. }
  523. }
  524. }
  525. while (1) {
  526. struct wc_entry *e2;
  527. if (flags & WFE_LOWEST_SEQ)
  528. node = rb_prev(&e->rb_node);
  529. else
  530. node = rb_next(&e->rb_node);
  531. if (unlikely(!node))
  532. return e;
  533. e2 = container_of(node, struct wc_entry, rb_node);
  534. if (read_original_sector(wc, e2) != block)
  535. return e;
  536. e = e2;
  537. }
  538. }
  539. static void writecache_insert_entry(struct dm_writecache *wc, struct wc_entry *ins)
  540. {
  541. struct wc_entry *e;
  542. struct rb_node **node = &wc->tree.rb_node, *parent = NULL;
  543. while (*node) {
  544. e = container_of(*node, struct wc_entry, rb_node);
  545. parent = &e->rb_node;
  546. if (read_original_sector(wc, e) > read_original_sector(wc, ins))
  547. node = &parent->rb_left;
  548. else
  549. node = &parent->rb_right;
  550. }
  551. rb_link_node(&ins->rb_node, parent, node);
  552. rb_insert_color(&ins->rb_node, &wc->tree);
  553. list_add(&ins->lru, &wc->lru);
  554. ins->age = jiffies;
  555. }
  556. static void writecache_unlink(struct dm_writecache *wc, struct wc_entry *e)
  557. {
  558. list_del(&e->lru);
  559. rb_erase(&e->rb_node, &wc->tree);
  560. }
  561. static void writecache_add_to_freelist(struct dm_writecache *wc, struct wc_entry *e)
  562. {
  563. if (WC_MODE_SORT_FREELIST(wc)) {
  564. struct rb_node **node = &wc->freetree.rb_node, *parent = NULL;
  565. if (unlikely(!*node))
  566. wc->current_free = e;
  567. while (*node) {
  568. parent = *node;
  569. if (&e->rb_node < *node)
  570. node = &parent->rb_left;
  571. else
  572. node = &parent->rb_right;
  573. }
  574. rb_link_node(&e->rb_node, parent, node);
  575. rb_insert_color(&e->rb_node, &wc->freetree);
  576. } else {
  577. list_add_tail(&e->lru, &wc->freelist);
  578. }
  579. wc->freelist_size++;
  580. }
  581. static inline void writecache_verify_watermark(struct dm_writecache *wc)
  582. {
  583. if (unlikely(wc->freelist_size + wc->writeback_size <= wc->freelist_high_watermark))
  584. queue_work(wc->writeback_wq, &wc->writeback_work);
  585. }
  586. static void writecache_max_age_timer(struct timer_list *t)
  587. {
  588. struct dm_writecache *wc = from_timer(wc, t, max_age_timer);
  589. if (!dm_suspended(wc->ti) && !writecache_has_error(wc)) {
  590. queue_work(wc->writeback_wq, &wc->writeback_work);
  591. mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
  592. }
  593. }
  594. static struct wc_entry *writecache_pop_from_freelist(struct dm_writecache *wc, sector_t expected_sector)
  595. {
  596. struct wc_entry *e;
  597. if (WC_MODE_SORT_FREELIST(wc)) {
  598. struct rb_node *next;
  599. if (unlikely(!wc->current_free))
  600. return NULL;
  601. e = wc->current_free;
  602. if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
  603. return NULL;
  604. next = rb_next(&e->rb_node);
  605. rb_erase(&e->rb_node, &wc->freetree);
  606. if (unlikely(!next))
  607. next = rb_first(&wc->freetree);
  608. wc->current_free = next ? container_of(next, struct wc_entry, rb_node) : NULL;
  609. } else {
  610. if (unlikely(list_empty(&wc->freelist)))
  611. return NULL;
  612. e = container_of(wc->freelist.next, struct wc_entry, lru);
  613. if (expected_sector != (sector_t)-1 && unlikely(cache_sector(wc, e) != expected_sector))
  614. return NULL;
  615. list_del(&e->lru);
  616. }
  617. wc->freelist_size--;
  618. writecache_verify_watermark(wc);
  619. return e;
  620. }
  621. static void writecache_free_entry(struct dm_writecache *wc, struct wc_entry *e)
  622. {
  623. writecache_unlink(wc, e);
  624. writecache_add_to_freelist(wc, e);
  625. clear_seq_count(wc, e);
  626. writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
  627. if (unlikely(waitqueue_active(&wc->freelist_wait)))
  628. wake_up(&wc->freelist_wait);
  629. }
  630. static void writecache_wait_on_freelist(struct dm_writecache *wc)
  631. {
  632. DEFINE_WAIT(wait);
  633. prepare_to_wait(&wc->freelist_wait, &wait, TASK_UNINTERRUPTIBLE);
  634. wc_unlock(wc);
  635. io_schedule();
  636. finish_wait(&wc->freelist_wait, &wait);
  637. wc_lock(wc);
  638. }
  639. static void writecache_poison_lists(struct dm_writecache *wc)
  640. {
  641. /*
  642. * Catch incorrect access to these values while the device is suspended.
  643. */
  644. memset(&wc->tree, -1, sizeof wc->tree);
  645. wc->lru.next = LIST_POISON1;
  646. wc->lru.prev = LIST_POISON2;
  647. wc->freelist.next = LIST_POISON1;
  648. wc->freelist.prev = LIST_POISON2;
  649. }
  650. static void writecache_flush_entry(struct dm_writecache *wc, struct wc_entry *e)
  651. {
  652. writecache_flush_region(wc, memory_entry(wc, e), sizeof(struct wc_memory_entry));
  653. if (WC_MODE_PMEM(wc))
  654. writecache_flush_region(wc, memory_data(wc, e), wc->block_size);
  655. }
  656. static bool writecache_entry_is_committed(struct dm_writecache *wc, struct wc_entry *e)
  657. {
  658. return read_seq_count(wc, e) < wc->seq_count;
  659. }
  660. static void writecache_flush(struct dm_writecache *wc)
  661. {
  662. struct wc_entry *e, *e2;
  663. bool need_flush_after_free;
  664. wc->uncommitted_blocks = 0;
  665. del_timer(&wc->autocommit_timer);
  666. if (list_empty(&wc->lru))
  667. return;
  668. e = container_of(wc->lru.next, struct wc_entry, lru);
  669. if (writecache_entry_is_committed(wc, e)) {
  670. if (wc->overwrote_committed) {
  671. writecache_wait_for_ios(wc, WRITE);
  672. writecache_disk_flush(wc, wc->ssd_dev);
  673. wc->overwrote_committed = false;
  674. }
  675. return;
  676. }
  677. while (1) {
  678. writecache_flush_entry(wc, e);
  679. if (unlikely(e->lru.next == &wc->lru))
  680. break;
  681. e2 = container_of(e->lru.next, struct wc_entry, lru);
  682. if (writecache_entry_is_committed(wc, e2))
  683. break;
  684. e = e2;
  685. cond_resched();
  686. }
  687. writecache_commit_flushed(wc, true);
  688. wc->seq_count++;
  689. pmem_assign(sb(wc)->seq_count, cpu_to_le64(wc->seq_count));
  690. if (WC_MODE_PMEM(wc))
  691. writecache_commit_flushed(wc, false);
  692. else
  693. ssd_commit_superblock(wc);
  694. wc->overwrote_committed = false;
  695. need_flush_after_free = false;
  696. while (1) {
  697. /* Free another committed entry with lower seq-count */
  698. struct rb_node *rb_node = rb_prev(&e->rb_node);
  699. if (rb_node) {
  700. e2 = container_of(rb_node, struct wc_entry, rb_node);
  701. if (read_original_sector(wc, e2) == read_original_sector(wc, e) &&
  702. likely(!e2->write_in_progress)) {
  703. writecache_free_entry(wc, e2);
  704. need_flush_after_free = true;
  705. }
  706. }
  707. if (unlikely(e->lru.prev == &wc->lru))
  708. break;
  709. e = container_of(e->lru.prev, struct wc_entry, lru);
  710. cond_resched();
  711. }
  712. if (need_flush_after_free)
  713. writecache_commit_flushed(wc, false);
  714. }
  715. static void writecache_flush_work(struct work_struct *work)
  716. {
  717. struct dm_writecache *wc = container_of(work, struct dm_writecache, flush_work);
  718. wc_lock(wc);
  719. writecache_flush(wc);
  720. wc_unlock(wc);
  721. }
  722. static void writecache_autocommit_timer(struct timer_list *t)
  723. {
  724. struct dm_writecache *wc = from_timer(wc, t, autocommit_timer);
  725. if (!writecache_has_error(wc))
  726. queue_work(wc->writeback_wq, &wc->flush_work);
  727. }
  728. static void writecache_schedule_autocommit(struct dm_writecache *wc)
  729. {
  730. if (!timer_pending(&wc->autocommit_timer))
  731. mod_timer(&wc->autocommit_timer, jiffies + wc->autocommit_jiffies);
  732. }
  733. static void writecache_discard(struct dm_writecache *wc, sector_t start, sector_t end)
  734. {
  735. struct wc_entry *e;
  736. bool discarded_something = false;
  737. e = writecache_find_entry(wc, start, WFE_RETURN_FOLLOWING | WFE_LOWEST_SEQ);
  738. if (unlikely(!e))
  739. return;
  740. while (read_original_sector(wc, e) < end) {
  741. struct rb_node *node = rb_next(&e->rb_node);
  742. if (likely(!e->write_in_progress)) {
  743. if (!discarded_something) {
  744. if (!WC_MODE_PMEM(wc)) {
  745. writecache_wait_for_ios(wc, READ);
  746. writecache_wait_for_ios(wc, WRITE);
  747. }
  748. discarded_something = true;
  749. }
  750. if (!writecache_entry_is_committed(wc, e))
  751. wc->uncommitted_blocks--;
  752. writecache_free_entry(wc, e);
  753. }
  754. if (unlikely(!node))
  755. break;
  756. e = container_of(node, struct wc_entry, rb_node);
  757. }
  758. if (discarded_something)
  759. writecache_commit_flushed(wc, false);
  760. }
  761. static bool writecache_wait_for_writeback(struct dm_writecache *wc)
  762. {
  763. if (wc->writeback_size) {
  764. writecache_wait_on_freelist(wc);
  765. return true;
  766. }
  767. return false;
  768. }
  769. static void writecache_suspend(struct dm_target *ti)
  770. {
  771. struct dm_writecache *wc = ti->private;
  772. bool flush_on_suspend;
  773. del_timer_sync(&wc->autocommit_timer);
  774. del_timer_sync(&wc->max_age_timer);
  775. wc_lock(wc);
  776. writecache_flush(wc);
  777. flush_on_suspend = wc->flush_on_suspend;
  778. if (flush_on_suspend) {
  779. wc->flush_on_suspend = false;
  780. wc->writeback_all++;
  781. queue_work(wc->writeback_wq, &wc->writeback_work);
  782. }
  783. wc_unlock(wc);
  784. drain_workqueue(wc->writeback_wq);
  785. wc_lock(wc);
  786. if (flush_on_suspend)
  787. wc->writeback_all--;
  788. while (writecache_wait_for_writeback(wc));
  789. if (WC_MODE_PMEM(wc))
  790. persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
  791. writecache_poison_lists(wc);
  792. wc_unlock(wc);
  793. }
  794. static int writecache_alloc_entries(struct dm_writecache *wc)
  795. {
  796. size_t b;
  797. if (wc->entries)
  798. return 0;
  799. wc->entries = vmalloc(array_size(sizeof(struct wc_entry), wc->n_blocks));
  800. if (!wc->entries)
  801. return -ENOMEM;
  802. for (b = 0; b < wc->n_blocks; b++) {
  803. struct wc_entry *e = &wc->entries[b];
  804. e->index = b;
  805. e->write_in_progress = false;
  806. cond_resched();
  807. }
  808. return 0;
  809. }
  810. static int writecache_read_metadata(struct dm_writecache *wc, sector_t n_sectors)
  811. {
  812. struct dm_io_region region;
  813. struct dm_io_request req;
  814. region.bdev = wc->ssd_dev->bdev;
  815. region.sector = wc->start_sector;
  816. region.count = n_sectors;
  817. req.bi_op = REQ_OP_READ;
  818. req.bi_op_flags = REQ_SYNC;
  819. req.mem.type = DM_IO_VMA;
  820. req.mem.ptr.vma = (char *)wc->memory_map;
  821. req.client = wc->dm_io;
  822. req.notify.fn = NULL;
  823. return dm_io(&req, 1, &region, NULL);
  824. }
  825. static void writecache_resume(struct dm_target *ti)
  826. {
  827. struct dm_writecache *wc = ti->private;
  828. size_t b;
  829. bool need_flush = false;
  830. __le64 sb_seq_count;
  831. int r;
  832. wc_lock(wc);
  833. wc->data_device_sectors = i_size_read(wc->dev->bdev->bd_inode) >> SECTOR_SHIFT;
  834. if (WC_MODE_PMEM(wc)) {
  835. persistent_memory_invalidate_cache(wc->memory_map, wc->memory_map_size);
  836. } else {
  837. r = writecache_read_metadata(wc, wc->metadata_sectors);
  838. if (r) {
  839. size_t sb_entries_offset;
  840. writecache_error(wc, r, "unable to read metadata: %d", r);
  841. sb_entries_offset = offsetof(struct wc_memory_superblock, entries);
  842. memset((char *)wc->memory_map + sb_entries_offset, -1,
  843. (wc->metadata_sectors << SECTOR_SHIFT) - sb_entries_offset);
  844. }
  845. }
  846. wc->tree = RB_ROOT;
  847. INIT_LIST_HEAD(&wc->lru);
  848. if (WC_MODE_SORT_FREELIST(wc)) {
  849. wc->freetree = RB_ROOT;
  850. wc->current_free = NULL;
  851. } else {
  852. INIT_LIST_HEAD(&wc->freelist);
  853. }
  854. wc->freelist_size = 0;
  855. r = copy_mc_to_kernel(&sb_seq_count, &sb(wc)->seq_count,
  856. sizeof(uint64_t));
  857. if (r) {
  858. writecache_error(wc, r, "hardware memory error when reading superblock: %d", r);
  859. sb_seq_count = cpu_to_le64(0);
  860. }
  861. wc->seq_count = le64_to_cpu(sb_seq_count);
  862. #ifdef DM_WRITECACHE_HANDLE_HARDWARE_ERRORS
  863. for (b = 0; b < wc->n_blocks; b++) {
  864. struct wc_entry *e = &wc->entries[b];
  865. struct wc_memory_entry wme;
  866. if (writecache_has_error(wc)) {
  867. e->original_sector = -1;
  868. e->seq_count = -1;
  869. continue;
  870. }
  871. r = copy_mc_to_kernel(&wme, memory_entry(wc, e),
  872. sizeof(struct wc_memory_entry));
  873. if (r) {
  874. writecache_error(wc, r, "hardware memory error when reading metadata entry %lu: %d",
  875. (unsigned long)b, r);
  876. e->original_sector = -1;
  877. e->seq_count = -1;
  878. } else {
  879. e->original_sector = le64_to_cpu(wme.original_sector);
  880. e->seq_count = le64_to_cpu(wme.seq_count);
  881. }
  882. cond_resched();
  883. }
  884. #endif
  885. for (b = 0; b < wc->n_blocks; b++) {
  886. struct wc_entry *e = &wc->entries[b];
  887. if (!writecache_entry_is_committed(wc, e)) {
  888. if (read_seq_count(wc, e) != -1) {
  889. erase_this:
  890. clear_seq_count(wc, e);
  891. need_flush = true;
  892. }
  893. writecache_add_to_freelist(wc, e);
  894. } else {
  895. struct wc_entry *old;
  896. old = writecache_find_entry(wc, read_original_sector(wc, e), 0);
  897. if (!old) {
  898. writecache_insert_entry(wc, e);
  899. } else {
  900. if (read_seq_count(wc, old) == read_seq_count(wc, e)) {
  901. writecache_error(wc, -EINVAL,
  902. "two identical entries, position %llu, sector %llu, sequence %llu",
  903. (unsigned long long)b, (unsigned long long)read_original_sector(wc, e),
  904. (unsigned long long)read_seq_count(wc, e));
  905. }
  906. if (read_seq_count(wc, old) > read_seq_count(wc, e)) {
  907. goto erase_this;
  908. } else {
  909. writecache_free_entry(wc, old);
  910. writecache_insert_entry(wc, e);
  911. need_flush = true;
  912. }
  913. }
  914. }
  915. cond_resched();
  916. }
  917. if (need_flush) {
  918. writecache_flush_all_metadata(wc);
  919. writecache_commit_flushed(wc, false);
  920. }
  921. writecache_verify_watermark(wc);
  922. if (wc->max_age != MAX_AGE_UNSPECIFIED)
  923. mod_timer(&wc->max_age_timer, jiffies + wc->max_age / MAX_AGE_DIV);
  924. wc_unlock(wc);
  925. }
  926. static int process_flush_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
  927. {
  928. if (argc != 1)
  929. return -EINVAL;
  930. wc_lock(wc);
  931. if (dm_suspended(wc->ti)) {
  932. wc_unlock(wc);
  933. return -EBUSY;
  934. }
  935. if (writecache_has_error(wc)) {
  936. wc_unlock(wc);
  937. return -EIO;
  938. }
  939. writecache_flush(wc);
  940. wc->writeback_all++;
  941. queue_work(wc->writeback_wq, &wc->writeback_work);
  942. wc_unlock(wc);
  943. flush_workqueue(wc->writeback_wq);
  944. wc_lock(wc);
  945. wc->writeback_all--;
  946. if (writecache_has_error(wc)) {
  947. wc_unlock(wc);
  948. return -EIO;
  949. }
  950. wc_unlock(wc);
  951. return 0;
  952. }
  953. static int process_flush_on_suspend_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
  954. {
  955. if (argc != 1)
  956. return -EINVAL;
  957. wc_lock(wc);
  958. wc->flush_on_suspend = true;
  959. wc_unlock(wc);
  960. return 0;
  961. }
  962. static void activate_cleaner(struct dm_writecache *wc)
  963. {
  964. wc->flush_on_suspend = true;
  965. wc->cleaner = true;
  966. wc->freelist_high_watermark = wc->n_blocks;
  967. wc->freelist_low_watermark = wc->n_blocks;
  968. }
  969. static int process_cleaner_mesg(unsigned argc, char **argv, struct dm_writecache *wc)
  970. {
  971. if (argc != 1)
  972. return -EINVAL;
  973. wc_lock(wc);
  974. activate_cleaner(wc);
  975. if (!dm_suspended(wc->ti))
  976. writecache_verify_watermark(wc);
  977. wc_unlock(wc);
  978. return 0;
  979. }
  980. static int writecache_message(struct dm_target *ti, unsigned argc, char **argv,
  981. char *result, unsigned maxlen)
  982. {
  983. int r = -EINVAL;
  984. struct dm_writecache *wc = ti->private;
  985. if (!strcasecmp(argv[0], "flush"))
  986. r = process_flush_mesg(argc, argv, wc);
  987. else if (!strcasecmp(argv[0], "flush_on_suspend"))
  988. r = process_flush_on_suspend_mesg(argc, argv, wc);
  989. else if (!strcasecmp(argv[0], "cleaner"))
  990. r = process_cleaner_mesg(argc, argv, wc);
  991. else
  992. DMERR("unrecognised message received: %s", argv[0]);
  993. return r;
  994. }
  995. static void memcpy_flushcache_optimized(void *dest, void *source, size_t size)
  996. {
  997. /*
  998. * clflushopt performs better with block size 1024, 2048, 4096
  999. * non-temporal stores perform better with block size 512
  1000. *
  1001. * block size 512 1024 2048 4096
  1002. * movnti 496 MB/s 642 MB/s 725 MB/s 744 MB/s
  1003. * clflushopt 373 MB/s 688 MB/s 1.1 GB/s 1.2 GB/s
  1004. *
  1005. * We see that movnti performs better for 512-byte blocks, and
  1006. * clflushopt performs better for 1024-byte and larger blocks. So, we
  1007. * prefer clflushopt for sizes >= 768.
  1008. *
  1009. * NOTE: this happens to be the case now (with dm-writecache's single
  1010. * threaded model) but re-evaluate this once memcpy_flushcache() is
  1011. * enabled to use movdir64b which might invalidate this performance
  1012. * advantage seen with cache-allocating-writes plus flushing.
  1013. */
  1014. #ifdef CONFIG_X86
  1015. if (static_cpu_has(X86_FEATURE_CLFLUSHOPT) &&
  1016. likely(boot_cpu_data.x86_clflush_size == 64) &&
  1017. likely(size >= 768)) {
  1018. do {
  1019. memcpy((void *)dest, (void *)source, 64);
  1020. clflushopt((void *)dest);
  1021. dest += 64;
  1022. source += 64;
  1023. size -= 64;
  1024. } while (size >= 64);
  1025. return;
  1026. }
  1027. #endif
  1028. memcpy_flushcache(dest, source, size);
  1029. }
  1030. static void bio_copy_block(struct dm_writecache *wc, struct bio *bio, void *data)
  1031. {
  1032. void *buf;
  1033. unsigned long flags;
  1034. unsigned size;
  1035. int rw = bio_data_dir(bio);
  1036. unsigned remaining_size = wc->block_size;
  1037. do {
  1038. struct bio_vec bv = bio_iter_iovec(bio, bio->bi_iter);
  1039. buf = bvec_kmap_irq(&bv, &flags);
  1040. size = bv.bv_len;
  1041. if (unlikely(size > remaining_size))
  1042. size = remaining_size;
  1043. if (rw == READ) {
  1044. int r;
  1045. r = copy_mc_to_kernel(buf, data, size);
  1046. flush_dcache_page(bio_page(bio));
  1047. if (unlikely(r)) {
  1048. writecache_error(wc, r, "hardware memory error when reading data: %d", r);
  1049. bio->bi_status = BLK_STS_IOERR;
  1050. }
  1051. } else {
  1052. flush_dcache_page(bio_page(bio));
  1053. memcpy_flushcache_optimized(data, buf, size);
  1054. }
  1055. bvec_kunmap_irq(buf, &flags);
  1056. data = (char *)data + size;
  1057. remaining_size -= size;
  1058. bio_advance(bio, size);
  1059. } while (unlikely(remaining_size));
  1060. }
  1061. static int writecache_flush_thread(void *data)
  1062. {
  1063. struct dm_writecache *wc = data;
  1064. while (1) {
  1065. struct bio *bio;
  1066. wc_lock(wc);
  1067. bio = bio_list_pop(&wc->flush_list);
  1068. if (!bio) {
  1069. set_current_state(TASK_INTERRUPTIBLE);
  1070. wc_unlock(wc);
  1071. if (unlikely(kthread_should_stop())) {
  1072. set_current_state(TASK_RUNNING);
  1073. break;
  1074. }
  1075. schedule();
  1076. continue;
  1077. }
  1078. if (bio_op(bio) == REQ_OP_DISCARD) {
  1079. writecache_discard(wc, bio->bi_iter.bi_sector,
  1080. bio_end_sector(bio));
  1081. wc_unlock(wc);
  1082. bio_set_dev(bio, wc->dev->bdev);
  1083. submit_bio_noacct(bio);
  1084. } else {
  1085. writecache_flush(wc);
  1086. wc_unlock(wc);
  1087. if (writecache_has_error(wc))
  1088. bio->bi_status = BLK_STS_IOERR;
  1089. bio_endio(bio);
  1090. }
  1091. }
  1092. return 0;
  1093. }
  1094. static void writecache_offload_bio(struct dm_writecache *wc, struct bio *bio)
  1095. {
  1096. if (bio_list_empty(&wc->flush_list))
  1097. wake_up_process(wc->flush_thread);
  1098. bio_list_add(&wc->flush_list, bio);
  1099. }
  1100. static int writecache_map(struct dm_target *ti, struct bio *bio)
  1101. {
  1102. struct wc_entry *e;
  1103. struct dm_writecache *wc = ti->private;
  1104. bio->bi_private = NULL;
  1105. wc_lock(wc);
  1106. if (unlikely(bio->bi_opf & REQ_PREFLUSH)) {
  1107. if (writecache_has_error(wc))
  1108. goto unlock_error;
  1109. if (WC_MODE_PMEM(wc)) {
  1110. writecache_flush(wc);
  1111. if (writecache_has_error(wc))
  1112. goto unlock_error;
  1113. if (unlikely(wc->cleaner))
  1114. goto unlock_remap_origin;
  1115. goto unlock_submit;
  1116. } else {
  1117. if (dm_bio_get_target_bio_nr(bio))
  1118. goto unlock_remap_origin;
  1119. writecache_offload_bio(wc, bio);
  1120. goto unlock_return;
  1121. }
  1122. }
  1123. bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
  1124. if (unlikely((((unsigned)bio->bi_iter.bi_sector | bio_sectors(bio)) &
  1125. (wc->block_size / 512 - 1)) != 0)) {
  1126. DMERR("I/O is not aligned, sector %llu, size %u, block size %u",
  1127. (unsigned long long)bio->bi_iter.bi_sector,
  1128. bio->bi_iter.bi_size, wc->block_size);
  1129. goto unlock_error;
  1130. }
  1131. if (unlikely(bio_op(bio) == REQ_OP_DISCARD)) {
  1132. if (writecache_has_error(wc))
  1133. goto unlock_error;
  1134. if (WC_MODE_PMEM(wc)) {
  1135. writecache_discard(wc, bio->bi_iter.bi_sector, bio_end_sector(bio));
  1136. goto unlock_remap_origin;
  1137. } else {
  1138. writecache_offload_bio(wc, bio);
  1139. goto unlock_return;
  1140. }
  1141. }
  1142. if (bio_data_dir(bio) == READ) {
  1143. read_next_block:
  1144. e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
  1145. if (e && read_original_sector(wc, e) == bio->bi_iter.bi_sector) {
  1146. if (WC_MODE_PMEM(wc)) {
  1147. bio_copy_block(wc, bio, memory_data(wc, e));
  1148. if (bio->bi_iter.bi_size)
  1149. goto read_next_block;
  1150. goto unlock_submit;
  1151. } else {
  1152. dm_accept_partial_bio(bio, wc->block_size >> SECTOR_SHIFT);
  1153. bio_set_dev(bio, wc->ssd_dev->bdev);
  1154. bio->bi_iter.bi_sector = cache_sector(wc, e);
  1155. if (!writecache_entry_is_committed(wc, e))
  1156. writecache_wait_for_ios(wc, WRITE);
  1157. goto unlock_remap;
  1158. }
  1159. } else {
  1160. if (e) {
  1161. sector_t next_boundary =
  1162. read_original_sector(wc, e) - bio->bi_iter.bi_sector;
  1163. if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
  1164. dm_accept_partial_bio(bio, next_boundary);
  1165. }
  1166. }
  1167. goto unlock_remap_origin;
  1168. }
  1169. } else {
  1170. do {
  1171. bool found_entry = false;
  1172. bool search_used = false;
  1173. if (writecache_has_error(wc))
  1174. goto unlock_error;
  1175. e = writecache_find_entry(wc, bio->bi_iter.bi_sector, 0);
  1176. if (e) {
  1177. if (!writecache_entry_is_committed(wc, e)) {
  1178. search_used = true;
  1179. goto bio_copy;
  1180. }
  1181. if (!WC_MODE_PMEM(wc) && !e->write_in_progress) {
  1182. wc->overwrote_committed = true;
  1183. search_used = true;
  1184. goto bio_copy;
  1185. }
  1186. found_entry = true;
  1187. } else {
  1188. if (unlikely(wc->cleaner))
  1189. goto direct_write;
  1190. }
  1191. e = writecache_pop_from_freelist(wc, (sector_t)-1);
  1192. if (unlikely(!e)) {
  1193. if (!WC_MODE_PMEM(wc) && !found_entry) {
  1194. direct_write:
  1195. e = writecache_find_entry(wc, bio->bi_iter.bi_sector, WFE_RETURN_FOLLOWING);
  1196. if (e) {
  1197. sector_t next_boundary = read_original_sector(wc, e) - bio->bi_iter.bi_sector;
  1198. BUG_ON(!next_boundary);
  1199. if (next_boundary < bio->bi_iter.bi_size >> SECTOR_SHIFT) {
  1200. dm_accept_partial_bio(bio, next_boundary);
  1201. }
  1202. }
  1203. goto unlock_remap_origin;
  1204. }
  1205. writecache_wait_on_freelist(wc);
  1206. continue;
  1207. }
  1208. write_original_sector_seq_count(wc, e, bio->bi_iter.bi_sector, wc->seq_count);
  1209. writecache_insert_entry(wc, e);
  1210. wc->uncommitted_blocks++;
  1211. bio_copy:
  1212. if (WC_MODE_PMEM(wc)) {
  1213. bio_copy_block(wc, bio, memory_data(wc, e));
  1214. } else {
  1215. unsigned bio_size = wc->block_size;
  1216. sector_t start_cache_sec = cache_sector(wc, e);
  1217. sector_t current_cache_sec = start_cache_sec + (bio_size >> SECTOR_SHIFT);
  1218. while (bio_size < bio->bi_iter.bi_size) {
  1219. if (!search_used) {
  1220. struct wc_entry *f = writecache_pop_from_freelist(wc, current_cache_sec);
  1221. if (!f)
  1222. break;
  1223. write_original_sector_seq_count(wc, f, bio->bi_iter.bi_sector +
  1224. (bio_size >> SECTOR_SHIFT), wc->seq_count);
  1225. writecache_insert_entry(wc, f);
  1226. wc->uncommitted_blocks++;
  1227. } else {
  1228. struct wc_entry *f;
  1229. struct rb_node *next = rb_next(&e->rb_node);
  1230. if (!next)
  1231. break;
  1232. f = container_of(next, struct wc_entry, rb_node);
  1233. if (f != e + 1)
  1234. break;
  1235. if (read_original_sector(wc, f) !=
  1236. read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
  1237. break;
  1238. if (unlikely(f->write_in_progress))
  1239. break;
  1240. if (writecache_entry_is_committed(wc, f))
  1241. wc->overwrote_committed = true;
  1242. e = f;
  1243. }
  1244. bio_size += wc->block_size;
  1245. current_cache_sec += wc->block_size >> SECTOR_SHIFT;
  1246. }
  1247. bio_set_dev(bio, wc->ssd_dev->bdev);
  1248. bio->bi_iter.bi_sector = start_cache_sec;
  1249. dm_accept_partial_bio(bio, bio_size >> SECTOR_SHIFT);
  1250. if (unlikely(wc->uncommitted_blocks >= wc->autocommit_blocks)) {
  1251. wc->uncommitted_blocks = 0;
  1252. queue_work(wc->writeback_wq, &wc->flush_work);
  1253. } else {
  1254. writecache_schedule_autocommit(wc);
  1255. }
  1256. goto unlock_remap;
  1257. }
  1258. } while (bio->bi_iter.bi_size);
  1259. if (unlikely(bio->bi_opf & REQ_FUA ||
  1260. wc->uncommitted_blocks >= wc->autocommit_blocks))
  1261. writecache_flush(wc);
  1262. else
  1263. writecache_schedule_autocommit(wc);
  1264. goto unlock_submit;
  1265. }
  1266. unlock_remap_origin:
  1267. bio_set_dev(bio, wc->dev->bdev);
  1268. wc_unlock(wc);
  1269. return DM_MAPIO_REMAPPED;
  1270. unlock_remap:
  1271. /* make sure that writecache_end_io decrements bio_in_progress: */
  1272. bio->bi_private = (void *)1;
  1273. atomic_inc(&wc->bio_in_progress[bio_data_dir(bio)]);
  1274. wc_unlock(wc);
  1275. return DM_MAPIO_REMAPPED;
  1276. unlock_submit:
  1277. wc_unlock(wc);
  1278. bio_endio(bio);
  1279. return DM_MAPIO_SUBMITTED;
  1280. unlock_return:
  1281. wc_unlock(wc);
  1282. return DM_MAPIO_SUBMITTED;
  1283. unlock_error:
  1284. wc_unlock(wc);
  1285. bio_io_error(bio);
  1286. return DM_MAPIO_SUBMITTED;
  1287. }
  1288. static int writecache_end_io(struct dm_target *ti, struct bio *bio, blk_status_t *status)
  1289. {
  1290. struct dm_writecache *wc = ti->private;
  1291. if (bio->bi_private != NULL) {
  1292. int dir = bio_data_dir(bio);
  1293. if (atomic_dec_and_test(&wc->bio_in_progress[dir]))
  1294. if (unlikely(waitqueue_active(&wc->bio_in_progress_wait[dir])))
  1295. wake_up(&wc->bio_in_progress_wait[dir]);
  1296. }
  1297. return 0;
  1298. }
  1299. static int writecache_iterate_devices(struct dm_target *ti,
  1300. iterate_devices_callout_fn fn, void *data)
  1301. {
  1302. struct dm_writecache *wc = ti->private;
  1303. return fn(ti, wc->dev, 0, ti->len, data);
  1304. }
  1305. static void writecache_io_hints(struct dm_target *ti, struct queue_limits *limits)
  1306. {
  1307. struct dm_writecache *wc = ti->private;
  1308. if (limits->logical_block_size < wc->block_size)
  1309. limits->logical_block_size = wc->block_size;
  1310. if (limits->physical_block_size < wc->block_size)
  1311. limits->physical_block_size = wc->block_size;
  1312. if (limits->io_min < wc->block_size)
  1313. limits->io_min = wc->block_size;
  1314. }
  1315. static void writecache_writeback_endio(struct bio *bio)
  1316. {
  1317. struct writeback_struct *wb = container_of(bio, struct writeback_struct, bio);
  1318. struct dm_writecache *wc = wb->wc;
  1319. unsigned long flags;
  1320. raw_spin_lock_irqsave(&wc->endio_list_lock, flags);
  1321. if (unlikely(list_empty(&wc->endio_list)))
  1322. wake_up_process(wc->endio_thread);
  1323. list_add_tail(&wb->endio_entry, &wc->endio_list);
  1324. raw_spin_unlock_irqrestore(&wc->endio_list_lock, flags);
  1325. }
  1326. static void writecache_copy_endio(int read_err, unsigned long write_err, void *ptr)
  1327. {
  1328. struct copy_struct *c = ptr;
  1329. struct dm_writecache *wc = c->wc;
  1330. c->error = likely(!(read_err | write_err)) ? 0 : -EIO;
  1331. raw_spin_lock_irq(&wc->endio_list_lock);
  1332. if (unlikely(list_empty(&wc->endio_list)))
  1333. wake_up_process(wc->endio_thread);
  1334. list_add_tail(&c->endio_entry, &wc->endio_list);
  1335. raw_spin_unlock_irq(&wc->endio_list_lock);
  1336. }
  1337. static void __writecache_endio_pmem(struct dm_writecache *wc, struct list_head *list)
  1338. {
  1339. unsigned i;
  1340. struct writeback_struct *wb;
  1341. struct wc_entry *e;
  1342. unsigned long n_walked = 0;
  1343. do {
  1344. wb = list_entry(list->next, struct writeback_struct, endio_entry);
  1345. list_del(&wb->endio_entry);
  1346. if (unlikely(wb->bio.bi_status != BLK_STS_OK))
  1347. writecache_error(wc, blk_status_to_errno(wb->bio.bi_status),
  1348. "write error %d", wb->bio.bi_status);
  1349. i = 0;
  1350. do {
  1351. e = wb->wc_list[i];
  1352. BUG_ON(!e->write_in_progress);
  1353. e->write_in_progress = false;
  1354. INIT_LIST_HEAD(&e->lru);
  1355. if (!writecache_has_error(wc))
  1356. writecache_free_entry(wc, e);
  1357. BUG_ON(!wc->writeback_size);
  1358. wc->writeback_size--;
  1359. n_walked++;
  1360. if (unlikely(n_walked >= ENDIO_LATENCY)) {
  1361. writecache_commit_flushed(wc, false);
  1362. wc_unlock(wc);
  1363. wc_lock(wc);
  1364. n_walked = 0;
  1365. }
  1366. } while (++i < wb->wc_list_n);
  1367. if (wb->wc_list != wb->wc_list_inline)
  1368. kfree(wb->wc_list);
  1369. bio_put(&wb->bio);
  1370. } while (!list_empty(list));
  1371. }
  1372. static void __writecache_endio_ssd(struct dm_writecache *wc, struct list_head *list)
  1373. {
  1374. struct copy_struct *c;
  1375. struct wc_entry *e;
  1376. do {
  1377. c = list_entry(list->next, struct copy_struct, endio_entry);
  1378. list_del(&c->endio_entry);
  1379. if (unlikely(c->error))
  1380. writecache_error(wc, c->error, "copy error");
  1381. e = c->e;
  1382. do {
  1383. BUG_ON(!e->write_in_progress);
  1384. e->write_in_progress = false;
  1385. INIT_LIST_HEAD(&e->lru);
  1386. if (!writecache_has_error(wc))
  1387. writecache_free_entry(wc, e);
  1388. BUG_ON(!wc->writeback_size);
  1389. wc->writeback_size--;
  1390. e++;
  1391. } while (--c->n_entries);
  1392. mempool_free(c, &wc->copy_pool);
  1393. } while (!list_empty(list));
  1394. }
  1395. static int writecache_endio_thread(void *data)
  1396. {
  1397. struct dm_writecache *wc = data;
  1398. while (1) {
  1399. struct list_head list;
  1400. raw_spin_lock_irq(&wc->endio_list_lock);
  1401. if (!list_empty(&wc->endio_list))
  1402. goto pop_from_list;
  1403. set_current_state(TASK_INTERRUPTIBLE);
  1404. raw_spin_unlock_irq(&wc->endio_list_lock);
  1405. if (unlikely(kthread_should_stop())) {
  1406. set_current_state(TASK_RUNNING);
  1407. break;
  1408. }
  1409. schedule();
  1410. continue;
  1411. pop_from_list:
  1412. list = wc->endio_list;
  1413. list.next->prev = list.prev->next = &list;
  1414. INIT_LIST_HEAD(&wc->endio_list);
  1415. raw_spin_unlock_irq(&wc->endio_list_lock);
  1416. if (!WC_MODE_FUA(wc))
  1417. writecache_disk_flush(wc, wc->dev);
  1418. wc_lock(wc);
  1419. if (WC_MODE_PMEM(wc)) {
  1420. __writecache_endio_pmem(wc, &list);
  1421. } else {
  1422. __writecache_endio_ssd(wc, &list);
  1423. writecache_wait_for_ios(wc, READ);
  1424. }
  1425. writecache_commit_flushed(wc, false);
  1426. wc_unlock(wc);
  1427. }
  1428. return 0;
  1429. }
  1430. static bool wc_add_block(struct writeback_struct *wb, struct wc_entry *e, gfp_t gfp)
  1431. {
  1432. struct dm_writecache *wc = wb->wc;
  1433. unsigned block_size = wc->block_size;
  1434. void *address = memory_data(wc, e);
  1435. persistent_memory_flush_cache(address, block_size);
  1436. if (unlikely(bio_end_sector(&wb->bio) >= wc->data_device_sectors))
  1437. return true;
  1438. return bio_add_page(&wb->bio, persistent_memory_page(address),
  1439. block_size, persistent_memory_page_offset(address)) != 0;
  1440. }
  1441. struct writeback_list {
  1442. struct list_head list;
  1443. size_t size;
  1444. };
  1445. static void __writeback_throttle(struct dm_writecache *wc, struct writeback_list *wbl)
  1446. {
  1447. if (unlikely(wc->max_writeback_jobs)) {
  1448. if (READ_ONCE(wc->writeback_size) - wbl->size >= wc->max_writeback_jobs) {
  1449. wc_lock(wc);
  1450. while (wc->writeback_size - wbl->size >= wc->max_writeback_jobs)
  1451. writecache_wait_on_freelist(wc);
  1452. wc_unlock(wc);
  1453. }
  1454. }
  1455. cond_resched();
  1456. }
  1457. static void __writecache_writeback_pmem(struct dm_writecache *wc, struct writeback_list *wbl)
  1458. {
  1459. struct wc_entry *e, *f;
  1460. struct bio *bio;
  1461. struct writeback_struct *wb;
  1462. unsigned max_pages;
  1463. while (wbl->size) {
  1464. wbl->size--;
  1465. e = container_of(wbl->list.prev, struct wc_entry, lru);
  1466. list_del(&e->lru);
  1467. max_pages = e->wc_list_contiguous;
  1468. bio = bio_alloc_bioset(GFP_NOIO, max_pages, &wc->bio_set);
  1469. wb = container_of(bio, struct writeback_struct, bio);
  1470. wb->wc = wc;
  1471. bio->bi_end_io = writecache_writeback_endio;
  1472. bio_set_dev(bio, wc->dev->bdev);
  1473. bio->bi_iter.bi_sector = read_original_sector(wc, e);
  1474. if (max_pages <= WB_LIST_INLINE ||
  1475. unlikely(!(wb->wc_list = kmalloc_array(max_pages, sizeof(struct wc_entry *),
  1476. GFP_NOIO | __GFP_NORETRY |
  1477. __GFP_NOMEMALLOC | __GFP_NOWARN)))) {
  1478. wb->wc_list = wb->wc_list_inline;
  1479. max_pages = WB_LIST_INLINE;
  1480. }
  1481. BUG_ON(!wc_add_block(wb, e, GFP_NOIO));
  1482. wb->wc_list[0] = e;
  1483. wb->wc_list_n = 1;
  1484. while (wbl->size && wb->wc_list_n < max_pages) {
  1485. f = container_of(wbl->list.prev, struct wc_entry, lru);
  1486. if (read_original_sector(wc, f) !=
  1487. read_original_sector(wc, e) + (wc->block_size >> SECTOR_SHIFT))
  1488. break;
  1489. if (!wc_add_block(wb, f, GFP_NOWAIT | __GFP_NOWARN))
  1490. break;
  1491. wbl->size--;
  1492. list_del(&f->lru);
  1493. wb->wc_list[wb->wc_list_n++] = f;
  1494. e = f;
  1495. }
  1496. bio_set_op_attrs(bio, REQ_OP_WRITE, WC_MODE_FUA(wc) * REQ_FUA);
  1497. if (writecache_has_error(wc)) {
  1498. bio->bi_status = BLK_STS_IOERR;
  1499. bio_endio(bio);
  1500. } else if (unlikely(!bio_sectors(bio))) {
  1501. bio->bi_status = BLK_STS_OK;
  1502. bio_endio(bio);
  1503. } else {
  1504. submit_bio(bio);
  1505. }
  1506. __writeback_throttle(wc, wbl);
  1507. }
  1508. }
  1509. static void __writecache_writeback_ssd(struct dm_writecache *wc, struct writeback_list *wbl)
  1510. {
  1511. struct wc_entry *e, *f;
  1512. struct dm_io_region from, to;
  1513. struct copy_struct *c;
  1514. while (wbl->size) {
  1515. unsigned n_sectors;
  1516. wbl->size--;
  1517. e = container_of(wbl->list.prev, struct wc_entry, lru);
  1518. list_del(&e->lru);
  1519. n_sectors = e->wc_list_contiguous << (wc->block_size_bits - SECTOR_SHIFT);
  1520. from.bdev = wc->ssd_dev->bdev;
  1521. from.sector = cache_sector(wc, e);
  1522. from.count = n_sectors;
  1523. to.bdev = wc->dev->bdev;
  1524. to.sector = read_original_sector(wc, e);
  1525. to.count = n_sectors;
  1526. c = mempool_alloc(&wc->copy_pool, GFP_NOIO);
  1527. c->wc = wc;
  1528. c->e = e;
  1529. c->n_entries = e->wc_list_contiguous;
  1530. while ((n_sectors -= wc->block_size >> SECTOR_SHIFT)) {
  1531. wbl->size--;
  1532. f = container_of(wbl->list.prev, struct wc_entry, lru);
  1533. BUG_ON(f != e + 1);
  1534. list_del(&f->lru);
  1535. e = f;
  1536. }
  1537. if (unlikely(to.sector + to.count > wc->data_device_sectors)) {
  1538. if (to.sector >= wc->data_device_sectors) {
  1539. writecache_copy_endio(0, 0, c);
  1540. continue;
  1541. }
  1542. from.count = to.count = wc->data_device_sectors - to.sector;
  1543. }
  1544. dm_kcopyd_copy(wc->dm_kcopyd, &from, 1, &to, 0, writecache_copy_endio, c);
  1545. __writeback_throttle(wc, wbl);
  1546. }
  1547. }
  1548. static void writecache_writeback(struct work_struct *work)
  1549. {
  1550. struct dm_writecache *wc = container_of(work, struct dm_writecache, writeback_work);
  1551. struct blk_plug plug;
  1552. struct wc_entry *f, *g, *e = NULL;
  1553. struct rb_node *node, *next_node;
  1554. struct list_head skipped;
  1555. struct writeback_list wbl;
  1556. unsigned long n_walked;
  1557. wc_lock(wc);
  1558. restart:
  1559. if (writecache_has_error(wc)) {
  1560. wc_unlock(wc);
  1561. return;
  1562. }
  1563. if (unlikely(wc->writeback_all)) {
  1564. if (writecache_wait_for_writeback(wc))
  1565. goto restart;
  1566. }
  1567. if (wc->overwrote_committed) {
  1568. writecache_wait_for_ios(wc, WRITE);
  1569. }
  1570. n_walked = 0;
  1571. INIT_LIST_HEAD(&skipped);
  1572. INIT_LIST_HEAD(&wbl.list);
  1573. wbl.size = 0;
  1574. while (!list_empty(&wc->lru) &&
  1575. (wc->writeback_all ||
  1576. wc->freelist_size + wc->writeback_size <= wc->freelist_low_watermark ||
  1577. (jiffies - container_of(wc->lru.prev, struct wc_entry, lru)->age >=
  1578. wc->max_age - wc->max_age / MAX_AGE_DIV))) {
  1579. n_walked++;
  1580. if (unlikely(n_walked > WRITEBACK_LATENCY) &&
  1581. likely(!wc->writeback_all) && likely(!dm_suspended(wc->ti))) {
  1582. queue_work(wc->writeback_wq, &wc->writeback_work);
  1583. break;
  1584. }
  1585. if (unlikely(wc->writeback_all)) {
  1586. if (unlikely(!e)) {
  1587. writecache_flush(wc);
  1588. e = container_of(rb_first(&wc->tree), struct wc_entry, rb_node);
  1589. } else
  1590. e = g;
  1591. } else
  1592. e = container_of(wc->lru.prev, struct wc_entry, lru);
  1593. BUG_ON(e->write_in_progress);
  1594. if (unlikely(!writecache_entry_is_committed(wc, e))) {
  1595. writecache_flush(wc);
  1596. }
  1597. node = rb_prev(&e->rb_node);
  1598. if (node) {
  1599. f = container_of(node, struct wc_entry, rb_node);
  1600. if (unlikely(read_original_sector(wc, f) ==
  1601. read_original_sector(wc, e))) {
  1602. BUG_ON(!f->write_in_progress);
  1603. list_del(&e->lru);
  1604. list_add(&e->lru, &skipped);
  1605. cond_resched();
  1606. continue;
  1607. }
  1608. }
  1609. wc->writeback_size++;
  1610. list_del(&e->lru);
  1611. list_add(&e->lru, &wbl.list);
  1612. wbl.size++;
  1613. e->write_in_progress = true;
  1614. e->wc_list_contiguous = 1;
  1615. f = e;
  1616. while (1) {
  1617. next_node = rb_next(&f->rb_node);
  1618. if (unlikely(!next_node))
  1619. break;
  1620. g = container_of(next_node, struct wc_entry, rb_node);
  1621. if (unlikely(read_original_sector(wc, g) ==
  1622. read_original_sector(wc, f))) {
  1623. f = g;
  1624. continue;
  1625. }
  1626. if (read_original_sector(wc, g) !=
  1627. read_original_sector(wc, f) + (wc->block_size >> SECTOR_SHIFT))
  1628. break;
  1629. if (unlikely(g->write_in_progress))
  1630. break;
  1631. if (unlikely(!writecache_entry_is_committed(wc, g)))
  1632. break;
  1633. if (!WC_MODE_PMEM(wc)) {
  1634. if (g != f + 1)
  1635. break;
  1636. }
  1637. n_walked++;
  1638. //if (unlikely(n_walked > WRITEBACK_LATENCY) && likely(!wc->writeback_all))
  1639. // break;
  1640. wc->writeback_size++;
  1641. list_del(&g->lru);
  1642. list_add(&g->lru, &wbl.list);
  1643. wbl.size++;
  1644. g->write_in_progress = true;
  1645. g->wc_list_contiguous = BIO_MAX_PAGES;
  1646. f = g;
  1647. e->wc_list_contiguous++;
  1648. if (unlikely(e->wc_list_contiguous == BIO_MAX_PAGES)) {
  1649. if (unlikely(wc->writeback_all)) {
  1650. next_node = rb_next(&f->rb_node);
  1651. if (likely(next_node))
  1652. g = container_of(next_node, struct wc_entry, rb_node);
  1653. }
  1654. break;
  1655. }
  1656. }
  1657. cond_resched();
  1658. }
  1659. if (!list_empty(&skipped)) {
  1660. list_splice_tail(&skipped, &wc->lru);
  1661. /*
  1662. * If we didn't do any progress, we must wait until some
  1663. * writeback finishes to avoid burning CPU in a loop
  1664. */
  1665. if (unlikely(!wbl.size))
  1666. writecache_wait_for_writeback(wc);
  1667. }
  1668. wc_unlock(wc);
  1669. blk_start_plug(&plug);
  1670. if (WC_MODE_PMEM(wc))
  1671. __writecache_writeback_pmem(wc, &wbl);
  1672. else
  1673. __writecache_writeback_ssd(wc, &wbl);
  1674. blk_finish_plug(&plug);
  1675. if (unlikely(wc->writeback_all)) {
  1676. wc_lock(wc);
  1677. while (writecache_wait_for_writeback(wc));
  1678. wc_unlock(wc);
  1679. }
  1680. }
  1681. static int calculate_memory_size(uint64_t device_size, unsigned block_size,
  1682. size_t *n_blocks_p, size_t *n_metadata_blocks_p)
  1683. {
  1684. uint64_t n_blocks, offset;
  1685. struct wc_entry e;
  1686. n_blocks = device_size;
  1687. do_div(n_blocks, block_size + sizeof(struct wc_memory_entry));
  1688. while (1) {
  1689. if (!n_blocks)
  1690. return -ENOSPC;
  1691. /* Verify the following entries[n_blocks] won't overflow */
  1692. if (n_blocks >= ((size_t)-sizeof(struct wc_memory_superblock) /
  1693. sizeof(struct wc_memory_entry)))
  1694. return -EFBIG;
  1695. offset = offsetof(struct wc_memory_superblock, entries[n_blocks]);
  1696. offset = (offset + block_size - 1) & ~(uint64_t)(block_size - 1);
  1697. if (offset + n_blocks * block_size <= device_size)
  1698. break;
  1699. n_blocks--;
  1700. }
  1701. /* check if the bit field overflows */
  1702. e.index = n_blocks;
  1703. if (e.index != n_blocks)
  1704. return -EFBIG;
  1705. if (n_blocks_p)
  1706. *n_blocks_p = n_blocks;
  1707. if (n_metadata_blocks_p)
  1708. *n_metadata_blocks_p = offset >> __ffs(block_size);
  1709. return 0;
  1710. }
  1711. static int init_memory(struct dm_writecache *wc)
  1712. {
  1713. size_t b;
  1714. int r;
  1715. r = calculate_memory_size(wc->memory_map_size, wc->block_size, &wc->n_blocks, NULL);
  1716. if (r)
  1717. return r;
  1718. r = writecache_alloc_entries(wc);
  1719. if (r)
  1720. return r;
  1721. for (b = 0; b < ARRAY_SIZE(sb(wc)->padding); b++)
  1722. pmem_assign(sb(wc)->padding[b], cpu_to_le64(0));
  1723. pmem_assign(sb(wc)->version, cpu_to_le32(MEMORY_SUPERBLOCK_VERSION));
  1724. pmem_assign(sb(wc)->block_size, cpu_to_le32(wc->block_size));
  1725. pmem_assign(sb(wc)->n_blocks, cpu_to_le64(wc->n_blocks));
  1726. pmem_assign(sb(wc)->seq_count, cpu_to_le64(0));
  1727. for (b = 0; b < wc->n_blocks; b++) {
  1728. write_original_sector_seq_count(wc, &wc->entries[b], -1, -1);
  1729. cond_resched();
  1730. }
  1731. writecache_flush_all_metadata(wc);
  1732. writecache_commit_flushed(wc, false);
  1733. pmem_assign(sb(wc)->magic, cpu_to_le32(MEMORY_SUPERBLOCK_MAGIC));
  1734. writecache_flush_region(wc, &sb(wc)->magic, sizeof sb(wc)->magic);
  1735. writecache_commit_flushed(wc, false);
  1736. return 0;
  1737. }
  1738. static void writecache_dtr(struct dm_target *ti)
  1739. {
  1740. struct dm_writecache *wc = ti->private;
  1741. if (!wc)
  1742. return;
  1743. if (wc->endio_thread)
  1744. kthread_stop(wc->endio_thread);
  1745. if (wc->flush_thread)
  1746. kthread_stop(wc->flush_thread);
  1747. bioset_exit(&wc->bio_set);
  1748. mempool_exit(&wc->copy_pool);
  1749. if (wc->writeback_wq)
  1750. destroy_workqueue(wc->writeback_wq);
  1751. if (wc->dev)
  1752. dm_put_device(ti, wc->dev);
  1753. if (wc->ssd_dev)
  1754. dm_put_device(ti, wc->ssd_dev);
  1755. if (wc->entries)
  1756. vfree(wc->entries);
  1757. if (wc->memory_map) {
  1758. if (WC_MODE_PMEM(wc))
  1759. persistent_memory_release(wc);
  1760. else
  1761. vfree(wc->memory_map);
  1762. }
  1763. if (wc->dm_kcopyd)
  1764. dm_kcopyd_client_destroy(wc->dm_kcopyd);
  1765. if (wc->dm_io)
  1766. dm_io_client_destroy(wc->dm_io);
  1767. if (wc->dirty_bitmap)
  1768. vfree(wc->dirty_bitmap);
  1769. kfree(wc);
  1770. }
  1771. static int writecache_ctr(struct dm_target *ti, unsigned argc, char **argv)
  1772. {
  1773. struct dm_writecache *wc;
  1774. struct dm_arg_set as;
  1775. const char *string;
  1776. unsigned opt_params;
  1777. size_t offset, data_size;
  1778. int i, r;
  1779. char dummy;
  1780. int high_wm_percent = HIGH_WATERMARK;
  1781. int low_wm_percent = LOW_WATERMARK;
  1782. uint64_t x;
  1783. struct wc_memory_superblock s;
  1784. static struct dm_arg _args[] = {
  1785. {0, 16, "Invalid number of feature args"},
  1786. };
  1787. as.argc = argc;
  1788. as.argv = argv;
  1789. wc = kzalloc(sizeof(struct dm_writecache), GFP_KERNEL);
  1790. if (!wc) {
  1791. ti->error = "Cannot allocate writecache structure";
  1792. r = -ENOMEM;
  1793. goto bad;
  1794. }
  1795. ti->private = wc;
  1796. wc->ti = ti;
  1797. mutex_init(&wc->lock);
  1798. wc->max_age = MAX_AGE_UNSPECIFIED;
  1799. writecache_poison_lists(wc);
  1800. init_waitqueue_head(&wc->freelist_wait);
  1801. timer_setup(&wc->autocommit_timer, writecache_autocommit_timer, 0);
  1802. timer_setup(&wc->max_age_timer, writecache_max_age_timer, 0);
  1803. for (i = 0; i < 2; i++) {
  1804. atomic_set(&wc->bio_in_progress[i], 0);
  1805. init_waitqueue_head(&wc->bio_in_progress_wait[i]);
  1806. }
  1807. wc->dm_io = dm_io_client_create();
  1808. if (IS_ERR(wc->dm_io)) {
  1809. r = PTR_ERR(wc->dm_io);
  1810. ti->error = "Unable to allocate dm-io client";
  1811. wc->dm_io = NULL;
  1812. goto bad;
  1813. }
  1814. wc->writeback_wq = alloc_workqueue("writecache-writeback", WQ_MEM_RECLAIM, 1);
  1815. if (!wc->writeback_wq) {
  1816. r = -ENOMEM;
  1817. ti->error = "Could not allocate writeback workqueue";
  1818. goto bad;
  1819. }
  1820. INIT_WORK(&wc->writeback_work, writecache_writeback);
  1821. INIT_WORK(&wc->flush_work, writecache_flush_work);
  1822. raw_spin_lock_init(&wc->endio_list_lock);
  1823. INIT_LIST_HEAD(&wc->endio_list);
  1824. wc->endio_thread = kthread_create(writecache_endio_thread, wc, "writecache_endio");
  1825. if (IS_ERR(wc->endio_thread)) {
  1826. r = PTR_ERR(wc->endio_thread);
  1827. wc->endio_thread = NULL;
  1828. ti->error = "Couldn't spawn endio thread";
  1829. goto bad;
  1830. }
  1831. wake_up_process(wc->endio_thread);
  1832. /*
  1833. * Parse the mode (pmem or ssd)
  1834. */
  1835. string = dm_shift_arg(&as);
  1836. if (!string)
  1837. goto bad_arguments;
  1838. if (!strcasecmp(string, "s")) {
  1839. wc->pmem_mode = false;
  1840. } else if (!strcasecmp(string, "p")) {
  1841. #ifdef DM_WRITECACHE_HAS_PMEM
  1842. wc->pmem_mode = true;
  1843. wc->writeback_fua = true;
  1844. #else
  1845. /*
  1846. * If the architecture doesn't support persistent memory or
  1847. * the kernel doesn't support any DAX drivers, this driver can
  1848. * only be used in SSD-only mode.
  1849. */
  1850. r = -EOPNOTSUPP;
  1851. ti->error = "Persistent memory or DAX not supported on this system";
  1852. goto bad;
  1853. #endif
  1854. } else {
  1855. goto bad_arguments;
  1856. }
  1857. if (WC_MODE_PMEM(wc)) {
  1858. r = bioset_init(&wc->bio_set, BIO_POOL_SIZE,
  1859. offsetof(struct writeback_struct, bio),
  1860. BIOSET_NEED_BVECS);
  1861. if (r) {
  1862. ti->error = "Could not allocate bio set";
  1863. goto bad;
  1864. }
  1865. } else {
  1866. r = mempool_init_kmalloc_pool(&wc->copy_pool, 1, sizeof(struct copy_struct));
  1867. if (r) {
  1868. ti->error = "Could not allocate mempool";
  1869. goto bad;
  1870. }
  1871. }
  1872. /*
  1873. * Parse the origin data device
  1874. */
  1875. string = dm_shift_arg(&as);
  1876. if (!string)
  1877. goto bad_arguments;
  1878. r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->dev);
  1879. if (r) {
  1880. ti->error = "Origin data device lookup failed";
  1881. goto bad;
  1882. }
  1883. /*
  1884. * Parse cache data device (be it pmem or ssd)
  1885. */
  1886. string = dm_shift_arg(&as);
  1887. if (!string)
  1888. goto bad_arguments;
  1889. r = dm_get_device(ti, string, dm_table_get_mode(ti->table), &wc->ssd_dev);
  1890. if (r) {
  1891. ti->error = "Cache data device lookup failed";
  1892. goto bad;
  1893. }
  1894. wc->memory_map_size = i_size_read(wc->ssd_dev->bdev->bd_inode);
  1895. /*
  1896. * Parse the cache block size
  1897. */
  1898. string = dm_shift_arg(&as);
  1899. if (!string)
  1900. goto bad_arguments;
  1901. if (sscanf(string, "%u%c", &wc->block_size, &dummy) != 1 ||
  1902. wc->block_size < 512 || wc->block_size > PAGE_SIZE ||
  1903. (wc->block_size & (wc->block_size - 1))) {
  1904. r = -EINVAL;
  1905. ti->error = "Invalid block size";
  1906. goto bad;
  1907. }
  1908. if (wc->block_size < bdev_logical_block_size(wc->dev->bdev) ||
  1909. wc->block_size < bdev_logical_block_size(wc->ssd_dev->bdev)) {
  1910. r = -EINVAL;
  1911. ti->error = "Block size is smaller than device logical block size";
  1912. goto bad;
  1913. }
  1914. wc->block_size_bits = __ffs(wc->block_size);
  1915. wc->max_writeback_jobs = MAX_WRITEBACK_JOBS;
  1916. wc->autocommit_blocks = !WC_MODE_PMEM(wc) ? AUTOCOMMIT_BLOCKS_SSD : AUTOCOMMIT_BLOCKS_PMEM;
  1917. wc->autocommit_jiffies = msecs_to_jiffies(AUTOCOMMIT_MSEC);
  1918. /*
  1919. * Parse optional arguments
  1920. */
  1921. r = dm_read_arg_group(_args, &as, &opt_params, &ti->error);
  1922. if (r)
  1923. goto bad;
  1924. while (opt_params) {
  1925. string = dm_shift_arg(&as), opt_params--;
  1926. if (!strcasecmp(string, "start_sector") && opt_params >= 1) {
  1927. unsigned long long start_sector;
  1928. string = dm_shift_arg(&as), opt_params--;
  1929. if (sscanf(string, "%llu%c", &start_sector, &dummy) != 1)
  1930. goto invalid_optional;
  1931. wc->start_sector = start_sector;
  1932. wc->start_sector_set = true;
  1933. if (wc->start_sector != start_sector ||
  1934. wc->start_sector >= wc->memory_map_size >> SECTOR_SHIFT)
  1935. goto invalid_optional;
  1936. } else if (!strcasecmp(string, "high_watermark") && opt_params >= 1) {
  1937. string = dm_shift_arg(&as), opt_params--;
  1938. if (sscanf(string, "%d%c", &high_wm_percent, &dummy) != 1)
  1939. goto invalid_optional;
  1940. if (high_wm_percent < 0 || high_wm_percent > 100)
  1941. goto invalid_optional;
  1942. wc->high_wm_percent_value = high_wm_percent;
  1943. wc->high_wm_percent_set = true;
  1944. } else if (!strcasecmp(string, "low_watermark") && opt_params >= 1) {
  1945. string = dm_shift_arg(&as), opt_params--;
  1946. if (sscanf(string, "%d%c", &low_wm_percent, &dummy) != 1)
  1947. goto invalid_optional;
  1948. if (low_wm_percent < 0 || low_wm_percent > 100)
  1949. goto invalid_optional;
  1950. wc->low_wm_percent_value = low_wm_percent;
  1951. wc->low_wm_percent_set = true;
  1952. } else if (!strcasecmp(string, "writeback_jobs") && opt_params >= 1) {
  1953. string = dm_shift_arg(&as), opt_params--;
  1954. if (sscanf(string, "%u%c", &wc->max_writeback_jobs, &dummy) != 1)
  1955. goto invalid_optional;
  1956. wc->max_writeback_jobs_set = true;
  1957. } else if (!strcasecmp(string, "autocommit_blocks") && opt_params >= 1) {
  1958. string = dm_shift_arg(&as), opt_params--;
  1959. if (sscanf(string, "%u%c", &wc->autocommit_blocks, &dummy) != 1)
  1960. goto invalid_optional;
  1961. wc->autocommit_blocks_set = true;
  1962. } else if (!strcasecmp(string, "autocommit_time") && opt_params >= 1) {
  1963. unsigned autocommit_msecs;
  1964. string = dm_shift_arg(&as), opt_params--;
  1965. if (sscanf(string, "%u%c", &autocommit_msecs, &dummy) != 1)
  1966. goto invalid_optional;
  1967. if (autocommit_msecs > 3600000)
  1968. goto invalid_optional;
  1969. wc->autocommit_jiffies = msecs_to_jiffies(autocommit_msecs);
  1970. wc->autocommit_time_value = autocommit_msecs;
  1971. wc->autocommit_time_set = true;
  1972. } else if (!strcasecmp(string, "max_age") && opt_params >= 1) {
  1973. unsigned max_age_msecs;
  1974. string = dm_shift_arg(&as), opt_params--;
  1975. if (sscanf(string, "%u%c", &max_age_msecs, &dummy) != 1)
  1976. goto invalid_optional;
  1977. if (max_age_msecs > 86400000)
  1978. goto invalid_optional;
  1979. wc->max_age = msecs_to_jiffies(max_age_msecs);
  1980. wc->max_age_set = true;
  1981. wc->max_age_value = max_age_msecs;
  1982. } else if (!strcasecmp(string, "cleaner")) {
  1983. wc->cleaner_set = true;
  1984. wc->cleaner = true;
  1985. } else if (!strcasecmp(string, "fua")) {
  1986. if (WC_MODE_PMEM(wc)) {
  1987. wc->writeback_fua = true;
  1988. wc->writeback_fua_set = true;
  1989. } else goto invalid_optional;
  1990. } else if (!strcasecmp(string, "nofua")) {
  1991. if (WC_MODE_PMEM(wc)) {
  1992. wc->writeback_fua = false;
  1993. wc->writeback_fua_set = true;
  1994. } else goto invalid_optional;
  1995. } else {
  1996. invalid_optional:
  1997. r = -EINVAL;
  1998. ti->error = "Invalid optional argument";
  1999. goto bad;
  2000. }
  2001. }
  2002. if (high_wm_percent < low_wm_percent) {
  2003. r = -EINVAL;
  2004. ti->error = "High watermark must be greater than or equal to low watermark";
  2005. goto bad;
  2006. }
  2007. if (WC_MODE_PMEM(wc)) {
  2008. if (!dax_synchronous(wc->ssd_dev->dax_dev)) {
  2009. r = -EOPNOTSUPP;
  2010. ti->error = "Asynchronous persistent memory not supported as pmem cache";
  2011. goto bad;
  2012. }
  2013. r = persistent_memory_claim(wc);
  2014. if (r) {
  2015. ti->error = "Unable to map persistent memory for cache";
  2016. goto bad;
  2017. }
  2018. } else {
  2019. size_t n_blocks, n_metadata_blocks;
  2020. uint64_t n_bitmap_bits;
  2021. wc->memory_map_size -= (uint64_t)wc->start_sector << SECTOR_SHIFT;
  2022. bio_list_init(&wc->flush_list);
  2023. wc->flush_thread = kthread_create(writecache_flush_thread, wc, "dm_writecache_flush");
  2024. if (IS_ERR(wc->flush_thread)) {
  2025. r = PTR_ERR(wc->flush_thread);
  2026. wc->flush_thread = NULL;
  2027. ti->error = "Couldn't spawn flush thread";
  2028. goto bad;
  2029. }
  2030. wake_up_process(wc->flush_thread);
  2031. r = calculate_memory_size(wc->memory_map_size, wc->block_size,
  2032. &n_blocks, &n_metadata_blocks);
  2033. if (r) {
  2034. ti->error = "Invalid device size";
  2035. goto bad;
  2036. }
  2037. n_bitmap_bits = (((uint64_t)n_metadata_blocks << wc->block_size_bits) +
  2038. BITMAP_GRANULARITY - 1) / BITMAP_GRANULARITY;
  2039. /* this is limitation of test_bit functions */
  2040. if (n_bitmap_bits > 1U << 31) {
  2041. r = -EFBIG;
  2042. ti->error = "Invalid device size";
  2043. goto bad;
  2044. }
  2045. wc->memory_map = vmalloc(n_metadata_blocks << wc->block_size_bits);
  2046. if (!wc->memory_map) {
  2047. r = -ENOMEM;
  2048. ti->error = "Unable to allocate memory for metadata";
  2049. goto bad;
  2050. }
  2051. wc->dm_kcopyd = dm_kcopyd_client_create(&dm_kcopyd_throttle);
  2052. if (IS_ERR(wc->dm_kcopyd)) {
  2053. r = PTR_ERR(wc->dm_kcopyd);
  2054. ti->error = "Unable to allocate dm-kcopyd client";
  2055. wc->dm_kcopyd = NULL;
  2056. goto bad;
  2057. }
  2058. wc->metadata_sectors = n_metadata_blocks << (wc->block_size_bits - SECTOR_SHIFT);
  2059. wc->dirty_bitmap_size = (n_bitmap_bits + BITS_PER_LONG - 1) /
  2060. BITS_PER_LONG * sizeof(unsigned long);
  2061. wc->dirty_bitmap = vzalloc(wc->dirty_bitmap_size);
  2062. if (!wc->dirty_bitmap) {
  2063. r = -ENOMEM;
  2064. ti->error = "Unable to allocate dirty bitmap";
  2065. goto bad;
  2066. }
  2067. r = writecache_read_metadata(wc, wc->block_size >> SECTOR_SHIFT);
  2068. if (r) {
  2069. ti->error = "Unable to read first block of metadata";
  2070. goto bad;
  2071. }
  2072. }
  2073. r = copy_mc_to_kernel(&s, sb(wc), sizeof(struct wc_memory_superblock));
  2074. if (r) {
  2075. ti->error = "Hardware memory error when reading superblock";
  2076. goto bad;
  2077. }
  2078. if (!le32_to_cpu(s.magic) && !le32_to_cpu(s.version)) {
  2079. r = init_memory(wc);
  2080. if (r) {
  2081. ti->error = "Unable to initialize device";
  2082. goto bad;
  2083. }
  2084. r = copy_mc_to_kernel(&s, sb(wc),
  2085. sizeof(struct wc_memory_superblock));
  2086. if (r) {
  2087. ti->error = "Hardware memory error when reading superblock";
  2088. goto bad;
  2089. }
  2090. }
  2091. if (le32_to_cpu(s.magic) != MEMORY_SUPERBLOCK_MAGIC) {
  2092. ti->error = "Invalid magic in the superblock";
  2093. r = -EINVAL;
  2094. goto bad;
  2095. }
  2096. if (le32_to_cpu(s.version) != MEMORY_SUPERBLOCK_VERSION) {
  2097. ti->error = "Invalid version in the superblock";
  2098. r = -EINVAL;
  2099. goto bad;
  2100. }
  2101. if (le32_to_cpu(s.block_size) != wc->block_size) {
  2102. ti->error = "Block size does not match superblock";
  2103. r = -EINVAL;
  2104. goto bad;
  2105. }
  2106. wc->n_blocks = le64_to_cpu(s.n_blocks);
  2107. offset = wc->n_blocks * sizeof(struct wc_memory_entry);
  2108. if (offset / sizeof(struct wc_memory_entry) != le64_to_cpu(sb(wc)->n_blocks)) {
  2109. overflow:
  2110. ti->error = "Overflow in size calculation";
  2111. r = -EINVAL;
  2112. goto bad;
  2113. }
  2114. offset += sizeof(struct wc_memory_superblock);
  2115. if (offset < sizeof(struct wc_memory_superblock))
  2116. goto overflow;
  2117. offset = (offset + wc->block_size - 1) & ~(size_t)(wc->block_size - 1);
  2118. data_size = wc->n_blocks * (size_t)wc->block_size;
  2119. if (!offset || (data_size / wc->block_size != wc->n_blocks) ||
  2120. (offset + data_size < offset))
  2121. goto overflow;
  2122. if (offset + data_size > wc->memory_map_size) {
  2123. ti->error = "Memory area is too small";
  2124. r = -EINVAL;
  2125. goto bad;
  2126. }
  2127. wc->metadata_sectors = offset >> SECTOR_SHIFT;
  2128. wc->block_start = (char *)sb(wc) + offset;
  2129. x = (uint64_t)wc->n_blocks * (100 - high_wm_percent);
  2130. x += 50;
  2131. do_div(x, 100);
  2132. wc->freelist_high_watermark = x;
  2133. x = (uint64_t)wc->n_blocks * (100 - low_wm_percent);
  2134. x += 50;
  2135. do_div(x, 100);
  2136. wc->freelist_low_watermark = x;
  2137. if (wc->cleaner)
  2138. activate_cleaner(wc);
  2139. r = writecache_alloc_entries(wc);
  2140. if (r) {
  2141. ti->error = "Cannot allocate memory";
  2142. goto bad;
  2143. }
  2144. ti->num_flush_bios = WC_MODE_PMEM(wc) ? 1 : 2;
  2145. ti->flush_supported = true;
  2146. ti->num_discard_bios = 1;
  2147. if (WC_MODE_PMEM(wc))
  2148. persistent_memory_flush_cache(wc->memory_map, wc->memory_map_size);
  2149. return 0;
  2150. bad_arguments:
  2151. r = -EINVAL;
  2152. ti->error = "Bad arguments";
  2153. bad:
  2154. writecache_dtr(ti);
  2155. return r;
  2156. }
  2157. static void writecache_status(struct dm_target *ti, status_type_t type,
  2158. unsigned status_flags, char *result, unsigned maxlen)
  2159. {
  2160. struct dm_writecache *wc = ti->private;
  2161. unsigned extra_args;
  2162. unsigned sz = 0;
  2163. switch (type) {
  2164. case STATUSTYPE_INFO:
  2165. DMEMIT("%ld %llu %llu %llu", writecache_has_error(wc),
  2166. (unsigned long long)wc->n_blocks, (unsigned long long)wc->freelist_size,
  2167. (unsigned long long)wc->writeback_size);
  2168. break;
  2169. case STATUSTYPE_TABLE:
  2170. DMEMIT("%c %s %s %u ", WC_MODE_PMEM(wc) ? 'p' : 's',
  2171. wc->dev->name, wc->ssd_dev->name, wc->block_size);
  2172. extra_args = 0;
  2173. if (wc->start_sector_set)
  2174. extra_args += 2;
  2175. if (wc->high_wm_percent_set)
  2176. extra_args += 2;
  2177. if (wc->low_wm_percent_set)
  2178. extra_args += 2;
  2179. if (wc->max_writeback_jobs_set)
  2180. extra_args += 2;
  2181. if (wc->autocommit_blocks_set)
  2182. extra_args += 2;
  2183. if (wc->autocommit_time_set)
  2184. extra_args += 2;
  2185. if (wc->max_age_set)
  2186. extra_args += 2;
  2187. if (wc->cleaner_set)
  2188. extra_args++;
  2189. if (wc->writeback_fua_set)
  2190. extra_args++;
  2191. DMEMIT("%u", extra_args);
  2192. if (wc->start_sector_set)
  2193. DMEMIT(" start_sector %llu", (unsigned long long)wc->start_sector);
  2194. if (wc->high_wm_percent_set)
  2195. DMEMIT(" high_watermark %u", wc->high_wm_percent_value);
  2196. if (wc->low_wm_percent_set)
  2197. DMEMIT(" low_watermark %u", wc->low_wm_percent_value);
  2198. if (wc->max_writeback_jobs_set)
  2199. DMEMIT(" writeback_jobs %u", wc->max_writeback_jobs);
  2200. if (wc->autocommit_blocks_set)
  2201. DMEMIT(" autocommit_blocks %u", wc->autocommit_blocks);
  2202. if (wc->autocommit_time_set)
  2203. DMEMIT(" autocommit_time %u", wc->autocommit_time_value);
  2204. if (wc->max_age_set)
  2205. DMEMIT(" max_age %u", wc->max_age_value);
  2206. if (wc->cleaner_set)
  2207. DMEMIT(" cleaner");
  2208. if (wc->writeback_fua_set)
  2209. DMEMIT(" %sfua", wc->writeback_fua ? "" : "no");
  2210. break;
  2211. }
  2212. }
  2213. static struct target_type writecache_target = {
  2214. .name = "writecache",
  2215. .version = {1, 4, 0},
  2216. .module = THIS_MODULE,
  2217. .ctr = writecache_ctr,
  2218. .dtr = writecache_dtr,
  2219. .status = writecache_status,
  2220. .postsuspend = writecache_suspend,
  2221. .resume = writecache_resume,
  2222. .message = writecache_message,
  2223. .map = writecache_map,
  2224. .end_io = writecache_end_io,
  2225. .iterate_devices = writecache_iterate_devices,
  2226. .io_hints = writecache_io_hints,
  2227. };
  2228. static int __init dm_writecache_init(void)
  2229. {
  2230. int r;
  2231. r = dm_register_target(&writecache_target);
  2232. if (r < 0) {
  2233. DMERR("register failed %d", r);
  2234. return r;
  2235. }
  2236. return 0;
  2237. }
  2238. static void __exit dm_writecache_exit(void)
  2239. {
  2240. dm_unregister_target(&writecache_target);
  2241. }
  2242. module_init(dm_writecache_init);
  2243. module_exit(dm_writecache_exit);
  2244. MODULE_DESCRIPTION(DM_NAME " writecache target");
  2245. MODULE_AUTHOR("Mikulas Patocka <dm-devel@redhat.com>");
  2246. MODULE_LICENSE("GPL");