dax.c 35 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * dax: direct host memory access
  4. * Copyright (C) 2020 Red Hat, Inc.
  5. */
  6. #include "fuse_i.h"
  7. #include <linux/delay.h>
  8. #include <linux/dax.h>
  9. #include <linux/uio.h>
  10. #include <linux/pfn_t.h>
  11. #include <linux/iomap.h>
  12. #include <linux/interval_tree.h>
  13. /*
  14. * Default memory range size. A power of 2 so it agrees with common FUSE_INIT
  15. * map_alignment values 4KB and 64KB.
  16. */
  17. #define FUSE_DAX_SHIFT 21
  18. #define FUSE_DAX_SZ (1 << FUSE_DAX_SHIFT)
  19. #define FUSE_DAX_PAGES (FUSE_DAX_SZ / PAGE_SIZE)
  20. /* Number of ranges reclaimer will try to free in one invocation */
  21. #define FUSE_DAX_RECLAIM_CHUNK (10)
  22. /*
  23. * Dax memory reclaim threshold in percetage of total ranges. When free
  24. * number of free ranges drops below this threshold, reclaim can trigger
  25. * Default is 20%
  26. */
  27. #define FUSE_DAX_RECLAIM_THRESHOLD (20)
  28. /** Translation information for file offsets to DAX window offsets */
  29. struct fuse_dax_mapping {
  30. /* Pointer to inode where this memory range is mapped */
  31. struct inode *inode;
  32. /* Will connect in fcd->free_ranges to keep track of free memory */
  33. struct list_head list;
  34. /* For interval tree in file/inode */
  35. struct interval_tree_node itn;
  36. /* Will connect in fc->busy_ranges to keep track busy memory */
  37. struct list_head busy_list;
  38. /** Position in DAX window */
  39. u64 window_offset;
  40. /** Length of mapping, in bytes */
  41. loff_t length;
  42. /* Is this mapping read-only or read-write */
  43. bool writable;
  44. /* reference count when the mapping is used by dax iomap. */
  45. refcount_t refcnt;
  46. };
  47. /* Per-inode dax map */
  48. struct fuse_inode_dax {
  49. /* Semaphore to protect modifications to the dmap tree */
  50. struct rw_semaphore sem;
  51. /* Sorted rb tree of struct fuse_dax_mapping elements */
  52. struct rb_root_cached tree;
  53. unsigned long nr;
  54. };
  55. struct fuse_conn_dax {
  56. /* DAX device */
  57. struct dax_device *dev;
  58. /* Lock protecting accessess to members of this structure */
  59. spinlock_t lock;
  60. /* List of memory ranges which are busy */
  61. unsigned long nr_busy_ranges;
  62. struct list_head busy_ranges;
  63. /* Worker to free up memory ranges */
  64. struct delayed_work free_work;
  65. /* Wait queue for a dax range to become free */
  66. wait_queue_head_t range_waitq;
  67. /* DAX Window Free Ranges */
  68. long nr_free_ranges;
  69. struct list_head free_ranges;
  70. unsigned long nr_ranges;
  71. };
  72. static inline struct fuse_dax_mapping *
  73. node_to_dmap(struct interval_tree_node *node)
  74. {
  75. if (!node)
  76. return NULL;
  77. return container_of(node, struct fuse_dax_mapping, itn);
  78. }
  79. static struct fuse_dax_mapping *
  80. alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode);
  81. static void
  82. __kick_dmap_free_worker(struct fuse_conn_dax *fcd, unsigned long delay_ms)
  83. {
  84. unsigned long free_threshold;
  85. /* If number of free ranges are below threshold, start reclaim */
  86. free_threshold = max_t(unsigned long, fcd->nr_ranges * FUSE_DAX_RECLAIM_THRESHOLD / 100,
  87. 1);
  88. if (fcd->nr_free_ranges < free_threshold)
  89. queue_delayed_work(system_long_wq, &fcd->free_work,
  90. msecs_to_jiffies(delay_ms));
  91. }
  92. static void kick_dmap_free_worker(struct fuse_conn_dax *fcd,
  93. unsigned long delay_ms)
  94. {
  95. spin_lock(&fcd->lock);
  96. __kick_dmap_free_worker(fcd, delay_ms);
  97. spin_unlock(&fcd->lock);
  98. }
  99. static struct fuse_dax_mapping *alloc_dax_mapping(struct fuse_conn_dax *fcd)
  100. {
  101. struct fuse_dax_mapping *dmap;
  102. spin_lock(&fcd->lock);
  103. dmap = list_first_entry_or_null(&fcd->free_ranges,
  104. struct fuse_dax_mapping, list);
  105. if (dmap) {
  106. list_del_init(&dmap->list);
  107. WARN_ON(fcd->nr_free_ranges <= 0);
  108. fcd->nr_free_ranges--;
  109. }
  110. spin_unlock(&fcd->lock);
  111. kick_dmap_free_worker(fcd, 0);
  112. return dmap;
  113. }
  114. /* This assumes fcd->lock is held */
  115. static void __dmap_remove_busy_list(struct fuse_conn_dax *fcd,
  116. struct fuse_dax_mapping *dmap)
  117. {
  118. list_del_init(&dmap->busy_list);
  119. WARN_ON(fcd->nr_busy_ranges == 0);
  120. fcd->nr_busy_ranges--;
  121. }
  122. static void dmap_remove_busy_list(struct fuse_conn_dax *fcd,
  123. struct fuse_dax_mapping *dmap)
  124. {
  125. spin_lock(&fcd->lock);
  126. __dmap_remove_busy_list(fcd, dmap);
  127. spin_unlock(&fcd->lock);
  128. }
  129. /* This assumes fcd->lock is held */
  130. static void __dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
  131. struct fuse_dax_mapping *dmap)
  132. {
  133. list_add_tail(&dmap->list, &fcd->free_ranges);
  134. fcd->nr_free_ranges++;
  135. wake_up(&fcd->range_waitq);
  136. }
  137. static void dmap_add_to_free_pool(struct fuse_conn_dax *fcd,
  138. struct fuse_dax_mapping *dmap)
  139. {
  140. /* Return fuse_dax_mapping to free list */
  141. spin_lock(&fcd->lock);
  142. __dmap_add_to_free_pool(fcd, dmap);
  143. spin_unlock(&fcd->lock);
  144. }
  145. static int fuse_setup_one_mapping(struct inode *inode, unsigned long start_idx,
  146. struct fuse_dax_mapping *dmap, bool writable,
  147. bool upgrade)
  148. {
  149. struct fuse_mount *fm = get_fuse_mount(inode);
  150. struct fuse_conn_dax *fcd = fm->fc->dax;
  151. struct fuse_inode *fi = get_fuse_inode(inode);
  152. struct fuse_setupmapping_in inarg;
  153. loff_t offset = start_idx << FUSE_DAX_SHIFT;
  154. FUSE_ARGS(args);
  155. ssize_t err;
  156. WARN_ON(fcd->nr_free_ranges < 0);
  157. /* Ask fuse daemon to setup mapping */
  158. memset(&inarg, 0, sizeof(inarg));
  159. inarg.foffset = offset;
  160. inarg.fh = -1;
  161. inarg.moffset = dmap->window_offset;
  162. inarg.len = FUSE_DAX_SZ;
  163. inarg.flags |= FUSE_SETUPMAPPING_FLAG_READ;
  164. if (writable)
  165. inarg.flags |= FUSE_SETUPMAPPING_FLAG_WRITE;
  166. args.opcode = FUSE_SETUPMAPPING;
  167. args.nodeid = fi->nodeid;
  168. args.in_numargs = 1;
  169. args.in_args[0].size = sizeof(inarg);
  170. args.in_args[0].value = &inarg;
  171. err = fuse_simple_request(fm, &args);
  172. if (err < 0)
  173. return err;
  174. dmap->writable = writable;
  175. if (!upgrade) {
  176. /*
  177. * We don't take a refernce on inode. inode is valid right now
  178. * and when inode is going away, cleanup logic should first
  179. * cleanup dmap entries.
  180. */
  181. dmap->inode = inode;
  182. dmap->itn.start = dmap->itn.last = start_idx;
  183. /* Protected by fi->dax->sem */
  184. interval_tree_insert(&dmap->itn, &fi->dax->tree);
  185. fi->dax->nr++;
  186. spin_lock(&fcd->lock);
  187. list_add_tail(&dmap->busy_list, &fcd->busy_ranges);
  188. fcd->nr_busy_ranges++;
  189. spin_unlock(&fcd->lock);
  190. }
  191. return 0;
  192. }
  193. static int fuse_send_removemapping(struct inode *inode,
  194. struct fuse_removemapping_in *inargp,
  195. struct fuse_removemapping_one *remove_one)
  196. {
  197. struct fuse_inode *fi = get_fuse_inode(inode);
  198. struct fuse_mount *fm = get_fuse_mount(inode);
  199. FUSE_ARGS(args);
  200. args.opcode = FUSE_REMOVEMAPPING;
  201. args.nodeid = fi->nodeid;
  202. args.in_numargs = 2;
  203. args.in_args[0].size = sizeof(*inargp);
  204. args.in_args[0].value = inargp;
  205. args.in_args[1].size = inargp->count * sizeof(*remove_one);
  206. args.in_args[1].value = remove_one;
  207. return fuse_simple_request(fm, &args);
  208. }
  209. static int dmap_removemapping_list(struct inode *inode, unsigned int num,
  210. struct list_head *to_remove)
  211. {
  212. struct fuse_removemapping_one *remove_one, *ptr;
  213. struct fuse_removemapping_in inarg;
  214. struct fuse_dax_mapping *dmap;
  215. int ret, i = 0, nr_alloc;
  216. nr_alloc = min_t(unsigned int, num, FUSE_REMOVEMAPPING_MAX_ENTRY);
  217. remove_one = kmalloc_array(nr_alloc, sizeof(*remove_one), GFP_NOFS);
  218. if (!remove_one)
  219. return -ENOMEM;
  220. ptr = remove_one;
  221. list_for_each_entry(dmap, to_remove, list) {
  222. ptr->moffset = dmap->window_offset;
  223. ptr->len = dmap->length;
  224. ptr++;
  225. i++;
  226. num--;
  227. if (i >= nr_alloc || num == 0) {
  228. memset(&inarg, 0, sizeof(inarg));
  229. inarg.count = i;
  230. ret = fuse_send_removemapping(inode, &inarg,
  231. remove_one);
  232. if (ret)
  233. goto out;
  234. ptr = remove_one;
  235. i = 0;
  236. }
  237. }
  238. out:
  239. kfree(remove_one);
  240. return ret;
  241. }
  242. /*
  243. * Cleanup dmap entry and add back to free list. This should be called with
  244. * fcd->lock held.
  245. */
  246. static void dmap_reinit_add_to_free_pool(struct fuse_conn_dax *fcd,
  247. struct fuse_dax_mapping *dmap)
  248. {
  249. pr_debug("fuse: freeing memory range start_idx=0x%lx end_idx=0x%lx window_offset=0x%llx length=0x%llx\n",
  250. dmap->itn.start, dmap->itn.last, dmap->window_offset,
  251. dmap->length);
  252. __dmap_remove_busy_list(fcd, dmap);
  253. dmap->inode = NULL;
  254. dmap->itn.start = dmap->itn.last = 0;
  255. __dmap_add_to_free_pool(fcd, dmap);
  256. }
  257. /*
  258. * Free inode dmap entries whose range falls inside [start, end].
  259. * Does not take any locks. At this point of time it should only be
  260. * called from evict_inode() path where we know all dmap entries can be
  261. * reclaimed.
  262. */
  263. static void inode_reclaim_dmap_range(struct fuse_conn_dax *fcd,
  264. struct inode *inode,
  265. loff_t start, loff_t end)
  266. {
  267. struct fuse_inode *fi = get_fuse_inode(inode);
  268. struct fuse_dax_mapping *dmap, *n;
  269. int err, num = 0;
  270. LIST_HEAD(to_remove);
  271. unsigned long start_idx = start >> FUSE_DAX_SHIFT;
  272. unsigned long end_idx = end >> FUSE_DAX_SHIFT;
  273. struct interval_tree_node *node;
  274. while (1) {
  275. node = interval_tree_iter_first(&fi->dax->tree, start_idx,
  276. end_idx);
  277. if (!node)
  278. break;
  279. dmap = node_to_dmap(node);
  280. /* inode is going away. There should not be any users of dmap */
  281. WARN_ON(refcount_read(&dmap->refcnt) > 1);
  282. interval_tree_remove(&dmap->itn, &fi->dax->tree);
  283. num++;
  284. list_add(&dmap->list, &to_remove);
  285. }
  286. /* Nothing to remove */
  287. if (list_empty(&to_remove))
  288. return;
  289. WARN_ON(fi->dax->nr < num);
  290. fi->dax->nr -= num;
  291. err = dmap_removemapping_list(inode, num, &to_remove);
  292. if (err && err != -ENOTCONN) {
  293. pr_warn("Failed to removemappings. start=0x%llx end=0x%llx\n",
  294. start, end);
  295. }
  296. spin_lock(&fcd->lock);
  297. list_for_each_entry_safe(dmap, n, &to_remove, list) {
  298. list_del_init(&dmap->list);
  299. dmap_reinit_add_to_free_pool(fcd, dmap);
  300. }
  301. spin_unlock(&fcd->lock);
  302. }
  303. static int dmap_removemapping_one(struct inode *inode,
  304. struct fuse_dax_mapping *dmap)
  305. {
  306. struct fuse_removemapping_one forget_one;
  307. struct fuse_removemapping_in inarg;
  308. memset(&inarg, 0, sizeof(inarg));
  309. inarg.count = 1;
  310. memset(&forget_one, 0, sizeof(forget_one));
  311. forget_one.moffset = dmap->window_offset;
  312. forget_one.len = dmap->length;
  313. return fuse_send_removemapping(inode, &inarg, &forget_one);
  314. }
  315. /*
  316. * It is called from evict_inode() and by that time inode is going away. So
  317. * this function does not take any locks like fi->dax->sem for traversing
  318. * that fuse inode interval tree. If that lock is taken then lock validator
  319. * complains of deadlock situation w.r.t fs_reclaim lock.
  320. */
  321. void fuse_dax_inode_cleanup(struct inode *inode)
  322. {
  323. struct fuse_conn *fc = get_fuse_conn(inode);
  324. struct fuse_inode *fi = get_fuse_inode(inode);
  325. /*
  326. * fuse_evict_inode() has already called truncate_inode_pages_final()
  327. * before we arrive here. So we should not have to worry about any
  328. * pages/exception entries still associated with inode.
  329. */
  330. inode_reclaim_dmap_range(fc->dax, inode, 0, -1);
  331. WARN_ON(fi->dax->nr);
  332. }
  333. static void fuse_fill_iomap_hole(struct iomap *iomap, loff_t length)
  334. {
  335. iomap->addr = IOMAP_NULL_ADDR;
  336. iomap->length = length;
  337. iomap->type = IOMAP_HOLE;
  338. }
  339. static void fuse_fill_iomap(struct inode *inode, loff_t pos, loff_t length,
  340. struct iomap *iomap, struct fuse_dax_mapping *dmap,
  341. unsigned int flags)
  342. {
  343. loff_t offset, len;
  344. loff_t i_size = i_size_read(inode);
  345. offset = pos - (dmap->itn.start << FUSE_DAX_SHIFT);
  346. len = min(length, dmap->length - offset);
  347. /* If length is beyond end of file, truncate further */
  348. if (pos + len > i_size)
  349. len = i_size - pos;
  350. if (len > 0) {
  351. iomap->addr = dmap->window_offset + offset;
  352. iomap->length = len;
  353. if (flags & IOMAP_FAULT)
  354. iomap->length = ALIGN(len, PAGE_SIZE);
  355. iomap->type = IOMAP_MAPPED;
  356. /*
  357. * increace refcnt so that reclaim code knows this dmap is in
  358. * use. This assumes fi->dax->sem mutex is held either
  359. * shared/exclusive.
  360. */
  361. refcount_inc(&dmap->refcnt);
  362. /* iomap->private should be NULL */
  363. WARN_ON_ONCE(iomap->private);
  364. iomap->private = dmap;
  365. } else {
  366. /* Mapping beyond end of file is hole */
  367. fuse_fill_iomap_hole(iomap, length);
  368. }
  369. }
  370. static int fuse_setup_new_dax_mapping(struct inode *inode, loff_t pos,
  371. loff_t length, unsigned int flags,
  372. struct iomap *iomap)
  373. {
  374. struct fuse_inode *fi = get_fuse_inode(inode);
  375. struct fuse_conn *fc = get_fuse_conn(inode);
  376. struct fuse_conn_dax *fcd = fc->dax;
  377. struct fuse_dax_mapping *dmap, *alloc_dmap = NULL;
  378. int ret;
  379. bool writable = flags & IOMAP_WRITE;
  380. unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
  381. struct interval_tree_node *node;
  382. /*
  383. * Can't do inline reclaim in fault path. We call
  384. * dax_layout_busy_page() before we free a range. And
  385. * fuse_wait_dax_page() drops fi->i_mmap_sem lock and requires it.
  386. * In fault path we enter with fi->i_mmap_sem held and can't drop
  387. * it. Also in fault path we hold fi->i_mmap_sem shared and not
  388. * exclusive, so that creates further issues with fuse_wait_dax_page().
  389. * Hence return -EAGAIN and fuse_dax_fault() will wait for a memory
  390. * range to become free and retry.
  391. */
  392. if (flags & IOMAP_FAULT) {
  393. alloc_dmap = alloc_dax_mapping(fcd);
  394. if (!alloc_dmap)
  395. return -EAGAIN;
  396. } else {
  397. alloc_dmap = alloc_dax_mapping_reclaim(fcd, inode);
  398. if (IS_ERR(alloc_dmap))
  399. return PTR_ERR(alloc_dmap);
  400. }
  401. /* If we are here, we should have memory allocated */
  402. if (WARN_ON(!alloc_dmap))
  403. return -EIO;
  404. /*
  405. * Take write lock so that only one caller can try to setup mapping
  406. * and other waits.
  407. */
  408. down_write(&fi->dax->sem);
  409. /*
  410. * We dropped lock. Check again if somebody else setup
  411. * mapping already.
  412. */
  413. node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
  414. if (node) {
  415. dmap = node_to_dmap(node);
  416. fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
  417. dmap_add_to_free_pool(fcd, alloc_dmap);
  418. up_write(&fi->dax->sem);
  419. return 0;
  420. }
  421. /* Setup one mapping */
  422. ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, alloc_dmap,
  423. writable, false);
  424. if (ret < 0) {
  425. dmap_add_to_free_pool(fcd, alloc_dmap);
  426. up_write(&fi->dax->sem);
  427. return ret;
  428. }
  429. fuse_fill_iomap(inode, pos, length, iomap, alloc_dmap, flags);
  430. up_write(&fi->dax->sem);
  431. return 0;
  432. }
  433. static int fuse_upgrade_dax_mapping(struct inode *inode, loff_t pos,
  434. loff_t length, unsigned int flags,
  435. struct iomap *iomap)
  436. {
  437. struct fuse_inode *fi = get_fuse_inode(inode);
  438. struct fuse_dax_mapping *dmap;
  439. int ret;
  440. unsigned long idx = pos >> FUSE_DAX_SHIFT;
  441. struct interval_tree_node *node;
  442. /*
  443. * Take exclusive lock so that only one caller can try to setup
  444. * mapping and others wait.
  445. */
  446. down_write(&fi->dax->sem);
  447. node = interval_tree_iter_first(&fi->dax->tree, idx, idx);
  448. /* We are holding either inode lock or i_mmap_sem, and that should
  449. * ensure that dmap can't be truncated. We are holding a reference
  450. * on dmap and that should make sure it can't be reclaimed. So dmap
  451. * should still be there in tree despite the fact we dropped and
  452. * re-acquired the fi->dax->sem lock.
  453. */
  454. ret = -EIO;
  455. if (WARN_ON(!node))
  456. goto out_err;
  457. dmap = node_to_dmap(node);
  458. /* We took an extra reference on dmap to make sure its not reclaimd.
  459. * Now we hold fi->dax->sem lock and that reference is not needed
  460. * anymore. Drop it.
  461. */
  462. if (refcount_dec_and_test(&dmap->refcnt)) {
  463. /* refcount should not hit 0. This object only goes
  464. * away when fuse connection goes away
  465. */
  466. WARN_ON_ONCE(1);
  467. }
  468. /* Maybe another thread already upgraded mapping while we were not
  469. * holding lock.
  470. */
  471. if (dmap->writable) {
  472. ret = 0;
  473. goto out_fill_iomap;
  474. }
  475. ret = fuse_setup_one_mapping(inode, pos >> FUSE_DAX_SHIFT, dmap, true,
  476. true);
  477. if (ret < 0)
  478. goto out_err;
  479. out_fill_iomap:
  480. fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
  481. out_err:
  482. up_write(&fi->dax->sem);
  483. return ret;
  484. }
  485. /* This is just for DAX and the mapping is ephemeral, do not use it for other
  486. * purposes since there is no block device with a permanent mapping.
  487. */
  488. static int fuse_iomap_begin(struct inode *inode, loff_t pos, loff_t length,
  489. unsigned int flags, struct iomap *iomap,
  490. struct iomap *srcmap)
  491. {
  492. struct fuse_inode *fi = get_fuse_inode(inode);
  493. struct fuse_conn *fc = get_fuse_conn(inode);
  494. struct fuse_dax_mapping *dmap;
  495. bool writable = flags & IOMAP_WRITE;
  496. unsigned long start_idx = pos >> FUSE_DAX_SHIFT;
  497. struct interval_tree_node *node;
  498. /* We don't support FIEMAP */
  499. if (WARN_ON(flags & IOMAP_REPORT))
  500. return -EIO;
  501. iomap->offset = pos;
  502. iomap->flags = 0;
  503. iomap->bdev = NULL;
  504. iomap->dax_dev = fc->dax->dev;
  505. /*
  506. * Both read/write and mmap path can race here. So we need something
  507. * to make sure if we are setting up mapping, then other path waits
  508. *
  509. * For now, use a semaphore for this. It probably needs to be
  510. * optimized later.
  511. */
  512. down_read(&fi->dax->sem);
  513. node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
  514. if (node) {
  515. dmap = node_to_dmap(node);
  516. if (writable && !dmap->writable) {
  517. /* Upgrade read-only mapping to read-write. This will
  518. * require exclusive fi->dax->sem lock as we don't want
  519. * two threads to be trying to this simultaneously
  520. * for same dmap. So drop shared lock and acquire
  521. * exclusive lock.
  522. *
  523. * Before dropping fi->dax->sem lock, take reference
  524. * on dmap so that its not freed by range reclaim.
  525. */
  526. refcount_inc(&dmap->refcnt);
  527. up_read(&fi->dax->sem);
  528. pr_debug("%s: Upgrading mapping at offset 0x%llx length 0x%llx\n",
  529. __func__, pos, length);
  530. return fuse_upgrade_dax_mapping(inode, pos, length,
  531. flags, iomap);
  532. } else {
  533. fuse_fill_iomap(inode, pos, length, iomap, dmap, flags);
  534. up_read(&fi->dax->sem);
  535. return 0;
  536. }
  537. } else {
  538. up_read(&fi->dax->sem);
  539. pr_debug("%s: no mapping at offset 0x%llx length 0x%llx\n",
  540. __func__, pos, length);
  541. if (pos >= i_size_read(inode))
  542. goto iomap_hole;
  543. return fuse_setup_new_dax_mapping(inode, pos, length, flags,
  544. iomap);
  545. }
  546. /*
  547. * If read beyond end of file happnes, fs code seems to return
  548. * it as hole
  549. */
  550. iomap_hole:
  551. fuse_fill_iomap_hole(iomap, length);
  552. pr_debug("%s returning hole mapping. pos=0x%llx length_asked=0x%llx length_returned=0x%llx\n",
  553. __func__, pos, length, iomap->length);
  554. return 0;
  555. }
  556. static int fuse_iomap_end(struct inode *inode, loff_t pos, loff_t length,
  557. ssize_t written, unsigned int flags,
  558. struct iomap *iomap)
  559. {
  560. struct fuse_dax_mapping *dmap = iomap->private;
  561. if (dmap) {
  562. if (refcount_dec_and_test(&dmap->refcnt)) {
  563. /* refcount should not hit 0. This object only goes
  564. * away when fuse connection goes away
  565. */
  566. WARN_ON_ONCE(1);
  567. }
  568. }
  569. /* DAX writes beyond end-of-file aren't handled using iomap, so the
  570. * file size is unchanged and there is nothing to do here.
  571. */
  572. return 0;
  573. }
  574. static const struct iomap_ops fuse_iomap_ops = {
  575. .iomap_begin = fuse_iomap_begin,
  576. .iomap_end = fuse_iomap_end,
  577. };
  578. static void fuse_wait_dax_page(struct inode *inode)
  579. {
  580. struct fuse_inode *fi = get_fuse_inode(inode);
  581. up_write(&fi->i_mmap_sem);
  582. schedule();
  583. down_write(&fi->i_mmap_sem);
  584. }
  585. /* Should be called with fi->i_mmap_sem lock held exclusively */
  586. static int __fuse_dax_break_layouts(struct inode *inode, bool *retry,
  587. loff_t start, loff_t end)
  588. {
  589. struct page *page;
  590. page = dax_layout_busy_page_range(inode->i_mapping, start, end);
  591. if (!page)
  592. return 0;
  593. *retry = true;
  594. return ___wait_var_event(&page->_refcount,
  595. atomic_read(&page->_refcount) == 1, TASK_INTERRUPTIBLE,
  596. 0, 0, fuse_wait_dax_page(inode));
  597. }
  598. /* dmap_end == 0 leads to unmapping of whole file */
  599. int fuse_dax_break_layouts(struct inode *inode, u64 dmap_start,
  600. u64 dmap_end)
  601. {
  602. bool retry;
  603. int ret;
  604. do {
  605. retry = false;
  606. ret = __fuse_dax_break_layouts(inode, &retry, dmap_start,
  607. dmap_end);
  608. } while (ret == 0 && retry);
  609. return ret;
  610. }
  611. ssize_t fuse_dax_read_iter(struct kiocb *iocb, struct iov_iter *to)
  612. {
  613. struct inode *inode = file_inode(iocb->ki_filp);
  614. ssize_t ret;
  615. if (iocb->ki_flags & IOCB_NOWAIT) {
  616. if (!inode_trylock_shared(inode))
  617. return -EAGAIN;
  618. } else {
  619. inode_lock_shared(inode);
  620. }
  621. ret = dax_iomap_rw(iocb, to, &fuse_iomap_ops);
  622. inode_unlock_shared(inode);
  623. /* TODO file_accessed(iocb->f_filp) */
  624. return ret;
  625. }
  626. static bool file_extending_write(struct kiocb *iocb, struct iov_iter *from)
  627. {
  628. struct inode *inode = file_inode(iocb->ki_filp);
  629. return (iov_iter_rw(from) == WRITE &&
  630. ((iocb->ki_pos) >= i_size_read(inode) ||
  631. (iocb->ki_pos + iov_iter_count(from) > i_size_read(inode))));
  632. }
  633. static ssize_t fuse_dax_direct_write(struct kiocb *iocb, struct iov_iter *from)
  634. {
  635. struct inode *inode = file_inode(iocb->ki_filp);
  636. struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
  637. ssize_t ret;
  638. ret = fuse_direct_io(&io, from, &iocb->ki_pos, FUSE_DIO_WRITE);
  639. if (ret < 0)
  640. return ret;
  641. fuse_invalidate_attr(inode);
  642. fuse_write_update_size(inode, iocb->ki_pos);
  643. return ret;
  644. }
  645. ssize_t fuse_dax_write_iter(struct kiocb *iocb, struct iov_iter *from)
  646. {
  647. struct inode *inode = file_inode(iocb->ki_filp);
  648. ssize_t ret;
  649. if (iocb->ki_flags & IOCB_NOWAIT) {
  650. if (!inode_trylock(inode))
  651. return -EAGAIN;
  652. } else {
  653. inode_lock(inode);
  654. }
  655. ret = generic_write_checks(iocb, from);
  656. if (ret <= 0)
  657. goto out;
  658. ret = file_remove_privs(iocb->ki_filp);
  659. if (ret)
  660. goto out;
  661. /* TODO file_update_time() but we don't want metadata I/O */
  662. /* Do not use dax for file extending writes as write and on
  663. * disk i_size increase are not atomic otherwise.
  664. */
  665. if (file_extending_write(iocb, from))
  666. ret = fuse_dax_direct_write(iocb, from);
  667. else
  668. ret = dax_iomap_rw(iocb, from, &fuse_iomap_ops);
  669. out:
  670. inode_unlock(inode);
  671. if (ret > 0)
  672. ret = generic_write_sync(iocb, ret);
  673. return ret;
  674. }
  675. static int fuse_dax_writepages(struct address_space *mapping,
  676. struct writeback_control *wbc)
  677. {
  678. struct inode *inode = mapping->host;
  679. struct fuse_conn *fc = get_fuse_conn(inode);
  680. return dax_writeback_mapping_range(mapping, fc->dax->dev, wbc);
  681. }
  682. static vm_fault_t __fuse_dax_fault(struct vm_fault *vmf,
  683. enum page_entry_size pe_size, bool write)
  684. {
  685. vm_fault_t ret;
  686. struct inode *inode = file_inode(vmf->vma->vm_file);
  687. struct super_block *sb = inode->i_sb;
  688. pfn_t pfn;
  689. int error = 0;
  690. struct fuse_conn *fc = get_fuse_conn(inode);
  691. struct fuse_conn_dax *fcd = fc->dax;
  692. bool retry = false;
  693. if (write)
  694. sb_start_pagefault(sb);
  695. retry:
  696. if (retry && !(fcd->nr_free_ranges > 0))
  697. wait_event(fcd->range_waitq, (fcd->nr_free_ranges > 0));
  698. /*
  699. * We need to serialize against not only truncate but also against
  700. * fuse dax memory range reclaim. While a range is being reclaimed,
  701. * we do not want any read/write/mmap to make progress and try
  702. * to populate page cache or access memory we are trying to free.
  703. */
  704. down_read(&get_fuse_inode(inode)->i_mmap_sem);
  705. ret = dax_iomap_fault(vmf, pe_size, &pfn, &error, &fuse_iomap_ops);
  706. if ((ret & VM_FAULT_ERROR) && error == -EAGAIN) {
  707. error = 0;
  708. retry = true;
  709. up_read(&get_fuse_inode(inode)->i_mmap_sem);
  710. goto retry;
  711. }
  712. if (ret & VM_FAULT_NEEDDSYNC)
  713. ret = dax_finish_sync_fault(vmf, pe_size, pfn);
  714. up_read(&get_fuse_inode(inode)->i_mmap_sem);
  715. if (write)
  716. sb_end_pagefault(sb);
  717. return ret;
  718. }
  719. static vm_fault_t fuse_dax_fault(struct vm_fault *vmf)
  720. {
  721. return __fuse_dax_fault(vmf, PE_SIZE_PTE,
  722. vmf->flags & FAULT_FLAG_WRITE);
  723. }
  724. static vm_fault_t fuse_dax_huge_fault(struct vm_fault *vmf,
  725. enum page_entry_size pe_size)
  726. {
  727. return __fuse_dax_fault(vmf, pe_size, vmf->flags & FAULT_FLAG_WRITE);
  728. }
  729. static vm_fault_t fuse_dax_page_mkwrite(struct vm_fault *vmf)
  730. {
  731. return __fuse_dax_fault(vmf, PE_SIZE_PTE, true);
  732. }
  733. static vm_fault_t fuse_dax_pfn_mkwrite(struct vm_fault *vmf)
  734. {
  735. return __fuse_dax_fault(vmf, PE_SIZE_PTE, true);
  736. }
  737. static const struct vm_operations_struct fuse_dax_vm_ops = {
  738. .fault = fuse_dax_fault,
  739. .huge_fault = fuse_dax_huge_fault,
  740. .page_mkwrite = fuse_dax_page_mkwrite,
  741. .pfn_mkwrite = fuse_dax_pfn_mkwrite,
  742. };
  743. int fuse_dax_mmap(struct file *file, struct vm_area_struct *vma)
  744. {
  745. file_accessed(file);
  746. vma->vm_ops = &fuse_dax_vm_ops;
  747. vma->vm_flags |= VM_MIXEDMAP | VM_HUGEPAGE;
  748. return 0;
  749. }
  750. static int dmap_writeback_invalidate(struct inode *inode,
  751. struct fuse_dax_mapping *dmap)
  752. {
  753. int ret;
  754. loff_t start_pos = dmap->itn.start << FUSE_DAX_SHIFT;
  755. loff_t end_pos = (start_pos + FUSE_DAX_SZ - 1);
  756. ret = filemap_fdatawrite_range(inode->i_mapping, start_pos, end_pos);
  757. if (ret) {
  758. pr_debug("fuse: filemap_fdatawrite_range() failed. err=%d start_pos=0x%llx, end_pos=0x%llx\n",
  759. ret, start_pos, end_pos);
  760. return ret;
  761. }
  762. ret = invalidate_inode_pages2_range(inode->i_mapping,
  763. start_pos >> PAGE_SHIFT,
  764. end_pos >> PAGE_SHIFT);
  765. if (ret)
  766. pr_debug("fuse: invalidate_inode_pages2_range() failed err=%d\n",
  767. ret);
  768. return ret;
  769. }
  770. static int reclaim_one_dmap_locked(struct inode *inode,
  771. struct fuse_dax_mapping *dmap)
  772. {
  773. int ret;
  774. struct fuse_inode *fi = get_fuse_inode(inode);
  775. /*
  776. * igrab() was done to make sure inode won't go under us, and this
  777. * further avoids the race with evict().
  778. */
  779. ret = dmap_writeback_invalidate(inode, dmap);
  780. if (ret)
  781. return ret;
  782. /* Remove dax mapping from inode interval tree now */
  783. interval_tree_remove(&dmap->itn, &fi->dax->tree);
  784. fi->dax->nr--;
  785. /* It is possible that umount/shutdown has killed the fuse connection
  786. * and worker thread is trying to reclaim memory in parallel. Don't
  787. * warn in that case.
  788. */
  789. ret = dmap_removemapping_one(inode, dmap);
  790. if (ret && ret != -ENOTCONN) {
  791. pr_warn("Failed to remove mapping. offset=0x%llx len=0x%llx ret=%d\n",
  792. dmap->window_offset, dmap->length, ret);
  793. }
  794. return 0;
  795. }
  796. /* Find first mapped dmap for an inode and return file offset. Caller needs
  797. * to hold fi->dax->sem lock either shared or exclusive.
  798. */
  799. static struct fuse_dax_mapping *inode_lookup_first_dmap(struct inode *inode)
  800. {
  801. struct fuse_inode *fi = get_fuse_inode(inode);
  802. struct fuse_dax_mapping *dmap;
  803. struct interval_tree_node *node;
  804. for (node = interval_tree_iter_first(&fi->dax->tree, 0, -1); node;
  805. node = interval_tree_iter_next(node, 0, -1)) {
  806. dmap = node_to_dmap(node);
  807. /* still in use. */
  808. if (refcount_read(&dmap->refcnt) > 1)
  809. continue;
  810. return dmap;
  811. }
  812. return NULL;
  813. }
  814. /*
  815. * Find first mapping in the tree and free it and return it. Do not add
  816. * it back to free pool.
  817. */
  818. static struct fuse_dax_mapping *
  819. inode_inline_reclaim_one_dmap(struct fuse_conn_dax *fcd, struct inode *inode,
  820. bool *retry)
  821. {
  822. struct fuse_inode *fi = get_fuse_inode(inode);
  823. struct fuse_dax_mapping *dmap;
  824. u64 dmap_start, dmap_end;
  825. unsigned long start_idx;
  826. int ret;
  827. struct interval_tree_node *node;
  828. down_write(&fi->i_mmap_sem);
  829. /* Lookup a dmap and corresponding file offset to reclaim. */
  830. down_read(&fi->dax->sem);
  831. dmap = inode_lookup_first_dmap(inode);
  832. if (dmap) {
  833. start_idx = dmap->itn.start;
  834. dmap_start = start_idx << FUSE_DAX_SHIFT;
  835. dmap_end = dmap_start + FUSE_DAX_SZ - 1;
  836. }
  837. up_read(&fi->dax->sem);
  838. if (!dmap)
  839. goto out_mmap_sem;
  840. /*
  841. * Make sure there are no references to inode pages using
  842. * get_user_pages()
  843. */
  844. ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
  845. if (ret) {
  846. pr_debug("fuse: fuse_dax_break_layouts() failed. err=%d\n",
  847. ret);
  848. dmap = ERR_PTR(ret);
  849. goto out_mmap_sem;
  850. }
  851. down_write(&fi->dax->sem);
  852. node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
  853. /* Range already got reclaimed by somebody else */
  854. if (!node) {
  855. if (retry)
  856. *retry = true;
  857. goto out_write_dmap_sem;
  858. }
  859. dmap = node_to_dmap(node);
  860. /* still in use. */
  861. if (refcount_read(&dmap->refcnt) > 1) {
  862. dmap = NULL;
  863. if (retry)
  864. *retry = true;
  865. goto out_write_dmap_sem;
  866. }
  867. ret = reclaim_one_dmap_locked(inode, dmap);
  868. if (ret < 0) {
  869. dmap = ERR_PTR(ret);
  870. goto out_write_dmap_sem;
  871. }
  872. /* Clean up dmap. Do not add back to free list */
  873. dmap_remove_busy_list(fcd, dmap);
  874. dmap->inode = NULL;
  875. dmap->itn.start = dmap->itn.last = 0;
  876. pr_debug("fuse: %s: inline reclaimed memory range. inode=%p, window_offset=0x%llx, length=0x%llx\n",
  877. __func__, inode, dmap->window_offset, dmap->length);
  878. out_write_dmap_sem:
  879. up_write(&fi->dax->sem);
  880. out_mmap_sem:
  881. up_write(&fi->i_mmap_sem);
  882. return dmap;
  883. }
  884. static struct fuse_dax_mapping *
  885. alloc_dax_mapping_reclaim(struct fuse_conn_dax *fcd, struct inode *inode)
  886. {
  887. struct fuse_dax_mapping *dmap;
  888. struct fuse_inode *fi = get_fuse_inode(inode);
  889. while (1) {
  890. bool retry = false;
  891. dmap = alloc_dax_mapping(fcd);
  892. if (dmap)
  893. return dmap;
  894. dmap = inode_inline_reclaim_one_dmap(fcd, inode, &retry);
  895. /*
  896. * Either we got a mapping or it is an error, return in both
  897. * the cases.
  898. */
  899. if (dmap)
  900. return dmap;
  901. /* If we could not reclaim a mapping because it
  902. * had a reference or some other temporary failure,
  903. * Try again. We want to give up inline reclaim only
  904. * if there is no range assigned to this node. Otherwise
  905. * if a deadlock is possible if we sleep with fi->i_mmap_sem
  906. * held and worker to free memory can't make progress due
  907. * to unavailability of fi->i_mmap_sem lock. So sleep
  908. * only if fi->dax->nr=0
  909. */
  910. if (retry)
  911. continue;
  912. /*
  913. * There are no mappings which can be reclaimed. Wait for one.
  914. * We are not holding fi->dax->sem. So it is possible
  915. * that range gets added now. But as we are not holding
  916. * fi->i_mmap_sem, worker should still be able to free up
  917. * a range and wake us up.
  918. */
  919. if (!fi->dax->nr && !(fcd->nr_free_ranges > 0)) {
  920. if (wait_event_killable_exclusive(fcd->range_waitq,
  921. (fcd->nr_free_ranges > 0))) {
  922. return ERR_PTR(-EINTR);
  923. }
  924. }
  925. }
  926. }
  927. static int lookup_and_reclaim_dmap_locked(struct fuse_conn_dax *fcd,
  928. struct inode *inode,
  929. unsigned long start_idx)
  930. {
  931. int ret;
  932. struct fuse_inode *fi = get_fuse_inode(inode);
  933. struct fuse_dax_mapping *dmap;
  934. struct interval_tree_node *node;
  935. /* Find fuse dax mapping at file offset inode. */
  936. node = interval_tree_iter_first(&fi->dax->tree, start_idx, start_idx);
  937. /* Range already got cleaned up by somebody else */
  938. if (!node)
  939. return 0;
  940. dmap = node_to_dmap(node);
  941. /* still in use. */
  942. if (refcount_read(&dmap->refcnt) > 1)
  943. return 0;
  944. ret = reclaim_one_dmap_locked(inode, dmap);
  945. if (ret < 0)
  946. return ret;
  947. /* Cleanup dmap entry and add back to free list */
  948. spin_lock(&fcd->lock);
  949. dmap_reinit_add_to_free_pool(fcd, dmap);
  950. spin_unlock(&fcd->lock);
  951. return ret;
  952. }
  953. /*
  954. * Free a range of memory.
  955. * Locking:
  956. * 1. Take fi->i_mmap_sem to block dax faults.
  957. * 2. Take fi->dax->sem to protect interval tree and also to make sure
  958. * read/write can not reuse a dmap which we might be freeing.
  959. */
  960. static int lookup_and_reclaim_dmap(struct fuse_conn_dax *fcd,
  961. struct inode *inode,
  962. unsigned long start_idx,
  963. unsigned long end_idx)
  964. {
  965. int ret;
  966. struct fuse_inode *fi = get_fuse_inode(inode);
  967. loff_t dmap_start = start_idx << FUSE_DAX_SHIFT;
  968. loff_t dmap_end = (dmap_start + FUSE_DAX_SZ) - 1;
  969. down_write(&fi->i_mmap_sem);
  970. ret = fuse_dax_break_layouts(inode, dmap_start, dmap_end);
  971. if (ret) {
  972. pr_debug("virtio_fs: fuse_dax_break_layouts() failed. err=%d\n",
  973. ret);
  974. goto out_mmap_sem;
  975. }
  976. down_write(&fi->dax->sem);
  977. ret = lookup_and_reclaim_dmap_locked(fcd, inode, start_idx);
  978. up_write(&fi->dax->sem);
  979. out_mmap_sem:
  980. up_write(&fi->i_mmap_sem);
  981. return ret;
  982. }
  983. static int try_to_free_dmap_chunks(struct fuse_conn_dax *fcd,
  984. unsigned long nr_to_free)
  985. {
  986. struct fuse_dax_mapping *dmap, *pos, *temp;
  987. int ret, nr_freed = 0;
  988. unsigned long start_idx = 0, end_idx = 0;
  989. struct inode *inode = NULL;
  990. /* Pick first busy range and free it for now*/
  991. while (1) {
  992. if (nr_freed >= nr_to_free)
  993. break;
  994. dmap = NULL;
  995. spin_lock(&fcd->lock);
  996. if (!fcd->nr_busy_ranges) {
  997. spin_unlock(&fcd->lock);
  998. return 0;
  999. }
  1000. list_for_each_entry_safe(pos, temp, &fcd->busy_ranges,
  1001. busy_list) {
  1002. /* skip this range if it's in use. */
  1003. if (refcount_read(&pos->refcnt) > 1)
  1004. continue;
  1005. inode = igrab(pos->inode);
  1006. /*
  1007. * This inode is going away. That will free
  1008. * up all the ranges anyway, continue to
  1009. * next range.
  1010. */
  1011. if (!inode)
  1012. continue;
  1013. /*
  1014. * Take this element off list and add it tail. If
  1015. * this element can't be freed, it will help with
  1016. * selecting new element in next iteration of loop.
  1017. */
  1018. dmap = pos;
  1019. list_move_tail(&dmap->busy_list, &fcd->busy_ranges);
  1020. start_idx = end_idx = dmap->itn.start;
  1021. break;
  1022. }
  1023. spin_unlock(&fcd->lock);
  1024. if (!dmap)
  1025. return 0;
  1026. ret = lookup_and_reclaim_dmap(fcd, inode, start_idx, end_idx);
  1027. iput(inode);
  1028. if (ret)
  1029. return ret;
  1030. nr_freed++;
  1031. }
  1032. return 0;
  1033. }
  1034. static void fuse_dax_free_mem_worker(struct work_struct *work)
  1035. {
  1036. int ret;
  1037. struct fuse_conn_dax *fcd = container_of(work, struct fuse_conn_dax,
  1038. free_work.work);
  1039. ret = try_to_free_dmap_chunks(fcd, FUSE_DAX_RECLAIM_CHUNK);
  1040. if (ret) {
  1041. pr_debug("fuse: try_to_free_dmap_chunks() failed with err=%d\n",
  1042. ret);
  1043. }
  1044. /* If number of free ranges are still below threhold, requeue */
  1045. kick_dmap_free_worker(fcd, 1);
  1046. }
  1047. static void fuse_free_dax_mem_ranges(struct list_head *mem_list)
  1048. {
  1049. struct fuse_dax_mapping *range, *temp;
  1050. /* Free All allocated elements */
  1051. list_for_each_entry_safe(range, temp, mem_list, list) {
  1052. list_del(&range->list);
  1053. if (!list_empty(&range->busy_list))
  1054. list_del(&range->busy_list);
  1055. kfree(range);
  1056. }
  1057. }
  1058. void fuse_dax_conn_free(struct fuse_conn *fc)
  1059. {
  1060. if (fc->dax) {
  1061. fuse_free_dax_mem_ranges(&fc->dax->free_ranges);
  1062. kfree(fc->dax);
  1063. }
  1064. }
  1065. static int fuse_dax_mem_range_init(struct fuse_conn_dax *fcd)
  1066. {
  1067. long nr_pages, nr_ranges;
  1068. void *kaddr;
  1069. pfn_t pfn;
  1070. struct fuse_dax_mapping *range;
  1071. int ret, id;
  1072. size_t dax_size = -1;
  1073. unsigned long i;
  1074. init_waitqueue_head(&fcd->range_waitq);
  1075. INIT_LIST_HEAD(&fcd->free_ranges);
  1076. INIT_LIST_HEAD(&fcd->busy_ranges);
  1077. INIT_DELAYED_WORK(&fcd->free_work, fuse_dax_free_mem_worker);
  1078. id = dax_read_lock();
  1079. nr_pages = dax_direct_access(fcd->dev, 0, PHYS_PFN(dax_size), &kaddr,
  1080. &pfn);
  1081. dax_read_unlock(id);
  1082. if (nr_pages < 0) {
  1083. pr_debug("dax_direct_access() returned %ld\n", nr_pages);
  1084. return nr_pages;
  1085. }
  1086. nr_ranges = nr_pages/FUSE_DAX_PAGES;
  1087. pr_debug("%s: dax mapped %ld pages. nr_ranges=%ld\n",
  1088. __func__, nr_pages, nr_ranges);
  1089. for (i = 0; i < nr_ranges; i++) {
  1090. range = kzalloc(sizeof(struct fuse_dax_mapping), GFP_KERNEL);
  1091. ret = -ENOMEM;
  1092. if (!range)
  1093. goto out_err;
  1094. /* TODO: This offset only works if virtio-fs driver is not
  1095. * having some memory hidden at the beginning. This needs
  1096. * better handling
  1097. */
  1098. range->window_offset = i * FUSE_DAX_SZ;
  1099. range->length = FUSE_DAX_SZ;
  1100. INIT_LIST_HEAD(&range->busy_list);
  1101. refcount_set(&range->refcnt, 1);
  1102. list_add_tail(&range->list, &fcd->free_ranges);
  1103. }
  1104. fcd->nr_free_ranges = nr_ranges;
  1105. fcd->nr_ranges = nr_ranges;
  1106. return 0;
  1107. out_err:
  1108. /* Free All allocated elements */
  1109. fuse_free_dax_mem_ranges(&fcd->free_ranges);
  1110. return ret;
  1111. }
  1112. int fuse_dax_conn_alloc(struct fuse_conn *fc, struct dax_device *dax_dev)
  1113. {
  1114. struct fuse_conn_dax *fcd;
  1115. int err;
  1116. if (!dax_dev)
  1117. return 0;
  1118. fcd = kzalloc(sizeof(*fcd), GFP_KERNEL);
  1119. if (!fcd)
  1120. return -ENOMEM;
  1121. spin_lock_init(&fcd->lock);
  1122. fcd->dev = dax_dev;
  1123. err = fuse_dax_mem_range_init(fcd);
  1124. if (err) {
  1125. kfree(fcd);
  1126. return err;
  1127. }
  1128. fc->dax = fcd;
  1129. return 0;
  1130. }
  1131. bool fuse_dax_inode_alloc(struct super_block *sb, struct fuse_inode *fi)
  1132. {
  1133. struct fuse_conn *fc = get_fuse_conn_super(sb);
  1134. fi->dax = NULL;
  1135. if (fc->dax) {
  1136. fi->dax = kzalloc(sizeof(*fi->dax), GFP_KERNEL_ACCOUNT);
  1137. if (!fi->dax)
  1138. return false;
  1139. init_rwsem(&fi->dax->sem);
  1140. fi->dax->tree = RB_ROOT_CACHED;
  1141. }
  1142. return true;
  1143. }
  1144. static const struct address_space_operations fuse_dax_file_aops = {
  1145. .writepages = fuse_dax_writepages,
  1146. .direct_IO = noop_direct_IO,
  1147. .set_page_dirty = noop_set_page_dirty,
  1148. .invalidatepage = noop_invalidatepage,
  1149. };
  1150. void fuse_dax_inode_init(struct inode *inode)
  1151. {
  1152. struct fuse_conn *fc = get_fuse_conn(inode);
  1153. if (!fc->dax)
  1154. return;
  1155. inode->i_flags |= S_DAX;
  1156. inode->i_data.a_ops = &fuse_dax_file_aops;
  1157. }
  1158. bool fuse_dax_check_alignment(struct fuse_conn *fc, unsigned int map_alignment)
  1159. {
  1160. if (fc->dax && (map_alignment > FUSE_DAX_SHIFT)) {
  1161. pr_warn("FUSE: map_alignment %u incompatible with dax mem range size %u\n",
  1162. map_alignment, FUSE_DAX_SZ);
  1163. return false;
  1164. }
  1165. return true;
  1166. }
  1167. void fuse_dax_cancel_work(struct fuse_conn *fc)
  1168. {
  1169. struct fuse_conn_dax *fcd = fc->dax;
  1170. if (fcd)
  1171. cancel_delayed_work_sync(&fcd->free_work);
  1172. }
  1173. EXPORT_SYMBOL_GPL(fuse_dax_cancel_work);