writeback.h 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * include/linux/writeback.h
  4. */
  5. #ifndef WRITEBACK_H
  6. #define WRITEBACK_H
  7. #include <linux/sched.h>
  8. #include <linux/workqueue.h>
  9. #include <linux/fs.h>
  10. #include <linux/flex_proportions.h>
  11. #include <linux/backing-dev-defs.h>
  12. #include <linux/blk_types.h>
  13. #include <linux/blk-cgroup.h>
  14. struct bio;
  15. DECLARE_PER_CPU(int, dirty_throttle_leaks);
  16. /*
  17. * The 1/4 region under the global dirty thresh is for smooth dirty throttling:
  18. *
  19. * (thresh - thresh/DIRTY_FULL_SCOPE, thresh)
  20. *
  21. * Further beyond, all dirtier tasks will enter a loop waiting (possibly long
  22. * time) for the dirty pages to drop, unless written enough pages.
  23. *
  24. * The global dirty threshold is normally equal to the global dirty limit,
  25. * except when the system suddenly allocates a lot of anonymous memory and
  26. * knocks down the global dirty threshold quickly, in which case the global
  27. * dirty limit will follow down slowly to prevent livelocking all dirtier tasks.
  28. */
  29. #define DIRTY_SCOPE 8
  30. #define DIRTY_FULL_SCOPE (DIRTY_SCOPE / 2)
  31. struct backing_dev_info;
  32. /*
  33. * fs/fs-writeback.c
  34. */
  35. enum writeback_sync_modes {
  36. WB_SYNC_NONE, /* Don't wait on anything */
  37. WB_SYNC_ALL, /* Wait on every mapping */
  38. };
  39. /*
  40. * A control structure which tells the writeback code what to do. These are
  41. * always on the stack, and hence need no locking. They are always initialised
  42. * in a manner such that unspecified fields are set to zero.
  43. */
  44. struct writeback_control {
  45. long nr_to_write; /* Write this many pages, and decrement
  46. this for each page written */
  47. long pages_skipped; /* Pages which were not written */
  48. /*
  49. * For a_ops->writepages(): if start or end are non-zero then this is
  50. * a hint that the filesystem need only write out the pages inside that
  51. * byterange. The byte at `end' is included in the writeout request.
  52. */
  53. loff_t range_start;
  54. loff_t range_end;
  55. enum writeback_sync_modes sync_mode;
  56. unsigned for_kupdate:1; /* A kupdate writeback */
  57. unsigned for_background:1; /* A background writeback */
  58. unsigned tagged_writepages:1; /* tag-and-write to avoid livelock */
  59. unsigned for_reclaim:1; /* Invoked from the page allocator */
  60. unsigned range_cyclic:1; /* range_start is cyclic */
  61. unsigned for_sync:1; /* sync(2) WB_SYNC_ALL writeback */
  62. /*
  63. * When writeback IOs are bounced through async layers, only the
  64. * initial synchronous phase should be accounted towards inode
  65. * cgroup ownership arbitration to avoid confusion. Later stages
  66. * can set the following flag to disable the accounting.
  67. */
  68. unsigned no_cgroup_owner:1;
  69. unsigned punt_to_cgroup:1; /* cgrp punting, see __REQ_CGROUP_PUNT */
  70. #ifdef CONFIG_CGROUP_WRITEBACK
  71. struct bdi_writeback *wb; /* wb this writeback is issued under */
  72. struct inode *inode; /* inode being written out */
  73. /* foreign inode detection, see wbc_detach_inode() */
  74. int wb_id; /* current wb id */
  75. int wb_lcand_id; /* last foreign candidate wb id */
  76. int wb_tcand_id; /* this foreign candidate wb id */
  77. size_t wb_bytes; /* bytes written by current wb */
  78. size_t wb_lcand_bytes; /* bytes written by last candidate */
  79. size_t wb_tcand_bytes; /* bytes written by this candidate */
  80. #endif
  81. };
  82. static inline int wbc_to_write_flags(struct writeback_control *wbc)
  83. {
  84. int flags = 0;
  85. if (wbc->punt_to_cgroup)
  86. flags = REQ_CGROUP_PUNT;
  87. if (wbc->sync_mode == WB_SYNC_ALL)
  88. flags |= REQ_SYNC;
  89. else if (wbc->for_kupdate || wbc->for_background)
  90. flags |= REQ_BACKGROUND;
  91. return flags;
  92. }
  93. static inline struct cgroup_subsys_state *
  94. wbc_blkcg_css(struct writeback_control *wbc)
  95. {
  96. #ifdef CONFIG_CGROUP_WRITEBACK
  97. if (wbc->wb)
  98. return wbc->wb->blkcg_css;
  99. #endif
  100. return blkcg_root_css;
  101. }
  102. /*
  103. * A wb_domain represents a domain that wb's (bdi_writeback's) belong to
  104. * and are measured against each other in. There always is one global
  105. * domain, global_wb_domain, that every wb in the system is a member of.
  106. * This allows measuring the relative bandwidth of each wb to distribute
  107. * dirtyable memory accordingly.
  108. */
  109. struct wb_domain {
  110. spinlock_t lock;
  111. /*
  112. * Scale the writeback cache size proportional to the relative
  113. * writeout speed.
  114. *
  115. * We do this by keeping a floating proportion between BDIs, based
  116. * on page writeback completions [end_page_writeback()]. Those
  117. * devices that write out pages fastest will get the larger share,
  118. * while the slower will get a smaller share.
  119. *
  120. * We use page writeout completions because we are interested in
  121. * getting rid of dirty pages. Having them written out is the
  122. * primary goal.
  123. *
  124. * We introduce a concept of time, a period over which we measure
  125. * these events, because demand can/will vary over time. The length
  126. * of this period itself is measured in page writeback completions.
  127. */
  128. struct fprop_global completions;
  129. struct timer_list period_timer; /* timer for aging of completions */
  130. unsigned long period_time;
  131. /*
  132. * The dirtyable memory and dirty threshold could be suddenly
  133. * knocked down by a large amount (eg. on the startup of KVM in a
  134. * swapless system). This may throw the system into deep dirty
  135. * exceeded state and throttle heavy/light dirtiers alike. To
  136. * retain good responsiveness, maintain global_dirty_limit for
  137. * tracking slowly down to the knocked down dirty threshold.
  138. *
  139. * Both fields are protected by ->lock.
  140. */
  141. unsigned long dirty_limit_tstamp;
  142. unsigned long dirty_limit;
  143. };
  144. /**
  145. * wb_domain_size_changed - memory available to a wb_domain has changed
  146. * @dom: wb_domain of interest
  147. *
  148. * This function should be called when the amount of memory available to
  149. * @dom has changed. It resets @dom's dirty limit parameters to prevent
  150. * the past values which don't match the current configuration from skewing
  151. * dirty throttling. Without this, when memory size of a wb_domain is
  152. * greatly reduced, the dirty throttling logic may allow too many pages to
  153. * be dirtied leading to consecutive unnecessary OOMs and may get stuck in
  154. * that situation.
  155. */
  156. static inline void wb_domain_size_changed(struct wb_domain *dom)
  157. {
  158. spin_lock(&dom->lock);
  159. dom->dirty_limit_tstamp = jiffies;
  160. dom->dirty_limit = 0;
  161. spin_unlock(&dom->lock);
  162. }
  163. /*
  164. * fs/fs-writeback.c
  165. */
  166. struct bdi_writeback;
  167. void writeback_inodes_sb(struct super_block *, enum wb_reason reason);
  168. void writeback_inodes_sb_nr(struct super_block *, unsigned long nr,
  169. enum wb_reason reason);
  170. void try_to_writeback_inodes_sb(struct super_block *sb, enum wb_reason reason);
  171. void sync_inodes_sb(struct super_block *);
  172. void wakeup_flusher_threads(enum wb_reason reason);
  173. void wakeup_flusher_threads_bdi(struct backing_dev_info *bdi,
  174. enum wb_reason reason);
  175. void inode_wait_for_writeback(struct inode *inode);
  176. void inode_io_list_del(struct inode *inode);
  177. /* writeback.h requires fs.h; it, too, is not included from here. */
  178. static inline void wait_on_inode(struct inode *inode)
  179. {
  180. might_sleep();
  181. wait_on_bit(&inode->i_state, __I_NEW, TASK_UNINTERRUPTIBLE);
  182. }
  183. #ifdef CONFIG_CGROUP_WRITEBACK
  184. #include <linux/cgroup.h>
  185. #include <linux/bio.h>
  186. void __inode_attach_wb(struct inode *inode, struct page *page);
  187. void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
  188. struct inode *inode)
  189. __releases(&inode->i_lock);
  190. void wbc_detach_inode(struct writeback_control *wbc);
  191. void wbc_account_cgroup_owner(struct writeback_control *wbc, struct page *page,
  192. size_t bytes);
  193. int cgroup_writeback_by_id(u64 bdi_id, int memcg_id, unsigned long nr_pages,
  194. enum wb_reason reason, struct wb_completion *done);
  195. void cgroup_writeback_umount(void);
  196. /**
  197. * inode_attach_wb - associate an inode with its wb
  198. * @inode: inode of interest
  199. * @page: page being dirtied (may be NULL)
  200. *
  201. * If @inode doesn't have its wb, associate it with the wb matching the
  202. * memcg of @page or, if @page is NULL, %current. May be called w/ or w/o
  203. * @inode->i_lock.
  204. */
  205. static inline void inode_attach_wb(struct inode *inode, struct page *page)
  206. {
  207. if (!inode->i_wb)
  208. __inode_attach_wb(inode, page);
  209. }
  210. /**
  211. * inode_detach_wb - disassociate an inode from its wb
  212. * @inode: inode of interest
  213. *
  214. * @inode is being freed. Detach from its wb.
  215. */
  216. static inline void inode_detach_wb(struct inode *inode)
  217. {
  218. if (inode->i_wb) {
  219. WARN_ON_ONCE(!(inode->i_state & I_CLEAR));
  220. wb_put(inode->i_wb);
  221. inode->i_wb = NULL;
  222. }
  223. }
  224. /**
  225. * wbc_attach_fdatawrite_inode - associate wbc and inode for fdatawrite
  226. * @wbc: writeback_control of interest
  227. * @inode: target inode
  228. *
  229. * This function is to be used by __filemap_fdatawrite_range(), which is an
  230. * alternative entry point into writeback code, and first ensures @inode is
  231. * associated with a bdi_writeback and attaches it to @wbc.
  232. */
  233. static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
  234. struct inode *inode)
  235. {
  236. spin_lock(&inode->i_lock);
  237. inode_attach_wb(inode, NULL);
  238. wbc_attach_and_unlock_inode(wbc, inode);
  239. }
  240. /**
  241. * wbc_init_bio - writeback specific initializtion of bio
  242. * @wbc: writeback_control for the writeback in progress
  243. * @bio: bio to be initialized
  244. *
  245. * @bio is a part of the writeback in progress controlled by @wbc. Perform
  246. * writeback specific initialization. This is used to apply the cgroup
  247. * writeback context. Must be called after the bio has been associated with
  248. * a device.
  249. */
  250. static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
  251. {
  252. /*
  253. * pageout() path doesn't attach @wbc to the inode being written
  254. * out. This is intentional as we don't want the function to block
  255. * behind a slow cgroup. Ultimately, we want pageout() to kick off
  256. * regular writeback instead of writing things out itself.
  257. */
  258. if (wbc->wb)
  259. bio_associate_blkg_from_css(bio, wbc->wb->blkcg_css);
  260. }
  261. #else /* CONFIG_CGROUP_WRITEBACK */
  262. static inline void inode_attach_wb(struct inode *inode, struct page *page)
  263. {
  264. }
  265. static inline void inode_detach_wb(struct inode *inode)
  266. {
  267. }
  268. static inline void wbc_attach_and_unlock_inode(struct writeback_control *wbc,
  269. struct inode *inode)
  270. __releases(&inode->i_lock)
  271. {
  272. spin_unlock(&inode->i_lock);
  273. }
  274. static inline void wbc_attach_fdatawrite_inode(struct writeback_control *wbc,
  275. struct inode *inode)
  276. {
  277. }
  278. static inline void wbc_detach_inode(struct writeback_control *wbc)
  279. {
  280. }
  281. static inline void wbc_init_bio(struct writeback_control *wbc, struct bio *bio)
  282. {
  283. }
  284. static inline void wbc_account_cgroup_owner(struct writeback_control *wbc,
  285. struct page *page, size_t bytes)
  286. {
  287. }
  288. static inline void cgroup_writeback_umount(void)
  289. {
  290. }
  291. #endif /* CONFIG_CGROUP_WRITEBACK */
  292. /*
  293. * mm/page-writeback.c
  294. */
  295. #ifdef CONFIG_BLOCK
  296. void laptop_io_completion(struct backing_dev_info *info);
  297. void laptop_sync_completion(void);
  298. void laptop_mode_sync(struct work_struct *work);
  299. void laptop_mode_timer_fn(struct timer_list *t);
  300. #else
  301. static inline void laptop_sync_completion(void) { }
  302. #endif
  303. bool node_dirty_ok(struct pglist_data *pgdat);
  304. int wb_domain_init(struct wb_domain *dom, gfp_t gfp);
  305. #ifdef CONFIG_CGROUP_WRITEBACK
  306. void wb_domain_exit(struct wb_domain *dom);
  307. #endif
  308. extern struct wb_domain global_wb_domain;
  309. /* These are exported to sysctl. */
  310. extern int dirty_background_ratio;
  311. extern unsigned long dirty_background_bytes;
  312. extern int vm_dirty_ratio;
  313. extern unsigned long vm_dirty_bytes;
  314. extern unsigned int dirty_writeback_interval;
  315. extern unsigned int dirty_expire_interval;
  316. extern unsigned int dirtytime_expire_interval;
  317. extern int vm_highmem_is_dirtyable;
  318. extern int block_dump;
  319. extern int laptop_mode;
  320. int dirty_background_ratio_handler(struct ctl_table *table, int write,
  321. void *buffer, size_t *lenp, loff_t *ppos);
  322. int dirty_background_bytes_handler(struct ctl_table *table, int write,
  323. void *buffer, size_t *lenp, loff_t *ppos);
  324. int dirty_ratio_handler(struct ctl_table *table, int write,
  325. void *buffer, size_t *lenp, loff_t *ppos);
  326. int dirty_bytes_handler(struct ctl_table *table, int write,
  327. void *buffer, size_t *lenp, loff_t *ppos);
  328. int dirtytime_interval_handler(struct ctl_table *table, int write,
  329. void *buffer, size_t *lenp, loff_t *ppos);
  330. int dirty_writeback_centisecs_handler(struct ctl_table *table, int write,
  331. void *buffer, size_t *lenp, loff_t *ppos);
  332. void global_dirty_limits(unsigned long *pbackground, unsigned long *pdirty);
  333. unsigned long wb_calc_thresh(struct bdi_writeback *wb, unsigned long thresh);
  334. void wb_update_bandwidth(struct bdi_writeback *wb, unsigned long start_time);
  335. void balance_dirty_pages_ratelimited(struct address_space *mapping);
  336. bool wb_over_bg_thresh(struct bdi_writeback *wb);
  337. typedef int (*writepage_t)(struct page *page, struct writeback_control *wbc,
  338. void *data);
  339. int generic_writepages(struct address_space *mapping,
  340. struct writeback_control *wbc);
  341. void tag_pages_for_writeback(struct address_space *mapping,
  342. pgoff_t start, pgoff_t end);
  343. int write_cache_pages(struct address_space *mapping,
  344. struct writeback_control *wbc, writepage_t writepage,
  345. void *data);
  346. int do_writepages(struct address_space *mapping, struct writeback_control *wbc);
  347. void writeback_set_ratelimit(void);
  348. void tag_pages_for_writeback(struct address_space *mapping,
  349. pgoff_t start, pgoff_t end);
  350. void account_page_redirty(struct page *page);
  351. void sb_mark_inode_writeback(struct inode *inode);
  352. void sb_clear_inode_writeback(struct inode *inode);
  353. #endif /* WRITEBACK_H */