page_io.c 9.9 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * linux/mm/page_io.c
  4. *
  5. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  6. *
  7. * Swap reorganised 29.12.95,
  8. * Asynchronous swapping added 30.12.95. Stephen Tweedie
  9. * Removed race in async swapping. 14.4.1996. Bruno Haible
  10. * Add swap of shared pages through the page cache. 20.2.1998. Stephen Tweedie
  11. * Always use brw_page, life becomes simpler. 12 May 1998 Eric Biederman
  12. */
  13. #include <linux/mm.h>
  14. #include <linux/kernel_stat.h>
  15. #include <linux/gfp.h>
  16. #include <linux/pagemap.h>
  17. #include <linux/swap.h>
  18. #include <linux/bio.h>
  19. #include <linux/swapops.h>
  20. #include <linux/buffer_head.h>
  21. #include <linux/writeback.h>
  22. #include <linux/frontswap.h>
  23. #include <linux/blkdev.h>
  24. #include <linux/psi.h>
  25. #include <linux/uio.h>
  26. #include <linux/sched/task.h>
  27. static struct bio *get_swap_bio(gfp_t gfp_flags,
  28. struct page *page, bio_end_io_t end_io)
  29. {
  30. struct bio *bio;
  31. bio = bio_alloc(gfp_flags, 1);
  32. if (bio) {
  33. struct block_device *bdev;
  34. bio->bi_iter.bi_sector = map_swap_page(page, &bdev);
  35. bio_set_dev(bio, bdev);
  36. bio->bi_iter.bi_sector <<= PAGE_SHIFT - 9;
  37. bio->bi_end_io = end_io;
  38. bio_add_page(bio, page, thp_size(page), 0);
  39. }
  40. return bio;
  41. }
  42. void end_swap_bio_write(struct bio *bio)
  43. {
  44. struct page *page = bio_first_page_all(bio);
  45. if (bio->bi_status) {
  46. SetPageError(page);
  47. /*
  48. * We failed to write the page out to swap-space.
  49. * Re-dirty the page in order to avoid it being reclaimed.
  50. * Also print a dire warning that things will go BAD (tm)
  51. * very quickly.
  52. *
  53. * Also clear PG_reclaim to avoid rotate_reclaimable_page()
  54. */
  55. set_page_dirty(page);
  56. pr_alert_ratelimited("Write-error on swap-device (%u:%u:%llu)\n",
  57. MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
  58. (unsigned long long)bio->bi_iter.bi_sector);
  59. ClearPageReclaim(page);
  60. }
  61. end_page_writeback(page);
  62. bio_put(bio);
  63. }
  64. static void end_swap_bio_read(struct bio *bio)
  65. {
  66. struct page *page = bio_first_page_all(bio);
  67. struct task_struct *waiter = bio->bi_private;
  68. if (bio->bi_status) {
  69. SetPageError(page);
  70. ClearPageUptodate(page);
  71. pr_alert_ratelimited("Read-error on swap-device (%u:%u:%llu)\n",
  72. MAJOR(bio_dev(bio)), MINOR(bio_dev(bio)),
  73. (unsigned long long)bio->bi_iter.bi_sector);
  74. goto out;
  75. }
  76. SetPageUptodate(page);
  77. out:
  78. unlock_page(page);
  79. WRITE_ONCE(bio->bi_private, NULL);
  80. bio_put(bio);
  81. if (waiter) {
  82. blk_wake_io_task(waiter);
  83. put_task_struct(waiter);
  84. }
  85. }
  86. int generic_swapfile_activate(struct swap_info_struct *sis,
  87. struct file *swap_file,
  88. sector_t *span)
  89. {
  90. struct address_space *mapping = swap_file->f_mapping;
  91. struct inode *inode = mapping->host;
  92. unsigned blocks_per_page;
  93. unsigned long page_no;
  94. unsigned blkbits;
  95. sector_t probe_block;
  96. sector_t last_block;
  97. sector_t lowest_block = -1;
  98. sector_t highest_block = 0;
  99. int nr_extents = 0;
  100. int ret;
  101. blkbits = inode->i_blkbits;
  102. blocks_per_page = PAGE_SIZE >> blkbits;
  103. /*
  104. * Map all the blocks into the extent tree. This code doesn't try
  105. * to be very smart.
  106. */
  107. probe_block = 0;
  108. page_no = 0;
  109. last_block = i_size_read(inode) >> blkbits;
  110. while ((probe_block + blocks_per_page) <= last_block &&
  111. page_no < sis->max) {
  112. unsigned block_in_page;
  113. sector_t first_block;
  114. cond_resched();
  115. first_block = probe_block;
  116. ret = bmap(inode, &first_block);
  117. if (ret || !first_block)
  118. goto bad_bmap;
  119. /*
  120. * It must be PAGE_SIZE aligned on-disk
  121. */
  122. if (first_block & (blocks_per_page - 1)) {
  123. probe_block++;
  124. goto reprobe;
  125. }
  126. for (block_in_page = 1; block_in_page < blocks_per_page;
  127. block_in_page++) {
  128. sector_t block;
  129. block = probe_block + block_in_page;
  130. ret = bmap(inode, &block);
  131. if (ret || !block)
  132. goto bad_bmap;
  133. if (block != first_block + block_in_page) {
  134. /* Discontiguity */
  135. probe_block++;
  136. goto reprobe;
  137. }
  138. }
  139. first_block >>= (PAGE_SHIFT - blkbits);
  140. if (page_no) { /* exclude the header page */
  141. if (first_block < lowest_block)
  142. lowest_block = first_block;
  143. if (first_block > highest_block)
  144. highest_block = first_block;
  145. }
  146. /*
  147. * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
  148. */
  149. ret = add_swap_extent(sis, page_no, 1, first_block);
  150. if (ret < 0)
  151. goto out;
  152. nr_extents += ret;
  153. page_no++;
  154. probe_block += blocks_per_page;
  155. reprobe:
  156. continue;
  157. }
  158. ret = nr_extents;
  159. *span = 1 + highest_block - lowest_block;
  160. if (page_no == 0)
  161. page_no = 1; /* force Empty message */
  162. sis->max = page_no;
  163. sis->pages = page_no - 1;
  164. sis->highest_bit = page_no - 1;
  165. out:
  166. return ret;
  167. bad_bmap:
  168. pr_err("swapon: swapfile has holes\n");
  169. ret = -EINVAL;
  170. goto out;
  171. }
  172. /*
  173. * We may have stale swap cache pages in memory: notice
  174. * them here and get rid of the unnecessary final write.
  175. */
  176. int swap_writepage(struct page *page, struct writeback_control *wbc)
  177. {
  178. int ret = 0;
  179. if (try_to_free_swap(page)) {
  180. unlock_page(page);
  181. goto out;
  182. }
  183. /*
  184. * Arch code may have to preserve more data than just the page
  185. * contents, e.g. memory tags.
  186. */
  187. ret = arch_prepare_to_swap(page);
  188. if (ret) {
  189. set_page_dirty(page);
  190. unlock_page(page);
  191. goto out;
  192. }
  193. if (frontswap_store(page) == 0) {
  194. set_page_writeback(page);
  195. unlock_page(page);
  196. end_page_writeback(page);
  197. goto out;
  198. }
  199. ret = __swap_writepage(page, wbc, end_swap_bio_write);
  200. out:
  201. return ret;
  202. }
  203. static inline void count_swpout_vm_event(struct page *page)
  204. {
  205. #ifdef CONFIG_TRANSPARENT_HUGEPAGE
  206. if (unlikely(PageTransHuge(page)))
  207. count_vm_event(THP_SWPOUT);
  208. #endif
  209. count_vm_events(PSWPOUT, thp_nr_pages(page));
  210. }
  211. #if defined(CONFIG_MEMCG) && defined(CONFIG_BLK_CGROUP)
  212. static void bio_associate_blkg_from_page(struct bio *bio, struct page *page)
  213. {
  214. struct cgroup_subsys_state *css;
  215. if (!page->mem_cgroup)
  216. return;
  217. rcu_read_lock();
  218. css = cgroup_e_css(page->mem_cgroup->css.cgroup, &io_cgrp_subsys);
  219. bio_associate_blkg_from_css(bio, css);
  220. rcu_read_unlock();
  221. }
  222. #else
  223. #define bio_associate_blkg_from_page(bio, page) do { } while (0)
  224. #endif /* CONFIG_MEMCG && CONFIG_BLK_CGROUP */
  225. int __swap_writepage(struct page *page, struct writeback_control *wbc,
  226. bio_end_io_t end_write_func)
  227. {
  228. struct bio *bio;
  229. int ret;
  230. struct swap_info_struct *sis = page_swap_info(page);
  231. VM_BUG_ON_PAGE(!PageSwapCache(page), page);
  232. if (data_race(sis->flags & SWP_FS_OPS)) {
  233. struct kiocb kiocb;
  234. struct file *swap_file = sis->swap_file;
  235. struct address_space *mapping = swap_file->f_mapping;
  236. struct bio_vec bv = {
  237. .bv_page = page,
  238. .bv_len = PAGE_SIZE,
  239. .bv_offset = 0
  240. };
  241. struct iov_iter from;
  242. iov_iter_bvec(&from, WRITE, &bv, 1, PAGE_SIZE);
  243. init_sync_kiocb(&kiocb, swap_file);
  244. kiocb.ki_pos = page_file_offset(page);
  245. set_page_writeback(page);
  246. unlock_page(page);
  247. ret = mapping->a_ops->direct_IO(&kiocb, &from);
  248. if (ret == PAGE_SIZE) {
  249. count_vm_event(PSWPOUT);
  250. ret = 0;
  251. } else {
  252. /*
  253. * In the case of swap-over-nfs, this can be a
  254. * temporary failure if the system has limited
  255. * memory for allocating transmit buffers.
  256. * Mark the page dirty and avoid
  257. * rotate_reclaimable_page but rate-limit the
  258. * messages but do not flag PageError like
  259. * the normal direct-to-bio case as it could
  260. * be temporary.
  261. */
  262. set_page_dirty(page);
  263. ClearPageReclaim(page);
  264. pr_err_ratelimited("Write error on dio swapfile (%llu)\n",
  265. page_file_offset(page));
  266. }
  267. end_page_writeback(page);
  268. return ret;
  269. }
  270. ret = bdev_write_page(sis->bdev, swap_page_sector(page), page, wbc);
  271. if (!ret) {
  272. count_swpout_vm_event(page);
  273. return 0;
  274. }
  275. bio = get_swap_bio(GFP_NOIO, page, end_write_func);
  276. if (bio == NULL) {
  277. set_page_dirty(page);
  278. unlock_page(page);
  279. return -ENOMEM;
  280. }
  281. bio->bi_opf = REQ_OP_WRITE | REQ_SWAP | wbc_to_write_flags(wbc);
  282. bio_associate_blkg_from_page(bio, page);
  283. count_swpout_vm_event(page);
  284. set_page_writeback(page);
  285. unlock_page(page);
  286. submit_bio(bio);
  287. return 0;
  288. }
  289. int swap_readpage(struct page *page, bool synchronous)
  290. {
  291. struct bio *bio;
  292. int ret = 0;
  293. struct swap_info_struct *sis = page_swap_info(page);
  294. blk_qc_t qc;
  295. struct gendisk *disk;
  296. unsigned long pflags;
  297. VM_BUG_ON_PAGE(!PageSwapCache(page) && !synchronous, page);
  298. VM_BUG_ON_PAGE(!PageLocked(page), page);
  299. VM_BUG_ON_PAGE(PageUptodate(page), page);
  300. /*
  301. * Count submission time as memory stall. When the device is congested,
  302. * or the submitting cgroup IO-throttled, submission can be a
  303. * significant part of overall IO time.
  304. */
  305. psi_memstall_enter(&pflags);
  306. if (frontswap_load(page) == 0) {
  307. SetPageUptodate(page);
  308. unlock_page(page);
  309. goto out;
  310. }
  311. if (data_race(sis->flags & SWP_FS_OPS)) {
  312. struct file *swap_file = sis->swap_file;
  313. struct address_space *mapping = swap_file->f_mapping;
  314. ret = mapping->a_ops->readpage(swap_file, page);
  315. if (!ret)
  316. count_vm_event(PSWPIN);
  317. goto out;
  318. }
  319. if (sis->flags & SWP_SYNCHRONOUS_IO) {
  320. ret = bdev_read_page(sis->bdev, swap_page_sector(page), page);
  321. if (!ret) {
  322. count_vm_event(PSWPIN);
  323. goto out;
  324. }
  325. }
  326. ret = 0;
  327. bio = get_swap_bio(GFP_KERNEL, page, end_swap_bio_read);
  328. if (bio == NULL) {
  329. unlock_page(page);
  330. ret = -ENOMEM;
  331. goto out;
  332. }
  333. disk = bio->bi_disk;
  334. /*
  335. * Keep this task valid during swap readpage because the oom killer may
  336. * attempt to access it in the page fault retry time check.
  337. */
  338. bio_set_op_attrs(bio, REQ_OP_READ, 0);
  339. if (synchronous) {
  340. bio->bi_opf |= REQ_HIPRI;
  341. get_task_struct(current);
  342. bio->bi_private = current;
  343. }
  344. count_vm_event(PSWPIN);
  345. bio_get(bio);
  346. qc = submit_bio(bio);
  347. while (synchronous) {
  348. set_current_state(TASK_UNINTERRUPTIBLE);
  349. if (!READ_ONCE(bio->bi_private))
  350. break;
  351. if (!blk_poll(disk->queue, qc, true))
  352. blk_io_schedule();
  353. }
  354. __set_current_state(TASK_RUNNING);
  355. bio_put(bio);
  356. out:
  357. psi_memstall_leave(&pflags);
  358. return ret;
  359. }
  360. int swap_set_page_dirty(struct page *page)
  361. {
  362. struct swap_info_struct *sis = page_swap_info(page);
  363. if (data_race(sis->flags & SWP_FS_OPS)) {
  364. struct address_space *mapping = sis->swap_file->f_mapping;
  365. VM_BUG_ON_PAGE(!PageSwapCache(page), page);
  366. return mapping->a_ops->set_page_dirty(page);
  367. } else {
  368. return __set_page_dirty_no_writeback(page);
  369. }
  370. }