truncate.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444
  1. /*
  2. * mm/truncate.c - code for taking down pages from address_spaces
  3. *
  4. * Copyright (C) 2002, Linus Torvalds
  5. *
  6. * 10Sep2002 akpm@zip.com.au
  7. * Initial version.
  8. */
  9. #include <linux/kernel.h>
  10. #include <linux/mm.h>
  11. #include <linux/swap.h>
  12. #include <linux/module.h>
  13. #include <linux/pagemap.h>
  14. #include <linux/pagevec.h>
  15. #include <linux/task_io_accounting_ops.h>
  16. #include <linux/buffer_head.h> /* grr. try_to_release_page,
  17. do_invalidatepage */
  18. /**
  19. * do_invalidatepage - invalidate part of all of a page
  20. * @page: the page which is affected
  21. * @offset: the index of the truncation point
  22. *
  23. * do_invalidatepage() is called when all or part of the page has become
  24. * invalidated by a truncate operation.
  25. *
  26. * do_invalidatepage() does not have to release all buffers, but it must
  27. * ensure that no dirty buffer is left outside @offset and that no I/O
  28. * is underway against any of the blocks which are outside the truncation
  29. * point. Because the caller is about to free (and possibly reuse) those
  30. * blocks on-disk.
  31. */
  32. void do_invalidatepage(struct page *page, unsigned long offset)
  33. {
  34. void (*invalidatepage)(struct page *, unsigned long);
  35. invalidatepage = page->mapping->a_ops->invalidatepage;
  36. #ifdef CONFIG_BLOCK
  37. if (!invalidatepage)
  38. invalidatepage = block_invalidatepage;
  39. #endif
  40. if (invalidatepage)
  41. (*invalidatepage)(page, offset);
  42. }
  43. static inline void truncate_partial_page(struct page *page, unsigned partial)
  44. {
  45. memclear_highpage_flush(page, partial, PAGE_CACHE_SIZE-partial);
  46. if (PagePrivate(page))
  47. do_invalidatepage(page, partial);
  48. }
  49. /*
  50. * This cancels just the dirty bit on the kernel page itself, it
  51. * does NOT actually remove dirty bits on any mmap's that may be
  52. * around. It also leaves the page tagged dirty, so any sync
  53. * activity will still find it on the dirty lists, and in particular,
  54. * clear_page_dirty_for_io() will still look at the dirty bits in
  55. * the VM.
  56. *
  57. * Doing this should *normally* only ever be done when a page
  58. * is truncated, and is not actually mapped anywhere at all. However,
  59. * fs/buffer.c does this when it notices that somebody has cleaned
  60. * out all the buffers on a page without actually doing it through
  61. * the VM. Can you say "ext3 is horribly ugly"? Tought you could.
  62. */
  63. void cancel_dirty_page(struct page *page, unsigned int account_size)
  64. {
  65. if (TestClearPageDirty(page)) {
  66. struct address_space *mapping = page->mapping;
  67. if (mapping && mapping_cap_account_dirty(mapping)) {
  68. dec_zone_page_state(page, NR_FILE_DIRTY);
  69. if (account_size)
  70. task_io_account_cancelled_write(account_size);
  71. }
  72. }
  73. }
  74. EXPORT_SYMBOL(cancel_dirty_page);
  75. /*
  76. * If truncate cannot remove the fs-private metadata from the page, the page
  77. * becomes anonymous. It will be left on the LRU and may even be mapped into
  78. * user pagetables if we're racing with filemap_nopage().
  79. *
  80. * We need to bale out if page->mapping is no longer equal to the original
  81. * mapping. This happens a) when the VM reclaimed the page while we waited on
  82. * its lock, b) when a concurrent invalidate_mapping_pages got there first and
  83. * c) when tmpfs swizzles a page between a tmpfs inode and swapper_space.
  84. */
  85. static void
  86. truncate_complete_page(struct address_space *mapping, struct page *page)
  87. {
  88. if (page->mapping != mapping)
  89. return;
  90. cancel_dirty_page(page, PAGE_CACHE_SIZE);
  91. if (PagePrivate(page))
  92. do_invalidatepage(page, 0);
  93. ClearPageUptodate(page);
  94. ClearPageMappedToDisk(page);
  95. remove_from_page_cache(page);
  96. page_cache_release(page); /* pagecache ref */
  97. }
  98. /*
  99. * This is for invalidate_mapping_pages(). That function can be called at
  100. * any time, and is not supposed to throw away dirty pages. But pages can
  101. * be marked dirty at any time too, so use remove_mapping which safely
  102. * discards clean, unused pages.
  103. *
  104. * Returns non-zero if the page was successfully invalidated.
  105. */
  106. static int
  107. invalidate_complete_page(struct address_space *mapping, struct page *page)
  108. {
  109. int ret;
  110. if (page->mapping != mapping)
  111. return 0;
  112. if (PagePrivate(page) && !try_to_release_page(page, 0))
  113. return 0;
  114. ret = remove_mapping(mapping, page);
  115. return ret;
  116. }
  117. /**
  118. * truncate_inode_pages - truncate range of pages specified by start and
  119. * end byte offsets
  120. * @mapping: mapping to truncate
  121. * @lstart: offset from which to truncate
  122. * @lend: offset to which to truncate
  123. *
  124. * Truncate the page cache, removing the pages that are between
  125. * specified offsets (and zeroing out partial page
  126. * (if lstart is not page aligned)).
  127. *
  128. * Truncate takes two passes - the first pass is nonblocking. It will not
  129. * block on page locks and it will not block on writeback. The second pass
  130. * will wait. This is to prevent as much IO as possible in the affected region.
  131. * The first pass will remove most pages, so the search cost of the second pass
  132. * is low.
  133. *
  134. * When looking at page->index outside the page lock we need to be careful to
  135. * copy it into a local to avoid races (it could change at any time).
  136. *
  137. * We pass down the cache-hot hint to the page freeing code. Even if the
  138. * mapping is large, it is probably the case that the final pages are the most
  139. * recently touched, and freeing happens in ascending file offset order.
  140. */
  141. void truncate_inode_pages_range(struct address_space *mapping,
  142. loff_t lstart, loff_t lend)
  143. {
  144. const pgoff_t start = (lstart + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
  145. pgoff_t end;
  146. const unsigned partial = lstart & (PAGE_CACHE_SIZE - 1);
  147. struct pagevec pvec;
  148. pgoff_t next;
  149. int i;
  150. if (mapping->nrpages == 0)
  151. return;
  152. BUG_ON((lend & (PAGE_CACHE_SIZE - 1)) != (PAGE_CACHE_SIZE - 1));
  153. end = (lend >> PAGE_CACHE_SHIFT);
  154. pagevec_init(&pvec, 0);
  155. next = start;
  156. while (next <= end &&
  157. pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
  158. for (i = 0; i < pagevec_count(&pvec); i++) {
  159. struct page *page = pvec.pages[i];
  160. pgoff_t page_index = page->index;
  161. if (page_index > end) {
  162. next = page_index;
  163. break;
  164. }
  165. if (page_index > next)
  166. next = page_index;
  167. next++;
  168. if (TestSetPageLocked(page))
  169. continue;
  170. if (PageWriteback(page)) {
  171. unlock_page(page);
  172. continue;
  173. }
  174. truncate_complete_page(mapping, page);
  175. unlock_page(page);
  176. }
  177. pagevec_release(&pvec);
  178. cond_resched();
  179. }
  180. if (partial) {
  181. struct page *page = find_lock_page(mapping, start - 1);
  182. if (page) {
  183. wait_on_page_writeback(page);
  184. truncate_partial_page(page, partial);
  185. unlock_page(page);
  186. page_cache_release(page);
  187. }
  188. }
  189. next = start;
  190. for ( ; ; ) {
  191. cond_resched();
  192. if (!pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
  193. if (next == start)
  194. break;
  195. next = start;
  196. continue;
  197. }
  198. if (pvec.pages[0]->index > end) {
  199. pagevec_release(&pvec);
  200. break;
  201. }
  202. for (i = 0; i < pagevec_count(&pvec); i++) {
  203. struct page *page = pvec.pages[i];
  204. if (page->index > end)
  205. break;
  206. lock_page(page);
  207. wait_on_page_writeback(page);
  208. if (page->index > next)
  209. next = page->index;
  210. next++;
  211. truncate_complete_page(mapping, page);
  212. unlock_page(page);
  213. }
  214. pagevec_release(&pvec);
  215. }
  216. }
  217. EXPORT_SYMBOL(truncate_inode_pages_range);
  218. /**
  219. * truncate_inode_pages - truncate *all* the pages from an offset
  220. * @mapping: mapping to truncate
  221. * @lstart: offset from which to truncate
  222. *
  223. * Called under (and serialised by) inode->i_mutex.
  224. */
  225. void truncate_inode_pages(struct address_space *mapping, loff_t lstart)
  226. {
  227. truncate_inode_pages_range(mapping, lstart, (loff_t)-1);
  228. }
  229. EXPORT_SYMBOL(truncate_inode_pages);
  230. /**
  231. * invalidate_mapping_pages - Invalidate all the unlocked pages of one inode
  232. * @mapping: the address_space which holds the pages to invalidate
  233. * @start: the offset 'from' which to invalidate
  234. * @end: the offset 'to' which to invalidate (inclusive)
  235. *
  236. * This function only removes the unlocked pages, if you want to
  237. * remove all the pages of one inode, you must call truncate_inode_pages.
  238. *
  239. * invalidate_mapping_pages() will not block on IO activity. It will not
  240. * invalidate pages which are dirty, locked, under writeback or mapped into
  241. * pagetables.
  242. */
  243. unsigned long invalidate_mapping_pages(struct address_space *mapping,
  244. pgoff_t start, pgoff_t end)
  245. {
  246. struct pagevec pvec;
  247. pgoff_t next = start;
  248. unsigned long ret = 0;
  249. int i;
  250. pagevec_init(&pvec, 0);
  251. while (next <= end &&
  252. pagevec_lookup(&pvec, mapping, next, PAGEVEC_SIZE)) {
  253. for (i = 0; i < pagevec_count(&pvec); i++) {
  254. struct page *page = pvec.pages[i];
  255. pgoff_t index;
  256. int lock_failed;
  257. lock_failed = TestSetPageLocked(page);
  258. /*
  259. * We really shouldn't be looking at the ->index of an
  260. * unlocked page. But we're not allowed to lock these
  261. * pages. So we rely upon nobody altering the ->index
  262. * of this (pinned-by-us) page.
  263. */
  264. index = page->index;
  265. if (index > next)
  266. next = index;
  267. next++;
  268. if (lock_failed)
  269. continue;
  270. if (PageDirty(page) || PageWriteback(page))
  271. goto unlock;
  272. if (page_mapped(page))
  273. goto unlock;
  274. ret += invalidate_complete_page(mapping, page);
  275. unlock:
  276. unlock_page(page);
  277. if (next > end)
  278. break;
  279. }
  280. pagevec_release(&pvec);
  281. }
  282. return ret;
  283. }
  284. EXPORT_SYMBOL(invalidate_mapping_pages);
  285. /*
  286. * This is like invalidate_complete_page(), except it ignores the page's
  287. * refcount. We do this because invalidate_inode_pages2() needs stronger
  288. * invalidation guarantees, and cannot afford to leave pages behind because
  289. * shrink_list() has a temp ref on them, or because they're transiently sitting
  290. * in the lru_cache_add() pagevecs.
  291. */
  292. static int
  293. invalidate_complete_page2(struct address_space *mapping, struct page *page)
  294. {
  295. if (page->mapping != mapping)
  296. return 0;
  297. if (PagePrivate(page) && !try_to_release_page(page, GFP_KERNEL))
  298. return 0;
  299. write_lock_irq(&mapping->tree_lock);
  300. if (PageDirty(page))
  301. goto failed;
  302. BUG_ON(PagePrivate(page));
  303. __remove_from_page_cache(page);
  304. write_unlock_irq(&mapping->tree_lock);
  305. ClearPageUptodate(page);
  306. page_cache_release(page); /* pagecache ref */
  307. return 1;
  308. failed:
  309. write_unlock_irq(&mapping->tree_lock);
  310. return 0;
  311. }
  312. static int do_launder_page(struct address_space *mapping, struct page *page)
  313. {
  314. if (!PageDirty(page))
  315. return 0;
  316. if (page->mapping != mapping || mapping->a_ops->launder_page == NULL)
  317. return 0;
  318. return mapping->a_ops->launder_page(page);
  319. }
  320. /**
  321. * invalidate_inode_pages2_range - remove range of pages from an address_space
  322. * @mapping: the address_space
  323. * @start: the page offset 'from' which to invalidate
  324. * @end: the page offset 'to' which to invalidate (inclusive)
  325. *
  326. * Any pages which are found to be mapped into pagetables are unmapped prior to
  327. * invalidation.
  328. *
  329. * Returns -EIO if any pages could not be invalidated.
  330. */
  331. int invalidate_inode_pages2_range(struct address_space *mapping,
  332. pgoff_t start, pgoff_t end)
  333. {
  334. struct pagevec pvec;
  335. pgoff_t next;
  336. int i;
  337. int ret = 0;
  338. int did_range_unmap = 0;
  339. int wrapped = 0;
  340. pagevec_init(&pvec, 0);
  341. next = start;
  342. while (next <= end && !wrapped &&
  343. pagevec_lookup(&pvec, mapping, next,
  344. min(end - next, (pgoff_t)PAGEVEC_SIZE - 1) + 1)) {
  345. for (i = 0; i < pagevec_count(&pvec); i++) {
  346. struct page *page = pvec.pages[i];
  347. pgoff_t page_index;
  348. lock_page(page);
  349. if (page->mapping != mapping) {
  350. unlock_page(page);
  351. continue;
  352. }
  353. page_index = page->index;
  354. next = page_index + 1;
  355. if (next == 0)
  356. wrapped = 1;
  357. if (page_index > end) {
  358. unlock_page(page);
  359. break;
  360. }
  361. wait_on_page_writeback(page);
  362. while (page_mapped(page)) {
  363. if (!did_range_unmap) {
  364. /*
  365. * Zap the rest of the file in one hit.
  366. */
  367. unmap_mapping_range(mapping,
  368. (loff_t)page_index<<PAGE_CACHE_SHIFT,
  369. (loff_t)(end - page_index + 1)
  370. << PAGE_CACHE_SHIFT,
  371. 0);
  372. did_range_unmap = 1;
  373. } else {
  374. /*
  375. * Just zap this page
  376. */
  377. unmap_mapping_range(mapping,
  378. (loff_t)page_index<<PAGE_CACHE_SHIFT,
  379. PAGE_CACHE_SIZE, 0);
  380. }
  381. }
  382. ret = do_launder_page(mapping, page);
  383. if (ret == 0 && !invalidate_complete_page2(mapping, page))
  384. ret = -EIO;
  385. unlock_page(page);
  386. }
  387. pagevec_release(&pvec);
  388. cond_resched();
  389. }
  390. return ret;
  391. }
  392. EXPORT_SYMBOL_GPL(invalidate_inode_pages2_range);
  393. /**
  394. * invalidate_inode_pages2 - remove all pages from an address_space
  395. * @mapping: the address_space
  396. *
  397. * Any pages which are found to be mapped into pagetables are unmapped prior to
  398. * invalidation.
  399. *
  400. * Returns -EIO if any pages could not be invalidated.
  401. */
  402. int invalidate_inode_pages2(struct address_space *mapping)
  403. {
  404. return invalidate_inode_pages2_range(mapping, 0, -1);
  405. }
  406. EXPORT_SYMBOL_GPL(invalidate_inode_pages2);