swap.c 12 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520
  1. /*
  2. * linux/mm/swap.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. */
  6. /*
  7. * This file contains the default values for the opereation of the
  8. * Linux VM subsystem. Fine-tuning documentation can be found in
  9. * Documentation/sysctl/vm.txt.
  10. * Started 18.12.91
  11. * Swap aging added 23.2.95, Stephen Tweedie.
  12. * Buffermem limits added 12.3.98, Rik van Riel.
  13. */
  14. #include <linux/mm.h>
  15. #include <linux/sched.h>
  16. #include <linux/kernel_stat.h>
  17. #include <linux/swap.h>
  18. #include <linux/mman.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/pagevec.h>
  21. #include <linux/init.h>
  22. #include <linux/module.h>
  23. #include <linux/mm_inline.h>
  24. #include <linux/buffer_head.h> /* for try_to_release_page() */
  25. #include <linux/module.h>
  26. #include <linux/percpu_counter.h>
  27. #include <linux/percpu.h>
  28. #include <linux/cpu.h>
  29. #include <linux/notifier.h>
  30. #include <linux/init.h>
  31. /* How many pages do we try to swap or page in/out together? */
  32. int page_cluster;
  33. /*
  34. * This path almost never happens for VM activity - pages are normally
  35. * freed via pagevecs. But it gets used by networking.
  36. */
  37. static void fastcall __page_cache_release(struct page *page)
  38. {
  39. if (PageLRU(page)) {
  40. unsigned long flags;
  41. struct zone *zone = page_zone(page);
  42. spin_lock_irqsave(&zone->lru_lock, flags);
  43. VM_BUG_ON(!PageLRU(page));
  44. __ClearPageLRU(page);
  45. del_page_from_lru(zone, page);
  46. spin_unlock_irqrestore(&zone->lru_lock, flags);
  47. }
  48. free_hot_page(page);
  49. }
  50. static void put_compound_page(struct page *page)
  51. {
  52. page = (struct page *)page_private(page);
  53. if (put_page_testzero(page)) {
  54. compound_page_dtor *dtor;
  55. dtor = get_compound_page_dtor(page);
  56. (*dtor)(page);
  57. }
  58. }
  59. void put_page(struct page *page)
  60. {
  61. if (unlikely(PageCompound(page)))
  62. put_compound_page(page);
  63. else if (put_page_testzero(page))
  64. __page_cache_release(page);
  65. }
  66. EXPORT_SYMBOL(put_page);
  67. /**
  68. * put_pages_list(): release a list of pages
  69. *
  70. * Release a list of pages which are strung together on page.lru. Currently
  71. * used by read_cache_pages() and related error recovery code.
  72. *
  73. * @pages: list of pages threaded on page->lru
  74. */
  75. void put_pages_list(struct list_head *pages)
  76. {
  77. while (!list_empty(pages)) {
  78. struct page *victim;
  79. victim = list_entry(pages->prev, struct page, lru);
  80. list_del(&victim->lru);
  81. page_cache_release(victim);
  82. }
  83. }
  84. EXPORT_SYMBOL(put_pages_list);
  85. /*
  86. * Writeback is about to end against a page which has been marked for immediate
  87. * reclaim. If it still appears to be reclaimable, move it to the tail of the
  88. * inactive list. The page still has PageWriteback set, which will pin it.
  89. *
  90. * We don't expect many pages to come through here, so don't bother batching
  91. * things up.
  92. *
  93. * To avoid placing the page at the tail of the LRU while PG_writeback is still
  94. * set, this function will clear PG_writeback before performing the page
  95. * motion. Do that inside the lru lock because once PG_writeback is cleared
  96. * we may not touch the page.
  97. *
  98. * Returns zero if it cleared PG_writeback.
  99. */
  100. int rotate_reclaimable_page(struct page *page)
  101. {
  102. struct zone *zone;
  103. unsigned long flags;
  104. if (PageLocked(page))
  105. return 1;
  106. if (PageDirty(page))
  107. return 1;
  108. if (PageActive(page))
  109. return 1;
  110. if (!PageLRU(page))
  111. return 1;
  112. zone = page_zone(page);
  113. spin_lock_irqsave(&zone->lru_lock, flags);
  114. if (PageLRU(page) && !PageActive(page)) {
  115. list_move_tail(&page->lru, &zone->inactive_list);
  116. __count_vm_event(PGROTATED);
  117. }
  118. if (!test_clear_page_writeback(page))
  119. BUG();
  120. spin_unlock_irqrestore(&zone->lru_lock, flags);
  121. return 0;
  122. }
  123. /*
  124. * FIXME: speed this up?
  125. */
  126. void fastcall activate_page(struct page *page)
  127. {
  128. struct zone *zone = page_zone(page);
  129. spin_lock_irq(&zone->lru_lock);
  130. if (PageLRU(page) && !PageActive(page)) {
  131. del_page_from_inactive_list(zone, page);
  132. SetPageActive(page);
  133. add_page_to_active_list(zone, page);
  134. __count_vm_event(PGACTIVATE);
  135. }
  136. spin_unlock_irq(&zone->lru_lock);
  137. }
  138. /*
  139. * Mark a page as having seen activity.
  140. *
  141. * inactive,unreferenced -> inactive,referenced
  142. * inactive,referenced -> active,unreferenced
  143. * active,unreferenced -> active,referenced
  144. */
  145. void fastcall mark_page_accessed(struct page *page)
  146. {
  147. if (!PageActive(page) && PageReferenced(page) && PageLRU(page)) {
  148. activate_page(page);
  149. ClearPageReferenced(page);
  150. } else if (!PageReferenced(page)) {
  151. SetPageReferenced(page);
  152. }
  153. }
  154. EXPORT_SYMBOL(mark_page_accessed);
  155. /**
  156. * lru_cache_add: add a page to the page lists
  157. * @page: the page to add
  158. */
  159. static DEFINE_PER_CPU(struct pagevec, lru_add_pvecs) = { 0, };
  160. static DEFINE_PER_CPU(struct pagevec, lru_add_active_pvecs) = { 0, };
  161. void fastcall lru_cache_add(struct page *page)
  162. {
  163. struct pagevec *pvec = &get_cpu_var(lru_add_pvecs);
  164. page_cache_get(page);
  165. if (!pagevec_add(pvec, page))
  166. __pagevec_lru_add(pvec);
  167. put_cpu_var(lru_add_pvecs);
  168. }
  169. void fastcall lru_cache_add_active(struct page *page)
  170. {
  171. struct pagevec *pvec = &get_cpu_var(lru_add_active_pvecs);
  172. page_cache_get(page);
  173. if (!pagevec_add(pvec, page))
  174. __pagevec_lru_add_active(pvec);
  175. put_cpu_var(lru_add_active_pvecs);
  176. }
  177. static void __lru_add_drain(int cpu)
  178. {
  179. struct pagevec *pvec = &per_cpu(lru_add_pvecs, cpu);
  180. /* CPU is dead, so no locking needed. */
  181. if (pagevec_count(pvec))
  182. __pagevec_lru_add(pvec);
  183. pvec = &per_cpu(lru_add_active_pvecs, cpu);
  184. if (pagevec_count(pvec))
  185. __pagevec_lru_add_active(pvec);
  186. }
  187. void lru_add_drain(void)
  188. {
  189. __lru_add_drain(get_cpu());
  190. put_cpu();
  191. }
  192. #ifdef CONFIG_NUMA
  193. static void lru_add_drain_per_cpu(struct work_struct *dummy)
  194. {
  195. lru_add_drain();
  196. }
  197. /*
  198. * Returns 0 for success
  199. */
  200. int lru_add_drain_all(void)
  201. {
  202. return schedule_on_each_cpu(lru_add_drain_per_cpu);
  203. }
  204. #else
  205. /*
  206. * Returns 0 for success
  207. */
  208. int lru_add_drain_all(void)
  209. {
  210. lru_add_drain();
  211. return 0;
  212. }
  213. #endif
  214. /*
  215. * Batched page_cache_release(). Decrement the reference count on all the
  216. * passed pages. If it fell to zero then remove the page from the LRU and
  217. * free it.
  218. *
  219. * Avoid taking zone->lru_lock if possible, but if it is taken, retain it
  220. * for the remainder of the operation.
  221. *
  222. * The locking in this function is against shrink_cache(): we recheck the
  223. * page count inside the lock to see whether shrink_cache grabbed the page
  224. * via the LRU. If it did, give up: shrink_cache will free it.
  225. */
  226. void release_pages(struct page **pages, int nr, int cold)
  227. {
  228. int i;
  229. struct pagevec pages_to_free;
  230. struct zone *zone = NULL;
  231. pagevec_init(&pages_to_free, cold);
  232. for (i = 0; i < nr; i++) {
  233. struct page *page = pages[i];
  234. if (unlikely(PageCompound(page))) {
  235. if (zone) {
  236. spin_unlock_irq(&zone->lru_lock);
  237. zone = NULL;
  238. }
  239. put_compound_page(page);
  240. continue;
  241. }
  242. if (!put_page_testzero(page))
  243. continue;
  244. if (PageLRU(page)) {
  245. struct zone *pagezone = page_zone(page);
  246. if (pagezone != zone) {
  247. if (zone)
  248. spin_unlock_irq(&zone->lru_lock);
  249. zone = pagezone;
  250. spin_lock_irq(&zone->lru_lock);
  251. }
  252. VM_BUG_ON(!PageLRU(page));
  253. __ClearPageLRU(page);
  254. del_page_from_lru(zone, page);
  255. }
  256. if (!pagevec_add(&pages_to_free, page)) {
  257. if (zone) {
  258. spin_unlock_irq(&zone->lru_lock);
  259. zone = NULL;
  260. }
  261. __pagevec_free(&pages_to_free);
  262. pagevec_reinit(&pages_to_free);
  263. }
  264. }
  265. if (zone)
  266. spin_unlock_irq(&zone->lru_lock);
  267. pagevec_free(&pages_to_free);
  268. }
  269. /*
  270. * The pages which we're about to release may be in the deferred lru-addition
  271. * queues. That would prevent them from really being freed right now. That's
  272. * OK from a correctness point of view but is inefficient - those pages may be
  273. * cache-warm and we want to give them back to the page allocator ASAP.
  274. *
  275. * So __pagevec_release() will drain those queues here. __pagevec_lru_add()
  276. * and __pagevec_lru_add_active() call release_pages() directly to avoid
  277. * mutual recursion.
  278. */
  279. void __pagevec_release(struct pagevec *pvec)
  280. {
  281. lru_add_drain();
  282. release_pages(pvec->pages, pagevec_count(pvec), pvec->cold);
  283. pagevec_reinit(pvec);
  284. }
  285. EXPORT_SYMBOL(__pagevec_release);
  286. /*
  287. * pagevec_release() for pages which are known to not be on the LRU
  288. *
  289. * This function reinitialises the caller's pagevec.
  290. */
  291. void __pagevec_release_nonlru(struct pagevec *pvec)
  292. {
  293. int i;
  294. struct pagevec pages_to_free;
  295. pagevec_init(&pages_to_free, pvec->cold);
  296. for (i = 0; i < pagevec_count(pvec); i++) {
  297. struct page *page = pvec->pages[i];
  298. VM_BUG_ON(PageLRU(page));
  299. if (put_page_testzero(page))
  300. pagevec_add(&pages_to_free, page);
  301. }
  302. pagevec_free(&pages_to_free);
  303. pagevec_reinit(pvec);
  304. }
  305. /*
  306. * Add the passed pages to the LRU, then drop the caller's refcount
  307. * on them. Reinitialises the caller's pagevec.
  308. */
  309. void __pagevec_lru_add(struct pagevec *pvec)
  310. {
  311. int i;
  312. struct zone *zone = NULL;
  313. for (i = 0; i < pagevec_count(pvec); i++) {
  314. struct page *page = pvec->pages[i];
  315. struct zone *pagezone = page_zone(page);
  316. if (pagezone != zone) {
  317. if (zone)
  318. spin_unlock_irq(&zone->lru_lock);
  319. zone = pagezone;
  320. spin_lock_irq(&zone->lru_lock);
  321. }
  322. VM_BUG_ON(PageLRU(page));
  323. SetPageLRU(page);
  324. add_page_to_inactive_list(zone, page);
  325. }
  326. if (zone)
  327. spin_unlock_irq(&zone->lru_lock);
  328. release_pages(pvec->pages, pvec->nr, pvec->cold);
  329. pagevec_reinit(pvec);
  330. }
  331. EXPORT_SYMBOL(__pagevec_lru_add);
  332. void __pagevec_lru_add_active(struct pagevec *pvec)
  333. {
  334. int i;
  335. struct zone *zone = NULL;
  336. for (i = 0; i < pagevec_count(pvec); i++) {
  337. struct page *page = pvec->pages[i];
  338. struct zone *pagezone = page_zone(page);
  339. if (pagezone != zone) {
  340. if (zone)
  341. spin_unlock_irq(&zone->lru_lock);
  342. zone = pagezone;
  343. spin_lock_irq(&zone->lru_lock);
  344. }
  345. VM_BUG_ON(PageLRU(page));
  346. SetPageLRU(page);
  347. VM_BUG_ON(PageActive(page));
  348. SetPageActive(page);
  349. add_page_to_active_list(zone, page);
  350. }
  351. if (zone)
  352. spin_unlock_irq(&zone->lru_lock);
  353. release_pages(pvec->pages, pvec->nr, pvec->cold);
  354. pagevec_reinit(pvec);
  355. }
  356. /*
  357. * Try to drop buffers from the pages in a pagevec
  358. */
  359. void pagevec_strip(struct pagevec *pvec)
  360. {
  361. int i;
  362. for (i = 0; i < pagevec_count(pvec); i++) {
  363. struct page *page = pvec->pages[i];
  364. if (PagePrivate(page) && !TestSetPageLocked(page)) {
  365. if (PagePrivate(page))
  366. try_to_release_page(page, 0);
  367. unlock_page(page);
  368. }
  369. }
  370. }
  371. /**
  372. * pagevec_lookup - gang pagecache lookup
  373. * @pvec: Where the resulting pages are placed
  374. * @mapping: The address_space to search
  375. * @start: The starting page index
  376. * @nr_pages: The maximum number of pages
  377. *
  378. * pagevec_lookup() will search for and return a group of up to @nr_pages pages
  379. * in the mapping. The pages are placed in @pvec. pagevec_lookup() takes a
  380. * reference against the pages in @pvec.
  381. *
  382. * The search returns a group of mapping-contiguous pages with ascending
  383. * indexes. There may be holes in the indices due to not-present pages.
  384. *
  385. * pagevec_lookup() returns the number of pages which were found.
  386. */
  387. unsigned pagevec_lookup(struct pagevec *pvec, struct address_space *mapping,
  388. pgoff_t start, unsigned nr_pages)
  389. {
  390. pvec->nr = find_get_pages(mapping, start, nr_pages, pvec->pages);
  391. return pagevec_count(pvec);
  392. }
  393. EXPORT_SYMBOL(pagevec_lookup);
  394. unsigned pagevec_lookup_tag(struct pagevec *pvec, struct address_space *mapping,
  395. pgoff_t *index, int tag, unsigned nr_pages)
  396. {
  397. pvec->nr = find_get_pages_tag(mapping, index, tag,
  398. nr_pages, pvec->pages);
  399. return pagevec_count(pvec);
  400. }
  401. EXPORT_SYMBOL(pagevec_lookup_tag);
  402. #ifdef CONFIG_SMP
  403. /*
  404. * We tolerate a little inaccuracy to avoid ping-ponging the counter between
  405. * CPUs
  406. */
  407. #define ACCT_THRESHOLD max(16, NR_CPUS * 2)
  408. static DEFINE_PER_CPU(long, committed_space) = 0;
  409. void vm_acct_memory(long pages)
  410. {
  411. long *local;
  412. preempt_disable();
  413. local = &__get_cpu_var(committed_space);
  414. *local += pages;
  415. if (*local > ACCT_THRESHOLD || *local < -ACCT_THRESHOLD) {
  416. atomic_add(*local, &vm_committed_space);
  417. *local = 0;
  418. }
  419. preempt_enable();
  420. }
  421. #ifdef CONFIG_HOTPLUG_CPU
  422. /* Drop the CPU's cached committed space back into the central pool. */
  423. static int cpu_swap_callback(struct notifier_block *nfb,
  424. unsigned long action,
  425. void *hcpu)
  426. {
  427. long *committed;
  428. committed = &per_cpu(committed_space, (long)hcpu);
  429. if (action == CPU_DEAD) {
  430. atomic_add(*committed, &vm_committed_space);
  431. *committed = 0;
  432. __lru_add_drain((long)hcpu);
  433. }
  434. return NOTIFY_OK;
  435. }
  436. #endif /* CONFIG_HOTPLUG_CPU */
  437. #endif /* CONFIG_SMP */
  438. /*
  439. * Perform any setup for the swap system
  440. */
  441. void __init swap_setup(void)
  442. {
  443. unsigned long megs = num_physpages >> (20 - PAGE_SHIFT);
  444. /* Use a smaller cluster for small-memory machines */
  445. if (megs < 16)
  446. page_cluster = 2;
  447. else
  448. page_cluster = 3;
  449. /*
  450. * Right now other parts of the system means that we
  451. * _really_ don't want to cluster much more
  452. */
  453. #ifdef CONFIG_HOTPLUG_CPU
  454. hotcpu_notifier(cpu_swap_callback, 0);
  455. #endif
  456. }