vmscan.c 47 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730
  1. /*
  2. * linux/mm/vmscan.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. *
  6. * Swap reorganised 29.12.95, Stephen Tweedie.
  7. * kswapd added: 7.1.96 sct
  8. * Removed kswapd_ctl limits, and swap out as many pages as needed
  9. * to bring the system back to freepages.high: 2.4.97, Rik van Riel.
  10. * Zone aware kswapd started 02/00, Kanoj Sarcar (kanoj@sgi.com).
  11. * Multiqueue VM started 5.8.00, Rik van Riel.
  12. */
  13. #include <linux/mm.h>
  14. #include <linux/module.h>
  15. #include <linux/slab.h>
  16. #include <linux/kernel_stat.h>
  17. #include <linux/swap.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/init.h>
  20. #include <linux/highmem.h>
  21. #include <linux/vmstat.h>
  22. #include <linux/file.h>
  23. #include <linux/writeback.h>
  24. #include <linux/blkdev.h>
  25. #include <linux/buffer_head.h> /* for try_to_release_page(),
  26. buffer_heads_over_limit */
  27. #include <linux/mm_inline.h>
  28. #include <linux/pagevec.h>
  29. #include <linux/backing-dev.h>
  30. #include <linux/rmap.h>
  31. #include <linux/topology.h>
  32. #include <linux/cpu.h>
  33. #include <linux/cpuset.h>
  34. #include <linux/notifier.h>
  35. #include <linux/rwsem.h>
  36. #include <linux/delay.h>
  37. #include <linux/kthread.h>
  38. #include <linux/freezer.h>
  39. #include <asm/tlbflush.h>
  40. #include <asm/div64.h>
  41. #include <linux/swapops.h>
  42. #include "internal.h"
  43. struct scan_control {
  44. /* Incremented by the number of inactive pages that were scanned */
  45. unsigned long nr_scanned;
  46. /* This context's GFP mask */
  47. gfp_t gfp_mask;
  48. int may_writepage;
  49. /* Can pages be swapped as part of reclaim? */
  50. int may_swap;
  51. /* This context's SWAP_CLUSTER_MAX. If freeing memory for
  52. * suspend, we effectively ignore SWAP_CLUSTER_MAX.
  53. * In this context, it doesn't matter that we scan the
  54. * whole list at once. */
  55. int swap_cluster_max;
  56. int swappiness;
  57. int all_unreclaimable;
  58. };
  59. /*
  60. * The list of shrinker callbacks used by to apply pressure to
  61. * ageable caches.
  62. */
  63. struct shrinker {
  64. shrinker_t shrinker;
  65. struct list_head list;
  66. int seeks; /* seeks to recreate an obj */
  67. long nr; /* objs pending delete */
  68. };
  69. #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
  70. #ifdef ARCH_HAS_PREFETCH
  71. #define prefetch_prev_lru_page(_page, _base, _field) \
  72. do { \
  73. if ((_page)->lru.prev != _base) { \
  74. struct page *prev; \
  75. \
  76. prev = lru_to_page(&(_page->lru)); \
  77. prefetch(&prev->_field); \
  78. } \
  79. } while (0)
  80. #else
  81. #define prefetch_prev_lru_page(_page, _base, _field) do { } while (0)
  82. #endif
  83. #ifdef ARCH_HAS_PREFETCHW
  84. #define prefetchw_prev_lru_page(_page, _base, _field) \
  85. do { \
  86. if ((_page)->lru.prev != _base) { \
  87. struct page *prev; \
  88. \
  89. prev = lru_to_page(&(_page->lru)); \
  90. prefetchw(&prev->_field); \
  91. } \
  92. } while (0)
  93. #else
  94. #define prefetchw_prev_lru_page(_page, _base, _field) do { } while (0)
  95. #endif
  96. /*
  97. * From 0 .. 100. Higher means more swappy.
  98. */
  99. int vm_swappiness = 60;
  100. long vm_total_pages; /* The total number of pages which the VM controls */
  101. static LIST_HEAD(shrinker_list);
  102. static DECLARE_RWSEM(shrinker_rwsem);
  103. /*
  104. * Add a shrinker callback to be called from the vm
  105. */
  106. struct shrinker *set_shrinker(int seeks, shrinker_t theshrinker)
  107. {
  108. struct shrinker *shrinker;
  109. shrinker = kmalloc(sizeof(*shrinker), GFP_KERNEL);
  110. if (shrinker) {
  111. shrinker->shrinker = theshrinker;
  112. shrinker->seeks = seeks;
  113. shrinker->nr = 0;
  114. down_write(&shrinker_rwsem);
  115. list_add_tail(&shrinker->list, &shrinker_list);
  116. up_write(&shrinker_rwsem);
  117. }
  118. return shrinker;
  119. }
  120. EXPORT_SYMBOL(set_shrinker);
  121. /*
  122. * Remove one
  123. */
  124. void remove_shrinker(struct shrinker *shrinker)
  125. {
  126. down_write(&shrinker_rwsem);
  127. list_del(&shrinker->list);
  128. up_write(&shrinker_rwsem);
  129. kfree(shrinker);
  130. }
  131. EXPORT_SYMBOL(remove_shrinker);
  132. #define SHRINK_BATCH 128
  133. /*
  134. * Call the shrink functions to age shrinkable caches
  135. *
  136. * Here we assume it costs one seek to replace a lru page and that it also
  137. * takes a seek to recreate a cache object. With this in mind we age equal
  138. * percentages of the lru and ageable caches. This should balance the seeks
  139. * generated by these structures.
  140. *
  141. * If the vm encounted mapped pages on the LRU it increase the pressure on
  142. * slab to avoid swapping.
  143. *
  144. * We do weird things to avoid (scanned*seeks*entries) overflowing 32 bits.
  145. *
  146. * `lru_pages' represents the number of on-LRU pages in all the zones which
  147. * are eligible for the caller's allocation attempt. It is used for balancing
  148. * slab reclaim versus page reclaim.
  149. *
  150. * Returns the number of slab objects which we shrunk.
  151. */
  152. unsigned long shrink_slab(unsigned long scanned, gfp_t gfp_mask,
  153. unsigned long lru_pages)
  154. {
  155. struct shrinker *shrinker;
  156. unsigned long ret = 0;
  157. if (scanned == 0)
  158. scanned = SWAP_CLUSTER_MAX;
  159. if (!down_read_trylock(&shrinker_rwsem))
  160. return 1; /* Assume we'll be able to shrink next time */
  161. list_for_each_entry(shrinker, &shrinker_list, list) {
  162. unsigned long long delta;
  163. unsigned long total_scan;
  164. unsigned long max_pass = (*shrinker->shrinker)(0, gfp_mask);
  165. delta = (4 * scanned) / shrinker->seeks;
  166. delta *= max_pass;
  167. do_div(delta, lru_pages + 1);
  168. shrinker->nr += delta;
  169. if (shrinker->nr < 0) {
  170. printk(KERN_ERR "%s: nr=%ld\n",
  171. __FUNCTION__, shrinker->nr);
  172. shrinker->nr = max_pass;
  173. }
  174. /*
  175. * Avoid risking looping forever due to too large nr value:
  176. * never try to free more than twice the estimate number of
  177. * freeable entries.
  178. */
  179. if (shrinker->nr > max_pass * 2)
  180. shrinker->nr = max_pass * 2;
  181. total_scan = shrinker->nr;
  182. shrinker->nr = 0;
  183. while (total_scan >= SHRINK_BATCH) {
  184. long this_scan = SHRINK_BATCH;
  185. int shrink_ret;
  186. int nr_before;
  187. nr_before = (*shrinker->shrinker)(0, gfp_mask);
  188. shrink_ret = (*shrinker->shrinker)(this_scan, gfp_mask);
  189. if (shrink_ret == -1)
  190. break;
  191. if (shrink_ret < nr_before)
  192. ret += nr_before - shrink_ret;
  193. count_vm_events(SLABS_SCANNED, this_scan);
  194. total_scan -= this_scan;
  195. cond_resched();
  196. }
  197. shrinker->nr += total_scan;
  198. }
  199. up_read(&shrinker_rwsem);
  200. return ret;
  201. }
  202. /* Called without lock on whether page is mapped, so answer is unstable */
  203. static inline int page_mapping_inuse(struct page *page)
  204. {
  205. struct address_space *mapping;
  206. /* Page is in somebody's page tables. */
  207. if (page_mapped(page))
  208. return 1;
  209. /* Be more reluctant to reclaim swapcache than pagecache */
  210. if (PageSwapCache(page))
  211. return 1;
  212. mapping = page_mapping(page);
  213. if (!mapping)
  214. return 0;
  215. /* File is mmap'd by somebody? */
  216. return mapping_mapped(mapping);
  217. }
  218. static inline int is_page_cache_freeable(struct page *page)
  219. {
  220. return page_count(page) - !!PagePrivate(page) == 2;
  221. }
  222. static int may_write_to_queue(struct backing_dev_info *bdi)
  223. {
  224. if (current->flags & PF_SWAPWRITE)
  225. return 1;
  226. if (!bdi_write_congested(bdi))
  227. return 1;
  228. if (bdi == current->backing_dev_info)
  229. return 1;
  230. return 0;
  231. }
  232. /*
  233. * We detected a synchronous write error writing a page out. Probably
  234. * -ENOSPC. We need to propagate that into the address_space for a subsequent
  235. * fsync(), msync() or close().
  236. *
  237. * The tricky part is that after writepage we cannot touch the mapping: nothing
  238. * prevents it from being freed up. But we have a ref on the page and once
  239. * that page is locked, the mapping is pinned.
  240. *
  241. * We're allowed to run sleeping lock_page() here because we know the caller has
  242. * __GFP_FS.
  243. */
  244. static void handle_write_error(struct address_space *mapping,
  245. struct page *page, int error)
  246. {
  247. lock_page(page);
  248. if (page_mapping(page) == mapping) {
  249. if (error == -ENOSPC)
  250. set_bit(AS_ENOSPC, &mapping->flags);
  251. else
  252. set_bit(AS_EIO, &mapping->flags);
  253. }
  254. unlock_page(page);
  255. }
  256. /* possible outcome of pageout() */
  257. typedef enum {
  258. /* failed to write page out, page is locked */
  259. PAGE_KEEP,
  260. /* move page to the active list, page is locked */
  261. PAGE_ACTIVATE,
  262. /* page has been sent to the disk successfully, page is unlocked */
  263. PAGE_SUCCESS,
  264. /* page is clean and locked */
  265. PAGE_CLEAN,
  266. } pageout_t;
  267. /*
  268. * pageout is called by shrink_page_list() for each dirty page.
  269. * Calls ->writepage().
  270. */
  271. static pageout_t pageout(struct page *page, struct address_space *mapping)
  272. {
  273. /*
  274. * If the page is dirty, only perform writeback if that write
  275. * will be non-blocking. To prevent this allocation from being
  276. * stalled by pagecache activity. But note that there may be
  277. * stalls if we need to run get_block(). We could test
  278. * PagePrivate for that.
  279. *
  280. * If this process is currently in generic_file_write() against
  281. * this page's queue, we can perform writeback even if that
  282. * will block.
  283. *
  284. * If the page is swapcache, write it back even if that would
  285. * block, for some throttling. This happens by accident, because
  286. * swap_backing_dev_info is bust: it doesn't reflect the
  287. * congestion state of the swapdevs. Easy to fix, if needed.
  288. * See swapfile.c:page_queue_congested().
  289. */
  290. if (!is_page_cache_freeable(page))
  291. return PAGE_KEEP;
  292. if (!mapping) {
  293. /*
  294. * Some data journaling orphaned pages can have
  295. * page->mapping == NULL while being dirty with clean buffers.
  296. */
  297. if (PagePrivate(page)) {
  298. if (try_to_free_buffers(page)) {
  299. ClearPageDirty(page);
  300. printk("%s: orphaned page\n", __FUNCTION__);
  301. return PAGE_CLEAN;
  302. }
  303. }
  304. return PAGE_KEEP;
  305. }
  306. if (mapping->a_ops->writepage == NULL)
  307. return PAGE_ACTIVATE;
  308. if (!may_write_to_queue(mapping->backing_dev_info))
  309. return PAGE_KEEP;
  310. if (clear_page_dirty_for_io(page)) {
  311. int res;
  312. struct writeback_control wbc = {
  313. .sync_mode = WB_SYNC_NONE,
  314. .nr_to_write = SWAP_CLUSTER_MAX,
  315. .range_start = 0,
  316. .range_end = LLONG_MAX,
  317. .nonblocking = 1,
  318. .for_reclaim = 1,
  319. };
  320. SetPageReclaim(page);
  321. res = mapping->a_ops->writepage(page, &wbc);
  322. if (res < 0)
  323. handle_write_error(mapping, page, res);
  324. if (res == AOP_WRITEPAGE_ACTIVATE) {
  325. ClearPageReclaim(page);
  326. return PAGE_ACTIVATE;
  327. }
  328. if (!PageWriteback(page)) {
  329. /* synchronous write or broken a_ops? */
  330. ClearPageReclaim(page);
  331. }
  332. inc_zone_page_state(page, NR_VMSCAN_WRITE);
  333. return PAGE_SUCCESS;
  334. }
  335. return PAGE_CLEAN;
  336. }
  337. /*
  338. * Attempt to detach a locked page from its ->mapping. If it is dirty or if
  339. * someone else has a ref on the page, abort and return 0. If it was
  340. * successfully detached, return 1. Assumes the caller has a single ref on
  341. * this page.
  342. */
  343. int remove_mapping(struct address_space *mapping, struct page *page)
  344. {
  345. BUG_ON(!PageLocked(page));
  346. BUG_ON(mapping != page_mapping(page));
  347. write_lock_irq(&mapping->tree_lock);
  348. /*
  349. * The non racy check for a busy page.
  350. *
  351. * Must be careful with the order of the tests. When someone has
  352. * a ref to the page, it may be possible that they dirty it then
  353. * drop the reference. So if PageDirty is tested before page_count
  354. * here, then the following race may occur:
  355. *
  356. * get_user_pages(&page);
  357. * [user mapping goes away]
  358. * write_to(page);
  359. * !PageDirty(page) [good]
  360. * SetPageDirty(page);
  361. * put_page(page);
  362. * !page_count(page) [good, discard it]
  363. *
  364. * [oops, our write_to data is lost]
  365. *
  366. * Reversing the order of the tests ensures such a situation cannot
  367. * escape unnoticed. The smp_rmb is needed to ensure the page->flags
  368. * load is not satisfied before that of page->_count.
  369. *
  370. * Note that if SetPageDirty is always performed via set_page_dirty,
  371. * and thus under tree_lock, then this ordering is not required.
  372. */
  373. if (unlikely(page_count(page) != 2))
  374. goto cannot_free;
  375. smp_rmb();
  376. if (unlikely(PageDirty(page)))
  377. goto cannot_free;
  378. if (PageSwapCache(page)) {
  379. swp_entry_t swap = { .val = page_private(page) };
  380. __delete_from_swap_cache(page);
  381. write_unlock_irq(&mapping->tree_lock);
  382. swap_free(swap);
  383. __put_page(page); /* The pagecache ref */
  384. return 1;
  385. }
  386. __remove_from_page_cache(page);
  387. write_unlock_irq(&mapping->tree_lock);
  388. __put_page(page);
  389. return 1;
  390. cannot_free:
  391. write_unlock_irq(&mapping->tree_lock);
  392. return 0;
  393. }
  394. /*
  395. * shrink_page_list() returns the number of reclaimed pages
  396. */
  397. static unsigned long shrink_page_list(struct list_head *page_list,
  398. struct scan_control *sc)
  399. {
  400. LIST_HEAD(ret_pages);
  401. struct pagevec freed_pvec;
  402. int pgactivate = 0;
  403. unsigned long nr_reclaimed = 0;
  404. cond_resched();
  405. pagevec_init(&freed_pvec, 1);
  406. while (!list_empty(page_list)) {
  407. struct address_space *mapping;
  408. struct page *page;
  409. int may_enter_fs;
  410. int referenced;
  411. cond_resched();
  412. page = lru_to_page(page_list);
  413. list_del(&page->lru);
  414. if (TestSetPageLocked(page))
  415. goto keep;
  416. VM_BUG_ON(PageActive(page));
  417. sc->nr_scanned++;
  418. if (!sc->may_swap && page_mapped(page))
  419. goto keep_locked;
  420. /* Double the slab pressure for mapped and swapcache pages */
  421. if (page_mapped(page) || PageSwapCache(page))
  422. sc->nr_scanned++;
  423. if (PageWriteback(page))
  424. goto keep_locked;
  425. referenced = page_referenced(page, 1);
  426. /* In active use or really unfreeable? Activate it. */
  427. if (referenced && page_mapping_inuse(page))
  428. goto activate_locked;
  429. #ifdef CONFIG_SWAP
  430. /*
  431. * Anonymous process memory has backing store?
  432. * Try to allocate it some swap space here.
  433. */
  434. if (PageAnon(page) && !PageSwapCache(page))
  435. if (!add_to_swap(page, GFP_ATOMIC))
  436. goto activate_locked;
  437. #endif /* CONFIG_SWAP */
  438. mapping = page_mapping(page);
  439. may_enter_fs = (sc->gfp_mask & __GFP_FS) ||
  440. (PageSwapCache(page) && (sc->gfp_mask & __GFP_IO));
  441. /*
  442. * The page is mapped into the page tables of one or more
  443. * processes. Try to unmap it here.
  444. */
  445. if (page_mapped(page) && mapping) {
  446. switch (try_to_unmap(page, 0)) {
  447. case SWAP_FAIL:
  448. goto activate_locked;
  449. case SWAP_AGAIN:
  450. goto keep_locked;
  451. case SWAP_SUCCESS:
  452. ; /* try to free the page below */
  453. }
  454. }
  455. if (PageDirty(page)) {
  456. if (referenced)
  457. goto keep_locked;
  458. if (!may_enter_fs)
  459. goto keep_locked;
  460. if (!sc->may_writepage)
  461. goto keep_locked;
  462. /* Page is dirty, try to write it out here */
  463. switch(pageout(page, mapping)) {
  464. case PAGE_KEEP:
  465. goto keep_locked;
  466. case PAGE_ACTIVATE:
  467. goto activate_locked;
  468. case PAGE_SUCCESS:
  469. if (PageWriteback(page) || PageDirty(page))
  470. goto keep;
  471. /*
  472. * A synchronous write - probably a ramdisk. Go
  473. * ahead and try to reclaim the page.
  474. */
  475. if (TestSetPageLocked(page))
  476. goto keep;
  477. if (PageDirty(page) || PageWriteback(page))
  478. goto keep_locked;
  479. mapping = page_mapping(page);
  480. case PAGE_CLEAN:
  481. ; /* try to free the page below */
  482. }
  483. }
  484. /*
  485. * If the page has buffers, try to free the buffer mappings
  486. * associated with this page. If we succeed we try to free
  487. * the page as well.
  488. *
  489. * We do this even if the page is PageDirty().
  490. * try_to_release_page() does not perform I/O, but it is
  491. * possible for a page to have PageDirty set, but it is actually
  492. * clean (all its buffers are clean). This happens if the
  493. * buffers were written out directly, with submit_bh(). ext3
  494. * will do this, as well as the blockdev mapping.
  495. * try_to_release_page() will discover that cleanness and will
  496. * drop the buffers and mark the page clean - it can be freed.
  497. *
  498. * Rarely, pages can have buffers and no ->mapping. These are
  499. * the pages which were not successfully invalidated in
  500. * truncate_complete_page(). We try to drop those buffers here
  501. * and if that worked, and the page is no longer mapped into
  502. * process address space (page_count == 1) it can be freed.
  503. * Otherwise, leave the page on the LRU so it is swappable.
  504. */
  505. if (PagePrivate(page)) {
  506. if (!try_to_release_page(page, sc->gfp_mask))
  507. goto activate_locked;
  508. if (!mapping && page_count(page) == 1)
  509. goto free_it;
  510. }
  511. if (!mapping || !remove_mapping(mapping, page))
  512. goto keep_locked;
  513. free_it:
  514. unlock_page(page);
  515. nr_reclaimed++;
  516. if (!pagevec_add(&freed_pvec, page))
  517. __pagevec_release_nonlru(&freed_pvec);
  518. continue;
  519. activate_locked:
  520. SetPageActive(page);
  521. pgactivate++;
  522. keep_locked:
  523. unlock_page(page);
  524. keep:
  525. list_add(&page->lru, &ret_pages);
  526. VM_BUG_ON(PageLRU(page));
  527. }
  528. list_splice(&ret_pages, page_list);
  529. if (pagevec_count(&freed_pvec))
  530. __pagevec_release_nonlru(&freed_pvec);
  531. count_vm_events(PGACTIVATE, pgactivate);
  532. return nr_reclaimed;
  533. }
  534. /*
  535. * zone->lru_lock is heavily contended. Some of the functions that
  536. * shrink the lists perform better by taking out a batch of pages
  537. * and working on them outside the LRU lock.
  538. *
  539. * For pagecache intensive workloads, this function is the hottest
  540. * spot in the kernel (apart from copy_*_user functions).
  541. *
  542. * Appropriate locks must be held before calling this function.
  543. *
  544. * @nr_to_scan: The number of pages to look through on the list.
  545. * @src: The LRU list to pull pages off.
  546. * @dst: The temp list to put pages on to.
  547. * @scanned: The number of pages that were scanned.
  548. *
  549. * returns how many pages were moved onto *@dst.
  550. */
  551. static unsigned long isolate_lru_pages(unsigned long nr_to_scan,
  552. struct list_head *src, struct list_head *dst,
  553. unsigned long *scanned)
  554. {
  555. unsigned long nr_taken = 0;
  556. struct page *page;
  557. unsigned long scan;
  558. for (scan = 0; scan < nr_to_scan && !list_empty(src); scan++) {
  559. struct list_head *target;
  560. page = lru_to_page(src);
  561. prefetchw_prev_lru_page(page, src, flags);
  562. VM_BUG_ON(!PageLRU(page));
  563. list_del(&page->lru);
  564. target = src;
  565. if (likely(get_page_unless_zero(page))) {
  566. /*
  567. * Be careful not to clear PageLRU until after we're
  568. * sure the page is not being freed elsewhere -- the
  569. * page release code relies on it.
  570. */
  571. ClearPageLRU(page);
  572. target = dst;
  573. nr_taken++;
  574. } /* else it is being freed elsewhere */
  575. list_add(&page->lru, target);
  576. }
  577. *scanned = scan;
  578. return nr_taken;
  579. }
  580. /*
  581. * shrink_inactive_list() is a helper for shrink_zone(). It returns the number
  582. * of reclaimed pages
  583. */
  584. static unsigned long shrink_inactive_list(unsigned long max_scan,
  585. struct zone *zone, struct scan_control *sc)
  586. {
  587. LIST_HEAD(page_list);
  588. struct pagevec pvec;
  589. unsigned long nr_scanned = 0;
  590. unsigned long nr_reclaimed = 0;
  591. pagevec_init(&pvec, 1);
  592. lru_add_drain();
  593. spin_lock_irq(&zone->lru_lock);
  594. do {
  595. struct page *page;
  596. unsigned long nr_taken;
  597. unsigned long nr_scan;
  598. unsigned long nr_freed;
  599. nr_taken = isolate_lru_pages(sc->swap_cluster_max,
  600. &zone->inactive_list,
  601. &page_list, &nr_scan);
  602. __mod_zone_page_state(zone, NR_INACTIVE, -nr_taken);
  603. zone->pages_scanned += nr_scan;
  604. spin_unlock_irq(&zone->lru_lock);
  605. nr_scanned += nr_scan;
  606. nr_freed = shrink_page_list(&page_list, sc);
  607. nr_reclaimed += nr_freed;
  608. local_irq_disable();
  609. if (current_is_kswapd()) {
  610. __count_zone_vm_events(PGSCAN_KSWAPD, zone, nr_scan);
  611. __count_vm_events(KSWAPD_STEAL, nr_freed);
  612. } else
  613. __count_zone_vm_events(PGSCAN_DIRECT, zone, nr_scan);
  614. __count_zone_vm_events(PGSTEAL, zone, nr_freed);
  615. if (nr_taken == 0)
  616. goto done;
  617. spin_lock(&zone->lru_lock);
  618. /*
  619. * Put back any unfreeable pages.
  620. */
  621. while (!list_empty(&page_list)) {
  622. page = lru_to_page(&page_list);
  623. VM_BUG_ON(PageLRU(page));
  624. SetPageLRU(page);
  625. list_del(&page->lru);
  626. if (PageActive(page))
  627. add_page_to_active_list(zone, page);
  628. else
  629. add_page_to_inactive_list(zone, page);
  630. if (!pagevec_add(&pvec, page)) {
  631. spin_unlock_irq(&zone->lru_lock);
  632. __pagevec_release(&pvec);
  633. spin_lock_irq(&zone->lru_lock);
  634. }
  635. }
  636. } while (nr_scanned < max_scan);
  637. spin_unlock(&zone->lru_lock);
  638. done:
  639. local_irq_enable();
  640. pagevec_release(&pvec);
  641. return nr_reclaimed;
  642. }
  643. /*
  644. * We are about to scan this zone at a certain priority level. If that priority
  645. * level is smaller (ie: more urgent) than the previous priority, then note
  646. * that priority level within the zone. This is done so that when the next
  647. * process comes in to scan this zone, it will immediately start out at this
  648. * priority level rather than having to build up its own scanning priority.
  649. * Here, this priority affects only the reclaim-mapped threshold.
  650. */
  651. static inline void note_zone_scanning_priority(struct zone *zone, int priority)
  652. {
  653. if (priority < zone->prev_priority)
  654. zone->prev_priority = priority;
  655. }
  656. static inline int zone_is_near_oom(struct zone *zone)
  657. {
  658. return zone->pages_scanned >= (zone_page_state(zone, NR_ACTIVE)
  659. + zone_page_state(zone, NR_INACTIVE))*3;
  660. }
  661. /*
  662. * This moves pages from the active list to the inactive list.
  663. *
  664. * We move them the other way if the page is referenced by one or more
  665. * processes, from rmap.
  666. *
  667. * If the pages are mostly unmapped, the processing is fast and it is
  668. * appropriate to hold zone->lru_lock across the whole operation. But if
  669. * the pages are mapped, the processing is slow (page_referenced()) so we
  670. * should drop zone->lru_lock around each page. It's impossible to balance
  671. * this, so instead we remove the pages from the LRU while processing them.
  672. * It is safe to rely on PG_active against the non-LRU pages in here because
  673. * nobody will play with that bit on a non-LRU page.
  674. *
  675. * The downside is that we have to touch page->_count against each page.
  676. * But we had to alter page->flags anyway.
  677. */
  678. static void shrink_active_list(unsigned long nr_pages, struct zone *zone,
  679. struct scan_control *sc, int priority)
  680. {
  681. unsigned long pgmoved;
  682. int pgdeactivate = 0;
  683. unsigned long pgscanned;
  684. LIST_HEAD(l_hold); /* The pages which were snipped off */
  685. LIST_HEAD(l_inactive); /* Pages to go onto the inactive_list */
  686. LIST_HEAD(l_active); /* Pages to go onto the active_list */
  687. struct page *page;
  688. struct pagevec pvec;
  689. int reclaim_mapped = 0;
  690. if (sc->may_swap) {
  691. long mapped_ratio;
  692. long distress;
  693. long swap_tendency;
  694. if (zone_is_near_oom(zone))
  695. goto force_reclaim_mapped;
  696. /*
  697. * `distress' is a measure of how much trouble we're having
  698. * reclaiming pages. 0 -> no problems. 100 -> great trouble.
  699. */
  700. distress = 100 >> min(zone->prev_priority, priority);
  701. /*
  702. * The point of this algorithm is to decide when to start
  703. * reclaiming mapped memory instead of just pagecache. Work out
  704. * how much memory
  705. * is mapped.
  706. */
  707. mapped_ratio = ((global_page_state(NR_FILE_MAPPED) +
  708. global_page_state(NR_ANON_PAGES)) * 100) /
  709. vm_total_pages;
  710. /*
  711. * Now decide how much we really want to unmap some pages. The
  712. * mapped ratio is downgraded - just because there's a lot of
  713. * mapped memory doesn't necessarily mean that page reclaim
  714. * isn't succeeding.
  715. *
  716. * The distress ratio is important - we don't want to start
  717. * going oom.
  718. *
  719. * A 100% value of vm_swappiness overrides this algorithm
  720. * altogether.
  721. */
  722. swap_tendency = mapped_ratio / 2 + distress + sc->swappiness;
  723. /*
  724. * Now use this metric to decide whether to start moving mapped
  725. * memory onto the inactive list.
  726. */
  727. if (swap_tendency >= 100)
  728. force_reclaim_mapped:
  729. reclaim_mapped = 1;
  730. }
  731. lru_add_drain();
  732. spin_lock_irq(&zone->lru_lock);
  733. pgmoved = isolate_lru_pages(nr_pages, &zone->active_list,
  734. &l_hold, &pgscanned);
  735. zone->pages_scanned += pgscanned;
  736. __mod_zone_page_state(zone, NR_ACTIVE, -pgmoved);
  737. spin_unlock_irq(&zone->lru_lock);
  738. while (!list_empty(&l_hold)) {
  739. cond_resched();
  740. page = lru_to_page(&l_hold);
  741. list_del(&page->lru);
  742. if (page_mapped(page)) {
  743. if (!reclaim_mapped ||
  744. (total_swap_pages == 0 && PageAnon(page)) ||
  745. page_referenced(page, 0)) {
  746. list_add(&page->lru, &l_active);
  747. continue;
  748. }
  749. }
  750. list_add(&page->lru, &l_inactive);
  751. }
  752. pagevec_init(&pvec, 1);
  753. pgmoved = 0;
  754. spin_lock_irq(&zone->lru_lock);
  755. while (!list_empty(&l_inactive)) {
  756. page = lru_to_page(&l_inactive);
  757. prefetchw_prev_lru_page(page, &l_inactive, flags);
  758. VM_BUG_ON(PageLRU(page));
  759. SetPageLRU(page);
  760. VM_BUG_ON(!PageActive(page));
  761. ClearPageActive(page);
  762. list_move(&page->lru, &zone->inactive_list);
  763. pgmoved++;
  764. if (!pagevec_add(&pvec, page)) {
  765. __mod_zone_page_state(zone, NR_INACTIVE, pgmoved);
  766. spin_unlock_irq(&zone->lru_lock);
  767. pgdeactivate += pgmoved;
  768. pgmoved = 0;
  769. if (buffer_heads_over_limit)
  770. pagevec_strip(&pvec);
  771. __pagevec_release(&pvec);
  772. spin_lock_irq(&zone->lru_lock);
  773. }
  774. }
  775. __mod_zone_page_state(zone, NR_INACTIVE, pgmoved);
  776. pgdeactivate += pgmoved;
  777. if (buffer_heads_over_limit) {
  778. spin_unlock_irq(&zone->lru_lock);
  779. pagevec_strip(&pvec);
  780. spin_lock_irq(&zone->lru_lock);
  781. }
  782. pgmoved = 0;
  783. while (!list_empty(&l_active)) {
  784. page = lru_to_page(&l_active);
  785. prefetchw_prev_lru_page(page, &l_active, flags);
  786. VM_BUG_ON(PageLRU(page));
  787. SetPageLRU(page);
  788. VM_BUG_ON(!PageActive(page));
  789. list_move(&page->lru, &zone->active_list);
  790. pgmoved++;
  791. if (!pagevec_add(&pvec, page)) {
  792. __mod_zone_page_state(zone, NR_ACTIVE, pgmoved);
  793. pgmoved = 0;
  794. spin_unlock_irq(&zone->lru_lock);
  795. __pagevec_release(&pvec);
  796. spin_lock_irq(&zone->lru_lock);
  797. }
  798. }
  799. __mod_zone_page_state(zone, NR_ACTIVE, pgmoved);
  800. __count_zone_vm_events(PGREFILL, zone, pgscanned);
  801. __count_vm_events(PGDEACTIVATE, pgdeactivate);
  802. spin_unlock_irq(&zone->lru_lock);
  803. pagevec_release(&pvec);
  804. }
  805. /*
  806. * This is a basic per-zone page freer. Used by both kswapd and direct reclaim.
  807. */
  808. static unsigned long shrink_zone(int priority, struct zone *zone,
  809. struct scan_control *sc)
  810. {
  811. unsigned long nr_active;
  812. unsigned long nr_inactive;
  813. unsigned long nr_to_scan;
  814. unsigned long nr_reclaimed = 0;
  815. atomic_inc(&zone->reclaim_in_progress);
  816. /*
  817. * Add one to `nr_to_scan' just to make sure that the kernel will
  818. * slowly sift through the active list.
  819. */
  820. zone->nr_scan_active +=
  821. (zone_page_state(zone, NR_ACTIVE) >> priority) + 1;
  822. nr_active = zone->nr_scan_active;
  823. if (nr_active >= sc->swap_cluster_max)
  824. zone->nr_scan_active = 0;
  825. else
  826. nr_active = 0;
  827. zone->nr_scan_inactive +=
  828. (zone_page_state(zone, NR_INACTIVE) >> priority) + 1;
  829. nr_inactive = zone->nr_scan_inactive;
  830. if (nr_inactive >= sc->swap_cluster_max)
  831. zone->nr_scan_inactive = 0;
  832. else
  833. nr_inactive = 0;
  834. while (nr_active || nr_inactive) {
  835. if (nr_active) {
  836. nr_to_scan = min(nr_active,
  837. (unsigned long)sc->swap_cluster_max);
  838. nr_active -= nr_to_scan;
  839. shrink_active_list(nr_to_scan, zone, sc, priority);
  840. }
  841. if (nr_inactive) {
  842. nr_to_scan = min(nr_inactive,
  843. (unsigned long)sc->swap_cluster_max);
  844. nr_inactive -= nr_to_scan;
  845. nr_reclaimed += shrink_inactive_list(nr_to_scan, zone,
  846. sc);
  847. }
  848. }
  849. throttle_vm_writeout(sc->gfp_mask);
  850. atomic_dec(&zone->reclaim_in_progress);
  851. return nr_reclaimed;
  852. }
  853. /*
  854. * This is the direct reclaim path, for page-allocating processes. We only
  855. * try to reclaim pages from zones which will satisfy the caller's allocation
  856. * request.
  857. *
  858. * We reclaim from a zone even if that zone is over pages_high. Because:
  859. * a) The caller may be trying to free *extra* pages to satisfy a higher-order
  860. * allocation or
  861. * b) The zones may be over pages_high but they must go *over* pages_high to
  862. * satisfy the `incremental min' zone defense algorithm.
  863. *
  864. * Returns the number of reclaimed pages.
  865. *
  866. * If a zone is deemed to be full of pinned pages then just give it a light
  867. * scan then give up on it.
  868. */
  869. static unsigned long shrink_zones(int priority, struct zone **zones,
  870. struct scan_control *sc)
  871. {
  872. unsigned long nr_reclaimed = 0;
  873. int i;
  874. sc->all_unreclaimable = 1;
  875. for (i = 0; zones[i] != NULL; i++) {
  876. struct zone *zone = zones[i];
  877. if (!populated_zone(zone))
  878. continue;
  879. if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
  880. continue;
  881. note_zone_scanning_priority(zone, priority);
  882. if (zone->all_unreclaimable && priority != DEF_PRIORITY)
  883. continue; /* Let kswapd poll it */
  884. sc->all_unreclaimable = 0;
  885. nr_reclaimed += shrink_zone(priority, zone, sc);
  886. }
  887. return nr_reclaimed;
  888. }
  889. /*
  890. * This is the main entry point to direct page reclaim.
  891. *
  892. * If a full scan of the inactive list fails to free enough memory then we
  893. * are "out of memory" and something needs to be killed.
  894. *
  895. * If the caller is !__GFP_FS then the probability of a failure is reasonably
  896. * high - the zone may be full of dirty or under-writeback pages, which this
  897. * caller can't do much about. We kick pdflush and take explicit naps in the
  898. * hope that some of these pages can be written. But if the allocating task
  899. * holds filesystem locks which prevent writeout this might not work, and the
  900. * allocation attempt will fail.
  901. */
  902. unsigned long try_to_free_pages(struct zone **zones, gfp_t gfp_mask)
  903. {
  904. int priority;
  905. int ret = 0;
  906. unsigned long total_scanned = 0;
  907. unsigned long nr_reclaimed = 0;
  908. struct reclaim_state *reclaim_state = current->reclaim_state;
  909. unsigned long lru_pages = 0;
  910. int i;
  911. struct scan_control sc = {
  912. .gfp_mask = gfp_mask,
  913. .may_writepage = !laptop_mode,
  914. .swap_cluster_max = SWAP_CLUSTER_MAX,
  915. .may_swap = 1,
  916. .swappiness = vm_swappiness,
  917. };
  918. count_vm_event(ALLOCSTALL);
  919. for (i = 0; zones[i] != NULL; i++) {
  920. struct zone *zone = zones[i];
  921. if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
  922. continue;
  923. lru_pages += zone_page_state(zone, NR_ACTIVE)
  924. + zone_page_state(zone, NR_INACTIVE);
  925. }
  926. for (priority = DEF_PRIORITY; priority >= 0; priority--) {
  927. sc.nr_scanned = 0;
  928. if (!priority)
  929. disable_swap_token();
  930. nr_reclaimed += shrink_zones(priority, zones, &sc);
  931. shrink_slab(sc.nr_scanned, gfp_mask, lru_pages);
  932. if (reclaim_state) {
  933. nr_reclaimed += reclaim_state->reclaimed_slab;
  934. reclaim_state->reclaimed_slab = 0;
  935. }
  936. total_scanned += sc.nr_scanned;
  937. if (nr_reclaimed >= sc.swap_cluster_max) {
  938. ret = 1;
  939. goto out;
  940. }
  941. /*
  942. * Try to write back as many pages as we just scanned. This
  943. * tends to cause slow streaming writers to write data to the
  944. * disk smoothly, at the dirtying rate, which is nice. But
  945. * that's undesirable in laptop mode, where we *want* lumpy
  946. * writeout. So in laptop mode, write out the whole world.
  947. */
  948. if (total_scanned > sc.swap_cluster_max +
  949. sc.swap_cluster_max / 2) {
  950. wakeup_pdflush(laptop_mode ? 0 : total_scanned);
  951. sc.may_writepage = 1;
  952. }
  953. /* Take a nap, wait for some writeback to complete */
  954. if (sc.nr_scanned && priority < DEF_PRIORITY - 2)
  955. congestion_wait(WRITE, HZ/10);
  956. }
  957. /* top priority shrink_caches still had more to do? don't OOM, then */
  958. if (!sc.all_unreclaimable)
  959. ret = 1;
  960. out:
  961. /*
  962. * Now that we've scanned all the zones at this priority level, note
  963. * that level within the zone so that the next thread which performs
  964. * scanning of this zone will immediately start out at this priority
  965. * level. This affects only the decision whether or not to bring
  966. * mapped pages onto the inactive list.
  967. */
  968. if (priority < 0)
  969. priority = 0;
  970. for (i = 0; zones[i] != 0; i++) {
  971. struct zone *zone = zones[i];
  972. if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
  973. continue;
  974. zone->prev_priority = priority;
  975. }
  976. return ret;
  977. }
  978. /*
  979. * For kswapd, balance_pgdat() will work across all this node's zones until
  980. * they are all at pages_high.
  981. *
  982. * Returns the number of pages which were actually freed.
  983. *
  984. * There is special handling here for zones which are full of pinned pages.
  985. * This can happen if the pages are all mlocked, or if they are all used by
  986. * device drivers (say, ZONE_DMA). Or if they are all in use by hugetlb.
  987. * What we do is to detect the case where all pages in the zone have been
  988. * scanned twice and there has been zero successful reclaim. Mark the zone as
  989. * dead and from now on, only perform a short scan. Basically we're polling
  990. * the zone for when the problem goes away.
  991. *
  992. * kswapd scans the zones in the highmem->normal->dma direction. It skips
  993. * zones which have free_pages > pages_high, but once a zone is found to have
  994. * free_pages <= pages_high, we scan that zone and the lower zones regardless
  995. * of the number of free pages in the lower zones. This interoperates with
  996. * the page allocator fallback scheme to ensure that aging of pages is balanced
  997. * across the zones.
  998. */
  999. static unsigned long balance_pgdat(pg_data_t *pgdat, int order)
  1000. {
  1001. int all_zones_ok;
  1002. int priority;
  1003. int i;
  1004. unsigned long total_scanned;
  1005. unsigned long nr_reclaimed;
  1006. struct reclaim_state *reclaim_state = current->reclaim_state;
  1007. struct scan_control sc = {
  1008. .gfp_mask = GFP_KERNEL,
  1009. .may_swap = 1,
  1010. .swap_cluster_max = SWAP_CLUSTER_MAX,
  1011. .swappiness = vm_swappiness,
  1012. };
  1013. /*
  1014. * temp_priority is used to remember the scanning priority at which
  1015. * this zone was successfully refilled to free_pages == pages_high.
  1016. */
  1017. int temp_priority[MAX_NR_ZONES];
  1018. loop_again:
  1019. total_scanned = 0;
  1020. nr_reclaimed = 0;
  1021. sc.may_writepage = !laptop_mode;
  1022. count_vm_event(PAGEOUTRUN);
  1023. for (i = 0; i < pgdat->nr_zones; i++)
  1024. temp_priority[i] = DEF_PRIORITY;
  1025. for (priority = DEF_PRIORITY; priority >= 0; priority--) {
  1026. int end_zone = 0; /* Inclusive. 0 = ZONE_DMA */
  1027. unsigned long lru_pages = 0;
  1028. /* The swap token gets in the way of swapout... */
  1029. if (!priority)
  1030. disable_swap_token();
  1031. all_zones_ok = 1;
  1032. /*
  1033. * Scan in the highmem->dma direction for the highest
  1034. * zone which needs scanning
  1035. */
  1036. for (i = pgdat->nr_zones - 1; i >= 0; i--) {
  1037. struct zone *zone = pgdat->node_zones + i;
  1038. if (!populated_zone(zone))
  1039. continue;
  1040. if (zone->all_unreclaimable && priority != DEF_PRIORITY)
  1041. continue;
  1042. if (!zone_watermark_ok(zone, order, zone->pages_high,
  1043. 0, 0)) {
  1044. end_zone = i;
  1045. break;
  1046. }
  1047. }
  1048. if (i < 0)
  1049. goto out;
  1050. for (i = 0; i <= end_zone; i++) {
  1051. struct zone *zone = pgdat->node_zones + i;
  1052. lru_pages += zone_page_state(zone, NR_ACTIVE)
  1053. + zone_page_state(zone, NR_INACTIVE);
  1054. }
  1055. /*
  1056. * Now scan the zone in the dma->highmem direction, stopping
  1057. * at the last zone which needs scanning.
  1058. *
  1059. * We do this because the page allocator works in the opposite
  1060. * direction. This prevents the page allocator from allocating
  1061. * pages behind kswapd's direction of progress, which would
  1062. * cause too much scanning of the lower zones.
  1063. */
  1064. for (i = 0; i <= end_zone; i++) {
  1065. struct zone *zone = pgdat->node_zones + i;
  1066. int nr_slab;
  1067. if (!populated_zone(zone))
  1068. continue;
  1069. if (zone->all_unreclaimable && priority != DEF_PRIORITY)
  1070. continue;
  1071. if (!zone_watermark_ok(zone, order, zone->pages_high,
  1072. end_zone, 0))
  1073. all_zones_ok = 0;
  1074. temp_priority[i] = priority;
  1075. sc.nr_scanned = 0;
  1076. note_zone_scanning_priority(zone, priority);
  1077. nr_reclaimed += shrink_zone(priority, zone, &sc);
  1078. reclaim_state->reclaimed_slab = 0;
  1079. nr_slab = shrink_slab(sc.nr_scanned, GFP_KERNEL,
  1080. lru_pages);
  1081. nr_reclaimed += reclaim_state->reclaimed_slab;
  1082. total_scanned += sc.nr_scanned;
  1083. if (zone->all_unreclaimable)
  1084. continue;
  1085. if (nr_slab == 0 && zone->pages_scanned >=
  1086. (zone_page_state(zone, NR_ACTIVE)
  1087. + zone_page_state(zone, NR_INACTIVE)) * 6)
  1088. zone->all_unreclaimable = 1;
  1089. /*
  1090. * If we've done a decent amount of scanning and
  1091. * the reclaim ratio is low, start doing writepage
  1092. * even in laptop mode
  1093. */
  1094. if (total_scanned > SWAP_CLUSTER_MAX * 2 &&
  1095. total_scanned > nr_reclaimed + nr_reclaimed / 2)
  1096. sc.may_writepage = 1;
  1097. }
  1098. if (all_zones_ok)
  1099. break; /* kswapd: all done */
  1100. /*
  1101. * OK, kswapd is getting into trouble. Take a nap, then take
  1102. * another pass across the zones.
  1103. */
  1104. if (total_scanned && priority < DEF_PRIORITY - 2)
  1105. congestion_wait(WRITE, HZ/10);
  1106. /*
  1107. * We do this so kswapd doesn't build up large priorities for
  1108. * example when it is freeing in parallel with allocators. It
  1109. * matches the direct reclaim path behaviour in terms of impact
  1110. * on zone->*_priority.
  1111. */
  1112. if (nr_reclaimed >= SWAP_CLUSTER_MAX)
  1113. break;
  1114. }
  1115. out:
  1116. /*
  1117. * Note within each zone the priority level at which this zone was
  1118. * brought into a happy state. So that the next thread which scans this
  1119. * zone will start out at that priority level.
  1120. */
  1121. for (i = 0; i < pgdat->nr_zones; i++) {
  1122. struct zone *zone = pgdat->node_zones + i;
  1123. zone->prev_priority = temp_priority[i];
  1124. }
  1125. if (!all_zones_ok) {
  1126. cond_resched();
  1127. try_to_freeze();
  1128. goto loop_again;
  1129. }
  1130. return nr_reclaimed;
  1131. }
  1132. /*
  1133. * The background pageout daemon, started as a kernel thread
  1134. * from the init process.
  1135. *
  1136. * This basically trickles out pages so that we have _some_
  1137. * free memory available even if there is no other activity
  1138. * that frees anything up. This is needed for things like routing
  1139. * etc, where we otherwise might have all activity going on in
  1140. * asynchronous contexts that cannot page things out.
  1141. *
  1142. * If there are applications that are active memory-allocators
  1143. * (most normal use), this basically shouldn't matter.
  1144. */
  1145. static int kswapd(void *p)
  1146. {
  1147. unsigned long order;
  1148. pg_data_t *pgdat = (pg_data_t*)p;
  1149. struct task_struct *tsk = current;
  1150. DEFINE_WAIT(wait);
  1151. struct reclaim_state reclaim_state = {
  1152. .reclaimed_slab = 0,
  1153. };
  1154. cpumask_t cpumask;
  1155. cpumask = node_to_cpumask(pgdat->node_id);
  1156. if (!cpus_empty(cpumask))
  1157. set_cpus_allowed(tsk, cpumask);
  1158. current->reclaim_state = &reclaim_state;
  1159. /*
  1160. * Tell the memory management that we're a "memory allocator",
  1161. * and that if we need more memory we should get access to it
  1162. * regardless (see "__alloc_pages()"). "kswapd" should
  1163. * never get caught in the normal page freeing logic.
  1164. *
  1165. * (Kswapd normally doesn't need memory anyway, but sometimes
  1166. * you need a small amount of memory in order to be able to
  1167. * page out something else, and this flag essentially protects
  1168. * us from recursively trying to free more memory as we're
  1169. * trying to free the first piece of memory in the first place).
  1170. */
  1171. tsk->flags |= PF_MEMALLOC | PF_SWAPWRITE | PF_KSWAPD;
  1172. order = 0;
  1173. for ( ; ; ) {
  1174. unsigned long new_order;
  1175. try_to_freeze();
  1176. prepare_to_wait(&pgdat->kswapd_wait, &wait, TASK_INTERRUPTIBLE);
  1177. new_order = pgdat->kswapd_max_order;
  1178. pgdat->kswapd_max_order = 0;
  1179. if (order < new_order) {
  1180. /*
  1181. * Don't sleep if someone wants a larger 'order'
  1182. * allocation
  1183. */
  1184. order = new_order;
  1185. } else {
  1186. schedule();
  1187. order = pgdat->kswapd_max_order;
  1188. }
  1189. finish_wait(&pgdat->kswapd_wait, &wait);
  1190. balance_pgdat(pgdat, order);
  1191. }
  1192. return 0;
  1193. }
  1194. /*
  1195. * A zone is low on free memory, so wake its kswapd task to service it.
  1196. */
  1197. void wakeup_kswapd(struct zone *zone, int order)
  1198. {
  1199. pg_data_t *pgdat;
  1200. if (!populated_zone(zone))
  1201. return;
  1202. pgdat = zone->zone_pgdat;
  1203. if (zone_watermark_ok(zone, order, zone->pages_low, 0, 0))
  1204. return;
  1205. if (pgdat->kswapd_max_order < order)
  1206. pgdat->kswapd_max_order = order;
  1207. if (!cpuset_zone_allowed_hardwall(zone, GFP_KERNEL))
  1208. return;
  1209. if (!waitqueue_active(&pgdat->kswapd_wait))
  1210. return;
  1211. wake_up_interruptible(&pgdat->kswapd_wait);
  1212. }
  1213. #ifdef CONFIG_PM
  1214. /*
  1215. * Helper function for shrink_all_memory(). Tries to reclaim 'nr_pages' pages
  1216. * from LRU lists system-wide, for given pass and priority, and returns the
  1217. * number of reclaimed pages
  1218. *
  1219. * For pass > 3 we also try to shrink the LRU lists that contain a few pages
  1220. */
  1221. static unsigned long shrink_all_zones(unsigned long nr_pages, int prio,
  1222. int pass, struct scan_control *sc)
  1223. {
  1224. struct zone *zone;
  1225. unsigned long nr_to_scan, ret = 0;
  1226. for_each_zone(zone) {
  1227. if (!populated_zone(zone))
  1228. continue;
  1229. if (zone->all_unreclaimable && prio != DEF_PRIORITY)
  1230. continue;
  1231. /* For pass = 0 we don't shrink the active list */
  1232. if (pass > 0) {
  1233. zone->nr_scan_active +=
  1234. (zone_page_state(zone, NR_ACTIVE) >> prio) + 1;
  1235. if (zone->nr_scan_active >= nr_pages || pass > 3) {
  1236. zone->nr_scan_active = 0;
  1237. nr_to_scan = min(nr_pages,
  1238. zone_page_state(zone, NR_ACTIVE));
  1239. shrink_active_list(nr_to_scan, zone, sc, prio);
  1240. }
  1241. }
  1242. zone->nr_scan_inactive +=
  1243. (zone_page_state(zone, NR_INACTIVE) >> prio) + 1;
  1244. if (zone->nr_scan_inactive >= nr_pages || pass > 3) {
  1245. zone->nr_scan_inactive = 0;
  1246. nr_to_scan = min(nr_pages,
  1247. zone_page_state(zone, NR_INACTIVE));
  1248. ret += shrink_inactive_list(nr_to_scan, zone, sc);
  1249. if (ret >= nr_pages)
  1250. return ret;
  1251. }
  1252. }
  1253. return ret;
  1254. }
  1255. static unsigned long count_lru_pages(void)
  1256. {
  1257. return global_page_state(NR_ACTIVE) + global_page_state(NR_INACTIVE);
  1258. }
  1259. /*
  1260. * Try to free `nr_pages' of memory, system-wide, and return the number of
  1261. * freed pages.
  1262. *
  1263. * Rather than trying to age LRUs the aim is to preserve the overall
  1264. * LRU order by reclaiming preferentially
  1265. * inactive > active > active referenced > active mapped
  1266. */
  1267. unsigned long shrink_all_memory(unsigned long nr_pages)
  1268. {
  1269. unsigned long lru_pages, nr_slab;
  1270. unsigned long ret = 0;
  1271. int pass;
  1272. struct reclaim_state reclaim_state;
  1273. struct scan_control sc = {
  1274. .gfp_mask = GFP_KERNEL,
  1275. .may_swap = 0,
  1276. .swap_cluster_max = nr_pages,
  1277. .may_writepage = 1,
  1278. .swappiness = vm_swappiness,
  1279. };
  1280. current->reclaim_state = &reclaim_state;
  1281. lru_pages = count_lru_pages();
  1282. nr_slab = global_page_state(NR_SLAB_RECLAIMABLE);
  1283. /* If slab caches are huge, it's better to hit them first */
  1284. while (nr_slab >= lru_pages) {
  1285. reclaim_state.reclaimed_slab = 0;
  1286. shrink_slab(nr_pages, sc.gfp_mask, lru_pages);
  1287. if (!reclaim_state.reclaimed_slab)
  1288. break;
  1289. ret += reclaim_state.reclaimed_slab;
  1290. if (ret >= nr_pages)
  1291. goto out;
  1292. nr_slab -= reclaim_state.reclaimed_slab;
  1293. }
  1294. /*
  1295. * We try to shrink LRUs in 5 passes:
  1296. * 0 = Reclaim from inactive_list only
  1297. * 1 = Reclaim from active list but don't reclaim mapped
  1298. * 2 = 2nd pass of type 1
  1299. * 3 = Reclaim mapped (normal reclaim)
  1300. * 4 = 2nd pass of type 3
  1301. */
  1302. for (pass = 0; pass < 5; pass++) {
  1303. int prio;
  1304. /* Force reclaiming mapped pages in the passes #3 and #4 */
  1305. if (pass > 2) {
  1306. sc.may_swap = 1;
  1307. sc.swappiness = 100;
  1308. }
  1309. for (prio = DEF_PRIORITY; prio >= 0; prio--) {
  1310. unsigned long nr_to_scan = nr_pages - ret;
  1311. sc.nr_scanned = 0;
  1312. ret += shrink_all_zones(nr_to_scan, prio, pass, &sc);
  1313. if (ret >= nr_pages)
  1314. goto out;
  1315. reclaim_state.reclaimed_slab = 0;
  1316. shrink_slab(sc.nr_scanned, sc.gfp_mask,
  1317. count_lru_pages());
  1318. ret += reclaim_state.reclaimed_slab;
  1319. if (ret >= nr_pages)
  1320. goto out;
  1321. if (sc.nr_scanned && prio < DEF_PRIORITY - 2)
  1322. congestion_wait(WRITE, HZ / 10);
  1323. }
  1324. }
  1325. /*
  1326. * If ret = 0, we could not shrink LRUs, but there may be something
  1327. * in slab caches
  1328. */
  1329. if (!ret) {
  1330. do {
  1331. reclaim_state.reclaimed_slab = 0;
  1332. shrink_slab(nr_pages, sc.gfp_mask, count_lru_pages());
  1333. ret += reclaim_state.reclaimed_slab;
  1334. } while (ret < nr_pages && reclaim_state.reclaimed_slab > 0);
  1335. }
  1336. out:
  1337. current->reclaim_state = NULL;
  1338. return ret;
  1339. }
  1340. #endif
  1341. /* It's optimal to keep kswapds on the same CPUs as their memory, but
  1342. not required for correctness. So if the last cpu in a node goes
  1343. away, we get changed to run anywhere: as the first one comes back,
  1344. restore their cpu bindings. */
  1345. static int __devinit cpu_callback(struct notifier_block *nfb,
  1346. unsigned long action, void *hcpu)
  1347. {
  1348. pg_data_t *pgdat;
  1349. cpumask_t mask;
  1350. if (action == CPU_ONLINE) {
  1351. for_each_online_pgdat(pgdat) {
  1352. mask = node_to_cpumask(pgdat->node_id);
  1353. if (any_online_cpu(mask) != NR_CPUS)
  1354. /* One of our CPUs online: restore mask */
  1355. set_cpus_allowed(pgdat->kswapd, mask);
  1356. }
  1357. }
  1358. return NOTIFY_OK;
  1359. }
  1360. /*
  1361. * This kswapd start function will be called by init and node-hot-add.
  1362. * On node-hot-add, kswapd will moved to proper cpus if cpus are hot-added.
  1363. */
  1364. int kswapd_run(int nid)
  1365. {
  1366. pg_data_t *pgdat = NODE_DATA(nid);
  1367. int ret = 0;
  1368. if (pgdat->kswapd)
  1369. return 0;
  1370. pgdat->kswapd = kthread_run(kswapd, pgdat, "kswapd%d", nid);
  1371. if (IS_ERR(pgdat->kswapd)) {
  1372. /* failure at boot is fatal */
  1373. BUG_ON(system_state == SYSTEM_BOOTING);
  1374. printk("Failed to start kswapd on node %d\n",nid);
  1375. ret = -1;
  1376. }
  1377. return ret;
  1378. }
  1379. static int __init kswapd_init(void)
  1380. {
  1381. int nid;
  1382. swap_setup();
  1383. for_each_online_node(nid)
  1384. kswapd_run(nid);
  1385. hotcpu_notifier(cpu_callback, 0);
  1386. return 0;
  1387. }
  1388. module_init(kswapd_init)
  1389. #ifdef CONFIG_NUMA
  1390. /*
  1391. * Zone reclaim mode
  1392. *
  1393. * If non-zero call zone_reclaim when the number of free pages falls below
  1394. * the watermarks.
  1395. */
  1396. int zone_reclaim_mode __read_mostly;
  1397. #define RECLAIM_OFF 0
  1398. #define RECLAIM_ZONE (1<<0) /* Run shrink_cache on the zone */
  1399. #define RECLAIM_WRITE (1<<1) /* Writeout pages during reclaim */
  1400. #define RECLAIM_SWAP (1<<2) /* Swap pages out during reclaim */
  1401. /*
  1402. * Priority for ZONE_RECLAIM. This determines the fraction of pages
  1403. * of a node considered for each zone_reclaim. 4 scans 1/16th of
  1404. * a zone.
  1405. */
  1406. #define ZONE_RECLAIM_PRIORITY 4
  1407. /*
  1408. * Percentage of pages in a zone that must be unmapped for zone_reclaim to
  1409. * occur.
  1410. */
  1411. int sysctl_min_unmapped_ratio = 1;
  1412. /*
  1413. * If the number of slab pages in a zone grows beyond this percentage then
  1414. * slab reclaim needs to occur.
  1415. */
  1416. int sysctl_min_slab_ratio = 5;
  1417. /*
  1418. * Try to free up some pages from this zone through reclaim.
  1419. */
  1420. static int __zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
  1421. {
  1422. /* Minimum pages needed in order to stay on node */
  1423. const unsigned long nr_pages = 1 << order;
  1424. struct task_struct *p = current;
  1425. struct reclaim_state reclaim_state;
  1426. int priority;
  1427. unsigned long nr_reclaimed = 0;
  1428. struct scan_control sc = {
  1429. .may_writepage = !!(zone_reclaim_mode & RECLAIM_WRITE),
  1430. .may_swap = !!(zone_reclaim_mode & RECLAIM_SWAP),
  1431. .swap_cluster_max = max_t(unsigned long, nr_pages,
  1432. SWAP_CLUSTER_MAX),
  1433. .gfp_mask = gfp_mask,
  1434. .swappiness = vm_swappiness,
  1435. };
  1436. unsigned long slab_reclaimable;
  1437. disable_swap_token();
  1438. cond_resched();
  1439. /*
  1440. * We need to be able to allocate from the reserves for RECLAIM_SWAP
  1441. * and we also need to be able to write out pages for RECLAIM_WRITE
  1442. * and RECLAIM_SWAP.
  1443. */
  1444. p->flags |= PF_MEMALLOC | PF_SWAPWRITE;
  1445. reclaim_state.reclaimed_slab = 0;
  1446. p->reclaim_state = &reclaim_state;
  1447. if (zone_page_state(zone, NR_FILE_PAGES) -
  1448. zone_page_state(zone, NR_FILE_MAPPED) >
  1449. zone->min_unmapped_pages) {
  1450. /*
  1451. * Free memory by calling shrink zone with increasing
  1452. * priorities until we have enough memory freed.
  1453. */
  1454. priority = ZONE_RECLAIM_PRIORITY;
  1455. do {
  1456. note_zone_scanning_priority(zone, priority);
  1457. nr_reclaimed += shrink_zone(priority, zone, &sc);
  1458. priority--;
  1459. } while (priority >= 0 && nr_reclaimed < nr_pages);
  1460. }
  1461. slab_reclaimable = zone_page_state(zone, NR_SLAB_RECLAIMABLE);
  1462. if (slab_reclaimable > zone->min_slab_pages) {
  1463. /*
  1464. * shrink_slab() does not currently allow us to determine how
  1465. * many pages were freed in this zone. So we take the current
  1466. * number of slab pages and shake the slab until it is reduced
  1467. * by the same nr_pages that we used for reclaiming unmapped
  1468. * pages.
  1469. *
  1470. * Note that shrink_slab will free memory on all zones and may
  1471. * take a long time.
  1472. */
  1473. while (shrink_slab(sc.nr_scanned, gfp_mask, order) &&
  1474. zone_page_state(zone, NR_SLAB_RECLAIMABLE) >
  1475. slab_reclaimable - nr_pages)
  1476. ;
  1477. /*
  1478. * Update nr_reclaimed by the number of slab pages we
  1479. * reclaimed from this zone.
  1480. */
  1481. nr_reclaimed += slab_reclaimable -
  1482. zone_page_state(zone, NR_SLAB_RECLAIMABLE);
  1483. }
  1484. p->reclaim_state = NULL;
  1485. current->flags &= ~(PF_MEMALLOC | PF_SWAPWRITE);
  1486. return nr_reclaimed >= nr_pages;
  1487. }
  1488. int zone_reclaim(struct zone *zone, gfp_t gfp_mask, unsigned int order)
  1489. {
  1490. cpumask_t mask;
  1491. int node_id;
  1492. /*
  1493. * Zone reclaim reclaims unmapped file backed pages and
  1494. * slab pages if we are over the defined limits.
  1495. *
  1496. * A small portion of unmapped file backed pages is needed for
  1497. * file I/O otherwise pages read by file I/O will be immediately
  1498. * thrown out if the zone is overallocated. So we do not reclaim
  1499. * if less than a specified percentage of the zone is used by
  1500. * unmapped file backed pages.
  1501. */
  1502. if (zone_page_state(zone, NR_FILE_PAGES) -
  1503. zone_page_state(zone, NR_FILE_MAPPED) <= zone->min_unmapped_pages
  1504. && zone_page_state(zone, NR_SLAB_RECLAIMABLE)
  1505. <= zone->min_slab_pages)
  1506. return 0;
  1507. /*
  1508. * Avoid concurrent zone reclaims, do not reclaim in a zone that does
  1509. * not have reclaimable pages and if we should not delay the allocation
  1510. * then do not scan.
  1511. */
  1512. if (!(gfp_mask & __GFP_WAIT) ||
  1513. zone->all_unreclaimable ||
  1514. atomic_read(&zone->reclaim_in_progress) > 0 ||
  1515. (current->flags & PF_MEMALLOC))
  1516. return 0;
  1517. /*
  1518. * Only run zone reclaim on the local zone or on zones that do not
  1519. * have associated processors. This will favor the local processor
  1520. * over remote processors and spread off node memory allocations
  1521. * as wide as possible.
  1522. */
  1523. node_id = zone_to_nid(zone);
  1524. mask = node_to_cpumask(node_id);
  1525. if (!cpus_empty(mask) && node_id != numa_node_id())
  1526. return 0;
  1527. return __zone_reclaim(zone, gfp_mask, order);
  1528. }
  1529. #endif