swapfile.c 44 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801
  1. /*
  2. * linux/mm/swapfile.c
  3. *
  4. * Copyright (C) 1991, 1992, 1993, 1994 Linus Torvalds
  5. * Swap reorganised 29.12.95, Stephen Tweedie
  6. */
  7. #include <linux/mm.h>
  8. #include <linux/hugetlb.h>
  9. #include <linux/mman.h>
  10. #include <linux/slab.h>
  11. #include <linux/kernel_stat.h>
  12. #include <linux/swap.h>
  13. #include <linux/vmalloc.h>
  14. #include <linux/pagemap.h>
  15. #include <linux/namei.h>
  16. #include <linux/shm.h>
  17. #include <linux/blkdev.h>
  18. #include <linux/writeback.h>
  19. #include <linux/proc_fs.h>
  20. #include <linux/seq_file.h>
  21. #include <linux/init.h>
  22. #include <linux/module.h>
  23. #include <linux/rmap.h>
  24. #include <linux/security.h>
  25. #include <linux/backing-dev.h>
  26. #include <linux/mutex.h>
  27. #include <linux/capability.h>
  28. #include <linux/syscalls.h>
  29. #include <asm/pgtable.h>
  30. #include <asm/tlbflush.h>
  31. #include <linux/swapops.h>
  32. DEFINE_SPINLOCK(swap_lock);
  33. unsigned int nr_swapfiles;
  34. long total_swap_pages;
  35. static int swap_overflow;
  36. static const char Bad_file[] = "Bad swap file entry ";
  37. static const char Unused_file[] = "Unused swap file entry ";
  38. static const char Bad_offset[] = "Bad swap offset entry ";
  39. static const char Unused_offset[] = "Unused swap offset entry ";
  40. struct swap_list_t swap_list = {-1, -1};
  41. static struct swap_info_struct swap_info[MAX_SWAPFILES];
  42. static DEFINE_MUTEX(swapon_mutex);
  43. /*
  44. * We need this because the bdev->unplug_fn can sleep and we cannot
  45. * hold swap_lock while calling the unplug_fn. And swap_lock
  46. * cannot be turned into a mutex.
  47. */
  48. static DECLARE_RWSEM(swap_unplug_sem);
  49. void swap_unplug_io_fn(struct backing_dev_info *unused_bdi, struct page *page)
  50. {
  51. swp_entry_t entry;
  52. down_read(&swap_unplug_sem);
  53. entry.val = page_private(page);
  54. if (PageSwapCache(page)) {
  55. struct block_device *bdev = swap_info[swp_type(entry)].bdev;
  56. struct backing_dev_info *bdi;
  57. /*
  58. * If the page is removed from swapcache from under us (with a
  59. * racy try_to_unuse/swapoff) we need an additional reference
  60. * count to avoid reading garbage from page_private(page) above.
  61. * If the WARN_ON triggers during a swapoff it maybe the race
  62. * condition and it's harmless. However if it triggers without
  63. * swapoff it signals a problem.
  64. */
  65. WARN_ON(page_count(page) <= 1);
  66. bdi = bdev->bd_inode->i_mapping->backing_dev_info;
  67. blk_run_backing_dev(bdi, page);
  68. }
  69. up_read(&swap_unplug_sem);
  70. }
  71. #define SWAPFILE_CLUSTER 256
  72. #define LATENCY_LIMIT 256
  73. static inline unsigned long scan_swap_map(struct swap_info_struct *si)
  74. {
  75. unsigned long offset, last_in_cluster;
  76. int latency_ration = LATENCY_LIMIT;
  77. /*
  78. * We try to cluster swap pages by allocating them sequentially
  79. * in swap. Once we've allocated SWAPFILE_CLUSTER pages this
  80. * way, however, we resort to first-free allocation, starting
  81. * a new cluster. This prevents us from scattering swap pages
  82. * all over the entire swap partition, so that we reduce
  83. * overall disk seek times between swap pages. -- sct
  84. * But we do now try to find an empty cluster. -Andrea
  85. */
  86. si->flags += SWP_SCANNING;
  87. if (unlikely(!si->cluster_nr)) {
  88. si->cluster_nr = SWAPFILE_CLUSTER - 1;
  89. if (si->pages - si->inuse_pages < SWAPFILE_CLUSTER)
  90. goto lowest;
  91. spin_unlock(&swap_lock);
  92. offset = si->lowest_bit;
  93. last_in_cluster = offset + SWAPFILE_CLUSTER - 1;
  94. /* Locate the first empty (unaligned) cluster */
  95. for (; last_in_cluster <= si->highest_bit; offset++) {
  96. if (si->swap_map[offset])
  97. last_in_cluster = offset + SWAPFILE_CLUSTER;
  98. else if (offset == last_in_cluster) {
  99. spin_lock(&swap_lock);
  100. si->cluster_next = offset-SWAPFILE_CLUSTER+1;
  101. goto cluster;
  102. }
  103. if (unlikely(--latency_ration < 0)) {
  104. cond_resched();
  105. latency_ration = LATENCY_LIMIT;
  106. }
  107. }
  108. spin_lock(&swap_lock);
  109. goto lowest;
  110. }
  111. si->cluster_nr--;
  112. cluster:
  113. offset = si->cluster_next;
  114. if (offset > si->highest_bit)
  115. lowest: offset = si->lowest_bit;
  116. checks: if (!(si->flags & SWP_WRITEOK))
  117. goto no_page;
  118. if (!si->highest_bit)
  119. goto no_page;
  120. if (!si->swap_map[offset]) {
  121. if (offset == si->lowest_bit)
  122. si->lowest_bit++;
  123. if (offset == si->highest_bit)
  124. si->highest_bit--;
  125. si->inuse_pages++;
  126. if (si->inuse_pages == si->pages) {
  127. si->lowest_bit = si->max;
  128. si->highest_bit = 0;
  129. }
  130. si->swap_map[offset] = 1;
  131. si->cluster_next = offset + 1;
  132. si->flags -= SWP_SCANNING;
  133. return offset;
  134. }
  135. spin_unlock(&swap_lock);
  136. while (++offset <= si->highest_bit) {
  137. if (!si->swap_map[offset]) {
  138. spin_lock(&swap_lock);
  139. goto checks;
  140. }
  141. if (unlikely(--latency_ration < 0)) {
  142. cond_resched();
  143. latency_ration = LATENCY_LIMIT;
  144. }
  145. }
  146. spin_lock(&swap_lock);
  147. goto lowest;
  148. no_page:
  149. si->flags -= SWP_SCANNING;
  150. return 0;
  151. }
  152. swp_entry_t get_swap_page(void)
  153. {
  154. struct swap_info_struct *si;
  155. pgoff_t offset;
  156. int type, next;
  157. int wrapped = 0;
  158. spin_lock(&swap_lock);
  159. if (nr_swap_pages <= 0)
  160. goto noswap;
  161. nr_swap_pages--;
  162. for (type = swap_list.next; type >= 0 && wrapped < 2; type = next) {
  163. si = swap_info + type;
  164. next = si->next;
  165. if (next < 0 ||
  166. (!wrapped && si->prio != swap_info[next].prio)) {
  167. next = swap_list.head;
  168. wrapped++;
  169. }
  170. if (!si->highest_bit)
  171. continue;
  172. if (!(si->flags & SWP_WRITEOK))
  173. continue;
  174. swap_list.next = next;
  175. offset = scan_swap_map(si);
  176. if (offset) {
  177. spin_unlock(&swap_lock);
  178. return swp_entry(type, offset);
  179. }
  180. next = swap_list.next;
  181. }
  182. nr_swap_pages++;
  183. noswap:
  184. spin_unlock(&swap_lock);
  185. return (swp_entry_t) {0};
  186. }
  187. swp_entry_t get_swap_page_of_type(int type)
  188. {
  189. struct swap_info_struct *si;
  190. pgoff_t offset;
  191. spin_lock(&swap_lock);
  192. si = swap_info + type;
  193. if (si->flags & SWP_WRITEOK) {
  194. nr_swap_pages--;
  195. offset = scan_swap_map(si);
  196. if (offset) {
  197. spin_unlock(&swap_lock);
  198. return swp_entry(type, offset);
  199. }
  200. nr_swap_pages++;
  201. }
  202. spin_unlock(&swap_lock);
  203. return (swp_entry_t) {0};
  204. }
  205. static struct swap_info_struct * swap_info_get(swp_entry_t entry)
  206. {
  207. struct swap_info_struct * p;
  208. unsigned long offset, type;
  209. if (!entry.val)
  210. goto out;
  211. type = swp_type(entry);
  212. if (type >= nr_swapfiles)
  213. goto bad_nofile;
  214. p = & swap_info[type];
  215. if (!(p->flags & SWP_USED))
  216. goto bad_device;
  217. offset = swp_offset(entry);
  218. if (offset >= p->max)
  219. goto bad_offset;
  220. if (!p->swap_map[offset])
  221. goto bad_free;
  222. spin_lock(&swap_lock);
  223. return p;
  224. bad_free:
  225. printk(KERN_ERR "swap_free: %s%08lx\n", Unused_offset, entry.val);
  226. goto out;
  227. bad_offset:
  228. printk(KERN_ERR "swap_free: %s%08lx\n", Bad_offset, entry.val);
  229. goto out;
  230. bad_device:
  231. printk(KERN_ERR "swap_free: %s%08lx\n", Unused_file, entry.val);
  232. goto out;
  233. bad_nofile:
  234. printk(KERN_ERR "swap_free: %s%08lx\n", Bad_file, entry.val);
  235. out:
  236. return NULL;
  237. }
  238. static int swap_entry_free(struct swap_info_struct *p, unsigned long offset)
  239. {
  240. int count = p->swap_map[offset];
  241. if (count < SWAP_MAP_MAX) {
  242. count--;
  243. p->swap_map[offset] = count;
  244. if (!count) {
  245. if (offset < p->lowest_bit)
  246. p->lowest_bit = offset;
  247. if (offset > p->highest_bit)
  248. p->highest_bit = offset;
  249. if (p->prio > swap_info[swap_list.next].prio)
  250. swap_list.next = p - swap_info;
  251. nr_swap_pages++;
  252. p->inuse_pages--;
  253. }
  254. }
  255. return count;
  256. }
  257. /*
  258. * Caller has made sure that the swapdevice corresponding to entry
  259. * is still around or has not been recycled.
  260. */
  261. void swap_free(swp_entry_t entry)
  262. {
  263. struct swap_info_struct * p;
  264. p = swap_info_get(entry);
  265. if (p) {
  266. swap_entry_free(p, swp_offset(entry));
  267. spin_unlock(&swap_lock);
  268. }
  269. }
  270. /*
  271. * How many references to page are currently swapped out?
  272. */
  273. static inline int page_swapcount(struct page *page)
  274. {
  275. int count = 0;
  276. struct swap_info_struct *p;
  277. swp_entry_t entry;
  278. entry.val = page_private(page);
  279. p = swap_info_get(entry);
  280. if (p) {
  281. /* Subtract the 1 for the swap cache itself */
  282. count = p->swap_map[swp_offset(entry)] - 1;
  283. spin_unlock(&swap_lock);
  284. }
  285. return count;
  286. }
  287. /*
  288. * We can use this swap cache entry directly
  289. * if there are no other references to it.
  290. */
  291. int can_share_swap_page(struct page *page)
  292. {
  293. int count;
  294. BUG_ON(!PageLocked(page));
  295. count = page_mapcount(page);
  296. if (count <= 1 && PageSwapCache(page))
  297. count += page_swapcount(page);
  298. return count == 1;
  299. }
  300. /*
  301. * Work out if there are any other processes sharing this
  302. * swap cache page. Free it if you can. Return success.
  303. */
  304. int remove_exclusive_swap_page(struct page *page)
  305. {
  306. int retval;
  307. struct swap_info_struct * p;
  308. swp_entry_t entry;
  309. BUG_ON(PagePrivate(page));
  310. BUG_ON(!PageLocked(page));
  311. if (!PageSwapCache(page))
  312. return 0;
  313. if (PageWriteback(page))
  314. return 0;
  315. if (page_count(page) != 2) /* 2: us + cache */
  316. return 0;
  317. entry.val = page_private(page);
  318. p = swap_info_get(entry);
  319. if (!p)
  320. return 0;
  321. /* Is the only swap cache user the cache itself? */
  322. retval = 0;
  323. if (p->swap_map[swp_offset(entry)] == 1) {
  324. /* Recheck the page count with the swapcache lock held.. */
  325. write_lock_irq(&swapper_space.tree_lock);
  326. if ((page_count(page) == 2) && !PageWriteback(page)) {
  327. __delete_from_swap_cache(page);
  328. SetPageDirty(page);
  329. retval = 1;
  330. }
  331. write_unlock_irq(&swapper_space.tree_lock);
  332. }
  333. spin_unlock(&swap_lock);
  334. if (retval) {
  335. swap_free(entry);
  336. page_cache_release(page);
  337. }
  338. return retval;
  339. }
  340. /*
  341. * Free the swap entry like above, but also try to
  342. * free the page cache entry if it is the last user.
  343. */
  344. void free_swap_and_cache(swp_entry_t entry)
  345. {
  346. struct swap_info_struct * p;
  347. struct page *page = NULL;
  348. if (is_migration_entry(entry))
  349. return;
  350. p = swap_info_get(entry);
  351. if (p) {
  352. if (swap_entry_free(p, swp_offset(entry)) == 1) {
  353. page = find_get_page(&swapper_space, entry.val);
  354. if (page && unlikely(TestSetPageLocked(page))) {
  355. page_cache_release(page);
  356. page = NULL;
  357. }
  358. }
  359. spin_unlock(&swap_lock);
  360. }
  361. if (page) {
  362. int one_user;
  363. BUG_ON(PagePrivate(page));
  364. one_user = (page_count(page) == 2);
  365. /* Only cache user (+us), or swap space full? Free it! */
  366. /* Also recheck PageSwapCache after page is locked (above) */
  367. if (PageSwapCache(page) && !PageWriteback(page) &&
  368. (one_user || vm_swap_full())) {
  369. delete_from_swap_cache(page);
  370. SetPageDirty(page);
  371. }
  372. unlock_page(page);
  373. page_cache_release(page);
  374. }
  375. }
  376. #ifdef CONFIG_SOFTWARE_SUSPEND
  377. /*
  378. * Find the swap type that corresponds to given device (if any).
  379. *
  380. * @offset - number of the PAGE_SIZE-sized block of the device, starting
  381. * from 0, in which the swap header is expected to be located.
  382. *
  383. * This is needed for the suspend to disk (aka swsusp).
  384. */
  385. int swap_type_of(dev_t device, sector_t offset, struct block_device **bdev_p)
  386. {
  387. struct block_device *bdev = NULL;
  388. int i;
  389. if (device)
  390. bdev = bdget(device);
  391. spin_lock(&swap_lock);
  392. for (i = 0; i < nr_swapfiles; i++) {
  393. struct swap_info_struct *sis = swap_info + i;
  394. if (!(sis->flags & SWP_WRITEOK))
  395. continue;
  396. if (!bdev) {
  397. if (bdev_p)
  398. *bdev_p = sis->bdev;
  399. spin_unlock(&swap_lock);
  400. return i;
  401. }
  402. if (bdev == sis->bdev) {
  403. struct swap_extent *se;
  404. se = list_entry(sis->extent_list.next,
  405. struct swap_extent, list);
  406. if (se->start_block == offset) {
  407. if (bdev_p)
  408. *bdev_p = sis->bdev;
  409. spin_unlock(&swap_lock);
  410. bdput(bdev);
  411. return i;
  412. }
  413. }
  414. }
  415. spin_unlock(&swap_lock);
  416. if (bdev)
  417. bdput(bdev);
  418. return -ENODEV;
  419. }
  420. /*
  421. * Return either the total number of swap pages of given type, or the number
  422. * of free pages of that type (depending on @free)
  423. *
  424. * This is needed for software suspend
  425. */
  426. unsigned int count_swap_pages(int type, int free)
  427. {
  428. unsigned int n = 0;
  429. if (type < nr_swapfiles) {
  430. spin_lock(&swap_lock);
  431. if (swap_info[type].flags & SWP_WRITEOK) {
  432. n = swap_info[type].pages;
  433. if (free)
  434. n -= swap_info[type].inuse_pages;
  435. }
  436. spin_unlock(&swap_lock);
  437. }
  438. return n;
  439. }
  440. #endif
  441. /*
  442. * No need to decide whether this PTE shares the swap entry with others,
  443. * just let do_wp_page work it out if a write is requested later - to
  444. * force COW, vm_page_prot omits write permission from any private vma.
  445. */
  446. static void unuse_pte(struct vm_area_struct *vma, pte_t *pte,
  447. unsigned long addr, swp_entry_t entry, struct page *page)
  448. {
  449. inc_mm_counter(vma->vm_mm, anon_rss);
  450. get_page(page);
  451. set_pte_at(vma->vm_mm, addr, pte,
  452. pte_mkold(mk_pte(page, vma->vm_page_prot)));
  453. page_add_anon_rmap(page, vma, addr);
  454. swap_free(entry);
  455. /*
  456. * Move the page to the active list so it is not
  457. * immediately swapped out again after swapon.
  458. */
  459. activate_page(page);
  460. }
  461. static int unuse_pte_range(struct vm_area_struct *vma, pmd_t *pmd,
  462. unsigned long addr, unsigned long end,
  463. swp_entry_t entry, struct page *page)
  464. {
  465. pte_t swp_pte = swp_entry_to_pte(entry);
  466. pte_t *pte;
  467. spinlock_t *ptl;
  468. int found = 0;
  469. pte = pte_offset_map_lock(vma->vm_mm, pmd, addr, &ptl);
  470. do {
  471. /*
  472. * swapoff spends a _lot_ of time in this loop!
  473. * Test inline before going to call unuse_pte.
  474. */
  475. if (unlikely(pte_same(*pte, swp_pte))) {
  476. unuse_pte(vma, pte++, addr, entry, page);
  477. found = 1;
  478. break;
  479. }
  480. } while (pte++, addr += PAGE_SIZE, addr != end);
  481. pte_unmap_unlock(pte - 1, ptl);
  482. return found;
  483. }
  484. static inline int unuse_pmd_range(struct vm_area_struct *vma, pud_t *pud,
  485. unsigned long addr, unsigned long end,
  486. swp_entry_t entry, struct page *page)
  487. {
  488. pmd_t *pmd;
  489. unsigned long next;
  490. pmd = pmd_offset(pud, addr);
  491. do {
  492. next = pmd_addr_end(addr, end);
  493. if (pmd_none_or_clear_bad(pmd))
  494. continue;
  495. if (unuse_pte_range(vma, pmd, addr, next, entry, page))
  496. return 1;
  497. } while (pmd++, addr = next, addr != end);
  498. return 0;
  499. }
  500. static inline int unuse_pud_range(struct vm_area_struct *vma, pgd_t *pgd,
  501. unsigned long addr, unsigned long end,
  502. swp_entry_t entry, struct page *page)
  503. {
  504. pud_t *pud;
  505. unsigned long next;
  506. pud = pud_offset(pgd, addr);
  507. do {
  508. next = pud_addr_end(addr, end);
  509. if (pud_none_or_clear_bad(pud))
  510. continue;
  511. if (unuse_pmd_range(vma, pud, addr, next, entry, page))
  512. return 1;
  513. } while (pud++, addr = next, addr != end);
  514. return 0;
  515. }
  516. static int unuse_vma(struct vm_area_struct *vma,
  517. swp_entry_t entry, struct page *page)
  518. {
  519. pgd_t *pgd;
  520. unsigned long addr, end, next;
  521. if (page->mapping) {
  522. addr = page_address_in_vma(page, vma);
  523. if (addr == -EFAULT)
  524. return 0;
  525. else
  526. end = addr + PAGE_SIZE;
  527. } else {
  528. addr = vma->vm_start;
  529. end = vma->vm_end;
  530. }
  531. pgd = pgd_offset(vma->vm_mm, addr);
  532. do {
  533. next = pgd_addr_end(addr, end);
  534. if (pgd_none_or_clear_bad(pgd))
  535. continue;
  536. if (unuse_pud_range(vma, pgd, addr, next, entry, page))
  537. return 1;
  538. } while (pgd++, addr = next, addr != end);
  539. return 0;
  540. }
  541. static int unuse_mm(struct mm_struct *mm,
  542. swp_entry_t entry, struct page *page)
  543. {
  544. struct vm_area_struct *vma;
  545. if (!down_read_trylock(&mm->mmap_sem)) {
  546. /*
  547. * Activate page so shrink_cache is unlikely to unmap its
  548. * ptes while lock is dropped, so swapoff can make progress.
  549. */
  550. activate_page(page);
  551. unlock_page(page);
  552. down_read(&mm->mmap_sem);
  553. lock_page(page);
  554. }
  555. for (vma = mm->mmap; vma; vma = vma->vm_next) {
  556. if (vma->anon_vma && unuse_vma(vma, entry, page))
  557. break;
  558. }
  559. up_read(&mm->mmap_sem);
  560. /*
  561. * Currently unuse_mm cannot fail, but leave error handling
  562. * at call sites for now, since we change it from time to time.
  563. */
  564. return 0;
  565. }
  566. /*
  567. * Scan swap_map from current position to next entry still in use.
  568. * Recycle to start on reaching the end, returning 0 when empty.
  569. */
  570. static unsigned int find_next_to_unuse(struct swap_info_struct *si,
  571. unsigned int prev)
  572. {
  573. unsigned int max = si->max;
  574. unsigned int i = prev;
  575. int count;
  576. /*
  577. * No need for swap_lock here: we're just looking
  578. * for whether an entry is in use, not modifying it; false
  579. * hits are okay, and sys_swapoff() has already prevented new
  580. * allocations from this area (while holding swap_lock).
  581. */
  582. for (;;) {
  583. if (++i >= max) {
  584. if (!prev) {
  585. i = 0;
  586. break;
  587. }
  588. /*
  589. * No entries in use at top of swap_map,
  590. * loop back to start and recheck there.
  591. */
  592. max = prev + 1;
  593. prev = 0;
  594. i = 1;
  595. }
  596. count = si->swap_map[i];
  597. if (count && count != SWAP_MAP_BAD)
  598. break;
  599. }
  600. return i;
  601. }
  602. /*
  603. * We completely avoid races by reading each swap page in advance,
  604. * and then search for the process using it. All the necessary
  605. * page table adjustments can then be made atomically.
  606. */
  607. static int try_to_unuse(unsigned int type)
  608. {
  609. struct swap_info_struct * si = &swap_info[type];
  610. struct mm_struct *start_mm;
  611. unsigned short *swap_map;
  612. unsigned short swcount;
  613. struct page *page;
  614. swp_entry_t entry;
  615. unsigned int i = 0;
  616. int retval = 0;
  617. int reset_overflow = 0;
  618. int shmem;
  619. /*
  620. * When searching mms for an entry, a good strategy is to
  621. * start at the first mm we freed the previous entry from
  622. * (though actually we don't notice whether we or coincidence
  623. * freed the entry). Initialize this start_mm with a hold.
  624. *
  625. * A simpler strategy would be to start at the last mm we
  626. * freed the previous entry from; but that would take less
  627. * advantage of mmlist ordering, which clusters forked mms
  628. * together, child after parent. If we race with dup_mmap(), we
  629. * prefer to resolve parent before child, lest we miss entries
  630. * duplicated after we scanned child: using last mm would invert
  631. * that. Though it's only a serious concern when an overflowed
  632. * swap count is reset from SWAP_MAP_MAX, preventing a rescan.
  633. */
  634. start_mm = &init_mm;
  635. atomic_inc(&init_mm.mm_users);
  636. /*
  637. * Keep on scanning until all entries have gone. Usually,
  638. * one pass through swap_map is enough, but not necessarily:
  639. * there are races when an instance of an entry might be missed.
  640. */
  641. while ((i = find_next_to_unuse(si, i)) != 0) {
  642. if (signal_pending(current)) {
  643. retval = -EINTR;
  644. break;
  645. }
  646. /*
  647. * Get a page for the entry, using the existing swap
  648. * cache page if there is one. Otherwise, get a clean
  649. * page and read the swap into it.
  650. */
  651. swap_map = &si->swap_map[i];
  652. entry = swp_entry(type, i);
  653. page = read_swap_cache_async(entry, NULL, 0);
  654. if (!page) {
  655. /*
  656. * Either swap_duplicate() failed because entry
  657. * has been freed independently, and will not be
  658. * reused since sys_swapoff() already disabled
  659. * allocation from here, or alloc_page() failed.
  660. */
  661. if (!*swap_map)
  662. continue;
  663. retval = -ENOMEM;
  664. break;
  665. }
  666. /*
  667. * Don't hold on to start_mm if it looks like exiting.
  668. */
  669. if (atomic_read(&start_mm->mm_users) == 1) {
  670. mmput(start_mm);
  671. start_mm = &init_mm;
  672. atomic_inc(&init_mm.mm_users);
  673. }
  674. /*
  675. * Wait for and lock page. When do_swap_page races with
  676. * try_to_unuse, do_swap_page can handle the fault much
  677. * faster than try_to_unuse can locate the entry. This
  678. * apparently redundant "wait_on_page_locked" lets try_to_unuse
  679. * defer to do_swap_page in such a case - in some tests,
  680. * do_swap_page and try_to_unuse repeatedly compete.
  681. */
  682. wait_on_page_locked(page);
  683. wait_on_page_writeback(page);
  684. lock_page(page);
  685. wait_on_page_writeback(page);
  686. /*
  687. * Remove all references to entry.
  688. * Whenever we reach init_mm, there's no address space
  689. * to search, but use it as a reminder to search shmem.
  690. */
  691. shmem = 0;
  692. swcount = *swap_map;
  693. if (swcount > 1) {
  694. if (start_mm == &init_mm)
  695. shmem = shmem_unuse(entry, page);
  696. else
  697. retval = unuse_mm(start_mm, entry, page);
  698. }
  699. if (*swap_map > 1) {
  700. int set_start_mm = (*swap_map >= swcount);
  701. struct list_head *p = &start_mm->mmlist;
  702. struct mm_struct *new_start_mm = start_mm;
  703. struct mm_struct *prev_mm = start_mm;
  704. struct mm_struct *mm;
  705. atomic_inc(&new_start_mm->mm_users);
  706. atomic_inc(&prev_mm->mm_users);
  707. spin_lock(&mmlist_lock);
  708. while (*swap_map > 1 && !retval &&
  709. (p = p->next) != &start_mm->mmlist) {
  710. mm = list_entry(p, struct mm_struct, mmlist);
  711. if (!atomic_inc_not_zero(&mm->mm_users))
  712. continue;
  713. spin_unlock(&mmlist_lock);
  714. mmput(prev_mm);
  715. prev_mm = mm;
  716. cond_resched();
  717. swcount = *swap_map;
  718. if (swcount <= 1)
  719. ;
  720. else if (mm == &init_mm) {
  721. set_start_mm = 1;
  722. shmem = shmem_unuse(entry, page);
  723. } else
  724. retval = unuse_mm(mm, entry, page);
  725. if (set_start_mm && *swap_map < swcount) {
  726. mmput(new_start_mm);
  727. atomic_inc(&mm->mm_users);
  728. new_start_mm = mm;
  729. set_start_mm = 0;
  730. }
  731. spin_lock(&mmlist_lock);
  732. }
  733. spin_unlock(&mmlist_lock);
  734. mmput(prev_mm);
  735. mmput(start_mm);
  736. start_mm = new_start_mm;
  737. }
  738. if (retval) {
  739. unlock_page(page);
  740. page_cache_release(page);
  741. break;
  742. }
  743. /*
  744. * How could swap count reach 0x7fff when the maximum
  745. * pid is 0x7fff, and there's no way to repeat a swap
  746. * page within an mm (except in shmem, where it's the
  747. * shared object which takes the reference count)?
  748. * We believe SWAP_MAP_MAX cannot occur in Linux 2.4.
  749. *
  750. * If that's wrong, then we should worry more about
  751. * exit_mmap() and do_munmap() cases described above:
  752. * we might be resetting SWAP_MAP_MAX too early here.
  753. * We know "Undead"s can happen, they're okay, so don't
  754. * report them; but do report if we reset SWAP_MAP_MAX.
  755. */
  756. if (*swap_map == SWAP_MAP_MAX) {
  757. spin_lock(&swap_lock);
  758. *swap_map = 1;
  759. spin_unlock(&swap_lock);
  760. reset_overflow = 1;
  761. }
  762. /*
  763. * If a reference remains (rare), we would like to leave
  764. * the page in the swap cache; but try_to_unmap could
  765. * then re-duplicate the entry once we drop page lock,
  766. * so we might loop indefinitely; also, that page could
  767. * not be swapped out to other storage meanwhile. So:
  768. * delete from cache even if there's another reference,
  769. * after ensuring that the data has been saved to disk -
  770. * since if the reference remains (rarer), it will be
  771. * read from disk into another page. Splitting into two
  772. * pages would be incorrect if swap supported "shared
  773. * private" pages, but they are handled by tmpfs files.
  774. *
  775. * Note shmem_unuse already deleted a swappage from
  776. * the swap cache, unless the move to filepage failed:
  777. * in which case it left swappage in cache, lowered its
  778. * swap count to pass quickly through the loops above,
  779. * and now we must reincrement count to try again later.
  780. */
  781. if ((*swap_map > 1) && PageDirty(page) && PageSwapCache(page)) {
  782. struct writeback_control wbc = {
  783. .sync_mode = WB_SYNC_NONE,
  784. };
  785. swap_writepage(page, &wbc);
  786. lock_page(page);
  787. wait_on_page_writeback(page);
  788. }
  789. if (PageSwapCache(page)) {
  790. if (shmem)
  791. swap_duplicate(entry);
  792. else
  793. delete_from_swap_cache(page);
  794. }
  795. /*
  796. * So we could skip searching mms once swap count went
  797. * to 1, we did not mark any present ptes as dirty: must
  798. * mark page dirty so shrink_list will preserve it.
  799. */
  800. SetPageDirty(page);
  801. unlock_page(page);
  802. page_cache_release(page);
  803. /*
  804. * Make sure that we aren't completely killing
  805. * interactive performance.
  806. */
  807. cond_resched();
  808. }
  809. mmput(start_mm);
  810. if (reset_overflow) {
  811. printk(KERN_WARNING "swapoff: cleared swap entry overflow\n");
  812. swap_overflow = 0;
  813. }
  814. return retval;
  815. }
  816. /*
  817. * After a successful try_to_unuse, if no swap is now in use, we know
  818. * we can empty the mmlist. swap_lock must be held on entry and exit.
  819. * Note that mmlist_lock nests inside swap_lock, and an mm must be
  820. * added to the mmlist just after page_duplicate - before would be racy.
  821. */
  822. static void drain_mmlist(void)
  823. {
  824. struct list_head *p, *next;
  825. unsigned int i;
  826. for (i = 0; i < nr_swapfiles; i++)
  827. if (swap_info[i].inuse_pages)
  828. return;
  829. spin_lock(&mmlist_lock);
  830. list_for_each_safe(p, next, &init_mm.mmlist)
  831. list_del_init(p);
  832. spin_unlock(&mmlist_lock);
  833. }
  834. /*
  835. * Use this swapdev's extent info to locate the (PAGE_SIZE) block which
  836. * corresponds to page offset `offset'.
  837. */
  838. sector_t map_swap_page(struct swap_info_struct *sis, pgoff_t offset)
  839. {
  840. struct swap_extent *se = sis->curr_swap_extent;
  841. struct swap_extent *start_se = se;
  842. for ( ; ; ) {
  843. struct list_head *lh;
  844. if (se->start_page <= offset &&
  845. offset < (se->start_page + se->nr_pages)) {
  846. return se->start_block + (offset - se->start_page);
  847. }
  848. lh = se->list.next;
  849. if (lh == &sis->extent_list)
  850. lh = lh->next;
  851. se = list_entry(lh, struct swap_extent, list);
  852. sis->curr_swap_extent = se;
  853. BUG_ON(se == start_se); /* It *must* be present */
  854. }
  855. }
  856. #ifdef CONFIG_SOFTWARE_SUSPEND
  857. /*
  858. * Get the (PAGE_SIZE) block corresponding to given offset on the swapdev
  859. * corresponding to given index in swap_info (swap type).
  860. */
  861. sector_t swapdev_block(int swap_type, pgoff_t offset)
  862. {
  863. struct swap_info_struct *sis;
  864. if (swap_type >= nr_swapfiles)
  865. return 0;
  866. sis = swap_info + swap_type;
  867. return (sis->flags & SWP_WRITEOK) ? map_swap_page(sis, offset) : 0;
  868. }
  869. #endif /* CONFIG_SOFTWARE_SUSPEND */
  870. /*
  871. * Free all of a swapdev's extent information
  872. */
  873. static void destroy_swap_extents(struct swap_info_struct *sis)
  874. {
  875. while (!list_empty(&sis->extent_list)) {
  876. struct swap_extent *se;
  877. se = list_entry(sis->extent_list.next,
  878. struct swap_extent, list);
  879. list_del(&se->list);
  880. kfree(se);
  881. }
  882. }
  883. /*
  884. * Add a block range (and the corresponding page range) into this swapdev's
  885. * extent list. The extent list is kept sorted in page order.
  886. *
  887. * This function rather assumes that it is called in ascending page order.
  888. */
  889. static int
  890. add_swap_extent(struct swap_info_struct *sis, unsigned long start_page,
  891. unsigned long nr_pages, sector_t start_block)
  892. {
  893. struct swap_extent *se;
  894. struct swap_extent *new_se;
  895. struct list_head *lh;
  896. lh = sis->extent_list.prev; /* The highest page extent */
  897. if (lh != &sis->extent_list) {
  898. se = list_entry(lh, struct swap_extent, list);
  899. BUG_ON(se->start_page + se->nr_pages != start_page);
  900. if (se->start_block + se->nr_pages == start_block) {
  901. /* Merge it */
  902. se->nr_pages += nr_pages;
  903. return 0;
  904. }
  905. }
  906. /*
  907. * No merge. Insert a new extent, preserving ordering.
  908. */
  909. new_se = kmalloc(sizeof(*se), GFP_KERNEL);
  910. if (new_se == NULL)
  911. return -ENOMEM;
  912. new_se->start_page = start_page;
  913. new_se->nr_pages = nr_pages;
  914. new_se->start_block = start_block;
  915. list_add_tail(&new_se->list, &sis->extent_list);
  916. return 1;
  917. }
  918. /*
  919. * A `swap extent' is a simple thing which maps a contiguous range of pages
  920. * onto a contiguous range of disk blocks. An ordered list of swap extents
  921. * is built at swapon time and is then used at swap_writepage/swap_readpage
  922. * time for locating where on disk a page belongs.
  923. *
  924. * If the swapfile is an S_ISBLK block device, a single extent is installed.
  925. * This is done so that the main operating code can treat S_ISBLK and S_ISREG
  926. * swap files identically.
  927. *
  928. * Whether the swapdev is an S_ISREG file or an S_ISBLK blockdev, the swap
  929. * extent list operates in PAGE_SIZE disk blocks. Both S_ISREG and S_ISBLK
  930. * swapfiles are handled *identically* after swapon time.
  931. *
  932. * For S_ISREG swapfiles, setup_swap_extents() will walk all the file's blocks
  933. * and will parse them into an ordered extent list, in PAGE_SIZE chunks. If
  934. * some stray blocks are found which do not fall within the PAGE_SIZE alignment
  935. * requirements, they are simply tossed out - we will never use those blocks
  936. * for swapping.
  937. *
  938. * For S_ISREG swapfiles we set S_SWAPFILE across the life of the swapon. This
  939. * prevents root from shooting her foot off by ftruncating an in-use swapfile,
  940. * which will scribble on the fs.
  941. *
  942. * The amount of disk space which a single swap extent represents varies.
  943. * Typically it is in the 1-4 megabyte range. So we can have hundreds of
  944. * extents in the list. To avoid much list walking, we cache the previous
  945. * search location in `curr_swap_extent', and start new searches from there.
  946. * This is extremely effective. The average number of iterations in
  947. * map_swap_page() has been measured at about 0.3 per page. - akpm.
  948. */
  949. static int setup_swap_extents(struct swap_info_struct *sis, sector_t *span)
  950. {
  951. struct inode *inode;
  952. unsigned blocks_per_page;
  953. unsigned long page_no;
  954. unsigned blkbits;
  955. sector_t probe_block;
  956. sector_t last_block;
  957. sector_t lowest_block = -1;
  958. sector_t highest_block = 0;
  959. int nr_extents = 0;
  960. int ret;
  961. inode = sis->swap_file->f_mapping->host;
  962. if (S_ISBLK(inode->i_mode)) {
  963. ret = add_swap_extent(sis, 0, sis->max, 0);
  964. *span = sis->pages;
  965. goto done;
  966. }
  967. blkbits = inode->i_blkbits;
  968. blocks_per_page = PAGE_SIZE >> blkbits;
  969. /*
  970. * Map all the blocks into the extent list. This code doesn't try
  971. * to be very smart.
  972. */
  973. probe_block = 0;
  974. page_no = 0;
  975. last_block = i_size_read(inode) >> blkbits;
  976. while ((probe_block + blocks_per_page) <= last_block &&
  977. page_no < sis->max) {
  978. unsigned block_in_page;
  979. sector_t first_block;
  980. first_block = bmap(inode, probe_block);
  981. if (first_block == 0)
  982. goto bad_bmap;
  983. /*
  984. * It must be PAGE_SIZE aligned on-disk
  985. */
  986. if (first_block & (blocks_per_page - 1)) {
  987. probe_block++;
  988. goto reprobe;
  989. }
  990. for (block_in_page = 1; block_in_page < blocks_per_page;
  991. block_in_page++) {
  992. sector_t block;
  993. block = bmap(inode, probe_block + block_in_page);
  994. if (block == 0)
  995. goto bad_bmap;
  996. if (block != first_block + block_in_page) {
  997. /* Discontiguity */
  998. probe_block++;
  999. goto reprobe;
  1000. }
  1001. }
  1002. first_block >>= (PAGE_SHIFT - blkbits);
  1003. if (page_no) { /* exclude the header page */
  1004. if (first_block < lowest_block)
  1005. lowest_block = first_block;
  1006. if (first_block > highest_block)
  1007. highest_block = first_block;
  1008. }
  1009. /*
  1010. * We found a PAGE_SIZE-length, PAGE_SIZE-aligned run of blocks
  1011. */
  1012. ret = add_swap_extent(sis, page_no, 1, first_block);
  1013. if (ret < 0)
  1014. goto out;
  1015. nr_extents += ret;
  1016. page_no++;
  1017. probe_block += blocks_per_page;
  1018. reprobe:
  1019. continue;
  1020. }
  1021. ret = nr_extents;
  1022. *span = 1 + highest_block - lowest_block;
  1023. if (page_no == 0)
  1024. page_no = 1; /* force Empty message */
  1025. sis->max = page_no;
  1026. sis->pages = page_no - 1;
  1027. sis->highest_bit = page_no - 1;
  1028. done:
  1029. sis->curr_swap_extent = list_entry(sis->extent_list.prev,
  1030. struct swap_extent, list);
  1031. goto out;
  1032. bad_bmap:
  1033. printk(KERN_ERR "swapon: swapfile has holes\n");
  1034. ret = -EINVAL;
  1035. out:
  1036. return ret;
  1037. }
  1038. #if 0 /* We don't need this yet */
  1039. #include <linux/backing-dev.h>
  1040. int page_queue_congested(struct page *page)
  1041. {
  1042. struct backing_dev_info *bdi;
  1043. BUG_ON(!PageLocked(page)); /* It pins the swap_info_struct */
  1044. if (PageSwapCache(page)) {
  1045. swp_entry_t entry = { .val = page_private(page) };
  1046. struct swap_info_struct *sis;
  1047. sis = get_swap_info_struct(swp_type(entry));
  1048. bdi = sis->bdev->bd_inode->i_mapping->backing_dev_info;
  1049. } else
  1050. bdi = page->mapping->backing_dev_info;
  1051. return bdi_write_congested(bdi);
  1052. }
  1053. #endif
  1054. asmlinkage long sys_swapoff(const char __user * specialfile)
  1055. {
  1056. struct swap_info_struct * p = NULL;
  1057. unsigned short *swap_map;
  1058. struct file *swap_file, *victim;
  1059. struct address_space *mapping;
  1060. struct inode *inode;
  1061. char * pathname;
  1062. int i, type, prev;
  1063. int err;
  1064. if (!capable(CAP_SYS_ADMIN))
  1065. return -EPERM;
  1066. pathname = getname(specialfile);
  1067. err = PTR_ERR(pathname);
  1068. if (IS_ERR(pathname))
  1069. goto out;
  1070. victim = filp_open(pathname, O_RDWR|O_LARGEFILE, 0);
  1071. putname(pathname);
  1072. err = PTR_ERR(victim);
  1073. if (IS_ERR(victim))
  1074. goto out;
  1075. mapping = victim->f_mapping;
  1076. prev = -1;
  1077. spin_lock(&swap_lock);
  1078. for (type = swap_list.head; type >= 0; type = swap_info[type].next) {
  1079. p = swap_info + type;
  1080. if ((p->flags & SWP_ACTIVE) == SWP_ACTIVE) {
  1081. if (p->swap_file->f_mapping == mapping)
  1082. break;
  1083. }
  1084. prev = type;
  1085. }
  1086. if (type < 0) {
  1087. err = -EINVAL;
  1088. spin_unlock(&swap_lock);
  1089. goto out_dput;
  1090. }
  1091. if (!security_vm_enough_memory(p->pages))
  1092. vm_unacct_memory(p->pages);
  1093. else {
  1094. err = -ENOMEM;
  1095. spin_unlock(&swap_lock);
  1096. goto out_dput;
  1097. }
  1098. if (prev < 0) {
  1099. swap_list.head = p->next;
  1100. } else {
  1101. swap_info[prev].next = p->next;
  1102. }
  1103. if (type == swap_list.next) {
  1104. /* just pick something that's safe... */
  1105. swap_list.next = swap_list.head;
  1106. }
  1107. nr_swap_pages -= p->pages;
  1108. total_swap_pages -= p->pages;
  1109. p->flags &= ~SWP_WRITEOK;
  1110. spin_unlock(&swap_lock);
  1111. current->flags |= PF_SWAPOFF;
  1112. err = try_to_unuse(type);
  1113. current->flags &= ~PF_SWAPOFF;
  1114. if (err) {
  1115. /* re-insert swap space back into swap_list */
  1116. spin_lock(&swap_lock);
  1117. for (prev = -1, i = swap_list.head; i >= 0; prev = i, i = swap_info[i].next)
  1118. if (p->prio >= swap_info[i].prio)
  1119. break;
  1120. p->next = i;
  1121. if (prev < 0)
  1122. swap_list.head = swap_list.next = p - swap_info;
  1123. else
  1124. swap_info[prev].next = p - swap_info;
  1125. nr_swap_pages += p->pages;
  1126. total_swap_pages += p->pages;
  1127. p->flags |= SWP_WRITEOK;
  1128. spin_unlock(&swap_lock);
  1129. goto out_dput;
  1130. }
  1131. /* wait for any unplug function to finish */
  1132. down_write(&swap_unplug_sem);
  1133. up_write(&swap_unplug_sem);
  1134. destroy_swap_extents(p);
  1135. mutex_lock(&swapon_mutex);
  1136. spin_lock(&swap_lock);
  1137. drain_mmlist();
  1138. /* wait for anyone still in scan_swap_map */
  1139. p->highest_bit = 0; /* cuts scans short */
  1140. while (p->flags >= SWP_SCANNING) {
  1141. spin_unlock(&swap_lock);
  1142. schedule_timeout_uninterruptible(1);
  1143. spin_lock(&swap_lock);
  1144. }
  1145. swap_file = p->swap_file;
  1146. p->swap_file = NULL;
  1147. p->max = 0;
  1148. swap_map = p->swap_map;
  1149. p->swap_map = NULL;
  1150. p->flags = 0;
  1151. spin_unlock(&swap_lock);
  1152. mutex_unlock(&swapon_mutex);
  1153. vfree(swap_map);
  1154. inode = mapping->host;
  1155. if (S_ISBLK(inode->i_mode)) {
  1156. struct block_device *bdev = I_BDEV(inode);
  1157. set_blocksize(bdev, p->old_block_size);
  1158. bd_release(bdev);
  1159. } else {
  1160. mutex_lock(&inode->i_mutex);
  1161. inode->i_flags &= ~S_SWAPFILE;
  1162. mutex_unlock(&inode->i_mutex);
  1163. }
  1164. filp_close(swap_file, NULL);
  1165. err = 0;
  1166. out_dput:
  1167. filp_close(victim, NULL);
  1168. out:
  1169. return err;
  1170. }
  1171. #ifdef CONFIG_PROC_FS
  1172. /* iterator */
  1173. static void *swap_start(struct seq_file *swap, loff_t *pos)
  1174. {
  1175. struct swap_info_struct *ptr = swap_info;
  1176. int i;
  1177. loff_t l = *pos;
  1178. mutex_lock(&swapon_mutex);
  1179. if (!l)
  1180. return SEQ_START_TOKEN;
  1181. for (i = 0; i < nr_swapfiles; i++, ptr++) {
  1182. if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
  1183. continue;
  1184. if (!--l)
  1185. return ptr;
  1186. }
  1187. return NULL;
  1188. }
  1189. static void *swap_next(struct seq_file *swap, void *v, loff_t *pos)
  1190. {
  1191. struct swap_info_struct *ptr;
  1192. struct swap_info_struct *endptr = swap_info + nr_swapfiles;
  1193. if (v == SEQ_START_TOKEN)
  1194. ptr = swap_info;
  1195. else {
  1196. ptr = v;
  1197. ptr++;
  1198. }
  1199. for (; ptr < endptr; ptr++) {
  1200. if (!(ptr->flags & SWP_USED) || !ptr->swap_map)
  1201. continue;
  1202. ++*pos;
  1203. return ptr;
  1204. }
  1205. return NULL;
  1206. }
  1207. static void swap_stop(struct seq_file *swap, void *v)
  1208. {
  1209. mutex_unlock(&swapon_mutex);
  1210. }
  1211. static int swap_show(struct seq_file *swap, void *v)
  1212. {
  1213. struct swap_info_struct *ptr = v;
  1214. struct file *file;
  1215. int len;
  1216. if (ptr == SEQ_START_TOKEN) {
  1217. seq_puts(swap,"Filename\t\t\t\tType\t\tSize\tUsed\tPriority\n");
  1218. return 0;
  1219. }
  1220. file = ptr->swap_file;
  1221. len = seq_path(swap, file->f_path.mnt, file->f_path.dentry, " \t\n\\");
  1222. seq_printf(swap, "%*s%s\t%u\t%u\t%d\n",
  1223. len < 40 ? 40 - len : 1, " ",
  1224. S_ISBLK(file->f_path.dentry->d_inode->i_mode) ?
  1225. "partition" : "file\t",
  1226. ptr->pages << (PAGE_SHIFT - 10),
  1227. ptr->inuse_pages << (PAGE_SHIFT - 10),
  1228. ptr->prio);
  1229. return 0;
  1230. }
  1231. static const struct seq_operations swaps_op = {
  1232. .start = swap_start,
  1233. .next = swap_next,
  1234. .stop = swap_stop,
  1235. .show = swap_show
  1236. };
  1237. static int swaps_open(struct inode *inode, struct file *file)
  1238. {
  1239. return seq_open(file, &swaps_op);
  1240. }
  1241. static const struct file_operations proc_swaps_operations = {
  1242. .open = swaps_open,
  1243. .read = seq_read,
  1244. .llseek = seq_lseek,
  1245. .release = seq_release,
  1246. };
  1247. static int __init procswaps_init(void)
  1248. {
  1249. struct proc_dir_entry *entry;
  1250. entry = create_proc_entry("swaps", 0, NULL);
  1251. if (entry)
  1252. entry->proc_fops = &proc_swaps_operations;
  1253. return 0;
  1254. }
  1255. __initcall(procswaps_init);
  1256. #endif /* CONFIG_PROC_FS */
  1257. /*
  1258. * Written 01/25/92 by Simmule Turner, heavily changed by Linus.
  1259. *
  1260. * The swapon system call
  1261. */
  1262. asmlinkage long sys_swapon(const char __user * specialfile, int swap_flags)
  1263. {
  1264. struct swap_info_struct * p;
  1265. char *name = NULL;
  1266. struct block_device *bdev = NULL;
  1267. struct file *swap_file = NULL;
  1268. struct address_space *mapping;
  1269. unsigned int type;
  1270. int i, prev;
  1271. int error;
  1272. static int least_priority;
  1273. union swap_header *swap_header = NULL;
  1274. int swap_header_version;
  1275. unsigned int nr_good_pages = 0;
  1276. int nr_extents = 0;
  1277. sector_t span;
  1278. unsigned long maxpages = 1;
  1279. int swapfilesize;
  1280. unsigned short *swap_map;
  1281. struct page *page = NULL;
  1282. struct inode *inode = NULL;
  1283. int did_down = 0;
  1284. if (!capable(CAP_SYS_ADMIN))
  1285. return -EPERM;
  1286. spin_lock(&swap_lock);
  1287. p = swap_info;
  1288. for (type = 0 ; type < nr_swapfiles ; type++,p++)
  1289. if (!(p->flags & SWP_USED))
  1290. break;
  1291. error = -EPERM;
  1292. if (type >= MAX_SWAPFILES) {
  1293. spin_unlock(&swap_lock);
  1294. goto out;
  1295. }
  1296. if (type >= nr_swapfiles)
  1297. nr_swapfiles = type+1;
  1298. INIT_LIST_HEAD(&p->extent_list);
  1299. p->flags = SWP_USED;
  1300. p->swap_file = NULL;
  1301. p->old_block_size = 0;
  1302. p->swap_map = NULL;
  1303. p->lowest_bit = 0;
  1304. p->highest_bit = 0;
  1305. p->cluster_nr = 0;
  1306. p->inuse_pages = 0;
  1307. p->next = -1;
  1308. if (swap_flags & SWAP_FLAG_PREFER) {
  1309. p->prio =
  1310. (swap_flags & SWAP_FLAG_PRIO_MASK)>>SWAP_FLAG_PRIO_SHIFT;
  1311. } else {
  1312. p->prio = --least_priority;
  1313. }
  1314. spin_unlock(&swap_lock);
  1315. name = getname(specialfile);
  1316. error = PTR_ERR(name);
  1317. if (IS_ERR(name)) {
  1318. name = NULL;
  1319. goto bad_swap_2;
  1320. }
  1321. swap_file = filp_open(name, O_RDWR|O_LARGEFILE, 0);
  1322. error = PTR_ERR(swap_file);
  1323. if (IS_ERR(swap_file)) {
  1324. swap_file = NULL;
  1325. goto bad_swap_2;
  1326. }
  1327. p->swap_file = swap_file;
  1328. mapping = swap_file->f_mapping;
  1329. inode = mapping->host;
  1330. error = -EBUSY;
  1331. for (i = 0; i < nr_swapfiles; i++) {
  1332. struct swap_info_struct *q = &swap_info[i];
  1333. if (i == type || !q->swap_file)
  1334. continue;
  1335. if (mapping == q->swap_file->f_mapping)
  1336. goto bad_swap;
  1337. }
  1338. error = -EINVAL;
  1339. if (S_ISBLK(inode->i_mode)) {
  1340. bdev = I_BDEV(inode);
  1341. error = bd_claim(bdev, sys_swapon);
  1342. if (error < 0) {
  1343. bdev = NULL;
  1344. error = -EINVAL;
  1345. goto bad_swap;
  1346. }
  1347. p->old_block_size = block_size(bdev);
  1348. error = set_blocksize(bdev, PAGE_SIZE);
  1349. if (error < 0)
  1350. goto bad_swap;
  1351. p->bdev = bdev;
  1352. } else if (S_ISREG(inode->i_mode)) {
  1353. p->bdev = inode->i_sb->s_bdev;
  1354. mutex_lock(&inode->i_mutex);
  1355. did_down = 1;
  1356. if (IS_SWAPFILE(inode)) {
  1357. error = -EBUSY;
  1358. goto bad_swap;
  1359. }
  1360. } else {
  1361. goto bad_swap;
  1362. }
  1363. swapfilesize = i_size_read(inode) >> PAGE_SHIFT;
  1364. /*
  1365. * Read the swap header.
  1366. */
  1367. if (!mapping->a_ops->readpage) {
  1368. error = -EINVAL;
  1369. goto bad_swap;
  1370. }
  1371. page = read_mapping_page(mapping, 0, swap_file);
  1372. if (IS_ERR(page)) {
  1373. error = PTR_ERR(page);
  1374. goto bad_swap;
  1375. }
  1376. wait_on_page_locked(page);
  1377. if (!PageUptodate(page))
  1378. goto bad_swap;
  1379. kmap(page);
  1380. swap_header = page_address(page);
  1381. if (!memcmp("SWAP-SPACE",swap_header->magic.magic,10))
  1382. swap_header_version = 1;
  1383. else if (!memcmp("SWAPSPACE2",swap_header->magic.magic,10))
  1384. swap_header_version = 2;
  1385. else {
  1386. printk(KERN_ERR "Unable to find swap-space signature\n");
  1387. error = -EINVAL;
  1388. goto bad_swap;
  1389. }
  1390. switch (swap_header_version) {
  1391. case 1:
  1392. printk(KERN_ERR "version 0 swap is no longer supported. "
  1393. "Use mkswap -v1 %s\n", name);
  1394. error = -EINVAL;
  1395. goto bad_swap;
  1396. case 2:
  1397. /* Check the swap header's sub-version and the size of
  1398. the swap file and bad block lists */
  1399. if (swap_header->info.version != 1) {
  1400. printk(KERN_WARNING
  1401. "Unable to handle swap header version %d\n",
  1402. swap_header->info.version);
  1403. error = -EINVAL;
  1404. goto bad_swap;
  1405. }
  1406. p->lowest_bit = 1;
  1407. p->cluster_next = 1;
  1408. /*
  1409. * Find out how many pages are allowed for a single swap
  1410. * device. There are two limiting factors: 1) the number of
  1411. * bits for the swap offset in the swp_entry_t type and
  1412. * 2) the number of bits in the a swap pte as defined by
  1413. * the different architectures. In order to find the
  1414. * largest possible bit mask a swap entry with swap type 0
  1415. * and swap offset ~0UL is created, encoded to a swap pte,
  1416. * decoded to a swp_entry_t again and finally the swap
  1417. * offset is extracted. This will mask all the bits from
  1418. * the initial ~0UL mask that can't be encoded in either
  1419. * the swp_entry_t or the architecture definition of a
  1420. * swap pte.
  1421. */
  1422. maxpages = swp_offset(pte_to_swp_entry(swp_entry_to_pte(swp_entry(0,~0UL)))) - 1;
  1423. if (maxpages > swap_header->info.last_page)
  1424. maxpages = swap_header->info.last_page;
  1425. p->highest_bit = maxpages - 1;
  1426. error = -EINVAL;
  1427. if (!maxpages)
  1428. goto bad_swap;
  1429. if (swapfilesize && maxpages > swapfilesize) {
  1430. printk(KERN_WARNING
  1431. "Swap area shorter than signature indicates\n");
  1432. goto bad_swap;
  1433. }
  1434. if (swap_header->info.nr_badpages && S_ISREG(inode->i_mode))
  1435. goto bad_swap;
  1436. if (swap_header->info.nr_badpages > MAX_SWAP_BADPAGES)
  1437. goto bad_swap;
  1438. /* OK, set up the swap map and apply the bad block list */
  1439. if (!(p->swap_map = vmalloc(maxpages * sizeof(short)))) {
  1440. error = -ENOMEM;
  1441. goto bad_swap;
  1442. }
  1443. error = 0;
  1444. memset(p->swap_map, 0, maxpages * sizeof(short));
  1445. for (i = 0; i < swap_header->info.nr_badpages; i++) {
  1446. int page_nr = swap_header->info.badpages[i];
  1447. if (page_nr <= 0 || page_nr >= swap_header->info.last_page)
  1448. error = -EINVAL;
  1449. else
  1450. p->swap_map[page_nr] = SWAP_MAP_BAD;
  1451. }
  1452. nr_good_pages = swap_header->info.last_page -
  1453. swap_header->info.nr_badpages -
  1454. 1 /* header page */;
  1455. if (error)
  1456. goto bad_swap;
  1457. }
  1458. if (nr_good_pages) {
  1459. p->swap_map[0] = SWAP_MAP_BAD;
  1460. p->max = maxpages;
  1461. p->pages = nr_good_pages;
  1462. nr_extents = setup_swap_extents(p, &span);
  1463. if (nr_extents < 0) {
  1464. error = nr_extents;
  1465. goto bad_swap;
  1466. }
  1467. nr_good_pages = p->pages;
  1468. }
  1469. if (!nr_good_pages) {
  1470. printk(KERN_WARNING "Empty swap-file\n");
  1471. error = -EINVAL;
  1472. goto bad_swap;
  1473. }
  1474. mutex_lock(&swapon_mutex);
  1475. spin_lock(&swap_lock);
  1476. p->flags = SWP_ACTIVE;
  1477. nr_swap_pages += nr_good_pages;
  1478. total_swap_pages += nr_good_pages;
  1479. printk(KERN_INFO "Adding %uk swap on %s. "
  1480. "Priority:%d extents:%d across:%lluk\n",
  1481. nr_good_pages<<(PAGE_SHIFT-10), name, p->prio,
  1482. nr_extents, (unsigned long long)span<<(PAGE_SHIFT-10));
  1483. /* insert swap space into swap_list: */
  1484. prev = -1;
  1485. for (i = swap_list.head; i >= 0; i = swap_info[i].next) {
  1486. if (p->prio >= swap_info[i].prio) {
  1487. break;
  1488. }
  1489. prev = i;
  1490. }
  1491. p->next = i;
  1492. if (prev < 0) {
  1493. swap_list.head = swap_list.next = p - swap_info;
  1494. } else {
  1495. swap_info[prev].next = p - swap_info;
  1496. }
  1497. spin_unlock(&swap_lock);
  1498. mutex_unlock(&swapon_mutex);
  1499. error = 0;
  1500. goto out;
  1501. bad_swap:
  1502. if (bdev) {
  1503. set_blocksize(bdev, p->old_block_size);
  1504. bd_release(bdev);
  1505. }
  1506. destroy_swap_extents(p);
  1507. bad_swap_2:
  1508. spin_lock(&swap_lock);
  1509. swap_map = p->swap_map;
  1510. p->swap_file = NULL;
  1511. p->swap_map = NULL;
  1512. p->flags = 0;
  1513. if (!(swap_flags & SWAP_FLAG_PREFER))
  1514. ++least_priority;
  1515. spin_unlock(&swap_lock);
  1516. vfree(swap_map);
  1517. if (swap_file)
  1518. filp_close(swap_file, NULL);
  1519. out:
  1520. if (page && !IS_ERR(page)) {
  1521. kunmap(page);
  1522. page_cache_release(page);
  1523. }
  1524. if (name)
  1525. putname(name);
  1526. if (did_down) {
  1527. if (!error)
  1528. inode->i_flags |= S_SWAPFILE;
  1529. mutex_unlock(&inode->i_mutex);
  1530. }
  1531. return error;
  1532. }
  1533. void si_swapinfo(struct sysinfo *val)
  1534. {
  1535. unsigned int i;
  1536. unsigned long nr_to_be_unused = 0;
  1537. spin_lock(&swap_lock);
  1538. for (i = 0; i < nr_swapfiles; i++) {
  1539. if (!(swap_info[i].flags & SWP_USED) ||
  1540. (swap_info[i].flags & SWP_WRITEOK))
  1541. continue;
  1542. nr_to_be_unused += swap_info[i].inuse_pages;
  1543. }
  1544. val->freeswap = nr_swap_pages + nr_to_be_unused;
  1545. val->totalswap = total_swap_pages + nr_to_be_unused;
  1546. spin_unlock(&swap_lock);
  1547. }
  1548. /*
  1549. * Verify that a swap entry is valid and increment its swap map count.
  1550. *
  1551. * Note: if swap_map[] reaches SWAP_MAP_MAX the entries are treated as
  1552. * "permanent", but will be reclaimed by the next swapoff.
  1553. */
  1554. int swap_duplicate(swp_entry_t entry)
  1555. {
  1556. struct swap_info_struct * p;
  1557. unsigned long offset, type;
  1558. int result = 0;
  1559. if (is_migration_entry(entry))
  1560. return 1;
  1561. type = swp_type(entry);
  1562. if (type >= nr_swapfiles)
  1563. goto bad_file;
  1564. p = type + swap_info;
  1565. offset = swp_offset(entry);
  1566. spin_lock(&swap_lock);
  1567. if (offset < p->max && p->swap_map[offset]) {
  1568. if (p->swap_map[offset] < SWAP_MAP_MAX - 1) {
  1569. p->swap_map[offset]++;
  1570. result = 1;
  1571. } else if (p->swap_map[offset] <= SWAP_MAP_MAX) {
  1572. if (swap_overflow++ < 5)
  1573. printk(KERN_WARNING "swap_dup: swap entry overflow\n");
  1574. p->swap_map[offset] = SWAP_MAP_MAX;
  1575. result = 1;
  1576. }
  1577. }
  1578. spin_unlock(&swap_lock);
  1579. out:
  1580. return result;
  1581. bad_file:
  1582. printk(KERN_ERR "swap_dup: %s%08lx\n", Bad_file, entry.val);
  1583. goto out;
  1584. }
  1585. struct swap_info_struct *
  1586. get_swap_info_struct(unsigned type)
  1587. {
  1588. return &swap_info[type];
  1589. }
  1590. /*
  1591. * swap_lock prevents swap_map being freed. Don't grab an extra
  1592. * reference on the swaphandle, it doesn't matter if it becomes unused.
  1593. */
  1594. int valid_swaphandles(swp_entry_t entry, unsigned long *offset)
  1595. {
  1596. int our_page_cluster = page_cluster;
  1597. int ret = 0, i = 1 << our_page_cluster;
  1598. unsigned long toff;
  1599. struct swap_info_struct *swapdev = swp_type(entry) + swap_info;
  1600. if (!our_page_cluster) /* no readahead */
  1601. return 0;
  1602. toff = (swp_offset(entry) >> our_page_cluster) << our_page_cluster;
  1603. if (!toff) /* first page is swap header */
  1604. toff++, i--;
  1605. *offset = toff;
  1606. spin_lock(&swap_lock);
  1607. do {
  1608. /* Don't read-ahead past the end of the swap area */
  1609. if (toff >= swapdev->max)
  1610. break;
  1611. /* Don't read in free or bad pages */
  1612. if (!swapdev->swap_map[toff])
  1613. break;
  1614. if (swapdev->swap_map[toff] == SWAP_MAP_BAD)
  1615. break;
  1616. toff++;
  1617. ret++;
  1618. } while (--i);
  1619. spin_unlock(&swap_lock);
  1620. return ret;
  1621. }