migrate.c 22 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017
  1. /*
  2. * Memory Migration functionality - linux/mm/migration.c
  3. *
  4. * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
  5. *
  6. * Page migration was first developed in the context of the memory hotplug
  7. * project. The main authors of the migration code are:
  8. *
  9. * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  10. * Hirokazu Takahashi <taka@valinux.co.jp>
  11. * Dave Hansen <haveblue@us.ibm.com>
  12. * Christoph Lameter <clameter@sgi.com>
  13. */
  14. #include <linux/migrate.h>
  15. #include <linux/module.h>
  16. #include <linux/swap.h>
  17. #include <linux/swapops.h>
  18. #include <linux/pagemap.h>
  19. #include <linux/buffer_head.h>
  20. #include <linux/mm_inline.h>
  21. #include <linux/pagevec.h>
  22. #include <linux/rmap.h>
  23. #include <linux/topology.h>
  24. #include <linux/cpu.h>
  25. #include <linux/cpuset.h>
  26. #include <linux/writeback.h>
  27. #include <linux/mempolicy.h>
  28. #include <linux/vmalloc.h>
  29. #include <linux/security.h>
  30. #include "internal.h"
  31. #define lru_to_page(_head) (list_entry((_head)->prev, struct page, lru))
  32. /*
  33. * Isolate one page from the LRU lists. If successful put it onto
  34. * the indicated list with elevated page count.
  35. *
  36. * Result:
  37. * -EBUSY: page not on LRU list
  38. * 0: page removed from LRU list and added to the specified list.
  39. */
  40. int isolate_lru_page(struct page *page, struct list_head *pagelist)
  41. {
  42. int ret = -EBUSY;
  43. if (PageLRU(page)) {
  44. struct zone *zone = page_zone(page);
  45. spin_lock_irq(&zone->lru_lock);
  46. if (PageLRU(page)) {
  47. ret = 0;
  48. get_page(page);
  49. ClearPageLRU(page);
  50. if (PageActive(page))
  51. del_page_from_active_list(zone, page);
  52. else
  53. del_page_from_inactive_list(zone, page);
  54. list_add_tail(&page->lru, pagelist);
  55. }
  56. spin_unlock_irq(&zone->lru_lock);
  57. }
  58. return ret;
  59. }
  60. /*
  61. * migrate_prep() needs to be called before we start compiling a list of pages
  62. * to be migrated using isolate_lru_page().
  63. */
  64. int migrate_prep(void)
  65. {
  66. /*
  67. * Clear the LRU lists so pages can be isolated.
  68. * Note that pages may be moved off the LRU after we have
  69. * drained them. Those pages will fail to migrate like other
  70. * pages that may be busy.
  71. */
  72. lru_add_drain_all();
  73. return 0;
  74. }
  75. static inline void move_to_lru(struct page *page)
  76. {
  77. if (PageActive(page)) {
  78. /*
  79. * lru_cache_add_active checks that
  80. * the PG_active bit is off.
  81. */
  82. ClearPageActive(page);
  83. lru_cache_add_active(page);
  84. } else {
  85. lru_cache_add(page);
  86. }
  87. put_page(page);
  88. }
  89. /*
  90. * Add isolated pages on the list back to the LRU.
  91. *
  92. * returns the number of pages put back.
  93. */
  94. int putback_lru_pages(struct list_head *l)
  95. {
  96. struct page *page;
  97. struct page *page2;
  98. int count = 0;
  99. list_for_each_entry_safe(page, page2, l, lru) {
  100. list_del(&page->lru);
  101. move_to_lru(page);
  102. count++;
  103. }
  104. return count;
  105. }
  106. static inline int is_swap_pte(pte_t pte)
  107. {
  108. return !pte_none(pte) && !pte_present(pte) && !pte_file(pte);
  109. }
  110. /*
  111. * Restore a potential migration pte to a working pte entry
  112. */
  113. static void remove_migration_pte(struct vm_area_struct *vma,
  114. struct page *old, struct page *new)
  115. {
  116. struct mm_struct *mm = vma->vm_mm;
  117. swp_entry_t entry;
  118. pgd_t *pgd;
  119. pud_t *pud;
  120. pmd_t *pmd;
  121. pte_t *ptep, pte;
  122. spinlock_t *ptl;
  123. unsigned long addr = page_address_in_vma(new, vma);
  124. if (addr == -EFAULT)
  125. return;
  126. pgd = pgd_offset(mm, addr);
  127. if (!pgd_present(*pgd))
  128. return;
  129. pud = pud_offset(pgd, addr);
  130. if (!pud_present(*pud))
  131. return;
  132. pmd = pmd_offset(pud, addr);
  133. if (!pmd_present(*pmd))
  134. return;
  135. ptep = pte_offset_map(pmd, addr);
  136. if (!is_swap_pte(*ptep)) {
  137. pte_unmap(ptep);
  138. return;
  139. }
  140. ptl = pte_lockptr(mm, pmd);
  141. spin_lock(ptl);
  142. pte = *ptep;
  143. if (!is_swap_pte(pte))
  144. goto out;
  145. entry = pte_to_swp_entry(pte);
  146. if (!is_migration_entry(entry) || migration_entry_to_page(entry) != old)
  147. goto out;
  148. get_page(new);
  149. pte = pte_mkold(mk_pte(new, vma->vm_page_prot));
  150. if (is_write_migration_entry(entry))
  151. pte = pte_mkwrite(pte);
  152. set_pte_at(mm, addr, ptep, pte);
  153. if (PageAnon(new))
  154. page_add_anon_rmap(new, vma, addr);
  155. else
  156. page_add_file_rmap(new);
  157. /* No need to invalidate - it was non-present before */
  158. update_mmu_cache(vma, addr, pte);
  159. lazy_mmu_prot_update(pte);
  160. out:
  161. pte_unmap_unlock(ptep, ptl);
  162. }
  163. /*
  164. * Note that remove_file_migration_ptes will only work on regular mappings,
  165. * Nonlinear mappings do not use migration entries.
  166. */
  167. static void remove_file_migration_ptes(struct page *old, struct page *new)
  168. {
  169. struct vm_area_struct *vma;
  170. struct address_space *mapping = page_mapping(new);
  171. struct prio_tree_iter iter;
  172. pgoff_t pgoff = new->index << (PAGE_CACHE_SHIFT - PAGE_SHIFT);
  173. if (!mapping)
  174. return;
  175. spin_lock(&mapping->i_mmap_lock);
  176. vma_prio_tree_foreach(vma, &iter, &mapping->i_mmap, pgoff, pgoff)
  177. remove_migration_pte(vma, old, new);
  178. spin_unlock(&mapping->i_mmap_lock);
  179. }
  180. /*
  181. * Must hold mmap_sem lock on at least one of the vmas containing
  182. * the page so that the anon_vma cannot vanish.
  183. */
  184. static void remove_anon_migration_ptes(struct page *old, struct page *new)
  185. {
  186. struct anon_vma *anon_vma;
  187. struct vm_area_struct *vma;
  188. unsigned long mapping;
  189. mapping = (unsigned long)new->mapping;
  190. if (!mapping || (mapping & PAGE_MAPPING_ANON) == 0)
  191. return;
  192. /*
  193. * We hold the mmap_sem lock. So no need to call page_lock_anon_vma.
  194. */
  195. anon_vma = (struct anon_vma *) (mapping - PAGE_MAPPING_ANON);
  196. spin_lock(&anon_vma->lock);
  197. list_for_each_entry(vma, &anon_vma->head, anon_vma_node)
  198. remove_migration_pte(vma, old, new);
  199. spin_unlock(&anon_vma->lock);
  200. }
  201. /*
  202. * Get rid of all migration entries and replace them by
  203. * references to the indicated page.
  204. */
  205. static void remove_migration_ptes(struct page *old, struct page *new)
  206. {
  207. if (PageAnon(new))
  208. remove_anon_migration_ptes(old, new);
  209. else
  210. remove_file_migration_ptes(old, new);
  211. }
  212. /*
  213. * Something used the pte of a page under migration. We need to
  214. * get to the page and wait until migration is finished.
  215. * When we return from this function the fault will be retried.
  216. *
  217. * This function is called from do_swap_page().
  218. */
  219. void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
  220. unsigned long address)
  221. {
  222. pte_t *ptep, pte;
  223. spinlock_t *ptl;
  224. swp_entry_t entry;
  225. struct page *page;
  226. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  227. pte = *ptep;
  228. if (!is_swap_pte(pte))
  229. goto out;
  230. entry = pte_to_swp_entry(pte);
  231. if (!is_migration_entry(entry))
  232. goto out;
  233. page = migration_entry_to_page(entry);
  234. get_page(page);
  235. pte_unmap_unlock(ptep, ptl);
  236. wait_on_page_locked(page);
  237. put_page(page);
  238. return;
  239. out:
  240. pte_unmap_unlock(ptep, ptl);
  241. }
  242. /*
  243. * Replace the page in the mapping.
  244. *
  245. * The number of remaining references must be:
  246. * 1 for anonymous pages without a mapping
  247. * 2 for pages with a mapping
  248. * 3 for pages with a mapping and PagePrivate set.
  249. */
  250. static int migrate_page_move_mapping(struct address_space *mapping,
  251. struct page *newpage, struct page *page)
  252. {
  253. void **pslot;
  254. if (!mapping) {
  255. /* Anonymous page without mapping */
  256. if (page_count(page) != 1)
  257. return -EAGAIN;
  258. return 0;
  259. }
  260. write_lock_irq(&mapping->tree_lock);
  261. pslot = radix_tree_lookup_slot(&mapping->page_tree,
  262. page_index(page));
  263. if (page_count(page) != 2 + !!PagePrivate(page) ||
  264. (struct page *)radix_tree_deref_slot(pslot) != page) {
  265. write_unlock_irq(&mapping->tree_lock);
  266. return -EAGAIN;
  267. }
  268. /*
  269. * Now we know that no one else is looking at the page.
  270. */
  271. get_page(newpage); /* add cache reference */
  272. #ifdef CONFIG_SWAP
  273. if (PageSwapCache(page)) {
  274. SetPageSwapCache(newpage);
  275. set_page_private(newpage, page_private(page));
  276. }
  277. #endif
  278. radix_tree_replace_slot(pslot, newpage);
  279. /*
  280. * Drop cache reference from old page.
  281. * We know this isn't the last reference.
  282. */
  283. __put_page(page);
  284. /*
  285. * If moved to a different zone then also account
  286. * the page for that zone. Other VM counters will be
  287. * taken care of when we establish references to the
  288. * new page and drop references to the old page.
  289. *
  290. * Note that anonymous pages are accounted for
  291. * via NR_FILE_PAGES and NR_ANON_PAGES if they
  292. * are mapped to swap space.
  293. */
  294. __dec_zone_page_state(page, NR_FILE_PAGES);
  295. __inc_zone_page_state(newpage, NR_FILE_PAGES);
  296. write_unlock_irq(&mapping->tree_lock);
  297. return 0;
  298. }
  299. /*
  300. * Copy the page to its new location
  301. */
  302. static void migrate_page_copy(struct page *newpage, struct page *page)
  303. {
  304. copy_highpage(newpage, page);
  305. if (PageError(page))
  306. SetPageError(newpage);
  307. if (PageReferenced(page))
  308. SetPageReferenced(newpage);
  309. if (PageUptodate(page))
  310. SetPageUptodate(newpage);
  311. if (PageActive(page))
  312. SetPageActive(newpage);
  313. if (PageChecked(page))
  314. SetPageChecked(newpage);
  315. if (PageMappedToDisk(page))
  316. SetPageMappedToDisk(newpage);
  317. if (PageDirty(page)) {
  318. clear_page_dirty_for_io(page);
  319. set_page_dirty(newpage);
  320. }
  321. #ifdef CONFIG_SWAP
  322. ClearPageSwapCache(page);
  323. #endif
  324. ClearPageActive(page);
  325. ClearPagePrivate(page);
  326. set_page_private(page, 0);
  327. page->mapping = NULL;
  328. /*
  329. * If any waiters have accumulated on the new page then
  330. * wake them up.
  331. */
  332. if (PageWriteback(newpage))
  333. end_page_writeback(newpage);
  334. }
  335. /************************************************************
  336. * Migration functions
  337. ***********************************************************/
  338. /* Always fail migration. Used for mappings that are not movable */
  339. int fail_migrate_page(struct address_space *mapping,
  340. struct page *newpage, struct page *page)
  341. {
  342. return -EIO;
  343. }
  344. EXPORT_SYMBOL(fail_migrate_page);
  345. /*
  346. * Common logic to directly migrate a single page suitable for
  347. * pages that do not use PagePrivate.
  348. *
  349. * Pages are locked upon entry and exit.
  350. */
  351. int migrate_page(struct address_space *mapping,
  352. struct page *newpage, struct page *page)
  353. {
  354. int rc;
  355. BUG_ON(PageWriteback(page)); /* Writeback must be complete */
  356. rc = migrate_page_move_mapping(mapping, newpage, page);
  357. if (rc)
  358. return rc;
  359. migrate_page_copy(newpage, page);
  360. return 0;
  361. }
  362. EXPORT_SYMBOL(migrate_page);
  363. #ifdef CONFIG_BLOCK
  364. /*
  365. * Migration function for pages with buffers. This function can only be used
  366. * if the underlying filesystem guarantees that no other references to "page"
  367. * exist.
  368. */
  369. int buffer_migrate_page(struct address_space *mapping,
  370. struct page *newpage, struct page *page)
  371. {
  372. struct buffer_head *bh, *head;
  373. int rc;
  374. if (!page_has_buffers(page))
  375. return migrate_page(mapping, newpage, page);
  376. head = page_buffers(page);
  377. rc = migrate_page_move_mapping(mapping, newpage, page);
  378. if (rc)
  379. return rc;
  380. bh = head;
  381. do {
  382. get_bh(bh);
  383. lock_buffer(bh);
  384. bh = bh->b_this_page;
  385. } while (bh != head);
  386. ClearPagePrivate(page);
  387. set_page_private(newpage, page_private(page));
  388. set_page_private(page, 0);
  389. put_page(page);
  390. get_page(newpage);
  391. bh = head;
  392. do {
  393. set_bh_page(bh, newpage, bh_offset(bh));
  394. bh = bh->b_this_page;
  395. } while (bh != head);
  396. SetPagePrivate(newpage);
  397. migrate_page_copy(newpage, page);
  398. bh = head;
  399. do {
  400. unlock_buffer(bh);
  401. put_bh(bh);
  402. bh = bh->b_this_page;
  403. } while (bh != head);
  404. return 0;
  405. }
  406. EXPORT_SYMBOL(buffer_migrate_page);
  407. #endif
  408. /*
  409. * Writeback a page to clean the dirty state
  410. */
  411. static int writeout(struct address_space *mapping, struct page *page)
  412. {
  413. struct writeback_control wbc = {
  414. .sync_mode = WB_SYNC_NONE,
  415. .nr_to_write = 1,
  416. .range_start = 0,
  417. .range_end = LLONG_MAX,
  418. .nonblocking = 1,
  419. .for_reclaim = 1
  420. };
  421. int rc;
  422. if (!mapping->a_ops->writepage)
  423. /* No write method for the address space */
  424. return -EINVAL;
  425. if (!clear_page_dirty_for_io(page))
  426. /* Someone else already triggered a write */
  427. return -EAGAIN;
  428. /*
  429. * A dirty page may imply that the underlying filesystem has
  430. * the page on some queue. So the page must be clean for
  431. * migration. Writeout may mean we loose the lock and the
  432. * page state is no longer what we checked for earlier.
  433. * At this point we know that the migration attempt cannot
  434. * be successful.
  435. */
  436. remove_migration_ptes(page, page);
  437. rc = mapping->a_ops->writepage(page, &wbc);
  438. if (rc < 0)
  439. /* I/O Error writing */
  440. return -EIO;
  441. if (rc != AOP_WRITEPAGE_ACTIVATE)
  442. /* unlocked. Relock */
  443. lock_page(page);
  444. return -EAGAIN;
  445. }
  446. /*
  447. * Default handling if a filesystem does not provide a migration function.
  448. */
  449. static int fallback_migrate_page(struct address_space *mapping,
  450. struct page *newpage, struct page *page)
  451. {
  452. if (PageDirty(page))
  453. return writeout(mapping, page);
  454. /*
  455. * Buffers may be managed in a filesystem specific way.
  456. * We must have no buffers or drop them.
  457. */
  458. if (PagePrivate(page) &&
  459. !try_to_release_page(page, GFP_KERNEL))
  460. return -EAGAIN;
  461. return migrate_page(mapping, newpage, page);
  462. }
  463. /*
  464. * Move a page to a newly allocated page
  465. * The page is locked and all ptes have been successfully removed.
  466. *
  467. * The new page will have replaced the old page if this function
  468. * is successful.
  469. */
  470. static int move_to_new_page(struct page *newpage, struct page *page)
  471. {
  472. struct address_space *mapping;
  473. int rc;
  474. /*
  475. * Block others from accessing the page when we get around to
  476. * establishing additional references. We are the only one
  477. * holding a reference to the new page at this point.
  478. */
  479. if (TestSetPageLocked(newpage))
  480. BUG();
  481. /* Prepare mapping for the new page.*/
  482. newpage->index = page->index;
  483. newpage->mapping = page->mapping;
  484. mapping = page_mapping(page);
  485. if (!mapping)
  486. rc = migrate_page(mapping, newpage, page);
  487. else if (mapping->a_ops->migratepage)
  488. /*
  489. * Most pages have a mapping and most filesystems
  490. * should provide a migration function. Anonymous
  491. * pages are part of swap space which also has its
  492. * own migration function. This is the most common
  493. * path for page migration.
  494. */
  495. rc = mapping->a_ops->migratepage(mapping,
  496. newpage, page);
  497. else
  498. rc = fallback_migrate_page(mapping, newpage, page);
  499. if (!rc)
  500. remove_migration_ptes(page, newpage);
  501. else
  502. newpage->mapping = NULL;
  503. unlock_page(newpage);
  504. return rc;
  505. }
  506. /*
  507. * Obtain the lock on page, remove all ptes and migrate the page
  508. * to the newly allocated page in newpage.
  509. */
  510. static int unmap_and_move(new_page_t get_new_page, unsigned long private,
  511. struct page *page, int force)
  512. {
  513. int rc = 0;
  514. int *result = NULL;
  515. struct page *newpage = get_new_page(page, private, &result);
  516. if (!newpage)
  517. return -ENOMEM;
  518. if (page_count(page) == 1)
  519. /* page was freed from under us. So we are done. */
  520. goto move_newpage;
  521. rc = -EAGAIN;
  522. if (TestSetPageLocked(page)) {
  523. if (!force)
  524. goto move_newpage;
  525. lock_page(page);
  526. }
  527. if (PageWriteback(page)) {
  528. if (!force)
  529. goto unlock;
  530. wait_on_page_writeback(page);
  531. }
  532. /*
  533. * Establish migration ptes or remove ptes
  534. */
  535. try_to_unmap(page, 1);
  536. if (!page_mapped(page))
  537. rc = move_to_new_page(newpage, page);
  538. if (rc)
  539. remove_migration_ptes(page, page);
  540. unlock:
  541. unlock_page(page);
  542. if (rc != -EAGAIN) {
  543. /*
  544. * A page that has been migrated has all references
  545. * removed and will be freed. A page that has not been
  546. * migrated will have kepts its references and be
  547. * restored.
  548. */
  549. list_del(&page->lru);
  550. move_to_lru(page);
  551. }
  552. move_newpage:
  553. /*
  554. * Move the new page to the LRU. If migration was not successful
  555. * then this will free the page.
  556. */
  557. move_to_lru(newpage);
  558. if (result) {
  559. if (rc)
  560. *result = rc;
  561. else
  562. *result = page_to_nid(newpage);
  563. }
  564. return rc;
  565. }
  566. /*
  567. * migrate_pages
  568. *
  569. * The function takes one list of pages to migrate and a function
  570. * that determines from the page to be migrated and the private data
  571. * the target of the move and allocates the page.
  572. *
  573. * The function returns after 10 attempts or if no pages
  574. * are movable anymore because to has become empty
  575. * or no retryable pages exist anymore. All pages will be
  576. * retruned to the LRU or freed.
  577. *
  578. * Return: Number of pages not migrated or error code.
  579. */
  580. int migrate_pages(struct list_head *from,
  581. new_page_t get_new_page, unsigned long private)
  582. {
  583. int retry = 1;
  584. int nr_failed = 0;
  585. int pass = 0;
  586. struct page *page;
  587. struct page *page2;
  588. int swapwrite = current->flags & PF_SWAPWRITE;
  589. int rc;
  590. if (!swapwrite)
  591. current->flags |= PF_SWAPWRITE;
  592. for(pass = 0; pass < 10 && retry; pass++) {
  593. retry = 0;
  594. list_for_each_entry_safe(page, page2, from, lru) {
  595. cond_resched();
  596. rc = unmap_and_move(get_new_page, private,
  597. page, pass > 2);
  598. switch(rc) {
  599. case -ENOMEM:
  600. goto out;
  601. case -EAGAIN:
  602. retry++;
  603. break;
  604. case 0:
  605. break;
  606. default:
  607. /* Permanent failure */
  608. nr_failed++;
  609. break;
  610. }
  611. }
  612. }
  613. rc = 0;
  614. out:
  615. if (!swapwrite)
  616. current->flags &= ~PF_SWAPWRITE;
  617. putback_lru_pages(from);
  618. if (rc)
  619. return rc;
  620. return nr_failed + retry;
  621. }
  622. #ifdef CONFIG_NUMA
  623. /*
  624. * Move a list of individual pages
  625. */
  626. struct page_to_node {
  627. unsigned long addr;
  628. struct page *page;
  629. int node;
  630. int status;
  631. };
  632. static struct page *new_page_node(struct page *p, unsigned long private,
  633. int **result)
  634. {
  635. struct page_to_node *pm = (struct page_to_node *)private;
  636. while (pm->node != MAX_NUMNODES && pm->page != p)
  637. pm++;
  638. if (pm->node == MAX_NUMNODES)
  639. return NULL;
  640. *result = &pm->status;
  641. return alloc_pages_node(pm->node, GFP_HIGHUSER | GFP_THISNODE, 0);
  642. }
  643. /*
  644. * Move a set of pages as indicated in the pm array. The addr
  645. * field must be set to the virtual address of the page to be moved
  646. * and the node number must contain a valid target node.
  647. */
  648. static int do_move_pages(struct mm_struct *mm, struct page_to_node *pm,
  649. int migrate_all)
  650. {
  651. int err;
  652. struct page_to_node *pp;
  653. LIST_HEAD(pagelist);
  654. down_read(&mm->mmap_sem);
  655. /*
  656. * Build a list of pages to migrate
  657. */
  658. migrate_prep();
  659. for (pp = pm; pp->node != MAX_NUMNODES; pp++) {
  660. struct vm_area_struct *vma;
  661. struct page *page;
  662. /*
  663. * A valid page pointer that will not match any of the
  664. * pages that will be moved.
  665. */
  666. pp->page = ZERO_PAGE(0);
  667. err = -EFAULT;
  668. vma = find_vma(mm, pp->addr);
  669. if (!vma || !vma_migratable(vma))
  670. goto set_status;
  671. page = follow_page(vma, pp->addr, FOLL_GET);
  672. err = -ENOENT;
  673. if (!page)
  674. goto set_status;
  675. if (PageReserved(page)) /* Check for zero page */
  676. goto put_and_set;
  677. pp->page = page;
  678. err = page_to_nid(page);
  679. if (err == pp->node)
  680. /*
  681. * Node already in the right place
  682. */
  683. goto put_and_set;
  684. err = -EACCES;
  685. if (page_mapcount(page) > 1 &&
  686. !migrate_all)
  687. goto put_and_set;
  688. err = isolate_lru_page(page, &pagelist);
  689. put_and_set:
  690. /*
  691. * Either remove the duplicate refcount from
  692. * isolate_lru_page() or drop the page ref if it was
  693. * not isolated.
  694. */
  695. put_page(page);
  696. set_status:
  697. pp->status = err;
  698. }
  699. if (!list_empty(&pagelist))
  700. err = migrate_pages(&pagelist, new_page_node,
  701. (unsigned long)pm);
  702. else
  703. err = -ENOENT;
  704. up_read(&mm->mmap_sem);
  705. return err;
  706. }
  707. /*
  708. * Determine the nodes of a list of pages. The addr in the pm array
  709. * must have been set to the virtual address of which we want to determine
  710. * the node number.
  711. */
  712. static int do_pages_stat(struct mm_struct *mm, struct page_to_node *pm)
  713. {
  714. down_read(&mm->mmap_sem);
  715. for ( ; pm->node != MAX_NUMNODES; pm++) {
  716. struct vm_area_struct *vma;
  717. struct page *page;
  718. int err;
  719. err = -EFAULT;
  720. vma = find_vma(mm, pm->addr);
  721. if (!vma)
  722. goto set_status;
  723. page = follow_page(vma, pm->addr, 0);
  724. err = -ENOENT;
  725. /* Use PageReserved to check for zero page */
  726. if (!page || PageReserved(page))
  727. goto set_status;
  728. err = page_to_nid(page);
  729. set_status:
  730. pm->status = err;
  731. }
  732. up_read(&mm->mmap_sem);
  733. return 0;
  734. }
  735. /*
  736. * Move a list of pages in the address space of the currently executing
  737. * process.
  738. */
  739. asmlinkage long sys_move_pages(pid_t pid, unsigned long nr_pages,
  740. const void __user * __user *pages,
  741. const int __user *nodes,
  742. int __user *status, int flags)
  743. {
  744. int err = 0;
  745. int i;
  746. struct task_struct *task;
  747. nodemask_t task_nodes;
  748. struct mm_struct *mm;
  749. struct page_to_node *pm = NULL;
  750. /* Check flags */
  751. if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
  752. return -EINVAL;
  753. if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
  754. return -EPERM;
  755. /* Find the mm_struct */
  756. read_lock(&tasklist_lock);
  757. task = pid ? find_task_by_pid(pid) : current;
  758. if (!task) {
  759. read_unlock(&tasklist_lock);
  760. return -ESRCH;
  761. }
  762. mm = get_task_mm(task);
  763. read_unlock(&tasklist_lock);
  764. if (!mm)
  765. return -EINVAL;
  766. /*
  767. * Check if this process has the right to modify the specified
  768. * process. The right exists if the process has administrative
  769. * capabilities, superuser privileges or the same
  770. * userid as the target process.
  771. */
  772. if ((current->euid != task->suid) && (current->euid != task->uid) &&
  773. (current->uid != task->suid) && (current->uid != task->uid) &&
  774. !capable(CAP_SYS_NICE)) {
  775. err = -EPERM;
  776. goto out2;
  777. }
  778. err = security_task_movememory(task);
  779. if (err)
  780. goto out2;
  781. task_nodes = cpuset_mems_allowed(task);
  782. /* Limit nr_pages so that the multiplication may not overflow */
  783. if (nr_pages >= ULONG_MAX / sizeof(struct page_to_node) - 1) {
  784. err = -E2BIG;
  785. goto out2;
  786. }
  787. pm = vmalloc((nr_pages + 1) * sizeof(struct page_to_node));
  788. if (!pm) {
  789. err = -ENOMEM;
  790. goto out2;
  791. }
  792. /*
  793. * Get parameters from user space and initialize the pm
  794. * array. Return various errors if the user did something wrong.
  795. */
  796. for (i = 0; i < nr_pages; i++) {
  797. const void *p;
  798. err = -EFAULT;
  799. if (get_user(p, pages + i))
  800. goto out;
  801. pm[i].addr = (unsigned long)p;
  802. if (nodes) {
  803. int node;
  804. if (get_user(node, nodes + i))
  805. goto out;
  806. err = -ENODEV;
  807. if (!node_online(node))
  808. goto out;
  809. err = -EACCES;
  810. if (!node_isset(node, task_nodes))
  811. goto out;
  812. pm[i].node = node;
  813. } else
  814. pm[i].node = 0; /* anything to not match MAX_NUMNODES */
  815. }
  816. /* End marker */
  817. pm[nr_pages].node = MAX_NUMNODES;
  818. if (nodes)
  819. err = do_move_pages(mm, pm, flags & MPOL_MF_MOVE_ALL);
  820. else
  821. err = do_pages_stat(mm, pm);
  822. if (err >= 0)
  823. /* Return status information */
  824. for (i = 0; i < nr_pages; i++)
  825. if (put_user(pm[i].status, status + i))
  826. err = -EFAULT;
  827. out:
  828. vfree(pm);
  829. out2:
  830. mmput(mm);
  831. return err;
  832. }
  833. #endif
  834. /*
  835. * Call migration functions in the vma_ops that may prepare
  836. * memory in a vm for migration. migration functions may perform
  837. * the migration for vmas that do not have an underlying page struct.
  838. */
  839. int migrate_vmas(struct mm_struct *mm, const nodemask_t *to,
  840. const nodemask_t *from, unsigned long flags)
  841. {
  842. struct vm_area_struct *vma;
  843. int err = 0;
  844. for(vma = mm->mmap; vma->vm_next && !err; vma = vma->vm_next) {
  845. if (vma->vm_ops && vma->vm_ops->migrate) {
  846. err = vma->vm_ops->migrate(vma, to, from, flags);
  847. if (err)
  848. break;
  849. }
  850. }
  851. return err;
  852. }