migrate.c 81 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544254525462547254825492550255125522553255425552556255725582559256025612562256325642565256625672568256925702571257225732574257525762577257825792580258125822583258425852586258725882589259025912592259325942595259625972598259926002601260226032604260526062607260826092610261126122613261426152616261726182619262026212622262326242625262626272628262926302631263226332634263526362637263826392640264126422643264426452646264726482649265026512652265326542655265626572658265926602661266226632664266526662667266826692670267126722673267426752676267726782679268026812682268326842685268626872688268926902691269226932694269526962697269826992700270127022703270427052706270727082709271027112712271327142715271627172718271927202721272227232724272527262727272827292730273127322733273427352736273727382739274027412742274327442745274627472748274927502751275227532754275527562757275827592760276127622763276427652766276727682769277027712772277327742775277627772778277927802781278227832784278527862787278827892790279127922793279427952796279727982799280028012802280328042805280628072808280928102811281228132814281528162817281828192820282128222823282428252826282728282829283028312832283328342835283628372838283928402841284228432844284528462847284828492850285128522853285428552856285728582859286028612862286328642865286628672868286928702871287228732874287528762877287828792880288128822883288428852886288728882889289028912892289328942895289628972898289929002901290229032904290529062907290829092910291129122913291429152916291729182919292029212922292329242925292629272928292929302931293229332934293529362937293829392940294129422943294429452946294729482949295029512952295329542955295629572958295929602961296229632964296529662967296829692970297129722973297429752976297729782979298029812982298329842985298629872988298929902991299229932994299529962997299829993000300130023003300430053006300730083009301030113012301330143015301630173018301930203021302230233024302530263027302830293030303130323033303430353036303730383039304030413042304330443045304630473048304930503051305230533054305530563057305830593060306130623063306430653066306730683069307030713072307330743075307630773078307930803081308230833084308530863087308830893090309130923093309430953096
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Memory Migration functionality - linux/mm/migrate.c
  4. *
  5. * Copyright (C) 2006 Silicon Graphics, Inc., Christoph Lameter
  6. *
  7. * Page migration was first developed in the context of the memory hotplug
  8. * project. The main authors of the migration code are:
  9. *
  10. * IWAMOTO Toshihiro <iwamoto@valinux.co.jp>
  11. * Hirokazu Takahashi <taka@valinux.co.jp>
  12. * Dave Hansen <haveblue@us.ibm.com>
  13. * Christoph Lameter
  14. */
  15. #include <linux/migrate.h>
  16. #include <linux/export.h>
  17. #include <linux/swap.h>
  18. #include <linux/swapops.h>
  19. #include <linux/pagemap.h>
  20. #include <linux/buffer_head.h>
  21. #include <linux/mm_inline.h>
  22. #include <linux/nsproxy.h>
  23. #include <linux/pagevec.h>
  24. #include <linux/ksm.h>
  25. #include <linux/rmap.h>
  26. #include <linux/topology.h>
  27. #include <linux/cpu.h>
  28. #include <linux/cpuset.h>
  29. #include <linux/writeback.h>
  30. #include <linux/mempolicy.h>
  31. #include <linux/vmalloc.h>
  32. #include <linux/security.h>
  33. #include <linux/backing-dev.h>
  34. #include <linux/compaction.h>
  35. #include <linux/syscalls.h>
  36. #include <linux/compat.h>
  37. #include <linux/hugetlb.h>
  38. #include <linux/hugetlb_cgroup.h>
  39. #include <linux/gfp.h>
  40. #include <linux/pagewalk.h>
  41. #include <linux/pfn_t.h>
  42. #include <linux/memremap.h>
  43. #include <linux/userfaultfd_k.h>
  44. #include <linux/balloon_compaction.h>
  45. #include <linux/mmu_notifier.h>
  46. #include <linux/page_idle.h>
  47. #include <linux/page_owner.h>
  48. #include <linux/sched/mm.h>
  49. #include <linux/ptrace.h>
  50. #include <linux/oom.h>
  51. #include <asm/tlbflush.h>
  52. #define CREATE_TRACE_POINTS
  53. #include <trace/events/migrate.h>
  54. #include "internal.h"
  55. int isolate_movable_page(struct page *page, isolate_mode_t mode)
  56. {
  57. struct address_space *mapping;
  58. /*
  59. * Avoid burning cycles with pages that are yet under __free_pages(),
  60. * or just got freed under us.
  61. *
  62. * In case we 'win' a race for a movable page being freed under us and
  63. * raise its refcount preventing __free_pages() from doing its job
  64. * the put_page() at the end of this block will take care of
  65. * release this page, thus avoiding a nasty leakage.
  66. */
  67. if (unlikely(!get_page_unless_zero(page)))
  68. goto out;
  69. /*
  70. * Check PageMovable before holding a PG_lock because page's owner
  71. * assumes anybody doesn't touch PG_lock of newly allocated page
  72. * so unconditionally grabbing the lock ruins page's owner side.
  73. */
  74. if (unlikely(!__PageMovable(page)))
  75. goto out_putpage;
  76. /*
  77. * As movable pages are not isolated from LRU lists, concurrent
  78. * compaction threads can race against page migration functions
  79. * as well as race against the releasing a page.
  80. *
  81. * In order to avoid having an already isolated movable page
  82. * being (wrongly) re-isolated while it is under migration,
  83. * or to avoid attempting to isolate pages being released,
  84. * lets be sure we have the page lock
  85. * before proceeding with the movable page isolation steps.
  86. */
  87. if (unlikely(!trylock_page(page)))
  88. goto out_putpage;
  89. if (!PageMovable(page) || PageIsolated(page))
  90. goto out_no_isolated;
  91. mapping = page_mapping(page);
  92. VM_BUG_ON_PAGE(!mapping, page);
  93. if (!mapping->a_ops->isolate_page(page, mode))
  94. goto out_no_isolated;
  95. /* Driver shouldn't use PG_isolated bit of page->flags */
  96. WARN_ON_ONCE(PageIsolated(page));
  97. SetPageIsolated(page);
  98. unlock_page(page);
  99. return 0;
  100. out_no_isolated:
  101. unlock_page(page);
  102. out_putpage:
  103. put_page(page);
  104. out:
  105. return -EBUSY;
  106. }
  107. /* It should be called on page which is PG_movable */
  108. void putback_movable_page(struct page *page)
  109. {
  110. struct address_space *mapping;
  111. VM_BUG_ON_PAGE(!PageLocked(page), page);
  112. VM_BUG_ON_PAGE(!PageMovable(page), page);
  113. VM_BUG_ON_PAGE(!PageIsolated(page), page);
  114. mapping = page_mapping(page);
  115. mapping->a_ops->putback_page(page);
  116. ClearPageIsolated(page);
  117. }
  118. /*
  119. * Put previously isolated pages back onto the appropriate lists
  120. * from where they were once taken off for compaction/migration.
  121. *
  122. * This function shall be used whenever the isolated pageset has been
  123. * built from lru, balloon, hugetlbfs page. See isolate_migratepages_range()
  124. * and isolate_huge_page().
  125. */
  126. void putback_movable_pages(struct list_head *l)
  127. {
  128. struct page *page;
  129. struct page *page2;
  130. list_for_each_entry_safe(page, page2, l, lru) {
  131. if (unlikely(PageHuge(page))) {
  132. putback_active_hugepage(page);
  133. continue;
  134. }
  135. list_del(&page->lru);
  136. /*
  137. * We isolated non-lru movable page so here we can use
  138. * __PageMovable because LRU page's mapping cannot have
  139. * PAGE_MAPPING_MOVABLE.
  140. */
  141. if (unlikely(__PageMovable(page))) {
  142. VM_BUG_ON_PAGE(!PageIsolated(page), page);
  143. lock_page(page);
  144. if (PageMovable(page))
  145. putback_movable_page(page);
  146. else
  147. ClearPageIsolated(page);
  148. unlock_page(page);
  149. put_page(page);
  150. } else {
  151. mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
  152. page_is_file_lru(page), -thp_nr_pages(page));
  153. putback_lru_page(page);
  154. }
  155. }
  156. }
  157. EXPORT_SYMBOL_GPL(putback_movable_pages);
  158. /*
  159. * Restore a potential migration pte to a working pte entry
  160. */
  161. static bool remove_migration_pte(struct page *page, struct vm_area_struct *vma,
  162. unsigned long addr, void *old)
  163. {
  164. struct page_vma_mapped_walk pvmw = {
  165. .page = old,
  166. .vma = vma,
  167. .address = addr,
  168. .flags = PVMW_SYNC | PVMW_MIGRATION,
  169. };
  170. struct page *new;
  171. pte_t pte;
  172. swp_entry_t entry;
  173. VM_BUG_ON_PAGE(PageTail(page), page);
  174. while (page_vma_mapped_walk(&pvmw)) {
  175. if (PageKsm(page))
  176. new = page;
  177. else
  178. new = page - pvmw.page->index +
  179. linear_page_index(vma, pvmw.address);
  180. #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
  181. /* PMD-mapped THP migration entry */
  182. if (!pvmw.pte) {
  183. VM_BUG_ON_PAGE(PageHuge(page) || !PageTransCompound(page), page);
  184. remove_migration_pmd(&pvmw, new);
  185. continue;
  186. }
  187. #endif
  188. get_page(new);
  189. pte = pte_mkold(mk_pte(new, READ_ONCE(vma->vm_page_prot)));
  190. if (pte_swp_soft_dirty(*pvmw.pte))
  191. pte = pte_mksoft_dirty(pte);
  192. /*
  193. * Recheck VMA as permissions can change since migration started
  194. */
  195. entry = pte_to_swp_entry(*pvmw.pte);
  196. if (is_write_migration_entry(entry))
  197. pte = maybe_mkwrite(pte, vma->vm_flags);
  198. else if (pte_swp_uffd_wp(*pvmw.pte))
  199. pte = pte_mkuffd_wp(pte);
  200. if (unlikely(is_device_private_page(new))) {
  201. entry = make_device_private_entry(new, pte_write(pte));
  202. pte = swp_entry_to_pte(entry);
  203. if (pte_swp_soft_dirty(*pvmw.pte))
  204. pte = pte_swp_mksoft_dirty(pte);
  205. if (pte_swp_uffd_wp(*pvmw.pte))
  206. pte = pte_swp_mkuffd_wp(pte);
  207. }
  208. #ifdef CONFIG_HUGETLB_PAGE
  209. if (PageHuge(new)) {
  210. pte = pte_mkhuge(pte);
  211. pte = arch_make_huge_pte(pte, vma, new, 0);
  212. set_huge_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
  213. if (PageAnon(new))
  214. hugepage_add_anon_rmap(new, vma, pvmw.address);
  215. else
  216. page_dup_rmap(new, true);
  217. } else
  218. #endif
  219. {
  220. set_pte_at(vma->vm_mm, pvmw.address, pvmw.pte, pte);
  221. if (PageAnon(new))
  222. page_add_anon_rmap(new, vma, pvmw.address, false);
  223. else
  224. page_add_file_rmap(new, false);
  225. }
  226. if (vma->vm_flags & VM_LOCKED && !PageTransCompound(new))
  227. mlock_vma_page(new);
  228. if (PageTransHuge(page) && PageMlocked(page))
  229. clear_page_mlock(page);
  230. /* No need to invalidate - it was non-present before */
  231. update_mmu_cache(vma, pvmw.address, pvmw.pte);
  232. }
  233. return true;
  234. }
  235. /*
  236. * Get rid of all migration entries and replace them by
  237. * references to the indicated page.
  238. */
  239. void remove_migration_ptes(struct page *old, struct page *new, bool locked)
  240. {
  241. struct rmap_walk_control rwc = {
  242. .rmap_one = remove_migration_pte,
  243. .arg = old,
  244. };
  245. if (locked)
  246. rmap_walk_locked(new, &rwc);
  247. else
  248. rmap_walk(new, &rwc);
  249. }
  250. /*
  251. * Something used the pte of a page under migration. We need to
  252. * get to the page and wait until migration is finished.
  253. * When we return from this function the fault will be retried.
  254. */
  255. void __migration_entry_wait(struct mm_struct *mm, pte_t *ptep,
  256. spinlock_t *ptl)
  257. {
  258. pte_t pte;
  259. swp_entry_t entry;
  260. struct page *page;
  261. spin_lock(ptl);
  262. pte = *ptep;
  263. if (!is_swap_pte(pte))
  264. goto out;
  265. entry = pte_to_swp_entry(pte);
  266. if (!is_migration_entry(entry))
  267. goto out;
  268. page = migration_entry_to_page(entry);
  269. page = compound_head(page);
  270. /*
  271. * Once page cache replacement of page migration started, page_count
  272. * is zero; but we must not call put_and_wait_on_page_locked() without
  273. * a ref. Use get_page_unless_zero(), and just fault again if it fails.
  274. */
  275. if (!get_page_unless_zero(page))
  276. goto out;
  277. pte_unmap_unlock(ptep, ptl);
  278. put_and_wait_on_page_locked(page);
  279. return;
  280. out:
  281. pte_unmap_unlock(ptep, ptl);
  282. }
  283. void migration_entry_wait(struct mm_struct *mm, pmd_t *pmd,
  284. unsigned long address)
  285. {
  286. spinlock_t *ptl = pte_lockptr(mm, pmd);
  287. pte_t *ptep = pte_offset_map(pmd, address);
  288. __migration_entry_wait(mm, ptep, ptl);
  289. }
  290. void migration_entry_wait_huge(struct vm_area_struct *vma,
  291. struct mm_struct *mm, pte_t *pte)
  292. {
  293. spinlock_t *ptl = huge_pte_lockptr(hstate_vma(vma), mm, pte);
  294. __migration_entry_wait(mm, pte, ptl);
  295. }
  296. #ifdef CONFIG_ARCH_ENABLE_THP_MIGRATION
  297. void pmd_migration_entry_wait(struct mm_struct *mm, pmd_t *pmd)
  298. {
  299. spinlock_t *ptl;
  300. struct page *page;
  301. ptl = pmd_lock(mm, pmd);
  302. if (!is_pmd_migration_entry(*pmd))
  303. goto unlock;
  304. page = migration_entry_to_page(pmd_to_swp_entry(*pmd));
  305. if (!get_page_unless_zero(page))
  306. goto unlock;
  307. spin_unlock(ptl);
  308. put_and_wait_on_page_locked(page);
  309. return;
  310. unlock:
  311. spin_unlock(ptl);
  312. }
  313. #endif
  314. static int expected_page_refs(struct address_space *mapping, struct page *page)
  315. {
  316. int expected_count = 1;
  317. /*
  318. * Device private pages have an extra refcount as they are
  319. * ZONE_DEVICE pages.
  320. */
  321. expected_count += is_device_private_page(page);
  322. if (mapping)
  323. expected_count += thp_nr_pages(page) + page_has_private(page);
  324. return expected_count;
  325. }
  326. /*
  327. * Replace the page in the mapping.
  328. *
  329. * The number of remaining references must be:
  330. * 1 for anonymous pages without a mapping
  331. * 2 for pages with a mapping
  332. * 3 for pages with a mapping and PagePrivate/PagePrivate2 set.
  333. */
  334. int migrate_page_move_mapping(struct address_space *mapping,
  335. struct page *newpage, struct page *page, int extra_count)
  336. {
  337. XA_STATE(xas, &mapping->i_pages, page_index(page));
  338. struct zone *oldzone, *newzone;
  339. int dirty;
  340. int expected_count = expected_page_refs(mapping, page) + extra_count;
  341. int nr = thp_nr_pages(page);
  342. if (!mapping) {
  343. /* Anonymous page without mapping */
  344. if (page_count(page) != expected_count)
  345. return -EAGAIN;
  346. /* No turning back from here */
  347. newpage->index = page->index;
  348. newpage->mapping = page->mapping;
  349. if (PageSwapBacked(page))
  350. __SetPageSwapBacked(newpage);
  351. return MIGRATEPAGE_SUCCESS;
  352. }
  353. oldzone = page_zone(page);
  354. newzone = page_zone(newpage);
  355. xas_lock_irq(&xas);
  356. if (page_count(page) != expected_count || xas_load(&xas) != page) {
  357. xas_unlock_irq(&xas);
  358. return -EAGAIN;
  359. }
  360. if (!page_ref_freeze(page, expected_count)) {
  361. xas_unlock_irq(&xas);
  362. return -EAGAIN;
  363. }
  364. /*
  365. * Now we know that no one else is looking at the page:
  366. * no turning back from here.
  367. */
  368. newpage->index = page->index;
  369. newpage->mapping = page->mapping;
  370. page_ref_add(newpage, nr); /* add cache reference */
  371. if (PageSwapBacked(page)) {
  372. __SetPageSwapBacked(newpage);
  373. if (PageSwapCache(page)) {
  374. SetPageSwapCache(newpage);
  375. set_page_private(newpage, page_private(page));
  376. }
  377. } else {
  378. VM_BUG_ON_PAGE(PageSwapCache(page), page);
  379. }
  380. /* Move dirty while page refs frozen and newpage not yet exposed */
  381. dirty = PageDirty(page);
  382. if (dirty) {
  383. ClearPageDirty(page);
  384. SetPageDirty(newpage);
  385. }
  386. xas_store(&xas, newpage);
  387. if (PageTransHuge(page)) {
  388. int i;
  389. for (i = 1; i < nr; i++) {
  390. xas_next(&xas);
  391. xas_store(&xas, newpage);
  392. }
  393. }
  394. /*
  395. * Drop cache reference from old page by unfreezing
  396. * to one less reference.
  397. * We know this isn't the last reference.
  398. */
  399. page_ref_unfreeze(page, expected_count - nr);
  400. xas_unlock(&xas);
  401. /* Leave irq disabled to prevent preemption while updating stats */
  402. /*
  403. * If moved to a different zone then also account
  404. * the page for that zone. Other VM counters will be
  405. * taken care of when we establish references to the
  406. * new page and drop references to the old page.
  407. *
  408. * Note that anonymous pages are accounted for
  409. * via NR_FILE_PAGES and NR_ANON_MAPPED if they
  410. * are mapped to swap space.
  411. */
  412. if (newzone != oldzone) {
  413. struct lruvec *old_lruvec, *new_lruvec;
  414. struct mem_cgroup *memcg;
  415. memcg = page_memcg(page);
  416. old_lruvec = mem_cgroup_lruvec(memcg, oldzone->zone_pgdat);
  417. new_lruvec = mem_cgroup_lruvec(memcg, newzone->zone_pgdat);
  418. __mod_lruvec_state(old_lruvec, NR_FILE_PAGES, -nr);
  419. __mod_lruvec_state(new_lruvec, NR_FILE_PAGES, nr);
  420. if (PageSwapBacked(page) && !PageSwapCache(page)) {
  421. __mod_lruvec_state(old_lruvec, NR_SHMEM, -nr);
  422. __mod_lruvec_state(new_lruvec, NR_SHMEM, nr);
  423. }
  424. if (dirty && mapping_can_writeback(mapping)) {
  425. __mod_lruvec_state(old_lruvec, NR_FILE_DIRTY, -nr);
  426. __mod_zone_page_state(oldzone, NR_ZONE_WRITE_PENDING, -nr);
  427. __mod_lruvec_state(new_lruvec, NR_FILE_DIRTY, nr);
  428. __mod_zone_page_state(newzone, NR_ZONE_WRITE_PENDING, nr);
  429. }
  430. }
  431. local_irq_enable();
  432. return MIGRATEPAGE_SUCCESS;
  433. }
  434. EXPORT_SYMBOL(migrate_page_move_mapping);
  435. /*
  436. * The expected number of remaining references is the same as that
  437. * of migrate_page_move_mapping().
  438. */
  439. int migrate_huge_page_move_mapping(struct address_space *mapping,
  440. struct page *newpage, struct page *page)
  441. {
  442. XA_STATE(xas, &mapping->i_pages, page_index(page));
  443. int expected_count;
  444. xas_lock_irq(&xas);
  445. expected_count = 2 + page_has_private(page);
  446. if (page_count(page) != expected_count || xas_load(&xas) != page) {
  447. xas_unlock_irq(&xas);
  448. return -EAGAIN;
  449. }
  450. if (!page_ref_freeze(page, expected_count)) {
  451. xas_unlock_irq(&xas);
  452. return -EAGAIN;
  453. }
  454. newpage->index = page->index;
  455. newpage->mapping = page->mapping;
  456. get_page(newpage);
  457. xas_store(&xas, newpage);
  458. page_ref_unfreeze(page, expected_count - 1);
  459. xas_unlock_irq(&xas);
  460. return MIGRATEPAGE_SUCCESS;
  461. }
  462. /*
  463. * Gigantic pages are so large that we do not guarantee that page++ pointer
  464. * arithmetic will work across the entire page. We need something more
  465. * specialized.
  466. */
  467. static void __copy_gigantic_page(struct page *dst, struct page *src,
  468. int nr_pages)
  469. {
  470. int i;
  471. struct page *dst_base = dst;
  472. struct page *src_base = src;
  473. for (i = 0; i < nr_pages; ) {
  474. cond_resched();
  475. copy_highpage(dst, src);
  476. i++;
  477. dst = mem_map_next(dst, dst_base, i);
  478. src = mem_map_next(src, src_base, i);
  479. }
  480. }
  481. static void copy_huge_page(struct page *dst, struct page *src)
  482. {
  483. int i;
  484. int nr_pages;
  485. if (PageHuge(src)) {
  486. /* hugetlbfs page */
  487. struct hstate *h = page_hstate(src);
  488. nr_pages = pages_per_huge_page(h);
  489. if (unlikely(nr_pages > MAX_ORDER_NR_PAGES)) {
  490. __copy_gigantic_page(dst, src, nr_pages);
  491. return;
  492. }
  493. } else {
  494. /* thp page */
  495. BUG_ON(!PageTransHuge(src));
  496. nr_pages = thp_nr_pages(src);
  497. }
  498. for (i = 0; i < nr_pages; i++) {
  499. cond_resched();
  500. copy_highpage(dst + i, src + i);
  501. }
  502. }
  503. /*
  504. * Copy the page to its new location
  505. */
  506. void migrate_page_states(struct page *newpage, struct page *page)
  507. {
  508. int cpupid;
  509. if (PageError(page))
  510. SetPageError(newpage);
  511. if (PageReferenced(page))
  512. SetPageReferenced(newpage);
  513. if (PageUptodate(page))
  514. SetPageUptodate(newpage);
  515. if (TestClearPageActive(page)) {
  516. VM_BUG_ON_PAGE(PageUnevictable(page), page);
  517. SetPageActive(newpage);
  518. } else if (TestClearPageUnevictable(page))
  519. SetPageUnevictable(newpage);
  520. if (PageWorkingset(page))
  521. SetPageWorkingset(newpage);
  522. if (PageChecked(page))
  523. SetPageChecked(newpage);
  524. if (PageMappedToDisk(page))
  525. SetPageMappedToDisk(newpage);
  526. /* Move dirty on pages not done by migrate_page_move_mapping() */
  527. if (PageDirty(page))
  528. SetPageDirty(newpage);
  529. if (page_is_young(page))
  530. set_page_young(newpage);
  531. if (page_is_idle(page))
  532. set_page_idle(newpage);
  533. /*
  534. * Copy NUMA information to the new page, to prevent over-eager
  535. * future migrations of this same page.
  536. */
  537. cpupid = page_cpupid_xchg_last(page, -1);
  538. page_cpupid_xchg_last(newpage, cpupid);
  539. ksm_migrate_page(newpage, page);
  540. /*
  541. * Please do not reorder this without considering how mm/ksm.c's
  542. * get_ksm_page() depends upon ksm_migrate_page() and PageSwapCache().
  543. */
  544. if (PageSwapCache(page))
  545. ClearPageSwapCache(page);
  546. ClearPagePrivate(page);
  547. set_page_private(page, 0);
  548. /*
  549. * If any waiters have accumulated on the new page then
  550. * wake them up.
  551. */
  552. if (PageWriteback(newpage))
  553. end_page_writeback(newpage);
  554. /*
  555. * PG_readahead shares the same bit with PG_reclaim. The above
  556. * end_page_writeback() may clear PG_readahead mistakenly, so set the
  557. * bit after that.
  558. */
  559. if (PageReadahead(page))
  560. SetPageReadahead(newpage);
  561. copy_page_owner(page, newpage);
  562. if (!PageHuge(page))
  563. mem_cgroup_migrate(page, newpage);
  564. }
  565. EXPORT_SYMBOL(migrate_page_states);
  566. void migrate_page_copy(struct page *newpage, struct page *page)
  567. {
  568. if (PageHuge(page) || PageTransHuge(page))
  569. copy_huge_page(newpage, page);
  570. else
  571. copy_highpage(newpage, page);
  572. migrate_page_states(newpage, page);
  573. }
  574. EXPORT_SYMBOL(migrate_page_copy);
  575. /************************************************************
  576. * Migration functions
  577. ***********************************************************/
  578. /*
  579. * Common logic to directly migrate a single LRU page suitable for
  580. * pages that do not use PagePrivate/PagePrivate2.
  581. *
  582. * Pages are locked upon entry and exit.
  583. */
  584. int migrate_page(struct address_space *mapping,
  585. struct page *newpage, struct page *page,
  586. enum migrate_mode mode)
  587. {
  588. int rc;
  589. BUG_ON(PageWriteback(page)); /* Writeback must be complete */
  590. rc = migrate_page_move_mapping(mapping, newpage, page, 0);
  591. if (rc != MIGRATEPAGE_SUCCESS)
  592. return rc;
  593. if (mode != MIGRATE_SYNC_NO_COPY)
  594. migrate_page_copy(newpage, page);
  595. else
  596. migrate_page_states(newpage, page);
  597. return MIGRATEPAGE_SUCCESS;
  598. }
  599. EXPORT_SYMBOL(migrate_page);
  600. #ifdef CONFIG_BLOCK
  601. /* Returns true if all buffers are successfully locked */
  602. static bool buffer_migrate_lock_buffers(struct buffer_head *head,
  603. enum migrate_mode mode)
  604. {
  605. struct buffer_head *bh = head;
  606. /* Simple case, sync compaction */
  607. if (mode != MIGRATE_ASYNC) {
  608. do {
  609. lock_buffer(bh);
  610. bh = bh->b_this_page;
  611. } while (bh != head);
  612. return true;
  613. }
  614. /* async case, we cannot block on lock_buffer so use trylock_buffer */
  615. do {
  616. if (!trylock_buffer(bh)) {
  617. /*
  618. * We failed to lock the buffer and cannot stall in
  619. * async migration. Release the taken locks
  620. */
  621. struct buffer_head *failed_bh = bh;
  622. bh = head;
  623. while (bh != failed_bh) {
  624. unlock_buffer(bh);
  625. bh = bh->b_this_page;
  626. }
  627. return false;
  628. }
  629. bh = bh->b_this_page;
  630. } while (bh != head);
  631. return true;
  632. }
  633. static int __buffer_migrate_page(struct address_space *mapping,
  634. struct page *newpage, struct page *page, enum migrate_mode mode,
  635. bool check_refs)
  636. {
  637. struct buffer_head *bh, *head;
  638. int rc;
  639. int expected_count;
  640. if (!page_has_buffers(page))
  641. return migrate_page(mapping, newpage, page, mode);
  642. /* Check whether page does not have extra refs before we do more work */
  643. expected_count = expected_page_refs(mapping, page);
  644. if (page_count(page) != expected_count)
  645. return -EAGAIN;
  646. head = page_buffers(page);
  647. if (!buffer_migrate_lock_buffers(head, mode))
  648. return -EAGAIN;
  649. if (check_refs) {
  650. bool busy;
  651. bool invalidated = false;
  652. recheck_buffers:
  653. busy = false;
  654. spin_lock(&mapping->private_lock);
  655. bh = head;
  656. do {
  657. if (atomic_read(&bh->b_count)) {
  658. busy = true;
  659. break;
  660. }
  661. bh = bh->b_this_page;
  662. } while (bh != head);
  663. if (busy) {
  664. if (invalidated) {
  665. rc = -EAGAIN;
  666. goto unlock_buffers;
  667. }
  668. spin_unlock(&mapping->private_lock);
  669. invalidate_bh_lrus();
  670. invalidated = true;
  671. goto recheck_buffers;
  672. }
  673. }
  674. rc = migrate_page_move_mapping(mapping, newpage, page, 0);
  675. if (rc != MIGRATEPAGE_SUCCESS)
  676. goto unlock_buffers;
  677. attach_page_private(newpage, detach_page_private(page));
  678. bh = head;
  679. do {
  680. set_bh_page(bh, newpage, bh_offset(bh));
  681. bh = bh->b_this_page;
  682. } while (bh != head);
  683. if (mode != MIGRATE_SYNC_NO_COPY)
  684. migrate_page_copy(newpage, page);
  685. else
  686. migrate_page_states(newpage, page);
  687. rc = MIGRATEPAGE_SUCCESS;
  688. unlock_buffers:
  689. if (check_refs)
  690. spin_unlock(&mapping->private_lock);
  691. bh = head;
  692. do {
  693. unlock_buffer(bh);
  694. bh = bh->b_this_page;
  695. } while (bh != head);
  696. return rc;
  697. }
  698. /*
  699. * Migration function for pages with buffers. This function can only be used
  700. * if the underlying filesystem guarantees that no other references to "page"
  701. * exist. For example attached buffer heads are accessed only under page lock.
  702. */
  703. int buffer_migrate_page(struct address_space *mapping,
  704. struct page *newpage, struct page *page, enum migrate_mode mode)
  705. {
  706. return __buffer_migrate_page(mapping, newpage, page, mode, false);
  707. }
  708. EXPORT_SYMBOL(buffer_migrate_page);
  709. /*
  710. * Same as above except that this variant is more careful and checks that there
  711. * are also no buffer head references. This function is the right one for
  712. * mappings where buffer heads are directly looked up and referenced (such as
  713. * block device mappings).
  714. */
  715. int buffer_migrate_page_norefs(struct address_space *mapping,
  716. struct page *newpage, struct page *page, enum migrate_mode mode)
  717. {
  718. return __buffer_migrate_page(mapping, newpage, page, mode, true);
  719. }
  720. #endif
  721. /*
  722. * Writeback a page to clean the dirty state
  723. */
  724. static int writeout(struct address_space *mapping, struct page *page)
  725. {
  726. struct writeback_control wbc = {
  727. .sync_mode = WB_SYNC_NONE,
  728. .nr_to_write = 1,
  729. .range_start = 0,
  730. .range_end = LLONG_MAX,
  731. .for_reclaim = 1
  732. };
  733. int rc;
  734. if (!mapping->a_ops->writepage)
  735. /* No write method for the address space */
  736. return -EINVAL;
  737. if (!clear_page_dirty_for_io(page))
  738. /* Someone else already triggered a write */
  739. return -EAGAIN;
  740. /*
  741. * A dirty page may imply that the underlying filesystem has
  742. * the page on some queue. So the page must be clean for
  743. * migration. Writeout may mean we loose the lock and the
  744. * page state is no longer what we checked for earlier.
  745. * At this point we know that the migration attempt cannot
  746. * be successful.
  747. */
  748. remove_migration_ptes(page, page, false);
  749. rc = mapping->a_ops->writepage(page, &wbc);
  750. if (rc != AOP_WRITEPAGE_ACTIVATE)
  751. /* unlocked. Relock */
  752. lock_page(page);
  753. return (rc < 0) ? -EIO : -EAGAIN;
  754. }
  755. /*
  756. * Default handling if a filesystem does not provide a migration function.
  757. */
  758. static int fallback_migrate_page(struct address_space *mapping,
  759. struct page *newpage, struct page *page, enum migrate_mode mode)
  760. {
  761. if (PageDirty(page)) {
  762. /* Only writeback pages in full synchronous migration */
  763. switch (mode) {
  764. case MIGRATE_SYNC:
  765. case MIGRATE_SYNC_NO_COPY:
  766. break;
  767. default:
  768. return -EBUSY;
  769. }
  770. return writeout(mapping, page);
  771. }
  772. /*
  773. * Buffers may be managed in a filesystem specific way.
  774. * We must have no buffers or drop them.
  775. */
  776. if (page_has_private(page) &&
  777. !try_to_release_page(page, GFP_KERNEL))
  778. return mode == MIGRATE_SYNC ? -EAGAIN : -EBUSY;
  779. return migrate_page(mapping, newpage, page, mode);
  780. }
  781. /*
  782. * Move a page to a newly allocated page
  783. * The page is locked and all ptes have been successfully removed.
  784. *
  785. * The new page will have replaced the old page if this function
  786. * is successful.
  787. *
  788. * Return value:
  789. * < 0 - error code
  790. * MIGRATEPAGE_SUCCESS - success
  791. */
  792. static int move_to_new_page(struct page *newpage, struct page *page,
  793. enum migrate_mode mode)
  794. {
  795. struct address_space *mapping;
  796. int rc = -EAGAIN;
  797. bool is_lru = !__PageMovable(page);
  798. VM_BUG_ON_PAGE(!PageLocked(page), page);
  799. VM_BUG_ON_PAGE(!PageLocked(newpage), newpage);
  800. mapping = page_mapping(page);
  801. if (likely(is_lru)) {
  802. if (!mapping)
  803. rc = migrate_page(mapping, newpage, page, mode);
  804. else if (mapping->a_ops->migratepage)
  805. /*
  806. * Most pages have a mapping and most filesystems
  807. * provide a migratepage callback. Anonymous pages
  808. * are part of swap space which also has its own
  809. * migratepage callback. This is the most common path
  810. * for page migration.
  811. */
  812. rc = mapping->a_ops->migratepage(mapping, newpage,
  813. page, mode);
  814. else
  815. rc = fallback_migrate_page(mapping, newpage,
  816. page, mode);
  817. } else {
  818. /*
  819. * In case of non-lru page, it could be released after
  820. * isolation step. In that case, we shouldn't try migration.
  821. */
  822. VM_BUG_ON_PAGE(!PageIsolated(page), page);
  823. if (!PageMovable(page)) {
  824. rc = MIGRATEPAGE_SUCCESS;
  825. ClearPageIsolated(page);
  826. goto out;
  827. }
  828. rc = mapping->a_ops->migratepage(mapping, newpage,
  829. page, mode);
  830. WARN_ON_ONCE(rc == MIGRATEPAGE_SUCCESS &&
  831. !PageIsolated(page));
  832. }
  833. /*
  834. * When successful, old pagecache page->mapping must be cleared before
  835. * page is freed; but stats require that PageAnon be left as PageAnon.
  836. */
  837. if (rc == MIGRATEPAGE_SUCCESS) {
  838. if (__PageMovable(page)) {
  839. VM_BUG_ON_PAGE(!PageIsolated(page), page);
  840. /*
  841. * We clear PG_movable under page_lock so any compactor
  842. * cannot try to migrate this page.
  843. */
  844. ClearPageIsolated(page);
  845. }
  846. /*
  847. * Anonymous and movable page->mapping will be cleared by
  848. * free_pages_prepare so don't reset it here for keeping
  849. * the type to work PageAnon, for example.
  850. */
  851. if (!PageMappingFlags(page))
  852. page->mapping = NULL;
  853. if (likely(!is_zone_device_page(newpage)))
  854. flush_dcache_page(newpage);
  855. }
  856. out:
  857. return rc;
  858. }
  859. static int __unmap_and_move(struct page *page, struct page *newpage,
  860. int force, enum migrate_mode mode)
  861. {
  862. int rc = -EAGAIN;
  863. int page_was_mapped = 0;
  864. struct anon_vma *anon_vma = NULL;
  865. bool is_lru = !__PageMovable(page);
  866. if (!trylock_page(page)) {
  867. if (!force || mode == MIGRATE_ASYNC)
  868. goto out;
  869. /*
  870. * It's not safe for direct compaction to call lock_page.
  871. * For example, during page readahead pages are added locked
  872. * to the LRU. Later, when the IO completes the pages are
  873. * marked uptodate and unlocked. However, the queueing
  874. * could be merging multiple pages for one bio (e.g.
  875. * mpage_readahead). If an allocation happens for the
  876. * second or third page, the process can end up locking
  877. * the same page twice and deadlocking. Rather than
  878. * trying to be clever about what pages can be locked,
  879. * avoid the use of lock_page for direct compaction
  880. * altogether.
  881. */
  882. if (current->flags & PF_MEMALLOC)
  883. goto out;
  884. lock_page(page);
  885. }
  886. if (PageWriteback(page)) {
  887. /*
  888. * Only in the case of a full synchronous migration is it
  889. * necessary to wait for PageWriteback. In the async case,
  890. * the retry loop is too short and in the sync-light case,
  891. * the overhead of stalling is too much
  892. */
  893. switch (mode) {
  894. case MIGRATE_SYNC:
  895. case MIGRATE_SYNC_NO_COPY:
  896. break;
  897. default:
  898. rc = -EBUSY;
  899. goto out_unlock;
  900. }
  901. if (!force)
  902. goto out_unlock;
  903. wait_on_page_writeback(page);
  904. }
  905. /*
  906. * By try_to_unmap(), page->mapcount goes down to 0 here. In this case,
  907. * we cannot notice that anon_vma is freed while we migrates a page.
  908. * This get_anon_vma() delays freeing anon_vma pointer until the end
  909. * of migration. File cache pages are no problem because of page_lock()
  910. * File Caches may use write_page() or lock_page() in migration, then,
  911. * just care Anon page here.
  912. *
  913. * Only page_get_anon_vma() understands the subtleties of
  914. * getting a hold on an anon_vma from outside one of its mms.
  915. * But if we cannot get anon_vma, then we won't need it anyway,
  916. * because that implies that the anon page is no longer mapped
  917. * (and cannot be remapped so long as we hold the page lock).
  918. */
  919. if (PageAnon(page) && !PageKsm(page))
  920. anon_vma = page_get_anon_vma(page);
  921. /*
  922. * Block others from accessing the new page when we get around to
  923. * establishing additional references. We are usually the only one
  924. * holding a reference to newpage at this point. We used to have a BUG
  925. * here if trylock_page(newpage) fails, but would like to allow for
  926. * cases where there might be a race with the previous use of newpage.
  927. * This is much like races on refcount of oldpage: just don't BUG().
  928. */
  929. if (unlikely(!trylock_page(newpage)))
  930. goto out_unlock;
  931. if (unlikely(!is_lru)) {
  932. rc = move_to_new_page(newpage, page, mode);
  933. goto out_unlock_both;
  934. }
  935. /*
  936. * Corner case handling:
  937. * 1. When a new swap-cache page is read into, it is added to the LRU
  938. * and treated as swapcache but it has no rmap yet.
  939. * Calling try_to_unmap() against a page->mapping==NULL page will
  940. * trigger a BUG. So handle it here.
  941. * 2. An orphaned page (see truncate_complete_page) might have
  942. * fs-private metadata. The page can be picked up due to memory
  943. * offlining. Everywhere else except page reclaim, the page is
  944. * invisible to the vm, so the page can not be migrated. So try to
  945. * free the metadata, so the page can be freed.
  946. */
  947. if (!page->mapping) {
  948. VM_BUG_ON_PAGE(PageAnon(page), page);
  949. if (page_has_private(page)) {
  950. try_to_free_buffers(page);
  951. goto out_unlock_both;
  952. }
  953. } else if (page_mapped(page)) {
  954. /* Establish migration ptes */
  955. VM_BUG_ON_PAGE(PageAnon(page) && !PageKsm(page) && !anon_vma,
  956. page);
  957. try_to_unmap(page, TTU_MIGRATION|TTU_IGNORE_MLOCK);
  958. page_was_mapped = 1;
  959. }
  960. if (!page_mapped(page))
  961. rc = move_to_new_page(newpage, page, mode);
  962. if (page_was_mapped)
  963. remove_migration_ptes(page,
  964. rc == MIGRATEPAGE_SUCCESS ? newpage : page, false);
  965. out_unlock_both:
  966. unlock_page(newpage);
  967. out_unlock:
  968. /* Drop an anon_vma reference if we took one */
  969. if (anon_vma)
  970. put_anon_vma(anon_vma);
  971. unlock_page(page);
  972. out:
  973. /*
  974. * If migration is successful, decrease refcount of the newpage
  975. * which will not free the page because new page owner increased
  976. * refcounter. As well, if it is LRU page, add the page to LRU
  977. * list in here. Use the old state of the isolated source page to
  978. * determine if we migrated a LRU page. newpage was already unlocked
  979. * and possibly modified by its owner - don't rely on the page
  980. * state.
  981. */
  982. if (rc == MIGRATEPAGE_SUCCESS) {
  983. if (unlikely(!is_lru))
  984. put_page(newpage);
  985. else
  986. putback_lru_page(newpage);
  987. }
  988. return rc;
  989. }
  990. /*
  991. * Obtain the lock on page, remove all ptes and migrate the page
  992. * to the newly allocated page in newpage.
  993. */
  994. static int unmap_and_move(new_page_t get_new_page,
  995. free_page_t put_new_page,
  996. unsigned long private, struct page *page,
  997. int force, enum migrate_mode mode,
  998. enum migrate_reason reason)
  999. {
  1000. int rc = MIGRATEPAGE_SUCCESS;
  1001. struct page *newpage = NULL;
  1002. if (!thp_migration_supported() && PageTransHuge(page))
  1003. return -ENOMEM;
  1004. if (page_count(page) == 1) {
  1005. /* page was freed from under us. So we are done. */
  1006. ClearPageActive(page);
  1007. ClearPageUnevictable(page);
  1008. if (unlikely(__PageMovable(page))) {
  1009. lock_page(page);
  1010. if (!PageMovable(page))
  1011. ClearPageIsolated(page);
  1012. unlock_page(page);
  1013. }
  1014. goto out;
  1015. }
  1016. newpage = get_new_page(page, private);
  1017. if (!newpage)
  1018. return -ENOMEM;
  1019. rc = __unmap_and_move(page, newpage, force, mode);
  1020. if (rc == MIGRATEPAGE_SUCCESS)
  1021. set_page_owner_migrate_reason(newpage, reason);
  1022. out:
  1023. if (rc != -EAGAIN) {
  1024. /*
  1025. * A page that has been migrated has all references
  1026. * removed and will be freed. A page that has not been
  1027. * migrated will have kept its references and be restored.
  1028. */
  1029. list_del(&page->lru);
  1030. /*
  1031. * Compaction can migrate also non-LRU pages which are
  1032. * not accounted to NR_ISOLATED_*. They can be recognized
  1033. * as __PageMovable
  1034. */
  1035. if (likely(!__PageMovable(page)))
  1036. mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON +
  1037. page_is_file_lru(page), -thp_nr_pages(page));
  1038. }
  1039. /*
  1040. * If migration is successful, releases reference grabbed during
  1041. * isolation. Otherwise, restore the page to right list unless
  1042. * we want to retry.
  1043. */
  1044. if (rc == MIGRATEPAGE_SUCCESS) {
  1045. if (reason != MR_MEMORY_FAILURE)
  1046. /*
  1047. * We release the page in page_handle_poison.
  1048. */
  1049. put_page(page);
  1050. } else {
  1051. if (rc != -EAGAIN) {
  1052. if (likely(!__PageMovable(page))) {
  1053. putback_lru_page(page);
  1054. goto put_new;
  1055. }
  1056. lock_page(page);
  1057. if (PageMovable(page))
  1058. putback_movable_page(page);
  1059. else
  1060. ClearPageIsolated(page);
  1061. unlock_page(page);
  1062. put_page(page);
  1063. }
  1064. put_new:
  1065. if (put_new_page)
  1066. put_new_page(newpage, private);
  1067. else
  1068. put_page(newpage);
  1069. }
  1070. return rc;
  1071. }
  1072. /*
  1073. * Counterpart of unmap_and_move_page() for hugepage migration.
  1074. *
  1075. * This function doesn't wait the completion of hugepage I/O
  1076. * because there is no race between I/O and migration for hugepage.
  1077. * Note that currently hugepage I/O occurs only in direct I/O
  1078. * where no lock is held and PG_writeback is irrelevant,
  1079. * and writeback status of all subpages are counted in the reference
  1080. * count of the head page (i.e. if all subpages of a 2MB hugepage are
  1081. * under direct I/O, the reference of the head page is 512 and a bit more.)
  1082. * This means that when we try to migrate hugepage whose subpages are
  1083. * doing direct I/O, some references remain after try_to_unmap() and
  1084. * hugepage migration fails without data corruption.
  1085. *
  1086. * There is also no race when direct I/O is issued on the page under migration,
  1087. * because then pte is replaced with migration swap entry and direct I/O code
  1088. * will wait in the page fault for migration to complete.
  1089. */
  1090. static int unmap_and_move_huge_page(new_page_t get_new_page,
  1091. free_page_t put_new_page, unsigned long private,
  1092. struct page *hpage, int force,
  1093. enum migrate_mode mode, int reason)
  1094. {
  1095. int rc = -EAGAIN;
  1096. int page_was_mapped = 0;
  1097. struct page *new_hpage;
  1098. struct anon_vma *anon_vma = NULL;
  1099. struct address_space *mapping = NULL;
  1100. /*
  1101. * Migratability of hugepages depends on architectures and their size.
  1102. * This check is necessary because some callers of hugepage migration
  1103. * like soft offline and memory hotremove don't walk through page
  1104. * tables or check whether the hugepage is pmd-based or not before
  1105. * kicking migration.
  1106. */
  1107. if (!hugepage_migration_supported(page_hstate(hpage))) {
  1108. putback_active_hugepage(hpage);
  1109. return -ENOSYS;
  1110. }
  1111. new_hpage = get_new_page(hpage, private);
  1112. if (!new_hpage)
  1113. return -ENOMEM;
  1114. if (!trylock_page(hpage)) {
  1115. if (!force)
  1116. goto out;
  1117. switch (mode) {
  1118. case MIGRATE_SYNC:
  1119. case MIGRATE_SYNC_NO_COPY:
  1120. break;
  1121. default:
  1122. goto out;
  1123. }
  1124. lock_page(hpage);
  1125. }
  1126. /*
  1127. * Check for pages which are in the process of being freed. Without
  1128. * page_mapping() set, hugetlbfs specific move page routine will not
  1129. * be called and we could leak usage counts for subpools.
  1130. */
  1131. if (page_private(hpage) && !page_mapping(hpage)) {
  1132. rc = -EBUSY;
  1133. goto out_unlock;
  1134. }
  1135. if (PageAnon(hpage))
  1136. anon_vma = page_get_anon_vma(hpage);
  1137. if (unlikely(!trylock_page(new_hpage)))
  1138. goto put_anon;
  1139. if (page_mapped(hpage)) {
  1140. bool mapping_locked = false;
  1141. enum ttu_flags ttu = TTU_MIGRATION|TTU_IGNORE_MLOCK;
  1142. if (!PageAnon(hpage)) {
  1143. /*
  1144. * In shared mappings, try_to_unmap could potentially
  1145. * call huge_pmd_unshare. Because of this, take
  1146. * semaphore in write mode here and set TTU_RMAP_LOCKED
  1147. * to let lower levels know we have taken the lock.
  1148. */
  1149. mapping = hugetlb_page_mapping_lock_write(hpage);
  1150. if (unlikely(!mapping))
  1151. goto unlock_put_anon;
  1152. mapping_locked = true;
  1153. ttu |= TTU_RMAP_LOCKED;
  1154. }
  1155. try_to_unmap(hpage, ttu);
  1156. page_was_mapped = 1;
  1157. if (mapping_locked)
  1158. i_mmap_unlock_write(mapping);
  1159. }
  1160. if (!page_mapped(hpage))
  1161. rc = move_to_new_page(new_hpage, hpage, mode);
  1162. if (page_was_mapped)
  1163. remove_migration_ptes(hpage,
  1164. rc == MIGRATEPAGE_SUCCESS ? new_hpage : hpage, false);
  1165. unlock_put_anon:
  1166. unlock_page(new_hpage);
  1167. put_anon:
  1168. if (anon_vma)
  1169. put_anon_vma(anon_vma);
  1170. if (rc == MIGRATEPAGE_SUCCESS) {
  1171. move_hugetlb_state(hpage, new_hpage, reason);
  1172. put_new_page = NULL;
  1173. }
  1174. out_unlock:
  1175. unlock_page(hpage);
  1176. out:
  1177. if (rc != -EAGAIN)
  1178. putback_active_hugepage(hpage);
  1179. /*
  1180. * If migration was not successful and there's a freeing callback, use
  1181. * it. Otherwise, put_page() will drop the reference grabbed during
  1182. * isolation.
  1183. */
  1184. if (put_new_page)
  1185. put_new_page(new_hpage, private);
  1186. else
  1187. putback_active_hugepage(new_hpage);
  1188. return rc;
  1189. }
  1190. /*
  1191. * migrate_pages - migrate the pages specified in a list, to the free pages
  1192. * supplied as the target for the page migration
  1193. *
  1194. * @from: The list of pages to be migrated.
  1195. * @get_new_page: The function used to allocate free pages to be used
  1196. * as the target of the page migration.
  1197. * @put_new_page: The function used to free target pages if migration
  1198. * fails, or NULL if no special handling is necessary.
  1199. * @private: Private data to be passed on to get_new_page()
  1200. * @mode: The migration mode that specifies the constraints for
  1201. * page migration, if any.
  1202. * @reason: The reason for page migration.
  1203. *
  1204. * The function returns after 10 attempts or if no pages are movable any more
  1205. * because the list has become empty or no retryable pages exist any more.
  1206. * The caller should call putback_movable_pages() to return pages to the LRU
  1207. * or free list only if ret != 0.
  1208. *
  1209. * Returns the number of pages that were not migrated, or an error code.
  1210. */
  1211. int migrate_pages(struct list_head *from, new_page_t get_new_page,
  1212. free_page_t put_new_page, unsigned long private,
  1213. enum migrate_mode mode, int reason)
  1214. {
  1215. int retry = 1;
  1216. int thp_retry = 1;
  1217. int nr_failed = 0;
  1218. int nr_succeeded = 0;
  1219. int nr_thp_succeeded = 0;
  1220. int nr_thp_failed = 0;
  1221. int nr_thp_split = 0;
  1222. int pass = 0;
  1223. bool is_thp = false;
  1224. struct page *page;
  1225. struct page *page2;
  1226. int swapwrite = current->flags & PF_SWAPWRITE;
  1227. int rc, nr_subpages;
  1228. trace_mm_migrate_pages_start(mode, reason);
  1229. if (!swapwrite)
  1230. current->flags |= PF_SWAPWRITE;
  1231. for (pass = 0; pass < 10 && (retry || thp_retry); pass++) {
  1232. retry = 0;
  1233. thp_retry = 0;
  1234. list_for_each_entry_safe(page, page2, from, lru) {
  1235. retry:
  1236. /*
  1237. * THP statistics is based on the source huge page.
  1238. * Capture required information that might get lost
  1239. * during migration.
  1240. */
  1241. is_thp = PageTransHuge(page) && !PageHuge(page);
  1242. nr_subpages = thp_nr_pages(page);
  1243. cond_resched();
  1244. if (PageHuge(page))
  1245. rc = unmap_and_move_huge_page(get_new_page,
  1246. put_new_page, private, page,
  1247. pass > 2, mode, reason);
  1248. else
  1249. rc = unmap_and_move(get_new_page, put_new_page,
  1250. private, page, pass > 2, mode,
  1251. reason);
  1252. switch(rc) {
  1253. case -ENOMEM:
  1254. /*
  1255. * THP migration might be unsupported or the
  1256. * allocation could've failed so we should
  1257. * retry on the same page with the THP split
  1258. * to base pages.
  1259. *
  1260. * Head page is retried immediately and tail
  1261. * pages are added to the tail of the list so
  1262. * we encounter them after the rest of the list
  1263. * is processed.
  1264. */
  1265. if (is_thp) {
  1266. lock_page(page);
  1267. rc = split_huge_page_to_list(page, from);
  1268. unlock_page(page);
  1269. if (!rc) {
  1270. list_safe_reset_next(page, page2, lru);
  1271. nr_thp_split++;
  1272. goto retry;
  1273. }
  1274. nr_thp_failed++;
  1275. nr_failed += nr_subpages;
  1276. goto out;
  1277. }
  1278. nr_failed++;
  1279. goto out;
  1280. case -EAGAIN:
  1281. if (is_thp) {
  1282. thp_retry++;
  1283. break;
  1284. }
  1285. retry++;
  1286. break;
  1287. case MIGRATEPAGE_SUCCESS:
  1288. if (is_thp) {
  1289. nr_thp_succeeded++;
  1290. nr_succeeded += nr_subpages;
  1291. break;
  1292. }
  1293. nr_succeeded++;
  1294. break;
  1295. default:
  1296. /*
  1297. * Permanent failure (-EBUSY, -ENOSYS, etc.):
  1298. * unlike -EAGAIN case, the failed page is
  1299. * removed from migration page list and not
  1300. * retried in the next outer loop.
  1301. */
  1302. if (is_thp) {
  1303. nr_thp_failed++;
  1304. nr_failed += nr_subpages;
  1305. break;
  1306. }
  1307. nr_failed++;
  1308. break;
  1309. }
  1310. }
  1311. }
  1312. nr_failed += retry + thp_retry;
  1313. nr_thp_failed += thp_retry;
  1314. rc = nr_failed;
  1315. out:
  1316. count_vm_events(PGMIGRATE_SUCCESS, nr_succeeded);
  1317. count_vm_events(PGMIGRATE_FAIL, nr_failed);
  1318. count_vm_events(THP_MIGRATION_SUCCESS, nr_thp_succeeded);
  1319. count_vm_events(THP_MIGRATION_FAIL, nr_thp_failed);
  1320. count_vm_events(THP_MIGRATION_SPLIT, nr_thp_split);
  1321. trace_mm_migrate_pages(nr_succeeded, nr_failed, nr_thp_succeeded,
  1322. nr_thp_failed, nr_thp_split, mode, reason);
  1323. if (!swapwrite)
  1324. current->flags &= ~PF_SWAPWRITE;
  1325. return rc;
  1326. }
  1327. EXPORT_SYMBOL_GPL(migrate_pages);
  1328. struct page *alloc_migration_target(struct page *page, unsigned long private)
  1329. {
  1330. struct migration_target_control *mtc;
  1331. gfp_t gfp_mask;
  1332. unsigned int order = 0;
  1333. struct page *new_page = NULL;
  1334. int nid;
  1335. int zidx;
  1336. mtc = (struct migration_target_control *)private;
  1337. gfp_mask = mtc->gfp_mask;
  1338. nid = mtc->nid;
  1339. if (nid == NUMA_NO_NODE)
  1340. nid = page_to_nid(page);
  1341. if (PageHuge(page)) {
  1342. struct hstate *h = page_hstate(compound_head(page));
  1343. gfp_mask = htlb_modify_alloc_mask(h, gfp_mask);
  1344. return alloc_huge_page_nodemask(h, nid, mtc->nmask, gfp_mask);
  1345. }
  1346. if (PageTransHuge(page)) {
  1347. /*
  1348. * clear __GFP_RECLAIM to make the migration callback
  1349. * consistent with regular THP allocations.
  1350. */
  1351. gfp_mask &= ~__GFP_RECLAIM;
  1352. gfp_mask |= GFP_TRANSHUGE;
  1353. order = HPAGE_PMD_ORDER;
  1354. }
  1355. zidx = zone_idx(page_zone(page));
  1356. if (is_highmem_idx(zidx) || zidx == ZONE_MOVABLE)
  1357. gfp_mask |= __GFP_HIGHMEM;
  1358. new_page = __alloc_pages_nodemask(gfp_mask, order, nid, mtc->nmask);
  1359. if (new_page && PageTransHuge(new_page))
  1360. prep_transhuge_page(new_page);
  1361. return new_page;
  1362. }
  1363. #ifdef CONFIG_NUMA
  1364. static int store_status(int __user *status, int start, int value, int nr)
  1365. {
  1366. while (nr-- > 0) {
  1367. if (put_user(value, status + start))
  1368. return -EFAULT;
  1369. start++;
  1370. }
  1371. return 0;
  1372. }
  1373. static int do_move_pages_to_node(struct mm_struct *mm,
  1374. struct list_head *pagelist, int node)
  1375. {
  1376. int err;
  1377. struct migration_target_control mtc = {
  1378. .nid = node,
  1379. .gfp_mask = GFP_HIGHUSER_MOVABLE | __GFP_THISNODE,
  1380. };
  1381. err = migrate_pages(pagelist, alloc_migration_target, NULL,
  1382. (unsigned long)&mtc, MIGRATE_SYNC, MR_SYSCALL);
  1383. if (err)
  1384. putback_movable_pages(pagelist);
  1385. return err;
  1386. }
  1387. /*
  1388. * Resolves the given address to a struct page, isolates it from the LRU and
  1389. * puts it to the given pagelist.
  1390. * Returns:
  1391. * errno - if the page cannot be found/isolated
  1392. * 0 - when it doesn't have to be migrated because it is already on the
  1393. * target node
  1394. * 1 - when it has been queued
  1395. */
  1396. static int add_page_for_migration(struct mm_struct *mm, unsigned long addr,
  1397. int node, struct list_head *pagelist, bool migrate_all)
  1398. {
  1399. struct vm_area_struct *vma;
  1400. struct page *page;
  1401. unsigned int follflags;
  1402. int err;
  1403. mmap_read_lock(mm);
  1404. err = -EFAULT;
  1405. vma = find_vma(mm, addr);
  1406. if (!vma || addr < vma->vm_start || !vma_migratable(vma))
  1407. goto out;
  1408. /* FOLL_DUMP to ignore special (like zero) pages */
  1409. follflags = FOLL_GET | FOLL_DUMP;
  1410. page = follow_page(vma, addr, follflags);
  1411. err = PTR_ERR(page);
  1412. if (IS_ERR(page))
  1413. goto out;
  1414. err = -ENOENT;
  1415. if (!page)
  1416. goto out;
  1417. err = 0;
  1418. if (page_to_nid(page) == node)
  1419. goto out_putpage;
  1420. err = -EACCES;
  1421. if (page_mapcount(page) > 1 && !migrate_all)
  1422. goto out_putpage;
  1423. if (PageHuge(page)) {
  1424. if (PageHead(page)) {
  1425. isolate_huge_page(page, pagelist);
  1426. err = 1;
  1427. }
  1428. } else {
  1429. struct page *head;
  1430. head = compound_head(page);
  1431. err = isolate_lru_page(head);
  1432. if (err)
  1433. goto out_putpage;
  1434. err = 1;
  1435. list_add_tail(&head->lru, pagelist);
  1436. mod_node_page_state(page_pgdat(head),
  1437. NR_ISOLATED_ANON + page_is_file_lru(head),
  1438. thp_nr_pages(head));
  1439. }
  1440. out_putpage:
  1441. /*
  1442. * Either remove the duplicate refcount from
  1443. * isolate_lru_page() or drop the page ref if it was
  1444. * not isolated.
  1445. */
  1446. put_user_page(page);
  1447. out:
  1448. mmap_read_unlock(mm);
  1449. return err;
  1450. }
  1451. static int move_pages_and_store_status(struct mm_struct *mm, int node,
  1452. struct list_head *pagelist, int __user *status,
  1453. int start, int i, unsigned long nr_pages)
  1454. {
  1455. int err;
  1456. if (list_empty(pagelist))
  1457. return 0;
  1458. err = do_move_pages_to_node(mm, pagelist, node);
  1459. if (err) {
  1460. /*
  1461. * Positive err means the number of failed
  1462. * pages to migrate. Since we are going to
  1463. * abort and return the number of non-migrated
  1464. * pages, so need to incude the rest of the
  1465. * nr_pages that have not been attempted as
  1466. * well.
  1467. */
  1468. if (err > 0)
  1469. err += nr_pages - i - 1;
  1470. return err;
  1471. }
  1472. return store_status(status, start, node, i - start);
  1473. }
  1474. /*
  1475. * Migrate an array of page address onto an array of nodes and fill
  1476. * the corresponding array of status.
  1477. */
  1478. static int do_pages_move(struct mm_struct *mm, nodemask_t task_nodes,
  1479. unsigned long nr_pages,
  1480. const void __user * __user *pages,
  1481. const int __user *nodes,
  1482. int __user *status, int flags)
  1483. {
  1484. int current_node = NUMA_NO_NODE;
  1485. LIST_HEAD(pagelist);
  1486. int start, i;
  1487. int err = 0, err1;
  1488. lru_cache_disable();
  1489. for (i = start = 0; i < nr_pages; i++) {
  1490. const void __user *p;
  1491. unsigned long addr;
  1492. int node;
  1493. err = -EFAULT;
  1494. if (get_user(p, pages + i))
  1495. goto out_flush;
  1496. if (get_user(node, nodes + i))
  1497. goto out_flush;
  1498. addr = (unsigned long)untagged_addr(p);
  1499. err = -ENODEV;
  1500. if (node < 0 || node >= MAX_NUMNODES)
  1501. goto out_flush;
  1502. if (!node_state(node, N_MEMORY))
  1503. goto out_flush;
  1504. err = -EACCES;
  1505. if (!node_isset(node, task_nodes))
  1506. goto out_flush;
  1507. if (current_node == NUMA_NO_NODE) {
  1508. current_node = node;
  1509. start = i;
  1510. } else if (node != current_node) {
  1511. err = move_pages_and_store_status(mm, current_node,
  1512. &pagelist, status, start, i, nr_pages);
  1513. if (err)
  1514. goto out;
  1515. start = i;
  1516. current_node = node;
  1517. }
  1518. /*
  1519. * Errors in the page lookup or isolation are not fatal and we simply
  1520. * report them via status
  1521. */
  1522. err = add_page_for_migration(mm, addr, current_node,
  1523. &pagelist, flags & MPOL_MF_MOVE_ALL);
  1524. if (err > 0) {
  1525. /* The page is successfully queued for migration */
  1526. continue;
  1527. }
  1528. /*
  1529. * If the page is already on the target node (!err), store the
  1530. * node, otherwise, store the err.
  1531. */
  1532. err = store_status(status, i, err ? : current_node, 1);
  1533. if (err)
  1534. goto out_flush;
  1535. err = move_pages_and_store_status(mm, current_node, &pagelist,
  1536. status, start, i, nr_pages);
  1537. if (err)
  1538. goto out;
  1539. current_node = NUMA_NO_NODE;
  1540. }
  1541. out_flush:
  1542. /* Make sure we do not overwrite the existing error */
  1543. err1 = move_pages_and_store_status(mm, current_node, &pagelist,
  1544. status, start, i, nr_pages);
  1545. if (err >= 0)
  1546. err = err1;
  1547. out:
  1548. lru_cache_enable();
  1549. return err;
  1550. }
  1551. /*
  1552. * Determine the nodes of an array of pages and store it in an array of status.
  1553. */
  1554. static void do_pages_stat_array(struct mm_struct *mm, unsigned long nr_pages,
  1555. const void __user **pages, int *status)
  1556. {
  1557. unsigned long i;
  1558. mmap_read_lock(mm);
  1559. for (i = 0; i < nr_pages; i++) {
  1560. unsigned long addr = (unsigned long)(*pages);
  1561. struct vm_area_struct *vma;
  1562. struct page *page;
  1563. int err = -EFAULT;
  1564. vma = find_vma(mm, addr);
  1565. if (!vma || addr < vma->vm_start)
  1566. goto set_status;
  1567. /* FOLL_DUMP to ignore special (like zero) pages */
  1568. page = follow_page(vma, addr, FOLL_DUMP);
  1569. err = PTR_ERR(page);
  1570. if (IS_ERR(page))
  1571. goto set_status;
  1572. err = page ? page_to_nid(page) : -ENOENT;
  1573. set_status:
  1574. *status = err;
  1575. pages++;
  1576. status++;
  1577. }
  1578. mmap_read_unlock(mm);
  1579. }
  1580. /*
  1581. * Determine the nodes of a user array of pages and store it in
  1582. * a user array of status.
  1583. */
  1584. static int do_pages_stat(struct mm_struct *mm, unsigned long nr_pages,
  1585. const void __user * __user *pages,
  1586. int __user *status)
  1587. {
  1588. #define DO_PAGES_STAT_CHUNK_NR 16
  1589. const void __user *chunk_pages[DO_PAGES_STAT_CHUNK_NR];
  1590. int chunk_status[DO_PAGES_STAT_CHUNK_NR];
  1591. while (nr_pages) {
  1592. unsigned long chunk_nr;
  1593. chunk_nr = nr_pages;
  1594. if (chunk_nr > DO_PAGES_STAT_CHUNK_NR)
  1595. chunk_nr = DO_PAGES_STAT_CHUNK_NR;
  1596. if (copy_from_user(chunk_pages, pages, chunk_nr * sizeof(*chunk_pages)))
  1597. break;
  1598. do_pages_stat_array(mm, chunk_nr, chunk_pages, chunk_status);
  1599. if (copy_to_user(status, chunk_status, chunk_nr * sizeof(*status)))
  1600. break;
  1601. pages += chunk_nr;
  1602. status += chunk_nr;
  1603. nr_pages -= chunk_nr;
  1604. }
  1605. return nr_pages ? -EFAULT : 0;
  1606. }
  1607. static struct mm_struct *find_mm_struct(pid_t pid, nodemask_t *mem_nodes)
  1608. {
  1609. struct task_struct *task;
  1610. struct mm_struct *mm;
  1611. /*
  1612. * There is no need to check if current process has the right to modify
  1613. * the specified process when they are same.
  1614. */
  1615. if (!pid) {
  1616. mmget(current->mm);
  1617. *mem_nodes = cpuset_mems_allowed(current);
  1618. return current->mm;
  1619. }
  1620. /* Find the mm_struct */
  1621. rcu_read_lock();
  1622. task = find_task_by_vpid(pid);
  1623. if (!task) {
  1624. rcu_read_unlock();
  1625. return ERR_PTR(-ESRCH);
  1626. }
  1627. get_task_struct(task);
  1628. /*
  1629. * Check if this process has the right to modify the specified
  1630. * process. Use the regular "ptrace_may_access()" checks.
  1631. */
  1632. if (!ptrace_may_access(task, PTRACE_MODE_READ_REALCREDS)) {
  1633. rcu_read_unlock();
  1634. mm = ERR_PTR(-EPERM);
  1635. goto out;
  1636. }
  1637. rcu_read_unlock();
  1638. mm = ERR_PTR(security_task_movememory(task));
  1639. if (IS_ERR(mm))
  1640. goto out;
  1641. *mem_nodes = cpuset_mems_allowed(task);
  1642. mm = get_task_mm(task);
  1643. out:
  1644. put_task_struct(task);
  1645. if (!mm)
  1646. mm = ERR_PTR(-EINVAL);
  1647. return mm;
  1648. }
  1649. /*
  1650. * Move a list of pages in the address space of the currently executing
  1651. * process.
  1652. */
  1653. static int kernel_move_pages(pid_t pid, unsigned long nr_pages,
  1654. const void __user * __user *pages,
  1655. const int __user *nodes,
  1656. int __user *status, int flags)
  1657. {
  1658. struct mm_struct *mm;
  1659. int err;
  1660. nodemask_t task_nodes;
  1661. /* Check flags */
  1662. if (flags & ~(MPOL_MF_MOVE|MPOL_MF_MOVE_ALL))
  1663. return -EINVAL;
  1664. if ((flags & MPOL_MF_MOVE_ALL) && !capable(CAP_SYS_NICE))
  1665. return -EPERM;
  1666. mm = find_mm_struct(pid, &task_nodes);
  1667. if (IS_ERR(mm))
  1668. return PTR_ERR(mm);
  1669. if (nodes)
  1670. err = do_pages_move(mm, task_nodes, nr_pages, pages,
  1671. nodes, status, flags);
  1672. else
  1673. err = do_pages_stat(mm, nr_pages, pages, status);
  1674. mmput(mm);
  1675. return err;
  1676. }
  1677. SYSCALL_DEFINE6(move_pages, pid_t, pid, unsigned long, nr_pages,
  1678. const void __user * __user *, pages,
  1679. const int __user *, nodes,
  1680. int __user *, status, int, flags)
  1681. {
  1682. return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
  1683. }
  1684. #ifdef CONFIG_COMPAT
  1685. COMPAT_SYSCALL_DEFINE6(move_pages, pid_t, pid, compat_ulong_t, nr_pages,
  1686. compat_uptr_t __user *, pages32,
  1687. const int __user *, nodes,
  1688. int __user *, status,
  1689. int, flags)
  1690. {
  1691. const void __user * __user *pages;
  1692. int i;
  1693. pages = compat_alloc_user_space(nr_pages * sizeof(void *));
  1694. for (i = 0; i < nr_pages; i++) {
  1695. compat_uptr_t p;
  1696. if (get_user(p, pages32 + i) ||
  1697. put_user(compat_ptr(p), pages + i))
  1698. return -EFAULT;
  1699. }
  1700. return kernel_move_pages(pid, nr_pages, pages, nodes, status, flags);
  1701. }
  1702. #endif /* CONFIG_COMPAT */
  1703. #ifdef CONFIG_NUMA_BALANCING
  1704. /*
  1705. * Returns true if this is a safe migration target node for misplaced NUMA
  1706. * pages. Currently it only checks the watermarks which crude
  1707. */
  1708. static bool migrate_balanced_pgdat(struct pglist_data *pgdat,
  1709. unsigned long nr_migrate_pages)
  1710. {
  1711. int z;
  1712. for (z = pgdat->nr_zones - 1; z >= 0; z--) {
  1713. struct zone *zone = pgdat->node_zones + z;
  1714. if (!populated_zone(zone))
  1715. continue;
  1716. /* Avoid waking kswapd by allocating pages_to_migrate pages. */
  1717. if (!zone_watermark_ok(zone, 0,
  1718. high_wmark_pages(zone) +
  1719. nr_migrate_pages,
  1720. ZONE_MOVABLE, 0))
  1721. continue;
  1722. return true;
  1723. }
  1724. return false;
  1725. }
  1726. static struct page *alloc_misplaced_dst_page(struct page *page,
  1727. unsigned long data)
  1728. {
  1729. int nid = (int) data;
  1730. struct page *newpage;
  1731. newpage = __alloc_pages_node(nid,
  1732. (GFP_HIGHUSER_MOVABLE |
  1733. __GFP_THISNODE | __GFP_NOMEMALLOC |
  1734. __GFP_NORETRY | __GFP_NOWARN) &
  1735. ~__GFP_RECLAIM, 0);
  1736. return newpage;
  1737. }
  1738. static int numamigrate_isolate_page(pg_data_t *pgdat, struct page *page)
  1739. {
  1740. int page_lru;
  1741. VM_BUG_ON_PAGE(compound_order(page) && !PageTransHuge(page), page);
  1742. /* Avoid migrating to a node that is nearly full */
  1743. if (!migrate_balanced_pgdat(pgdat, compound_nr(page)))
  1744. return 0;
  1745. if (isolate_lru_page(page))
  1746. return 0;
  1747. /*
  1748. * migrate_misplaced_transhuge_page() skips page migration's usual
  1749. * check on page_count(), so we must do it here, now that the page
  1750. * has been isolated: a GUP pin, or any other pin, prevents migration.
  1751. * The expected page count is 3: 1 for page's mapcount and 1 for the
  1752. * caller's pin and 1 for the reference taken by isolate_lru_page().
  1753. */
  1754. if (PageTransHuge(page) && page_count(page) != 3) {
  1755. putback_lru_page(page);
  1756. return 0;
  1757. }
  1758. page_lru = page_is_file_lru(page);
  1759. mod_node_page_state(page_pgdat(page), NR_ISOLATED_ANON + page_lru,
  1760. thp_nr_pages(page));
  1761. /*
  1762. * Isolating the page has taken another reference, so the
  1763. * caller's reference can be safely dropped without the page
  1764. * disappearing underneath us during migration.
  1765. */
  1766. put_page(page);
  1767. return 1;
  1768. }
  1769. bool pmd_trans_migrating(pmd_t pmd)
  1770. {
  1771. struct page *page = pmd_page(pmd);
  1772. return PageLocked(page);
  1773. }
  1774. /*
  1775. * Attempt to migrate a misplaced page to the specified destination
  1776. * node. Caller is expected to have an elevated reference count on
  1777. * the page that will be dropped by this function before returning.
  1778. */
  1779. int migrate_misplaced_page(struct page *page, struct vm_fault *vmf,
  1780. int node)
  1781. {
  1782. pg_data_t *pgdat = NODE_DATA(node);
  1783. int isolated;
  1784. int nr_remaining;
  1785. LIST_HEAD(migratepages);
  1786. /*
  1787. * Don't migrate file pages that are mapped in multiple processes
  1788. * with execute permissions as they are probably shared libraries.
  1789. */
  1790. if (page_mapcount(page) != 1 && page_is_file_lru(page) &&
  1791. (vmf->vma_flags & VM_EXEC))
  1792. goto out;
  1793. /*
  1794. * Also do not migrate dirty pages as not all filesystems can move
  1795. * dirty pages in MIGRATE_ASYNC mode which is a waste of cycles.
  1796. */
  1797. if (page_is_file_lru(page) && PageDirty(page))
  1798. goto out;
  1799. isolated = numamigrate_isolate_page(pgdat, page);
  1800. if (!isolated)
  1801. goto out;
  1802. list_add(&page->lru, &migratepages);
  1803. nr_remaining = migrate_pages(&migratepages, alloc_misplaced_dst_page,
  1804. NULL, node, MIGRATE_ASYNC,
  1805. MR_NUMA_MISPLACED);
  1806. if (nr_remaining) {
  1807. if (!list_empty(&migratepages)) {
  1808. list_del(&page->lru);
  1809. dec_node_page_state(page, NR_ISOLATED_ANON +
  1810. page_is_file_lru(page));
  1811. putback_lru_page(page);
  1812. }
  1813. isolated = 0;
  1814. } else
  1815. count_vm_numa_event(NUMA_PAGE_MIGRATE);
  1816. BUG_ON(!list_empty(&migratepages));
  1817. return isolated;
  1818. out:
  1819. put_page(page);
  1820. return 0;
  1821. }
  1822. #endif /* CONFIG_NUMA_BALANCING */
  1823. #if defined(CONFIG_NUMA_BALANCING) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
  1824. /*
  1825. * Migrates a THP to a given target node. page must be locked and is unlocked
  1826. * before returning.
  1827. */
  1828. int migrate_misplaced_transhuge_page(struct mm_struct *mm,
  1829. struct vm_area_struct *vma,
  1830. pmd_t *pmd, pmd_t entry,
  1831. unsigned long address,
  1832. struct page *page, int node)
  1833. {
  1834. spinlock_t *ptl;
  1835. pg_data_t *pgdat = NODE_DATA(node);
  1836. int isolated = 0;
  1837. struct page *new_page = NULL;
  1838. int page_lru = page_is_file_lru(page);
  1839. unsigned long start = address & HPAGE_PMD_MASK;
  1840. new_page = alloc_pages_node(node,
  1841. (GFP_TRANSHUGE_LIGHT | __GFP_THISNODE),
  1842. HPAGE_PMD_ORDER);
  1843. if (!new_page)
  1844. goto out_fail;
  1845. prep_transhuge_page(new_page);
  1846. isolated = numamigrate_isolate_page(pgdat, page);
  1847. if (!isolated) {
  1848. put_page(new_page);
  1849. goto out_fail;
  1850. }
  1851. /* Prepare a page as a migration target */
  1852. __SetPageLocked(new_page);
  1853. if (PageSwapBacked(page))
  1854. __SetPageSwapBacked(new_page);
  1855. /* anon mapping, we can simply copy page->mapping to the new page: */
  1856. new_page->mapping = page->mapping;
  1857. new_page->index = page->index;
  1858. /* flush the cache before copying using the kernel virtual address */
  1859. flush_cache_range(vma, start, start + HPAGE_PMD_SIZE);
  1860. migrate_page_copy(new_page, page);
  1861. WARN_ON(PageLRU(new_page));
  1862. /* Recheck the target PMD */
  1863. ptl = pmd_lock(mm, pmd);
  1864. if (unlikely(!pmd_same(*pmd, entry) || !page_ref_freeze(page, 2))) {
  1865. spin_unlock(ptl);
  1866. /* Reverse changes made by migrate_page_copy() */
  1867. if (TestClearPageActive(new_page))
  1868. SetPageActive(page);
  1869. if (TestClearPageUnevictable(new_page))
  1870. SetPageUnevictable(page);
  1871. unlock_page(new_page);
  1872. put_page(new_page); /* Free it */
  1873. /* Retake the callers reference and putback on LRU */
  1874. get_page(page);
  1875. putback_lru_page(page);
  1876. mod_node_page_state(page_pgdat(page),
  1877. NR_ISOLATED_ANON + page_lru, -HPAGE_PMD_NR);
  1878. goto out_unlock;
  1879. }
  1880. entry = mk_huge_pmd(new_page, vma->vm_page_prot);
  1881. entry = maybe_pmd_mkwrite(pmd_mkdirty(entry), vma);
  1882. /*
  1883. * Overwrite the old entry under pagetable lock and establish
  1884. * the new PTE. Any parallel GUP will either observe the old
  1885. * page blocking on the page lock, block on the page table
  1886. * lock or observe the new page. The SetPageUptodate on the
  1887. * new page and page_add_new_anon_rmap guarantee the copy is
  1888. * visible before the pagetable update.
  1889. */
  1890. page_add_anon_rmap(new_page, vma, start, true);
  1891. /*
  1892. * At this point the pmd is numa/protnone (i.e. non present) and the TLB
  1893. * has already been flushed globally. So no TLB can be currently
  1894. * caching this non present pmd mapping. There's no need to clear the
  1895. * pmd before doing set_pmd_at(), nor to flush the TLB after
  1896. * set_pmd_at(). Clearing the pmd here would introduce a race
  1897. * condition against MADV_DONTNEED, because MADV_DONTNEED only holds the
  1898. * mmap_lock for reading. If the pmd is set to NULL at any given time,
  1899. * MADV_DONTNEED won't wait on the pmd lock and it'll skip clearing this
  1900. * pmd.
  1901. */
  1902. set_pmd_at(mm, start, pmd, entry);
  1903. update_mmu_cache_pmd(vma, address, &entry);
  1904. page_ref_unfreeze(page, 2);
  1905. mlock_migrate_page(new_page, page);
  1906. page_remove_rmap(page, true);
  1907. set_page_owner_migrate_reason(new_page, MR_NUMA_MISPLACED);
  1908. spin_unlock(ptl);
  1909. /* Take an "isolate" reference and put new page on the LRU. */
  1910. get_page(new_page);
  1911. putback_lru_page(new_page);
  1912. unlock_page(new_page);
  1913. unlock_page(page);
  1914. put_page(page); /* Drop the rmap reference */
  1915. put_page(page); /* Drop the LRU isolation reference */
  1916. count_vm_events(PGMIGRATE_SUCCESS, HPAGE_PMD_NR);
  1917. count_vm_numa_events(NUMA_PAGE_MIGRATE, HPAGE_PMD_NR);
  1918. mod_node_page_state(page_pgdat(page),
  1919. NR_ISOLATED_ANON + page_lru,
  1920. -HPAGE_PMD_NR);
  1921. return isolated;
  1922. out_fail:
  1923. count_vm_events(PGMIGRATE_FAIL, HPAGE_PMD_NR);
  1924. ptl = pmd_lock(mm, pmd);
  1925. if (pmd_same(*pmd, entry)) {
  1926. entry = pmd_modify(entry, vma->vm_page_prot);
  1927. set_pmd_at(mm, start, pmd, entry);
  1928. update_mmu_cache_pmd(vma, address, &entry);
  1929. }
  1930. spin_unlock(ptl);
  1931. out_unlock:
  1932. unlock_page(page);
  1933. put_page(page);
  1934. return 0;
  1935. }
  1936. #endif /* CONFIG_NUMA_BALANCING */
  1937. #endif /* CONFIG_NUMA */
  1938. #ifdef CONFIG_DEVICE_PRIVATE
  1939. static int migrate_vma_collect_hole(unsigned long start,
  1940. unsigned long end,
  1941. __always_unused int depth,
  1942. struct mm_walk *walk)
  1943. {
  1944. struct migrate_vma *migrate = walk->private;
  1945. unsigned long addr;
  1946. /* Only allow populating anonymous memory. */
  1947. if (!vma_is_anonymous(walk->vma)) {
  1948. for (addr = start; addr < end; addr += PAGE_SIZE) {
  1949. migrate->src[migrate->npages] = 0;
  1950. migrate->dst[migrate->npages] = 0;
  1951. migrate->npages++;
  1952. }
  1953. return 0;
  1954. }
  1955. for (addr = start; addr < end; addr += PAGE_SIZE) {
  1956. migrate->src[migrate->npages] = MIGRATE_PFN_MIGRATE;
  1957. migrate->dst[migrate->npages] = 0;
  1958. migrate->npages++;
  1959. migrate->cpages++;
  1960. }
  1961. return 0;
  1962. }
  1963. static int migrate_vma_collect_skip(unsigned long start,
  1964. unsigned long end,
  1965. struct mm_walk *walk)
  1966. {
  1967. struct migrate_vma *migrate = walk->private;
  1968. unsigned long addr;
  1969. for (addr = start; addr < end; addr += PAGE_SIZE) {
  1970. migrate->dst[migrate->npages] = 0;
  1971. migrate->src[migrate->npages++] = 0;
  1972. }
  1973. return 0;
  1974. }
  1975. static int migrate_vma_collect_pmd(pmd_t *pmdp,
  1976. unsigned long start,
  1977. unsigned long end,
  1978. struct mm_walk *walk)
  1979. {
  1980. struct migrate_vma *migrate = walk->private;
  1981. struct vm_area_struct *vma = walk->vma;
  1982. struct mm_struct *mm = vma->vm_mm;
  1983. unsigned long addr = start, unmapped = 0;
  1984. spinlock_t *ptl;
  1985. pte_t *ptep;
  1986. again:
  1987. if (pmd_none(*pmdp))
  1988. return migrate_vma_collect_hole(start, end, -1, walk);
  1989. if (pmd_trans_huge(*pmdp)) {
  1990. struct page *page;
  1991. ptl = pmd_lock(mm, pmdp);
  1992. if (unlikely(!pmd_trans_huge(*pmdp))) {
  1993. spin_unlock(ptl);
  1994. goto again;
  1995. }
  1996. page = pmd_page(*pmdp);
  1997. if (is_huge_zero_page(page)) {
  1998. spin_unlock(ptl);
  1999. split_huge_pmd(vma, pmdp, addr);
  2000. if (pmd_trans_unstable(pmdp))
  2001. return migrate_vma_collect_skip(start, end,
  2002. walk);
  2003. } else {
  2004. int ret;
  2005. get_page(page);
  2006. spin_unlock(ptl);
  2007. if (unlikely(!trylock_page(page)))
  2008. return migrate_vma_collect_skip(start, end,
  2009. walk);
  2010. ret = split_huge_page(page);
  2011. unlock_page(page);
  2012. put_page(page);
  2013. if (ret)
  2014. return migrate_vma_collect_skip(start, end,
  2015. walk);
  2016. if (pmd_none(*pmdp))
  2017. return migrate_vma_collect_hole(start, end, -1,
  2018. walk);
  2019. }
  2020. }
  2021. if (unlikely(pmd_bad(*pmdp)))
  2022. return migrate_vma_collect_skip(start, end, walk);
  2023. ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
  2024. arch_enter_lazy_mmu_mode();
  2025. for (; addr < end; addr += PAGE_SIZE, ptep++) {
  2026. unsigned long mpfn = 0, pfn;
  2027. struct page *page;
  2028. swp_entry_t entry;
  2029. pte_t pte;
  2030. pte = *ptep;
  2031. if (pte_none(pte)) {
  2032. if (vma_is_anonymous(vma)) {
  2033. mpfn = MIGRATE_PFN_MIGRATE;
  2034. migrate->cpages++;
  2035. }
  2036. goto next;
  2037. }
  2038. if (!pte_present(pte)) {
  2039. /*
  2040. * Only care about unaddressable device page special
  2041. * page table entry. Other special swap entries are not
  2042. * migratable, and we ignore regular swapped page.
  2043. */
  2044. entry = pte_to_swp_entry(pte);
  2045. if (!is_device_private_entry(entry))
  2046. goto next;
  2047. page = device_private_entry_to_page(entry);
  2048. if (!(migrate->flags &
  2049. MIGRATE_VMA_SELECT_DEVICE_PRIVATE) ||
  2050. page->pgmap->owner != migrate->pgmap_owner)
  2051. goto next;
  2052. mpfn = migrate_pfn(page_to_pfn(page)) |
  2053. MIGRATE_PFN_MIGRATE;
  2054. if (is_write_device_private_entry(entry))
  2055. mpfn |= MIGRATE_PFN_WRITE;
  2056. } else {
  2057. if (!(migrate->flags & MIGRATE_VMA_SELECT_SYSTEM))
  2058. goto next;
  2059. pfn = pte_pfn(pte);
  2060. if (is_zero_pfn(pfn)) {
  2061. mpfn = MIGRATE_PFN_MIGRATE;
  2062. migrate->cpages++;
  2063. goto next;
  2064. }
  2065. page = vm_normal_page(migrate->vma, addr, pte);
  2066. mpfn = migrate_pfn(pfn) | MIGRATE_PFN_MIGRATE;
  2067. mpfn |= pte_write(pte) ? MIGRATE_PFN_WRITE : 0;
  2068. }
  2069. /* FIXME support THP */
  2070. if (!page || !page->mapping || PageTransCompound(page)) {
  2071. mpfn = 0;
  2072. goto next;
  2073. }
  2074. /*
  2075. * By getting a reference on the page we pin it and that blocks
  2076. * any kind of migration. Side effect is that it "freezes" the
  2077. * pte.
  2078. *
  2079. * We drop this reference after isolating the page from the lru
  2080. * for non device page (device page are not on the lru and thus
  2081. * can't be dropped from it).
  2082. */
  2083. get_page(page);
  2084. migrate->cpages++;
  2085. /*
  2086. * Optimize for the common case where page is only mapped once
  2087. * in one process. If we can lock the page, then we can safely
  2088. * set up a special migration page table entry now.
  2089. */
  2090. if (trylock_page(page)) {
  2091. pte_t swp_pte;
  2092. mpfn |= MIGRATE_PFN_LOCKED;
  2093. ptep_get_and_clear(mm, addr, ptep);
  2094. /* Setup special migration page table entry */
  2095. entry = make_migration_entry(page, mpfn &
  2096. MIGRATE_PFN_WRITE);
  2097. swp_pte = swp_entry_to_pte(entry);
  2098. if (pte_present(pte)) {
  2099. if (pte_soft_dirty(pte))
  2100. swp_pte = pte_swp_mksoft_dirty(swp_pte);
  2101. if (pte_uffd_wp(pte))
  2102. swp_pte = pte_swp_mkuffd_wp(swp_pte);
  2103. } else {
  2104. if (pte_swp_soft_dirty(pte))
  2105. swp_pte = pte_swp_mksoft_dirty(swp_pte);
  2106. if (pte_swp_uffd_wp(pte))
  2107. swp_pte = pte_swp_mkuffd_wp(swp_pte);
  2108. }
  2109. set_pte_at(mm, addr, ptep, swp_pte);
  2110. /*
  2111. * This is like regular unmap: we remove the rmap and
  2112. * drop page refcount. Page won't be freed, as we took
  2113. * a reference just above.
  2114. */
  2115. page_remove_rmap(page, false);
  2116. put_page(page);
  2117. if (pte_present(pte))
  2118. unmapped++;
  2119. }
  2120. next:
  2121. migrate->dst[migrate->npages] = 0;
  2122. migrate->src[migrate->npages++] = mpfn;
  2123. }
  2124. arch_leave_lazy_mmu_mode();
  2125. pte_unmap_unlock(ptep - 1, ptl);
  2126. /* Only flush the TLB if we actually modified any entries */
  2127. if (unmapped)
  2128. flush_tlb_range(walk->vma, start, end);
  2129. return 0;
  2130. }
  2131. static const struct mm_walk_ops migrate_vma_walk_ops = {
  2132. .pmd_entry = migrate_vma_collect_pmd,
  2133. .pte_hole = migrate_vma_collect_hole,
  2134. };
  2135. /*
  2136. * migrate_vma_collect() - collect pages over a range of virtual addresses
  2137. * @migrate: migrate struct containing all migration information
  2138. *
  2139. * This will walk the CPU page table. For each virtual address backed by a
  2140. * valid page, it updates the src array and takes a reference on the page, in
  2141. * order to pin the page until we lock it and unmap it.
  2142. */
  2143. static void migrate_vma_collect(struct migrate_vma *migrate)
  2144. {
  2145. struct mmu_notifier_range range;
  2146. /*
  2147. * Note that the pgmap_owner is passed to the mmu notifier callback so
  2148. * that the registered device driver can skip invalidating device
  2149. * private page mappings that won't be migrated.
  2150. */
  2151. mmu_notifier_range_init_migrate(&range, 0, migrate->vma,
  2152. migrate->vma->vm_mm, migrate->start, migrate->end,
  2153. migrate->pgmap_owner);
  2154. mmu_notifier_invalidate_range_start(&range);
  2155. walk_page_range(migrate->vma->vm_mm, migrate->start, migrate->end,
  2156. &migrate_vma_walk_ops, migrate);
  2157. mmu_notifier_invalidate_range_end(&range);
  2158. migrate->end = migrate->start + (migrate->npages << PAGE_SHIFT);
  2159. }
  2160. /*
  2161. * migrate_vma_check_page() - check if page is pinned or not
  2162. * @page: struct page to check
  2163. *
  2164. * Pinned pages cannot be migrated. This is the same test as in
  2165. * migrate_page_move_mapping(), except that here we allow migration of a
  2166. * ZONE_DEVICE page.
  2167. */
  2168. static bool migrate_vma_check_page(struct page *page)
  2169. {
  2170. /*
  2171. * One extra ref because caller holds an extra reference, either from
  2172. * isolate_lru_page() for a regular page, or migrate_vma_collect() for
  2173. * a device page.
  2174. */
  2175. int extra = 1;
  2176. /*
  2177. * FIXME support THP (transparent huge page), it is bit more complex to
  2178. * check them than regular pages, because they can be mapped with a pmd
  2179. * or with a pte (split pte mapping).
  2180. */
  2181. if (PageCompound(page))
  2182. return false;
  2183. /* Page from ZONE_DEVICE have one extra reference */
  2184. if (is_zone_device_page(page)) {
  2185. /*
  2186. * Private page can never be pin as they have no valid pte and
  2187. * GUP will fail for those. Yet if there is a pending migration
  2188. * a thread might try to wait on the pte migration entry and
  2189. * will bump the page reference count. Sadly there is no way to
  2190. * differentiate a regular pin from migration wait. Hence to
  2191. * avoid 2 racing thread trying to migrate back to CPU to enter
  2192. * infinite loop (one stoping migration because the other is
  2193. * waiting on pte migration entry). We always return true here.
  2194. *
  2195. * FIXME proper solution is to rework migration_entry_wait() so
  2196. * it does not need to take a reference on page.
  2197. */
  2198. return is_device_private_page(page);
  2199. }
  2200. /* For file back page */
  2201. if (page_mapping(page))
  2202. extra += 1 + page_has_private(page);
  2203. if ((page_count(page) - extra) > page_mapcount(page))
  2204. return false;
  2205. return true;
  2206. }
  2207. /*
  2208. * migrate_vma_prepare() - lock pages and isolate them from the lru
  2209. * @migrate: migrate struct containing all migration information
  2210. *
  2211. * This locks pages that have been collected by migrate_vma_collect(). Once each
  2212. * page is locked it is isolated from the lru (for non-device pages). Finally,
  2213. * the ref taken by migrate_vma_collect() is dropped, as locked pages cannot be
  2214. * migrated by concurrent kernel threads.
  2215. */
  2216. static void migrate_vma_prepare(struct migrate_vma *migrate)
  2217. {
  2218. const unsigned long npages = migrate->npages;
  2219. const unsigned long start = migrate->start;
  2220. unsigned long addr, i, restore = 0;
  2221. bool allow_drain = true;
  2222. lru_add_drain();
  2223. for (i = 0; (i < npages) && migrate->cpages; i++) {
  2224. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2225. bool remap = true;
  2226. if (!page)
  2227. continue;
  2228. if (!(migrate->src[i] & MIGRATE_PFN_LOCKED)) {
  2229. /*
  2230. * Because we are migrating several pages there can be
  2231. * a deadlock between 2 concurrent migration where each
  2232. * are waiting on each other page lock.
  2233. *
  2234. * Make migrate_vma() a best effort thing and backoff
  2235. * for any page we can not lock right away.
  2236. */
  2237. if (!trylock_page(page)) {
  2238. migrate->src[i] = 0;
  2239. migrate->cpages--;
  2240. put_page(page);
  2241. continue;
  2242. }
  2243. remap = false;
  2244. migrate->src[i] |= MIGRATE_PFN_LOCKED;
  2245. }
  2246. /* ZONE_DEVICE pages are not on LRU */
  2247. if (!is_zone_device_page(page)) {
  2248. if (!PageLRU(page) && allow_drain) {
  2249. /* Drain CPU's pagevec */
  2250. lru_add_drain_all();
  2251. allow_drain = false;
  2252. }
  2253. if (isolate_lru_page(page)) {
  2254. if (remap) {
  2255. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2256. migrate->cpages--;
  2257. restore++;
  2258. } else {
  2259. migrate->src[i] = 0;
  2260. unlock_page(page);
  2261. migrate->cpages--;
  2262. put_page(page);
  2263. }
  2264. continue;
  2265. }
  2266. /* Drop the reference we took in collect */
  2267. put_page(page);
  2268. }
  2269. if (!migrate_vma_check_page(page)) {
  2270. if (remap) {
  2271. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2272. migrate->cpages--;
  2273. restore++;
  2274. if (!is_zone_device_page(page)) {
  2275. get_page(page);
  2276. putback_lru_page(page);
  2277. }
  2278. } else {
  2279. migrate->src[i] = 0;
  2280. unlock_page(page);
  2281. migrate->cpages--;
  2282. if (!is_zone_device_page(page))
  2283. putback_lru_page(page);
  2284. else
  2285. put_page(page);
  2286. }
  2287. }
  2288. }
  2289. for (i = 0, addr = start; i < npages && restore; i++, addr += PAGE_SIZE) {
  2290. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2291. if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
  2292. continue;
  2293. remove_migration_pte(page, migrate->vma, addr, page);
  2294. migrate->src[i] = 0;
  2295. unlock_page(page);
  2296. put_page(page);
  2297. restore--;
  2298. }
  2299. }
  2300. /*
  2301. * migrate_vma_unmap() - replace page mapping with special migration pte entry
  2302. * @migrate: migrate struct containing all migration information
  2303. *
  2304. * Replace page mapping (CPU page table pte) with a special migration pte entry
  2305. * and check again if it has been pinned. Pinned pages are restored because we
  2306. * cannot migrate them.
  2307. *
  2308. * This is the last step before we call the device driver callback to allocate
  2309. * destination memory and copy contents of original page over to new page.
  2310. */
  2311. static void migrate_vma_unmap(struct migrate_vma *migrate)
  2312. {
  2313. int flags = TTU_MIGRATION | TTU_IGNORE_MLOCK;
  2314. const unsigned long npages = migrate->npages;
  2315. const unsigned long start = migrate->start;
  2316. unsigned long addr, i, restore = 0;
  2317. for (i = 0; i < npages; i++) {
  2318. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2319. if (!page || !(migrate->src[i] & MIGRATE_PFN_MIGRATE))
  2320. continue;
  2321. if (page_mapped(page)) {
  2322. try_to_unmap(page, flags);
  2323. if (page_mapped(page))
  2324. goto restore;
  2325. }
  2326. if (migrate_vma_check_page(page))
  2327. continue;
  2328. restore:
  2329. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2330. migrate->cpages--;
  2331. restore++;
  2332. }
  2333. for (addr = start, i = 0; i < npages && restore; addr += PAGE_SIZE, i++) {
  2334. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2335. if (!page || (migrate->src[i] & MIGRATE_PFN_MIGRATE))
  2336. continue;
  2337. remove_migration_ptes(page, page, false);
  2338. migrate->src[i] = 0;
  2339. unlock_page(page);
  2340. restore--;
  2341. if (is_zone_device_page(page))
  2342. put_page(page);
  2343. else
  2344. putback_lru_page(page);
  2345. }
  2346. }
  2347. /**
  2348. * migrate_vma_setup() - prepare to migrate a range of memory
  2349. * @args: contains the vma, start, and pfns arrays for the migration
  2350. *
  2351. * Returns: negative errno on failures, 0 when 0 or more pages were migrated
  2352. * without an error.
  2353. *
  2354. * Prepare to migrate a range of memory virtual address range by collecting all
  2355. * the pages backing each virtual address in the range, saving them inside the
  2356. * src array. Then lock those pages and unmap them. Once the pages are locked
  2357. * and unmapped, check whether each page is pinned or not. Pages that aren't
  2358. * pinned have the MIGRATE_PFN_MIGRATE flag set (by this function) in the
  2359. * corresponding src array entry. Then restores any pages that are pinned, by
  2360. * remapping and unlocking those pages.
  2361. *
  2362. * The caller should then allocate destination memory and copy source memory to
  2363. * it for all those entries (ie with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE
  2364. * flag set). Once these are allocated and copied, the caller must update each
  2365. * corresponding entry in the dst array with the pfn value of the destination
  2366. * page and with the MIGRATE_PFN_VALID and MIGRATE_PFN_LOCKED flags set
  2367. * (destination pages must have their struct pages locked, via lock_page()).
  2368. *
  2369. * Note that the caller does not have to migrate all the pages that are marked
  2370. * with MIGRATE_PFN_MIGRATE flag in src array unless this is a migration from
  2371. * device memory to system memory. If the caller cannot migrate a device page
  2372. * back to system memory, then it must return VM_FAULT_SIGBUS, which has severe
  2373. * consequences for the userspace process, so it must be avoided if at all
  2374. * possible.
  2375. *
  2376. * For empty entries inside CPU page table (pte_none() or pmd_none() is true) we
  2377. * do set MIGRATE_PFN_MIGRATE flag inside the corresponding source array thus
  2378. * allowing the caller to allocate device memory for those unback virtual
  2379. * address. For this the caller simply has to allocate device memory and
  2380. * properly set the destination entry like for regular migration. Note that
  2381. * this can still fails and thus inside the device driver must check if the
  2382. * migration was successful for those entries after calling migrate_vma_pages()
  2383. * just like for regular migration.
  2384. *
  2385. * After that, the callers must call migrate_vma_pages() to go over each entry
  2386. * in the src array that has the MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag
  2387. * set. If the corresponding entry in dst array has MIGRATE_PFN_VALID flag set,
  2388. * then migrate_vma_pages() to migrate struct page information from the source
  2389. * struct page to the destination struct page. If it fails to migrate the
  2390. * struct page information, then it clears the MIGRATE_PFN_MIGRATE flag in the
  2391. * src array.
  2392. *
  2393. * At this point all successfully migrated pages have an entry in the src
  2394. * array with MIGRATE_PFN_VALID and MIGRATE_PFN_MIGRATE flag set and the dst
  2395. * array entry with MIGRATE_PFN_VALID flag set.
  2396. *
  2397. * Once migrate_vma_pages() returns the caller may inspect which pages were
  2398. * successfully migrated, and which were not. Successfully migrated pages will
  2399. * have the MIGRATE_PFN_MIGRATE flag set for their src array entry.
  2400. *
  2401. * It is safe to update device page table after migrate_vma_pages() because
  2402. * both destination and source page are still locked, and the mmap_lock is held
  2403. * in read mode (hence no one can unmap the range being migrated).
  2404. *
  2405. * Once the caller is done cleaning up things and updating its page table (if it
  2406. * chose to do so, this is not an obligation) it finally calls
  2407. * migrate_vma_finalize() to update the CPU page table to point to new pages
  2408. * for successfully migrated pages or otherwise restore the CPU page table to
  2409. * point to the original source pages.
  2410. */
  2411. int migrate_vma_setup(struct migrate_vma *args)
  2412. {
  2413. long nr_pages = (args->end - args->start) >> PAGE_SHIFT;
  2414. args->start &= PAGE_MASK;
  2415. args->end &= PAGE_MASK;
  2416. if (!args->vma || is_vm_hugetlb_page(args->vma) ||
  2417. (args->vma->vm_flags & VM_SPECIAL) || vma_is_dax(args->vma))
  2418. return -EINVAL;
  2419. if (nr_pages <= 0)
  2420. return -EINVAL;
  2421. if (args->start < args->vma->vm_start ||
  2422. args->start >= args->vma->vm_end)
  2423. return -EINVAL;
  2424. if (args->end <= args->vma->vm_start || args->end > args->vma->vm_end)
  2425. return -EINVAL;
  2426. if (!args->src || !args->dst)
  2427. return -EINVAL;
  2428. memset(args->src, 0, sizeof(*args->src) * nr_pages);
  2429. args->cpages = 0;
  2430. args->npages = 0;
  2431. migrate_vma_collect(args);
  2432. if (args->cpages)
  2433. migrate_vma_prepare(args);
  2434. if (args->cpages)
  2435. migrate_vma_unmap(args);
  2436. /*
  2437. * At this point pages are locked and unmapped, and thus they have
  2438. * stable content and can safely be copied to destination memory that
  2439. * is allocated by the drivers.
  2440. */
  2441. return 0;
  2442. }
  2443. EXPORT_SYMBOL(migrate_vma_setup);
  2444. /*
  2445. * This code closely matches the code in:
  2446. * __handle_mm_fault()
  2447. * handle_pte_fault()
  2448. * do_anonymous_page()
  2449. * to map in an anonymous zero page but the struct page will be a ZONE_DEVICE
  2450. * private page.
  2451. */
  2452. static void migrate_vma_insert_page(struct migrate_vma *migrate,
  2453. unsigned long addr,
  2454. struct page *page,
  2455. unsigned long *src,
  2456. unsigned long *dst)
  2457. {
  2458. struct vm_area_struct *vma = migrate->vma;
  2459. struct mm_struct *mm = vma->vm_mm;
  2460. bool flush = false;
  2461. spinlock_t *ptl;
  2462. pte_t entry;
  2463. pgd_t *pgdp;
  2464. p4d_t *p4dp;
  2465. pud_t *pudp;
  2466. pmd_t *pmdp;
  2467. pte_t *ptep;
  2468. /* Only allow populating anonymous memory */
  2469. if (!vma_is_anonymous(vma))
  2470. goto abort;
  2471. pgdp = pgd_offset(mm, addr);
  2472. p4dp = p4d_alloc(mm, pgdp, addr);
  2473. if (!p4dp)
  2474. goto abort;
  2475. pudp = pud_alloc(mm, p4dp, addr);
  2476. if (!pudp)
  2477. goto abort;
  2478. pmdp = pmd_alloc(mm, pudp, addr);
  2479. if (!pmdp)
  2480. goto abort;
  2481. if (pmd_trans_huge(*pmdp) || pmd_devmap(*pmdp))
  2482. goto abort;
  2483. /*
  2484. * Use pte_alloc() instead of pte_alloc_map(). We can't run
  2485. * pte_offset_map() on pmds where a huge pmd might be created
  2486. * from a different thread.
  2487. *
  2488. * pte_alloc_map() is safe to use under mmap_write_lock(mm) or when
  2489. * parallel threads are excluded by other means.
  2490. *
  2491. * Here we only have mmap_read_lock(mm).
  2492. */
  2493. if (pte_alloc(mm, pmdp))
  2494. goto abort;
  2495. /* See the comment in pte_alloc_one_map() */
  2496. if (unlikely(pmd_trans_unstable(pmdp)))
  2497. goto abort;
  2498. if (unlikely(anon_vma_prepare(vma)))
  2499. goto abort;
  2500. if (mem_cgroup_charge(page, vma->vm_mm, GFP_KERNEL))
  2501. goto abort;
  2502. /*
  2503. * The memory barrier inside __SetPageUptodate makes sure that
  2504. * preceding stores to the page contents become visible before
  2505. * the set_pte_at() write.
  2506. */
  2507. __SetPageUptodate(page);
  2508. if (is_zone_device_page(page)) {
  2509. if (is_device_private_page(page)) {
  2510. swp_entry_t swp_entry;
  2511. swp_entry = make_device_private_entry(page, vma->vm_flags & VM_WRITE);
  2512. entry = swp_entry_to_pte(swp_entry);
  2513. } else {
  2514. /*
  2515. * For now we only support migrating to un-addressable
  2516. * device memory.
  2517. */
  2518. pr_warn_once("Unsupported ZONE_DEVICE page type.\n");
  2519. goto abort;
  2520. }
  2521. } else {
  2522. entry = mk_pte(page, vma->vm_page_prot);
  2523. if (vma->vm_flags & VM_WRITE)
  2524. entry = pte_mkwrite(pte_mkdirty(entry));
  2525. }
  2526. ptep = pte_offset_map_lock(mm, pmdp, addr, &ptl);
  2527. if (check_stable_address_space(mm))
  2528. goto unlock_abort;
  2529. if (pte_present(*ptep)) {
  2530. unsigned long pfn = pte_pfn(*ptep);
  2531. if (!is_zero_pfn(pfn))
  2532. goto unlock_abort;
  2533. flush = true;
  2534. } else if (!pte_none(*ptep))
  2535. goto unlock_abort;
  2536. /*
  2537. * Check for userfaultfd but do not deliver the fault. Instead,
  2538. * just back off.
  2539. */
  2540. if (userfaultfd_missing(vma))
  2541. goto unlock_abort;
  2542. inc_mm_counter(mm, MM_ANONPAGES);
  2543. page_add_new_anon_rmap(page, vma, addr, false);
  2544. if (!is_zone_device_page(page))
  2545. lru_cache_add_inactive_or_unevictable(page, vma);
  2546. get_page(page);
  2547. if (flush) {
  2548. flush_cache_page(vma, addr, pte_pfn(*ptep));
  2549. ptep_clear_flush_notify(vma, addr, ptep);
  2550. set_pte_at_notify(mm, addr, ptep, entry);
  2551. update_mmu_cache(vma, addr, ptep);
  2552. } else {
  2553. /* No need to invalidate - it was non-present before */
  2554. set_pte_at(mm, addr, ptep, entry);
  2555. update_mmu_cache(vma, addr, ptep);
  2556. }
  2557. pte_unmap_unlock(ptep, ptl);
  2558. *src = MIGRATE_PFN_MIGRATE;
  2559. return;
  2560. unlock_abort:
  2561. pte_unmap_unlock(ptep, ptl);
  2562. abort:
  2563. *src &= ~MIGRATE_PFN_MIGRATE;
  2564. }
  2565. /**
  2566. * migrate_vma_pages() - migrate meta-data from src page to dst page
  2567. * @migrate: migrate struct containing all migration information
  2568. *
  2569. * This migrates struct page meta-data from source struct page to destination
  2570. * struct page. This effectively finishes the migration from source page to the
  2571. * destination page.
  2572. */
  2573. void migrate_vma_pages(struct migrate_vma *migrate)
  2574. {
  2575. const unsigned long npages = migrate->npages;
  2576. const unsigned long start = migrate->start;
  2577. struct mmu_notifier_range range;
  2578. unsigned long addr, i;
  2579. bool notified = false;
  2580. for (i = 0, addr = start; i < npages; addr += PAGE_SIZE, i++) {
  2581. struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
  2582. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2583. struct address_space *mapping;
  2584. int r;
  2585. if (!newpage) {
  2586. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2587. continue;
  2588. }
  2589. if (!page) {
  2590. if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE))
  2591. continue;
  2592. if (!notified) {
  2593. notified = true;
  2594. mmu_notifier_range_init(&range,
  2595. MMU_NOTIFY_CLEAR, 0,
  2596. NULL,
  2597. migrate->vma->vm_mm,
  2598. addr, migrate->end);
  2599. mmu_notifier_invalidate_range_start(&range);
  2600. }
  2601. migrate_vma_insert_page(migrate, addr, newpage,
  2602. &migrate->src[i],
  2603. &migrate->dst[i]);
  2604. continue;
  2605. }
  2606. mapping = page_mapping(page);
  2607. if (is_zone_device_page(newpage)) {
  2608. if (is_device_private_page(newpage)) {
  2609. /*
  2610. * For now only support private anonymous when
  2611. * migrating to un-addressable device memory.
  2612. */
  2613. if (mapping) {
  2614. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2615. continue;
  2616. }
  2617. } else {
  2618. /*
  2619. * Other types of ZONE_DEVICE page are not
  2620. * supported.
  2621. */
  2622. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2623. continue;
  2624. }
  2625. }
  2626. r = migrate_page(mapping, newpage, page, MIGRATE_SYNC_NO_COPY);
  2627. if (r != MIGRATEPAGE_SUCCESS)
  2628. migrate->src[i] &= ~MIGRATE_PFN_MIGRATE;
  2629. }
  2630. /*
  2631. * No need to double call mmu_notifier->invalidate_range() callback as
  2632. * the above ptep_clear_flush_notify() inside migrate_vma_insert_page()
  2633. * did already call it.
  2634. */
  2635. if (notified)
  2636. mmu_notifier_invalidate_range_only_end(&range);
  2637. }
  2638. EXPORT_SYMBOL(migrate_vma_pages);
  2639. /**
  2640. * migrate_vma_finalize() - restore CPU page table entry
  2641. * @migrate: migrate struct containing all migration information
  2642. *
  2643. * This replaces the special migration pte entry with either a mapping to the
  2644. * new page if migration was successful for that page, or to the original page
  2645. * otherwise.
  2646. *
  2647. * This also unlocks the pages and puts them back on the lru, or drops the extra
  2648. * refcount, for device pages.
  2649. */
  2650. void migrate_vma_finalize(struct migrate_vma *migrate)
  2651. {
  2652. const unsigned long npages = migrate->npages;
  2653. unsigned long i;
  2654. for (i = 0; i < npages; i++) {
  2655. struct page *newpage = migrate_pfn_to_page(migrate->dst[i]);
  2656. struct page *page = migrate_pfn_to_page(migrate->src[i]);
  2657. if (!page) {
  2658. if (newpage) {
  2659. unlock_page(newpage);
  2660. put_page(newpage);
  2661. }
  2662. continue;
  2663. }
  2664. if (!(migrate->src[i] & MIGRATE_PFN_MIGRATE) || !newpage) {
  2665. if (newpage) {
  2666. unlock_page(newpage);
  2667. put_page(newpage);
  2668. }
  2669. newpage = page;
  2670. }
  2671. remove_migration_ptes(page, newpage, false);
  2672. unlock_page(page);
  2673. if (is_zone_device_page(page))
  2674. put_page(page);
  2675. else
  2676. putback_lru_page(page);
  2677. if (newpage != page) {
  2678. unlock_page(newpage);
  2679. if (is_zone_device_page(newpage))
  2680. put_page(newpage);
  2681. else
  2682. putback_lru_page(newpage);
  2683. }
  2684. }
  2685. }
  2686. EXPORT_SYMBOL(migrate_vma_finalize);
  2687. #endif /* CONFIG_DEVICE_PRIVATE */