gup.c 85 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. #include <linux/kernel.h>
  3. #include <linux/errno.h>
  4. #include <linux/err.h>
  5. #include <linux/spinlock.h>
  6. #include <linux/mm.h>
  7. #include <linux/memremap.h>
  8. #include <linux/pagemap.h>
  9. #include <linux/rmap.h>
  10. #include <linux/swap.h>
  11. #include <linux/swapops.h>
  12. #include <linux/sched/signal.h>
  13. #include <linux/rwsem.h>
  14. #include <linux/hugetlb.h>
  15. #include <linux/migrate.h>
  16. #include <linux/mm_inline.h>
  17. #include <linux/sched/mm.h>
  18. #include <linux/page_pinner.h>
  19. #include <asm/mmu_context.h>
  20. #include <asm/tlbflush.h>
  21. #include "internal.h"
  22. struct follow_page_context {
  23. struct dev_pagemap *pgmap;
  24. unsigned int page_mask;
  25. };
  26. static void hpage_pincount_add(struct page *page, int refs)
  27. {
  28. VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
  29. VM_BUG_ON_PAGE(page != compound_head(page), page);
  30. atomic_add(refs, compound_pincount_ptr(page));
  31. }
  32. static void hpage_pincount_sub(struct page *page, int refs)
  33. {
  34. VM_BUG_ON_PAGE(!hpage_pincount_available(page), page);
  35. VM_BUG_ON_PAGE(page != compound_head(page), page);
  36. atomic_sub(refs, compound_pincount_ptr(page));
  37. }
  38. /* Equivalent to calling put_page() @refs times. */
  39. static void put_page_refs(struct page *page, int refs)
  40. {
  41. #ifdef CONFIG_DEBUG_VM
  42. if (VM_WARN_ON_ONCE_PAGE(page_ref_count(page) < refs, page))
  43. return;
  44. #endif
  45. /*
  46. * Calling put_page() for each ref is unnecessarily slow. Only the last
  47. * ref needs a put_page().
  48. */
  49. if (refs > 1)
  50. page_ref_sub(page, refs - 1);
  51. put_page(page);
  52. }
  53. /*
  54. * Return the compound head page with ref appropriately incremented,
  55. * or NULL if that failed.
  56. */
  57. static inline struct page *try_get_compound_head(struct page *page, int refs)
  58. {
  59. struct page *head = compound_head(page);
  60. if (WARN_ON_ONCE(page_ref_count(head) < 0))
  61. return NULL;
  62. if (unlikely(!page_cache_add_speculative(head, refs)))
  63. return NULL;
  64. /*
  65. * At this point we have a stable reference to the head page; but it
  66. * could be that between the compound_head() lookup and the refcount
  67. * increment, the compound page was split, in which case we'd end up
  68. * holding a reference on a page that has nothing to do with the page
  69. * we were given anymore.
  70. * So now that the head page is stable, recheck that the pages still
  71. * belong together.
  72. */
  73. if (unlikely(compound_head(page) != head)) {
  74. put_page_refs(head, refs);
  75. return NULL;
  76. }
  77. return head;
  78. }
  79. /*
  80. * try_grab_compound_head() - attempt to elevate a page's refcount, by a
  81. * flags-dependent amount.
  82. *
  83. * "grab" names in this file mean, "look at flags to decide whether to use
  84. * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
  85. *
  86. * Either FOLL_PIN or FOLL_GET (or neither) must be set, but not both at the
  87. * same time. (That's true throughout the get_user_pages*() and
  88. * pin_user_pages*() APIs.) Cases:
  89. *
  90. * FOLL_GET: page's refcount will be incremented by 1.
  91. * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
  92. *
  93. * Return: head page (with refcount appropriately incremented) for success, or
  94. * NULL upon failure. If neither FOLL_GET nor FOLL_PIN was set, that's
  95. * considered failure, and furthermore, a likely bug in the caller, so a warning
  96. * is also emitted.
  97. */
  98. static __maybe_unused struct page *try_grab_compound_head(struct page *page,
  99. int refs,
  100. unsigned int flags)
  101. {
  102. if (flags & FOLL_GET) {
  103. struct page *head = try_get_compound_head(page, refs);
  104. if (head)
  105. set_page_pinner(head, compound_order(head));
  106. return head;
  107. } else if (flags & FOLL_PIN) {
  108. int orig_refs = refs;
  109. /*
  110. * Can't do FOLL_LONGTERM + FOLL_PIN with CMA in the gup fast
  111. * path, so fail and let the caller fall back to the slow path.
  112. */
  113. if (unlikely(flags & FOLL_LONGTERM) &&
  114. is_migrate_cma_page(page))
  115. return NULL;
  116. /*
  117. * CAUTION: Don't use compound_head() on the page before this
  118. * point, the result won't be stable.
  119. */
  120. page = try_get_compound_head(page, refs);
  121. if (!page)
  122. return NULL;
  123. /*
  124. * When pinning a compound page of order > 1 (which is what
  125. * hpage_pincount_available() checks for), use an exact count to
  126. * track it, via hpage_pincount_add/_sub().
  127. *
  128. * However, be sure to *also* increment the normal page refcount
  129. * field at least once, so that the page really is pinned.
  130. */
  131. if (hpage_pincount_available(page))
  132. hpage_pincount_add(page, refs);
  133. else
  134. page_ref_add(page, refs * (GUP_PIN_COUNTING_BIAS - 1));
  135. mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED,
  136. orig_refs);
  137. return page;
  138. }
  139. WARN_ON_ONCE(1);
  140. return NULL;
  141. }
  142. static void put_compound_head(struct page *page, int refs, unsigned int flags)
  143. {
  144. if (flags & FOLL_PIN) {
  145. mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_RELEASED,
  146. refs);
  147. if (hpage_pincount_available(page))
  148. hpage_pincount_sub(page, refs);
  149. else
  150. refs *= GUP_PIN_COUNTING_BIAS;
  151. }
  152. if (flags & FOLL_GET)
  153. reset_page_pinner(page, compound_order(page));
  154. put_page_refs(page, refs);
  155. }
  156. /**
  157. * try_grab_page() - elevate a page's refcount by a flag-dependent amount
  158. *
  159. * This might not do anything at all, depending on the flags argument.
  160. *
  161. * "grab" names in this file mean, "look at flags to decide whether to use
  162. * FOLL_PIN or FOLL_GET behavior, when incrementing the page's refcount.
  163. *
  164. * @page: pointer to page to be grabbed
  165. * @flags: gup flags: these are the FOLL_* flag values.
  166. *
  167. * Either FOLL_PIN or FOLL_GET (or neither) may be set, but not both at the same
  168. * time. Cases:
  169. *
  170. * FOLL_GET: page's refcount will be incremented by 1.
  171. * FOLL_PIN: page's refcount will be incremented by GUP_PIN_COUNTING_BIAS.
  172. *
  173. * Return: true for success, or if no action was required (if neither FOLL_PIN
  174. * nor FOLL_GET was set, nothing is done). False for failure: FOLL_GET or
  175. * FOLL_PIN was set, but the page could not be grabbed.
  176. */
  177. bool __must_check try_grab_page(struct page *page, unsigned int flags)
  178. {
  179. WARN_ON_ONCE((flags & (FOLL_GET | FOLL_PIN)) == (FOLL_GET | FOLL_PIN));
  180. if (flags & FOLL_GET) {
  181. bool ret = try_get_page(page);
  182. if (ret) {
  183. page = compound_head(page);
  184. set_page_pinner(page, compound_order(page));
  185. }
  186. return ret;
  187. } else if (flags & FOLL_PIN) {
  188. int refs = 1;
  189. page = compound_head(page);
  190. if (WARN_ON_ONCE(page_ref_count(page) <= 0))
  191. return false;
  192. if (hpage_pincount_available(page))
  193. hpage_pincount_add(page, 1);
  194. else
  195. refs = GUP_PIN_COUNTING_BIAS;
  196. /*
  197. * Similar to try_grab_compound_head(): even if using the
  198. * hpage_pincount_add/_sub() routines, be sure to
  199. * *also* increment the normal page refcount field at least
  200. * once, so that the page really is pinned.
  201. */
  202. page_ref_add(page, refs);
  203. mod_node_page_state(page_pgdat(page), NR_FOLL_PIN_ACQUIRED, 1);
  204. }
  205. return true;
  206. }
  207. /**
  208. * unpin_user_page() - release a dma-pinned page
  209. * @page: pointer to page to be released
  210. *
  211. * Pages that were pinned via pin_user_pages*() must be released via either
  212. * unpin_user_page(), or one of the unpin_user_pages*() routines. This is so
  213. * that such pages can be separately tracked and uniquely handled. In
  214. * particular, interactions with RDMA and filesystems need special handling.
  215. */
  216. void unpin_user_page(struct page *page)
  217. {
  218. put_compound_head(compound_head(page), 1, FOLL_PIN);
  219. }
  220. EXPORT_SYMBOL(unpin_user_page);
  221. /*
  222. * put_user_page() - release a page obtained using get_user_pages() or
  223. * follow_page(FOLL_GET)
  224. * @page: pointer to page to be released
  225. *
  226. * Pages that were obtained via get_user_pages()/follow_page(FOLL_GET) must be
  227. * released via put_user_page.
  228. * note: If it's not a page from GUP or follow_page(FOLL_GET), it's harmless.
  229. */
  230. void put_user_page(struct page *page)
  231. {
  232. struct page *head = compound_head(page);
  233. reset_page_pinner(head, compound_order(head));
  234. put_page(page);
  235. }
  236. EXPORT_SYMBOL(put_user_page);
  237. /**
  238. * unpin_user_pages_dirty_lock() - release and optionally dirty gup-pinned pages
  239. * @pages: array of pages to be maybe marked dirty, and definitely released.
  240. * @npages: number of pages in the @pages array.
  241. * @make_dirty: whether to mark the pages dirty
  242. *
  243. * "gup-pinned page" refers to a page that has had one of the get_user_pages()
  244. * variants called on that page.
  245. *
  246. * For each page in the @pages array, make that page (or its head page, if a
  247. * compound page) dirty, if @make_dirty is true, and if the page was previously
  248. * listed as clean. In any case, releases all pages using unpin_user_page(),
  249. * possibly via unpin_user_pages(), for the non-dirty case.
  250. *
  251. * Please see the unpin_user_page() documentation for details.
  252. *
  253. * set_page_dirty_lock() is used internally. If instead, set_page_dirty() is
  254. * required, then the caller should a) verify that this is really correct,
  255. * because _lock() is usually required, and b) hand code it:
  256. * set_page_dirty_lock(), unpin_user_page().
  257. *
  258. */
  259. void unpin_user_pages_dirty_lock(struct page **pages, unsigned long npages,
  260. bool make_dirty)
  261. {
  262. unsigned long index;
  263. /*
  264. * TODO: this can be optimized for huge pages: if a series of pages is
  265. * physically contiguous and part of the same compound page, then a
  266. * single operation to the head page should suffice.
  267. */
  268. if (!make_dirty) {
  269. unpin_user_pages(pages, npages);
  270. return;
  271. }
  272. for (index = 0; index < npages; index++) {
  273. struct page *page = compound_head(pages[index]);
  274. /*
  275. * Checking PageDirty at this point may race with
  276. * clear_page_dirty_for_io(), but that's OK. Two key
  277. * cases:
  278. *
  279. * 1) This code sees the page as already dirty, so it
  280. * skips the call to set_page_dirty(). That could happen
  281. * because clear_page_dirty_for_io() called
  282. * page_mkclean(), followed by set_page_dirty().
  283. * However, now the page is going to get written back,
  284. * which meets the original intention of setting it
  285. * dirty, so all is well: clear_page_dirty_for_io() goes
  286. * on to call TestClearPageDirty(), and write the page
  287. * back.
  288. *
  289. * 2) This code sees the page as clean, so it calls
  290. * set_page_dirty(). The page stays dirty, despite being
  291. * written back, so it gets written back again in the
  292. * next writeback cycle. This is harmless.
  293. */
  294. if (!PageDirty(page))
  295. set_page_dirty_lock(page);
  296. unpin_user_page(page);
  297. }
  298. }
  299. EXPORT_SYMBOL(unpin_user_pages_dirty_lock);
  300. /**
  301. * unpin_user_pages() - release an array of gup-pinned pages.
  302. * @pages: array of pages to be marked dirty and released.
  303. * @npages: number of pages in the @pages array.
  304. *
  305. * For each page in the @pages array, release the page using unpin_user_page().
  306. *
  307. * Please see the unpin_user_page() documentation for details.
  308. */
  309. void unpin_user_pages(struct page **pages, unsigned long npages)
  310. {
  311. unsigned long index;
  312. /*
  313. * If this WARN_ON() fires, then the system *might* be leaking pages (by
  314. * leaving them pinned), but probably not. More likely, gup/pup returned
  315. * a hard -ERRNO error to the caller, who erroneously passed it here.
  316. */
  317. if (WARN_ON(IS_ERR_VALUE(npages)))
  318. return;
  319. /*
  320. * TODO: this can be optimized for huge pages: if a series of pages is
  321. * physically contiguous and part of the same compound page, then a
  322. * single operation to the head page should suffice.
  323. */
  324. for (index = 0; index < npages; index++)
  325. unpin_user_page(pages[index]);
  326. }
  327. EXPORT_SYMBOL(unpin_user_pages);
  328. #ifdef CONFIG_MMU
  329. static struct page *no_page_table(struct vm_area_struct *vma,
  330. unsigned int flags)
  331. {
  332. /*
  333. * When core dumping an enormous anonymous area that nobody
  334. * has touched so far, we don't want to allocate unnecessary pages or
  335. * page tables. Return error instead of NULL to skip handle_mm_fault,
  336. * then get_dump_page() will return NULL to leave a hole in the dump.
  337. * But we can only make this optimization where a hole would surely
  338. * be zero-filled if handle_mm_fault() actually did handle it.
  339. */
  340. if ((flags & FOLL_DUMP) &&
  341. (vma_is_anonymous(vma) || !vma->vm_ops->fault))
  342. return ERR_PTR(-EFAULT);
  343. return NULL;
  344. }
  345. static int follow_pfn_pte(struct vm_area_struct *vma, unsigned long address,
  346. pte_t *pte, unsigned int flags)
  347. {
  348. /* No page to get reference */
  349. if (flags & FOLL_GET)
  350. return -EFAULT;
  351. if (flags & FOLL_TOUCH) {
  352. pte_t entry = *pte;
  353. if (flags & FOLL_WRITE)
  354. entry = pte_mkdirty(entry);
  355. entry = pte_mkyoung(entry);
  356. if (!pte_same(*pte, entry)) {
  357. set_pte_at(vma->vm_mm, address, pte, entry);
  358. update_mmu_cache(vma, address, pte);
  359. }
  360. }
  361. /* Proper page table entry exists, but no corresponding struct page */
  362. return -EEXIST;
  363. }
  364. /*
  365. * FOLL_FORCE can write to even unwritable pte's, but only
  366. * after we've gone through a COW cycle and they are dirty.
  367. */
  368. static inline bool can_follow_write_pte(pte_t pte, unsigned int flags)
  369. {
  370. return pte_write(pte) ||
  371. ((flags & FOLL_FORCE) && (flags & FOLL_COW) && pte_dirty(pte));
  372. }
  373. static struct page *follow_page_pte(struct vm_area_struct *vma,
  374. unsigned long address, pmd_t *pmd, unsigned int flags,
  375. struct dev_pagemap **pgmap)
  376. {
  377. struct mm_struct *mm = vma->vm_mm;
  378. struct page *page;
  379. spinlock_t *ptl;
  380. pte_t *ptep, pte;
  381. int ret;
  382. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  383. if (WARN_ON_ONCE((flags & (FOLL_PIN | FOLL_GET)) ==
  384. (FOLL_PIN | FOLL_GET)))
  385. return ERR_PTR(-EINVAL);
  386. retry:
  387. if (unlikely(pmd_bad(*pmd)))
  388. return no_page_table(vma, flags);
  389. ptep = pte_offset_map_lock(mm, pmd, address, &ptl);
  390. pte = *ptep;
  391. if (!pte_present(pte)) {
  392. swp_entry_t entry;
  393. /*
  394. * KSM's break_ksm() relies upon recognizing a ksm page
  395. * even while it is being migrated, so for that case we
  396. * need migration_entry_wait().
  397. */
  398. if (likely(!(flags & FOLL_MIGRATION)))
  399. goto no_page;
  400. if (pte_none(pte))
  401. goto no_page;
  402. entry = pte_to_swp_entry(pte);
  403. if (!is_migration_entry(entry))
  404. goto no_page;
  405. pte_unmap_unlock(ptep, ptl);
  406. migration_entry_wait(mm, pmd, address);
  407. goto retry;
  408. }
  409. if ((flags & FOLL_NUMA) && pte_protnone(pte))
  410. goto no_page;
  411. if ((flags & FOLL_WRITE) && !can_follow_write_pte(pte, flags)) {
  412. pte_unmap_unlock(ptep, ptl);
  413. return NULL;
  414. }
  415. page = vm_normal_page(vma, address, pte);
  416. if (!page && pte_devmap(pte) && (flags & (FOLL_GET | FOLL_PIN))) {
  417. /*
  418. * Only return device mapping pages in the FOLL_GET or FOLL_PIN
  419. * case since they are only valid while holding the pgmap
  420. * reference.
  421. */
  422. *pgmap = get_dev_pagemap(pte_pfn(pte), *pgmap);
  423. if (*pgmap)
  424. page = pte_page(pte);
  425. else
  426. goto no_page;
  427. } else if (unlikely(!page)) {
  428. if (flags & FOLL_DUMP) {
  429. /* Avoid special (like zero) pages in core dumps */
  430. page = ERR_PTR(-EFAULT);
  431. goto out;
  432. }
  433. if (is_zero_pfn(pte_pfn(pte))) {
  434. page = pte_page(pte);
  435. } else {
  436. ret = follow_pfn_pte(vma, address, ptep, flags);
  437. page = ERR_PTR(ret);
  438. goto out;
  439. }
  440. }
  441. if (flags & FOLL_SPLIT && PageTransCompound(page)) {
  442. get_page(page);
  443. pte_unmap_unlock(ptep, ptl);
  444. lock_page(page);
  445. ret = split_huge_page(page);
  446. unlock_page(page);
  447. put_page(page);
  448. if (ret)
  449. return ERR_PTR(ret);
  450. goto retry;
  451. }
  452. /* try_grab_page() does nothing unless FOLL_GET or FOLL_PIN is set. */
  453. if (unlikely(!try_grab_page(page, flags))) {
  454. page = ERR_PTR(-ENOMEM);
  455. goto out;
  456. }
  457. /*
  458. * We need to make the page accessible if and only if we are going
  459. * to access its content (the FOLL_PIN case). Please see
  460. * Documentation/core-api/pin_user_pages.rst for details.
  461. */
  462. if (flags & FOLL_PIN) {
  463. ret = arch_make_page_accessible(page);
  464. if (ret) {
  465. unpin_user_page(page);
  466. page = ERR_PTR(ret);
  467. goto out;
  468. }
  469. }
  470. if (flags & FOLL_TOUCH) {
  471. if ((flags & FOLL_WRITE) &&
  472. !pte_dirty(pte) && !PageDirty(page))
  473. set_page_dirty(page);
  474. /*
  475. * pte_mkyoung() would be more correct here, but atomic care
  476. * is needed to avoid losing the dirty bit: it is easier to use
  477. * mark_page_accessed().
  478. */
  479. mark_page_accessed(page);
  480. }
  481. if ((flags & FOLL_MLOCK) && (vma->vm_flags & VM_LOCKED)) {
  482. /* Do not mlock pte-mapped THP */
  483. if (PageTransCompound(page))
  484. goto out;
  485. /*
  486. * The preliminary mapping check is mainly to avoid the
  487. * pointless overhead of lock_page on the ZERO_PAGE
  488. * which might bounce very badly if there is contention.
  489. *
  490. * If the page is already locked, we don't need to
  491. * handle it now - vmscan will handle it later if and
  492. * when it attempts to reclaim the page.
  493. */
  494. if (page->mapping && trylock_page(page)) {
  495. lru_add_drain(); /* push cached pages to LRU */
  496. /*
  497. * Because we lock page here, and migration is
  498. * blocked by the pte's page reference, and we
  499. * know the page is still mapped, we don't even
  500. * need to check for file-cache page truncation.
  501. */
  502. mlock_vma_page(page);
  503. unlock_page(page);
  504. }
  505. }
  506. out:
  507. pte_unmap_unlock(ptep, ptl);
  508. return page;
  509. no_page:
  510. pte_unmap_unlock(ptep, ptl);
  511. if (!pte_none(pte))
  512. return NULL;
  513. return no_page_table(vma, flags);
  514. }
  515. static struct page *follow_pmd_mask(struct vm_area_struct *vma,
  516. unsigned long address, pud_t *pudp,
  517. unsigned int flags,
  518. struct follow_page_context *ctx)
  519. {
  520. pmd_t *pmd, pmdval;
  521. spinlock_t *ptl;
  522. struct page *page;
  523. struct mm_struct *mm = vma->vm_mm;
  524. pmd = pmd_offset(pudp, address);
  525. /*
  526. * The READ_ONCE() will stabilize the pmdval in a register or
  527. * on the stack so that it will stop changing under the code.
  528. */
  529. pmdval = READ_ONCE(*pmd);
  530. if (pmd_none(pmdval))
  531. return no_page_table(vma, flags);
  532. if (pmd_huge(pmdval) && is_vm_hugetlb_page(vma)) {
  533. page = follow_huge_pmd(mm, address, pmd, flags);
  534. if (page)
  535. return page;
  536. return no_page_table(vma, flags);
  537. }
  538. if (is_hugepd(__hugepd(pmd_val(pmdval)))) {
  539. page = follow_huge_pd(vma, address,
  540. __hugepd(pmd_val(pmdval)), flags,
  541. PMD_SHIFT);
  542. if (page)
  543. return page;
  544. return no_page_table(vma, flags);
  545. }
  546. retry:
  547. if (!pmd_present(pmdval)) {
  548. if (likely(!(flags & FOLL_MIGRATION)))
  549. return no_page_table(vma, flags);
  550. VM_BUG_ON(thp_migration_supported() &&
  551. !is_pmd_migration_entry(pmdval));
  552. if (is_pmd_migration_entry(pmdval))
  553. pmd_migration_entry_wait(mm, pmd);
  554. pmdval = READ_ONCE(*pmd);
  555. /*
  556. * MADV_DONTNEED may convert the pmd to null because
  557. * mmap_lock is held in read mode
  558. */
  559. if (pmd_none(pmdval))
  560. return no_page_table(vma, flags);
  561. goto retry;
  562. }
  563. if (pmd_devmap(pmdval)) {
  564. ptl = pmd_lock(mm, pmd);
  565. page = follow_devmap_pmd(vma, address, pmd, flags, &ctx->pgmap);
  566. spin_unlock(ptl);
  567. if (page)
  568. return page;
  569. }
  570. if (likely(!pmd_trans_huge(pmdval)))
  571. return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
  572. if ((flags & FOLL_NUMA) && pmd_protnone(pmdval))
  573. return no_page_table(vma, flags);
  574. retry_locked:
  575. ptl = pmd_lock(mm, pmd);
  576. if (unlikely(pmd_none(*pmd))) {
  577. spin_unlock(ptl);
  578. return no_page_table(vma, flags);
  579. }
  580. if (unlikely(!pmd_present(*pmd))) {
  581. spin_unlock(ptl);
  582. if (likely(!(flags & FOLL_MIGRATION)))
  583. return no_page_table(vma, flags);
  584. pmd_migration_entry_wait(mm, pmd);
  585. goto retry_locked;
  586. }
  587. if (unlikely(!pmd_trans_huge(*pmd))) {
  588. spin_unlock(ptl);
  589. return follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
  590. }
  591. if (flags & (FOLL_SPLIT | FOLL_SPLIT_PMD)) {
  592. int ret;
  593. page = pmd_page(*pmd);
  594. if (is_huge_zero_page(page)) {
  595. spin_unlock(ptl);
  596. ret = 0;
  597. split_huge_pmd(vma, pmd, address);
  598. if (pmd_trans_unstable(pmd))
  599. ret = -EBUSY;
  600. } else if (flags & FOLL_SPLIT) {
  601. if (unlikely(!try_get_page(page))) {
  602. spin_unlock(ptl);
  603. return ERR_PTR(-ENOMEM);
  604. }
  605. spin_unlock(ptl);
  606. lock_page(page);
  607. ret = split_huge_page(page);
  608. unlock_page(page);
  609. put_page(page);
  610. if (pmd_none(*pmd))
  611. return no_page_table(vma, flags);
  612. } else { /* flags & FOLL_SPLIT_PMD */
  613. spin_unlock(ptl);
  614. split_huge_pmd(vma, pmd, address);
  615. ret = pte_alloc(mm, pmd) ? -ENOMEM : 0;
  616. }
  617. return ret ? ERR_PTR(ret) :
  618. follow_page_pte(vma, address, pmd, flags, &ctx->pgmap);
  619. }
  620. page = follow_trans_huge_pmd(vma, address, pmd, flags);
  621. spin_unlock(ptl);
  622. ctx->page_mask = HPAGE_PMD_NR - 1;
  623. return page;
  624. }
  625. static struct page *follow_pud_mask(struct vm_area_struct *vma,
  626. unsigned long address, p4d_t *p4dp,
  627. unsigned int flags,
  628. struct follow_page_context *ctx)
  629. {
  630. pud_t *pud;
  631. spinlock_t *ptl;
  632. struct page *page;
  633. struct mm_struct *mm = vma->vm_mm;
  634. pud = pud_offset(p4dp, address);
  635. if (pud_none(*pud))
  636. return no_page_table(vma, flags);
  637. if (pud_huge(*pud) && is_vm_hugetlb_page(vma)) {
  638. page = follow_huge_pud(mm, address, pud, flags);
  639. if (page)
  640. return page;
  641. return no_page_table(vma, flags);
  642. }
  643. if (is_hugepd(__hugepd(pud_val(*pud)))) {
  644. page = follow_huge_pd(vma, address,
  645. __hugepd(pud_val(*pud)), flags,
  646. PUD_SHIFT);
  647. if (page)
  648. return page;
  649. return no_page_table(vma, flags);
  650. }
  651. if (pud_devmap(*pud)) {
  652. ptl = pud_lock(mm, pud);
  653. page = follow_devmap_pud(vma, address, pud, flags, &ctx->pgmap);
  654. spin_unlock(ptl);
  655. if (page)
  656. return page;
  657. }
  658. if (unlikely(pud_bad(*pud)))
  659. return no_page_table(vma, flags);
  660. return follow_pmd_mask(vma, address, pud, flags, ctx);
  661. }
  662. static struct page *follow_p4d_mask(struct vm_area_struct *vma,
  663. unsigned long address, pgd_t *pgdp,
  664. unsigned int flags,
  665. struct follow_page_context *ctx)
  666. {
  667. p4d_t *p4d;
  668. struct page *page;
  669. p4d = p4d_offset(pgdp, address);
  670. if (p4d_none(*p4d))
  671. return no_page_table(vma, flags);
  672. BUILD_BUG_ON(p4d_huge(*p4d));
  673. if (unlikely(p4d_bad(*p4d)))
  674. return no_page_table(vma, flags);
  675. if (is_hugepd(__hugepd(p4d_val(*p4d)))) {
  676. page = follow_huge_pd(vma, address,
  677. __hugepd(p4d_val(*p4d)), flags,
  678. P4D_SHIFT);
  679. if (page)
  680. return page;
  681. return no_page_table(vma, flags);
  682. }
  683. return follow_pud_mask(vma, address, p4d, flags, ctx);
  684. }
  685. /**
  686. * follow_page_mask - look up a page descriptor from a user-virtual address
  687. * @vma: vm_area_struct mapping @address
  688. * @address: virtual address to look up
  689. * @flags: flags modifying lookup behaviour
  690. * @ctx: contains dev_pagemap for %ZONE_DEVICE memory pinning and a
  691. * pointer to output page_mask
  692. *
  693. * @flags can have FOLL_ flags set, defined in <linux/mm.h>
  694. *
  695. * When getting pages from ZONE_DEVICE memory, the @ctx->pgmap caches
  696. * the device's dev_pagemap metadata to avoid repeating expensive lookups.
  697. *
  698. * On output, the @ctx->page_mask is set according to the size of the page.
  699. *
  700. * Return: the mapped (struct page *), %NULL if no mapping exists, or
  701. * an error pointer if there is a mapping to something not represented
  702. * by a page descriptor (see also vm_normal_page()).
  703. */
  704. static struct page *follow_page_mask(struct vm_area_struct *vma,
  705. unsigned long address, unsigned int flags,
  706. struct follow_page_context *ctx)
  707. {
  708. pgd_t *pgd;
  709. struct page *page;
  710. struct mm_struct *mm = vma->vm_mm;
  711. ctx->page_mask = 0;
  712. /* make this handle hugepd */
  713. page = follow_huge_addr(mm, address, flags & FOLL_WRITE);
  714. if (!IS_ERR(page)) {
  715. WARN_ON_ONCE(flags & (FOLL_GET | FOLL_PIN));
  716. return page;
  717. }
  718. pgd = pgd_offset(mm, address);
  719. if (pgd_none(*pgd) || unlikely(pgd_bad(*pgd)))
  720. return no_page_table(vma, flags);
  721. if (pgd_huge(*pgd)) {
  722. page = follow_huge_pgd(mm, address, pgd, flags);
  723. if (page)
  724. return page;
  725. return no_page_table(vma, flags);
  726. }
  727. if (is_hugepd(__hugepd(pgd_val(*pgd)))) {
  728. page = follow_huge_pd(vma, address,
  729. __hugepd(pgd_val(*pgd)), flags,
  730. PGDIR_SHIFT);
  731. if (page)
  732. return page;
  733. return no_page_table(vma, flags);
  734. }
  735. return follow_p4d_mask(vma, address, pgd, flags, ctx);
  736. }
  737. struct page *follow_page(struct vm_area_struct *vma, unsigned long address,
  738. unsigned int foll_flags)
  739. {
  740. struct follow_page_context ctx = { NULL };
  741. struct page *page;
  742. page = follow_page_mask(vma, address, foll_flags, &ctx);
  743. if (ctx.pgmap)
  744. put_dev_pagemap(ctx.pgmap);
  745. return page;
  746. }
  747. static int get_gate_page(struct mm_struct *mm, unsigned long address,
  748. unsigned int gup_flags, struct vm_area_struct **vma,
  749. struct page **page)
  750. {
  751. pgd_t *pgd;
  752. p4d_t *p4d;
  753. pud_t *pud;
  754. pmd_t *pmd;
  755. pte_t *pte;
  756. int ret = -EFAULT;
  757. /* user gate pages are read-only */
  758. if (gup_flags & FOLL_WRITE)
  759. return -EFAULT;
  760. if (address > TASK_SIZE)
  761. pgd = pgd_offset_k(address);
  762. else
  763. pgd = pgd_offset_gate(mm, address);
  764. if (pgd_none(*pgd))
  765. return -EFAULT;
  766. p4d = p4d_offset(pgd, address);
  767. if (p4d_none(*p4d))
  768. return -EFAULT;
  769. pud = pud_offset(p4d, address);
  770. if (pud_none(*pud))
  771. return -EFAULT;
  772. pmd = pmd_offset(pud, address);
  773. if (!pmd_present(*pmd))
  774. return -EFAULT;
  775. VM_BUG_ON(pmd_trans_huge(*pmd));
  776. pte = pte_offset_map(pmd, address);
  777. if (pte_none(*pte))
  778. goto unmap;
  779. *vma = get_gate_vma(mm);
  780. if (!page)
  781. goto out;
  782. *page = vm_normal_page(*vma, address, *pte);
  783. if (!*page) {
  784. if ((gup_flags & FOLL_DUMP) || !is_zero_pfn(pte_pfn(*pte)))
  785. goto unmap;
  786. *page = pte_page(*pte);
  787. }
  788. if (unlikely(!try_grab_page(*page, gup_flags))) {
  789. ret = -ENOMEM;
  790. goto unmap;
  791. }
  792. out:
  793. ret = 0;
  794. unmap:
  795. pte_unmap(pte);
  796. return ret;
  797. }
  798. /*
  799. * mmap_lock must be held on entry. If @locked != NULL and *@flags
  800. * does not include FOLL_NOWAIT, the mmap_lock may be released. If it
  801. * is, *@locked will be set to 0 and -EBUSY returned.
  802. */
  803. static int faultin_page(struct vm_area_struct *vma,
  804. unsigned long address, unsigned int *flags, int *locked)
  805. {
  806. unsigned int fault_flags = 0;
  807. vm_fault_t ret;
  808. /* mlock all present pages, but do not fault in new pages */
  809. if ((*flags & (FOLL_POPULATE | FOLL_MLOCK)) == FOLL_MLOCK)
  810. return -ENOENT;
  811. if (*flags & FOLL_WRITE)
  812. fault_flags |= FAULT_FLAG_WRITE;
  813. if (*flags & FOLL_REMOTE)
  814. fault_flags |= FAULT_FLAG_REMOTE;
  815. if (locked)
  816. fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
  817. if (*flags & FOLL_NOWAIT)
  818. fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_RETRY_NOWAIT;
  819. if (*flags & FOLL_TRIED) {
  820. /*
  821. * Note: FAULT_FLAG_ALLOW_RETRY and FAULT_FLAG_TRIED
  822. * can co-exist
  823. */
  824. fault_flags |= FAULT_FLAG_TRIED;
  825. }
  826. ret = handle_mm_fault(vma, address, fault_flags, NULL);
  827. if (ret & VM_FAULT_ERROR) {
  828. int err = vm_fault_to_errno(ret, *flags);
  829. if (err)
  830. return err;
  831. BUG();
  832. }
  833. if (ret & VM_FAULT_RETRY) {
  834. if (locked && !(fault_flags & FAULT_FLAG_RETRY_NOWAIT))
  835. *locked = 0;
  836. return -EBUSY;
  837. }
  838. /*
  839. * The VM_FAULT_WRITE bit tells us that do_wp_page has broken COW when
  840. * necessary, even if maybe_mkwrite decided not to set pte_write. We
  841. * can thus safely do subsequent page lookups as if they were reads.
  842. * But only do so when looping for pte_write is futile: in some cases
  843. * userspace may also be wanting to write to the gotten user page,
  844. * which a read fault here might prevent (a readonly page might get
  845. * reCOWed by userspace write).
  846. */
  847. if ((ret & VM_FAULT_WRITE) && !(vma->vm_flags & VM_WRITE))
  848. *flags |= FOLL_COW;
  849. return 0;
  850. }
  851. static int check_vma_flags(struct vm_area_struct *vma, unsigned long gup_flags)
  852. {
  853. vm_flags_t vm_flags = vma->vm_flags;
  854. int write = (gup_flags & FOLL_WRITE);
  855. int foreign = (gup_flags & FOLL_REMOTE);
  856. if (vm_flags & (VM_IO | VM_PFNMAP))
  857. return -EFAULT;
  858. if (gup_flags & FOLL_ANON && !vma_is_anonymous(vma))
  859. return -EFAULT;
  860. if ((gup_flags & FOLL_LONGTERM) && vma_is_fsdax(vma))
  861. return -EOPNOTSUPP;
  862. if (write) {
  863. if (!(vm_flags & VM_WRITE)) {
  864. if (!(gup_flags & FOLL_FORCE))
  865. return -EFAULT;
  866. /*
  867. * We used to let the write,force case do COW in a
  868. * VM_MAYWRITE VM_SHARED !VM_WRITE vma, so ptrace could
  869. * set a breakpoint in a read-only mapping of an
  870. * executable, without corrupting the file (yet only
  871. * when that file had been opened for writing!).
  872. * Anon pages in shared mappings are surprising: now
  873. * just reject it.
  874. */
  875. if (!is_cow_mapping(vm_flags))
  876. return -EFAULT;
  877. }
  878. } else if (!(vm_flags & VM_READ)) {
  879. if (!(gup_flags & FOLL_FORCE))
  880. return -EFAULT;
  881. /*
  882. * Is there actually any vma we can reach here which does not
  883. * have VM_MAYREAD set?
  884. */
  885. if (!(vm_flags & VM_MAYREAD))
  886. return -EFAULT;
  887. }
  888. /*
  889. * gups are always data accesses, not instruction
  890. * fetches, so execute=false here
  891. */
  892. if (!arch_vma_access_permitted(vma, write, false, foreign))
  893. return -EFAULT;
  894. return 0;
  895. }
  896. /**
  897. * __get_user_pages() - pin user pages in memory
  898. * @mm: mm_struct of target mm
  899. * @start: starting user address
  900. * @nr_pages: number of pages from start to pin
  901. * @gup_flags: flags modifying pin behaviour
  902. * @pages: array that receives pointers to the pages pinned.
  903. * Should be at least nr_pages long. Or NULL, if caller
  904. * only intends to ensure the pages are faulted in.
  905. * @vmas: array of pointers to vmas corresponding to each page.
  906. * Or NULL if the caller does not require them.
  907. * @locked: whether we're still with the mmap_lock held
  908. *
  909. * Returns either number of pages pinned (which may be less than the
  910. * number requested), or an error. Details about the return value:
  911. *
  912. * -- If nr_pages is 0, returns 0.
  913. * -- If nr_pages is >0, but no pages were pinned, returns -errno.
  914. * -- If nr_pages is >0, and some pages were pinned, returns the number of
  915. * pages pinned. Again, this may be less than nr_pages.
  916. * -- 0 return value is possible when the fault would need to be retried.
  917. *
  918. * The caller is responsible for releasing returned @pages, via put_page().
  919. *
  920. * @vmas are valid only as long as mmap_lock is held.
  921. *
  922. * Must be called with mmap_lock held. It may be released. See below.
  923. *
  924. * __get_user_pages walks a process's page tables and takes a reference to
  925. * each struct page that each user address corresponds to at a given
  926. * instant. That is, it takes the page that would be accessed if a user
  927. * thread accesses the given user virtual address at that instant.
  928. *
  929. * This does not guarantee that the page exists in the user mappings when
  930. * __get_user_pages returns, and there may even be a completely different
  931. * page there in some cases (eg. if mmapped pagecache has been invalidated
  932. * and subsequently re faulted). However it does guarantee that the page
  933. * won't be freed completely. And mostly callers simply care that the page
  934. * contains data that was valid *at some point in time*. Typically, an IO
  935. * or similar operation cannot guarantee anything stronger anyway because
  936. * locks can't be held over the syscall boundary.
  937. *
  938. * If @gup_flags & FOLL_WRITE == 0, the page must not be written to. If
  939. * the page is written to, set_page_dirty (or set_page_dirty_lock, as
  940. * appropriate) must be called after the page is finished with, and
  941. * before put_page is called.
  942. *
  943. * If @locked != NULL, *@locked will be set to 0 when mmap_lock is
  944. * released by an up_read(). That can happen if @gup_flags does not
  945. * have FOLL_NOWAIT.
  946. *
  947. * A caller using such a combination of @locked and @gup_flags
  948. * must therefore hold the mmap_lock for reading only, and recognize
  949. * when it's been released. Otherwise, it must be held for either
  950. * reading or writing and will not be released.
  951. *
  952. * In most cases, get_user_pages or get_user_pages_fast should be used
  953. * instead of __get_user_pages. __get_user_pages should be used only if
  954. * you need some special @gup_flags.
  955. */
  956. static long __get_user_pages(struct mm_struct *mm,
  957. unsigned long start, unsigned long nr_pages,
  958. unsigned int gup_flags, struct page **pages,
  959. struct vm_area_struct **vmas, int *locked)
  960. {
  961. long ret = 0, i = 0;
  962. struct vm_area_struct *vma = NULL;
  963. struct follow_page_context ctx = { NULL };
  964. if (!nr_pages)
  965. return 0;
  966. start = untagged_addr(start);
  967. VM_BUG_ON(!!pages != !!(gup_flags & (FOLL_GET | FOLL_PIN)));
  968. /*
  969. * If FOLL_FORCE is set then do not force a full fault as the hinting
  970. * fault information is unrelated to the reference behaviour of a task
  971. * using the address space
  972. */
  973. if (!(gup_flags & FOLL_FORCE))
  974. gup_flags |= FOLL_NUMA;
  975. do {
  976. struct page *page;
  977. unsigned int foll_flags = gup_flags;
  978. unsigned int page_increm;
  979. /* first iteration or cross vma bound */
  980. if (!vma || start >= vma->vm_end) {
  981. vma = find_extend_vma(mm, start);
  982. if (!vma && in_gate_area(mm, start)) {
  983. ret = get_gate_page(mm, start & PAGE_MASK,
  984. gup_flags, &vma,
  985. pages ? &pages[i] : NULL);
  986. if (ret)
  987. goto out;
  988. ctx.page_mask = 0;
  989. goto next_page;
  990. }
  991. if (!vma) {
  992. ret = -EFAULT;
  993. goto out;
  994. }
  995. ret = check_vma_flags(vma, gup_flags);
  996. if (ret)
  997. goto out;
  998. if (is_vm_hugetlb_page(vma)) {
  999. i = follow_hugetlb_page(mm, vma, pages, vmas,
  1000. &start, &nr_pages, i,
  1001. gup_flags, locked);
  1002. if (locked && *locked == 0) {
  1003. /*
  1004. * We've got a VM_FAULT_RETRY
  1005. * and we've lost mmap_lock.
  1006. * We must stop here.
  1007. */
  1008. BUG_ON(gup_flags & FOLL_NOWAIT);
  1009. BUG_ON(ret != 0);
  1010. goto out;
  1011. }
  1012. continue;
  1013. }
  1014. }
  1015. retry:
  1016. /*
  1017. * If we have a pending SIGKILL, don't keep faulting pages and
  1018. * potentially allocating memory.
  1019. */
  1020. if (fatal_signal_pending(current)) {
  1021. ret = -EINTR;
  1022. goto out;
  1023. }
  1024. cond_resched();
  1025. page = follow_page_mask(vma, start, foll_flags, &ctx);
  1026. if (!page) {
  1027. ret = faultin_page(vma, start, &foll_flags, locked);
  1028. switch (ret) {
  1029. case 0:
  1030. goto retry;
  1031. case -EBUSY:
  1032. ret = 0;
  1033. fallthrough;
  1034. case -EFAULT:
  1035. case -ENOMEM:
  1036. case -EHWPOISON:
  1037. goto out;
  1038. case -ENOENT:
  1039. goto next_page;
  1040. }
  1041. BUG();
  1042. } else if (PTR_ERR(page) == -EEXIST) {
  1043. /*
  1044. * Proper page table entry exists, but no corresponding
  1045. * struct page.
  1046. */
  1047. goto next_page;
  1048. } else if (IS_ERR(page)) {
  1049. ret = PTR_ERR(page);
  1050. goto out;
  1051. }
  1052. if (pages) {
  1053. pages[i] = page;
  1054. flush_anon_page(vma, page, start);
  1055. flush_dcache_page(page);
  1056. ctx.page_mask = 0;
  1057. }
  1058. next_page:
  1059. if (vmas) {
  1060. vmas[i] = vma;
  1061. ctx.page_mask = 0;
  1062. }
  1063. page_increm = 1 + (~(start >> PAGE_SHIFT) & ctx.page_mask);
  1064. if (page_increm > nr_pages)
  1065. page_increm = nr_pages;
  1066. i += page_increm;
  1067. start += page_increm * PAGE_SIZE;
  1068. nr_pages -= page_increm;
  1069. } while (nr_pages);
  1070. out:
  1071. if (ctx.pgmap)
  1072. put_dev_pagemap(ctx.pgmap);
  1073. return i ? i : ret;
  1074. }
  1075. static bool vma_permits_fault(struct vm_area_struct *vma,
  1076. unsigned int fault_flags)
  1077. {
  1078. bool write = !!(fault_flags & FAULT_FLAG_WRITE);
  1079. bool foreign = !!(fault_flags & FAULT_FLAG_REMOTE);
  1080. vm_flags_t vm_flags = write ? VM_WRITE : VM_READ;
  1081. if (!(vm_flags & vma->vm_flags))
  1082. return false;
  1083. /*
  1084. * The architecture might have a hardware protection
  1085. * mechanism other than read/write that can deny access.
  1086. *
  1087. * gup always represents data access, not instruction
  1088. * fetches, so execute=false here:
  1089. */
  1090. if (!arch_vma_access_permitted(vma, write, false, foreign))
  1091. return false;
  1092. return true;
  1093. }
  1094. /**
  1095. * fixup_user_fault() - manually resolve a user page fault
  1096. * @mm: mm_struct of target mm
  1097. * @address: user address
  1098. * @fault_flags:flags to pass down to handle_mm_fault()
  1099. * @unlocked: did we unlock the mmap_lock while retrying, maybe NULL if caller
  1100. * does not allow retry. If NULL, the caller must guarantee
  1101. * that fault_flags does not contain FAULT_FLAG_ALLOW_RETRY.
  1102. *
  1103. * This is meant to be called in the specific scenario where for locking reasons
  1104. * we try to access user memory in atomic context (within a pagefault_disable()
  1105. * section), this returns -EFAULT, and we want to resolve the user fault before
  1106. * trying again.
  1107. *
  1108. * Typically this is meant to be used by the futex code.
  1109. *
  1110. * The main difference with get_user_pages() is that this function will
  1111. * unconditionally call handle_mm_fault() which will in turn perform all the
  1112. * necessary SW fixup of the dirty and young bits in the PTE, while
  1113. * get_user_pages() only guarantees to update these in the struct page.
  1114. *
  1115. * This is important for some architectures where those bits also gate the
  1116. * access permission to the page because they are maintained in software. On
  1117. * such architectures, gup() will not be enough to make a subsequent access
  1118. * succeed.
  1119. *
  1120. * This function will not return with an unlocked mmap_lock. So it has not the
  1121. * same semantics wrt the @mm->mmap_lock as does filemap_fault().
  1122. */
  1123. int fixup_user_fault(struct mm_struct *mm,
  1124. unsigned long address, unsigned int fault_flags,
  1125. bool *unlocked)
  1126. {
  1127. struct vm_area_struct *vma;
  1128. vm_fault_t ret, major = 0;
  1129. address = untagged_addr(address);
  1130. if (unlocked)
  1131. fault_flags |= FAULT_FLAG_ALLOW_RETRY | FAULT_FLAG_KILLABLE;
  1132. retry:
  1133. vma = find_extend_vma(mm, address);
  1134. if (!vma || address < vma->vm_start)
  1135. return -EFAULT;
  1136. if (!vma_permits_fault(vma, fault_flags))
  1137. return -EFAULT;
  1138. if ((fault_flags & FAULT_FLAG_KILLABLE) &&
  1139. fatal_signal_pending(current))
  1140. return -EINTR;
  1141. ret = handle_mm_fault(vma, address, fault_flags, NULL);
  1142. major |= ret & VM_FAULT_MAJOR;
  1143. if (ret & VM_FAULT_ERROR) {
  1144. int err = vm_fault_to_errno(ret, 0);
  1145. if (err)
  1146. return err;
  1147. BUG();
  1148. }
  1149. if (ret & VM_FAULT_RETRY) {
  1150. mmap_read_lock(mm);
  1151. *unlocked = true;
  1152. fault_flags |= FAULT_FLAG_TRIED;
  1153. goto retry;
  1154. }
  1155. return 0;
  1156. }
  1157. EXPORT_SYMBOL_GPL(fixup_user_fault);
  1158. /*
  1159. * Please note that this function, unlike __get_user_pages will not
  1160. * return 0 for nr_pages > 0 without FOLL_NOWAIT
  1161. */
  1162. static __always_inline long __get_user_pages_locked(struct mm_struct *mm,
  1163. unsigned long start,
  1164. unsigned long nr_pages,
  1165. struct page **pages,
  1166. struct vm_area_struct **vmas,
  1167. int *locked,
  1168. unsigned int flags)
  1169. {
  1170. long ret, pages_done;
  1171. bool lock_dropped;
  1172. if (locked) {
  1173. /* if VM_FAULT_RETRY can be returned, vmas become invalid */
  1174. BUG_ON(vmas);
  1175. /* check caller initialized locked */
  1176. BUG_ON(*locked != 1);
  1177. }
  1178. if (flags & FOLL_PIN)
  1179. atomic_set(&mm->has_pinned, 1);
  1180. /*
  1181. * FOLL_PIN and FOLL_GET are mutually exclusive. Traditional behavior
  1182. * is to set FOLL_GET if the caller wants pages[] filled in (but has
  1183. * carelessly failed to specify FOLL_GET), so keep doing that, but only
  1184. * for FOLL_GET, not for the newer FOLL_PIN.
  1185. *
  1186. * FOLL_PIN always expects pages to be non-null, but no need to assert
  1187. * that here, as any failures will be obvious enough.
  1188. */
  1189. if (pages && !(flags & FOLL_PIN))
  1190. flags |= FOLL_GET;
  1191. pages_done = 0;
  1192. lock_dropped = false;
  1193. for (;;) {
  1194. ret = __get_user_pages(mm, start, nr_pages, flags, pages,
  1195. vmas, locked);
  1196. if (!locked)
  1197. /* VM_FAULT_RETRY couldn't trigger, bypass */
  1198. return ret;
  1199. /* VM_FAULT_RETRY cannot return errors */
  1200. if (!*locked) {
  1201. BUG_ON(ret < 0);
  1202. BUG_ON(ret >= nr_pages);
  1203. }
  1204. if (ret > 0) {
  1205. nr_pages -= ret;
  1206. pages_done += ret;
  1207. if (!nr_pages)
  1208. break;
  1209. }
  1210. if (*locked) {
  1211. /*
  1212. * VM_FAULT_RETRY didn't trigger or it was a
  1213. * FOLL_NOWAIT.
  1214. */
  1215. if (!pages_done)
  1216. pages_done = ret;
  1217. break;
  1218. }
  1219. /*
  1220. * VM_FAULT_RETRY triggered, so seek to the faulting offset.
  1221. * For the prefault case (!pages) we only update counts.
  1222. */
  1223. if (likely(pages))
  1224. pages += ret;
  1225. start += ret << PAGE_SHIFT;
  1226. lock_dropped = true;
  1227. retry:
  1228. /*
  1229. * Repeat on the address that fired VM_FAULT_RETRY
  1230. * with both FAULT_FLAG_ALLOW_RETRY and
  1231. * FAULT_FLAG_TRIED. Note that GUP can be interrupted
  1232. * by fatal signals, so we need to check it before we
  1233. * start trying again otherwise it can loop forever.
  1234. */
  1235. if (fatal_signal_pending(current)) {
  1236. if (!pages_done)
  1237. pages_done = -EINTR;
  1238. break;
  1239. }
  1240. ret = mmap_read_lock_killable(mm);
  1241. if (ret) {
  1242. BUG_ON(ret > 0);
  1243. if (!pages_done)
  1244. pages_done = ret;
  1245. break;
  1246. }
  1247. *locked = 1;
  1248. ret = __get_user_pages(mm, start, 1, flags | FOLL_TRIED,
  1249. pages, NULL, locked);
  1250. if (!*locked) {
  1251. /* Continue to retry until we succeeded */
  1252. BUG_ON(ret != 0);
  1253. goto retry;
  1254. }
  1255. if (ret != 1) {
  1256. BUG_ON(ret > 1);
  1257. if (!pages_done)
  1258. pages_done = ret;
  1259. break;
  1260. }
  1261. nr_pages--;
  1262. pages_done++;
  1263. if (!nr_pages)
  1264. break;
  1265. if (likely(pages))
  1266. pages++;
  1267. start += PAGE_SIZE;
  1268. }
  1269. if (lock_dropped && *locked) {
  1270. /*
  1271. * We must let the caller know we temporarily dropped the lock
  1272. * and so the critical section protected by it was lost.
  1273. */
  1274. mmap_read_unlock(mm);
  1275. *locked = 0;
  1276. }
  1277. return pages_done;
  1278. }
  1279. /**
  1280. * populate_vma_page_range() - populate a range of pages in the vma.
  1281. * @vma: target vma
  1282. * @start: start address
  1283. * @end: end address
  1284. * @locked: whether the mmap_lock is still held
  1285. *
  1286. * This takes care of mlocking the pages too if VM_LOCKED is set.
  1287. *
  1288. * Return either number of pages pinned in the vma, or a negative error
  1289. * code on error.
  1290. *
  1291. * vma->vm_mm->mmap_lock must be held.
  1292. *
  1293. * If @locked is NULL, it may be held for read or write and will
  1294. * be unperturbed.
  1295. *
  1296. * If @locked is non-NULL, it must held for read only and may be
  1297. * released. If it's released, *@locked will be set to 0.
  1298. */
  1299. long populate_vma_page_range(struct vm_area_struct *vma,
  1300. unsigned long start, unsigned long end, int *locked)
  1301. {
  1302. struct mm_struct *mm = vma->vm_mm;
  1303. unsigned long nr_pages = (end - start) / PAGE_SIZE;
  1304. int gup_flags;
  1305. VM_BUG_ON(start & ~PAGE_MASK);
  1306. VM_BUG_ON(end & ~PAGE_MASK);
  1307. VM_BUG_ON_VMA(start < vma->vm_start, vma);
  1308. VM_BUG_ON_VMA(end > vma->vm_end, vma);
  1309. mmap_assert_locked(mm);
  1310. gup_flags = FOLL_TOUCH | FOLL_POPULATE | FOLL_MLOCK;
  1311. if (vma->vm_flags & VM_LOCKONFAULT)
  1312. gup_flags &= ~FOLL_POPULATE;
  1313. /*
  1314. * We want to touch writable mappings with a write fault in order
  1315. * to break COW, except for shared mappings because these don't COW
  1316. * and we would not want to dirty them for nothing.
  1317. */
  1318. if ((vma->vm_flags & (VM_WRITE | VM_SHARED)) == VM_WRITE)
  1319. gup_flags |= FOLL_WRITE;
  1320. /*
  1321. * We want mlock to succeed for regions that have any permissions
  1322. * other than PROT_NONE.
  1323. */
  1324. if (vma_is_accessible(vma))
  1325. gup_flags |= FOLL_FORCE;
  1326. /*
  1327. * We made sure addr is within a VMA, so the following will
  1328. * not result in a stack expansion that recurses back here.
  1329. */
  1330. return __get_user_pages(mm, start, nr_pages, gup_flags,
  1331. NULL, NULL, locked);
  1332. }
  1333. /*
  1334. * __mm_populate - populate and/or mlock pages within a range of address space.
  1335. *
  1336. * This is used to implement mlock() and the MAP_POPULATE / MAP_LOCKED mmap
  1337. * flags. VMAs must be already marked with the desired vm_flags, and
  1338. * mmap_lock must not be held.
  1339. */
  1340. int __mm_populate(unsigned long start, unsigned long len, int ignore_errors)
  1341. {
  1342. struct mm_struct *mm = current->mm;
  1343. unsigned long end, nstart, nend;
  1344. struct vm_area_struct *vma = NULL;
  1345. int locked = 0;
  1346. long ret = 0;
  1347. end = start + len;
  1348. for (nstart = start; nstart < end; nstart = nend) {
  1349. /*
  1350. * We want to fault in pages for [nstart; end) address range.
  1351. * Find first corresponding VMA.
  1352. */
  1353. if (!locked) {
  1354. locked = 1;
  1355. mmap_read_lock(mm);
  1356. vma = find_vma(mm, nstart);
  1357. } else if (nstart >= vma->vm_end)
  1358. vma = vma->vm_next;
  1359. if (!vma || vma->vm_start >= end)
  1360. break;
  1361. /*
  1362. * Set [nstart; nend) to intersection of desired address
  1363. * range with the first VMA. Also, skip undesirable VMA types.
  1364. */
  1365. nend = min(end, vma->vm_end);
  1366. if (vma->vm_flags & (VM_IO | VM_PFNMAP))
  1367. continue;
  1368. if (nstart < vma->vm_start)
  1369. nstart = vma->vm_start;
  1370. /*
  1371. * Now fault in a range of pages. populate_vma_page_range()
  1372. * double checks the vma flags, so that it won't mlock pages
  1373. * if the vma was already munlocked.
  1374. */
  1375. ret = populate_vma_page_range(vma, nstart, nend, &locked);
  1376. if (ret < 0) {
  1377. if (ignore_errors) {
  1378. ret = 0;
  1379. continue; /* continue at next VMA */
  1380. }
  1381. break;
  1382. }
  1383. nend = nstart + ret * PAGE_SIZE;
  1384. ret = 0;
  1385. }
  1386. if (locked)
  1387. mmap_read_unlock(mm);
  1388. return ret; /* 0 or negative error code */
  1389. }
  1390. #else /* CONFIG_MMU */
  1391. static long __get_user_pages_locked(struct mm_struct *mm, unsigned long start,
  1392. unsigned long nr_pages, struct page **pages,
  1393. struct vm_area_struct **vmas, int *locked,
  1394. unsigned int foll_flags)
  1395. {
  1396. struct vm_area_struct *vma;
  1397. unsigned long vm_flags;
  1398. int i;
  1399. /* calculate required read or write permissions.
  1400. * If FOLL_FORCE is set, we only require the "MAY" flags.
  1401. */
  1402. vm_flags = (foll_flags & FOLL_WRITE) ?
  1403. (VM_WRITE | VM_MAYWRITE) : (VM_READ | VM_MAYREAD);
  1404. vm_flags &= (foll_flags & FOLL_FORCE) ?
  1405. (VM_MAYREAD | VM_MAYWRITE) : (VM_READ | VM_WRITE);
  1406. for (i = 0; i < nr_pages; i++) {
  1407. vma = find_vma(mm, start);
  1408. if (!vma)
  1409. goto finish_or_fault;
  1410. /* protect what we can, including chardevs */
  1411. if ((vma->vm_flags & (VM_IO | VM_PFNMAP)) ||
  1412. !(vm_flags & vma->vm_flags))
  1413. goto finish_or_fault;
  1414. if (pages) {
  1415. pages[i] = virt_to_page(start);
  1416. if (pages[i])
  1417. get_page(pages[i]);
  1418. }
  1419. if (vmas)
  1420. vmas[i] = vma;
  1421. start = (start + PAGE_SIZE) & PAGE_MASK;
  1422. }
  1423. return i;
  1424. finish_or_fault:
  1425. return i ? : -EFAULT;
  1426. }
  1427. #endif /* !CONFIG_MMU */
  1428. /**
  1429. * get_dump_page() - pin user page in memory while writing it to core dump
  1430. * @addr: user address
  1431. *
  1432. * Returns struct page pointer of user page pinned for dump,
  1433. * to be freed afterwards by put_page().
  1434. *
  1435. * Returns NULL on any kind of failure - a hole must then be inserted into
  1436. * the corefile, to preserve alignment with its headers; and also returns
  1437. * NULL wherever the ZERO_PAGE, or an anonymous pte_none, has been found -
  1438. * allowing a hole to be left in the corefile to save diskspace.
  1439. *
  1440. * Called without mmap_lock (takes and releases the mmap_lock by itself).
  1441. */
  1442. #ifdef CONFIG_ELF_CORE
  1443. struct page *get_dump_page(unsigned long addr)
  1444. {
  1445. struct mm_struct *mm = current->mm;
  1446. struct page *page;
  1447. int locked = 1;
  1448. int ret;
  1449. if (mmap_read_lock_killable(mm))
  1450. return NULL;
  1451. ret = __get_user_pages_locked(mm, addr, 1, &page, NULL, &locked,
  1452. FOLL_FORCE | FOLL_DUMP | FOLL_GET);
  1453. if (locked)
  1454. mmap_read_unlock(mm);
  1455. return (ret == 1) ? page : NULL;
  1456. }
  1457. #endif /* CONFIG_ELF_CORE */
  1458. #ifdef CONFIG_CMA
  1459. static long check_and_migrate_cma_pages(struct mm_struct *mm,
  1460. unsigned long start,
  1461. unsigned long nr_pages,
  1462. struct page **pages,
  1463. struct vm_area_struct **vmas,
  1464. unsigned int gup_flags)
  1465. {
  1466. unsigned long i, isolation_error_count;
  1467. bool drain_allow;
  1468. LIST_HEAD(cma_page_list);
  1469. long ret = nr_pages;
  1470. struct page *prev_head, *head;
  1471. struct migration_target_control mtc = {
  1472. .nid = NUMA_NO_NODE,
  1473. .gfp_mask = GFP_USER | __GFP_MOVABLE | __GFP_NOWARN,
  1474. };
  1475. check_again:
  1476. prev_head = NULL;
  1477. isolation_error_count = 0;
  1478. drain_allow = true;
  1479. for (i = 0; i < nr_pages; i++) {
  1480. head = compound_head(pages[i]);
  1481. if (head == prev_head)
  1482. continue;
  1483. prev_head = head;
  1484. /*
  1485. * If we get a page from the CMA zone, since we are going to
  1486. * be pinning these entries, we might as well move them out
  1487. * of the CMA zone if possible.
  1488. */
  1489. if (is_migrate_cma_page(head)) {
  1490. if (PageHuge(head)) {
  1491. if (!isolate_huge_page(head, &cma_page_list))
  1492. isolation_error_count++;
  1493. } else {
  1494. if (!PageLRU(head) && drain_allow) {
  1495. lru_add_drain_all();
  1496. drain_allow = false;
  1497. }
  1498. if (isolate_lru_page(head)) {
  1499. isolation_error_count++;
  1500. continue;
  1501. }
  1502. list_add_tail(&head->lru, &cma_page_list);
  1503. mod_node_page_state(page_pgdat(head),
  1504. NR_ISOLATED_ANON +
  1505. page_is_file_lru(head),
  1506. thp_nr_pages(head));
  1507. }
  1508. }
  1509. }
  1510. /*
  1511. * If list is empty, and no isolation errors, means that all pages are
  1512. * in the correct zone.
  1513. */
  1514. if (list_empty(&cma_page_list) && !isolation_error_count)
  1515. return ret;
  1516. if (!list_empty(&cma_page_list)) {
  1517. /*
  1518. * drop the above get_user_pages reference.
  1519. */
  1520. if (gup_flags & FOLL_PIN)
  1521. unpin_user_pages(pages, nr_pages);
  1522. else
  1523. for (i = 0; i < nr_pages; i++)
  1524. put_page(pages[i]);
  1525. ret = migrate_pages(&cma_page_list, alloc_migration_target,
  1526. NULL, (unsigned long)&mtc, MIGRATE_SYNC,
  1527. MR_CONTIG_RANGE);
  1528. if (ret) {
  1529. if (!list_empty(&cma_page_list))
  1530. putback_movable_pages(&cma_page_list);
  1531. return ret > 0 ? -ENOMEM : ret;
  1532. }
  1533. /* We unpinned pages before migration, pin them again */
  1534. ret = __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
  1535. NULL, gup_flags);
  1536. if (ret <= 0)
  1537. return ret;
  1538. nr_pages = ret;
  1539. }
  1540. /*
  1541. * check again because pages were unpinned, and we also might have
  1542. * had isolation errors and need more pages to migrate.
  1543. */
  1544. goto check_again;
  1545. }
  1546. #else
  1547. static long check_and_migrate_cma_pages(struct mm_struct *mm,
  1548. unsigned long start,
  1549. unsigned long nr_pages,
  1550. struct page **pages,
  1551. struct vm_area_struct **vmas,
  1552. unsigned int gup_flags)
  1553. {
  1554. return nr_pages;
  1555. }
  1556. #endif /* CONFIG_CMA */
  1557. /*
  1558. * __gup_longterm_locked() is a wrapper for __get_user_pages_locked which
  1559. * allows us to process the FOLL_LONGTERM flag.
  1560. */
  1561. static long __gup_longterm_locked(struct mm_struct *mm,
  1562. unsigned long start,
  1563. unsigned long nr_pages,
  1564. struct page **pages,
  1565. struct vm_area_struct **vmas,
  1566. unsigned int gup_flags)
  1567. {
  1568. unsigned long flags = 0;
  1569. long rc;
  1570. if (gup_flags & FOLL_LONGTERM)
  1571. flags = memalloc_nocma_save();
  1572. rc = __get_user_pages_locked(mm, start, nr_pages, pages, vmas, NULL,
  1573. gup_flags);
  1574. if (gup_flags & FOLL_LONGTERM) {
  1575. if (rc > 0)
  1576. rc = check_and_migrate_cma_pages(mm, start, rc, pages,
  1577. vmas, gup_flags);
  1578. memalloc_nocma_restore(flags);
  1579. }
  1580. return rc;
  1581. }
  1582. static bool is_valid_gup_flags(unsigned int gup_flags)
  1583. {
  1584. /*
  1585. * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
  1586. * never directly by the caller, so enforce that with an assertion:
  1587. */
  1588. if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
  1589. return false;
  1590. /*
  1591. * FOLL_PIN is a prerequisite to FOLL_LONGTERM. Another way of saying
  1592. * that is, FOLL_LONGTERM is a specific case, more restrictive case of
  1593. * FOLL_PIN.
  1594. */
  1595. if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
  1596. return false;
  1597. return true;
  1598. }
  1599. #ifdef CONFIG_MMU
  1600. static long __get_user_pages_remote(struct mm_struct *mm,
  1601. unsigned long start, unsigned long nr_pages,
  1602. unsigned int gup_flags, struct page **pages,
  1603. struct vm_area_struct **vmas, int *locked)
  1604. {
  1605. /*
  1606. * Parts of FOLL_LONGTERM behavior are incompatible with
  1607. * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
  1608. * vmas. However, this only comes up if locked is set, and there are
  1609. * callers that do request FOLL_LONGTERM, but do not set locked. So,
  1610. * allow what we can.
  1611. */
  1612. if (gup_flags & FOLL_LONGTERM) {
  1613. if (WARN_ON_ONCE(locked))
  1614. return -EINVAL;
  1615. /*
  1616. * This will check the vmas (even if our vmas arg is NULL)
  1617. * and return -ENOTSUPP if DAX isn't allowed in this case:
  1618. */
  1619. return __gup_longterm_locked(mm, start, nr_pages, pages,
  1620. vmas, gup_flags | FOLL_TOUCH |
  1621. FOLL_REMOTE);
  1622. }
  1623. return __get_user_pages_locked(mm, start, nr_pages, pages, vmas,
  1624. locked,
  1625. gup_flags | FOLL_TOUCH | FOLL_REMOTE);
  1626. }
  1627. /**
  1628. * get_user_pages_remote() - pin user pages in memory
  1629. * @mm: mm_struct of target mm
  1630. * @start: starting user address
  1631. * @nr_pages: number of pages from start to pin
  1632. * @gup_flags: flags modifying lookup behaviour
  1633. * @pages: array that receives pointers to the pages pinned.
  1634. * Should be at least nr_pages long. Or NULL, if caller
  1635. * only intends to ensure the pages are faulted in.
  1636. * @vmas: array of pointers to vmas corresponding to each page.
  1637. * Or NULL if the caller does not require them.
  1638. * @locked: pointer to lock flag indicating whether lock is held and
  1639. * subsequently whether VM_FAULT_RETRY functionality can be
  1640. * utilised. Lock must initially be held.
  1641. *
  1642. * Returns either number of pages pinned (which may be less than the
  1643. * number requested), or an error. Details about the return value:
  1644. *
  1645. * -- If nr_pages is 0, returns 0.
  1646. * -- If nr_pages is >0, but no pages were pinned, returns -errno.
  1647. * -- If nr_pages is >0, and some pages were pinned, returns the number of
  1648. * pages pinned. Again, this may be less than nr_pages.
  1649. *
  1650. * The caller is responsible for releasing returned @pages, via put_page().
  1651. *
  1652. * @vmas are valid only as long as mmap_lock is held.
  1653. *
  1654. * Must be called with mmap_lock held for read or write.
  1655. *
  1656. * get_user_pages_remote walks a process's page tables and takes a reference
  1657. * to each struct page that each user address corresponds to at a given
  1658. * instant. That is, it takes the page that would be accessed if a user
  1659. * thread accesses the given user virtual address at that instant.
  1660. *
  1661. * This does not guarantee that the page exists in the user mappings when
  1662. * get_user_pages_remote returns, and there may even be a completely different
  1663. * page there in some cases (eg. if mmapped pagecache has been invalidated
  1664. * and subsequently re faulted). However it does guarantee that the page
  1665. * won't be freed completely. And mostly callers simply care that the page
  1666. * contains data that was valid *at some point in time*. Typically, an IO
  1667. * or similar operation cannot guarantee anything stronger anyway because
  1668. * locks can't be held over the syscall boundary.
  1669. *
  1670. * If gup_flags & FOLL_WRITE == 0, the page must not be written to. If the page
  1671. * is written to, set_page_dirty (or set_page_dirty_lock, as appropriate) must
  1672. * be called after the page is finished with, and before put_page is called.
  1673. *
  1674. * get_user_pages_remote is typically used for fewer-copy IO operations,
  1675. * to get a handle on the memory by some means other than accesses
  1676. * via the user virtual addresses. The pages may be submitted for
  1677. * DMA to devices or accessed via their kernel linear mapping (via the
  1678. * kmap APIs). Care should be taken to use the correct cache flushing APIs.
  1679. *
  1680. * See also get_user_pages_fast, for performance critical applications.
  1681. *
  1682. * get_user_pages_remote should be phased out in favor of
  1683. * get_user_pages_locked|unlocked or get_user_pages_fast. Nothing
  1684. * should use get_user_pages_remote because it cannot pass
  1685. * FAULT_FLAG_ALLOW_RETRY to handle_mm_fault.
  1686. */
  1687. long get_user_pages_remote(struct mm_struct *mm,
  1688. unsigned long start, unsigned long nr_pages,
  1689. unsigned int gup_flags, struct page **pages,
  1690. struct vm_area_struct **vmas, int *locked)
  1691. {
  1692. if (!is_valid_gup_flags(gup_flags))
  1693. return -EINVAL;
  1694. return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
  1695. pages, vmas, locked);
  1696. }
  1697. EXPORT_SYMBOL(get_user_pages_remote);
  1698. #else /* CONFIG_MMU */
  1699. long get_user_pages_remote(struct mm_struct *mm,
  1700. unsigned long start, unsigned long nr_pages,
  1701. unsigned int gup_flags, struct page **pages,
  1702. struct vm_area_struct **vmas, int *locked)
  1703. {
  1704. return 0;
  1705. }
  1706. static long __get_user_pages_remote(struct mm_struct *mm,
  1707. unsigned long start, unsigned long nr_pages,
  1708. unsigned int gup_flags, struct page **pages,
  1709. struct vm_area_struct **vmas, int *locked)
  1710. {
  1711. return 0;
  1712. }
  1713. #endif /* !CONFIG_MMU */
  1714. /**
  1715. * get_user_pages() - pin user pages in memory
  1716. * @start: starting user address
  1717. * @nr_pages: number of pages from start to pin
  1718. * @gup_flags: flags modifying lookup behaviour
  1719. * @pages: array that receives pointers to the pages pinned.
  1720. * Should be at least nr_pages long. Or NULL, if caller
  1721. * only intends to ensure the pages are faulted in.
  1722. * @vmas: array of pointers to vmas corresponding to each page.
  1723. * Or NULL if the caller does not require them.
  1724. *
  1725. * This is the same as get_user_pages_remote(), just with a less-flexible
  1726. * calling convention where we assume that the mm being operated on belongs to
  1727. * the current task, and doesn't allow passing of a locked parameter. We also
  1728. * obviously don't pass FOLL_REMOTE in here.
  1729. */
  1730. long get_user_pages(unsigned long start, unsigned long nr_pages,
  1731. unsigned int gup_flags, struct page **pages,
  1732. struct vm_area_struct **vmas)
  1733. {
  1734. if (!is_valid_gup_flags(gup_flags))
  1735. return -EINVAL;
  1736. return __gup_longterm_locked(current->mm, start, nr_pages,
  1737. pages, vmas, gup_flags | FOLL_TOUCH);
  1738. }
  1739. EXPORT_SYMBOL(get_user_pages);
  1740. /**
  1741. * get_user_pages_locked() is suitable to replace the form:
  1742. *
  1743. * mmap_read_lock(mm);
  1744. * do_something()
  1745. * get_user_pages(mm, ..., pages, NULL);
  1746. * mmap_read_unlock(mm);
  1747. *
  1748. * to:
  1749. *
  1750. * int locked = 1;
  1751. * mmap_read_lock(mm);
  1752. * do_something()
  1753. * get_user_pages_locked(mm, ..., pages, &locked);
  1754. * if (locked)
  1755. * mmap_read_unlock(mm);
  1756. *
  1757. * @start: starting user address
  1758. * @nr_pages: number of pages from start to pin
  1759. * @gup_flags: flags modifying lookup behaviour
  1760. * @pages: array that receives pointers to the pages pinned.
  1761. * Should be at least nr_pages long. Or NULL, if caller
  1762. * only intends to ensure the pages are faulted in.
  1763. * @locked: pointer to lock flag indicating whether lock is held and
  1764. * subsequently whether VM_FAULT_RETRY functionality can be
  1765. * utilised. Lock must initially be held.
  1766. *
  1767. * We can leverage the VM_FAULT_RETRY functionality in the page fault
  1768. * paths better by using either get_user_pages_locked() or
  1769. * get_user_pages_unlocked().
  1770. *
  1771. */
  1772. long get_user_pages_locked(unsigned long start, unsigned long nr_pages,
  1773. unsigned int gup_flags, struct page **pages,
  1774. int *locked)
  1775. {
  1776. /*
  1777. * FIXME: Current FOLL_LONGTERM behavior is incompatible with
  1778. * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
  1779. * vmas. As there are no users of this flag in this call we simply
  1780. * disallow this option for now.
  1781. */
  1782. if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
  1783. return -EINVAL;
  1784. /*
  1785. * FOLL_PIN must only be set internally by the pin_user_pages*() APIs,
  1786. * never directly by the caller, so enforce that:
  1787. */
  1788. if (WARN_ON_ONCE(gup_flags & FOLL_PIN))
  1789. return -EINVAL;
  1790. return __get_user_pages_locked(current->mm, start, nr_pages,
  1791. pages, NULL, locked,
  1792. gup_flags | FOLL_TOUCH);
  1793. }
  1794. EXPORT_SYMBOL(get_user_pages_locked);
  1795. /*
  1796. * get_user_pages_unlocked() is suitable to replace the form:
  1797. *
  1798. * mmap_read_lock(mm);
  1799. * get_user_pages(mm, ..., pages, NULL);
  1800. * mmap_read_unlock(mm);
  1801. *
  1802. * with:
  1803. *
  1804. * get_user_pages_unlocked(mm, ..., pages);
  1805. *
  1806. * It is functionally equivalent to get_user_pages_fast so
  1807. * get_user_pages_fast should be used instead if specific gup_flags
  1808. * (e.g. FOLL_FORCE) are not required.
  1809. */
  1810. long get_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
  1811. struct page **pages, unsigned int gup_flags)
  1812. {
  1813. struct mm_struct *mm = current->mm;
  1814. int locked = 1;
  1815. long ret;
  1816. /*
  1817. * FIXME: Current FOLL_LONGTERM behavior is incompatible with
  1818. * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
  1819. * vmas. As there are no users of this flag in this call we simply
  1820. * disallow this option for now.
  1821. */
  1822. if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
  1823. return -EINVAL;
  1824. mmap_read_lock(mm);
  1825. ret = __get_user_pages_locked(mm, start, nr_pages, pages, NULL,
  1826. &locked, gup_flags | FOLL_TOUCH);
  1827. if (locked)
  1828. mmap_read_unlock(mm);
  1829. return ret;
  1830. }
  1831. EXPORT_SYMBOL(get_user_pages_unlocked);
  1832. /*
  1833. * Fast GUP
  1834. *
  1835. * get_user_pages_fast attempts to pin user pages by walking the page
  1836. * tables directly and avoids taking locks. Thus the walker needs to be
  1837. * protected from page table pages being freed from under it, and should
  1838. * block any THP splits.
  1839. *
  1840. * One way to achieve this is to have the walker disable interrupts, and
  1841. * rely on IPIs from the TLB flushing code blocking before the page table
  1842. * pages are freed. This is unsuitable for architectures that do not need
  1843. * to broadcast an IPI when invalidating TLBs.
  1844. *
  1845. * Another way to achieve this is to batch up page table containing pages
  1846. * belonging to more than one mm_user, then rcu_sched a callback to free those
  1847. * pages. Disabling interrupts will allow the fast_gup walker to both block
  1848. * the rcu_sched callback, and an IPI that we broadcast for splitting THPs
  1849. * (which is a relatively rare event). The code below adopts this strategy.
  1850. *
  1851. * Before activating this code, please be aware that the following assumptions
  1852. * are currently made:
  1853. *
  1854. * *) Either MMU_GATHER_RCU_TABLE_FREE is enabled, and tlb_remove_table() is used to
  1855. * free pages containing page tables or TLB flushing requires IPI broadcast.
  1856. *
  1857. * *) ptes can be read atomically by the architecture.
  1858. *
  1859. * *) access_ok is sufficient to validate userspace address ranges.
  1860. *
  1861. * The last two assumptions can be relaxed by the addition of helper functions.
  1862. *
  1863. * This code is based heavily on the PowerPC implementation by Nick Piggin.
  1864. */
  1865. #ifdef CONFIG_HAVE_FAST_GUP
  1866. #ifdef CONFIG_GUP_GET_PTE_LOW_HIGH
  1867. /*
  1868. * WARNING: only to be used in the get_user_pages_fast() implementation.
  1869. *
  1870. * With get_user_pages_fast(), we walk down the pagetables without taking any
  1871. * locks. For this we would like to load the pointers atomically, but sometimes
  1872. * that is not possible (e.g. without expensive cmpxchg8b on x86_32 PAE). What
  1873. * we do have is the guarantee that a PTE will only either go from not present
  1874. * to present, or present to not present or both -- it will not switch to a
  1875. * completely different present page without a TLB flush in between; something
  1876. * that we are blocking by holding interrupts off.
  1877. *
  1878. * Setting ptes from not present to present goes:
  1879. *
  1880. * ptep->pte_high = h;
  1881. * smp_wmb();
  1882. * ptep->pte_low = l;
  1883. *
  1884. * And present to not present goes:
  1885. *
  1886. * ptep->pte_low = 0;
  1887. * smp_wmb();
  1888. * ptep->pte_high = 0;
  1889. *
  1890. * We must ensure here that the load of pte_low sees 'l' IFF pte_high sees 'h'.
  1891. * We load pte_high *after* loading pte_low, which ensures we don't see an older
  1892. * value of pte_high. *Then* we recheck pte_low, which ensures that we haven't
  1893. * picked up a changed pte high. We might have gotten rubbish values from
  1894. * pte_low and pte_high, but we are guaranteed that pte_low will not have the
  1895. * present bit set *unless* it is 'l'. Because get_user_pages_fast() only
  1896. * operates on present ptes we're safe.
  1897. */
  1898. static inline pte_t gup_get_pte(pte_t *ptep)
  1899. {
  1900. pte_t pte;
  1901. do {
  1902. pte.pte_low = ptep->pte_low;
  1903. smp_rmb();
  1904. pte.pte_high = ptep->pte_high;
  1905. smp_rmb();
  1906. } while (unlikely(pte.pte_low != ptep->pte_low));
  1907. return pte;
  1908. }
  1909. #else /* CONFIG_GUP_GET_PTE_LOW_HIGH */
  1910. /*
  1911. * We require that the PTE can be read atomically.
  1912. */
  1913. static inline pte_t gup_get_pte(pte_t *ptep)
  1914. {
  1915. return ptep_get(ptep);
  1916. }
  1917. #endif /* CONFIG_GUP_GET_PTE_LOW_HIGH */
  1918. static void __maybe_unused undo_dev_pagemap(int *nr, int nr_start,
  1919. unsigned int flags,
  1920. struct page **pages)
  1921. {
  1922. while ((*nr) - nr_start) {
  1923. struct page *page = pages[--(*nr)];
  1924. ClearPageReferenced(page);
  1925. if (flags & FOLL_PIN)
  1926. unpin_user_page(page);
  1927. else
  1928. put_page(page);
  1929. }
  1930. }
  1931. #ifdef CONFIG_ARCH_HAS_PTE_SPECIAL
  1932. static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
  1933. unsigned int flags, struct page **pages, int *nr)
  1934. {
  1935. struct dev_pagemap *pgmap = NULL;
  1936. int nr_start = *nr, ret = 0;
  1937. pte_t *ptep, *ptem;
  1938. ptem = ptep = pte_offset_map(&pmd, addr);
  1939. do {
  1940. pte_t pte = gup_get_pte(ptep);
  1941. struct page *head, *page;
  1942. /*
  1943. * Similar to the PMD case below, NUMA hinting must take slow
  1944. * path using the pte_protnone check.
  1945. */
  1946. if (pte_protnone(pte))
  1947. goto pte_unmap;
  1948. if (!pte_access_permitted(pte, flags & FOLL_WRITE))
  1949. goto pte_unmap;
  1950. if (pte_devmap(pte)) {
  1951. if (unlikely(flags & FOLL_LONGTERM))
  1952. goto pte_unmap;
  1953. pgmap = get_dev_pagemap(pte_pfn(pte), pgmap);
  1954. if (unlikely(!pgmap)) {
  1955. undo_dev_pagemap(nr, nr_start, flags, pages);
  1956. goto pte_unmap;
  1957. }
  1958. } else if (pte_special(pte))
  1959. goto pte_unmap;
  1960. VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  1961. page = pte_page(pte);
  1962. head = try_grab_compound_head(page, 1, flags);
  1963. if (!head)
  1964. goto pte_unmap;
  1965. if (unlikely(pte_val(pte) != pte_val(*ptep))) {
  1966. put_compound_head(head, 1, flags);
  1967. goto pte_unmap;
  1968. }
  1969. VM_BUG_ON_PAGE(compound_head(page) != head, page);
  1970. /*
  1971. * We need to make the page accessible if and only if we are
  1972. * going to access its content (the FOLL_PIN case). Please
  1973. * see Documentation/core-api/pin_user_pages.rst for
  1974. * details.
  1975. */
  1976. if (flags & FOLL_PIN) {
  1977. ret = arch_make_page_accessible(page);
  1978. if (ret) {
  1979. unpin_user_page(page);
  1980. goto pte_unmap;
  1981. }
  1982. }
  1983. SetPageReferenced(page);
  1984. pages[*nr] = page;
  1985. (*nr)++;
  1986. } while (ptep++, addr += PAGE_SIZE, addr != end);
  1987. ret = 1;
  1988. pte_unmap:
  1989. if (pgmap)
  1990. put_dev_pagemap(pgmap);
  1991. pte_unmap(ptem);
  1992. return ret;
  1993. }
  1994. #else
  1995. /*
  1996. * If we can't determine whether or not a pte is special, then fail immediately
  1997. * for ptes. Note, we can still pin HugeTLB and THP as these are guaranteed not
  1998. * to be special.
  1999. *
  2000. * For a futex to be placed on a THP tail page, get_futex_key requires a
  2001. * get_user_pages_fast_only implementation that can pin pages. Thus it's still
  2002. * useful to have gup_huge_pmd even if we can't operate on ptes.
  2003. */
  2004. static int gup_pte_range(pmd_t pmd, unsigned long addr, unsigned long end,
  2005. unsigned int flags, struct page **pages, int *nr)
  2006. {
  2007. return 0;
  2008. }
  2009. #endif /* CONFIG_ARCH_HAS_PTE_SPECIAL */
  2010. #if defined(CONFIG_ARCH_HAS_PTE_DEVMAP) && defined(CONFIG_TRANSPARENT_HUGEPAGE)
  2011. static int __gup_device_huge(unsigned long pfn, unsigned long addr,
  2012. unsigned long end, unsigned int flags,
  2013. struct page **pages, int *nr)
  2014. {
  2015. int nr_start = *nr;
  2016. struct dev_pagemap *pgmap = NULL;
  2017. do {
  2018. struct page *page = pfn_to_page(pfn);
  2019. pgmap = get_dev_pagemap(pfn, pgmap);
  2020. if (unlikely(!pgmap)) {
  2021. undo_dev_pagemap(nr, nr_start, flags, pages);
  2022. return 0;
  2023. }
  2024. SetPageReferenced(page);
  2025. pages[*nr] = page;
  2026. if (unlikely(!try_grab_page(page, flags))) {
  2027. undo_dev_pagemap(nr, nr_start, flags, pages);
  2028. return 0;
  2029. }
  2030. (*nr)++;
  2031. pfn++;
  2032. } while (addr += PAGE_SIZE, addr != end);
  2033. if (pgmap)
  2034. put_dev_pagemap(pgmap);
  2035. return 1;
  2036. }
  2037. static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
  2038. unsigned long end, unsigned int flags,
  2039. struct page **pages, int *nr)
  2040. {
  2041. unsigned long fault_pfn;
  2042. int nr_start = *nr;
  2043. fault_pfn = pmd_pfn(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  2044. if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
  2045. return 0;
  2046. if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
  2047. undo_dev_pagemap(nr, nr_start, flags, pages);
  2048. return 0;
  2049. }
  2050. return 1;
  2051. }
  2052. static int __gup_device_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
  2053. unsigned long end, unsigned int flags,
  2054. struct page **pages, int *nr)
  2055. {
  2056. unsigned long fault_pfn;
  2057. int nr_start = *nr;
  2058. fault_pfn = pud_pfn(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
  2059. if (!__gup_device_huge(fault_pfn, addr, end, flags, pages, nr))
  2060. return 0;
  2061. if (unlikely(pud_val(orig) != pud_val(*pudp))) {
  2062. undo_dev_pagemap(nr, nr_start, flags, pages);
  2063. return 0;
  2064. }
  2065. return 1;
  2066. }
  2067. #else
  2068. static int __gup_device_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
  2069. unsigned long end, unsigned int flags,
  2070. struct page **pages, int *nr)
  2071. {
  2072. BUILD_BUG();
  2073. return 0;
  2074. }
  2075. static int __gup_device_huge_pud(pud_t pud, pud_t *pudp, unsigned long addr,
  2076. unsigned long end, unsigned int flags,
  2077. struct page **pages, int *nr)
  2078. {
  2079. BUILD_BUG();
  2080. return 0;
  2081. }
  2082. #endif
  2083. static int record_subpages(struct page *page, unsigned long addr,
  2084. unsigned long end, struct page **pages)
  2085. {
  2086. int nr;
  2087. for (nr = 0; addr != end; addr += PAGE_SIZE)
  2088. pages[nr++] = page++;
  2089. return nr;
  2090. }
  2091. #ifdef CONFIG_ARCH_HAS_HUGEPD
  2092. static unsigned long hugepte_addr_end(unsigned long addr, unsigned long end,
  2093. unsigned long sz)
  2094. {
  2095. unsigned long __boundary = (addr + sz) & ~(sz-1);
  2096. return (__boundary - 1 < end - 1) ? __boundary : end;
  2097. }
  2098. static int gup_hugepte(pte_t *ptep, unsigned long sz, unsigned long addr,
  2099. unsigned long end, unsigned int flags,
  2100. struct page **pages, int *nr)
  2101. {
  2102. unsigned long pte_end;
  2103. struct page *head, *page;
  2104. pte_t pte;
  2105. int refs;
  2106. pte_end = (addr + sz) & ~(sz-1);
  2107. if (pte_end < end)
  2108. end = pte_end;
  2109. pte = huge_ptep_get(ptep);
  2110. if (!pte_access_permitted(pte, flags & FOLL_WRITE))
  2111. return 0;
  2112. /* hugepages are never "special" */
  2113. VM_BUG_ON(!pfn_valid(pte_pfn(pte)));
  2114. head = pte_page(pte);
  2115. page = head + ((addr & (sz-1)) >> PAGE_SHIFT);
  2116. refs = record_subpages(page, addr, end, pages + *nr);
  2117. head = try_grab_compound_head(head, refs, flags);
  2118. if (!head)
  2119. return 0;
  2120. if (unlikely(pte_val(pte) != pte_val(*ptep))) {
  2121. put_compound_head(head, refs, flags);
  2122. return 0;
  2123. }
  2124. *nr += refs;
  2125. SetPageReferenced(head);
  2126. return 1;
  2127. }
  2128. static int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
  2129. unsigned int pdshift, unsigned long end, unsigned int flags,
  2130. struct page **pages, int *nr)
  2131. {
  2132. pte_t *ptep;
  2133. unsigned long sz = 1UL << hugepd_shift(hugepd);
  2134. unsigned long next;
  2135. ptep = hugepte_offset(hugepd, addr, pdshift);
  2136. do {
  2137. next = hugepte_addr_end(addr, end, sz);
  2138. if (!gup_hugepte(ptep, sz, addr, end, flags, pages, nr))
  2139. return 0;
  2140. } while (ptep++, addr = next, addr != end);
  2141. return 1;
  2142. }
  2143. #else
  2144. static inline int gup_huge_pd(hugepd_t hugepd, unsigned long addr,
  2145. unsigned int pdshift, unsigned long end, unsigned int flags,
  2146. struct page **pages, int *nr)
  2147. {
  2148. return 0;
  2149. }
  2150. #endif /* CONFIG_ARCH_HAS_HUGEPD */
  2151. static int gup_huge_pmd(pmd_t orig, pmd_t *pmdp, unsigned long addr,
  2152. unsigned long end, unsigned int flags,
  2153. struct page **pages, int *nr)
  2154. {
  2155. struct page *head, *page;
  2156. int refs;
  2157. if (!pmd_access_permitted(orig, flags & FOLL_WRITE))
  2158. return 0;
  2159. if (pmd_devmap(orig)) {
  2160. if (unlikely(flags & FOLL_LONGTERM))
  2161. return 0;
  2162. return __gup_device_huge_pmd(orig, pmdp, addr, end, flags,
  2163. pages, nr);
  2164. }
  2165. page = pmd_page(orig) + ((addr & ~PMD_MASK) >> PAGE_SHIFT);
  2166. refs = record_subpages(page, addr, end, pages + *nr);
  2167. head = try_grab_compound_head(pmd_page(orig), refs, flags);
  2168. if (!head)
  2169. return 0;
  2170. if (unlikely(pmd_val(orig) != pmd_val(*pmdp))) {
  2171. put_compound_head(head, refs, flags);
  2172. return 0;
  2173. }
  2174. *nr += refs;
  2175. SetPageReferenced(head);
  2176. return 1;
  2177. }
  2178. static int gup_huge_pud(pud_t orig, pud_t *pudp, unsigned long addr,
  2179. unsigned long end, unsigned int flags,
  2180. struct page **pages, int *nr)
  2181. {
  2182. struct page *head, *page;
  2183. int refs;
  2184. if (!pud_access_permitted(orig, flags & FOLL_WRITE))
  2185. return 0;
  2186. if (pud_devmap(orig)) {
  2187. if (unlikely(flags & FOLL_LONGTERM))
  2188. return 0;
  2189. return __gup_device_huge_pud(orig, pudp, addr, end, flags,
  2190. pages, nr);
  2191. }
  2192. page = pud_page(orig) + ((addr & ~PUD_MASK) >> PAGE_SHIFT);
  2193. refs = record_subpages(page, addr, end, pages + *nr);
  2194. head = try_grab_compound_head(pud_page(orig), refs, flags);
  2195. if (!head)
  2196. return 0;
  2197. if (unlikely(pud_val(orig) != pud_val(*pudp))) {
  2198. put_compound_head(head, refs, flags);
  2199. return 0;
  2200. }
  2201. *nr += refs;
  2202. SetPageReferenced(head);
  2203. return 1;
  2204. }
  2205. static int gup_huge_pgd(pgd_t orig, pgd_t *pgdp, unsigned long addr,
  2206. unsigned long end, unsigned int flags,
  2207. struct page **pages, int *nr)
  2208. {
  2209. int refs;
  2210. struct page *head, *page;
  2211. if (!pgd_access_permitted(orig, flags & FOLL_WRITE))
  2212. return 0;
  2213. BUILD_BUG_ON(pgd_devmap(orig));
  2214. page = pgd_page(orig) + ((addr & ~PGDIR_MASK) >> PAGE_SHIFT);
  2215. refs = record_subpages(page, addr, end, pages + *nr);
  2216. head = try_grab_compound_head(pgd_page(orig), refs, flags);
  2217. if (!head)
  2218. return 0;
  2219. if (unlikely(pgd_val(orig) != pgd_val(*pgdp))) {
  2220. put_compound_head(head, refs, flags);
  2221. return 0;
  2222. }
  2223. *nr += refs;
  2224. SetPageReferenced(head);
  2225. return 1;
  2226. }
  2227. static int gup_pmd_range(pud_t *pudp, pud_t pud, unsigned long addr, unsigned long end,
  2228. unsigned int flags, struct page **pages, int *nr)
  2229. {
  2230. unsigned long next;
  2231. pmd_t *pmdp;
  2232. pmdp = pmd_offset_lockless(pudp, pud, addr);
  2233. do {
  2234. pmd_t pmd = READ_ONCE(*pmdp);
  2235. next = pmd_addr_end(addr, end);
  2236. if (!pmd_present(pmd))
  2237. return 0;
  2238. if (unlikely(pmd_trans_huge(pmd) || pmd_huge(pmd) ||
  2239. pmd_devmap(pmd))) {
  2240. /*
  2241. * NUMA hinting faults need to be handled in the GUP
  2242. * slowpath for accounting purposes and so that they
  2243. * can be serialised against THP migration.
  2244. */
  2245. if (pmd_protnone(pmd))
  2246. return 0;
  2247. if (!gup_huge_pmd(pmd, pmdp, addr, next, flags,
  2248. pages, nr))
  2249. return 0;
  2250. } else if (unlikely(is_hugepd(__hugepd(pmd_val(pmd))))) {
  2251. /*
  2252. * architecture have different format for hugetlbfs
  2253. * pmd format and THP pmd format
  2254. */
  2255. if (!gup_huge_pd(__hugepd(pmd_val(pmd)), addr,
  2256. PMD_SHIFT, next, flags, pages, nr))
  2257. return 0;
  2258. } else if (!gup_pte_range(pmd, addr, next, flags, pages, nr))
  2259. return 0;
  2260. } while (pmdp++, addr = next, addr != end);
  2261. return 1;
  2262. }
  2263. static int gup_pud_range(p4d_t *p4dp, p4d_t p4d, unsigned long addr, unsigned long end,
  2264. unsigned int flags, struct page **pages, int *nr)
  2265. {
  2266. unsigned long next;
  2267. pud_t *pudp;
  2268. pudp = pud_offset_lockless(p4dp, p4d, addr);
  2269. do {
  2270. pud_t pud = READ_ONCE(*pudp);
  2271. next = pud_addr_end(addr, end);
  2272. if (unlikely(!pud_present(pud)))
  2273. return 0;
  2274. if (unlikely(pud_huge(pud))) {
  2275. if (!gup_huge_pud(pud, pudp, addr, next, flags,
  2276. pages, nr))
  2277. return 0;
  2278. } else if (unlikely(is_hugepd(__hugepd(pud_val(pud))))) {
  2279. if (!gup_huge_pd(__hugepd(pud_val(pud)), addr,
  2280. PUD_SHIFT, next, flags, pages, nr))
  2281. return 0;
  2282. } else if (!gup_pmd_range(pudp, pud, addr, next, flags, pages, nr))
  2283. return 0;
  2284. } while (pudp++, addr = next, addr != end);
  2285. return 1;
  2286. }
  2287. static int gup_p4d_range(pgd_t *pgdp, pgd_t pgd, unsigned long addr, unsigned long end,
  2288. unsigned int flags, struct page **pages, int *nr)
  2289. {
  2290. unsigned long next;
  2291. p4d_t *p4dp;
  2292. p4dp = p4d_offset_lockless(pgdp, pgd, addr);
  2293. do {
  2294. p4d_t p4d = READ_ONCE(*p4dp);
  2295. next = p4d_addr_end(addr, end);
  2296. if (p4d_none(p4d))
  2297. return 0;
  2298. BUILD_BUG_ON(p4d_huge(p4d));
  2299. if (unlikely(is_hugepd(__hugepd(p4d_val(p4d))))) {
  2300. if (!gup_huge_pd(__hugepd(p4d_val(p4d)), addr,
  2301. P4D_SHIFT, next, flags, pages, nr))
  2302. return 0;
  2303. } else if (!gup_pud_range(p4dp, p4d, addr, next, flags, pages, nr))
  2304. return 0;
  2305. } while (p4dp++, addr = next, addr != end);
  2306. return 1;
  2307. }
  2308. static void gup_pgd_range(unsigned long addr, unsigned long end,
  2309. unsigned int flags, struct page **pages, int *nr)
  2310. {
  2311. unsigned long next;
  2312. pgd_t *pgdp;
  2313. pgdp = pgd_offset(current->mm, addr);
  2314. do {
  2315. pgd_t pgd = READ_ONCE(*pgdp);
  2316. next = pgd_addr_end(addr, end);
  2317. if (pgd_none(pgd))
  2318. return;
  2319. if (unlikely(pgd_huge(pgd))) {
  2320. if (!gup_huge_pgd(pgd, pgdp, addr, next, flags,
  2321. pages, nr))
  2322. return;
  2323. } else if (unlikely(is_hugepd(__hugepd(pgd_val(pgd))))) {
  2324. if (!gup_huge_pd(__hugepd(pgd_val(pgd)), addr,
  2325. PGDIR_SHIFT, next, flags, pages, nr))
  2326. return;
  2327. } else if (!gup_p4d_range(pgdp, pgd, addr, next, flags, pages, nr))
  2328. return;
  2329. } while (pgdp++, addr = next, addr != end);
  2330. }
  2331. #else
  2332. static inline void gup_pgd_range(unsigned long addr, unsigned long end,
  2333. unsigned int flags, struct page **pages, int *nr)
  2334. {
  2335. }
  2336. #endif /* CONFIG_HAVE_FAST_GUP */
  2337. #ifndef gup_fast_permitted
  2338. /*
  2339. * Check if it's allowed to use get_user_pages_fast_only() for the range, or
  2340. * we need to fall back to the slow version:
  2341. */
  2342. static bool gup_fast_permitted(unsigned long start, unsigned long end)
  2343. {
  2344. return true;
  2345. }
  2346. #endif
  2347. static int __gup_longterm_unlocked(unsigned long start, int nr_pages,
  2348. unsigned int gup_flags, struct page **pages)
  2349. {
  2350. int ret;
  2351. /*
  2352. * FIXME: FOLL_LONGTERM does not work with
  2353. * get_user_pages_unlocked() (see comments in that function)
  2354. */
  2355. if (gup_flags & FOLL_LONGTERM) {
  2356. mmap_read_lock(current->mm);
  2357. ret = __gup_longterm_locked(current->mm,
  2358. start, nr_pages,
  2359. pages, NULL, gup_flags);
  2360. mmap_read_unlock(current->mm);
  2361. } else {
  2362. ret = get_user_pages_unlocked(start, nr_pages,
  2363. pages, gup_flags);
  2364. }
  2365. return ret;
  2366. }
  2367. static unsigned long lockless_pages_from_mm(unsigned long start,
  2368. unsigned long end,
  2369. unsigned int gup_flags,
  2370. struct page **pages)
  2371. {
  2372. unsigned long flags;
  2373. int nr_pinned = 0;
  2374. unsigned seq;
  2375. if (!IS_ENABLED(CONFIG_HAVE_FAST_GUP) ||
  2376. !gup_fast_permitted(start, end))
  2377. return 0;
  2378. if (gup_flags & FOLL_PIN) {
  2379. seq = raw_read_seqcount(&current->mm->write_protect_seq);
  2380. if (seq & 1)
  2381. return 0;
  2382. }
  2383. /*
  2384. * Disable interrupts. The nested form is used, in order to allow full,
  2385. * general purpose use of this routine.
  2386. *
  2387. * With interrupts disabled, we block page table pages from being freed
  2388. * from under us. See struct mmu_table_batch comments in
  2389. * include/asm-generic/tlb.h for more details.
  2390. *
  2391. * We do not adopt an rcu_read_lock() here as we also want to block IPIs
  2392. * that come from THPs splitting.
  2393. */
  2394. local_irq_save(flags);
  2395. gup_pgd_range(start, end, gup_flags, pages, &nr_pinned);
  2396. local_irq_restore(flags);
  2397. /*
  2398. * When pinning pages for DMA there could be a concurrent write protect
  2399. * from fork() via copy_page_range(), in this case always fail fast GUP.
  2400. */
  2401. if (gup_flags & FOLL_PIN) {
  2402. if (read_seqcount_retry(&current->mm->write_protect_seq, seq)) {
  2403. unpin_user_pages(pages, nr_pinned);
  2404. return 0;
  2405. }
  2406. }
  2407. return nr_pinned;
  2408. }
  2409. static int internal_get_user_pages_fast(unsigned long start,
  2410. unsigned long nr_pages,
  2411. unsigned int gup_flags,
  2412. struct page **pages)
  2413. {
  2414. unsigned long len, end;
  2415. unsigned long nr_pinned;
  2416. int ret;
  2417. if (WARN_ON_ONCE(gup_flags & ~(FOLL_WRITE | FOLL_LONGTERM |
  2418. FOLL_FORCE | FOLL_PIN | FOLL_GET |
  2419. FOLL_FAST_ONLY)))
  2420. return -EINVAL;
  2421. if (gup_flags & FOLL_PIN)
  2422. atomic_set(&current->mm->has_pinned, 1);
  2423. if (!(gup_flags & FOLL_FAST_ONLY))
  2424. might_lock_read(&current->mm->mmap_lock);
  2425. start = untagged_addr(start) & PAGE_MASK;
  2426. len = nr_pages << PAGE_SHIFT;
  2427. if (check_add_overflow(start, len, &end))
  2428. return 0;
  2429. if (unlikely(!access_ok((void __user *)start, len)))
  2430. return -EFAULT;
  2431. nr_pinned = lockless_pages_from_mm(start, end, gup_flags, pages);
  2432. if (nr_pinned == nr_pages || gup_flags & FOLL_FAST_ONLY)
  2433. return nr_pinned;
  2434. /* Slow path: try to get the remaining pages with get_user_pages */
  2435. start += nr_pinned << PAGE_SHIFT;
  2436. pages += nr_pinned;
  2437. ret = __gup_longterm_unlocked(start, nr_pages - nr_pinned, gup_flags,
  2438. pages);
  2439. if (ret < 0) {
  2440. /*
  2441. * The caller has to unpin the pages we already pinned so
  2442. * returning -errno is not an option
  2443. */
  2444. if (nr_pinned)
  2445. return nr_pinned;
  2446. return ret;
  2447. }
  2448. return ret + nr_pinned;
  2449. }
  2450. /**
  2451. * get_user_pages_fast_only() - pin user pages in memory
  2452. * @start: starting user address
  2453. * @nr_pages: number of pages from start to pin
  2454. * @gup_flags: flags modifying pin behaviour
  2455. * @pages: array that receives pointers to the pages pinned.
  2456. * Should be at least nr_pages long.
  2457. *
  2458. * Like get_user_pages_fast() except it's IRQ-safe in that it won't fall back to
  2459. * the regular GUP.
  2460. * Note a difference with get_user_pages_fast: this always returns the
  2461. * number of pages pinned, 0 if no pages were pinned.
  2462. *
  2463. * If the architecture does not support this function, simply return with no
  2464. * pages pinned.
  2465. *
  2466. * Careful, careful! COW breaking can go either way, so a non-write
  2467. * access can get ambiguous page results. If you call this function without
  2468. * 'write' set, you'd better be sure that you're ok with that ambiguity.
  2469. */
  2470. int get_user_pages_fast_only(unsigned long start, int nr_pages,
  2471. unsigned int gup_flags, struct page **pages)
  2472. {
  2473. int nr_pinned;
  2474. /*
  2475. * Internally (within mm/gup.c), gup fast variants must set FOLL_GET,
  2476. * because gup fast is always a "pin with a +1 page refcount" request.
  2477. *
  2478. * FOLL_FAST_ONLY is required in order to match the API description of
  2479. * this routine: no fall back to regular ("slow") GUP.
  2480. */
  2481. gup_flags |= FOLL_GET | FOLL_FAST_ONLY;
  2482. nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
  2483. pages);
  2484. /*
  2485. * As specified in the API description above, this routine is not
  2486. * allowed to return negative values. However, the common core
  2487. * routine internal_get_user_pages_fast() *can* return -errno.
  2488. * Therefore, correct for that here:
  2489. */
  2490. if (nr_pinned < 0)
  2491. nr_pinned = 0;
  2492. return nr_pinned;
  2493. }
  2494. EXPORT_SYMBOL_GPL(get_user_pages_fast_only);
  2495. /**
  2496. * get_user_pages_fast() - pin user pages in memory
  2497. * @start: starting user address
  2498. * @nr_pages: number of pages from start to pin
  2499. * @gup_flags: flags modifying pin behaviour
  2500. * @pages: array that receives pointers to the pages pinned.
  2501. * Should be at least nr_pages long.
  2502. *
  2503. * Attempt to pin user pages in memory without taking mm->mmap_lock.
  2504. * If not successful, it will fall back to taking the lock and
  2505. * calling get_user_pages().
  2506. *
  2507. * Returns number of pages pinned. This may be fewer than the number requested.
  2508. * If nr_pages is 0 or negative, returns 0. If no pages were pinned, returns
  2509. * -errno.
  2510. */
  2511. int get_user_pages_fast(unsigned long start, int nr_pages,
  2512. unsigned int gup_flags, struct page **pages)
  2513. {
  2514. if (!is_valid_gup_flags(gup_flags))
  2515. return -EINVAL;
  2516. /*
  2517. * The caller may or may not have explicitly set FOLL_GET; either way is
  2518. * OK. However, internally (within mm/gup.c), gup fast variants must set
  2519. * FOLL_GET, because gup fast is always a "pin with a +1 page refcount"
  2520. * request.
  2521. */
  2522. gup_flags |= FOLL_GET;
  2523. return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
  2524. }
  2525. EXPORT_SYMBOL_GPL(get_user_pages_fast);
  2526. /**
  2527. * pin_user_pages_fast() - pin user pages in memory without taking locks
  2528. *
  2529. * @start: starting user address
  2530. * @nr_pages: number of pages from start to pin
  2531. * @gup_flags: flags modifying pin behaviour
  2532. * @pages: array that receives pointers to the pages pinned.
  2533. * Should be at least nr_pages long.
  2534. *
  2535. * Nearly the same as get_user_pages_fast(), except that FOLL_PIN is set. See
  2536. * get_user_pages_fast() for documentation on the function arguments, because
  2537. * the arguments here are identical.
  2538. *
  2539. * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
  2540. * see Documentation/core-api/pin_user_pages.rst for further details.
  2541. */
  2542. int pin_user_pages_fast(unsigned long start, int nr_pages,
  2543. unsigned int gup_flags, struct page **pages)
  2544. {
  2545. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  2546. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2547. return -EINVAL;
  2548. gup_flags |= FOLL_PIN;
  2549. return internal_get_user_pages_fast(start, nr_pages, gup_flags, pages);
  2550. }
  2551. EXPORT_SYMBOL_GPL(pin_user_pages_fast);
  2552. /*
  2553. * This is the FOLL_PIN equivalent of get_user_pages_fast_only(). Behavior
  2554. * is the same, except that this one sets FOLL_PIN instead of FOLL_GET.
  2555. *
  2556. * The API rules are the same, too: no negative values may be returned.
  2557. */
  2558. int pin_user_pages_fast_only(unsigned long start, int nr_pages,
  2559. unsigned int gup_flags, struct page **pages)
  2560. {
  2561. int nr_pinned;
  2562. /*
  2563. * FOLL_GET and FOLL_PIN are mutually exclusive. Note that the API
  2564. * rules require returning 0, rather than -errno:
  2565. */
  2566. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2567. return 0;
  2568. /*
  2569. * FOLL_FAST_ONLY is required in order to match the API description of
  2570. * this routine: no fall back to regular ("slow") GUP.
  2571. */
  2572. gup_flags |= (FOLL_PIN | FOLL_FAST_ONLY);
  2573. nr_pinned = internal_get_user_pages_fast(start, nr_pages, gup_flags,
  2574. pages);
  2575. /*
  2576. * This routine is not allowed to return negative values. However,
  2577. * internal_get_user_pages_fast() *can* return -errno. Therefore,
  2578. * correct for that here:
  2579. */
  2580. if (nr_pinned < 0)
  2581. nr_pinned = 0;
  2582. return nr_pinned;
  2583. }
  2584. EXPORT_SYMBOL_GPL(pin_user_pages_fast_only);
  2585. /**
  2586. * pin_user_pages_remote() - pin pages of a remote process
  2587. *
  2588. * @mm: mm_struct of target mm
  2589. * @start: starting user address
  2590. * @nr_pages: number of pages from start to pin
  2591. * @gup_flags: flags modifying lookup behaviour
  2592. * @pages: array that receives pointers to the pages pinned.
  2593. * Should be at least nr_pages long. Or NULL, if caller
  2594. * only intends to ensure the pages are faulted in.
  2595. * @vmas: array of pointers to vmas corresponding to each page.
  2596. * Or NULL if the caller does not require them.
  2597. * @locked: pointer to lock flag indicating whether lock is held and
  2598. * subsequently whether VM_FAULT_RETRY functionality can be
  2599. * utilised. Lock must initially be held.
  2600. *
  2601. * Nearly the same as get_user_pages_remote(), except that FOLL_PIN is set. See
  2602. * get_user_pages_remote() for documentation on the function arguments, because
  2603. * the arguments here are identical.
  2604. *
  2605. * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
  2606. * see Documentation/core-api/pin_user_pages.rst for details.
  2607. */
  2608. long pin_user_pages_remote(struct mm_struct *mm,
  2609. unsigned long start, unsigned long nr_pages,
  2610. unsigned int gup_flags, struct page **pages,
  2611. struct vm_area_struct **vmas, int *locked)
  2612. {
  2613. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  2614. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2615. return -EINVAL;
  2616. gup_flags |= FOLL_PIN;
  2617. return __get_user_pages_remote(mm, start, nr_pages, gup_flags,
  2618. pages, vmas, locked);
  2619. }
  2620. EXPORT_SYMBOL(pin_user_pages_remote);
  2621. /**
  2622. * pin_user_pages() - pin user pages in memory for use by other devices
  2623. *
  2624. * @start: starting user address
  2625. * @nr_pages: number of pages from start to pin
  2626. * @gup_flags: flags modifying lookup behaviour
  2627. * @pages: array that receives pointers to the pages pinned.
  2628. * Should be at least nr_pages long. Or NULL, if caller
  2629. * only intends to ensure the pages are faulted in.
  2630. * @vmas: array of pointers to vmas corresponding to each page.
  2631. * Or NULL if the caller does not require them.
  2632. *
  2633. * Nearly the same as get_user_pages(), except that FOLL_TOUCH is not set, and
  2634. * FOLL_PIN is set.
  2635. *
  2636. * FOLL_PIN means that the pages must be released via unpin_user_page(). Please
  2637. * see Documentation/core-api/pin_user_pages.rst for details.
  2638. */
  2639. long pin_user_pages(unsigned long start, unsigned long nr_pages,
  2640. unsigned int gup_flags, struct page **pages,
  2641. struct vm_area_struct **vmas)
  2642. {
  2643. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  2644. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2645. return -EINVAL;
  2646. gup_flags |= FOLL_PIN;
  2647. return __gup_longterm_locked(current->mm, start, nr_pages,
  2648. pages, vmas, gup_flags);
  2649. }
  2650. EXPORT_SYMBOL(pin_user_pages);
  2651. /*
  2652. * pin_user_pages_unlocked() is the FOLL_PIN variant of
  2653. * get_user_pages_unlocked(). Behavior is the same, except that this one sets
  2654. * FOLL_PIN and rejects FOLL_GET.
  2655. */
  2656. long pin_user_pages_unlocked(unsigned long start, unsigned long nr_pages,
  2657. struct page **pages, unsigned int gup_flags)
  2658. {
  2659. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  2660. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2661. return -EINVAL;
  2662. gup_flags |= FOLL_PIN;
  2663. return get_user_pages_unlocked(start, nr_pages, pages, gup_flags);
  2664. }
  2665. EXPORT_SYMBOL(pin_user_pages_unlocked);
  2666. /*
  2667. * pin_user_pages_locked() is the FOLL_PIN variant of get_user_pages_locked().
  2668. * Behavior is the same, except that this one sets FOLL_PIN and rejects
  2669. * FOLL_GET.
  2670. */
  2671. long pin_user_pages_locked(unsigned long start, unsigned long nr_pages,
  2672. unsigned int gup_flags, struct page **pages,
  2673. int *locked)
  2674. {
  2675. /*
  2676. * FIXME: Current FOLL_LONGTERM behavior is incompatible with
  2677. * FAULT_FLAG_ALLOW_RETRY because of the FS DAX check requirement on
  2678. * vmas. As there are no users of this flag in this call we simply
  2679. * disallow this option for now.
  2680. */
  2681. if (WARN_ON_ONCE(gup_flags & FOLL_LONGTERM))
  2682. return -EINVAL;
  2683. /* FOLL_GET and FOLL_PIN are mutually exclusive. */
  2684. if (WARN_ON_ONCE(gup_flags & FOLL_GET))
  2685. return -EINVAL;
  2686. gup_flags |= FOLL_PIN;
  2687. return __get_user_pages_locked(current->mm, start, nr_pages,
  2688. pages, NULL, locked,
  2689. gup_flags | FOLL_TOUCH);
  2690. }
  2691. EXPORT_SYMBOL(pin_user_pages_locked);