filemap.c 63 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463
  1. /*
  2. * linux/mm/filemap.c
  3. *
  4. * Copyright (C) 1994-1999 Linus Torvalds
  5. */
  6. /*
  7. * This file handles the generic file mmap semantics used by
  8. * most "normal" filesystems (but you don't /have/ to use this:
  9. * the NFS filesystem used to do this differently, for example)
  10. */
  11. #include <linux/module.h>
  12. #include <linux/slab.h>
  13. #include <linux/compiler.h>
  14. #include <linux/fs.h>
  15. #include <linux/uaccess.h>
  16. #include <linux/aio.h>
  17. #include <linux/capability.h>
  18. #include <linux/kernel_stat.h>
  19. #include <linux/mm.h>
  20. #include <linux/swap.h>
  21. #include <linux/mman.h>
  22. #include <linux/pagemap.h>
  23. #include <linux/file.h>
  24. #include <linux/uio.h>
  25. #include <linux/hash.h>
  26. #include <linux/writeback.h>
  27. #include <linux/pagevec.h>
  28. #include <linux/blkdev.h>
  29. #include <linux/security.h>
  30. #include <linux/syscalls.h>
  31. #include <linux/cpuset.h>
  32. #include "filemap.h"
  33. #include "internal.h"
  34. /*
  35. * FIXME: remove all knowledge of the buffer layer from the core VM
  36. */
  37. #include <linux/buffer_head.h> /* for generic_osync_inode */
  38. #include <asm/mman.h>
  39. static ssize_t
  40. generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  41. loff_t offset, unsigned long nr_segs);
  42. /*
  43. * Shared mappings implemented 30.11.1994. It's not fully working yet,
  44. * though.
  45. *
  46. * Shared mappings now work. 15.8.1995 Bruno.
  47. *
  48. * finished 'unifying' the page and buffer cache and SMP-threaded the
  49. * page-cache, 21.05.1999, Ingo Molnar <mingo@redhat.com>
  50. *
  51. * SMP-threaded pagemap-LRU 1999, Andrea Arcangeli <andrea@suse.de>
  52. */
  53. /*
  54. * Lock ordering:
  55. *
  56. * ->i_mmap_lock (vmtruncate)
  57. * ->private_lock (__free_pte->__set_page_dirty_buffers)
  58. * ->swap_lock (exclusive_swap_page, others)
  59. * ->mapping->tree_lock
  60. *
  61. * ->i_mutex
  62. * ->i_mmap_lock (truncate->unmap_mapping_range)
  63. *
  64. * ->mmap_sem
  65. * ->i_mmap_lock
  66. * ->page_table_lock or pte_lock (various, mainly in memory.c)
  67. * ->mapping->tree_lock (arch-dependent flush_dcache_mmap_lock)
  68. *
  69. * ->mmap_sem
  70. * ->lock_page (access_process_vm)
  71. *
  72. * ->i_mutex (generic_file_buffered_write)
  73. * ->mmap_sem (fault_in_pages_readable->do_page_fault)
  74. *
  75. * ->i_mutex
  76. * ->i_alloc_sem (various)
  77. *
  78. * ->inode_lock
  79. * ->sb_lock (fs/fs-writeback.c)
  80. * ->mapping->tree_lock (__sync_single_inode)
  81. *
  82. * ->i_mmap_lock
  83. * ->anon_vma.lock (vma_adjust)
  84. *
  85. * ->anon_vma.lock
  86. * ->page_table_lock or pte_lock (anon_vma_prepare and various)
  87. *
  88. * ->page_table_lock or pte_lock
  89. * ->swap_lock (try_to_unmap_one)
  90. * ->private_lock (try_to_unmap_one)
  91. * ->tree_lock (try_to_unmap_one)
  92. * ->zone.lru_lock (follow_page->mark_page_accessed)
  93. * ->zone.lru_lock (check_pte_range->isolate_lru_page)
  94. * ->private_lock (page_remove_rmap->set_page_dirty)
  95. * ->tree_lock (page_remove_rmap->set_page_dirty)
  96. * ->inode_lock (page_remove_rmap->set_page_dirty)
  97. * ->inode_lock (zap_pte_range->set_page_dirty)
  98. * ->private_lock (zap_pte_range->__set_page_dirty_buffers)
  99. *
  100. * ->task->proc_lock
  101. * ->dcache_lock (proc_pid_lookup)
  102. */
  103. /*
  104. * Remove a page from the page cache and free it. Caller has to make
  105. * sure the page is locked and that nobody else uses it - or that usage
  106. * is safe. The caller must hold a write_lock on the mapping's tree_lock.
  107. */
  108. void __remove_from_page_cache(struct page *page)
  109. {
  110. struct address_space *mapping = page->mapping;
  111. radix_tree_delete(&mapping->page_tree, page->index);
  112. page->mapping = NULL;
  113. mapping->nrpages--;
  114. __dec_zone_page_state(page, NR_FILE_PAGES);
  115. }
  116. void remove_from_page_cache(struct page *page)
  117. {
  118. struct address_space *mapping = page->mapping;
  119. BUG_ON(!PageLocked(page));
  120. write_lock_irq(&mapping->tree_lock);
  121. __remove_from_page_cache(page);
  122. write_unlock_irq(&mapping->tree_lock);
  123. }
  124. static int sync_page(void *word)
  125. {
  126. struct address_space *mapping;
  127. struct page *page;
  128. page = container_of((unsigned long *)word, struct page, flags);
  129. /*
  130. * page_mapping() is being called without PG_locked held.
  131. * Some knowledge of the state and use of the page is used to
  132. * reduce the requirements down to a memory barrier.
  133. * The danger here is of a stale page_mapping() return value
  134. * indicating a struct address_space different from the one it's
  135. * associated with when it is associated with one.
  136. * After smp_mb(), it's either the correct page_mapping() for
  137. * the page, or an old page_mapping() and the page's own
  138. * page_mapping() has gone NULL.
  139. * The ->sync_page() address_space operation must tolerate
  140. * page_mapping() going NULL. By an amazing coincidence,
  141. * this comes about because none of the users of the page
  142. * in the ->sync_page() methods make essential use of the
  143. * page_mapping(), merely passing the page down to the backing
  144. * device's unplug functions when it's non-NULL, which in turn
  145. * ignore it for all cases but swap, where only page_private(page) is
  146. * of interest. When page_mapping() does go NULL, the entire
  147. * call stack gracefully ignores the page and returns.
  148. * -- wli
  149. */
  150. smp_mb();
  151. mapping = page_mapping(page);
  152. if (mapping && mapping->a_ops && mapping->a_ops->sync_page)
  153. mapping->a_ops->sync_page(page);
  154. io_schedule();
  155. return 0;
  156. }
  157. /**
  158. * __filemap_fdatawrite_range - start writeback on mapping dirty pages in range
  159. * @mapping: address space structure to write
  160. * @start: offset in bytes where the range starts
  161. * @end: offset in bytes where the range ends (inclusive)
  162. * @sync_mode: enable synchronous operation
  163. *
  164. * Start writeback against all of a mapping's dirty pages that lie
  165. * within the byte offsets <start, end> inclusive.
  166. *
  167. * If sync_mode is WB_SYNC_ALL then this is a "data integrity" operation, as
  168. * opposed to a regular memory cleansing writeback. The difference between
  169. * these two operations is that if a dirty page/buffer is encountered, it must
  170. * be waited upon, and not just skipped over.
  171. */
  172. int __filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  173. loff_t end, int sync_mode)
  174. {
  175. int ret;
  176. struct writeback_control wbc = {
  177. .sync_mode = sync_mode,
  178. .nr_to_write = mapping->nrpages * 2,
  179. .range_start = start,
  180. .range_end = end,
  181. };
  182. if (!mapping_cap_writeback_dirty(mapping))
  183. return 0;
  184. ret = do_writepages(mapping, &wbc);
  185. return ret;
  186. }
  187. static inline int __filemap_fdatawrite(struct address_space *mapping,
  188. int sync_mode)
  189. {
  190. return __filemap_fdatawrite_range(mapping, 0, LLONG_MAX, sync_mode);
  191. }
  192. int filemap_fdatawrite(struct address_space *mapping)
  193. {
  194. return __filemap_fdatawrite(mapping, WB_SYNC_ALL);
  195. }
  196. EXPORT_SYMBOL(filemap_fdatawrite);
  197. static int filemap_fdatawrite_range(struct address_space *mapping, loff_t start,
  198. loff_t end)
  199. {
  200. return __filemap_fdatawrite_range(mapping, start, end, WB_SYNC_ALL);
  201. }
  202. /**
  203. * filemap_flush - mostly a non-blocking flush
  204. * @mapping: target address_space
  205. *
  206. * This is a mostly non-blocking flush. Not suitable for data-integrity
  207. * purposes - I/O may not be started against all dirty pages.
  208. */
  209. int filemap_flush(struct address_space *mapping)
  210. {
  211. return __filemap_fdatawrite(mapping, WB_SYNC_NONE);
  212. }
  213. EXPORT_SYMBOL(filemap_flush);
  214. /**
  215. * wait_on_page_writeback_range - wait for writeback to complete
  216. * @mapping: target address_space
  217. * @start: beginning page index
  218. * @end: ending page index
  219. *
  220. * Wait for writeback to complete against pages indexed by start->end
  221. * inclusive
  222. */
  223. int wait_on_page_writeback_range(struct address_space *mapping,
  224. pgoff_t start, pgoff_t end)
  225. {
  226. struct pagevec pvec;
  227. int nr_pages;
  228. int ret = 0;
  229. pgoff_t index;
  230. if (end < start)
  231. return 0;
  232. pagevec_init(&pvec, 0);
  233. index = start;
  234. while ((index <= end) &&
  235. (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
  236. PAGECACHE_TAG_WRITEBACK,
  237. min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1)) != 0) {
  238. unsigned i;
  239. for (i = 0; i < nr_pages; i++) {
  240. struct page *page = pvec.pages[i];
  241. /* until radix tree lookup accepts end_index */
  242. if (page->index > end)
  243. continue;
  244. wait_on_page_writeback(page);
  245. if (PageError(page))
  246. ret = -EIO;
  247. }
  248. pagevec_release(&pvec);
  249. cond_resched();
  250. }
  251. /* Check for outstanding write errors */
  252. if (test_and_clear_bit(AS_ENOSPC, &mapping->flags))
  253. ret = -ENOSPC;
  254. if (test_and_clear_bit(AS_EIO, &mapping->flags))
  255. ret = -EIO;
  256. return ret;
  257. }
  258. /**
  259. * sync_page_range - write and wait on all pages in the passed range
  260. * @inode: target inode
  261. * @mapping: target address_space
  262. * @pos: beginning offset in pages to write
  263. * @count: number of bytes to write
  264. *
  265. * Write and wait upon all the pages in the passed range. This is a "data
  266. * integrity" operation. It waits upon in-flight writeout before starting and
  267. * waiting upon new writeout. If there was an IO error, return it.
  268. *
  269. * We need to re-take i_mutex during the generic_osync_inode list walk because
  270. * it is otherwise livelockable.
  271. */
  272. int sync_page_range(struct inode *inode, struct address_space *mapping,
  273. loff_t pos, loff_t count)
  274. {
  275. pgoff_t start = pos >> PAGE_CACHE_SHIFT;
  276. pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
  277. int ret;
  278. if (!mapping_cap_writeback_dirty(mapping) || !count)
  279. return 0;
  280. ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
  281. if (ret == 0) {
  282. mutex_lock(&inode->i_mutex);
  283. ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  284. mutex_unlock(&inode->i_mutex);
  285. }
  286. if (ret == 0)
  287. ret = wait_on_page_writeback_range(mapping, start, end);
  288. return ret;
  289. }
  290. EXPORT_SYMBOL(sync_page_range);
  291. /**
  292. * sync_page_range_nolock
  293. * @inode: target inode
  294. * @mapping: target address_space
  295. * @pos: beginning offset in pages to write
  296. * @count: number of bytes to write
  297. *
  298. * Note: Holding i_mutex across sync_page_range_nolock() is not a good idea
  299. * as it forces O_SYNC writers to different parts of the same file
  300. * to be serialised right until io completion.
  301. */
  302. int sync_page_range_nolock(struct inode *inode, struct address_space *mapping,
  303. loff_t pos, loff_t count)
  304. {
  305. pgoff_t start = pos >> PAGE_CACHE_SHIFT;
  306. pgoff_t end = (pos + count - 1) >> PAGE_CACHE_SHIFT;
  307. int ret;
  308. if (!mapping_cap_writeback_dirty(mapping) || !count)
  309. return 0;
  310. ret = filemap_fdatawrite_range(mapping, pos, pos + count - 1);
  311. if (ret == 0)
  312. ret = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  313. if (ret == 0)
  314. ret = wait_on_page_writeback_range(mapping, start, end);
  315. return ret;
  316. }
  317. EXPORT_SYMBOL(sync_page_range_nolock);
  318. /**
  319. * filemap_fdatawait - wait for all under-writeback pages to complete
  320. * @mapping: address space structure to wait for
  321. *
  322. * Walk the list of under-writeback pages of the given address space
  323. * and wait for all of them.
  324. */
  325. int filemap_fdatawait(struct address_space *mapping)
  326. {
  327. loff_t i_size = i_size_read(mapping->host);
  328. if (i_size == 0)
  329. return 0;
  330. return wait_on_page_writeback_range(mapping, 0,
  331. (i_size - 1) >> PAGE_CACHE_SHIFT);
  332. }
  333. EXPORT_SYMBOL(filemap_fdatawait);
  334. int filemap_write_and_wait(struct address_space *mapping)
  335. {
  336. int err = 0;
  337. if (mapping->nrpages) {
  338. err = filemap_fdatawrite(mapping);
  339. /*
  340. * Even if the above returned error, the pages may be
  341. * written partially (e.g. -ENOSPC), so we wait for it.
  342. * But the -EIO is special case, it may indicate the worst
  343. * thing (e.g. bug) happened, so we avoid waiting for it.
  344. */
  345. if (err != -EIO) {
  346. int err2 = filemap_fdatawait(mapping);
  347. if (!err)
  348. err = err2;
  349. }
  350. }
  351. return err;
  352. }
  353. EXPORT_SYMBOL(filemap_write_and_wait);
  354. /**
  355. * filemap_write_and_wait_range - write out & wait on a file range
  356. * @mapping: the address_space for the pages
  357. * @lstart: offset in bytes where the range starts
  358. * @lend: offset in bytes where the range ends (inclusive)
  359. *
  360. * Write out and wait upon file offsets lstart->lend, inclusive.
  361. *
  362. * Note that `lend' is inclusive (describes the last byte to be written) so
  363. * that this function can be used to write to the very end-of-file (end = -1).
  364. */
  365. int filemap_write_and_wait_range(struct address_space *mapping,
  366. loff_t lstart, loff_t lend)
  367. {
  368. int err = 0;
  369. if (mapping->nrpages) {
  370. err = __filemap_fdatawrite_range(mapping, lstart, lend,
  371. WB_SYNC_ALL);
  372. /* See comment of filemap_write_and_wait() */
  373. if (err != -EIO) {
  374. int err2 = wait_on_page_writeback_range(mapping,
  375. lstart >> PAGE_CACHE_SHIFT,
  376. lend >> PAGE_CACHE_SHIFT);
  377. if (!err)
  378. err = err2;
  379. }
  380. }
  381. return err;
  382. }
  383. /**
  384. * add_to_page_cache - add newly allocated pagecache pages
  385. * @page: page to add
  386. * @mapping: the page's address_space
  387. * @offset: page index
  388. * @gfp_mask: page allocation mode
  389. *
  390. * This function is used to add newly allocated pagecache pages;
  391. * the page is new, so we can just run SetPageLocked() against it.
  392. * The other page state flags were set by rmqueue().
  393. *
  394. * This function does not add the page to the LRU. The caller must do that.
  395. */
  396. int add_to_page_cache(struct page *page, struct address_space *mapping,
  397. pgoff_t offset, gfp_t gfp_mask)
  398. {
  399. int error = radix_tree_preload(gfp_mask & ~__GFP_HIGHMEM);
  400. if (error == 0) {
  401. write_lock_irq(&mapping->tree_lock);
  402. error = radix_tree_insert(&mapping->page_tree, offset, page);
  403. if (!error) {
  404. page_cache_get(page);
  405. SetPageLocked(page);
  406. page->mapping = mapping;
  407. page->index = offset;
  408. mapping->nrpages++;
  409. __inc_zone_page_state(page, NR_FILE_PAGES);
  410. }
  411. write_unlock_irq(&mapping->tree_lock);
  412. radix_tree_preload_end();
  413. }
  414. return error;
  415. }
  416. EXPORT_SYMBOL(add_to_page_cache);
  417. int add_to_page_cache_lru(struct page *page, struct address_space *mapping,
  418. pgoff_t offset, gfp_t gfp_mask)
  419. {
  420. int ret = add_to_page_cache(page, mapping, offset, gfp_mask);
  421. if (ret == 0)
  422. lru_cache_add(page);
  423. return ret;
  424. }
  425. #ifdef CONFIG_NUMA
  426. struct page *__page_cache_alloc(gfp_t gfp)
  427. {
  428. if (cpuset_do_page_mem_spread()) {
  429. int n = cpuset_mem_spread_node();
  430. return alloc_pages_node(n, gfp, 0);
  431. }
  432. return alloc_pages(gfp, 0);
  433. }
  434. EXPORT_SYMBOL(__page_cache_alloc);
  435. #endif
  436. static int __sleep_on_page_lock(void *word)
  437. {
  438. io_schedule();
  439. return 0;
  440. }
  441. /*
  442. * In order to wait for pages to become available there must be
  443. * waitqueues associated with pages. By using a hash table of
  444. * waitqueues where the bucket discipline is to maintain all
  445. * waiters on the same queue and wake all when any of the pages
  446. * become available, and for the woken contexts to check to be
  447. * sure the appropriate page became available, this saves space
  448. * at a cost of "thundering herd" phenomena during rare hash
  449. * collisions.
  450. */
  451. static wait_queue_head_t *page_waitqueue(struct page *page)
  452. {
  453. const struct zone *zone = page_zone(page);
  454. return &zone->wait_table[hash_ptr(page, zone->wait_table_bits)];
  455. }
  456. static inline void wake_up_page(struct page *page, int bit)
  457. {
  458. __wake_up_bit(page_waitqueue(page), &page->flags, bit);
  459. }
  460. void fastcall wait_on_page_bit(struct page *page, int bit_nr)
  461. {
  462. DEFINE_WAIT_BIT(wait, &page->flags, bit_nr);
  463. if (test_bit(bit_nr, &page->flags))
  464. __wait_on_bit(page_waitqueue(page), &wait, sync_page,
  465. TASK_UNINTERRUPTIBLE);
  466. }
  467. EXPORT_SYMBOL(wait_on_page_bit);
  468. /**
  469. * unlock_page - unlock a locked page
  470. * @page: the page
  471. *
  472. * Unlocks the page and wakes up sleepers in ___wait_on_page_locked().
  473. * Also wakes sleepers in wait_on_page_writeback() because the wakeup
  474. * mechananism between PageLocked pages and PageWriteback pages is shared.
  475. * But that's OK - sleepers in wait_on_page_writeback() just go back to sleep.
  476. *
  477. * The first mb is necessary to safely close the critical section opened by the
  478. * TestSetPageLocked(), the second mb is necessary to enforce ordering between
  479. * the clear_bit and the read of the waitqueue (to avoid SMP races with a
  480. * parallel wait_on_page_locked()).
  481. */
  482. void fastcall unlock_page(struct page *page)
  483. {
  484. smp_mb__before_clear_bit();
  485. if (!TestClearPageLocked(page))
  486. BUG();
  487. smp_mb__after_clear_bit();
  488. wake_up_page(page, PG_locked);
  489. }
  490. EXPORT_SYMBOL(unlock_page);
  491. /**
  492. * end_page_writeback - end writeback against a page
  493. * @page: the page
  494. */
  495. void end_page_writeback(struct page *page)
  496. {
  497. if (!TestClearPageReclaim(page) || rotate_reclaimable_page(page)) {
  498. if (!test_clear_page_writeback(page))
  499. BUG();
  500. }
  501. smp_mb__after_clear_bit();
  502. wake_up_page(page, PG_writeback);
  503. }
  504. EXPORT_SYMBOL(end_page_writeback);
  505. /**
  506. * __lock_page - get a lock on the page, assuming we need to sleep to get it
  507. * @page: the page to lock
  508. *
  509. * Ugly. Running sync_page() in state TASK_UNINTERRUPTIBLE is scary. If some
  510. * random driver's requestfn sets TASK_RUNNING, we could busywait. However
  511. * chances are that on the second loop, the block layer's plug list is empty,
  512. * so sync_page() will then return in state TASK_UNINTERRUPTIBLE.
  513. */
  514. void fastcall __lock_page(struct page *page)
  515. {
  516. DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
  517. __wait_on_bit_lock(page_waitqueue(page), &wait, sync_page,
  518. TASK_UNINTERRUPTIBLE);
  519. }
  520. EXPORT_SYMBOL(__lock_page);
  521. /*
  522. * Variant of lock_page that does not require the caller to hold a reference
  523. * on the page's mapping.
  524. */
  525. void fastcall __lock_page_nosync(struct page *page)
  526. {
  527. DEFINE_WAIT_BIT(wait, &page->flags, PG_locked);
  528. __wait_on_bit_lock(page_waitqueue(page), &wait, __sleep_on_page_lock,
  529. TASK_UNINTERRUPTIBLE);
  530. }
  531. /**
  532. * find_get_page - find and get a page reference
  533. * @mapping: the address_space to search
  534. * @offset: the page index
  535. *
  536. * Is there a pagecache struct page at the given (mapping, offset) tuple?
  537. * If yes, increment its refcount and return it; if no, return NULL.
  538. */
  539. struct page * find_get_page(struct address_space *mapping, unsigned long offset)
  540. {
  541. struct page *page;
  542. read_lock_irq(&mapping->tree_lock);
  543. page = radix_tree_lookup(&mapping->page_tree, offset);
  544. if (page)
  545. page_cache_get(page);
  546. read_unlock_irq(&mapping->tree_lock);
  547. return page;
  548. }
  549. EXPORT_SYMBOL(find_get_page);
  550. /**
  551. * find_lock_page - locate, pin and lock a pagecache page
  552. * @mapping: the address_space to search
  553. * @offset: the page index
  554. *
  555. * Locates the desired pagecache page, locks it, increments its reference
  556. * count and returns its address.
  557. *
  558. * Returns zero if the page was not present. find_lock_page() may sleep.
  559. */
  560. struct page *find_lock_page(struct address_space *mapping,
  561. unsigned long offset)
  562. {
  563. struct page *page;
  564. read_lock_irq(&mapping->tree_lock);
  565. repeat:
  566. page = radix_tree_lookup(&mapping->page_tree, offset);
  567. if (page) {
  568. page_cache_get(page);
  569. if (TestSetPageLocked(page)) {
  570. read_unlock_irq(&mapping->tree_lock);
  571. __lock_page(page);
  572. read_lock_irq(&mapping->tree_lock);
  573. /* Has the page been truncated while we slept? */
  574. if (unlikely(page->mapping != mapping ||
  575. page->index != offset)) {
  576. unlock_page(page);
  577. page_cache_release(page);
  578. goto repeat;
  579. }
  580. }
  581. }
  582. read_unlock_irq(&mapping->tree_lock);
  583. return page;
  584. }
  585. EXPORT_SYMBOL(find_lock_page);
  586. /**
  587. * find_or_create_page - locate or add a pagecache page
  588. * @mapping: the page's address_space
  589. * @index: the page's index into the mapping
  590. * @gfp_mask: page allocation mode
  591. *
  592. * Locates a page in the pagecache. If the page is not present, a new page
  593. * is allocated using @gfp_mask and is added to the pagecache and to the VM's
  594. * LRU list. The returned page is locked and has its reference count
  595. * incremented.
  596. *
  597. * find_or_create_page() may sleep, even if @gfp_flags specifies an atomic
  598. * allocation!
  599. *
  600. * find_or_create_page() returns the desired page's address, or zero on
  601. * memory exhaustion.
  602. */
  603. struct page *find_or_create_page(struct address_space *mapping,
  604. unsigned long index, gfp_t gfp_mask)
  605. {
  606. struct page *page, *cached_page = NULL;
  607. int err;
  608. repeat:
  609. page = find_lock_page(mapping, index);
  610. if (!page) {
  611. if (!cached_page) {
  612. cached_page = alloc_page(gfp_mask);
  613. if (!cached_page)
  614. return NULL;
  615. }
  616. err = add_to_page_cache_lru(cached_page, mapping,
  617. index, gfp_mask);
  618. if (!err) {
  619. page = cached_page;
  620. cached_page = NULL;
  621. } else if (err == -EEXIST)
  622. goto repeat;
  623. }
  624. if (cached_page)
  625. page_cache_release(cached_page);
  626. return page;
  627. }
  628. EXPORT_SYMBOL(find_or_create_page);
  629. /**
  630. * find_get_pages - gang pagecache lookup
  631. * @mapping: The address_space to search
  632. * @start: The starting page index
  633. * @nr_pages: The maximum number of pages
  634. * @pages: Where the resulting pages are placed
  635. *
  636. * find_get_pages() will search for and return a group of up to
  637. * @nr_pages pages in the mapping. The pages are placed at @pages.
  638. * find_get_pages() takes a reference against the returned pages.
  639. *
  640. * The search returns a group of mapping-contiguous pages with ascending
  641. * indexes. There may be holes in the indices due to not-present pages.
  642. *
  643. * find_get_pages() returns the number of pages which were found.
  644. */
  645. unsigned find_get_pages(struct address_space *mapping, pgoff_t start,
  646. unsigned int nr_pages, struct page **pages)
  647. {
  648. unsigned int i;
  649. unsigned int ret;
  650. read_lock_irq(&mapping->tree_lock);
  651. ret = radix_tree_gang_lookup(&mapping->page_tree,
  652. (void **)pages, start, nr_pages);
  653. for (i = 0; i < ret; i++)
  654. page_cache_get(pages[i]);
  655. read_unlock_irq(&mapping->tree_lock);
  656. return ret;
  657. }
  658. /**
  659. * find_get_pages_contig - gang contiguous pagecache lookup
  660. * @mapping: The address_space to search
  661. * @index: The starting page index
  662. * @nr_pages: The maximum number of pages
  663. * @pages: Where the resulting pages are placed
  664. *
  665. * find_get_pages_contig() works exactly like find_get_pages(), except
  666. * that the returned number of pages are guaranteed to be contiguous.
  667. *
  668. * find_get_pages_contig() returns the number of pages which were found.
  669. */
  670. unsigned find_get_pages_contig(struct address_space *mapping, pgoff_t index,
  671. unsigned int nr_pages, struct page **pages)
  672. {
  673. unsigned int i;
  674. unsigned int ret;
  675. read_lock_irq(&mapping->tree_lock);
  676. ret = radix_tree_gang_lookup(&mapping->page_tree,
  677. (void **)pages, index, nr_pages);
  678. for (i = 0; i < ret; i++) {
  679. if (pages[i]->mapping == NULL || pages[i]->index != index)
  680. break;
  681. page_cache_get(pages[i]);
  682. index++;
  683. }
  684. read_unlock_irq(&mapping->tree_lock);
  685. return i;
  686. }
  687. /**
  688. * find_get_pages_tag - find and return pages that match @tag
  689. * @mapping: the address_space to search
  690. * @index: the starting page index
  691. * @tag: the tag index
  692. * @nr_pages: the maximum number of pages
  693. * @pages: where the resulting pages are placed
  694. *
  695. * Like find_get_pages, except we only return pages which are tagged with
  696. * @tag. We update @index to index the next page for the traversal.
  697. */
  698. unsigned find_get_pages_tag(struct address_space *mapping, pgoff_t *index,
  699. int tag, unsigned int nr_pages, struct page **pages)
  700. {
  701. unsigned int i;
  702. unsigned int ret;
  703. read_lock_irq(&mapping->tree_lock);
  704. ret = radix_tree_gang_lookup_tag(&mapping->page_tree,
  705. (void **)pages, *index, nr_pages, tag);
  706. for (i = 0; i < ret; i++)
  707. page_cache_get(pages[i]);
  708. if (ret)
  709. *index = pages[ret - 1]->index + 1;
  710. read_unlock_irq(&mapping->tree_lock);
  711. return ret;
  712. }
  713. /**
  714. * grab_cache_page_nowait - returns locked page at given index in given cache
  715. * @mapping: target address_space
  716. * @index: the page index
  717. *
  718. * Same as grab_cache_page(), but do not wait if the page is unavailable.
  719. * This is intended for speculative data generators, where the data can
  720. * be regenerated if the page couldn't be grabbed. This routine should
  721. * be safe to call while holding the lock for another page.
  722. *
  723. * Clear __GFP_FS when allocating the page to avoid recursion into the fs
  724. * and deadlock against the caller's locked page.
  725. */
  726. struct page *
  727. grab_cache_page_nowait(struct address_space *mapping, unsigned long index)
  728. {
  729. struct page *page = find_get_page(mapping, index);
  730. if (page) {
  731. if (!TestSetPageLocked(page))
  732. return page;
  733. page_cache_release(page);
  734. return NULL;
  735. }
  736. page = __page_cache_alloc(mapping_gfp_mask(mapping) & ~__GFP_FS);
  737. if (page && add_to_page_cache_lru(page, mapping, index, GFP_KERNEL)) {
  738. page_cache_release(page);
  739. page = NULL;
  740. }
  741. return page;
  742. }
  743. EXPORT_SYMBOL(grab_cache_page_nowait);
  744. /*
  745. * CD/DVDs are error prone. When a medium error occurs, the driver may fail
  746. * a _large_ part of the i/o request. Imagine the worst scenario:
  747. *
  748. * ---R__________________________________________B__________
  749. * ^ reading here ^ bad block(assume 4k)
  750. *
  751. * read(R) => miss => readahead(R...B) => media error => frustrating retries
  752. * => failing the whole request => read(R) => read(R+1) =>
  753. * readahead(R+1...B+1) => bang => read(R+2) => read(R+3) =>
  754. * readahead(R+3...B+2) => bang => read(R+3) => read(R+4) =>
  755. * readahead(R+4...B+3) => bang => read(R+4) => read(R+5) => ......
  756. *
  757. * It is going insane. Fix it by quickly scaling down the readahead size.
  758. */
  759. static void shrink_readahead_size_eio(struct file *filp,
  760. struct file_ra_state *ra)
  761. {
  762. if (!ra->ra_pages)
  763. return;
  764. ra->ra_pages /= 4;
  765. }
  766. /**
  767. * do_generic_mapping_read - generic file read routine
  768. * @mapping: address_space to be read
  769. * @_ra: file's readahead state
  770. * @filp: the file to read
  771. * @ppos: current file position
  772. * @desc: read_descriptor
  773. * @actor: read method
  774. *
  775. * This is a generic file read routine, and uses the
  776. * mapping->a_ops->readpage() function for the actual low-level stuff.
  777. *
  778. * This is really ugly. But the goto's actually try to clarify some
  779. * of the logic when it comes to error handling etc.
  780. *
  781. * Note the struct file* is only passed for the use of readpage.
  782. * It may be NULL.
  783. */
  784. void do_generic_mapping_read(struct address_space *mapping,
  785. struct file_ra_state *_ra,
  786. struct file *filp,
  787. loff_t *ppos,
  788. read_descriptor_t *desc,
  789. read_actor_t actor)
  790. {
  791. struct inode *inode = mapping->host;
  792. unsigned long index;
  793. unsigned long end_index;
  794. unsigned long offset;
  795. unsigned long last_index;
  796. unsigned long next_index;
  797. unsigned long prev_index;
  798. loff_t isize;
  799. struct page *cached_page;
  800. int error;
  801. struct file_ra_state ra = *_ra;
  802. cached_page = NULL;
  803. index = *ppos >> PAGE_CACHE_SHIFT;
  804. next_index = index;
  805. prev_index = ra.prev_page;
  806. last_index = (*ppos + desc->count + PAGE_CACHE_SIZE-1) >> PAGE_CACHE_SHIFT;
  807. offset = *ppos & ~PAGE_CACHE_MASK;
  808. isize = i_size_read(inode);
  809. if (!isize)
  810. goto out;
  811. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  812. for (;;) {
  813. struct page *page;
  814. unsigned long nr, ret;
  815. /* nr is the maximum number of bytes to copy from this page */
  816. nr = PAGE_CACHE_SIZE;
  817. if (index >= end_index) {
  818. if (index > end_index)
  819. goto out;
  820. nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  821. if (nr <= offset) {
  822. goto out;
  823. }
  824. }
  825. nr = nr - offset;
  826. cond_resched();
  827. if (index == next_index)
  828. next_index = page_cache_readahead(mapping, &ra, filp,
  829. index, last_index - index);
  830. find_page:
  831. page = find_get_page(mapping, index);
  832. if (unlikely(page == NULL)) {
  833. handle_ra_miss(mapping, &ra, index);
  834. goto no_cached_page;
  835. }
  836. if (!PageUptodate(page))
  837. goto page_not_up_to_date;
  838. page_ok:
  839. /* If users can be writing to this page using arbitrary
  840. * virtual addresses, take care about potential aliasing
  841. * before reading the page on the kernel side.
  842. */
  843. if (mapping_writably_mapped(mapping))
  844. flush_dcache_page(page);
  845. /*
  846. * When (part of) the same page is read multiple times
  847. * in succession, only mark it as accessed the first time.
  848. */
  849. if (prev_index != index)
  850. mark_page_accessed(page);
  851. prev_index = index;
  852. /*
  853. * Ok, we have the page, and it's up-to-date, so
  854. * now we can copy it to user space...
  855. *
  856. * The actor routine returns how many bytes were actually used..
  857. * NOTE! This may not be the same as how much of a user buffer
  858. * we filled up (we may be padding etc), so we can only update
  859. * "pos" here (the actor routine has to update the user buffer
  860. * pointers and the remaining count).
  861. */
  862. ret = actor(desc, page, offset, nr);
  863. offset += ret;
  864. index += offset >> PAGE_CACHE_SHIFT;
  865. offset &= ~PAGE_CACHE_MASK;
  866. page_cache_release(page);
  867. if (ret == nr && desc->count)
  868. continue;
  869. goto out;
  870. page_not_up_to_date:
  871. /* Get exclusive access to the page ... */
  872. lock_page(page);
  873. /* Did it get truncated before we got the lock? */
  874. if (!page->mapping) {
  875. unlock_page(page);
  876. page_cache_release(page);
  877. continue;
  878. }
  879. /* Did somebody else fill it already? */
  880. if (PageUptodate(page)) {
  881. unlock_page(page);
  882. goto page_ok;
  883. }
  884. readpage:
  885. /* Start the actual read. The read will unlock the page. */
  886. error = mapping->a_ops->readpage(filp, page);
  887. if (unlikely(error)) {
  888. if (error == AOP_TRUNCATED_PAGE) {
  889. page_cache_release(page);
  890. goto find_page;
  891. }
  892. goto readpage_error;
  893. }
  894. if (!PageUptodate(page)) {
  895. lock_page(page);
  896. if (!PageUptodate(page)) {
  897. if (page->mapping == NULL) {
  898. /*
  899. * invalidate_inode_pages got it
  900. */
  901. unlock_page(page);
  902. page_cache_release(page);
  903. goto find_page;
  904. }
  905. unlock_page(page);
  906. error = -EIO;
  907. shrink_readahead_size_eio(filp, &ra);
  908. goto readpage_error;
  909. }
  910. unlock_page(page);
  911. }
  912. /*
  913. * i_size must be checked after we have done ->readpage.
  914. *
  915. * Checking i_size after the readpage allows us to calculate
  916. * the correct value for "nr", which means the zero-filled
  917. * part of the page is not copied back to userspace (unless
  918. * another truncate extends the file - this is desired though).
  919. */
  920. isize = i_size_read(inode);
  921. end_index = (isize - 1) >> PAGE_CACHE_SHIFT;
  922. if (unlikely(!isize || index > end_index)) {
  923. page_cache_release(page);
  924. goto out;
  925. }
  926. /* nr is the maximum number of bytes to copy from this page */
  927. nr = PAGE_CACHE_SIZE;
  928. if (index == end_index) {
  929. nr = ((isize - 1) & ~PAGE_CACHE_MASK) + 1;
  930. if (nr <= offset) {
  931. page_cache_release(page);
  932. goto out;
  933. }
  934. }
  935. nr = nr - offset;
  936. goto page_ok;
  937. readpage_error:
  938. /* UHHUH! A synchronous read error occurred. Report it */
  939. desc->error = error;
  940. page_cache_release(page);
  941. goto out;
  942. no_cached_page:
  943. /*
  944. * Ok, it wasn't cached, so we need to create a new
  945. * page..
  946. */
  947. if (!cached_page) {
  948. cached_page = page_cache_alloc_cold(mapping);
  949. if (!cached_page) {
  950. desc->error = -ENOMEM;
  951. goto out;
  952. }
  953. }
  954. error = add_to_page_cache_lru(cached_page, mapping,
  955. index, GFP_KERNEL);
  956. if (error) {
  957. if (error == -EEXIST)
  958. goto find_page;
  959. desc->error = error;
  960. goto out;
  961. }
  962. page = cached_page;
  963. cached_page = NULL;
  964. goto readpage;
  965. }
  966. out:
  967. *_ra = ra;
  968. *ppos = ((loff_t) index << PAGE_CACHE_SHIFT) + offset;
  969. if (cached_page)
  970. page_cache_release(cached_page);
  971. if (filp)
  972. file_accessed(filp);
  973. }
  974. EXPORT_SYMBOL(do_generic_mapping_read);
  975. int file_read_actor(read_descriptor_t *desc, struct page *page,
  976. unsigned long offset, unsigned long size)
  977. {
  978. char *kaddr;
  979. unsigned long left, count = desc->count;
  980. if (size > count)
  981. size = count;
  982. /*
  983. * Faults on the destination of a read are common, so do it before
  984. * taking the kmap.
  985. */
  986. if (!fault_in_pages_writeable(desc->arg.buf, size)) {
  987. kaddr = kmap_atomic(page, KM_USER0);
  988. left = __copy_to_user_inatomic(desc->arg.buf,
  989. kaddr + offset, size);
  990. kunmap_atomic(kaddr, KM_USER0);
  991. if (left == 0)
  992. goto success;
  993. }
  994. /* Do it the slow way */
  995. kaddr = kmap(page);
  996. left = __copy_to_user(desc->arg.buf, kaddr + offset, size);
  997. kunmap(page);
  998. if (left) {
  999. size -= left;
  1000. desc->error = -EFAULT;
  1001. }
  1002. success:
  1003. desc->count = count - size;
  1004. desc->written += size;
  1005. desc->arg.buf += size;
  1006. return size;
  1007. }
  1008. /**
  1009. * generic_file_aio_read - generic filesystem read routine
  1010. * @iocb: kernel I/O control block
  1011. * @iov: io vector request
  1012. * @nr_segs: number of segments in the iovec
  1013. * @pos: current file position
  1014. *
  1015. * This is the "read()" routine for all filesystems
  1016. * that can use the page cache directly.
  1017. */
  1018. ssize_t
  1019. generic_file_aio_read(struct kiocb *iocb, const struct iovec *iov,
  1020. unsigned long nr_segs, loff_t pos)
  1021. {
  1022. struct file *filp = iocb->ki_filp;
  1023. ssize_t retval;
  1024. unsigned long seg;
  1025. size_t count;
  1026. loff_t *ppos = &iocb->ki_pos;
  1027. count = 0;
  1028. for (seg = 0; seg < nr_segs; seg++) {
  1029. const struct iovec *iv = &iov[seg];
  1030. /*
  1031. * If any segment has a negative length, or the cumulative
  1032. * length ever wraps negative then return -EINVAL.
  1033. */
  1034. count += iv->iov_len;
  1035. if (unlikely((ssize_t)(count|iv->iov_len) < 0))
  1036. return -EINVAL;
  1037. if (access_ok(VERIFY_WRITE, iv->iov_base, iv->iov_len))
  1038. continue;
  1039. if (seg == 0)
  1040. return -EFAULT;
  1041. nr_segs = seg;
  1042. count -= iv->iov_len; /* This segment is no good */
  1043. break;
  1044. }
  1045. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  1046. if (filp->f_flags & O_DIRECT) {
  1047. loff_t size;
  1048. struct address_space *mapping;
  1049. struct inode *inode;
  1050. mapping = filp->f_mapping;
  1051. inode = mapping->host;
  1052. retval = 0;
  1053. if (!count)
  1054. goto out; /* skip atime */
  1055. size = i_size_read(inode);
  1056. if (pos < size) {
  1057. retval = generic_file_direct_IO(READ, iocb,
  1058. iov, pos, nr_segs);
  1059. if (retval > 0)
  1060. *ppos = pos + retval;
  1061. }
  1062. if (likely(retval != 0)) {
  1063. file_accessed(filp);
  1064. goto out;
  1065. }
  1066. }
  1067. retval = 0;
  1068. if (count) {
  1069. for (seg = 0; seg < nr_segs; seg++) {
  1070. read_descriptor_t desc;
  1071. desc.written = 0;
  1072. desc.arg.buf = iov[seg].iov_base;
  1073. desc.count = iov[seg].iov_len;
  1074. if (desc.count == 0)
  1075. continue;
  1076. desc.error = 0;
  1077. do_generic_file_read(filp,ppos,&desc,file_read_actor);
  1078. retval += desc.written;
  1079. if (desc.error) {
  1080. retval = retval ?: desc.error;
  1081. break;
  1082. }
  1083. }
  1084. }
  1085. out:
  1086. return retval;
  1087. }
  1088. EXPORT_SYMBOL(generic_file_aio_read);
  1089. int file_send_actor(read_descriptor_t * desc, struct page *page, unsigned long offset, unsigned long size)
  1090. {
  1091. ssize_t written;
  1092. unsigned long count = desc->count;
  1093. struct file *file = desc->arg.data;
  1094. if (size > count)
  1095. size = count;
  1096. written = file->f_op->sendpage(file, page, offset,
  1097. size, &file->f_pos, size<count);
  1098. if (written < 0) {
  1099. desc->error = written;
  1100. written = 0;
  1101. }
  1102. desc->count = count - written;
  1103. desc->written += written;
  1104. return written;
  1105. }
  1106. ssize_t generic_file_sendfile(struct file *in_file, loff_t *ppos,
  1107. size_t count, read_actor_t actor, void *target)
  1108. {
  1109. read_descriptor_t desc;
  1110. if (!count)
  1111. return 0;
  1112. desc.written = 0;
  1113. desc.count = count;
  1114. desc.arg.data = target;
  1115. desc.error = 0;
  1116. do_generic_file_read(in_file, ppos, &desc, actor);
  1117. if (desc.written)
  1118. return desc.written;
  1119. return desc.error;
  1120. }
  1121. EXPORT_SYMBOL(generic_file_sendfile);
  1122. static ssize_t
  1123. do_readahead(struct address_space *mapping, struct file *filp,
  1124. unsigned long index, unsigned long nr)
  1125. {
  1126. if (!mapping || !mapping->a_ops || !mapping->a_ops->readpage)
  1127. return -EINVAL;
  1128. force_page_cache_readahead(mapping, filp, index,
  1129. max_sane_readahead(nr));
  1130. return 0;
  1131. }
  1132. asmlinkage ssize_t sys_readahead(int fd, loff_t offset, size_t count)
  1133. {
  1134. ssize_t ret;
  1135. struct file *file;
  1136. ret = -EBADF;
  1137. file = fget(fd);
  1138. if (file) {
  1139. if (file->f_mode & FMODE_READ) {
  1140. struct address_space *mapping = file->f_mapping;
  1141. unsigned long start = offset >> PAGE_CACHE_SHIFT;
  1142. unsigned long end = (offset + count - 1) >> PAGE_CACHE_SHIFT;
  1143. unsigned long len = end - start + 1;
  1144. ret = do_readahead(mapping, file, start, len);
  1145. }
  1146. fput(file);
  1147. }
  1148. return ret;
  1149. }
  1150. #ifdef CONFIG_MMU
  1151. static int FASTCALL(page_cache_read(struct file * file, unsigned long offset));
  1152. /**
  1153. * page_cache_read - adds requested page to the page cache if not already there
  1154. * @file: file to read
  1155. * @offset: page index
  1156. *
  1157. * This adds the requested page to the page cache if it isn't already there,
  1158. * and schedules an I/O to read in its contents from disk.
  1159. */
  1160. static int fastcall page_cache_read(struct file * file, unsigned long offset)
  1161. {
  1162. struct address_space *mapping = file->f_mapping;
  1163. struct page *page;
  1164. int ret;
  1165. do {
  1166. page = page_cache_alloc_cold(mapping);
  1167. if (!page)
  1168. return -ENOMEM;
  1169. ret = add_to_page_cache_lru(page, mapping, offset, GFP_KERNEL);
  1170. if (ret == 0)
  1171. ret = mapping->a_ops->readpage(file, page);
  1172. else if (ret == -EEXIST)
  1173. ret = 0; /* losing race to add is OK */
  1174. page_cache_release(page);
  1175. } while (ret == AOP_TRUNCATED_PAGE);
  1176. return ret;
  1177. }
  1178. #define MMAP_LOTSAMISS (100)
  1179. /**
  1180. * filemap_nopage - read in file data for page fault handling
  1181. * @area: the applicable vm_area
  1182. * @address: target address to read in
  1183. * @type: returned with VM_FAULT_{MINOR,MAJOR} if not %NULL
  1184. *
  1185. * filemap_nopage() is invoked via the vma operations vector for a
  1186. * mapped memory region to read in file data during a page fault.
  1187. *
  1188. * The goto's are kind of ugly, but this streamlines the normal case of having
  1189. * it in the page cache, and handles the special cases reasonably without
  1190. * having a lot of duplicated code.
  1191. */
  1192. struct page *filemap_nopage(struct vm_area_struct *area,
  1193. unsigned long address, int *type)
  1194. {
  1195. int error;
  1196. struct file *file = area->vm_file;
  1197. struct address_space *mapping = file->f_mapping;
  1198. struct file_ra_state *ra = &file->f_ra;
  1199. struct inode *inode = mapping->host;
  1200. struct page *page;
  1201. unsigned long size, pgoff;
  1202. int did_readaround = 0, majmin = VM_FAULT_MINOR;
  1203. pgoff = ((address-area->vm_start) >> PAGE_CACHE_SHIFT) + area->vm_pgoff;
  1204. retry_all:
  1205. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1206. if (pgoff >= size)
  1207. goto outside_data_content;
  1208. /* If we don't want any read-ahead, don't bother */
  1209. if (VM_RandomReadHint(area))
  1210. goto no_cached_page;
  1211. /*
  1212. * The readahead code wants to be told about each and every page
  1213. * so it can build and shrink its windows appropriately
  1214. *
  1215. * For sequential accesses, we use the generic readahead logic.
  1216. */
  1217. if (VM_SequentialReadHint(area))
  1218. page_cache_readahead(mapping, ra, file, pgoff, 1);
  1219. /*
  1220. * Do we have something in the page cache already?
  1221. */
  1222. retry_find:
  1223. page = find_get_page(mapping, pgoff);
  1224. if (!page) {
  1225. unsigned long ra_pages;
  1226. if (VM_SequentialReadHint(area)) {
  1227. handle_ra_miss(mapping, ra, pgoff);
  1228. goto no_cached_page;
  1229. }
  1230. ra->mmap_miss++;
  1231. /*
  1232. * Do we miss much more than hit in this file? If so,
  1233. * stop bothering with read-ahead. It will only hurt.
  1234. */
  1235. if (ra->mmap_miss > ra->mmap_hit + MMAP_LOTSAMISS)
  1236. goto no_cached_page;
  1237. /*
  1238. * To keep the pgmajfault counter straight, we need to
  1239. * check did_readaround, as this is an inner loop.
  1240. */
  1241. if (!did_readaround) {
  1242. majmin = VM_FAULT_MAJOR;
  1243. count_vm_event(PGMAJFAULT);
  1244. }
  1245. did_readaround = 1;
  1246. ra_pages = max_sane_readahead(file->f_ra.ra_pages);
  1247. if (ra_pages) {
  1248. pgoff_t start = 0;
  1249. if (pgoff > ra_pages / 2)
  1250. start = pgoff - ra_pages / 2;
  1251. do_page_cache_readahead(mapping, file, start, ra_pages);
  1252. }
  1253. page = find_get_page(mapping, pgoff);
  1254. if (!page)
  1255. goto no_cached_page;
  1256. }
  1257. if (!did_readaround)
  1258. ra->mmap_hit++;
  1259. /*
  1260. * Ok, found a page in the page cache, now we need to check
  1261. * that it's up-to-date.
  1262. */
  1263. if (!PageUptodate(page))
  1264. goto page_not_uptodate;
  1265. success:
  1266. /*
  1267. * Found the page and have a reference on it.
  1268. */
  1269. mark_page_accessed(page);
  1270. if (type)
  1271. *type = majmin;
  1272. return page;
  1273. outside_data_content:
  1274. /*
  1275. * An external ptracer can access pages that normally aren't
  1276. * accessible..
  1277. */
  1278. if (area->vm_mm == current->mm)
  1279. return NOPAGE_SIGBUS;
  1280. /* Fall through to the non-read-ahead case */
  1281. no_cached_page:
  1282. /*
  1283. * We're only likely to ever get here if MADV_RANDOM is in
  1284. * effect.
  1285. */
  1286. error = page_cache_read(file, pgoff);
  1287. /*
  1288. * The page we want has now been added to the page cache.
  1289. * In the unlikely event that someone removed it in the
  1290. * meantime, we'll just come back here and read it again.
  1291. */
  1292. if (error >= 0)
  1293. goto retry_find;
  1294. /*
  1295. * An error return from page_cache_read can result if the
  1296. * system is low on memory, or a problem occurs while trying
  1297. * to schedule I/O.
  1298. */
  1299. if (error == -ENOMEM)
  1300. return NOPAGE_OOM;
  1301. return NOPAGE_SIGBUS;
  1302. page_not_uptodate:
  1303. if (!did_readaround) {
  1304. majmin = VM_FAULT_MAJOR;
  1305. count_vm_event(PGMAJFAULT);
  1306. }
  1307. lock_page(page);
  1308. /* Did it get unhashed while we waited for it? */
  1309. if (!page->mapping) {
  1310. unlock_page(page);
  1311. page_cache_release(page);
  1312. goto retry_all;
  1313. }
  1314. /* Did somebody else get it up-to-date? */
  1315. if (PageUptodate(page)) {
  1316. unlock_page(page);
  1317. goto success;
  1318. }
  1319. error = mapping->a_ops->readpage(file, page);
  1320. if (!error) {
  1321. wait_on_page_locked(page);
  1322. if (PageUptodate(page))
  1323. goto success;
  1324. } else if (error == AOP_TRUNCATED_PAGE) {
  1325. page_cache_release(page);
  1326. goto retry_find;
  1327. }
  1328. /*
  1329. * Umm, take care of errors if the page isn't up-to-date.
  1330. * Try to re-read it _once_. We do this synchronously,
  1331. * because there really aren't any performance issues here
  1332. * and we need to check for errors.
  1333. */
  1334. lock_page(page);
  1335. /* Somebody truncated the page on us? */
  1336. if (!page->mapping) {
  1337. unlock_page(page);
  1338. page_cache_release(page);
  1339. goto retry_all;
  1340. }
  1341. /* Somebody else successfully read it in? */
  1342. if (PageUptodate(page)) {
  1343. unlock_page(page);
  1344. goto success;
  1345. }
  1346. ClearPageError(page);
  1347. error = mapping->a_ops->readpage(file, page);
  1348. if (!error) {
  1349. wait_on_page_locked(page);
  1350. if (PageUptodate(page))
  1351. goto success;
  1352. } else if (error == AOP_TRUNCATED_PAGE) {
  1353. page_cache_release(page);
  1354. goto retry_find;
  1355. }
  1356. /*
  1357. * Things didn't work out. Return zero to tell the
  1358. * mm layer so, possibly freeing the page cache page first.
  1359. */
  1360. shrink_readahead_size_eio(file, ra);
  1361. page_cache_release(page);
  1362. return NOPAGE_SIGBUS;
  1363. }
  1364. EXPORT_SYMBOL(filemap_nopage);
  1365. static struct page * filemap_getpage(struct file *file, unsigned long pgoff,
  1366. int nonblock)
  1367. {
  1368. struct address_space *mapping = file->f_mapping;
  1369. struct page *page;
  1370. int error;
  1371. /*
  1372. * Do we have something in the page cache already?
  1373. */
  1374. retry_find:
  1375. page = find_get_page(mapping, pgoff);
  1376. if (!page) {
  1377. if (nonblock)
  1378. return NULL;
  1379. goto no_cached_page;
  1380. }
  1381. /*
  1382. * Ok, found a page in the page cache, now we need to check
  1383. * that it's up-to-date.
  1384. */
  1385. if (!PageUptodate(page)) {
  1386. if (nonblock) {
  1387. page_cache_release(page);
  1388. return NULL;
  1389. }
  1390. goto page_not_uptodate;
  1391. }
  1392. success:
  1393. /*
  1394. * Found the page and have a reference on it.
  1395. */
  1396. mark_page_accessed(page);
  1397. return page;
  1398. no_cached_page:
  1399. error = page_cache_read(file, pgoff);
  1400. /*
  1401. * The page we want has now been added to the page cache.
  1402. * In the unlikely event that someone removed it in the
  1403. * meantime, we'll just come back here and read it again.
  1404. */
  1405. if (error >= 0)
  1406. goto retry_find;
  1407. /*
  1408. * An error return from page_cache_read can result if the
  1409. * system is low on memory, or a problem occurs while trying
  1410. * to schedule I/O.
  1411. */
  1412. return NULL;
  1413. page_not_uptodate:
  1414. lock_page(page);
  1415. /* Did it get truncated while we waited for it? */
  1416. if (!page->mapping) {
  1417. unlock_page(page);
  1418. goto err;
  1419. }
  1420. /* Did somebody else get it up-to-date? */
  1421. if (PageUptodate(page)) {
  1422. unlock_page(page);
  1423. goto success;
  1424. }
  1425. error = mapping->a_ops->readpage(file, page);
  1426. if (!error) {
  1427. wait_on_page_locked(page);
  1428. if (PageUptodate(page))
  1429. goto success;
  1430. } else if (error == AOP_TRUNCATED_PAGE) {
  1431. page_cache_release(page);
  1432. goto retry_find;
  1433. }
  1434. /*
  1435. * Umm, take care of errors if the page isn't up-to-date.
  1436. * Try to re-read it _once_. We do this synchronously,
  1437. * because there really aren't any performance issues here
  1438. * and we need to check for errors.
  1439. */
  1440. lock_page(page);
  1441. /* Somebody truncated the page on us? */
  1442. if (!page->mapping) {
  1443. unlock_page(page);
  1444. goto err;
  1445. }
  1446. /* Somebody else successfully read it in? */
  1447. if (PageUptodate(page)) {
  1448. unlock_page(page);
  1449. goto success;
  1450. }
  1451. ClearPageError(page);
  1452. error = mapping->a_ops->readpage(file, page);
  1453. if (!error) {
  1454. wait_on_page_locked(page);
  1455. if (PageUptodate(page))
  1456. goto success;
  1457. } else if (error == AOP_TRUNCATED_PAGE) {
  1458. page_cache_release(page);
  1459. goto retry_find;
  1460. }
  1461. /*
  1462. * Things didn't work out. Return zero to tell the
  1463. * mm layer so, possibly freeing the page cache page first.
  1464. */
  1465. err:
  1466. page_cache_release(page);
  1467. return NULL;
  1468. }
  1469. int filemap_populate(struct vm_area_struct *vma, unsigned long addr,
  1470. unsigned long len, pgprot_t prot, unsigned long pgoff,
  1471. int nonblock)
  1472. {
  1473. struct file *file = vma->vm_file;
  1474. struct address_space *mapping = file->f_mapping;
  1475. struct inode *inode = mapping->host;
  1476. unsigned long size;
  1477. struct mm_struct *mm = vma->vm_mm;
  1478. struct page *page;
  1479. int err;
  1480. if (!nonblock)
  1481. force_page_cache_readahead(mapping, vma->vm_file,
  1482. pgoff, len >> PAGE_CACHE_SHIFT);
  1483. repeat:
  1484. size = (i_size_read(inode) + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT;
  1485. if (pgoff + (len >> PAGE_CACHE_SHIFT) > size)
  1486. return -EINVAL;
  1487. page = filemap_getpage(file, pgoff, nonblock);
  1488. /* XXX: This is wrong, a filesystem I/O error may have happened. Fix that as
  1489. * done in shmem_populate calling shmem_getpage */
  1490. if (!page && !nonblock)
  1491. return -ENOMEM;
  1492. if (page) {
  1493. err = install_page(mm, vma, addr, page, prot);
  1494. if (err) {
  1495. page_cache_release(page);
  1496. return err;
  1497. }
  1498. } else if (vma->vm_flags & VM_NONLINEAR) {
  1499. /* No page was found just because we can't read it in now (being
  1500. * here implies nonblock != 0), but the page may exist, so set
  1501. * the PTE to fault it in later. */
  1502. err = install_file_pte(mm, vma, addr, pgoff, prot);
  1503. if (err)
  1504. return err;
  1505. }
  1506. len -= PAGE_SIZE;
  1507. addr += PAGE_SIZE;
  1508. pgoff++;
  1509. if (len)
  1510. goto repeat;
  1511. return 0;
  1512. }
  1513. EXPORT_SYMBOL(filemap_populate);
  1514. struct vm_operations_struct generic_file_vm_ops = {
  1515. .nopage = filemap_nopage,
  1516. .populate = filemap_populate,
  1517. };
  1518. /* This is used for a general mmap of a disk file */
  1519. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1520. {
  1521. struct address_space *mapping = file->f_mapping;
  1522. if (!mapping->a_ops->readpage)
  1523. return -ENOEXEC;
  1524. file_accessed(file);
  1525. vma->vm_ops = &generic_file_vm_ops;
  1526. return 0;
  1527. }
  1528. /*
  1529. * This is for filesystems which do not implement ->writepage.
  1530. */
  1531. int generic_file_readonly_mmap(struct file *file, struct vm_area_struct *vma)
  1532. {
  1533. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  1534. return -EINVAL;
  1535. return generic_file_mmap(file, vma);
  1536. }
  1537. #else
  1538. int generic_file_mmap(struct file * file, struct vm_area_struct * vma)
  1539. {
  1540. return -ENOSYS;
  1541. }
  1542. int generic_file_readonly_mmap(struct file * file, struct vm_area_struct * vma)
  1543. {
  1544. return -ENOSYS;
  1545. }
  1546. #endif /* CONFIG_MMU */
  1547. EXPORT_SYMBOL(generic_file_mmap);
  1548. EXPORT_SYMBOL(generic_file_readonly_mmap);
  1549. static inline struct page *__read_cache_page(struct address_space *mapping,
  1550. unsigned long index,
  1551. int (*filler)(void *,struct page*),
  1552. void *data)
  1553. {
  1554. struct page *page, *cached_page = NULL;
  1555. int err;
  1556. repeat:
  1557. page = find_get_page(mapping, index);
  1558. if (!page) {
  1559. if (!cached_page) {
  1560. cached_page = page_cache_alloc_cold(mapping);
  1561. if (!cached_page)
  1562. return ERR_PTR(-ENOMEM);
  1563. }
  1564. err = add_to_page_cache_lru(cached_page, mapping,
  1565. index, GFP_KERNEL);
  1566. if (err == -EEXIST)
  1567. goto repeat;
  1568. if (err < 0) {
  1569. /* Presumably ENOMEM for radix tree node */
  1570. page_cache_release(cached_page);
  1571. return ERR_PTR(err);
  1572. }
  1573. page = cached_page;
  1574. cached_page = NULL;
  1575. err = filler(data, page);
  1576. if (err < 0) {
  1577. page_cache_release(page);
  1578. page = ERR_PTR(err);
  1579. }
  1580. }
  1581. if (cached_page)
  1582. page_cache_release(cached_page);
  1583. return page;
  1584. }
  1585. /**
  1586. * read_cache_page - read into page cache, fill it if needed
  1587. * @mapping: the page's address_space
  1588. * @index: the page index
  1589. * @filler: function to perform the read
  1590. * @data: destination for read data
  1591. *
  1592. * Read into the page cache. If a page already exists,
  1593. * and PageUptodate() is not set, try to fill the page.
  1594. */
  1595. struct page *read_cache_page(struct address_space *mapping,
  1596. unsigned long index,
  1597. int (*filler)(void *,struct page*),
  1598. void *data)
  1599. {
  1600. struct page *page;
  1601. int err;
  1602. retry:
  1603. page = __read_cache_page(mapping, index, filler, data);
  1604. if (IS_ERR(page))
  1605. goto out;
  1606. mark_page_accessed(page);
  1607. if (PageUptodate(page))
  1608. goto out;
  1609. lock_page(page);
  1610. if (!page->mapping) {
  1611. unlock_page(page);
  1612. page_cache_release(page);
  1613. goto retry;
  1614. }
  1615. if (PageUptodate(page)) {
  1616. unlock_page(page);
  1617. goto out;
  1618. }
  1619. err = filler(data, page);
  1620. if (err < 0) {
  1621. page_cache_release(page);
  1622. page = ERR_PTR(err);
  1623. }
  1624. out:
  1625. return page;
  1626. }
  1627. EXPORT_SYMBOL(read_cache_page);
  1628. /*
  1629. * If the page was newly created, increment its refcount and add it to the
  1630. * caller's lru-buffering pagevec. This function is specifically for
  1631. * generic_file_write().
  1632. */
  1633. static inline struct page *
  1634. __grab_cache_page(struct address_space *mapping, unsigned long index,
  1635. struct page **cached_page, struct pagevec *lru_pvec)
  1636. {
  1637. int err;
  1638. struct page *page;
  1639. repeat:
  1640. page = find_lock_page(mapping, index);
  1641. if (!page) {
  1642. if (!*cached_page) {
  1643. *cached_page = page_cache_alloc(mapping);
  1644. if (!*cached_page)
  1645. return NULL;
  1646. }
  1647. err = add_to_page_cache(*cached_page, mapping,
  1648. index, GFP_KERNEL);
  1649. if (err == -EEXIST)
  1650. goto repeat;
  1651. if (err == 0) {
  1652. page = *cached_page;
  1653. page_cache_get(page);
  1654. if (!pagevec_add(lru_pvec, page))
  1655. __pagevec_lru_add(lru_pvec);
  1656. *cached_page = NULL;
  1657. }
  1658. }
  1659. return page;
  1660. }
  1661. /*
  1662. * The logic we want is
  1663. *
  1664. * if suid or (sgid and xgrp)
  1665. * remove privs
  1666. */
  1667. int should_remove_suid(struct dentry *dentry)
  1668. {
  1669. mode_t mode = dentry->d_inode->i_mode;
  1670. int kill = 0;
  1671. /* suid always must be killed */
  1672. if (unlikely(mode & S_ISUID))
  1673. kill = ATTR_KILL_SUID;
  1674. /*
  1675. * sgid without any exec bits is just a mandatory locking mark; leave
  1676. * it alone. If some exec bits are set, it's a real sgid; kill it.
  1677. */
  1678. if (unlikely((mode & S_ISGID) && (mode & S_IXGRP)))
  1679. kill |= ATTR_KILL_SGID;
  1680. if (unlikely(kill && !capable(CAP_FSETID)))
  1681. return kill;
  1682. return 0;
  1683. }
  1684. EXPORT_SYMBOL(should_remove_suid);
  1685. int __remove_suid(struct dentry *dentry, int kill)
  1686. {
  1687. struct iattr newattrs;
  1688. newattrs.ia_valid = ATTR_FORCE | kill;
  1689. return notify_change(dentry, &newattrs);
  1690. }
  1691. int remove_suid(struct dentry *dentry)
  1692. {
  1693. int kill = should_remove_suid(dentry);
  1694. if (unlikely(kill))
  1695. return __remove_suid(dentry, kill);
  1696. return 0;
  1697. }
  1698. EXPORT_SYMBOL(remove_suid);
  1699. size_t
  1700. __filemap_copy_from_user_iovec_inatomic(char *vaddr,
  1701. const struct iovec *iov, size_t base, size_t bytes)
  1702. {
  1703. size_t copied = 0, left = 0;
  1704. while (bytes) {
  1705. char __user *buf = iov->iov_base + base;
  1706. int copy = min(bytes, iov->iov_len - base);
  1707. base = 0;
  1708. left = __copy_from_user_inatomic_nocache(vaddr, buf, copy);
  1709. copied += copy;
  1710. bytes -= copy;
  1711. vaddr += copy;
  1712. iov++;
  1713. if (unlikely(left))
  1714. break;
  1715. }
  1716. return copied - left;
  1717. }
  1718. /*
  1719. * Performs necessary checks before doing a write
  1720. *
  1721. * Can adjust writing position or amount of bytes to write.
  1722. * Returns appropriate error code that caller should return or
  1723. * zero in case that write should be allowed.
  1724. */
  1725. inline int generic_write_checks(struct file *file, loff_t *pos, size_t *count, int isblk)
  1726. {
  1727. struct inode *inode = file->f_mapping->host;
  1728. unsigned long limit = current->signal->rlim[RLIMIT_FSIZE].rlim_cur;
  1729. if (unlikely(*pos < 0))
  1730. return -EINVAL;
  1731. if (!isblk) {
  1732. /* FIXME: this is for backwards compatibility with 2.4 */
  1733. if (file->f_flags & O_APPEND)
  1734. *pos = i_size_read(inode);
  1735. if (limit != RLIM_INFINITY) {
  1736. if (*pos >= limit) {
  1737. send_sig(SIGXFSZ, current, 0);
  1738. return -EFBIG;
  1739. }
  1740. if (*count > limit - (typeof(limit))*pos) {
  1741. *count = limit - (typeof(limit))*pos;
  1742. }
  1743. }
  1744. }
  1745. /*
  1746. * LFS rule
  1747. */
  1748. if (unlikely(*pos + *count > MAX_NON_LFS &&
  1749. !(file->f_flags & O_LARGEFILE))) {
  1750. if (*pos >= MAX_NON_LFS) {
  1751. send_sig(SIGXFSZ, current, 0);
  1752. return -EFBIG;
  1753. }
  1754. if (*count > MAX_NON_LFS - (unsigned long)*pos) {
  1755. *count = MAX_NON_LFS - (unsigned long)*pos;
  1756. }
  1757. }
  1758. /*
  1759. * Are we about to exceed the fs block limit ?
  1760. *
  1761. * If we have written data it becomes a short write. If we have
  1762. * exceeded without writing data we send a signal and return EFBIG.
  1763. * Linus frestrict idea will clean these up nicely..
  1764. */
  1765. if (likely(!isblk)) {
  1766. if (unlikely(*pos >= inode->i_sb->s_maxbytes)) {
  1767. if (*count || *pos > inode->i_sb->s_maxbytes) {
  1768. send_sig(SIGXFSZ, current, 0);
  1769. return -EFBIG;
  1770. }
  1771. /* zero-length writes at ->s_maxbytes are OK */
  1772. }
  1773. if (unlikely(*pos + *count > inode->i_sb->s_maxbytes))
  1774. *count = inode->i_sb->s_maxbytes - *pos;
  1775. } else {
  1776. #ifdef CONFIG_BLOCK
  1777. loff_t isize;
  1778. if (bdev_read_only(I_BDEV(inode)))
  1779. return -EPERM;
  1780. isize = i_size_read(inode);
  1781. if (*pos >= isize) {
  1782. if (*count || *pos > isize)
  1783. return -ENOSPC;
  1784. }
  1785. if (*pos + *count > isize)
  1786. *count = isize - *pos;
  1787. #else
  1788. return -EPERM;
  1789. #endif
  1790. }
  1791. return 0;
  1792. }
  1793. EXPORT_SYMBOL(generic_write_checks);
  1794. ssize_t
  1795. generic_file_direct_write(struct kiocb *iocb, const struct iovec *iov,
  1796. unsigned long *nr_segs, loff_t pos, loff_t *ppos,
  1797. size_t count, size_t ocount)
  1798. {
  1799. struct file *file = iocb->ki_filp;
  1800. struct address_space *mapping = file->f_mapping;
  1801. struct inode *inode = mapping->host;
  1802. ssize_t written;
  1803. if (count != ocount)
  1804. *nr_segs = iov_shorten((struct iovec *)iov, *nr_segs, count);
  1805. written = generic_file_direct_IO(WRITE, iocb, iov, pos, *nr_segs);
  1806. if (written > 0) {
  1807. loff_t end = pos + written;
  1808. if (end > i_size_read(inode) && !S_ISBLK(inode->i_mode)) {
  1809. i_size_write(inode, end);
  1810. mark_inode_dirty(inode);
  1811. }
  1812. *ppos = end;
  1813. }
  1814. /*
  1815. * Sync the fs metadata but not the minor inode changes and
  1816. * of course not the data as we did direct DMA for the IO.
  1817. * i_mutex is held, which protects generic_osync_inode() from
  1818. * livelocking. AIO O_DIRECT ops attempt to sync metadata here.
  1819. */
  1820. if ((written >= 0 || written == -EIOCBQUEUED) &&
  1821. ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  1822. int err = generic_osync_inode(inode, mapping, OSYNC_METADATA);
  1823. if (err < 0)
  1824. written = err;
  1825. }
  1826. return written;
  1827. }
  1828. EXPORT_SYMBOL(generic_file_direct_write);
  1829. ssize_t
  1830. generic_file_buffered_write(struct kiocb *iocb, const struct iovec *iov,
  1831. unsigned long nr_segs, loff_t pos, loff_t *ppos,
  1832. size_t count, ssize_t written)
  1833. {
  1834. struct file *file = iocb->ki_filp;
  1835. struct address_space * mapping = file->f_mapping;
  1836. const struct address_space_operations *a_ops = mapping->a_ops;
  1837. struct inode *inode = mapping->host;
  1838. long status = 0;
  1839. struct page *page;
  1840. struct page *cached_page = NULL;
  1841. size_t bytes;
  1842. struct pagevec lru_pvec;
  1843. const struct iovec *cur_iov = iov; /* current iovec */
  1844. size_t iov_base = 0; /* offset in the current iovec */
  1845. char __user *buf;
  1846. pagevec_init(&lru_pvec, 0);
  1847. /*
  1848. * handle partial DIO write. Adjust cur_iov if needed.
  1849. */
  1850. if (likely(nr_segs == 1))
  1851. buf = iov->iov_base + written;
  1852. else {
  1853. filemap_set_next_iovec(&cur_iov, &iov_base, written);
  1854. buf = cur_iov->iov_base + iov_base;
  1855. }
  1856. do {
  1857. unsigned long index;
  1858. unsigned long offset;
  1859. size_t copied;
  1860. offset = (pos & (PAGE_CACHE_SIZE -1)); /* Within page */
  1861. index = pos >> PAGE_CACHE_SHIFT;
  1862. bytes = PAGE_CACHE_SIZE - offset;
  1863. /* Limit the size of the copy to the caller's write size */
  1864. bytes = min(bytes, count);
  1865. /* We only need to worry about prefaulting when writes are from
  1866. * user-space. NFSd uses vfs_writev with several non-aligned
  1867. * segments in the vector, and limiting to one segment a time is
  1868. * a noticeable performance for re-write
  1869. */
  1870. if (!segment_eq(get_fs(), KERNEL_DS)) {
  1871. /*
  1872. * Limit the size of the copy to that of the current
  1873. * segment, because fault_in_pages_readable() doesn't
  1874. * know how to walk segments.
  1875. */
  1876. bytes = min(bytes, cur_iov->iov_len - iov_base);
  1877. /*
  1878. * Bring in the user page that we will copy from
  1879. * _first_. Otherwise there's a nasty deadlock on
  1880. * copying from the same page as we're writing to,
  1881. * without it being marked up-to-date.
  1882. */
  1883. fault_in_pages_readable(buf, bytes);
  1884. }
  1885. page = __grab_cache_page(mapping,index,&cached_page,&lru_pvec);
  1886. if (!page) {
  1887. status = -ENOMEM;
  1888. break;
  1889. }
  1890. if (unlikely(bytes == 0)) {
  1891. status = 0;
  1892. copied = 0;
  1893. goto zero_length_segment;
  1894. }
  1895. status = a_ops->prepare_write(file, page, offset, offset+bytes);
  1896. if (unlikely(status)) {
  1897. loff_t isize = i_size_read(inode);
  1898. if (status != AOP_TRUNCATED_PAGE)
  1899. unlock_page(page);
  1900. page_cache_release(page);
  1901. if (status == AOP_TRUNCATED_PAGE)
  1902. continue;
  1903. /*
  1904. * prepare_write() may have instantiated a few blocks
  1905. * outside i_size. Trim these off again.
  1906. */
  1907. if (pos + bytes > isize)
  1908. vmtruncate(inode, isize);
  1909. break;
  1910. }
  1911. if (likely(nr_segs == 1))
  1912. copied = filemap_copy_from_user(page, offset,
  1913. buf, bytes);
  1914. else
  1915. copied = filemap_copy_from_user_iovec(page, offset,
  1916. cur_iov, iov_base, bytes);
  1917. flush_dcache_page(page);
  1918. status = a_ops->commit_write(file, page, offset, offset+bytes);
  1919. if (status == AOP_TRUNCATED_PAGE) {
  1920. page_cache_release(page);
  1921. continue;
  1922. }
  1923. zero_length_segment:
  1924. if (likely(copied >= 0)) {
  1925. if (!status)
  1926. status = copied;
  1927. if (status >= 0) {
  1928. written += status;
  1929. count -= status;
  1930. pos += status;
  1931. buf += status;
  1932. if (unlikely(nr_segs > 1)) {
  1933. filemap_set_next_iovec(&cur_iov,
  1934. &iov_base, status);
  1935. if (count)
  1936. buf = cur_iov->iov_base +
  1937. iov_base;
  1938. } else {
  1939. iov_base += status;
  1940. }
  1941. }
  1942. }
  1943. if (unlikely(copied != bytes))
  1944. if (status >= 0)
  1945. status = -EFAULT;
  1946. unlock_page(page);
  1947. mark_page_accessed(page);
  1948. page_cache_release(page);
  1949. if (status < 0)
  1950. break;
  1951. balance_dirty_pages_ratelimited(mapping);
  1952. cond_resched();
  1953. } while (count);
  1954. *ppos = pos;
  1955. if (cached_page)
  1956. page_cache_release(cached_page);
  1957. /*
  1958. * For now, when the user asks for O_SYNC, we'll actually give O_DSYNC
  1959. */
  1960. if (likely(status >= 0)) {
  1961. if (unlikely((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  1962. if (!a_ops->writepage || !is_sync_kiocb(iocb))
  1963. status = generic_osync_inode(inode, mapping,
  1964. OSYNC_METADATA|OSYNC_DATA);
  1965. }
  1966. }
  1967. /*
  1968. * If we get here for O_DIRECT writes then we must have fallen through
  1969. * to buffered writes (block instantiation inside i_size). So we sync
  1970. * the file data here, to try to honour O_DIRECT expectations.
  1971. */
  1972. if (unlikely(file->f_flags & O_DIRECT) && written)
  1973. status = filemap_write_and_wait(mapping);
  1974. pagevec_lru_add(&lru_pvec);
  1975. return written ? written : status;
  1976. }
  1977. EXPORT_SYMBOL(generic_file_buffered_write);
  1978. static ssize_t
  1979. __generic_file_aio_write_nolock(struct kiocb *iocb, const struct iovec *iov,
  1980. unsigned long nr_segs, loff_t *ppos)
  1981. {
  1982. struct file *file = iocb->ki_filp;
  1983. struct address_space * mapping = file->f_mapping;
  1984. size_t ocount; /* original count */
  1985. size_t count; /* after file limit checks */
  1986. struct inode *inode = mapping->host;
  1987. unsigned long seg;
  1988. loff_t pos;
  1989. ssize_t written;
  1990. ssize_t err;
  1991. ocount = 0;
  1992. for (seg = 0; seg < nr_segs; seg++) {
  1993. const struct iovec *iv = &iov[seg];
  1994. /*
  1995. * If any segment has a negative length, or the cumulative
  1996. * length ever wraps negative then return -EINVAL.
  1997. */
  1998. ocount += iv->iov_len;
  1999. if (unlikely((ssize_t)(ocount|iv->iov_len) < 0))
  2000. return -EINVAL;
  2001. if (access_ok(VERIFY_READ, iv->iov_base, iv->iov_len))
  2002. continue;
  2003. if (seg == 0)
  2004. return -EFAULT;
  2005. nr_segs = seg;
  2006. ocount -= iv->iov_len; /* This segment is no good */
  2007. break;
  2008. }
  2009. count = ocount;
  2010. pos = *ppos;
  2011. vfs_check_frozen(inode->i_sb, SB_FREEZE_WRITE);
  2012. /* We can write back this queue in page reclaim */
  2013. current->backing_dev_info = mapping->backing_dev_info;
  2014. written = 0;
  2015. err = generic_write_checks(file, &pos, &count, S_ISBLK(inode->i_mode));
  2016. if (err)
  2017. goto out;
  2018. if (count == 0)
  2019. goto out;
  2020. err = remove_suid(file->f_path.dentry);
  2021. if (err)
  2022. goto out;
  2023. file_update_time(file);
  2024. /* coalesce the iovecs and go direct-to-BIO for O_DIRECT */
  2025. if (unlikely(file->f_flags & O_DIRECT)) {
  2026. loff_t endbyte;
  2027. ssize_t written_buffered;
  2028. written = generic_file_direct_write(iocb, iov, &nr_segs, pos,
  2029. ppos, count, ocount);
  2030. if (written < 0 || written == count)
  2031. goto out;
  2032. /*
  2033. * direct-io write to a hole: fall through to buffered I/O
  2034. * for completing the rest of the request.
  2035. */
  2036. pos += written;
  2037. count -= written;
  2038. written_buffered = generic_file_buffered_write(iocb, iov,
  2039. nr_segs, pos, ppos, count,
  2040. written);
  2041. /*
  2042. * If generic_file_buffered_write() retuned a synchronous error
  2043. * then we want to return the number of bytes which were
  2044. * direct-written, or the error code if that was zero. Note
  2045. * that this differs from normal direct-io semantics, which
  2046. * will return -EFOO even if some bytes were written.
  2047. */
  2048. if (written_buffered < 0) {
  2049. err = written_buffered;
  2050. goto out;
  2051. }
  2052. /*
  2053. * We need to ensure that the page cache pages are written to
  2054. * disk and invalidated to preserve the expected O_DIRECT
  2055. * semantics.
  2056. */
  2057. endbyte = pos + written_buffered - written - 1;
  2058. err = do_sync_file_range(file, pos, endbyte,
  2059. SYNC_FILE_RANGE_WAIT_BEFORE|
  2060. SYNC_FILE_RANGE_WRITE|
  2061. SYNC_FILE_RANGE_WAIT_AFTER);
  2062. if (err == 0) {
  2063. written = written_buffered;
  2064. invalidate_mapping_pages(mapping,
  2065. pos >> PAGE_CACHE_SHIFT,
  2066. endbyte >> PAGE_CACHE_SHIFT);
  2067. } else {
  2068. /*
  2069. * We don't know how much we wrote, so just return
  2070. * the number of bytes which were direct-written
  2071. */
  2072. }
  2073. } else {
  2074. written = generic_file_buffered_write(iocb, iov, nr_segs,
  2075. pos, ppos, count, written);
  2076. }
  2077. out:
  2078. current->backing_dev_info = NULL;
  2079. return written ? written : err;
  2080. }
  2081. ssize_t generic_file_aio_write_nolock(struct kiocb *iocb,
  2082. const struct iovec *iov, unsigned long nr_segs, loff_t pos)
  2083. {
  2084. struct file *file = iocb->ki_filp;
  2085. struct address_space *mapping = file->f_mapping;
  2086. struct inode *inode = mapping->host;
  2087. ssize_t ret;
  2088. BUG_ON(iocb->ki_pos != pos);
  2089. ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
  2090. &iocb->ki_pos);
  2091. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2092. ssize_t err;
  2093. err = sync_page_range_nolock(inode, mapping, pos, ret);
  2094. if (err < 0)
  2095. ret = err;
  2096. }
  2097. return ret;
  2098. }
  2099. EXPORT_SYMBOL(generic_file_aio_write_nolock);
  2100. ssize_t generic_file_aio_write(struct kiocb *iocb, const struct iovec *iov,
  2101. unsigned long nr_segs, loff_t pos)
  2102. {
  2103. struct file *file = iocb->ki_filp;
  2104. struct address_space *mapping = file->f_mapping;
  2105. struct inode *inode = mapping->host;
  2106. ssize_t ret;
  2107. BUG_ON(iocb->ki_pos != pos);
  2108. mutex_lock(&inode->i_mutex);
  2109. ret = __generic_file_aio_write_nolock(iocb, iov, nr_segs,
  2110. &iocb->ki_pos);
  2111. mutex_unlock(&inode->i_mutex);
  2112. if (ret > 0 && ((file->f_flags & O_SYNC) || IS_SYNC(inode))) {
  2113. ssize_t err;
  2114. err = sync_page_range(inode, mapping, pos, ret);
  2115. if (err < 0)
  2116. ret = err;
  2117. }
  2118. return ret;
  2119. }
  2120. EXPORT_SYMBOL(generic_file_aio_write);
  2121. /*
  2122. * Called under i_mutex for writes to S_ISREG files. Returns -EIO if something
  2123. * went wrong during pagecache shootdown.
  2124. */
  2125. static ssize_t
  2126. generic_file_direct_IO(int rw, struct kiocb *iocb, const struct iovec *iov,
  2127. loff_t offset, unsigned long nr_segs)
  2128. {
  2129. struct file *file = iocb->ki_filp;
  2130. struct address_space *mapping = file->f_mapping;
  2131. ssize_t retval;
  2132. size_t write_len;
  2133. pgoff_t end = 0; /* silence gcc */
  2134. /*
  2135. * If it's a write, unmap all mmappings of the file up-front. This
  2136. * will cause any pte dirty bits to be propagated into the pageframes
  2137. * for the subsequent filemap_write_and_wait().
  2138. */
  2139. if (rw == WRITE) {
  2140. write_len = iov_length(iov, nr_segs);
  2141. end = (offset + write_len - 1) >> PAGE_CACHE_SHIFT;
  2142. if (mapping_mapped(mapping))
  2143. unmap_mapping_range(mapping, offset, write_len, 0);
  2144. }
  2145. retval = filemap_write_and_wait(mapping);
  2146. if (retval)
  2147. goto out;
  2148. /*
  2149. * After a write we want buffered reads to be sure to go to disk to get
  2150. * the new data. We invalidate clean cached page from the region we're
  2151. * about to write. We do this *before* the write so that we can return
  2152. * -EIO without clobbering -EIOCBQUEUED from ->direct_IO().
  2153. */
  2154. if (rw == WRITE && mapping->nrpages) {
  2155. retval = invalidate_inode_pages2_range(mapping,
  2156. offset >> PAGE_CACHE_SHIFT, end);
  2157. if (retval)
  2158. goto out;
  2159. }
  2160. retval = mapping->a_ops->direct_IO(rw, iocb, iov, offset, nr_segs);
  2161. if (retval)
  2162. goto out;
  2163. /*
  2164. * Finally, try again to invalidate clean pages which might have been
  2165. * faulted in by get_user_pages() if the source of the write was an
  2166. * mmap()ed region of the file we're writing. That's a pretty crazy
  2167. * thing to do, so we don't support it 100%. If this invalidation
  2168. * fails and we have -EIOCBQUEUED we ignore the failure.
  2169. */
  2170. if (rw == WRITE && mapping->nrpages) {
  2171. int err = invalidate_inode_pages2_range(mapping,
  2172. offset >> PAGE_CACHE_SHIFT, end);
  2173. if (err && retval >= 0)
  2174. retval = err;
  2175. }
  2176. out:
  2177. return retval;
  2178. }
  2179. /**
  2180. * try_to_release_page() - release old fs-specific metadata on a page
  2181. *
  2182. * @page: the page which the kernel is trying to free
  2183. * @gfp_mask: memory allocation flags (and I/O mode)
  2184. *
  2185. * The address_space is to try to release any data against the page
  2186. * (presumably at page->private). If the release was successful, return `1'.
  2187. * Otherwise return zero.
  2188. *
  2189. * The @gfp_mask argument specifies whether I/O may be performed to release
  2190. * this page (__GFP_IO), and whether the call may block (__GFP_WAIT).
  2191. *
  2192. * NOTE: @gfp_mask may go away, and this function may become non-blocking.
  2193. */
  2194. int try_to_release_page(struct page *page, gfp_t gfp_mask)
  2195. {
  2196. struct address_space * const mapping = page->mapping;
  2197. BUG_ON(!PageLocked(page));
  2198. if (PageWriteback(page))
  2199. return 0;
  2200. if (mapping && mapping->a_ops->releasepage)
  2201. return mapping->a_ops->releasepage(page, gfp_mask);
  2202. return try_to_free_buffers(page);
  2203. }
  2204. EXPORT_SYMBOL(try_to_release_page);