file.c 87 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645264626472648264926502651265226532654265526562657265826592660266126622663266426652666266726682669267026712672267326742675267626772678267926802681268226832684268526862687268826892690269126922693269426952696269726982699270027012702270327042705270627072708270927102711271227132714271527162717271827192720272127222723272427252726272727282729273027312732273327342735273627372738273927402741274227432744274527462747274827492750275127522753275427552756275727582759276027612762276327642765276627672768276927702771277227732774277527762777277827792780278127822783278427852786278727882789279027912792279327942795279627972798279928002801280228032804280528062807280828092810281128122813281428152816281728182819282028212822282328242825282628272828282928302831283228332834283528362837283828392840284128422843284428452846284728482849285028512852285328542855285628572858285928602861286228632864286528662867286828692870287128722873287428752876287728782879288028812882288328842885288628872888288928902891289228932894289528962897289828992900290129022903290429052906290729082909291029112912291329142915291629172918291929202921292229232924292529262927292829292930293129322933293429352936293729382939294029412942294329442945294629472948294929502951295229532954295529562957295829592960296129622963296429652966296729682969297029712972297329742975297629772978297929802981298229832984298529862987298829892990299129922993299429952996299729982999300030013002300330043005300630073008300930103011301230133014301530163017301830193020302130223023302430253026302730283029303030313032303330343035303630373038303930403041304230433044304530463047304830493050305130523053305430553056305730583059306030613062306330643065306630673068306930703071307230733074307530763077307830793080308130823083308430853086308730883089309030913092309330943095309630973098309931003101310231033104310531063107310831093110311131123113311431153116311731183119312031213122312331243125312631273128312931303131313231333134313531363137313831393140314131423143314431453146314731483149315031513152315331543155315631573158315931603161316231633164316531663167316831693170317131723173317431753176317731783179318031813182318331843185318631873188318931903191319231933194319531963197319831993200320132023203320432053206320732083209321032113212321332143215321632173218321932203221322232233224322532263227322832293230323132323233323432353236323732383239324032413242324332443245324632473248324932503251325232533254325532563257325832593260326132623263326432653266326732683269327032713272327332743275327632773278327932803281328232833284328532863287328832893290329132923293329432953296329732983299330033013302330333043305330633073308330933103311331233133314331533163317331833193320332133223323332433253326332733283329333033313332333333343335333633373338333933403341334233433344334533463347334833493350335133523353335433553356335733583359336033613362336333643365336633673368336933703371337233733374337533763377337833793380338133823383338433853386338733883389339033913392339333943395339633973398339934003401340234033404340534063407340834093410341134123413341434153416341734183419342034213422342334243425342634273428342934303431343234333434343534363437343834393440344134423443344434453446344734483449345034513452345334543455345634573458345934603461346234633464346534663467346834693470347134723473347434753476347734783479348034813482348334843485348634873488348934903491349234933494349534963497349834993500350135023503350435053506350735083509351035113512351335143515351635173518351935203521352235233524352535263527352835293530353135323533353435353536353735383539354035413542
  1. /*
  2. FUSE: Filesystem in Userspace
  3. Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
  4. This program can be distributed under the terms of the GNU GPL.
  5. See the file COPYING.
  6. */
  7. #include "fuse_i.h"
  8. #include <linux/pagemap.h>
  9. #include <linux/slab.h>
  10. #include <linux/kernel.h>
  11. #include <linux/sched.h>
  12. #include <linux/sched/signal.h>
  13. #include <linux/module.h>
  14. #include <linux/compat.h>
  15. #include <linux/swap.h>
  16. #include <linux/falloc.h>
  17. #include <linux/uio.h>
  18. #include <linux/fs.h>
  19. static struct page **fuse_pages_alloc(unsigned int npages, gfp_t flags,
  20. struct fuse_page_desc **desc)
  21. {
  22. struct page **pages;
  23. pages = kzalloc(npages * (sizeof(struct page *) +
  24. sizeof(struct fuse_page_desc)), flags);
  25. *desc = (void *) (pages + npages);
  26. return pages;
  27. }
  28. static int fuse_send_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
  29. int opcode, struct fuse_open_out *outargp)
  30. {
  31. struct fuse_open_in inarg;
  32. FUSE_ARGS(args);
  33. memset(&inarg, 0, sizeof(inarg));
  34. inarg.flags = file->f_flags & ~(O_CREAT | O_EXCL | O_NOCTTY);
  35. if (!fm->fc->atomic_o_trunc)
  36. inarg.flags &= ~O_TRUNC;
  37. args.opcode = opcode;
  38. args.nodeid = nodeid;
  39. args.in_numargs = 1;
  40. args.in_args[0].size = sizeof(inarg);
  41. args.in_args[0].value = &inarg;
  42. args.out_numargs = 1;
  43. args.out_args[0].size = sizeof(*outargp);
  44. args.out_args[0].value = outargp;
  45. return fuse_simple_request(fm, &args);
  46. }
  47. struct fuse_release_args {
  48. struct fuse_args args;
  49. struct fuse_release_in inarg;
  50. struct inode *inode;
  51. };
  52. struct fuse_file *fuse_file_alloc(struct fuse_mount *fm)
  53. {
  54. struct fuse_file *ff;
  55. ff = kzalloc(sizeof(struct fuse_file), GFP_KERNEL_ACCOUNT);
  56. if (unlikely(!ff))
  57. return NULL;
  58. ff->fm = fm;
  59. ff->release_args = kzalloc(sizeof(*ff->release_args),
  60. GFP_KERNEL_ACCOUNT);
  61. if (!ff->release_args) {
  62. kfree(ff);
  63. return NULL;
  64. }
  65. INIT_LIST_HEAD(&ff->write_entry);
  66. mutex_init(&ff->readdir.lock);
  67. refcount_set(&ff->count, 1);
  68. RB_CLEAR_NODE(&ff->polled_node);
  69. init_waitqueue_head(&ff->poll_wait);
  70. ff->kh = atomic64_inc_return(&fm->fc->khctr);
  71. return ff;
  72. }
  73. void fuse_file_free(struct fuse_file *ff)
  74. {
  75. kfree(ff->release_args);
  76. mutex_destroy(&ff->readdir.lock);
  77. kfree(ff);
  78. }
  79. static struct fuse_file *fuse_file_get(struct fuse_file *ff)
  80. {
  81. refcount_inc(&ff->count);
  82. return ff;
  83. }
  84. static void fuse_release_end(struct fuse_mount *fm, struct fuse_args *args,
  85. int error)
  86. {
  87. struct fuse_release_args *ra = container_of(args, typeof(*ra), args);
  88. iput(ra->inode);
  89. kfree(ra);
  90. }
  91. static void fuse_file_put(struct fuse_file *ff, bool sync, bool isdir)
  92. {
  93. if (refcount_dec_and_test(&ff->count)) {
  94. struct fuse_args *args = &ff->release_args->args;
  95. if (isdir ? ff->fm->fc->no_opendir : ff->fm->fc->no_open) {
  96. /* Do nothing when client does not implement 'open' */
  97. fuse_release_end(ff->fm, args, 0);
  98. } else if (sync) {
  99. fuse_simple_request(ff->fm, args);
  100. fuse_release_end(ff->fm, args, 0);
  101. } else {
  102. args->end = fuse_release_end;
  103. if (fuse_simple_background(ff->fm, args,
  104. GFP_KERNEL | __GFP_NOFAIL))
  105. fuse_release_end(ff->fm, args, -ENOTCONN);
  106. }
  107. kfree(ff);
  108. }
  109. }
  110. int fuse_do_open(struct fuse_mount *fm, u64 nodeid, struct file *file,
  111. bool isdir)
  112. {
  113. struct fuse_conn *fc = fm->fc;
  114. struct fuse_file *ff;
  115. int opcode = isdir ? FUSE_OPENDIR : FUSE_OPEN;
  116. ff = fuse_file_alloc(fm);
  117. if (!ff)
  118. return -ENOMEM;
  119. ff->fh = 0;
  120. /* Default for no-open */
  121. ff->open_flags = FOPEN_KEEP_CACHE | (isdir ? FOPEN_CACHE_DIR : 0);
  122. if (isdir ? !fc->no_opendir : !fc->no_open) {
  123. struct fuse_open_out outarg;
  124. int err;
  125. err = fuse_send_open(fm, nodeid, file, opcode, &outarg);
  126. if (!err) {
  127. ff->fh = outarg.fh;
  128. ff->open_flags = outarg.open_flags;
  129. fuse_passthrough_setup(fc, ff, &outarg);
  130. } else if (err != -ENOSYS) {
  131. fuse_file_free(ff);
  132. return err;
  133. } else {
  134. if (isdir)
  135. fc->no_opendir = 1;
  136. else
  137. fc->no_open = 1;
  138. }
  139. }
  140. if (isdir)
  141. ff->open_flags &= ~FOPEN_DIRECT_IO;
  142. ff->nodeid = nodeid;
  143. file->private_data = ff;
  144. return 0;
  145. }
  146. EXPORT_SYMBOL_GPL(fuse_do_open);
  147. static void fuse_link_write_file(struct file *file)
  148. {
  149. struct inode *inode = file_inode(file);
  150. struct fuse_inode *fi = get_fuse_inode(inode);
  151. struct fuse_file *ff = file->private_data;
  152. /*
  153. * file may be written through mmap, so chain it onto the
  154. * inodes's write_file list
  155. */
  156. spin_lock(&fi->lock);
  157. if (list_empty(&ff->write_entry))
  158. list_add(&ff->write_entry, &fi->write_files);
  159. spin_unlock(&fi->lock);
  160. }
  161. void fuse_finish_open(struct inode *inode, struct file *file)
  162. {
  163. struct fuse_file *ff = file->private_data;
  164. struct fuse_conn *fc = get_fuse_conn(inode);
  165. if (ff->open_flags & FOPEN_STREAM)
  166. stream_open(inode, file);
  167. else if (ff->open_flags & FOPEN_NONSEEKABLE)
  168. nonseekable_open(inode, file);
  169. if (fc->atomic_o_trunc && (file->f_flags & O_TRUNC)) {
  170. struct fuse_inode *fi = get_fuse_inode(inode);
  171. spin_lock(&fi->lock);
  172. fi->attr_version = atomic64_inc_return(&fc->attr_version);
  173. i_size_write(inode, 0);
  174. spin_unlock(&fi->lock);
  175. truncate_pagecache(inode, 0);
  176. fuse_invalidate_attr(inode);
  177. if (fc->writeback_cache)
  178. file_update_time(file);
  179. } else if (!(ff->open_flags & FOPEN_KEEP_CACHE)) {
  180. invalidate_inode_pages2(inode->i_mapping);
  181. }
  182. if ((file->f_mode & FMODE_WRITE) && fc->writeback_cache)
  183. fuse_link_write_file(file);
  184. }
  185. int fuse_open_common(struct inode *inode, struct file *file, bool isdir)
  186. {
  187. struct fuse_mount *fm = get_fuse_mount(inode);
  188. struct fuse_conn *fc = fm->fc;
  189. int err;
  190. bool is_wb_truncate = (file->f_flags & O_TRUNC) &&
  191. fc->atomic_o_trunc &&
  192. fc->writeback_cache;
  193. bool dax_truncate = (file->f_flags & O_TRUNC) &&
  194. fc->atomic_o_trunc && FUSE_IS_DAX(inode);
  195. if (fuse_is_bad(inode))
  196. return -EIO;
  197. err = generic_file_open(inode, file);
  198. if (err)
  199. return err;
  200. if (is_wb_truncate || dax_truncate) {
  201. inode_lock(inode);
  202. fuse_set_nowrite(inode);
  203. }
  204. if (dax_truncate) {
  205. down_write(&get_fuse_inode(inode)->i_mmap_sem);
  206. err = fuse_dax_break_layouts(inode, 0, 0);
  207. if (err)
  208. goto out;
  209. }
  210. err = fuse_do_open(fm, get_node_id(inode), file, isdir);
  211. if (!err)
  212. fuse_finish_open(inode, file);
  213. out:
  214. if (dax_truncate)
  215. up_write(&get_fuse_inode(inode)->i_mmap_sem);
  216. if (is_wb_truncate | dax_truncate) {
  217. fuse_release_nowrite(inode);
  218. inode_unlock(inode);
  219. }
  220. return err;
  221. }
  222. static void fuse_prepare_release(struct fuse_inode *fi, struct fuse_file *ff,
  223. int flags, int opcode)
  224. {
  225. struct fuse_conn *fc = ff->fm->fc;
  226. struct fuse_release_args *ra = ff->release_args;
  227. /* Inode is NULL on error path of fuse_create_open() */
  228. if (likely(fi)) {
  229. spin_lock(&fi->lock);
  230. list_del(&ff->write_entry);
  231. spin_unlock(&fi->lock);
  232. }
  233. spin_lock(&fc->lock);
  234. if (!RB_EMPTY_NODE(&ff->polled_node))
  235. rb_erase(&ff->polled_node, &fc->polled_files);
  236. spin_unlock(&fc->lock);
  237. wake_up_interruptible_all(&ff->poll_wait);
  238. ra->inarg.fh = ff->fh;
  239. ra->inarg.flags = flags;
  240. ra->args.in_numargs = 1;
  241. ra->args.in_args[0].size = sizeof(struct fuse_release_in);
  242. ra->args.in_args[0].value = &ra->inarg;
  243. ra->args.opcode = opcode;
  244. ra->args.nodeid = ff->nodeid;
  245. ra->args.force = true;
  246. ra->args.nocreds = true;
  247. }
  248. void fuse_release_common(struct file *file, bool isdir)
  249. {
  250. struct fuse_inode *fi = get_fuse_inode(file_inode(file));
  251. struct fuse_file *ff = file->private_data;
  252. struct fuse_release_args *ra = ff->release_args;
  253. int opcode = isdir ? FUSE_RELEASEDIR : FUSE_RELEASE;
  254. fuse_passthrough_release(&ff->passthrough);
  255. fuse_prepare_release(fi, ff, file->f_flags, opcode);
  256. if (ff->flock) {
  257. ra->inarg.release_flags |= FUSE_RELEASE_FLOCK_UNLOCK;
  258. ra->inarg.lock_owner = fuse_lock_owner_id(ff->fm->fc,
  259. (fl_owner_t) file);
  260. }
  261. /* Hold inode until release is finished */
  262. ra->inode = igrab(file_inode(file));
  263. /*
  264. * Normally this will send the RELEASE request, however if
  265. * some asynchronous READ or WRITE requests are outstanding,
  266. * the sending will be delayed.
  267. *
  268. * Make the release synchronous if this is a fuseblk mount,
  269. * synchronous RELEASE is allowed (and desirable) in this case
  270. * because the server can be trusted not to screw up.
  271. */
  272. fuse_file_put(ff, ff->fm->fc->destroy, isdir);
  273. }
  274. static int fuse_open(struct inode *inode, struct file *file)
  275. {
  276. return fuse_open_common(inode, file, false);
  277. }
  278. static int fuse_release(struct inode *inode, struct file *file)
  279. {
  280. struct fuse_conn *fc = get_fuse_conn(inode);
  281. /* see fuse_vma_close() for !writeback_cache case */
  282. if (fc->writeback_cache)
  283. write_inode_now(inode, 1);
  284. fuse_release_common(file, false);
  285. /* return value is ignored by VFS */
  286. return 0;
  287. }
  288. void fuse_sync_release(struct fuse_inode *fi, struct fuse_file *ff, int flags)
  289. {
  290. WARN_ON(refcount_read(&ff->count) > 1);
  291. fuse_prepare_release(fi, ff, flags, FUSE_RELEASE);
  292. /*
  293. * iput(NULL) is a no-op and since the refcount is 1 and everything's
  294. * synchronous, we are fine with not doing igrab() here"
  295. */
  296. fuse_file_put(ff, true, false);
  297. }
  298. EXPORT_SYMBOL_GPL(fuse_sync_release);
  299. /*
  300. * Scramble the ID space with XTEA, so that the value of the files_struct
  301. * pointer is not exposed to userspace.
  302. */
  303. u64 fuse_lock_owner_id(struct fuse_conn *fc, fl_owner_t id)
  304. {
  305. u32 *k = fc->scramble_key;
  306. u64 v = (unsigned long) id;
  307. u32 v0 = v;
  308. u32 v1 = v >> 32;
  309. u32 sum = 0;
  310. int i;
  311. for (i = 0; i < 32; i++) {
  312. v0 += ((v1 << 4 ^ v1 >> 5) + v1) ^ (sum + k[sum & 3]);
  313. sum += 0x9E3779B9;
  314. v1 += ((v0 << 4 ^ v0 >> 5) + v0) ^ (sum + k[sum>>11 & 3]);
  315. }
  316. return (u64) v0 + ((u64) v1 << 32);
  317. }
  318. struct fuse_writepage_args {
  319. struct fuse_io_args ia;
  320. struct rb_node writepages_entry;
  321. struct list_head queue_entry;
  322. struct fuse_writepage_args *next;
  323. struct inode *inode;
  324. };
  325. static struct fuse_writepage_args *fuse_find_writeback(struct fuse_inode *fi,
  326. pgoff_t idx_from, pgoff_t idx_to)
  327. {
  328. struct rb_node *n;
  329. n = fi->writepages.rb_node;
  330. while (n) {
  331. struct fuse_writepage_args *wpa;
  332. pgoff_t curr_index;
  333. wpa = rb_entry(n, struct fuse_writepage_args, writepages_entry);
  334. WARN_ON(get_fuse_inode(wpa->inode) != fi);
  335. curr_index = wpa->ia.write.in.offset >> PAGE_SHIFT;
  336. if (idx_from >= curr_index + wpa->ia.ap.num_pages)
  337. n = n->rb_right;
  338. else if (idx_to < curr_index)
  339. n = n->rb_left;
  340. else
  341. return wpa;
  342. }
  343. return NULL;
  344. }
  345. /*
  346. * Check if any page in a range is under writeback
  347. *
  348. * This is currently done by walking the list of writepage requests
  349. * for the inode, which can be pretty inefficient.
  350. */
  351. static bool fuse_range_is_writeback(struct inode *inode, pgoff_t idx_from,
  352. pgoff_t idx_to)
  353. {
  354. struct fuse_inode *fi = get_fuse_inode(inode);
  355. bool found;
  356. spin_lock(&fi->lock);
  357. found = fuse_find_writeback(fi, idx_from, idx_to);
  358. spin_unlock(&fi->lock);
  359. return found;
  360. }
  361. static inline bool fuse_page_is_writeback(struct inode *inode, pgoff_t index)
  362. {
  363. return fuse_range_is_writeback(inode, index, index);
  364. }
  365. /*
  366. * Wait for page writeback to be completed.
  367. *
  368. * Since fuse doesn't rely on the VM writeback tracking, this has to
  369. * use some other means.
  370. */
  371. static void fuse_wait_on_page_writeback(struct inode *inode, pgoff_t index)
  372. {
  373. struct fuse_inode *fi = get_fuse_inode(inode);
  374. wait_event(fi->page_waitq, !fuse_page_is_writeback(inode, index));
  375. }
  376. /*
  377. * Wait for all pending writepages on the inode to finish.
  378. *
  379. * This is currently done by blocking further writes with FUSE_NOWRITE
  380. * and waiting for all sent writes to complete.
  381. *
  382. * This must be called under i_mutex, otherwise the FUSE_NOWRITE usage
  383. * could conflict with truncation.
  384. */
  385. static void fuse_sync_writes(struct inode *inode)
  386. {
  387. fuse_set_nowrite(inode);
  388. fuse_release_nowrite(inode);
  389. }
  390. static int fuse_flush(struct file *file, fl_owner_t id)
  391. {
  392. struct inode *inode = file_inode(file);
  393. struct fuse_mount *fm = get_fuse_mount(inode);
  394. struct fuse_file *ff = file->private_data;
  395. struct fuse_flush_in inarg;
  396. FUSE_ARGS(args);
  397. int err;
  398. if (fuse_is_bad(inode))
  399. return -EIO;
  400. err = write_inode_now(inode, 1);
  401. if (err)
  402. return err;
  403. inode_lock(inode);
  404. fuse_sync_writes(inode);
  405. inode_unlock(inode);
  406. err = filemap_check_errors(file->f_mapping);
  407. if (err)
  408. return err;
  409. err = 0;
  410. if (fm->fc->no_flush)
  411. goto inval_attr_out;
  412. memset(&inarg, 0, sizeof(inarg));
  413. inarg.fh = ff->fh;
  414. inarg.lock_owner = fuse_lock_owner_id(fm->fc, id);
  415. args.opcode = FUSE_FLUSH;
  416. args.nodeid = get_node_id(inode);
  417. args.in_numargs = 1;
  418. args.in_args[0].size = sizeof(inarg);
  419. args.in_args[0].value = &inarg;
  420. args.force = true;
  421. err = fuse_simple_request(fm, &args);
  422. if (err == -ENOSYS) {
  423. fm->fc->no_flush = 1;
  424. err = 0;
  425. }
  426. inval_attr_out:
  427. /*
  428. * In memory i_blocks is not maintained by fuse, if writeback cache is
  429. * enabled, i_blocks from cached attr may not be accurate.
  430. */
  431. if (!err && fm->fc->writeback_cache)
  432. fuse_invalidate_attr(inode);
  433. return err;
  434. }
  435. int fuse_fsync_common(struct file *file, loff_t start, loff_t end,
  436. int datasync, int opcode)
  437. {
  438. struct inode *inode = file->f_mapping->host;
  439. struct fuse_mount *fm = get_fuse_mount(inode);
  440. struct fuse_file *ff = file->private_data;
  441. FUSE_ARGS(args);
  442. struct fuse_fsync_in inarg;
  443. memset(&inarg, 0, sizeof(inarg));
  444. inarg.fh = ff->fh;
  445. inarg.fsync_flags = datasync ? FUSE_FSYNC_FDATASYNC : 0;
  446. args.opcode = opcode;
  447. args.nodeid = get_node_id(inode);
  448. args.in_numargs = 1;
  449. args.in_args[0].size = sizeof(inarg);
  450. args.in_args[0].value = &inarg;
  451. return fuse_simple_request(fm, &args);
  452. }
  453. static int fuse_fsync(struct file *file, loff_t start, loff_t end,
  454. int datasync)
  455. {
  456. struct inode *inode = file->f_mapping->host;
  457. struct fuse_conn *fc = get_fuse_conn(inode);
  458. int err;
  459. if (fuse_is_bad(inode))
  460. return -EIO;
  461. inode_lock(inode);
  462. /*
  463. * Start writeback against all dirty pages of the inode, then
  464. * wait for all outstanding writes, before sending the FSYNC
  465. * request.
  466. */
  467. err = file_write_and_wait_range(file, start, end);
  468. if (err)
  469. goto out;
  470. fuse_sync_writes(inode);
  471. /*
  472. * Due to implementation of fuse writeback
  473. * file_write_and_wait_range() does not catch errors.
  474. * We have to do this directly after fuse_sync_writes()
  475. */
  476. err = file_check_and_advance_wb_err(file);
  477. if (err)
  478. goto out;
  479. err = sync_inode_metadata(inode, 1);
  480. if (err)
  481. goto out;
  482. if (fc->no_fsync)
  483. goto out;
  484. err = fuse_fsync_common(file, start, end, datasync, FUSE_FSYNC);
  485. if (err == -ENOSYS) {
  486. fc->no_fsync = 1;
  487. err = 0;
  488. }
  489. out:
  490. inode_unlock(inode);
  491. return err;
  492. }
  493. void fuse_read_args_fill(struct fuse_io_args *ia, struct file *file, loff_t pos,
  494. size_t count, int opcode)
  495. {
  496. struct fuse_file *ff = file->private_data;
  497. struct fuse_args *args = &ia->ap.args;
  498. ia->read.in.fh = ff->fh;
  499. ia->read.in.offset = pos;
  500. ia->read.in.size = count;
  501. ia->read.in.flags = file->f_flags;
  502. args->opcode = opcode;
  503. args->nodeid = ff->nodeid;
  504. args->in_numargs = 1;
  505. args->in_args[0].size = sizeof(ia->read.in);
  506. args->in_args[0].value = &ia->read.in;
  507. args->out_argvar = true;
  508. args->out_numargs = 1;
  509. args->out_args[0].size = count;
  510. }
  511. static void fuse_release_user_pages(struct fuse_args_pages *ap,
  512. bool should_dirty)
  513. {
  514. unsigned int i;
  515. for (i = 0; i < ap->num_pages; i++) {
  516. if (should_dirty)
  517. set_page_dirty_lock(ap->pages[i]);
  518. put_page(ap->pages[i]);
  519. }
  520. }
  521. static void fuse_io_release(struct kref *kref)
  522. {
  523. kfree(container_of(kref, struct fuse_io_priv, refcnt));
  524. }
  525. static ssize_t fuse_get_res_by_io(struct fuse_io_priv *io)
  526. {
  527. if (io->err)
  528. return io->err;
  529. if (io->bytes >= 0 && io->write)
  530. return -EIO;
  531. return io->bytes < 0 ? io->size : io->bytes;
  532. }
  533. /**
  534. * In case of short read, the caller sets 'pos' to the position of
  535. * actual end of fuse request in IO request. Otherwise, if bytes_requested
  536. * == bytes_transferred or rw == WRITE, the caller sets 'pos' to -1.
  537. *
  538. * An example:
  539. * User requested DIO read of 64K. It was splitted into two 32K fuse requests,
  540. * both submitted asynchronously. The first of them was ACKed by userspace as
  541. * fully completed (req->out.args[0].size == 32K) resulting in pos == -1. The
  542. * second request was ACKed as short, e.g. only 1K was read, resulting in
  543. * pos == 33K.
  544. *
  545. * Thus, when all fuse requests are completed, the minimal non-negative 'pos'
  546. * will be equal to the length of the longest contiguous fragment of
  547. * transferred data starting from the beginning of IO request.
  548. */
  549. static void fuse_aio_complete(struct fuse_io_priv *io, int err, ssize_t pos)
  550. {
  551. int left;
  552. spin_lock(&io->lock);
  553. if (err)
  554. io->err = io->err ? : err;
  555. else if (pos >= 0 && (io->bytes < 0 || pos < io->bytes))
  556. io->bytes = pos;
  557. left = --io->reqs;
  558. if (!left && io->blocking)
  559. complete(io->done);
  560. spin_unlock(&io->lock);
  561. if (!left && !io->blocking) {
  562. ssize_t res = fuse_get_res_by_io(io);
  563. if (res >= 0) {
  564. struct inode *inode = file_inode(io->iocb->ki_filp);
  565. struct fuse_conn *fc = get_fuse_conn(inode);
  566. struct fuse_inode *fi = get_fuse_inode(inode);
  567. spin_lock(&fi->lock);
  568. fi->attr_version = atomic64_inc_return(&fc->attr_version);
  569. spin_unlock(&fi->lock);
  570. }
  571. io->iocb->ki_complete(io->iocb, res, 0);
  572. }
  573. kref_put(&io->refcnt, fuse_io_release);
  574. }
  575. static struct fuse_io_args *fuse_io_alloc(struct fuse_io_priv *io,
  576. unsigned int npages)
  577. {
  578. struct fuse_io_args *ia;
  579. ia = kzalloc(sizeof(*ia), GFP_KERNEL);
  580. if (ia) {
  581. ia->io = io;
  582. ia->ap.pages = fuse_pages_alloc(npages, GFP_KERNEL,
  583. &ia->ap.descs);
  584. if (!ia->ap.pages) {
  585. kfree(ia);
  586. ia = NULL;
  587. }
  588. }
  589. return ia;
  590. }
  591. static void fuse_io_free(struct fuse_io_args *ia)
  592. {
  593. kfree(ia->ap.pages);
  594. kfree(ia);
  595. }
  596. static void fuse_aio_complete_req(struct fuse_mount *fm, struct fuse_args *args,
  597. int err)
  598. {
  599. struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
  600. struct fuse_io_priv *io = ia->io;
  601. ssize_t pos = -1;
  602. fuse_release_user_pages(&ia->ap, io->should_dirty);
  603. if (err) {
  604. /* Nothing */
  605. } else if (io->write) {
  606. if (ia->write.out.size > ia->write.in.size) {
  607. err = -EIO;
  608. } else if (ia->write.in.size != ia->write.out.size) {
  609. pos = ia->write.in.offset - io->offset +
  610. ia->write.out.size;
  611. }
  612. } else {
  613. u32 outsize = args->out_args[0].size;
  614. if (ia->read.in.size != outsize)
  615. pos = ia->read.in.offset - io->offset + outsize;
  616. }
  617. fuse_aio_complete(io, err, pos);
  618. fuse_io_free(ia);
  619. }
  620. static ssize_t fuse_async_req_send(struct fuse_mount *fm,
  621. struct fuse_io_args *ia, size_t num_bytes)
  622. {
  623. ssize_t err;
  624. struct fuse_io_priv *io = ia->io;
  625. spin_lock(&io->lock);
  626. kref_get(&io->refcnt);
  627. io->size += num_bytes;
  628. io->reqs++;
  629. spin_unlock(&io->lock);
  630. ia->ap.args.end = fuse_aio_complete_req;
  631. ia->ap.args.may_block = io->should_dirty;
  632. err = fuse_simple_background(fm, &ia->ap.args, GFP_KERNEL);
  633. if (err)
  634. fuse_aio_complete_req(fm, &ia->ap.args, err);
  635. return num_bytes;
  636. }
  637. static ssize_t fuse_send_read(struct fuse_io_args *ia, loff_t pos, size_t count,
  638. fl_owner_t owner)
  639. {
  640. struct file *file = ia->io->iocb->ki_filp;
  641. struct fuse_file *ff = file->private_data;
  642. struct fuse_mount *fm = ff->fm;
  643. fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
  644. if (owner != NULL) {
  645. ia->read.in.read_flags |= FUSE_READ_LOCKOWNER;
  646. ia->read.in.lock_owner = fuse_lock_owner_id(fm->fc, owner);
  647. }
  648. if (ia->io->async)
  649. return fuse_async_req_send(fm, ia, count);
  650. return fuse_simple_request(fm, &ia->ap.args);
  651. }
  652. static void fuse_read_update_size(struct inode *inode, loff_t size,
  653. u64 attr_ver)
  654. {
  655. struct fuse_conn *fc = get_fuse_conn(inode);
  656. struct fuse_inode *fi = get_fuse_inode(inode);
  657. spin_lock(&fi->lock);
  658. if (attr_ver == fi->attr_version && size < inode->i_size &&
  659. !test_bit(FUSE_I_SIZE_UNSTABLE, &fi->state)) {
  660. fi->attr_version = atomic64_inc_return(&fc->attr_version);
  661. i_size_write(inode, size);
  662. }
  663. spin_unlock(&fi->lock);
  664. }
  665. static void fuse_short_read(struct inode *inode, u64 attr_ver, size_t num_read,
  666. struct fuse_args_pages *ap)
  667. {
  668. struct fuse_conn *fc = get_fuse_conn(inode);
  669. if (fc->writeback_cache) {
  670. /*
  671. * A hole in a file. Some data after the hole are in page cache,
  672. * but have not reached the client fs yet. So, the hole is not
  673. * present there.
  674. */
  675. int i;
  676. int start_idx = num_read >> PAGE_SHIFT;
  677. size_t off = num_read & (PAGE_SIZE - 1);
  678. for (i = start_idx; i < ap->num_pages; i++) {
  679. zero_user_segment(ap->pages[i], off, PAGE_SIZE);
  680. off = 0;
  681. }
  682. } else {
  683. loff_t pos = page_offset(ap->pages[0]) + num_read;
  684. fuse_read_update_size(inode, pos, attr_ver);
  685. }
  686. }
  687. static int fuse_do_readpage(struct file *file, struct page *page)
  688. {
  689. struct inode *inode = page->mapping->host;
  690. struct fuse_mount *fm = get_fuse_mount(inode);
  691. loff_t pos = page_offset(page);
  692. struct fuse_page_desc desc = { .length = PAGE_SIZE };
  693. struct fuse_io_args ia = {
  694. .ap.args.page_zeroing = true,
  695. .ap.args.out_pages = true,
  696. .ap.num_pages = 1,
  697. .ap.pages = &page,
  698. .ap.descs = &desc,
  699. };
  700. ssize_t res;
  701. u64 attr_ver;
  702. /*
  703. * Page writeback can extend beyond the lifetime of the
  704. * page-cache page, so make sure we read a properly synced
  705. * page.
  706. */
  707. fuse_wait_on_page_writeback(inode, page->index);
  708. attr_ver = fuse_get_attr_version(fm->fc);
  709. /* Don't overflow end offset */
  710. if (pos + (desc.length - 1) == LLONG_MAX)
  711. desc.length--;
  712. fuse_read_args_fill(&ia, file, pos, desc.length, FUSE_READ);
  713. res = fuse_simple_request(fm, &ia.ap.args);
  714. if (res < 0)
  715. return res;
  716. /*
  717. * Short read means EOF. If file size is larger, truncate it
  718. */
  719. if (res < desc.length)
  720. fuse_short_read(inode, attr_ver, res, &ia.ap);
  721. SetPageUptodate(page);
  722. return 0;
  723. }
  724. static int fuse_readpage(struct file *file, struct page *page)
  725. {
  726. struct inode *inode = page->mapping->host;
  727. int err;
  728. err = -EIO;
  729. if (fuse_is_bad(inode))
  730. goto out;
  731. err = fuse_do_readpage(file, page);
  732. fuse_invalidate_atime(inode);
  733. out:
  734. unlock_page(page);
  735. return err;
  736. }
  737. static void fuse_readpages_end(struct fuse_mount *fm, struct fuse_args *args,
  738. int err)
  739. {
  740. int i;
  741. struct fuse_io_args *ia = container_of(args, typeof(*ia), ap.args);
  742. struct fuse_args_pages *ap = &ia->ap;
  743. size_t count = ia->read.in.size;
  744. size_t num_read = args->out_args[0].size;
  745. struct address_space *mapping = NULL;
  746. for (i = 0; mapping == NULL && i < ap->num_pages; i++)
  747. mapping = ap->pages[i]->mapping;
  748. if (mapping) {
  749. struct inode *inode = mapping->host;
  750. /*
  751. * Short read means EOF. If file size is larger, truncate it
  752. */
  753. if (!err && num_read < count)
  754. fuse_short_read(inode, ia->read.attr_ver, num_read, ap);
  755. fuse_invalidate_atime(inode);
  756. }
  757. for (i = 0; i < ap->num_pages; i++) {
  758. struct page *page = ap->pages[i];
  759. if (!err)
  760. SetPageUptodate(page);
  761. else
  762. SetPageError(page);
  763. unlock_page(page);
  764. put_page(page);
  765. }
  766. if (ia->ff)
  767. fuse_file_put(ia->ff, false, false);
  768. fuse_io_free(ia);
  769. }
  770. static void fuse_send_readpages(struct fuse_io_args *ia, struct file *file)
  771. {
  772. struct fuse_file *ff = file->private_data;
  773. struct fuse_mount *fm = ff->fm;
  774. struct fuse_args_pages *ap = &ia->ap;
  775. loff_t pos = page_offset(ap->pages[0]);
  776. size_t count = ap->num_pages << PAGE_SHIFT;
  777. ssize_t res;
  778. int err;
  779. ap->args.out_pages = true;
  780. ap->args.page_zeroing = true;
  781. ap->args.page_replace = true;
  782. /* Don't overflow end offset */
  783. if (pos + (count - 1) == LLONG_MAX) {
  784. count--;
  785. ap->descs[ap->num_pages - 1].length--;
  786. }
  787. WARN_ON((loff_t) (pos + count) < 0);
  788. fuse_read_args_fill(ia, file, pos, count, FUSE_READ);
  789. ia->read.attr_ver = fuse_get_attr_version(fm->fc);
  790. if (fm->fc->async_read) {
  791. ia->ff = fuse_file_get(ff);
  792. ap->args.end = fuse_readpages_end;
  793. err = fuse_simple_background(fm, &ap->args, GFP_KERNEL);
  794. if (!err)
  795. return;
  796. } else {
  797. res = fuse_simple_request(fm, &ap->args);
  798. err = res < 0 ? res : 0;
  799. }
  800. fuse_readpages_end(fm, &ap->args, err);
  801. }
  802. static void fuse_readahead(struct readahead_control *rac)
  803. {
  804. struct inode *inode = rac->mapping->host;
  805. struct fuse_conn *fc = get_fuse_conn(inode);
  806. unsigned int i, max_pages, nr_pages = 0;
  807. if (fuse_is_bad(inode))
  808. return;
  809. max_pages = min_t(unsigned int, fc->max_pages,
  810. fc->max_read / PAGE_SIZE);
  811. for (;;) {
  812. struct fuse_io_args *ia;
  813. struct fuse_args_pages *ap;
  814. nr_pages = readahead_count(rac) - nr_pages;
  815. if (nr_pages > max_pages)
  816. nr_pages = max_pages;
  817. if (nr_pages == 0)
  818. break;
  819. ia = fuse_io_alloc(NULL, nr_pages);
  820. if (!ia)
  821. return;
  822. ap = &ia->ap;
  823. nr_pages = __readahead_batch(rac, ap->pages, nr_pages);
  824. for (i = 0; i < nr_pages; i++) {
  825. fuse_wait_on_page_writeback(inode,
  826. readahead_index(rac) + i);
  827. ap->descs[i].length = PAGE_SIZE;
  828. }
  829. ap->num_pages = nr_pages;
  830. fuse_send_readpages(ia, rac->file);
  831. }
  832. }
  833. static ssize_t fuse_cache_read_iter(struct kiocb *iocb, struct iov_iter *to)
  834. {
  835. struct inode *inode = iocb->ki_filp->f_mapping->host;
  836. struct fuse_conn *fc = get_fuse_conn(inode);
  837. /*
  838. * In auto invalidate mode, always update attributes on read.
  839. * Otherwise, only update if we attempt to read past EOF (to ensure
  840. * i_size is up to date).
  841. */
  842. if (fc->auto_inval_data ||
  843. (iocb->ki_pos + iov_iter_count(to) > i_size_read(inode))) {
  844. int err;
  845. err = fuse_update_attributes(inode, iocb->ki_filp);
  846. if (err)
  847. return err;
  848. }
  849. return generic_file_read_iter(iocb, to);
  850. }
  851. static void fuse_write_args_fill(struct fuse_io_args *ia, struct fuse_file *ff,
  852. loff_t pos, size_t count)
  853. {
  854. struct fuse_args *args = &ia->ap.args;
  855. ia->write.in.fh = ff->fh;
  856. ia->write.in.offset = pos;
  857. ia->write.in.size = count;
  858. args->opcode = FUSE_WRITE;
  859. args->nodeid = ff->nodeid;
  860. args->in_numargs = 2;
  861. if (ff->fm->fc->minor < 9)
  862. args->in_args[0].size = FUSE_COMPAT_WRITE_IN_SIZE;
  863. else
  864. args->in_args[0].size = sizeof(ia->write.in);
  865. args->in_args[0].value = &ia->write.in;
  866. args->in_args[1].size = count;
  867. args->out_numargs = 1;
  868. args->out_args[0].size = sizeof(ia->write.out);
  869. args->out_args[0].value = &ia->write.out;
  870. }
  871. static unsigned int fuse_write_flags(struct kiocb *iocb)
  872. {
  873. unsigned int flags = iocb->ki_filp->f_flags;
  874. if (iocb->ki_flags & IOCB_DSYNC)
  875. flags |= O_DSYNC;
  876. if (iocb->ki_flags & IOCB_SYNC)
  877. flags |= O_SYNC;
  878. return flags;
  879. }
  880. static ssize_t fuse_send_write(struct fuse_io_args *ia, loff_t pos,
  881. size_t count, fl_owner_t owner)
  882. {
  883. struct kiocb *iocb = ia->io->iocb;
  884. struct file *file = iocb->ki_filp;
  885. struct fuse_file *ff = file->private_data;
  886. struct fuse_mount *fm = ff->fm;
  887. struct fuse_write_in *inarg = &ia->write.in;
  888. ssize_t err;
  889. fuse_write_args_fill(ia, ff, pos, count);
  890. inarg->flags = fuse_write_flags(iocb);
  891. if (owner != NULL) {
  892. inarg->write_flags |= FUSE_WRITE_LOCKOWNER;
  893. inarg->lock_owner = fuse_lock_owner_id(fm->fc, owner);
  894. }
  895. if (ia->io->async)
  896. return fuse_async_req_send(fm, ia, count);
  897. err = fuse_simple_request(fm, &ia->ap.args);
  898. if (!err && ia->write.out.size > count)
  899. err = -EIO;
  900. return err ?: ia->write.out.size;
  901. }
  902. bool fuse_write_update_size(struct inode *inode, loff_t pos)
  903. {
  904. struct fuse_conn *fc = get_fuse_conn(inode);
  905. struct fuse_inode *fi = get_fuse_inode(inode);
  906. bool ret = false;
  907. spin_lock(&fi->lock);
  908. fi->attr_version = atomic64_inc_return(&fc->attr_version);
  909. if (pos > inode->i_size) {
  910. i_size_write(inode, pos);
  911. ret = true;
  912. }
  913. spin_unlock(&fi->lock);
  914. return ret;
  915. }
  916. static ssize_t fuse_send_write_pages(struct fuse_io_args *ia,
  917. struct kiocb *iocb, struct inode *inode,
  918. loff_t pos, size_t count)
  919. {
  920. struct fuse_args_pages *ap = &ia->ap;
  921. struct file *file = iocb->ki_filp;
  922. struct fuse_file *ff = file->private_data;
  923. struct fuse_mount *fm = ff->fm;
  924. unsigned int offset, i;
  925. bool short_write;
  926. int err;
  927. for (i = 0; i < ap->num_pages; i++)
  928. fuse_wait_on_page_writeback(inode, ap->pages[i]->index);
  929. fuse_write_args_fill(ia, ff, pos, count);
  930. ia->write.in.flags = fuse_write_flags(iocb);
  931. err = fuse_simple_request(fm, &ap->args);
  932. if (!err && ia->write.out.size > count)
  933. err = -EIO;
  934. short_write = ia->write.out.size < count;
  935. offset = ap->descs[0].offset;
  936. count = ia->write.out.size;
  937. for (i = 0; i < ap->num_pages; i++) {
  938. struct page *page = ap->pages[i];
  939. if (err) {
  940. ClearPageUptodate(page);
  941. } else {
  942. if (count >= PAGE_SIZE - offset)
  943. count -= PAGE_SIZE - offset;
  944. else {
  945. if (short_write)
  946. ClearPageUptodate(page);
  947. count = 0;
  948. }
  949. offset = 0;
  950. }
  951. if (ia->write.page_locked && (i == ap->num_pages - 1))
  952. unlock_page(page);
  953. put_page(page);
  954. }
  955. return err;
  956. }
  957. static ssize_t fuse_fill_write_pages(struct fuse_io_args *ia,
  958. struct address_space *mapping,
  959. struct iov_iter *ii, loff_t pos,
  960. unsigned int max_pages)
  961. {
  962. struct fuse_args_pages *ap = &ia->ap;
  963. struct fuse_conn *fc = get_fuse_conn(mapping->host);
  964. unsigned offset = pos & (PAGE_SIZE - 1);
  965. size_t count = 0;
  966. int err;
  967. ap->args.in_pages = true;
  968. ap->descs[0].offset = offset;
  969. do {
  970. size_t tmp;
  971. struct page *page;
  972. pgoff_t index = pos >> PAGE_SHIFT;
  973. size_t bytes = min_t(size_t, PAGE_SIZE - offset,
  974. iov_iter_count(ii));
  975. bytes = min_t(size_t, bytes, fc->max_write - count);
  976. again:
  977. err = -EFAULT;
  978. if (iov_iter_fault_in_readable(ii, bytes))
  979. break;
  980. err = -ENOMEM;
  981. page = grab_cache_page_write_begin(mapping, index, 0);
  982. if (!page)
  983. break;
  984. if (mapping_writably_mapped(mapping))
  985. flush_dcache_page(page);
  986. tmp = iov_iter_copy_from_user_atomic(page, ii, offset, bytes);
  987. flush_dcache_page(page);
  988. iov_iter_advance(ii, tmp);
  989. if (!tmp) {
  990. unlock_page(page);
  991. put_page(page);
  992. bytes = min(bytes, iov_iter_single_seg_count(ii));
  993. goto again;
  994. }
  995. err = 0;
  996. ap->pages[ap->num_pages] = page;
  997. ap->descs[ap->num_pages].length = tmp;
  998. ap->num_pages++;
  999. count += tmp;
  1000. pos += tmp;
  1001. offset += tmp;
  1002. if (offset == PAGE_SIZE)
  1003. offset = 0;
  1004. /* If we copied full page, mark it uptodate */
  1005. if (tmp == PAGE_SIZE)
  1006. SetPageUptodate(page);
  1007. if (PageUptodate(page)) {
  1008. unlock_page(page);
  1009. } else {
  1010. ia->write.page_locked = true;
  1011. break;
  1012. }
  1013. if (!fc->big_writes)
  1014. break;
  1015. } while (iov_iter_count(ii) && count < fc->max_write &&
  1016. ap->num_pages < max_pages && offset == 0);
  1017. return count > 0 ? count : err;
  1018. }
  1019. static inline unsigned int fuse_wr_pages(loff_t pos, size_t len,
  1020. unsigned int max_pages)
  1021. {
  1022. return min_t(unsigned int,
  1023. ((pos + len - 1) >> PAGE_SHIFT) -
  1024. (pos >> PAGE_SHIFT) + 1,
  1025. max_pages);
  1026. }
  1027. static ssize_t fuse_perform_write(struct kiocb *iocb,
  1028. struct address_space *mapping,
  1029. struct iov_iter *ii, loff_t pos)
  1030. {
  1031. struct inode *inode = mapping->host;
  1032. struct fuse_conn *fc = get_fuse_conn(inode);
  1033. struct fuse_inode *fi = get_fuse_inode(inode);
  1034. int err = 0;
  1035. ssize_t res = 0;
  1036. if (inode->i_size < pos + iov_iter_count(ii))
  1037. set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
  1038. do {
  1039. ssize_t count;
  1040. struct fuse_io_args ia = {};
  1041. struct fuse_args_pages *ap = &ia.ap;
  1042. unsigned int nr_pages = fuse_wr_pages(pos, iov_iter_count(ii),
  1043. fc->max_pages);
  1044. ap->pages = fuse_pages_alloc(nr_pages, GFP_KERNEL, &ap->descs);
  1045. if (!ap->pages) {
  1046. err = -ENOMEM;
  1047. break;
  1048. }
  1049. count = fuse_fill_write_pages(&ia, mapping, ii, pos, nr_pages);
  1050. if (count <= 0) {
  1051. err = count;
  1052. } else {
  1053. err = fuse_send_write_pages(&ia, iocb, inode,
  1054. pos, count);
  1055. if (!err) {
  1056. size_t num_written = ia.write.out.size;
  1057. res += num_written;
  1058. pos += num_written;
  1059. /* break out of the loop on short write */
  1060. if (num_written != count)
  1061. err = -EIO;
  1062. }
  1063. }
  1064. kfree(ap->pages);
  1065. } while (!err && iov_iter_count(ii));
  1066. if (res > 0)
  1067. fuse_write_update_size(inode, pos);
  1068. clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
  1069. fuse_invalidate_attr(inode);
  1070. return res > 0 ? res : err;
  1071. }
  1072. static ssize_t fuse_cache_write_iter(struct kiocb *iocb, struct iov_iter *from)
  1073. {
  1074. struct file *file = iocb->ki_filp;
  1075. struct address_space *mapping = file->f_mapping;
  1076. ssize_t written = 0;
  1077. ssize_t written_buffered = 0;
  1078. struct inode *inode = mapping->host;
  1079. ssize_t err;
  1080. loff_t endbyte = 0;
  1081. if (get_fuse_conn(inode)->writeback_cache) {
  1082. /* Update size (EOF optimization) and mode (SUID clearing) */
  1083. err = fuse_update_attributes(mapping->host, file);
  1084. if (err)
  1085. return err;
  1086. return generic_file_write_iter(iocb, from);
  1087. }
  1088. inode_lock(inode);
  1089. /* We can write back this queue in page reclaim */
  1090. current->backing_dev_info = inode_to_bdi(inode);
  1091. err = generic_write_checks(iocb, from);
  1092. if (err <= 0)
  1093. goto out;
  1094. err = file_remove_privs(file);
  1095. if (err)
  1096. goto out;
  1097. err = file_update_time(file);
  1098. if (err)
  1099. goto out;
  1100. if (iocb->ki_flags & IOCB_DIRECT) {
  1101. loff_t pos = iocb->ki_pos;
  1102. written = generic_file_direct_write(iocb, from);
  1103. if (written < 0 || !iov_iter_count(from))
  1104. goto out;
  1105. pos += written;
  1106. written_buffered = fuse_perform_write(iocb, mapping, from, pos);
  1107. if (written_buffered < 0) {
  1108. err = written_buffered;
  1109. goto out;
  1110. }
  1111. endbyte = pos + written_buffered - 1;
  1112. err = filemap_write_and_wait_range(file->f_mapping, pos,
  1113. endbyte);
  1114. if (err)
  1115. goto out;
  1116. invalidate_mapping_pages(file->f_mapping,
  1117. pos >> PAGE_SHIFT,
  1118. endbyte >> PAGE_SHIFT);
  1119. written += written_buffered;
  1120. iocb->ki_pos = pos + written_buffered;
  1121. } else {
  1122. written = fuse_perform_write(iocb, mapping, from, iocb->ki_pos);
  1123. if (written >= 0)
  1124. iocb->ki_pos += written;
  1125. }
  1126. out:
  1127. current->backing_dev_info = NULL;
  1128. inode_unlock(inode);
  1129. if (written > 0)
  1130. written = generic_write_sync(iocb, written);
  1131. return written ? written : err;
  1132. }
  1133. static inline void fuse_page_descs_length_init(struct fuse_page_desc *descs,
  1134. unsigned int index,
  1135. unsigned int nr_pages)
  1136. {
  1137. int i;
  1138. for (i = index; i < index + nr_pages; i++)
  1139. descs[i].length = PAGE_SIZE - descs[i].offset;
  1140. }
  1141. static inline unsigned long fuse_get_user_addr(const struct iov_iter *ii)
  1142. {
  1143. return (unsigned long)ii->iov->iov_base + ii->iov_offset;
  1144. }
  1145. static inline size_t fuse_get_frag_size(const struct iov_iter *ii,
  1146. size_t max_size)
  1147. {
  1148. return min(iov_iter_single_seg_count(ii), max_size);
  1149. }
  1150. static int fuse_get_user_pages(struct fuse_args_pages *ap, struct iov_iter *ii,
  1151. size_t *nbytesp, int write,
  1152. unsigned int max_pages)
  1153. {
  1154. size_t nbytes = 0; /* # bytes already packed in req */
  1155. ssize_t ret = 0;
  1156. /* Special case for kernel I/O: can copy directly into the buffer */
  1157. if (iov_iter_is_kvec(ii)) {
  1158. unsigned long user_addr = fuse_get_user_addr(ii);
  1159. size_t frag_size = fuse_get_frag_size(ii, *nbytesp);
  1160. if (write)
  1161. ap->args.in_args[1].value = (void *) user_addr;
  1162. else
  1163. ap->args.out_args[0].value = (void *) user_addr;
  1164. iov_iter_advance(ii, frag_size);
  1165. *nbytesp = frag_size;
  1166. return 0;
  1167. }
  1168. while (nbytes < *nbytesp && ap->num_pages < max_pages) {
  1169. unsigned npages;
  1170. size_t start;
  1171. ret = iov_iter_get_pages(ii, &ap->pages[ap->num_pages],
  1172. *nbytesp - nbytes,
  1173. max_pages - ap->num_pages,
  1174. &start);
  1175. if (ret < 0)
  1176. break;
  1177. iov_iter_advance(ii, ret);
  1178. nbytes += ret;
  1179. ret += start;
  1180. npages = (ret + PAGE_SIZE - 1) / PAGE_SIZE;
  1181. ap->descs[ap->num_pages].offset = start;
  1182. fuse_page_descs_length_init(ap->descs, ap->num_pages, npages);
  1183. ap->num_pages += npages;
  1184. ap->descs[ap->num_pages - 1].length -=
  1185. (PAGE_SIZE - ret) & (PAGE_SIZE - 1);
  1186. }
  1187. ap->args.user_pages = true;
  1188. if (write)
  1189. ap->args.in_pages = true;
  1190. else
  1191. ap->args.out_pages = true;
  1192. *nbytesp = nbytes;
  1193. return ret < 0 ? ret : 0;
  1194. }
  1195. ssize_t fuse_direct_io(struct fuse_io_priv *io, struct iov_iter *iter,
  1196. loff_t *ppos, int flags)
  1197. {
  1198. int write = flags & FUSE_DIO_WRITE;
  1199. int cuse = flags & FUSE_DIO_CUSE;
  1200. struct file *file = io->iocb->ki_filp;
  1201. struct inode *inode = file->f_mapping->host;
  1202. struct fuse_file *ff = file->private_data;
  1203. struct fuse_conn *fc = ff->fm->fc;
  1204. size_t nmax = write ? fc->max_write : fc->max_read;
  1205. loff_t pos = *ppos;
  1206. size_t count = iov_iter_count(iter);
  1207. pgoff_t idx_from = pos >> PAGE_SHIFT;
  1208. pgoff_t idx_to = (pos + count - 1) >> PAGE_SHIFT;
  1209. ssize_t res = 0;
  1210. int err = 0;
  1211. struct fuse_io_args *ia;
  1212. unsigned int max_pages;
  1213. max_pages = iov_iter_npages(iter, fc->max_pages);
  1214. ia = fuse_io_alloc(io, max_pages);
  1215. if (!ia)
  1216. return -ENOMEM;
  1217. ia->io = io;
  1218. if (!cuse && fuse_range_is_writeback(inode, idx_from, idx_to)) {
  1219. if (!write)
  1220. inode_lock(inode);
  1221. fuse_sync_writes(inode);
  1222. if (!write)
  1223. inode_unlock(inode);
  1224. }
  1225. io->should_dirty = !write && iter_is_iovec(iter);
  1226. while (count) {
  1227. ssize_t nres;
  1228. fl_owner_t owner = current->files;
  1229. size_t nbytes = min(count, nmax);
  1230. err = fuse_get_user_pages(&ia->ap, iter, &nbytes, write,
  1231. max_pages);
  1232. if (err && !nbytes)
  1233. break;
  1234. if (write) {
  1235. if (!capable(CAP_FSETID))
  1236. ia->write.in.write_flags |= FUSE_WRITE_KILL_PRIV;
  1237. nres = fuse_send_write(ia, pos, nbytes, owner);
  1238. } else {
  1239. nres = fuse_send_read(ia, pos, nbytes, owner);
  1240. }
  1241. if (!io->async || nres < 0) {
  1242. fuse_release_user_pages(&ia->ap, io->should_dirty);
  1243. fuse_io_free(ia);
  1244. }
  1245. ia = NULL;
  1246. if (nres < 0) {
  1247. iov_iter_revert(iter, nbytes);
  1248. err = nres;
  1249. break;
  1250. }
  1251. WARN_ON(nres > nbytes);
  1252. count -= nres;
  1253. res += nres;
  1254. pos += nres;
  1255. if (nres != nbytes) {
  1256. iov_iter_revert(iter, nbytes - nres);
  1257. break;
  1258. }
  1259. if (count) {
  1260. max_pages = iov_iter_npages(iter, fc->max_pages);
  1261. ia = fuse_io_alloc(io, max_pages);
  1262. if (!ia)
  1263. break;
  1264. }
  1265. }
  1266. if (ia)
  1267. fuse_io_free(ia);
  1268. if (res > 0)
  1269. *ppos = pos;
  1270. return res > 0 ? res : err;
  1271. }
  1272. EXPORT_SYMBOL_GPL(fuse_direct_io);
  1273. static ssize_t __fuse_direct_read(struct fuse_io_priv *io,
  1274. struct iov_iter *iter,
  1275. loff_t *ppos)
  1276. {
  1277. ssize_t res;
  1278. struct inode *inode = file_inode(io->iocb->ki_filp);
  1279. res = fuse_direct_io(io, iter, ppos, 0);
  1280. fuse_invalidate_atime(inode);
  1281. return res;
  1282. }
  1283. static ssize_t fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter);
  1284. static ssize_t fuse_direct_read_iter(struct kiocb *iocb, struct iov_iter *to)
  1285. {
  1286. ssize_t res;
  1287. if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
  1288. res = fuse_direct_IO(iocb, to);
  1289. } else {
  1290. struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
  1291. res = __fuse_direct_read(&io, to, &iocb->ki_pos);
  1292. }
  1293. return res;
  1294. }
  1295. static ssize_t fuse_direct_write_iter(struct kiocb *iocb, struct iov_iter *from)
  1296. {
  1297. struct inode *inode = file_inode(iocb->ki_filp);
  1298. struct fuse_io_priv io = FUSE_IO_PRIV_SYNC(iocb);
  1299. ssize_t res;
  1300. /* Don't allow parallel writes to the same file */
  1301. inode_lock(inode);
  1302. res = generic_write_checks(iocb, from);
  1303. if (res > 0) {
  1304. if (!is_sync_kiocb(iocb) && iocb->ki_flags & IOCB_DIRECT) {
  1305. res = fuse_direct_IO(iocb, from);
  1306. } else {
  1307. res = fuse_direct_io(&io, from, &iocb->ki_pos,
  1308. FUSE_DIO_WRITE);
  1309. }
  1310. }
  1311. fuse_invalidate_attr(inode);
  1312. if (res > 0)
  1313. fuse_write_update_size(inode, iocb->ki_pos);
  1314. inode_unlock(inode);
  1315. return res;
  1316. }
  1317. static ssize_t fuse_file_read_iter(struct kiocb *iocb, struct iov_iter *to)
  1318. {
  1319. struct file *file = iocb->ki_filp;
  1320. struct fuse_file *ff = file->private_data;
  1321. struct inode *inode = file_inode(file);
  1322. if (fuse_is_bad(inode))
  1323. return -EIO;
  1324. if (FUSE_IS_DAX(inode))
  1325. return fuse_dax_read_iter(iocb, to);
  1326. if (ff->passthrough.filp)
  1327. return fuse_passthrough_read_iter(iocb, to);
  1328. else if (!(ff->open_flags & FOPEN_DIRECT_IO))
  1329. return fuse_cache_read_iter(iocb, to);
  1330. else
  1331. return fuse_direct_read_iter(iocb, to);
  1332. }
  1333. static ssize_t fuse_file_write_iter(struct kiocb *iocb, struct iov_iter *from)
  1334. {
  1335. struct file *file = iocb->ki_filp;
  1336. struct fuse_file *ff = file->private_data;
  1337. struct inode *inode = file_inode(file);
  1338. if (fuse_is_bad(inode))
  1339. return -EIO;
  1340. if (FUSE_IS_DAX(inode))
  1341. return fuse_dax_write_iter(iocb, from);
  1342. if (ff->passthrough.filp)
  1343. return fuse_passthrough_write_iter(iocb, from);
  1344. else if (!(ff->open_flags & FOPEN_DIRECT_IO))
  1345. return fuse_cache_write_iter(iocb, from);
  1346. else
  1347. return fuse_direct_write_iter(iocb, from);
  1348. }
  1349. static void fuse_writepage_free(struct fuse_writepage_args *wpa)
  1350. {
  1351. struct fuse_args_pages *ap = &wpa->ia.ap;
  1352. int i;
  1353. for (i = 0; i < ap->num_pages; i++)
  1354. __free_page(ap->pages[i]);
  1355. if (wpa->ia.ff)
  1356. fuse_file_put(wpa->ia.ff, false, false);
  1357. kfree(ap->pages);
  1358. kfree(wpa);
  1359. }
  1360. static void fuse_writepage_finish(struct fuse_mount *fm,
  1361. struct fuse_writepage_args *wpa)
  1362. {
  1363. struct fuse_args_pages *ap = &wpa->ia.ap;
  1364. struct inode *inode = wpa->inode;
  1365. struct fuse_inode *fi = get_fuse_inode(inode);
  1366. struct backing_dev_info *bdi = inode_to_bdi(inode);
  1367. int i;
  1368. for (i = 0; i < ap->num_pages; i++) {
  1369. dec_wb_stat(&bdi->wb, WB_WRITEBACK);
  1370. dec_node_page_state(ap->pages[i], NR_WRITEBACK_TEMP);
  1371. wb_writeout_inc(&bdi->wb);
  1372. }
  1373. wake_up(&fi->page_waitq);
  1374. }
  1375. /* Called under fi->lock, may release and reacquire it */
  1376. static void fuse_send_writepage(struct fuse_mount *fm,
  1377. struct fuse_writepage_args *wpa, loff_t size)
  1378. __releases(fi->lock)
  1379. __acquires(fi->lock)
  1380. {
  1381. struct fuse_writepage_args *aux, *next;
  1382. struct fuse_inode *fi = get_fuse_inode(wpa->inode);
  1383. struct fuse_write_in *inarg = &wpa->ia.write.in;
  1384. struct fuse_args *args = &wpa->ia.ap.args;
  1385. __u64 data_size = wpa->ia.ap.num_pages * PAGE_SIZE;
  1386. int err;
  1387. fi->writectr++;
  1388. if (inarg->offset + data_size <= size) {
  1389. inarg->size = data_size;
  1390. } else if (inarg->offset < size) {
  1391. inarg->size = size - inarg->offset;
  1392. } else {
  1393. /* Got truncated off completely */
  1394. goto out_free;
  1395. }
  1396. args->in_args[1].size = inarg->size;
  1397. args->force = true;
  1398. args->nocreds = true;
  1399. err = fuse_simple_background(fm, args, GFP_ATOMIC);
  1400. if (err == -ENOMEM) {
  1401. spin_unlock(&fi->lock);
  1402. err = fuse_simple_background(fm, args, GFP_NOFS | __GFP_NOFAIL);
  1403. spin_lock(&fi->lock);
  1404. }
  1405. /* Fails on broken connection only */
  1406. if (unlikely(err))
  1407. goto out_free;
  1408. return;
  1409. out_free:
  1410. fi->writectr--;
  1411. rb_erase(&wpa->writepages_entry, &fi->writepages);
  1412. fuse_writepage_finish(fm, wpa);
  1413. spin_unlock(&fi->lock);
  1414. /* After fuse_writepage_finish() aux request list is private */
  1415. for (aux = wpa->next; aux; aux = next) {
  1416. next = aux->next;
  1417. aux->next = NULL;
  1418. fuse_writepage_free(aux);
  1419. }
  1420. fuse_writepage_free(wpa);
  1421. spin_lock(&fi->lock);
  1422. }
  1423. /*
  1424. * If fi->writectr is positive (no truncate or fsync going on) send
  1425. * all queued writepage requests.
  1426. *
  1427. * Called with fi->lock
  1428. */
  1429. void fuse_flush_writepages(struct inode *inode)
  1430. __releases(fi->lock)
  1431. __acquires(fi->lock)
  1432. {
  1433. struct fuse_mount *fm = get_fuse_mount(inode);
  1434. struct fuse_inode *fi = get_fuse_inode(inode);
  1435. loff_t crop = i_size_read(inode);
  1436. struct fuse_writepage_args *wpa;
  1437. while (fi->writectr >= 0 && !list_empty(&fi->queued_writes)) {
  1438. wpa = list_entry(fi->queued_writes.next,
  1439. struct fuse_writepage_args, queue_entry);
  1440. list_del_init(&wpa->queue_entry);
  1441. fuse_send_writepage(fm, wpa, crop);
  1442. }
  1443. }
  1444. static struct fuse_writepage_args *fuse_insert_writeback(struct rb_root *root,
  1445. struct fuse_writepage_args *wpa)
  1446. {
  1447. pgoff_t idx_from = wpa->ia.write.in.offset >> PAGE_SHIFT;
  1448. pgoff_t idx_to = idx_from + wpa->ia.ap.num_pages - 1;
  1449. struct rb_node **p = &root->rb_node;
  1450. struct rb_node *parent = NULL;
  1451. WARN_ON(!wpa->ia.ap.num_pages);
  1452. while (*p) {
  1453. struct fuse_writepage_args *curr;
  1454. pgoff_t curr_index;
  1455. parent = *p;
  1456. curr = rb_entry(parent, struct fuse_writepage_args,
  1457. writepages_entry);
  1458. WARN_ON(curr->inode != wpa->inode);
  1459. curr_index = curr->ia.write.in.offset >> PAGE_SHIFT;
  1460. if (idx_from >= curr_index + curr->ia.ap.num_pages)
  1461. p = &(*p)->rb_right;
  1462. else if (idx_to < curr_index)
  1463. p = &(*p)->rb_left;
  1464. else
  1465. return curr;
  1466. }
  1467. rb_link_node(&wpa->writepages_entry, parent, p);
  1468. rb_insert_color(&wpa->writepages_entry, root);
  1469. return NULL;
  1470. }
  1471. static void tree_insert(struct rb_root *root, struct fuse_writepage_args *wpa)
  1472. {
  1473. WARN_ON(fuse_insert_writeback(root, wpa));
  1474. }
  1475. static void fuse_writepage_end(struct fuse_mount *fm, struct fuse_args *args,
  1476. int error)
  1477. {
  1478. struct fuse_writepage_args *wpa =
  1479. container_of(args, typeof(*wpa), ia.ap.args);
  1480. struct inode *inode = wpa->inode;
  1481. struct fuse_inode *fi = get_fuse_inode(inode);
  1482. struct fuse_conn *fc = get_fuse_conn(inode);
  1483. mapping_set_error(inode->i_mapping, error);
  1484. /*
  1485. * A writeback finished and this might have updated mtime/ctime on
  1486. * server making local mtime/ctime stale. Hence invalidate attrs.
  1487. * Do this only if writeback_cache is not enabled. If writeback_cache
  1488. * is enabled, we trust local ctime/mtime.
  1489. */
  1490. if (!fc->writeback_cache)
  1491. fuse_invalidate_attr(inode);
  1492. spin_lock(&fi->lock);
  1493. rb_erase(&wpa->writepages_entry, &fi->writepages);
  1494. while (wpa->next) {
  1495. struct fuse_mount *fm = get_fuse_mount(inode);
  1496. struct fuse_write_in *inarg = &wpa->ia.write.in;
  1497. struct fuse_writepage_args *next = wpa->next;
  1498. wpa->next = next->next;
  1499. next->next = NULL;
  1500. next->ia.ff = fuse_file_get(wpa->ia.ff);
  1501. tree_insert(&fi->writepages, next);
  1502. /*
  1503. * Skip fuse_flush_writepages() to make it easy to crop requests
  1504. * based on primary request size.
  1505. *
  1506. * 1st case (trivial): there are no concurrent activities using
  1507. * fuse_set/release_nowrite. Then we're on safe side because
  1508. * fuse_flush_writepages() would call fuse_send_writepage()
  1509. * anyway.
  1510. *
  1511. * 2nd case: someone called fuse_set_nowrite and it is waiting
  1512. * now for completion of all in-flight requests. This happens
  1513. * rarely and no more than once per page, so this should be
  1514. * okay.
  1515. *
  1516. * 3rd case: someone (e.g. fuse_do_setattr()) is in the middle
  1517. * of fuse_set_nowrite..fuse_release_nowrite section. The fact
  1518. * that fuse_set_nowrite returned implies that all in-flight
  1519. * requests were completed along with all of their secondary
  1520. * requests. Further primary requests are blocked by negative
  1521. * writectr. Hence there cannot be any in-flight requests and
  1522. * no invocations of fuse_writepage_end() while we're in
  1523. * fuse_set_nowrite..fuse_release_nowrite section.
  1524. */
  1525. fuse_send_writepage(fm, next, inarg->offset + inarg->size);
  1526. }
  1527. fi->writectr--;
  1528. fuse_writepage_finish(fm, wpa);
  1529. spin_unlock(&fi->lock);
  1530. fuse_writepage_free(wpa);
  1531. }
  1532. static struct fuse_file *__fuse_write_file_get(struct fuse_conn *fc,
  1533. struct fuse_inode *fi)
  1534. {
  1535. struct fuse_file *ff = NULL;
  1536. spin_lock(&fi->lock);
  1537. if (!list_empty(&fi->write_files)) {
  1538. ff = list_entry(fi->write_files.next, struct fuse_file,
  1539. write_entry);
  1540. fuse_file_get(ff);
  1541. }
  1542. spin_unlock(&fi->lock);
  1543. return ff;
  1544. }
  1545. static struct fuse_file *fuse_write_file_get(struct fuse_conn *fc,
  1546. struct fuse_inode *fi)
  1547. {
  1548. struct fuse_file *ff = __fuse_write_file_get(fc, fi);
  1549. WARN_ON(!ff);
  1550. return ff;
  1551. }
  1552. int fuse_write_inode(struct inode *inode, struct writeback_control *wbc)
  1553. {
  1554. struct fuse_conn *fc = get_fuse_conn(inode);
  1555. struct fuse_inode *fi = get_fuse_inode(inode);
  1556. struct fuse_file *ff;
  1557. int err;
  1558. /*
  1559. * Inode is always written before the last reference is dropped and
  1560. * hence this should not be reached from reclaim.
  1561. *
  1562. * Writing back the inode from reclaim can deadlock if the request
  1563. * processing itself needs an allocation. Allocations triggering
  1564. * reclaim while serving a request can't be prevented, because it can
  1565. * involve any number of unrelated userspace processes.
  1566. */
  1567. WARN_ON(wbc->for_reclaim);
  1568. ff = __fuse_write_file_get(fc, fi);
  1569. err = fuse_flush_times(inode, ff);
  1570. if (ff)
  1571. fuse_file_put(ff, false, false);
  1572. return err;
  1573. }
  1574. static struct fuse_writepage_args *fuse_writepage_args_alloc(void)
  1575. {
  1576. struct fuse_writepage_args *wpa;
  1577. struct fuse_args_pages *ap;
  1578. wpa = kzalloc(sizeof(*wpa), GFP_NOFS);
  1579. if (wpa) {
  1580. ap = &wpa->ia.ap;
  1581. ap->num_pages = 0;
  1582. ap->pages = fuse_pages_alloc(1, GFP_NOFS, &ap->descs);
  1583. if (!ap->pages) {
  1584. kfree(wpa);
  1585. wpa = NULL;
  1586. }
  1587. }
  1588. return wpa;
  1589. }
  1590. static int fuse_writepage_locked(struct page *page)
  1591. {
  1592. struct address_space *mapping = page->mapping;
  1593. struct inode *inode = mapping->host;
  1594. struct fuse_conn *fc = get_fuse_conn(inode);
  1595. struct fuse_inode *fi = get_fuse_inode(inode);
  1596. struct fuse_writepage_args *wpa;
  1597. struct fuse_args_pages *ap;
  1598. struct page *tmp_page;
  1599. int error = -ENOMEM;
  1600. set_page_writeback(page);
  1601. wpa = fuse_writepage_args_alloc();
  1602. if (!wpa)
  1603. goto err;
  1604. ap = &wpa->ia.ap;
  1605. tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  1606. if (!tmp_page)
  1607. goto err_free;
  1608. error = -EIO;
  1609. wpa->ia.ff = fuse_write_file_get(fc, fi);
  1610. if (!wpa->ia.ff)
  1611. goto err_nofile;
  1612. fuse_write_args_fill(&wpa->ia, wpa->ia.ff, page_offset(page), 0);
  1613. copy_highpage(tmp_page, page);
  1614. wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
  1615. wpa->next = NULL;
  1616. ap->args.in_pages = true;
  1617. ap->num_pages = 1;
  1618. ap->pages[0] = tmp_page;
  1619. ap->descs[0].offset = 0;
  1620. ap->descs[0].length = PAGE_SIZE;
  1621. ap->args.end = fuse_writepage_end;
  1622. wpa->inode = inode;
  1623. inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
  1624. inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
  1625. spin_lock(&fi->lock);
  1626. tree_insert(&fi->writepages, wpa);
  1627. list_add_tail(&wpa->queue_entry, &fi->queued_writes);
  1628. fuse_flush_writepages(inode);
  1629. spin_unlock(&fi->lock);
  1630. end_page_writeback(page);
  1631. return 0;
  1632. err_nofile:
  1633. __free_page(tmp_page);
  1634. err_free:
  1635. kfree(wpa);
  1636. err:
  1637. mapping_set_error(page->mapping, error);
  1638. end_page_writeback(page);
  1639. return error;
  1640. }
  1641. static int fuse_writepage(struct page *page, struct writeback_control *wbc)
  1642. {
  1643. int err;
  1644. if (fuse_page_is_writeback(page->mapping->host, page->index)) {
  1645. /*
  1646. * ->writepages() should be called for sync() and friends. We
  1647. * should only get here on direct reclaim and then we are
  1648. * allowed to skip a page which is already in flight
  1649. */
  1650. WARN_ON(wbc->sync_mode == WB_SYNC_ALL);
  1651. redirty_page_for_writepage(wbc, page);
  1652. unlock_page(page);
  1653. return 0;
  1654. }
  1655. err = fuse_writepage_locked(page);
  1656. unlock_page(page);
  1657. return err;
  1658. }
  1659. struct fuse_fill_wb_data {
  1660. struct fuse_writepage_args *wpa;
  1661. struct fuse_file *ff;
  1662. struct inode *inode;
  1663. struct page **orig_pages;
  1664. unsigned int max_pages;
  1665. };
  1666. static bool fuse_pages_realloc(struct fuse_fill_wb_data *data)
  1667. {
  1668. struct fuse_args_pages *ap = &data->wpa->ia.ap;
  1669. struct fuse_conn *fc = get_fuse_conn(data->inode);
  1670. struct page **pages;
  1671. struct fuse_page_desc *descs;
  1672. unsigned int npages = min_t(unsigned int,
  1673. max_t(unsigned int, data->max_pages * 2,
  1674. FUSE_DEFAULT_MAX_PAGES_PER_REQ),
  1675. fc->max_pages);
  1676. WARN_ON(npages <= data->max_pages);
  1677. pages = fuse_pages_alloc(npages, GFP_NOFS, &descs);
  1678. if (!pages)
  1679. return false;
  1680. memcpy(pages, ap->pages, sizeof(struct page *) * ap->num_pages);
  1681. memcpy(descs, ap->descs, sizeof(struct fuse_page_desc) * ap->num_pages);
  1682. kfree(ap->pages);
  1683. ap->pages = pages;
  1684. ap->descs = descs;
  1685. data->max_pages = npages;
  1686. return true;
  1687. }
  1688. static void fuse_writepages_send(struct fuse_fill_wb_data *data)
  1689. {
  1690. struct fuse_writepage_args *wpa = data->wpa;
  1691. struct inode *inode = data->inode;
  1692. struct fuse_inode *fi = get_fuse_inode(inode);
  1693. int num_pages = wpa->ia.ap.num_pages;
  1694. int i;
  1695. wpa->ia.ff = fuse_file_get(data->ff);
  1696. spin_lock(&fi->lock);
  1697. list_add_tail(&wpa->queue_entry, &fi->queued_writes);
  1698. fuse_flush_writepages(inode);
  1699. spin_unlock(&fi->lock);
  1700. for (i = 0; i < num_pages; i++)
  1701. end_page_writeback(data->orig_pages[i]);
  1702. }
  1703. /*
  1704. * Check under fi->lock if the page is under writeback, and insert it onto the
  1705. * rb_tree if not. Otherwise iterate auxiliary write requests, to see if there's
  1706. * one already added for a page at this offset. If there's none, then insert
  1707. * this new request onto the auxiliary list, otherwise reuse the existing one by
  1708. * swapping the new temp page with the old one.
  1709. */
  1710. static bool fuse_writepage_add(struct fuse_writepage_args *new_wpa,
  1711. struct page *page)
  1712. {
  1713. struct fuse_inode *fi = get_fuse_inode(new_wpa->inode);
  1714. struct fuse_writepage_args *tmp;
  1715. struct fuse_writepage_args *old_wpa;
  1716. struct fuse_args_pages *new_ap = &new_wpa->ia.ap;
  1717. WARN_ON(new_ap->num_pages != 0);
  1718. new_ap->num_pages = 1;
  1719. spin_lock(&fi->lock);
  1720. old_wpa = fuse_insert_writeback(&fi->writepages, new_wpa);
  1721. if (!old_wpa) {
  1722. spin_unlock(&fi->lock);
  1723. return true;
  1724. }
  1725. for (tmp = old_wpa->next; tmp; tmp = tmp->next) {
  1726. pgoff_t curr_index;
  1727. WARN_ON(tmp->inode != new_wpa->inode);
  1728. curr_index = tmp->ia.write.in.offset >> PAGE_SHIFT;
  1729. if (curr_index == page->index) {
  1730. WARN_ON(tmp->ia.ap.num_pages != 1);
  1731. swap(tmp->ia.ap.pages[0], new_ap->pages[0]);
  1732. break;
  1733. }
  1734. }
  1735. if (!tmp) {
  1736. new_wpa->next = old_wpa->next;
  1737. old_wpa->next = new_wpa;
  1738. }
  1739. spin_unlock(&fi->lock);
  1740. if (tmp) {
  1741. struct backing_dev_info *bdi = inode_to_bdi(new_wpa->inode);
  1742. dec_wb_stat(&bdi->wb, WB_WRITEBACK);
  1743. dec_node_page_state(new_ap->pages[0], NR_WRITEBACK_TEMP);
  1744. wb_writeout_inc(&bdi->wb);
  1745. fuse_writepage_free(new_wpa);
  1746. }
  1747. return false;
  1748. }
  1749. static bool fuse_writepage_need_send(struct fuse_conn *fc, struct page *page,
  1750. struct fuse_args_pages *ap,
  1751. struct fuse_fill_wb_data *data)
  1752. {
  1753. WARN_ON(!ap->num_pages);
  1754. /*
  1755. * Being under writeback is unlikely but possible. For example direct
  1756. * read to an mmaped fuse file will set the page dirty twice; once when
  1757. * the pages are faulted with get_user_pages(), and then after the read
  1758. * completed.
  1759. */
  1760. if (fuse_page_is_writeback(data->inode, page->index))
  1761. return true;
  1762. /* Reached max pages */
  1763. if (ap->num_pages == fc->max_pages)
  1764. return true;
  1765. /* Reached max write bytes */
  1766. if ((ap->num_pages + 1) * PAGE_SIZE > fc->max_write)
  1767. return true;
  1768. /* Discontinuity */
  1769. if (data->orig_pages[ap->num_pages - 1]->index + 1 != page->index)
  1770. return true;
  1771. /* Need to grow the pages array? If so, did the expansion fail? */
  1772. if (ap->num_pages == data->max_pages && !fuse_pages_realloc(data))
  1773. return true;
  1774. return false;
  1775. }
  1776. static int fuse_writepages_fill(struct page *page,
  1777. struct writeback_control *wbc, void *_data)
  1778. {
  1779. struct fuse_fill_wb_data *data = _data;
  1780. struct fuse_writepage_args *wpa = data->wpa;
  1781. struct fuse_args_pages *ap = &wpa->ia.ap;
  1782. struct inode *inode = data->inode;
  1783. struct fuse_inode *fi = get_fuse_inode(inode);
  1784. struct fuse_conn *fc = get_fuse_conn(inode);
  1785. struct page *tmp_page;
  1786. int err;
  1787. if (!data->ff) {
  1788. err = -EIO;
  1789. data->ff = fuse_write_file_get(fc, fi);
  1790. if (!data->ff)
  1791. goto out_unlock;
  1792. }
  1793. if (wpa && fuse_writepage_need_send(fc, page, ap, data)) {
  1794. fuse_writepages_send(data);
  1795. data->wpa = NULL;
  1796. }
  1797. err = -ENOMEM;
  1798. tmp_page = alloc_page(GFP_NOFS | __GFP_HIGHMEM);
  1799. if (!tmp_page)
  1800. goto out_unlock;
  1801. /*
  1802. * The page must not be redirtied until the writeout is completed
  1803. * (i.e. userspace has sent a reply to the write request). Otherwise
  1804. * there could be more than one temporary page instance for each real
  1805. * page.
  1806. *
  1807. * This is ensured by holding the page lock in page_mkwrite() while
  1808. * checking fuse_page_is_writeback(). We already hold the page lock
  1809. * since clear_page_dirty_for_io() and keep it held until we add the
  1810. * request to the fi->writepages list and increment ap->num_pages.
  1811. * After this fuse_page_is_writeback() will indicate that the page is
  1812. * under writeback, so we can release the page lock.
  1813. */
  1814. if (data->wpa == NULL) {
  1815. err = -ENOMEM;
  1816. wpa = fuse_writepage_args_alloc();
  1817. if (!wpa) {
  1818. __free_page(tmp_page);
  1819. goto out_unlock;
  1820. }
  1821. data->max_pages = 1;
  1822. ap = &wpa->ia.ap;
  1823. fuse_write_args_fill(&wpa->ia, data->ff, page_offset(page), 0);
  1824. wpa->ia.write.in.write_flags |= FUSE_WRITE_CACHE;
  1825. wpa->next = NULL;
  1826. ap->args.in_pages = true;
  1827. ap->args.end = fuse_writepage_end;
  1828. ap->num_pages = 0;
  1829. wpa->inode = inode;
  1830. }
  1831. set_page_writeback(page);
  1832. copy_highpage(tmp_page, page);
  1833. ap->pages[ap->num_pages] = tmp_page;
  1834. ap->descs[ap->num_pages].offset = 0;
  1835. ap->descs[ap->num_pages].length = PAGE_SIZE;
  1836. data->orig_pages[ap->num_pages] = page;
  1837. inc_wb_stat(&inode_to_bdi(inode)->wb, WB_WRITEBACK);
  1838. inc_node_page_state(tmp_page, NR_WRITEBACK_TEMP);
  1839. err = 0;
  1840. if (data->wpa) {
  1841. /*
  1842. * Protected by fi->lock against concurrent access by
  1843. * fuse_page_is_writeback().
  1844. */
  1845. spin_lock(&fi->lock);
  1846. ap->num_pages++;
  1847. spin_unlock(&fi->lock);
  1848. } else if (fuse_writepage_add(wpa, page)) {
  1849. data->wpa = wpa;
  1850. } else {
  1851. end_page_writeback(page);
  1852. }
  1853. out_unlock:
  1854. unlock_page(page);
  1855. return err;
  1856. }
  1857. static int fuse_writepages(struct address_space *mapping,
  1858. struct writeback_control *wbc)
  1859. {
  1860. struct inode *inode = mapping->host;
  1861. struct fuse_conn *fc = get_fuse_conn(inode);
  1862. struct fuse_fill_wb_data data;
  1863. int err;
  1864. err = -EIO;
  1865. if (fuse_is_bad(inode))
  1866. goto out;
  1867. data.inode = inode;
  1868. data.wpa = NULL;
  1869. data.ff = NULL;
  1870. err = -ENOMEM;
  1871. data.orig_pages = kcalloc(fc->max_pages,
  1872. sizeof(struct page *),
  1873. GFP_NOFS);
  1874. if (!data.orig_pages)
  1875. goto out;
  1876. err = write_cache_pages(mapping, wbc, fuse_writepages_fill, &data);
  1877. if (data.wpa) {
  1878. WARN_ON(!data.wpa->ia.ap.num_pages);
  1879. fuse_writepages_send(&data);
  1880. }
  1881. if (data.ff)
  1882. fuse_file_put(data.ff, false, false);
  1883. kfree(data.orig_pages);
  1884. out:
  1885. return err;
  1886. }
  1887. /*
  1888. * It's worthy to make sure that space is reserved on disk for the write,
  1889. * but how to implement it without killing performance need more thinking.
  1890. */
  1891. static int fuse_write_begin(struct file *file, struct address_space *mapping,
  1892. loff_t pos, unsigned len, unsigned flags,
  1893. struct page **pagep, void **fsdata)
  1894. {
  1895. pgoff_t index = pos >> PAGE_SHIFT;
  1896. struct fuse_conn *fc = get_fuse_conn(file_inode(file));
  1897. struct page *page;
  1898. loff_t fsize;
  1899. int err = -ENOMEM;
  1900. WARN_ON(!fc->writeback_cache);
  1901. page = grab_cache_page_write_begin(mapping, index, flags);
  1902. if (!page)
  1903. goto error;
  1904. fuse_wait_on_page_writeback(mapping->host, page->index);
  1905. if (PageUptodate(page) || len == PAGE_SIZE)
  1906. goto success;
  1907. /*
  1908. * Check if the start this page comes after the end of file, in which
  1909. * case the readpage can be optimized away.
  1910. */
  1911. fsize = i_size_read(mapping->host);
  1912. if (fsize <= (pos & PAGE_MASK)) {
  1913. size_t off = pos & ~PAGE_MASK;
  1914. if (off)
  1915. zero_user_segment(page, 0, off);
  1916. goto success;
  1917. }
  1918. err = fuse_do_readpage(file, page);
  1919. if (err)
  1920. goto cleanup;
  1921. success:
  1922. *pagep = page;
  1923. return 0;
  1924. cleanup:
  1925. unlock_page(page);
  1926. put_page(page);
  1927. error:
  1928. return err;
  1929. }
  1930. static int fuse_write_end(struct file *file, struct address_space *mapping,
  1931. loff_t pos, unsigned len, unsigned copied,
  1932. struct page *page, void *fsdata)
  1933. {
  1934. struct inode *inode = page->mapping->host;
  1935. /* Haven't copied anything? Skip zeroing, size extending, dirtying. */
  1936. if (!copied)
  1937. goto unlock;
  1938. if (!PageUptodate(page)) {
  1939. /* Zero any unwritten bytes at the end of the page */
  1940. size_t endoff = (pos + copied) & ~PAGE_MASK;
  1941. if (endoff)
  1942. zero_user_segment(page, endoff, PAGE_SIZE);
  1943. SetPageUptodate(page);
  1944. }
  1945. fuse_write_update_size(inode, pos + copied);
  1946. set_page_dirty(page);
  1947. unlock:
  1948. unlock_page(page);
  1949. put_page(page);
  1950. return copied;
  1951. }
  1952. static int fuse_launder_page(struct page *page)
  1953. {
  1954. int err = 0;
  1955. if (clear_page_dirty_for_io(page)) {
  1956. struct inode *inode = page->mapping->host;
  1957. err = fuse_writepage_locked(page);
  1958. if (!err)
  1959. fuse_wait_on_page_writeback(inode, page->index);
  1960. }
  1961. return err;
  1962. }
  1963. /*
  1964. * Write back dirty pages now, because there may not be any suitable
  1965. * open files later
  1966. */
  1967. static void fuse_vma_close(struct vm_area_struct *vma)
  1968. {
  1969. filemap_write_and_wait(vma->vm_file->f_mapping);
  1970. }
  1971. /*
  1972. * Wait for writeback against this page to complete before allowing it
  1973. * to be marked dirty again, and hence written back again, possibly
  1974. * before the previous writepage completed.
  1975. *
  1976. * Block here, instead of in ->writepage(), so that the userspace fs
  1977. * can only block processes actually operating on the filesystem.
  1978. *
  1979. * Otherwise unprivileged userspace fs would be able to block
  1980. * unrelated:
  1981. *
  1982. * - page migration
  1983. * - sync(2)
  1984. * - try_to_free_pages() with order > PAGE_ALLOC_COSTLY_ORDER
  1985. */
  1986. static vm_fault_t fuse_page_mkwrite(struct vm_fault *vmf)
  1987. {
  1988. struct page *page = vmf->page;
  1989. struct inode *inode = file_inode(vmf->vma->vm_file);
  1990. file_update_time(vmf->vma->vm_file);
  1991. lock_page(page);
  1992. if (page->mapping != inode->i_mapping) {
  1993. unlock_page(page);
  1994. return VM_FAULT_NOPAGE;
  1995. }
  1996. fuse_wait_on_page_writeback(inode, page->index);
  1997. return VM_FAULT_LOCKED;
  1998. }
  1999. static const struct vm_operations_struct fuse_file_vm_ops = {
  2000. .close = fuse_vma_close,
  2001. .fault = filemap_fault,
  2002. .map_pages = filemap_map_pages,
  2003. .page_mkwrite = fuse_page_mkwrite,
  2004. };
  2005. static int fuse_file_mmap(struct file *file, struct vm_area_struct *vma)
  2006. {
  2007. struct fuse_file *ff = file->private_data;
  2008. /* DAX mmap is superior to direct_io mmap */
  2009. if (FUSE_IS_DAX(file_inode(file)))
  2010. return fuse_dax_mmap(file, vma);
  2011. if (ff->passthrough.filp)
  2012. return fuse_passthrough_mmap(file, vma);
  2013. if (ff->open_flags & FOPEN_DIRECT_IO) {
  2014. /* Can't provide the coherency needed for MAP_SHARED */
  2015. if (vma->vm_flags & VM_MAYSHARE)
  2016. return -ENODEV;
  2017. invalidate_inode_pages2(file->f_mapping);
  2018. return generic_file_mmap(file, vma);
  2019. }
  2020. if ((vma->vm_flags & VM_SHARED) && (vma->vm_flags & VM_MAYWRITE))
  2021. fuse_link_write_file(file);
  2022. file_accessed(file);
  2023. vma->vm_ops = &fuse_file_vm_ops;
  2024. return 0;
  2025. }
  2026. static int convert_fuse_file_lock(struct fuse_conn *fc,
  2027. const struct fuse_file_lock *ffl,
  2028. struct file_lock *fl)
  2029. {
  2030. switch (ffl->type) {
  2031. case F_UNLCK:
  2032. break;
  2033. case F_RDLCK:
  2034. case F_WRLCK:
  2035. if (ffl->start > OFFSET_MAX || ffl->end > OFFSET_MAX ||
  2036. ffl->end < ffl->start)
  2037. return -EIO;
  2038. fl->fl_start = ffl->start;
  2039. fl->fl_end = ffl->end;
  2040. /*
  2041. * Convert pid into init's pid namespace. The locks API will
  2042. * translate it into the caller's pid namespace.
  2043. */
  2044. rcu_read_lock();
  2045. fl->fl_pid = pid_nr_ns(find_pid_ns(ffl->pid, fc->pid_ns), &init_pid_ns);
  2046. rcu_read_unlock();
  2047. break;
  2048. default:
  2049. return -EIO;
  2050. }
  2051. fl->fl_type = ffl->type;
  2052. return 0;
  2053. }
  2054. static void fuse_lk_fill(struct fuse_args *args, struct file *file,
  2055. const struct file_lock *fl, int opcode, pid_t pid,
  2056. int flock, struct fuse_lk_in *inarg)
  2057. {
  2058. struct inode *inode = file_inode(file);
  2059. struct fuse_conn *fc = get_fuse_conn(inode);
  2060. struct fuse_file *ff = file->private_data;
  2061. memset(inarg, 0, sizeof(*inarg));
  2062. inarg->fh = ff->fh;
  2063. inarg->owner = fuse_lock_owner_id(fc, fl->fl_owner);
  2064. inarg->lk.start = fl->fl_start;
  2065. inarg->lk.end = fl->fl_end;
  2066. inarg->lk.type = fl->fl_type;
  2067. inarg->lk.pid = pid;
  2068. if (flock)
  2069. inarg->lk_flags |= FUSE_LK_FLOCK;
  2070. args->opcode = opcode;
  2071. args->nodeid = get_node_id(inode);
  2072. args->in_numargs = 1;
  2073. args->in_args[0].size = sizeof(*inarg);
  2074. args->in_args[0].value = inarg;
  2075. }
  2076. static int fuse_getlk(struct file *file, struct file_lock *fl)
  2077. {
  2078. struct inode *inode = file_inode(file);
  2079. struct fuse_mount *fm = get_fuse_mount(inode);
  2080. FUSE_ARGS(args);
  2081. struct fuse_lk_in inarg;
  2082. struct fuse_lk_out outarg;
  2083. int err;
  2084. fuse_lk_fill(&args, file, fl, FUSE_GETLK, 0, 0, &inarg);
  2085. args.out_numargs = 1;
  2086. args.out_args[0].size = sizeof(outarg);
  2087. args.out_args[0].value = &outarg;
  2088. err = fuse_simple_request(fm, &args);
  2089. if (!err)
  2090. err = convert_fuse_file_lock(fm->fc, &outarg.lk, fl);
  2091. return err;
  2092. }
  2093. static int fuse_setlk(struct file *file, struct file_lock *fl, int flock)
  2094. {
  2095. struct inode *inode = file_inode(file);
  2096. struct fuse_mount *fm = get_fuse_mount(inode);
  2097. FUSE_ARGS(args);
  2098. struct fuse_lk_in inarg;
  2099. int opcode = (fl->fl_flags & FL_SLEEP) ? FUSE_SETLKW : FUSE_SETLK;
  2100. struct pid *pid = fl->fl_type != F_UNLCK ? task_tgid(current) : NULL;
  2101. pid_t pid_nr = pid_nr_ns(pid, fm->fc->pid_ns);
  2102. int err;
  2103. if (fl->fl_lmops && fl->fl_lmops->lm_grant) {
  2104. /* NLM needs asynchronous locks, which we don't support yet */
  2105. return -ENOLCK;
  2106. }
  2107. /* Unlock on close is handled by the flush method */
  2108. if ((fl->fl_flags & FL_CLOSE_POSIX) == FL_CLOSE_POSIX)
  2109. return 0;
  2110. fuse_lk_fill(&args, file, fl, opcode, pid_nr, flock, &inarg);
  2111. err = fuse_simple_request(fm, &args);
  2112. /* locking is restartable */
  2113. if (err == -EINTR)
  2114. err = -ERESTARTSYS;
  2115. return err;
  2116. }
  2117. static int fuse_file_lock(struct file *file, int cmd, struct file_lock *fl)
  2118. {
  2119. struct inode *inode = file_inode(file);
  2120. struct fuse_conn *fc = get_fuse_conn(inode);
  2121. int err;
  2122. if (cmd == F_CANCELLK) {
  2123. err = 0;
  2124. } else if (cmd == F_GETLK) {
  2125. if (fc->no_lock) {
  2126. posix_test_lock(file, fl);
  2127. err = 0;
  2128. } else
  2129. err = fuse_getlk(file, fl);
  2130. } else {
  2131. if (fc->no_lock)
  2132. err = posix_lock_file(file, fl, NULL);
  2133. else
  2134. err = fuse_setlk(file, fl, 0);
  2135. }
  2136. return err;
  2137. }
  2138. static int fuse_file_flock(struct file *file, int cmd, struct file_lock *fl)
  2139. {
  2140. struct inode *inode = file_inode(file);
  2141. struct fuse_conn *fc = get_fuse_conn(inode);
  2142. int err;
  2143. if (fc->no_flock) {
  2144. err = locks_lock_file_wait(file, fl);
  2145. } else {
  2146. struct fuse_file *ff = file->private_data;
  2147. /* emulate flock with POSIX locks */
  2148. ff->flock = true;
  2149. err = fuse_setlk(file, fl, 1);
  2150. }
  2151. return err;
  2152. }
  2153. static sector_t fuse_bmap(struct address_space *mapping, sector_t block)
  2154. {
  2155. struct inode *inode = mapping->host;
  2156. struct fuse_mount *fm = get_fuse_mount(inode);
  2157. FUSE_ARGS(args);
  2158. struct fuse_bmap_in inarg;
  2159. struct fuse_bmap_out outarg;
  2160. int err;
  2161. if (!inode->i_sb->s_bdev || fm->fc->no_bmap)
  2162. return 0;
  2163. memset(&inarg, 0, sizeof(inarg));
  2164. inarg.block = block;
  2165. inarg.blocksize = inode->i_sb->s_blocksize;
  2166. args.opcode = FUSE_BMAP;
  2167. args.nodeid = get_node_id(inode);
  2168. args.in_numargs = 1;
  2169. args.in_args[0].size = sizeof(inarg);
  2170. args.in_args[0].value = &inarg;
  2171. args.out_numargs = 1;
  2172. args.out_args[0].size = sizeof(outarg);
  2173. args.out_args[0].value = &outarg;
  2174. err = fuse_simple_request(fm, &args);
  2175. if (err == -ENOSYS)
  2176. fm->fc->no_bmap = 1;
  2177. return err ? 0 : outarg.block;
  2178. }
  2179. static loff_t fuse_lseek(struct file *file, loff_t offset, int whence)
  2180. {
  2181. struct inode *inode = file->f_mapping->host;
  2182. struct fuse_mount *fm = get_fuse_mount(inode);
  2183. struct fuse_file *ff = file->private_data;
  2184. FUSE_ARGS(args);
  2185. struct fuse_lseek_in inarg = {
  2186. .fh = ff->fh,
  2187. .offset = offset,
  2188. .whence = whence
  2189. };
  2190. struct fuse_lseek_out outarg;
  2191. int err;
  2192. if (fm->fc->no_lseek)
  2193. goto fallback;
  2194. args.opcode = FUSE_LSEEK;
  2195. args.nodeid = ff->nodeid;
  2196. args.in_numargs = 1;
  2197. args.in_args[0].size = sizeof(inarg);
  2198. args.in_args[0].value = &inarg;
  2199. args.out_numargs = 1;
  2200. args.out_args[0].size = sizeof(outarg);
  2201. args.out_args[0].value = &outarg;
  2202. err = fuse_simple_request(fm, &args);
  2203. if (err) {
  2204. if (err == -ENOSYS) {
  2205. fm->fc->no_lseek = 1;
  2206. goto fallback;
  2207. }
  2208. return err;
  2209. }
  2210. return vfs_setpos(file, outarg.offset, inode->i_sb->s_maxbytes);
  2211. fallback:
  2212. err = fuse_update_attributes(inode, file);
  2213. if (!err)
  2214. return generic_file_llseek(file, offset, whence);
  2215. else
  2216. return err;
  2217. }
  2218. static loff_t fuse_file_llseek(struct file *file, loff_t offset, int whence)
  2219. {
  2220. loff_t retval;
  2221. struct inode *inode = file_inode(file);
  2222. switch (whence) {
  2223. case SEEK_SET:
  2224. case SEEK_CUR:
  2225. /* No i_mutex protection necessary for SEEK_CUR and SEEK_SET */
  2226. retval = generic_file_llseek(file, offset, whence);
  2227. break;
  2228. case SEEK_END:
  2229. inode_lock(inode);
  2230. retval = fuse_update_attributes(inode, file);
  2231. if (!retval)
  2232. retval = generic_file_llseek(file, offset, whence);
  2233. inode_unlock(inode);
  2234. break;
  2235. case SEEK_HOLE:
  2236. case SEEK_DATA:
  2237. inode_lock(inode);
  2238. retval = fuse_lseek(file, offset, whence);
  2239. inode_unlock(inode);
  2240. break;
  2241. default:
  2242. retval = -EINVAL;
  2243. }
  2244. return retval;
  2245. }
  2246. /*
  2247. * CUSE servers compiled on 32bit broke on 64bit kernels because the
  2248. * ABI was defined to be 'struct iovec' which is different on 32bit
  2249. * and 64bit. Fortunately we can determine which structure the server
  2250. * used from the size of the reply.
  2251. */
  2252. static int fuse_copy_ioctl_iovec_old(struct iovec *dst, void *src,
  2253. size_t transferred, unsigned count,
  2254. bool is_compat)
  2255. {
  2256. #ifdef CONFIG_COMPAT
  2257. if (count * sizeof(struct compat_iovec) == transferred) {
  2258. struct compat_iovec *ciov = src;
  2259. unsigned i;
  2260. /*
  2261. * With this interface a 32bit server cannot support
  2262. * non-compat (i.e. ones coming from 64bit apps) ioctl
  2263. * requests
  2264. */
  2265. if (!is_compat)
  2266. return -EINVAL;
  2267. for (i = 0; i < count; i++) {
  2268. dst[i].iov_base = compat_ptr(ciov[i].iov_base);
  2269. dst[i].iov_len = ciov[i].iov_len;
  2270. }
  2271. return 0;
  2272. }
  2273. #endif
  2274. if (count * sizeof(struct iovec) != transferred)
  2275. return -EIO;
  2276. memcpy(dst, src, transferred);
  2277. return 0;
  2278. }
  2279. /* Make sure iov_length() won't overflow */
  2280. static int fuse_verify_ioctl_iov(struct fuse_conn *fc, struct iovec *iov,
  2281. size_t count)
  2282. {
  2283. size_t n;
  2284. u32 max = fc->max_pages << PAGE_SHIFT;
  2285. for (n = 0; n < count; n++, iov++) {
  2286. if (iov->iov_len > (size_t) max)
  2287. return -ENOMEM;
  2288. max -= iov->iov_len;
  2289. }
  2290. return 0;
  2291. }
  2292. static int fuse_copy_ioctl_iovec(struct fuse_conn *fc, struct iovec *dst,
  2293. void *src, size_t transferred, unsigned count,
  2294. bool is_compat)
  2295. {
  2296. unsigned i;
  2297. struct fuse_ioctl_iovec *fiov = src;
  2298. if (fc->minor < 16) {
  2299. return fuse_copy_ioctl_iovec_old(dst, src, transferred,
  2300. count, is_compat);
  2301. }
  2302. if (count * sizeof(struct fuse_ioctl_iovec) != transferred)
  2303. return -EIO;
  2304. for (i = 0; i < count; i++) {
  2305. /* Did the server supply an inappropriate value? */
  2306. if (fiov[i].base != (unsigned long) fiov[i].base ||
  2307. fiov[i].len != (unsigned long) fiov[i].len)
  2308. return -EIO;
  2309. dst[i].iov_base = (void __user *) (unsigned long) fiov[i].base;
  2310. dst[i].iov_len = (size_t) fiov[i].len;
  2311. #ifdef CONFIG_COMPAT
  2312. if (is_compat &&
  2313. (ptr_to_compat(dst[i].iov_base) != fiov[i].base ||
  2314. (compat_size_t) dst[i].iov_len != fiov[i].len))
  2315. return -EIO;
  2316. #endif
  2317. }
  2318. return 0;
  2319. }
  2320. /*
  2321. * For ioctls, there is no generic way to determine how much memory
  2322. * needs to be read and/or written. Furthermore, ioctls are allowed
  2323. * to dereference the passed pointer, so the parameter requires deep
  2324. * copying but FUSE has no idea whatsoever about what to copy in or
  2325. * out.
  2326. *
  2327. * This is solved by allowing FUSE server to retry ioctl with
  2328. * necessary in/out iovecs. Let's assume the ioctl implementation
  2329. * needs to read in the following structure.
  2330. *
  2331. * struct a {
  2332. * char *buf;
  2333. * size_t buflen;
  2334. * }
  2335. *
  2336. * On the first callout to FUSE server, inarg->in_size and
  2337. * inarg->out_size will be NULL; then, the server completes the ioctl
  2338. * with FUSE_IOCTL_RETRY set in out->flags, out->in_iovs set to 1 and
  2339. * the actual iov array to
  2340. *
  2341. * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) } }
  2342. *
  2343. * which tells FUSE to copy in the requested area and retry the ioctl.
  2344. * On the second round, the server has access to the structure and
  2345. * from that it can tell what to look for next, so on the invocation,
  2346. * it sets FUSE_IOCTL_RETRY, out->in_iovs to 2 and iov array to
  2347. *
  2348. * { { .iov_base = inarg.arg, .iov_len = sizeof(struct a) },
  2349. * { .iov_base = a.buf, .iov_len = a.buflen } }
  2350. *
  2351. * FUSE will copy both struct a and the pointed buffer from the
  2352. * process doing the ioctl and retry ioctl with both struct a and the
  2353. * buffer.
  2354. *
  2355. * This time, FUSE server has everything it needs and completes ioctl
  2356. * without FUSE_IOCTL_RETRY which finishes the ioctl call.
  2357. *
  2358. * Copying data out works the same way.
  2359. *
  2360. * Note that if FUSE_IOCTL_UNRESTRICTED is clear, the kernel
  2361. * automatically initializes in and out iovs by decoding @cmd with
  2362. * _IOC_* macros and the server is not allowed to request RETRY. This
  2363. * limits ioctl data transfers to well-formed ioctls and is the forced
  2364. * behavior for all FUSE servers.
  2365. */
  2366. long fuse_do_ioctl(struct file *file, unsigned int cmd, unsigned long arg,
  2367. unsigned int flags)
  2368. {
  2369. struct fuse_file *ff = file->private_data;
  2370. struct fuse_mount *fm = ff->fm;
  2371. struct fuse_ioctl_in inarg = {
  2372. .fh = ff->fh,
  2373. .cmd = cmd,
  2374. .arg = arg,
  2375. .flags = flags
  2376. };
  2377. struct fuse_ioctl_out outarg;
  2378. struct iovec *iov_page = NULL;
  2379. struct iovec *in_iov = NULL, *out_iov = NULL;
  2380. unsigned int in_iovs = 0, out_iovs = 0, max_pages;
  2381. size_t in_size, out_size, c;
  2382. ssize_t transferred;
  2383. int err, i;
  2384. struct iov_iter ii;
  2385. struct fuse_args_pages ap = {};
  2386. #if BITS_PER_LONG == 32
  2387. inarg.flags |= FUSE_IOCTL_32BIT;
  2388. #else
  2389. if (flags & FUSE_IOCTL_COMPAT) {
  2390. inarg.flags |= FUSE_IOCTL_32BIT;
  2391. #ifdef CONFIG_X86_X32
  2392. if (in_x32_syscall())
  2393. inarg.flags |= FUSE_IOCTL_COMPAT_X32;
  2394. #endif
  2395. }
  2396. #endif
  2397. /* assume all the iovs returned by client always fits in a page */
  2398. BUILD_BUG_ON(sizeof(struct fuse_ioctl_iovec) * FUSE_IOCTL_MAX_IOV > PAGE_SIZE);
  2399. err = -ENOMEM;
  2400. ap.pages = fuse_pages_alloc(fm->fc->max_pages, GFP_KERNEL, &ap.descs);
  2401. iov_page = (struct iovec *) __get_free_page(GFP_KERNEL);
  2402. if (!ap.pages || !iov_page)
  2403. goto out;
  2404. fuse_page_descs_length_init(ap.descs, 0, fm->fc->max_pages);
  2405. /*
  2406. * If restricted, initialize IO parameters as encoded in @cmd.
  2407. * RETRY from server is not allowed.
  2408. */
  2409. if (!(flags & FUSE_IOCTL_UNRESTRICTED)) {
  2410. struct iovec *iov = iov_page;
  2411. iov->iov_base = (void __user *)arg;
  2412. switch (cmd) {
  2413. case FS_IOC_GETFLAGS:
  2414. case FS_IOC_SETFLAGS:
  2415. iov->iov_len = sizeof(int);
  2416. break;
  2417. default:
  2418. iov->iov_len = _IOC_SIZE(cmd);
  2419. break;
  2420. }
  2421. if (_IOC_DIR(cmd) & _IOC_WRITE) {
  2422. in_iov = iov;
  2423. in_iovs = 1;
  2424. }
  2425. if (_IOC_DIR(cmd) & _IOC_READ) {
  2426. out_iov = iov;
  2427. out_iovs = 1;
  2428. }
  2429. }
  2430. retry:
  2431. inarg.in_size = in_size = iov_length(in_iov, in_iovs);
  2432. inarg.out_size = out_size = iov_length(out_iov, out_iovs);
  2433. /*
  2434. * Out data can be used either for actual out data or iovs,
  2435. * make sure there always is at least one page.
  2436. */
  2437. out_size = max_t(size_t, out_size, PAGE_SIZE);
  2438. max_pages = DIV_ROUND_UP(max(in_size, out_size), PAGE_SIZE);
  2439. /* make sure there are enough buffer pages and init request with them */
  2440. err = -ENOMEM;
  2441. if (max_pages > fm->fc->max_pages)
  2442. goto out;
  2443. while (ap.num_pages < max_pages) {
  2444. ap.pages[ap.num_pages] = alloc_page(GFP_KERNEL | __GFP_HIGHMEM);
  2445. if (!ap.pages[ap.num_pages])
  2446. goto out;
  2447. ap.num_pages++;
  2448. }
  2449. /* okay, let's send it to the client */
  2450. ap.args.opcode = FUSE_IOCTL;
  2451. ap.args.nodeid = ff->nodeid;
  2452. ap.args.in_numargs = 1;
  2453. ap.args.in_args[0].size = sizeof(inarg);
  2454. ap.args.in_args[0].value = &inarg;
  2455. if (in_size) {
  2456. ap.args.in_numargs++;
  2457. ap.args.in_args[1].size = in_size;
  2458. ap.args.in_pages = true;
  2459. err = -EFAULT;
  2460. iov_iter_init(&ii, WRITE, in_iov, in_iovs, in_size);
  2461. for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
  2462. c = copy_page_from_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
  2463. if (c != PAGE_SIZE && iov_iter_count(&ii))
  2464. goto out;
  2465. }
  2466. }
  2467. ap.args.out_numargs = 2;
  2468. ap.args.out_args[0].size = sizeof(outarg);
  2469. ap.args.out_args[0].value = &outarg;
  2470. ap.args.out_args[1].size = out_size;
  2471. ap.args.out_pages = true;
  2472. ap.args.out_argvar = true;
  2473. transferred = fuse_simple_request(fm, &ap.args);
  2474. err = transferred;
  2475. if (transferred < 0)
  2476. goto out;
  2477. /* did it ask for retry? */
  2478. if (outarg.flags & FUSE_IOCTL_RETRY) {
  2479. void *vaddr;
  2480. /* no retry if in restricted mode */
  2481. err = -EIO;
  2482. if (!(flags & FUSE_IOCTL_UNRESTRICTED))
  2483. goto out;
  2484. in_iovs = outarg.in_iovs;
  2485. out_iovs = outarg.out_iovs;
  2486. /*
  2487. * Make sure things are in boundary, separate checks
  2488. * are to protect against overflow.
  2489. */
  2490. err = -ENOMEM;
  2491. if (in_iovs > FUSE_IOCTL_MAX_IOV ||
  2492. out_iovs > FUSE_IOCTL_MAX_IOV ||
  2493. in_iovs + out_iovs > FUSE_IOCTL_MAX_IOV)
  2494. goto out;
  2495. vaddr = kmap_atomic(ap.pages[0]);
  2496. err = fuse_copy_ioctl_iovec(fm->fc, iov_page, vaddr,
  2497. transferred, in_iovs + out_iovs,
  2498. (flags & FUSE_IOCTL_COMPAT) != 0);
  2499. kunmap_atomic(vaddr);
  2500. if (err)
  2501. goto out;
  2502. in_iov = iov_page;
  2503. out_iov = in_iov + in_iovs;
  2504. err = fuse_verify_ioctl_iov(fm->fc, in_iov, in_iovs);
  2505. if (err)
  2506. goto out;
  2507. err = fuse_verify_ioctl_iov(fm->fc, out_iov, out_iovs);
  2508. if (err)
  2509. goto out;
  2510. goto retry;
  2511. }
  2512. err = -EIO;
  2513. if (transferred > inarg.out_size)
  2514. goto out;
  2515. err = -EFAULT;
  2516. iov_iter_init(&ii, READ, out_iov, out_iovs, transferred);
  2517. for (i = 0; iov_iter_count(&ii) && !WARN_ON(i >= ap.num_pages); i++) {
  2518. c = copy_page_to_iter(ap.pages[i], 0, PAGE_SIZE, &ii);
  2519. if (c != PAGE_SIZE && iov_iter_count(&ii))
  2520. goto out;
  2521. }
  2522. err = 0;
  2523. out:
  2524. free_page((unsigned long) iov_page);
  2525. while (ap.num_pages)
  2526. __free_page(ap.pages[--ap.num_pages]);
  2527. kfree(ap.pages);
  2528. return err ? err : outarg.result;
  2529. }
  2530. EXPORT_SYMBOL_GPL(fuse_do_ioctl);
  2531. long fuse_ioctl_common(struct file *file, unsigned int cmd,
  2532. unsigned long arg, unsigned int flags)
  2533. {
  2534. struct inode *inode = file_inode(file);
  2535. struct fuse_conn *fc = get_fuse_conn(inode);
  2536. if (!fuse_allow_current_process(fc))
  2537. return -EACCES;
  2538. if (fuse_is_bad(inode))
  2539. return -EIO;
  2540. return fuse_do_ioctl(file, cmd, arg, flags);
  2541. }
  2542. static long fuse_file_ioctl(struct file *file, unsigned int cmd,
  2543. unsigned long arg)
  2544. {
  2545. return fuse_ioctl_common(file, cmd, arg, 0);
  2546. }
  2547. static long fuse_file_compat_ioctl(struct file *file, unsigned int cmd,
  2548. unsigned long arg)
  2549. {
  2550. return fuse_ioctl_common(file, cmd, arg, FUSE_IOCTL_COMPAT);
  2551. }
  2552. /*
  2553. * All files which have been polled are linked to RB tree
  2554. * fuse_conn->polled_files which is indexed by kh. Walk the tree and
  2555. * find the matching one.
  2556. */
  2557. static struct rb_node **fuse_find_polled_node(struct fuse_conn *fc, u64 kh,
  2558. struct rb_node **parent_out)
  2559. {
  2560. struct rb_node **link = &fc->polled_files.rb_node;
  2561. struct rb_node *last = NULL;
  2562. while (*link) {
  2563. struct fuse_file *ff;
  2564. last = *link;
  2565. ff = rb_entry(last, struct fuse_file, polled_node);
  2566. if (kh < ff->kh)
  2567. link = &last->rb_left;
  2568. else if (kh > ff->kh)
  2569. link = &last->rb_right;
  2570. else
  2571. return link;
  2572. }
  2573. if (parent_out)
  2574. *parent_out = last;
  2575. return link;
  2576. }
  2577. /*
  2578. * The file is about to be polled. Make sure it's on the polled_files
  2579. * RB tree. Note that files once added to the polled_files tree are
  2580. * not removed before the file is released. This is because a file
  2581. * polled once is likely to be polled again.
  2582. */
  2583. static void fuse_register_polled_file(struct fuse_conn *fc,
  2584. struct fuse_file *ff)
  2585. {
  2586. spin_lock(&fc->lock);
  2587. if (RB_EMPTY_NODE(&ff->polled_node)) {
  2588. struct rb_node **link, *parent;
  2589. link = fuse_find_polled_node(fc, ff->kh, &parent);
  2590. BUG_ON(*link);
  2591. rb_link_node(&ff->polled_node, parent, link);
  2592. rb_insert_color(&ff->polled_node, &fc->polled_files);
  2593. }
  2594. spin_unlock(&fc->lock);
  2595. }
  2596. __poll_t fuse_file_poll(struct file *file, poll_table *wait)
  2597. {
  2598. struct fuse_file *ff = file->private_data;
  2599. struct fuse_mount *fm = ff->fm;
  2600. struct fuse_poll_in inarg = { .fh = ff->fh, .kh = ff->kh };
  2601. struct fuse_poll_out outarg;
  2602. FUSE_ARGS(args);
  2603. int err;
  2604. if (fm->fc->no_poll)
  2605. return DEFAULT_POLLMASK;
  2606. poll_wait(file, &ff->poll_wait, wait);
  2607. inarg.events = mangle_poll(poll_requested_events(wait));
  2608. /*
  2609. * Ask for notification iff there's someone waiting for it.
  2610. * The client may ignore the flag and always notify.
  2611. */
  2612. if (waitqueue_active(&ff->poll_wait)) {
  2613. inarg.flags |= FUSE_POLL_SCHEDULE_NOTIFY;
  2614. fuse_register_polled_file(fm->fc, ff);
  2615. }
  2616. args.opcode = FUSE_POLL;
  2617. args.nodeid = ff->nodeid;
  2618. args.in_numargs = 1;
  2619. args.in_args[0].size = sizeof(inarg);
  2620. args.in_args[0].value = &inarg;
  2621. args.out_numargs = 1;
  2622. args.out_args[0].size = sizeof(outarg);
  2623. args.out_args[0].value = &outarg;
  2624. err = fuse_simple_request(fm, &args);
  2625. if (!err)
  2626. return demangle_poll(outarg.revents);
  2627. if (err == -ENOSYS) {
  2628. fm->fc->no_poll = 1;
  2629. return DEFAULT_POLLMASK;
  2630. }
  2631. return EPOLLERR;
  2632. }
  2633. EXPORT_SYMBOL_GPL(fuse_file_poll);
  2634. /*
  2635. * This is called from fuse_handle_notify() on FUSE_NOTIFY_POLL and
  2636. * wakes up the poll waiters.
  2637. */
  2638. int fuse_notify_poll_wakeup(struct fuse_conn *fc,
  2639. struct fuse_notify_poll_wakeup_out *outarg)
  2640. {
  2641. u64 kh = outarg->kh;
  2642. struct rb_node **link;
  2643. spin_lock(&fc->lock);
  2644. link = fuse_find_polled_node(fc, kh, NULL);
  2645. if (*link) {
  2646. struct fuse_file *ff;
  2647. ff = rb_entry(*link, struct fuse_file, polled_node);
  2648. wake_up_interruptible_sync(&ff->poll_wait);
  2649. }
  2650. spin_unlock(&fc->lock);
  2651. return 0;
  2652. }
  2653. static void fuse_do_truncate(struct file *file)
  2654. {
  2655. struct inode *inode = file->f_mapping->host;
  2656. struct iattr attr;
  2657. attr.ia_valid = ATTR_SIZE;
  2658. attr.ia_size = i_size_read(inode);
  2659. attr.ia_file = file;
  2660. attr.ia_valid |= ATTR_FILE;
  2661. fuse_do_setattr(file_dentry(file), &attr, file);
  2662. }
  2663. static inline loff_t fuse_round_up(struct fuse_conn *fc, loff_t off)
  2664. {
  2665. return round_up(off, fc->max_pages << PAGE_SHIFT);
  2666. }
  2667. static ssize_t
  2668. fuse_direct_IO(struct kiocb *iocb, struct iov_iter *iter)
  2669. {
  2670. DECLARE_COMPLETION_ONSTACK(wait);
  2671. ssize_t ret = 0;
  2672. struct file *file = iocb->ki_filp;
  2673. struct fuse_file *ff = file->private_data;
  2674. loff_t pos = 0;
  2675. struct inode *inode;
  2676. loff_t i_size;
  2677. size_t count = iov_iter_count(iter), shortened = 0;
  2678. loff_t offset = iocb->ki_pos;
  2679. struct fuse_io_priv *io;
  2680. pos = offset;
  2681. inode = file->f_mapping->host;
  2682. i_size = i_size_read(inode);
  2683. if ((iov_iter_rw(iter) == READ) && (offset >= i_size))
  2684. return 0;
  2685. io = kmalloc(sizeof(struct fuse_io_priv), GFP_KERNEL);
  2686. if (!io)
  2687. return -ENOMEM;
  2688. spin_lock_init(&io->lock);
  2689. kref_init(&io->refcnt);
  2690. io->reqs = 1;
  2691. io->bytes = -1;
  2692. io->size = 0;
  2693. io->offset = offset;
  2694. io->write = (iov_iter_rw(iter) == WRITE);
  2695. io->err = 0;
  2696. /*
  2697. * By default, we want to optimize all I/Os with async request
  2698. * submission to the client filesystem if supported.
  2699. */
  2700. io->async = ff->fm->fc->async_dio;
  2701. io->iocb = iocb;
  2702. io->blocking = is_sync_kiocb(iocb);
  2703. /* optimization for short read */
  2704. if (io->async && !io->write && offset + count > i_size) {
  2705. iov_iter_truncate(iter, fuse_round_up(ff->fm->fc, i_size - offset));
  2706. shortened = count - iov_iter_count(iter);
  2707. count -= shortened;
  2708. }
  2709. /*
  2710. * We cannot asynchronously extend the size of a file.
  2711. * In such case the aio will behave exactly like sync io.
  2712. */
  2713. if ((offset + count > i_size) && io->write)
  2714. io->blocking = true;
  2715. if (io->async && io->blocking) {
  2716. /*
  2717. * Additional reference to keep io around after
  2718. * calling fuse_aio_complete()
  2719. */
  2720. kref_get(&io->refcnt);
  2721. io->done = &wait;
  2722. }
  2723. if (iov_iter_rw(iter) == WRITE) {
  2724. ret = fuse_direct_io(io, iter, &pos, FUSE_DIO_WRITE);
  2725. fuse_invalidate_attr(inode);
  2726. } else {
  2727. ret = __fuse_direct_read(io, iter, &pos);
  2728. }
  2729. iov_iter_reexpand(iter, iov_iter_count(iter) + shortened);
  2730. if (io->async) {
  2731. bool blocking = io->blocking;
  2732. fuse_aio_complete(io, ret < 0 ? ret : 0, -1);
  2733. /* we have a non-extending, async request, so return */
  2734. if (!blocking)
  2735. return -EIOCBQUEUED;
  2736. wait_for_completion(&wait);
  2737. ret = fuse_get_res_by_io(io);
  2738. }
  2739. kref_put(&io->refcnt, fuse_io_release);
  2740. if (iov_iter_rw(iter) == WRITE) {
  2741. if (ret > 0)
  2742. fuse_write_update_size(inode, pos);
  2743. else if (ret < 0 && offset + count > i_size)
  2744. fuse_do_truncate(file);
  2745. }
  2746. return ret;
  2747. }
  2748. static int fuse_writeback_range(struct inode *inode, loff_t start, loff_t end)
  2749. {
  2750. int err = filemap_write_and_wait_range(inode->i_mapping, start, LLONG_MAX);
  2751. if (!err)
  2752. fuse_sync_writes(inode);
  2753. return err;
  2754. }
  2755. static long fuse_file_fallocate(struct file *file, int mode, loff_t offset,
  2756. loff_t length)
  2757. {
  2758. struct fuse_file *ff = file->private_data;
  2759. struct inode *inode = file_inode(file);
  2760. struct fuse_inode *fi = get_fuse_inode(inode);
  2761. struct fuse_mount *fm = ff->fm;
  2762. FUSE_ARGS(args);
  2763. struct fuse_fallocate_in inarg = {
  2764. .fh = ff->fh,
  2765. .offset = offset,
  2766. .length = length,
  2767. .mode = mode
  2768. };
  2769. int err;
  2770. bool lock_inode = !(mode & FALLOC_FL_KEEP_SIZE) ||
  2771. (mode & FALLOC_FL_PUNCH_HOLE);
  2772. bool block_faults = FUSE_IS_DAX(inode) && lock_inode;
  2773. if (mode & ~(FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
  2774. return -EOPNOTSUPP;
  2775. if (fm->fc->no_fallocate)
  2776. return -EOPNOTSUPP;
  2777. if (lock_inode) {
  2778. inode_lock(inode);
  2779. if (block_faults) {
  2780. down_write(&fi->i_mmap_sem);
  2781. err = fuse_dax_break_layouts(inode, 0, 0);
  2782. if (err)
  2783. goto out;
  2784. }
  2785. if (mode & FALLOC_FL_PUNCH_HOLE) {
  2786. loff_t endbyte = offset + length - 1;
  2787. err = fuse_writeback_range(inode, offset, endbyte);
  2788. if (err)
  2789. goto out;
  2790. }
  2791. }
  2792. if (!(mode & FALLOC_FL_KEEP_SIZE) &&
  2793. offset + length > i_size_read(inode)) {
  2794. err = inode_newsize_ok(inode, offset + length);
  2795. if (err)
  2796. goto out;
  2797. }
  2798. if (!(mode & FALLOC_FL_KEEP_SIZE))
  2799. set_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
  2800. args.opcode = FUSE_FALLOCATE;
  2801. args.nodeid = ff->nodeid;
  2802. args.in_numargs = 1;
  2803. args.in_args[0].size = sizeof(inarg);
  2804. args.in_args[0].value = &inarg;
  2805. err = fuse_simple_request(fm, &args);
  2806. if (err == -ENOSYS) {
  2807. fm->fc->no_fallocate = 1;
  2808. err = -EOPNOTSUPP;
  2809. }
  2810. if (err)
  2811. goto out;
  2812. /* we could have extended the file */
  2813. if (!(mode & FALLOC_FL_KEEP_SIZE)) {
  2814. bool changed = fuse_write_update_size(inode, offset + length);
  2815. if (changed && fm->fc->writeback_cache)
  2816. file_update_time(file);
  2817. }
  2818. if (mode & FALLOC_FL_PUNCH_HOLE)
  2819. truncate_pagecache_range(inode, offset, offset + length - 1);
  2820. fuse_invalidate_attr(inode);
  2821. out:
  2822. if (!(mode & FALLOC_FL_KEEP_SIZE))
  2823. clear_bit(FUSE_I_SIZE_UNSTABLE, &fi->state);
  2824. if (block_faults)
  2825. up_write(&fi->i_mmap_sem);
  2826. if (lock_inode)
  2827. inode_unlock(inode);
  2828. fuse_flush_time_update(inode);
  2829. return err;
  2830. }
  2831. static ssize_t __fuse_copy_file_range(struct file *file_in, loff_t pos_in,
  2832. struct file *file_out, loff_t pos_out,
  2833. size_t len, unsigned int flags)
  2834. {
  2835. struct fuse_file *ff_in = file_in->private_data;
  2836. struct fuse_file *ff_out = file_out->private_data;
  2837. struct inode *inode_in = file_inode(file_in);
  2838. struct inode *inode_out = file_inode(file_out);
  2839. struct fuse_inode *fi_out = get_fuse_inode(inode_out);
  2840. struct fuse_mount *fm = ff_in->fm;
  2841. struct fuse_conn *fc = fm->fc;
  2842. FUSE_ARGS(args);
  2843. struct fuse_copy_file_range_in inarg = {
  2844. .fh_in = ff_in->fh,
  2845. .off_in = pos_in,
  2846. .nodeid_out = ff_out->nodeid,
  2847. .fh_out = ff_out->fh,
  2848. .off_out = pos_out,
  2849. .len = len,
  2850. .flags = flags
  2851. };
  2852. struct fuse_write_out outarg;
  2853. ssize_t err;
  2854. /* mark unstable when write-back is not used, and file_out gets
  2855. * extended */
  2856. bool is_unstable = (!fc->writeback_cache) &&
  2857. ((pos_out + len) > inode_out->i_size);
  2858. if (fc->no_copy_file_range)
  2859. return -EOPNOTSUPP;
  2860. if (file_inode(file_in)->i_sb != file_inode(file_out)->i_sb)
  2861. return -EXDEV;
  2862. inode_lock(inode_in);
  2863. err = fuse_writeback_range(inode_in, pos_in, pos_in + len - 1);
  2864. inode_unlock(inode_in);
  2865. if (err)
  2866. return err;
  2867. inode_lock(inode_out);
  2868. err = file_modified(file_out);
  2869. if (err)
  2870. goto out;
  2871. /*
  2872. * Write out dirty pages in the destination file before sending the COPY
  2873. * request to userspace. After the request is completed, truncate off
  2874. * pages (including partial ones) from the cache that have been copied,
  2875. * since these contain stale data at that point.
  2876. *
  2877. * This should be mostly correct, but if the COPY writes to partial
  2878. * pages (at the start or end) and the parts not covered by the COPY are
  2879. * written through a memory map after calling fuse_writeback_range(),
  2880. * then these partial page modifications will be lost on truncation.
  2881. *
  2882. * It is unlikely that someone would rely on such mixed style
  2883. * modifications. Yet this does give less guarantees than if the
  2884. * copying was performed with write(2).
  2885. *
  2886. * To fix this a i_mmap_sem style lock could be used to prevent new
  2887. * faults while the copy is ongoing.
  2888. */
  2889. err = fuse_writeback_range(inode_out, pos_out, pos_out + len - 1);
  2890. if (err)
  2891. goto out;
  2892. if (is_unstable)
  2893. set_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
  2894. args.opcode = FUSE_COPY_FILE_RANGE;
  2895. args.nodeid = ff_in->nodeid;
  2896. args.in_numargs = 1;
  2897. args.in_args[0].size = sizeof(inarg);
  2898. args.in_args[0].value = &inarg;
  2899. args.out_numargs = 1;
  2900. args.out_args[0].size = sizeof(outarg);
  2901. args.out_args[0].value = &outarg;
  2902. err = fuse_simple_request(fm, &args);
  2903. if (err == -ENOSYS) {
  2904. fc->no_copy_file_range = 1;
  2905. err = -EOPNOTSUPP;
  2906. }
  2907. if (err)
  2908. goto out;
  2909. truncate_inode_pages_range(inode_out->i_mapping,
  2910. ALIGN_DOWN(pos_out, PAGE_SIZE),
  2911. ALIGN(pos_out + outarg.size, PAGE_SIZE) - 1);
  2912. if (fc->writeback_cache) {
  2913. fuse_write_update_size(inode_out, pos_out + outarg.size);
  2914. file_update_time(file_out);
  2915. }
  2916. fuse_invalidate_attr(inode_out);
  2917. err = outarg.size;
  2918. out:
  2919. if (is_unstable)
  2920. clear_bit(FUSE_I_SIZE_UNSTABLE, &fi_out->state);
  2921. inode_unlock(inode_out);
  2922. file_accessed(file_in);
  2923. fuse_flush_time_update(inode_out);
  2924. return err;
  2925. }
  2926. static ssize_t fuse_copy_file_range(struct file *src_file, loff_t src_off,
  2927. struct file *dst_file, loff_t dst_off,
  2928. size_t len, unsigned int flags)
  2929. {
  2930. ssize_t ret;
  2931. ret = __fuse_copy_file_range(src_file, src_off, dst_file, dst_off,
  2932. len, flags);
  2933. if (ret == -EOPNOTSUPP || ret == -EXDEV)
  2934. ret = generic_copy_file_range(src_file, src_off, dst_file,
  2935. dst_off, len, flags);
  2936. return ret;
  2937. }
  2938. static const struct file_operations fuse_file_operations = {
  2939. .llseek = fuse_file_llseek,
  2940. .read_iter = fuse_file_read_iter,
  2941. .write_iter = fuse_file_write_iter,
  2942. .mmap = fuse_file_mmap,
  2943. .open = fuse_open,
  2944. .flush = fuse_flush,
  2945. .release = fuse_release,
  2946. .fsync = fuse_fsync,
  2947. .lock = fuse_file_lock,
  2948. .get_unmapped_area = thp_get_unmapped_area,
  2949. .flock = fuse_file_flock,
  2950. .splice_read = generic_file_splice_read,
  2951. .splice_write = iter_file_splice_write,
  2952. .unlocked_ioctl = fuse_file_ioctl,
  2953. .compat_ioctl = fuse_file_compat_ioctl,
  2954. .poll = fuse_file_poll,
  2955. .fallocate = fuse_file_fallocate,
  2956. .copy_file_range = fuse_copy_file_range,
  2957. };
  2958. static const struct address_space_operations fuse_file_aops = {
  2959. .readpage = fuse_readpage,
  2960. .readahead = fuse_readahead,
  2961. .writepage = fuse_writepage,
  2962. .writepages = fuse_writepages,
  2963. .launder_page = fuse_launder_page,
  2964. .set_page_dirty = __set_page_dirty_nobuffers,
  2965. .bmap = fuse_bmap,
  2966. .direct_IO = fuse_direct_IO,
  2967. .write_begin = fuse_write_begin,
  2968. .write_end = fuse_write_end,
  2969. };
  2970. void fuse_init_file_inode(struct inode *inode)
  2971. {
  2972. struct fuse_inode *fi = get_fuse_inode(inode);
  2973. inode->i_fop = &fuse_file_operations;
  2974. inode->i_data.a_ops = &fuse_file_aops;
  2975. INIT_LIST_HEAD(&fi->write_files);
  2976. INIT_LIST_HEAD(&fi->queued_writes);
  2977. fi->writectr = 0;
  2978. init_waitqueue_head(&fi->page_waitq);
  2979. fi->writepages = RB_ROOT;
  2980. if (IS_ENABLED(CONFIG_FUSE_DAX))
  2981. fuse_dax_inode_init(inode);
  2982. }