dev.c 53 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372
  1. /*
  2. FUSE: Filesystem in Userspace
  3. Copyright (C) 2001-2008 Miklos Szeredi <miklos@szeredi.hu>
  4. This program can be distributed under the terms of the GNU GPL.
  5. See the file COPYING.
  6. */
  7. #include "fuse_i.h"
  8. #include <linux/init.h>
  9. #include <linux/module.h>
  10. #include <linux/poll.h>
  11. #include <linux/sched/signal.h>
  12. #include <linux/uio.h>
  13. #include <linux/miscdevice.h>
  14. #include <linux/namei.h>
  15. #include <linux/pagemap.h>
  16. #include <linux/file.h>
  17. #include <linux/slab.h>
  18. #include <linux/pipe_fs_i.h>
  19. #include <linux/swap.h>
  20. #include <linux/splice.h>
  21. #include <linux/sched.h>
  22. MODULE_ALIAS_MISCDEV(FUSE_MINOR);
  23. MODULE_ALIAS("devname:fuse");
  24. /* Ordinary requests have even IDs, while interrupts IDs are odd */
  25. #define FUSE_INT_REQ_BIT (1ULL << 0)
  26. #define FUSE_REQ_ID_STEP (1ULL << 1)
  27. static struct kmem_cache *fuse_req_cachep;
  28. static struct fuse_dev *fuse_get_dev(struct file *file)
  29. {
  30. /*
  31. * Lockless access is OK, because file->private data is set
  32. * once during mount and is valid until the file is released.
  33. */
  34. return READ_ONCE(file->private_data);
  35. }
  36. static void fuse_request_init(struct fuse_mount *fm, struct fuse_req *req)
  37. {
  38. INIT_LIST_HEAD(&req->list);
  39. INIT_LIST_HEAD(&req->intr_entry);
  40. init_waitqueue_head(&req->waitq);
  41. refcount_set(&req->count, 1);
  42. __set_bit(FR_PENDING, &req->flags);
  43. req->fm = fm;
  44. }
  45. static struct fuse_req *fuse_request_alloc(struct fuse_mount *fm, gfp_t flags)
  46. {
  47. struct fuse_req *req = kmem_cache_zalloc(fuse_req_cachep, flags);
  48. if (req)
  49. fuse_request_init(fm, req);
  50. return req;
  51. }
  52. static void fuse_request_free(struct fuse_req *req)
  53. {
  54. kmem_cache_free(fuse_req_cachep, req);
  55. }
  56. static void __fuse_get_request(struct fuse_req *req)
  57. {
  58. refcount_inc(&req->count);
  59. }
  60. /* Must be called with > 1 refcount */
  61. static void __fuse_put_request(struct fuse_req *req)
  62. {
  63. refcount_dec(&req->count);
  64. }
  65. void fuse_set_initialized(struct fuse_conn *fc)
  66. {
  67. /* Make sure stores before this are seen on another CPU */
  68. smp_wmb();
  69. fc->initialized = 1;
  70. }
  71. static bool fuse_block_alloc(struct fuse_conn *fc, bool for_background)
  72. {
  73. return !fc->initialized || (for_background && fc->blocked);
  74. }
  75. static void fuse_drop_waiting(struct fuse_conn *fc)
  76. {
  77. /*
  78. * lockess check of fc->connected is okay, because atomic_dec_and_test()
  79. * provides a memory barrier mached with the one in fuse_wait_aborted()
  80. * to ensure no wake-up is missed.
  81. */
  82. if (atomic_dec_and_test(&fc->num_waiting) &&
  83. !READ_ONCE(fc->connected)) {
  84. /* wake up aborters */
  85. wake_up_all(&fc->blocked_waitq);
  86. }
  87. }
  88. static void fuse_put_request(struct fuse_req *req);
  89. static struct fuse_req *fuse_get_req(struct fuse_mount *fm, bool for_background)
  90. {
  91. struct fuse_conn *fc = fm->fc;
  92. struct fuse_req *req;
  93. int err;
  94. atomic_inc(&fc->num_waiting);
  95. if (fuse_block_alloc(fc, for_background)) {
  96. err = -EINTR;
  97. if (wait_event_killable_exclusive(fc->blocked_waitq,
  98. !fuse_block_alloc(fc, for_background)))
  99. goto out;
  100. }
  101. /* Matches smp_wmb() in fuse_set_initialized() */
  102. smp_rmb();
  103. err = -ENOTCONN;
  104. if (!fc->connected)
  105. goto out;
  106. err = -ECONNREFUSED;
  107. if (fc->conn_error)
  108. goto out;
  109. req = fuse_request_alloc(fm, GFP_KERNEL);
  110. err = -ENOMEM;
  111. if (!req) {
  112. if (for_background)
  113. wake_up(&fc->blocked_waitq);
  114. goto out;
  115. }
  116. req->in.h.uid = from_kuid(fc->user_ns, current_fsuid());
  117. req->in.h.gid = from_kgid(fc->user_ns, current_fsgid());
  118. req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
  119. __set_bit(FR_WAITING, &req->flags);
  120. if (for_background)
  121. __set_bit(FR_BACKGROUND, &req->flags);
  122. if (unlikely(req->in.h.uid == ((uid_t)-1) ||
  123. req->in.h.gid == ((gid_t)-1))) {
  124. fuse_put_request(req);
  125. return ERR_PTR(-EOVERFLOW);
  126. }
  127. return req;
  128. out:
  129. fuse_drop_waiting(fc);
  130. return ERR_PTR(err);
  131. }
  132. static void fuse_put_request(struct fuse_req *req)
  133. {
  134. struct fuse_conn *fc = req->fm->fc;
  135. if (refcount_dec_and_test(&req->count)) {
  136. if (test_bit(FR_BACKGROUND, &req->flags)) {
  137. /*
  138. * We get here in the unlikely case that a background
  139. * request was allocated but not sent
  140. */
  141. spin_lock(&fc->bg_lock);
  142. if (!fc->blocked)
  143. wake_up(&fc->blocked_waitq);
  144. spin_unlock(&fc->bg_lock);
  145. }
  146. if (test_bit(FR_WAITING, &req->flags)) {
  147. __clear_bit(FR_WAITING, &req->flags);
  148. fuse_drop_waiting(fc);
  149. }
  150. fuse_request_free(req);
  151. }
  152. }
  153. unsigned int fuse_len_args(unsigned int numargs, struct fuse_arg *args)
  154. {
  155. unsigned nbytes = 0;
  156. unsigned i;
  157. for (i = 0; i < numargs; i++)
  158. nbytes += args[i].size;
  159. return nbytes;
  160. }
  161. EXPORT_SYMBOL_GPL(fuse_len_args);
  162. u64 fuse_get_unique(struct fuse_iqueue *fiq)
  163. {
  164. fiq->reqctr += FUSE_REQ_ID_STEP;
  165. return fiq->reqctr;
  166. }
  167. EXPORT_SYMBOL_GPL(fuse_get_unique);
  168. static unsigned int fuse_req_hash(u64 unique)
  169. {
  170. return hash_long(unique & ~FUSE_INT_REQ_BIT, FUSE_PQ_HASH_BITS);
  171. }
  172. /**
  173. * A new request is available, wake fiq->waitq
  174. */
  175. static void fuse_dev_wake_and_unlock(struct fuse_iqueue *fiq)
  176. __releases(fiq->lock)
  177. {
  178. wake_up(&fiq->waitq);
  179. kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
  180. spin_unlock(&fiq->lock);
  181. }
  182. const struct fuse_iqueue_ops fuse_dev_fiq_ops = {
  183. .wake_forget_and_unlock = fuse_dev_wake_and_unlock,
  184. .wake_interrupt_and_unlock = fuse_dev_wake_and_unlock,
  185. .wake_pending_and_unlock = fuse_dev_wake_and_unlock,
  186. };
  187. EXPORT_SYMBOL_GPL(fuse_dev_fiq_ops);
  188. static void queue_request_and_unlock(struct fuse_iqueue *fiq,
  189. struct fuse_req *req)
  190. __releases(fiq->lock)
  191. {
  192. req->in.h.len = sizeof(struct fuse_in_header) +
  193. fuse_len_args(req->args->in_numargs,
  194. (struct fuse_arg *) req->args->in_args);
  195. list_add_tail(&req->list, &fiq->pending);
  196. fiq->ops->wake_pending_and_unlock(fiq);
  197. }
  198. void fuse_queue_forget(struct fuse_conn *fc, struct fuse_forget_link *forget,
  199. u64 nodeid, u64 nlookup)
  200. {
  201. struct fuse_iqueue *fiq = &fc->iq;
  202. forget->forget_one.nodeid = nodeid;
  203. forget->forget_one.nlookup = nlookup;
  204. spin_lock(&fiq->lock);
  205. if (fiq->connected) {
  206. fiq->forget_list_tail->next = forget;
  207. fiq->forget_list_tail = forget;
  208. fiq->ops->wake_forget_and_unlock(fiq);
  209. } else {
  210. kfree(forget);
  211. spin_unlock(&fiq->lock);
  212. }
  213. }
  214. static void flush_bg_queue(struct fuse_conn *fc)
  215. {
  216. struct fuse_iqueue *fiq = &fc->iq;
  217. while (fc->active_background < fc->max_background &&
  218. !list_empty(&fc->bg_queue)) {
  219. struct fuse_req *req;
  220. req = list_first_entry(&fc->bg_queue, struct fuse_req, list);
  221. list_del(&req->list);
  222. fc->active_background++;
  223. spin_lock(&fiq->lock);
  224. req->in.h.unique = fuse_get_unique(fiq);
  225. queue_request_and_unlock(fiq, req);
  226. }
  227. }
  228. /*
  229. * This function is called when a request is finished. Either a reply
  230. * has arrived or it was aborted (and not yet sent) or some error
  231. * occurred during communication with userspace, or the device file
  232. * was closed. The requester thread is woken up (if still waiting),
  233. * the 'end' callback is called if given, else the reference to the
  234. * request is released
  235. */
  236. void fuse_request_end(struct fuse_req *req)
  237. {
  238. struct fuse_mount *fm = req->fm;
  239. struct fuse_conn *fc = fm->fc;
  240. struct fuse_iqueue *fiq = &fc->iq;
  241. if (test_and_set_bit(FR_FINISHED, &req->flags))
  242. goto put_request;
  243. /*
  244. * test_and_set_bit() implies smp_mb() between bit
  245. * changing and below FR_INTERRUPTED check. Pairs with
  246. * smp_mb() from queue_interrupt().
  247. */
  248. if (test_bit(FR_INTERRUPTED, &req->flags)) {
  249. spin_lock(&fiq->lock);
  250. list_del_init(&req->intr_entry);
  251. spin_unlock(&fiq->lock);
  252. }
  253. WARN_ON(test_bit(FR_PENDING, &req->flags));
  254. WARN_ON(test_bit(FR_SENT, &req->flags));
  255. if (test_bit(FR_BACKGROUND, &req->flags)) {
  256. spin_lock(&fc->bg_lock);
  257. clear_bit(FR_BACKGROUND, &req->flags);
  258. if (fc->num_background == fc->max_background) {
  259. fc->blocked = 0;
  260. wake_up(&fc->blocked_waitq);
  261. } else if (!fc->blocked) {
  262. /*
  263. * Wake up next waiter, if any. It's okay to use
  264. * waitqueue_active(), as we've already synced up
  265. * fc->blocked with waiters with the wake_up() call
  266. * above.
  267. */
  268. if (waitqueue_active(&fc->blocked_waitq))
  269. wake_up(&fc->blocked_waitq);
  270. }
  271. if (fc->num_background == fc->congestion_threshold && fm->sb) {
  272. clear_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC);
  273. clear_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC);
  274. }
  275. fc->num_background--;
  276. fc->active_background--;
  277. flush_bg_queue(fc);
  278. spin_unlock(&fc->bg_lock);
  279. } else {
  280. /* Wake up waiter sleeping in request_wait_answer() */
  281. wake_up(&req->waitq);
  282. }
  283. if (test_bit(FR_ASYNC, &req->flags))
  284. req->args->end(fm, req->args, req->out.h.error);
  285. put_request:
  286. fuse_put_request(req);
  287. }
  288. EXPORT_SYMBOL_GPL(fuse_request_end);
  289. static int queue_interrupt(struct fuse_req *req)
  290. {
  291. struct fuse_iqueue *fiq = &req->fm->fc->iq;
  292. spin_lock(&fiq->lock);
  293. /* Check for we've sent request to interrupt this req */
  294. if (unlikely(!test_bit(FR_INTERRUPTED, &req->flags))) {
  295. spin_unlock(&fiq->lock);
  296. return -EINVAL;
  297. }
  298. if (list_empty(&req->intr_entry)) {
  299. list_add_tail(&req->intr_entry, &fiq->interrupts);
  300. /*
  301. * Pairs with smp_mb() implied by test_and_set_bit()
  302. * from fuse_request_end().
  303. */
  304. smp_mb();
  305. if (test_bit(FR_FINISHED, &req->flags)) {
  306. list_del_init(&req->intr_entry);
  307. spin_unlock(&fiq->lock);
  308. return 0;
  309. }
  310. fiq->ops->wake_interrupt_and_unlock(fiq);
  311. } else {
  312. spin_unlock(&fiq->lock);
  313. }
  314. return 0;
  315. }
  316. static void request_wait_answer(struct fuse_req *req)
  317. {
  318. struct fuse_conn *fc = req->fm->fc;
  319. struct fuse_iqueue *fiq = &fc->iq;
  320. int err;
  321. if (!fc->no_interrupt) {
  322. /* Any signal may interrupt this */
  323. err = wait_event_interruptible(req->waitq,
  324. test_bit(FR_FINISHED, &req->flags));
  325. if (!err)
  326. return;
  327. set_bit(FR_INTERRUPTED, &req->flags);
  328. /* matches barrier in fuse_dev_do_read() */
  329. smp_mb__after_atomic();
  330. if (test_bit(FR_SENT, &req->flags))
  331. queue_interrupt(req);
  332. }
  333. if (!test_bit(FR_FORCE, &req->flags)) {
  334. /* Only fatal signals may interrupt this */
  335. err = wait_event_killable(req->waitq,
  336. test_bit(FR_FINISHED, &req->flags));
  337. if (!err)
  338. return;
  339. spin_lock(&fiq->lock);
  340. /* Request is not yet in userspace, bail out */
  341. if (test_bit(FR_PENDING, &req->flags)) {
  342. list_del(&req->list);
  343. spin_unlock(&fiq->lock);
  344. __fuse_put_request(req);
  345. req->out.h.error = -EINTR;
  346. return;
  347. }
  348. spin_unlock(&fiq->lock);
  349. }
  350. /*
  351. * Either request is already in userspace, or it was forced.
  352. * Wait it out.
  353. */
  354. wait_event(req->waitq, test_bit(FR_FINISHED, &req->flags));
  355. }
  356. static void __fuse_request_send(struct fuse_req *req)
  357. {
  358. struct fuse_iqueue *fiq = &req->fm->fc->iq;
  359. BUG_ON(test_bit(FR_BACKGROUND, &req->flags));
  360. spin_lock(&fiq->lock);
  361. if (!fiq->connected) {
  362. spin_unlock(&fiq->lock);
  363. req->out.h.error = -ENOTCONN;
  364. } else {
  365. req->in.h.unique = fuse_get_unique(fiq);
  366. /* acquire extra reference, since request is still needed
  367. after fuse_request_end() */
  368. __fuse_get_request(req);
  369. queue_request_and_unlock(fiq, req);
  370. request_wait_answer(req);
  371. /* Pairs with smp_wmb() in fuse_request_end() */
  372. smp_rmb();
  373. }
  374. }
  375. static void fuse_adjust_compat(struct fuse_conn *fc, struct fuse_args *args)
  376. {
  377. if (fc->minor < 4 && args->opcode == FUSE_STATFS)
  378. args->out_args[0].size = FUSE_COMPAT_STATFS_SIZE;
  379. if (fc->minor < 9) {
  380. switch (args->opcode) {
  381. case FUSE_LOOKUP:
  382. case FUSE_CREATE:
  383. case FUSE_MKNOD:
  384. case FUSE_MKDIR:
  385. case FUSE_SYMLINK:
  386. case FUSE_LINK:
  387. args->out_args[0].size = FUSE_COMPAT_ENTRY_OUT_SIZE;
  388. break;
  389. case FUSE_GETATTR:
  390. case FUSE_SETATTR:
  391. args->out_args[0].size = FUSE_COMPAT_ATTR_OUT_SIZE;
  392. break;
  393. }
  394. }
  395. if (fc->minor < 12) {
  396. switch (args->opcode) {
  397. case FUSE_CREATE:
  398. args->in_args[0].size = sizeof(struct fuse_open_in);
  399. break;
  400. case FUSE_MKNOD:
  401. args->in_args[0].size = FUSE_COMPAT_MKNOD_IN_SIZE;
  402. break;
  403. }
  404. }
  405. }
  406. static void fuse_force_creds(struct fuse_req *req)
  407. {
  408. struct fuse_conn *fc = req->fm->fc;
  409. req->in.h.uid = from_kuid_munged(fc->user_ns, current_fsuid());
  410. req->in.h.gid = from_kgid_munged(fc->user_ns, current_fsgid());
  411. req->in.h.pid = pid_nr_ns(task_pid(current), fc->pid_ns);
  412. }
  413. static void fuse_args_to_req(struct fuse_req *req, struct fuse_args *args)
  414. {
  415. req->in.h.opcode = args->opcode;
  416. req->in.h.nodeid = args->nodeid;
  417. req->args = args;
  418. if (args->end)
  419. __set_bit(FR_ASYNC, &req->flags);
  420. }
  421. ssize_t fuse_simple_request(struct fuse_mount *fm, struct fuse_args *args)
  422. {
  423. struct fuse_conn *fc = fm->fc;
  424. struct fuse_req *req;
  425. ssize_t ret;
  426. if (args->force) {
  427. atomic_inc(&fc->num_waiting);
  428. req = fuse_request_alloc(fm, GFP_KERNEL | __GFP_NOFAIL);
  429. if (!args->nocreds)
  430. fuse_force_creds(req);
  431. __set_bit(FR_WAITING, &req->flags);
  432. __set_bit(FR_FORCE, &req->flags);
  433. } else {
  434. WARN_ON(args->nocreds);
  435. req = fuse_get_req(fm, false);
  436. if (IS_ERR(req))
  437. return PTR_ERR(req);
  438. }
  439. /* Needs to be done after fuse_get_req() so that fc->minor is valid */
  440. fuse_adjust_compat(fc, args);
  441. fuse_args_to_req(req, args);
  442. if (!args->noreply)
  443. __set_bit(FR_ISREPLY, &req->flags);
  444. __fuse_request_send(req);
  445. ret = req->out.h.error;
  446. if (!ret && args->out_argvar) {
  447. BUG_ON(args->out_numargs == 0);
  448. ret = args->out_args[args->out_numargs - 1].size;
  449. }
  450. fuse_put_request(req);
  451. return ret;
  452. }
  453. static bool fuse_request_queue_background(struct fuse_req *req)
  454. {
  455. struct fuse_mount *fm = req->fm;
  456. struct fuse_conn *fc = fm->fc;
  457. bool queued = false;
  458. WARN_ON(!test_bit(FR_BACKGROUND, &req->flags));
  459. if (!test_bit(FR_WAITING, &req->flags)) {
  460. __set_bit(FR_WAITING, &req->flags);
  461. atomic_inc(&fc->num_waiting);
  462. }
  463. __set_bit(FR_ISREPLY, &req->flags);
  464. spin_lock(&fc->bg_lock);
  465. if (likely(fc->connected)) {
  466. fc->num_background++;
  467. if (fc->num_background == fc->max_background)
  468. fc->blocked = 1;
  469. if (fc->num_background == fc->congestion_threshold && fm->sb) {
  470. set_bdi_congested(fm->sb->s_bdi, BLK_RW_SYNC);
  471. set_bdi_congested(fm->sb->s_bdi, BLK_RW_ASYNC);
  472. }
  473. list_add_tail(&req->list, &fc->bg_queue);
  474. flush_bg_queue(fc);
  475. queued = true;
  476. }
  477. spin_unlock(&fc->bg_lock);
  478. return queued;
  479. }
  480. int fuse_simple_background(struct fuse_mount *fm, struct fuse_args *args,
  481. gfp_t gfp_flags)
  482. {
  483. struct fuse_req *req;
  484. if (args->force) {
  485. WARN_ON(!args->nocreds);
  486. req = fuse_request_alloc(fm, gfp_flags);
  487. if (!req)
  488. return -ENOMEM;
  489. __set_bit(FR_BACKGROUND, &req->flags);
  490. } else {
  491. WARN_ON(args->nocreds);
  492. req = fuse_get_req(fm, true);
  493. if (IS_ERR(req))
  494. return PTR_ERR(req);
  495. }
  496. fuse_args_to_req(req, args);
  497. if (!fuse_request_queue_background(req)) {
  498. fuse_put_request(req);
  499. return -ENOTCONN;
  500. }
  501. return 0;
  502. }
  503. EXPORT_SYMBOL_GPL(fuse_simple_background);
  504. static int fuse_simple_notify_reply(struct fuse_mount *fm,
  505. struct fuse_args *args, u64 unique)
  506. {
  507. struct fuse_req *req;
  508. struct fuse_iqueue *fiq = &fm->fc->iq;
  509. int err = 0;
  510. req = fuse_get_req(fm, false);
  511. if (IS_ERR(req))
  512. return PTR_ERR(req);
  513. __clear_bit(FR_ISREPLY, &req->flags);
  514. req->in.h.unique = unique;
  515. fuse_args_to_req(req, args);
  516. spin_lock(&fiq->lock);
  517. if (fiq->connected) {
  518. queue_request_and_unlock(fiq, req);
  519. } else {
  520. err = -ENODEV;
  521. spin_unlock(&fiq->lock);
  522. fuse_put_request(req);
  523. }
  524. return err;
  525. }
  526. /*
  527. * Lock the request. Up to the next unlock_request() there mustn't be
  528. * anything that could cause a page-fault. If the request was already
  529. * aborted bail out.
  530. */
  531. static int lock_request(struct fuse_req *req)
  532. {
  533. int err = 0;
  534. if (req) {
  535. spin_lock(&req->waitq.lock);
  536. if (test_bit(FR_ABORTED, &req->flags))
  537. err = -ENOENT;
  538. else
  539. set_bit(FR_LOCKED, &req->flags);
  540. spin_unlock(&req->waitq.lock);
  541. }
  542. return err;
  543. }
  544. /*
  545. * Unlock request. If it was aborted while locked, caller is responsible
  546. * for unlocking and ending the request.
  547. */
  548. static int unlock_request(struct fuse_req *req)
  549. {
  550. int err = 0;
  551. if (req) {
  552. spin_lock(&req->waitq.lock);
  553. if (test_bit(FR_ABORTED, &req->flags))
  554. err = -ENOENT;
  555. else
  556. clear_bit(FR_LOCKED, &req->flags);
  557. spin_unlock(&req->waitq.lock);
  558. }
  559. return err;
  560. }
  561. struct fuse_copy_state {
  562. int write;
  563. struct fuse_req *req;
  564. struct iov_iter *iter;
  565. struct pipe_buffer *pipebufs;
  566. struct pipe_buffer *currbuf;
  567. struct pipe_inode_info *pipe;
  568. unsigned long nr_segs;
  569. struct page *pg;
  570. unsigned len;
  571. unsigned offset;
  572. unsigned move_pages:1;
  573. };
  574. static void fuse_copy_init(struct fuse_copy_state *cs, int write,
  575. struct iov_iter *iter)
  576. {
  577. memset(cs, 0, sizeof(*cs));
  578. cs->write = write;
  579. cs->iter = iter;
  580. }
  581. /* Unmap and put previous page of userspace buffer */
  582. static void fuse_copy_finish(struct fuse_copy_state *cs)
  583. {
  584. if (cs->currbuf) {
  585. struct pipe_buffer *buf = cs->currbuf;
  586. if (cs->write)
  587. buf->len = PAGE_SIZE - cs->len;
  588. cs->currbuf = NULL;
  589. } else if (cs->pg) {
  590. if (cs->write) {
  591. flush_dcache_page(cs->pg);
  592. set_page_dirty_lock(cs->pg);
  593. }
  594. /*
  595. * The page could be GUP page(see iov_iter_get_pages in
  596. * fuse_copy_fill) so use put_user_page to release it.
  597. */
  598. put_user_page(cs->pg);
  599. }
  600. cs->pg = NULL;
  601. }
  602. /*
  603. * Get another pagefull of userspace buffer, and map it to kernel
  604. * address space, and lock request
  605. */
  606. static int fuse_copy_fill(struct fuse_copy_state *cs)
  607. {
  608. struct page *page;
  609. int err;
  610. err = unlock_request(cs->req);
  611. if (err)
  612. return err;
  613. fuse_copy_finish(cs);
  614. if (cs->pipebufs) {
  615. struct pipe_buffer *buf = cs->pipebufs;
  616. if (!cs->write) {
  617. err = pipe_buf_confirm(cs->pipe, buf);
  618. if (err)
  619. return err;
  620. BUG_ON(!cs->nr_segs);
  621. cs->currbuf = buf;
  622. cs->pg = buf->page;
  623. cs->offset = buf->offset;
  624. cs->len = buf->len;
  625. cs->pipebufs++;
  626. cs->nr_segs--;
  627. } else {
  628. if (cs->nr_segs >= cs->pipe->max_usage)
  629. return -EIO;
  630. page = alloc_page(GFP_HIGHUSER);
  631. if (!page)
  632. return -ENOMEM;
  633. buf->page = page;
  634. buf->offset = 0;
  635. buf->len = 0;
  636. cs->currbuf = buf;
  637. cs->pg = page;
  638. cs->offset = 0;
  639. cs->len = PAGE_SIZE;
  640. cs->pipebufs++;
  641. cs->nr_segs++;
  642. }
  643. } else {
  644. size_t off;
  645. err = iov_iter_get_pages(cs->iter, &page, PAGE_SIZE, 1, &off);
  646. if (err < 0)
  647. return err;
  648. BUG_ON(!err);
  649. cs->len = err;
  650. cs->offset = off;
  651. cs->pg = page;
  652. iov_iter_advance(cs->iter, err);
  653. }
  654. return lock_request(cs->req);
  655. }
  656. /* Do as much copy to/from userspace buffer as we can */
  657. static int fuse_copy_do(struct fuse_copy_state *cs, void **val, unsigned *size)
  658. {
  659. unsigned ncpy = min(*size, cs->len);
  660. if (val) {
  661. void *pgaddr = kmap_atomic(cs->pg);
  662. void *buf = pgaddr + cs->offset;
  663. if (cs->write)
  664. memcpy(buf, *val, ncpy);
  665. else
  666. memcpy(*val, buf, ncpy);
  667. kunmap_atomic(pgaddr);
  668. *val += ncpy;
  669. }
  670. *size -= ncpy;
  671. cs->len -= ncpy;
  672. cs->offset += ncpy;
  673. return ncpy;
  674. }
  675. static int fuse_check_page(struct page *page)
  676. {
  677. if (page_mapcount(page) ||
  678. page->mapping != NULL ||
  679. (page->flags & PAGE_FLAGS_CHECK_AT_PREP &
  680. ~(1 << PG_locked |
  681. 1 << PG_referenced |
  682. 1 << PG_uptodate |
  683. 1 << PG_lru |
  684. 1 << PG_active |
  685. 1 << PG_workingset |
  686. 1 << PG_reclaim |
  687. 1 << PG_waiters))) {
  688. dump_page(page, "fuse: trying to steal weird page");
  689. return 1;
  690. }
  691. return 0;
  692. }
  693. static int fuse_try_move_page(struct fuse_copy_state *cs, struct page **pagep)
  694. {
  695. int err;
  696. struct page *oldpage = *pagep;
  697. struct page *newpage;
  698. struct pipe_buffer *buf = cs->pipebufs;
  699. get_page(oldpage);
  700. err = unlock_request(cs->req);
  701. if (err)
  702. goto out_put_old;
  703. fuse_copy_finish(cs);
  704. err = pipe_buf_confirm(cs->pipe, buf);
  705. if (err)
  706. goto out_put_old;
  707. BUG_ON(!cs->nr_segs);
  708. cs->currbuf = buf;
  709. cs->len = buf->len;
  710. cs->pipebufs++;
  711. cs->nr_segs--;
  712. if (cs->len != PAGE_SIZE)
  713. goto out_fallback;
  714. if (!pipe_buf_try_steal(cs->pipe, buf))
  715. goto out_fallback;
  716. newpage = buf->page;
  717. if (!PageUptodate(newpage))
  718. SetPageUptodate(newpage);
  719. ClearPageMappedToDisk(newpage);
  720. if (fuse_check_page(newpage) != 0)
  721. goto out_fallback_unlock;
  722. /*
  723. * This is a new and locked page, it shouldn't be mapped or
  724. * have any special flags on it
  725. */
  726. if (WARN_ON(page_mapped(oldpage)))
  727. goto out_fallback_unlock;
  728. if (WARN_ON(page_has_private(oldpage)))
  729. goto out_fallback_unlock;
  730. if (WARN_ON(PageDirty(oldpage) || PageWriteback(oldpage)))
  731. goto out_fallback_unlock;
  732. if (WARN_ON(PageMlocked(oldpage)))
  733. goto out_fallback_unlock;
  734. err = replace_page_cache_page(oldpage, newpage, GFP_KERNEL);
  735. if (err) {
  736. unlock_page(newpage);
  737. goto out_put_old;
  738. }
  739. get_page(newpage);
  740. if (!(buf->flags & PIPE_BUF_FLAG_LRU))
  741. lru_cache_add(newpage);
  742. /*
  743. * Release while we have extra ref on stolen page. Otherwise
  744. * anon_pipe_buf_release() might think the page can be reused.
  745. */
  746. pipe_buf_release(cs->pipe, buf);
  747. err = 0;
  748. spin_lock(&cs->req->waitq.lock);
  749. if (test_bit(FR_ABORTED, &cs->req->flags))
  750. err = -ENOENT;
  751. else
  752. *pagep = newpage;
  753. spin_unlock(&cs->req->waitq.lock);
  754. if (err) {
  755. unlock_page(newpage);
  756. put_page(newpage);
  757. goto out_put_old;
  758. }
  759. unlock_page(oldpage);
  760. /* Drop ref for ap->pages[] array */
  761. put_page(oldpage);
  762. cs->len = 0;
  763. err = 0;
  764. out_put_old:
  765. /* Drop ref obtained in this function */
  766. put_page(oldpage);
  767. return err;
  768. out_fallback_unlock:
  769. unlock_page(newpage);
  770. out_fallback:
  771. cs->pg = buf->page;
  772. cs->offset = buf->offset;
  773. err = lock_request(cs->req);
  774. if (!err)
  775. err = 1;
  776. goto out_put_old;
  777. }
  778. static int fuse_ref_page(struct fuse_copy_state *cs, struct page *page,
  779. unsigned offset, unsigned count)
  780. {
  781. struct pipe_buffer *buf;
  782. int err;
  783. if (cs->nr_segs >= cs->pipe->max_usage)
  784. return -EIO;
  785. get_page(page);
  786. err = unlock_request(cs->req);
  787. if (err) {
  788. put_page(page);
  789. return err;
  790. }
  791. fuse_copy_finish(cs);
  792. buf = cs->pipebufs;
  793. buf->page = page;
  794. buf->offset = offset;
  795. buf->len = count;
  796. cs->pipebufs++;
  797. cs->nr_segs++;
  798. cs->len = 0;
  799. return 0;
  800. }
  801. /*
  802. * Copy a page in the request to/from the userspace buffer. Must be
  803. * done atomically
  804. */
  805. static int fuse_copy_page(struct fuse_copy_state *cs, struct page **pagep,
  806. unsigned offset, unsigned count, int zeroing)
  807. {
  808. int err;
  809. struct page *page = *pagep;
  810. if (page && zeroing && count < PAGE_SIZE)
  811. clear_highpage(page);
  812. while (count) {
  813. if (cs->write && cs->pipebufs && page) {
  814. /*
  815. * Can't control lifetime of pipe buffers, so always
  816. * copy user pages.
  817. */
  818. if (cs->req->args->user_pages) {
  819. err = fuse_copy_fill(cs);
  820. if (err)
  821. return err;
  822. } else {
  823. return fuse_ref_page(cs, page, offset, count);
  824. }
  825. } else if (!cs->len) {
  826. if (cs->move_pages && page &&
  827. offset == 0 && count == PAGE_SIZE) {
  828. err = fuse_try_move_page(cs, pagep);
  829. if (err <= 0)
  830. return err;
  831. } else {
  832. err = fuse_copy_fill(cs);
  833. if (err)
  834. return err;
  835. }
  836. }
  837. if (page) {
  838. void *mapaddr = kmap_atomic(page);
  839. void *buf = mapaddr + offset;
  840. offset += fuse_copy_do(cs, &buf, &count);
  841. kunmap_atomic(mapaddr);
  842. } else
  843. offset += fuse_copy_do(cs, NULL, &count);
  844. }
  845. if (page && !cs->write)
  846. flush_dcache_page(page);
  847. return 0;
  848. }
  849. /* Copy pages in the request to/from userspace buffer */
  850. static int fuse_copy_pages(struct fuse_copy_state *cs, unsigned nbytes,
  851. int zeroing)
  852. {
  853. unsigned i;
  854. struct fuse_req *req = cs->req;
  855. struct fuse_args_pages *ap = container_of(req->args, typeof(*ap), args);
  856. for (i = 0; i < ap->num_pages && (nbytes || zeroing); i++) {
  857. int err;
  858. unsigned int offset = ap->descs[i].offset;
  859. unsigned int count = min(nbytes, ap->descs[i].length);
  860. err = fuse_copy_page(cs, &ap->pages[i], offset, count, zeroing);
  861. if (err)
  862. return err;
  863. nbytes -= count;
  864. }
  865. return 0;
  866. }
  867. /* Copy a single argument in the request to/from userspace buffer */
  868. static int fuse_copy_one(struct fuse_copy_state *cs, void *val, unsigned size)
  869. {
  870. while (size) {
  871. if (!cs->len) {
  872. int err = fuse_copy_fill(cs);
  873. if (err)
  874. return err;
  875. }
  876. fuse_copy_do(cs, &val, &size);
  877. }
  878. return 0;
  879. }
  880. /* Copy request arguments to/from userspace buffer */
  881. static int fuse_copy_args(struct fuse_copy_state *cs, unsigned numargs,
  882. unsigned argpages, struct fuse_arg *args,
  883. int zeroing)
  884. {
  885. int err = 0;
  886. unsigned i;
  887. for (i = 0; !err && i < numargs; i++) {
  888. struct fuse_arg *arg = &args[i];
  889. if (i == numargs - 1 && argpages)
  890. err = fuse_copy_pages(cs, arg->size, zeroing);
  891. else
  892. err = fuse_copy_one(cs, arg->value, arg->size);
  893. }
  894. return err;
  895. }
  896. static int forget_pending(struct fuse_iqueue *fiq)
  897. {
  898. return fiq->forget_list_head.next != NULL;
  899. }
  900. static int request_pending(struct fuse_iqueue *fiq)
  901. {
  902. return !list_empty(&fiq->pending) || !list_empty(&fiq->interrupts) ||
  903. forget_pending(fiq);
  904. }
  905. /*
  906. * Transfer an interrupt request to userspace
  907. *
  908. * Unlike other requests this is assembled on demand, without a need
  909. * to allocate a separate fuse_req structure.
  910. *
  911. * Called with fiq->lock held, releases it
  912. */
  913. static int fuse_read_interrupt(struct fuse_iqueue *fiq,
  914. struct fuse_copy_state *cs,
  915. size_t nbytes, struct fuse_req *req)
  916. __releases(fiq->lock)
  917. {
  918. struct fuse_in_header ih;
  919. struct fuse_interrupt_in arg;
  920. unsigned reqsize = sizeof(ih) + sizeof(arg);
  921. int err;
  922. list_del_init(&req->intr_entry);
  923. memset(&ih, 0, sizeof(ih));
  924. memset(&arg, 0, sizeof(arg));
  925. ih.len = reqsize;
  926. ih.opcode = FUSE_INTERRUPT;
  927. ih.unique = (req->in.h.unique | FUSE_INT_REQ_BIT);
  928. arg.unique = req->in.h.unique;
  929. spin_unlock(&fiq->lock);
  930. if (nbytes < reqsize)
  931. return -EINVAL;
  932. err = fuse_copy_one(cs, &ih, sizeof(ih));
  933. if (!err)
  934. err = fuse_copy_one(cs, &arg, sizeof(arg));
  935. fuse_copy_finish(cs);
  936. return err ? err : reqsize;
  937. }
  938. struct fuse_forget_link *fuse_dequeue_forget(struct fuse_iqueue *fiq,
  939. unsigned int max,
  940. unsigned int *countp)
  941. {
  942. struct fuse_forget_link *head = fiq->forget_list_head.next;
  943. struct fuse_forget_link **newhead = &head;
  944. unsigned count;
  945. for (count = 0; *newhead != NULL && count < max; count++)
  946. newhead = &(*newhead)->next;
  947. fiq->forget_list_head.next = *newhead;
  948. *newhead = NULL;
  949. if (fiq->forget_list_head.next == NULL)
  950. fiq->forget_list_tail = &fiq->forget_list_head;
  951. if (countp != NULL)
  952. *countp = count;
  953. return head;
  954. }
  955. EXPORT_SYMBOL(fuse_dequeue_forget);
  956. static int fuse_read_single_forget(struct fuse_iqueue *fiq,
  957. struct fuse_copy_state *cs,
  958. size_t nbytes)
  959. __releases(fiq->lock)
  960. {
  961. int err;
  962. struct fuse_forget_link *forget = fuse_dequeue_forget(fiq, 1, NULL);
  963. struct fuse_forget_in arg = {
  964. .nlookup = forget->forget_one.nlookup,
  965. };
  966. struct fuse_in_header ih = {
  967. .opcode = FUSE_FORGET,
  968. .nodeid = forget->forget_one.nodeid,
  969. .unique = fuse_get_unique(fiq),
  970. .len = sizeof(ih) + sizeof(arg),
  971. };
  972. spin_unlock(&fiq->lock);
  973. kfree(forget);
  974. if (nbytes < ih.len)
  975. return -EINVAL;
  976. err = fuse_copy_one(cs, &ih, sizeof(ih));
  977. if (!err)
  978. err = fuse_copy_one(cs, &arg, sizeof(arg));
  979. fuse_copy_finish(cs);
  980. if (err)
  981. return err;
  982. return ih.len;
  983. }
  984. static int fuse_read_batch_forget(struct fuse_iqueue *fiq,
  985. struct fuse_copy_state *cs, size_t nbytes)
  986. __releases(fiq->lock)
  987. {
  988. int err;
  989. unsigned max_forgets;
  990. unsigned count;
  991. struct fuse_forget_link *head;
  992. struct fuse_batch_forget_in arg = { .count = 0 };
  993. struct fuse_in_header ih = {
  994. .opcode = FUSE_BATCH_FORGET,
  995. .unique = fuse_get_unique(fiq),
  996. .len = sizeof(ih) + sizeof(arg),
  997. };
  998. if (nbytes < ih.len) {
  999. spin_unlock(&fiq->lock);
  1000. return -EINVAL;
  1001. }
  1002. max_forgets = (nbytes - ih.len) / sizeof(struct fuse_forget_one);
  1003. head = fuse_dequeue_forget(fiq, max_forgets, &count);
  1004. spin_unlock(&fiq->lock);
  1005. arg.count = count;
  1006. ih.len += count * sizeof(struct fuse_forget_one);
  1007. err = fuse_copy_one(cs, &ih, sizeof(ih));
  1008. if (!err)
  1009. err = fuse_copy_one(cs, &arg, sizeof(arg));
  1010. while (head) {
  1011. struct fuse_forget_link *forget = head;
  1012. if (!err) {
  1013. err = fuse_copy_one(cs, &forget->forget_one,
  1014. sizeof(forget->forget_one));
  1015. }
  1016. head = forget->next;
  1017. kfree(forget);
  1018. }
  1019. fuse_copy_finish(cs);
  1020. if (err)
  1021. return err;
  1022. return ih.len;
  1023. }
  1024. static int fuse_read_forget(struct fuse_conn *fc, struct fuse_iqueue *fiq,
  1025. struct fuse_copy_state *cs,
  1026. size_t nbytes)
  1027. __releases(fiq->lock)
  1028. {
  1029. if (fc->minor < 16 || fiq->forget_list_head.next->next == NULL)
  1030. return fuse_read_single_forget(fiq, cs, nbytes);
  1031. else
  1032. return fuse_read_batch_forget(fiq, cs, nbytes);
  1033. }
  1034. /*
  1035. * Read a single request into the userspace filesystem's buffer. This
  1036. * function waits until a request is available, then removes it from
  1037. * the pending list and copies request data to userspace buffer. If
  1038. * no reply is needed (FORGET) or request has been aborted or there
  1039. * was an error during the copying then it's finished by calling
  1040. * fuse_request_end(). Otherwise add it to the processing list, and set
  1041. * the 'sent' flag.
  1042. */
  1043. static ssize_t fuse_dev_do_read(struct fuse_dev *fud, struct file *file,
  1044. struct fuse_copy_state *cs, size_t nbytes)
  1045. {
  1046. ssize_t err;
  1047. struct fuse_conn *fc = fud->fc;
  1048. struct fuse_iqueue *fiq = &fc->iq;
  1049. struct fuse_pqueue *fpq = &fud->pq;
  1050. struct fuse_req *req;
  1051. struct fuse_args *args;
  1052. unsigned reqsize;
  1053. unsigned int hash;
  1054. /*
  1055. * Require sane minimum read buffer - that has capacity for fixed part
  1056. * of any request header + negotiated max_write room for data.
  1057. *
  1058. * Historically libfuse reserves 4K for fixed header room, but e.g.
  1059. * GlusterFS reserves only 80 bytes
  1060. *
  1061. * = `sizeof(fuse_in_header) + sizeof(fuse_write_in)`
  1062. *
  1063. * which is the absolute minimum any sane filesystem should be using
  1064. * for header room.
  1065. */
  1066. if (nbytes < max_t(size_t, FUSE_MIN_READ_BUFFER,
  1067. sizeof(struct fuse_in_header) +
  1068. sizeof(struct fuse_write_in) +
  1069. fc->max_write))
  1070. return -EINVAL;
  1071. restart:
  1072. for (;;) {
  1073. spin_lock(&fiq->lock);
  1074. if (!fiq->connected || request_pending(fiq))
  1075. break;
  1076. spin_unlock(&fiq->lock);
  1077. if (file->f_flags & O_NONBLOCK)
  1078. return -EAGAIN;
  1079. err = wait_event_interruptible_exclusive(fiq->waitq,
  1080. !fiq->connected || request_pending(fiq));
  1081. if (err)
  1082. return err;
  1083. }
  1084. if (!fiq->connected) {
  1085. err = fc->aborted ? -ECONNABORTED : -ENODEV;
  1086. goto err_unlock;
  1087. }
  1088. if (!list_empty(&fiq->interrupts)) {
  1089. req = list_entry(fiq->interrupts.next, struct fuse_req,
  1090. intr_entry);
  1091. return fuse_read_interrupt(fiq, cs, nbytes, req);
  1092. }
  1093. if (forget_pending(fiq)) {
  1094. if (list_empty(&fiq->pending) || fiq->forget_batch-- > 0)
  1095. return fuse_read_forget(fc, fiq, cs, nbytes);
  1096. if (fiq->forget_batch <= -8)
  1097. fiq->forget_batch = 16;
  1098. }
  1099. req = list_entry(fiq->pending.next, struct fuse_req, list);
  1100. clear_bit(FR_PENDING, &req->flags);
  1101. list_del_init(&req->list);
  1102. spin_unlock(&fiq->lock);
  1103. args = req->args;
  1104. reqsize = req->in.h.len;
  1105. /* If request is too large, reply with an error and restart the read */
  1106. if (nbytes < reqsize) {
  1107. req->out.h.error = -EIO;
  1108. /* SETXATTR is special, since it may contain too large data */
  1109. if (args->opcode == FUSE_SETXATTR)
  1110. req->out.h.error = -E2BIG;
  1111. fuse_request_end(req);
  1112. goto restart;
  1113. }
  1114. spin_lock(&fpq->lock);
  1115. /*
  1116. * Must not put request on fpq->io queue after having been shut down by
  1117. * fuse_abort_conn()
  1118. */
  1119. if (!fpq->connected) {
  1120. req->out.h.error = err = -ECONNABORTED;
  1121. goto out_end;
  1122. }
  1123. list_add(&req->list, &fpq->io);
  1124. spin_unlock(&fpq->lock);
  1125. cs->req = req;
  1126. err = fuse_copy_one(cs, &req->in.h, sizeof(req->in.h));
  1127. if (!err)
  1128. err = fuse_copy_args(cs, args->in_numargs, args->in_pages,
  1129. (struct fuse_arg *) args->in_args, 0);
  1130. fuse_copy_finish(cs);
  1131. spin_lock(&fpq->lock);
  1132. clear_bit(FR_LOCKED, &req->flags);
  1133. if (!fpq->connected) {
  1134. err = fc->aborted ? -ECONNABORTED : -ENODEV;
  1135. goto out_end;
  1136. }
  1137. if (err) {
  1138. req->out.h.error = -EIO;
  1139. goto out_end;
  1140. }
  1141. if (!test_bit(FR_ISREPLY, &req->flags)) {
  1142. err = reqsize;
  1143. goto out_end;
  1144. }
  1145. hash = fuse_req_hash(req->in.h.unique);
  1146. list_move_tail(&req->list, &fpq->processing[hash]);
  1147. __fuse_get_request(req);
  1148. set_bit(FR_SENT, &req->flags);
  1149. spin_unlock(&fpq->lock);
  1150. /* matches barrier in request_wait_answer() */
  1151. smp_mb__after_atomic();
  1152. if (test_bit(FR_INTERRUPTED, &req->flags))
  1153. queue_interrupt(req);
  1154. fuse_put_request(req);
  1155. return reqsize;
  1156. out_end:
  1157. if (!test_bit(FR_PRIVATE, &req->flags))
  1158. list_del_init(&req->list);
  1159. spin_unlock(&fpq->lock);
  1160. fuse_request_end(req);
  1161. return err;
  1162. err_unlock:
  1163. spin_unlock(&fiq->lock);
  1164. return err;
  1165. }
  1166. static int fuse_dev_open(struct inode *inode, struct file *file)
  1167. {
  1168. /*
  1169. * The fuse device's file's private_data is used to hold
  1170. * the fuse_conn(ection) when it is mounted, and is used to
  1171. * keep track of whether the file has been mounted already.
  1172. */
  1173. file->private_data = NULL;
  1174. return 0;
  1175. }
  1176. static ssize_t fuse_dev_read(struct kiocb *iocb, struct iov_iter *to)
  1177. {
  1178. struct fuse_copy_state cs;
  1179. struct file *file = iocb->ki_filp;
  1180. struct fuse_dev *fud = fuse_get_dev(file);
  1181. if (!fud)
  1182. return -EPERM;
  1183. if (!iter_is_iovec(to))
  1184. return -EINVAL;
  1185. fuse_copy_init(&cs, 1, to);
  1186. return fuse_dev_do_read(fud, file, &cs, iov_iter_count(to));
  1187. }
  1188. static ssize_t fuse_dev_splice_read(struct file *in, loff_t *ppos,
  1189. struct pipe_inode_info *pipe,
  1190. size_t len, unsigned int flags)
  1191. {
  1192. int total, ret;
  1193. int page_nr = 0;
  1194. struct pipe_buffer *bufs;
  1195. struct fuse_copy_state cs;
  1196. struct fuse_dev *fud = fuse_get_dev(in);
  1197. if (!fud)
  1198. return -EPERM;
  1199. bufs = kvmalloc_array(pipe->max_usage, sizeof(struct pipe_buffer),
  1200. GFP_KERNEL);
  1201. if (!bufs)
  1202. return -ENOMEM;
  1203. fuse_copy_init(&cs, 1, NULL);
  1204. cs.pipebufs = bufs;
  1205. cs.pipe = pipe;
  1206. ret = fuse_dev_do_read(fud, in, &cs, len);
  1207. if (ret < 0)
  1208. goto out;
  1209. if (pipe_occupancy(pipe->head, pipe->tail) + cs.nr_segs > pipe->max_usage) {
  1210. ret = -EIO;
  1211. goto out;
  1212. }
  1213. for (ret = total = 0; page_nr < cs.nr_segs; total += ret) {
  1214. /*
  1215. * Need to be careful about this. Having buf->ops in module
  1216. * code can Oops if the buffer persists after module unload.
  1217. */
  1218. bufs[page_nr].ops = &nosteal_pipe_buf_ops;
  1219. bufs[page_nr].flags = 0;
  1220. ret = add_to_pipe(pipe, &bufs[page_nr++]);
  1221. if (unlikely(ret < 0))
  1222. break;
  1223. }
  1224. if (total)
  1225. ret = total;
  1226. out:
  1227. for (; page_nr < cs.nr_segs; page_nr++)
  1228. put_page(bufs[page_nr].page);
  1229. kvfree(bufs);
  1230. return ret;
  1231. }
  1232. static int fuse_notify_poll(struct fuse_conn *fc, unsigned int size,
  1233. struct fuse_copy_state *cs)
  1234. {
  1235. struct fuse_notify_poll_wakeup_out outarg;
  1236. int err = -EINVAL;
  1237. if (size != sizeof(outarg))
  1238. goto err;
  1239. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1240. if (err)
  1241. goto err;
  1242. fuse_copy_finish(cs);
  1243. return fuse_notify_poll_wakeup(fc, &outarg);
  1244. err:
  1245. fuse_copy_finish(cs);
  1246. return err;
  1247. }
  1248. static int fuse_notify_inval_inode(struct fuse_conn *fc, unsigned int size,
  1249. struct fuse_copy_state *cs)
  1250. {
  1251. struct fuse_notify_inval_inode_out outarg;
  1252. int err = -EINVAL;
  1253. if (size != sizeof(outarg))
  1254. goto err;
  1255. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1256. if (err)
  1257. goto err;
  1258. fuse_copy_finish(cs);
  1259. down_read(&fc->killsb);
  1260. err = fuse_reverse_inval_inode(fc, outarg.ino,
  1261. outarg.off, outarg.len);
  1262. up_read(&fc->killsb);
  1263. return err;
  1264. err:
  1265. fuse_copy_finish(cs);
  1266. return err;
  1267. }
  1268. static int fuse_notify_inval_entry(struct fuse_conn *fc, unsigned int size,
  1269. struct fuse_copy_state *cs)
  1270. {
  1271. struct fuse_notify_inval_entry_out outarg;
  1272. int err = -ENOMEM;
  1273. char *buf;
  1274. struct qstr name;
  1275. buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
  1276. if (!buf)
  1277. goto err;
  1278. err = -EINVAL;
  1279. if (size < sizeof(outarg))
  1280. goto err;
  1281. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1282. if (err)
  1283. goto err;
  1284. err = -ENAMETOOLONG;
  1285. if (outarg.namelen > FUSE_NAME_MAX)
  1286. goto err;
  1287. err = -EINVAL;
  1288. if (size != sizeof(outarg) + outarg.namelen + 1)
  1289. goto err;
  1290. name.name = buf;
  1291. name.len = outarg.namelen;
  1292. err = fuse_copy_one(cs, buf, outarg.namelen + 1);
  1293. if (err)
  1294. goto err;
  1295. fuse_copy_finish(cs);
  1296. buf[outarg.namelen] = 0;
  1297. down_read(&fc->killsb);
  1298. err = fuse_reverse_inval_entry(fc, outarg.parent, 0, &name);
  1299. up_read(&fc->killsb);
  1300. kfree(buf);
  1301. return err;
  1302. err:
  1303. kfree(buf);
  1304. fuse_copy_finish(cs);
  1305. return err;
  1306. }
  1307. static int fuse_notify_delete(struct fuse_conn *fc, unsigned int size,
  1308. struct fuse_copy_state *cs)
  1309. {
  1310. struct fuse_notify_delete_out outarg;
  1311. int err = -ENOMEM;
  1312. char *buf;
  1313. struct qstr name;
  1314. buf = kzalloc(FUSE_NAME_MAX + 1, GFP_KERNEL);
  1315. if (!buf)
  1316. goto err;
  1317. err = -EINVAL;
  1318. if (size < sizeof(outarg))
  1319. goto err;
  1320. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1321. if (err)
  1322. goto err;
  1323. err = -ENAMETOOLONG;
  1324. if (outarg.namelen > FUSE_NAME_MAX)
  1325. goto err;
  1326. err = -EINVAL;
  1327. if (size != sizeof(outarg) + outarg.namelen + 1)
  1328. goto err;
  1329. name.name = buf;
  1330. name.len = outarg.namelen;
  1331. err = fuse_copy_one(cs, buf, outarg.namelen + 1);
  1332. if (err)
  1333. goto err;
  1334. fuse_copy_finish(cs);
  1335. buf[outarg.namelen] = 0;
  1336. down_read(&fc->killsb);
  1337. err = fuse_reverse_inval_entry(fc, outarg.parent, outarg.child, &name);
  1338. up_read(&fc->killsb);
  1339. kfree(buf);
  1340. return err;
  1341. err:
  1342. kfree(buf);
  1343. fuse_copy_finish(cs);
  1344. return err;
  1345. }
  1346. static int fuse_notify_store(struct fuse_conn *fc, unsigned int size,
  1347. struct fuse_copy_state *cs)
  1348. {
  1349. struct fuse_notify_store_out outarg;
  1350. struct inode *inode;
  1351. struct address_space *mapping;
  1352. u64 nodeid;
  1353. int err;
  1354. pgoff_t index;
  1355. unsigned int offset;
  1356. unsigned int num;
  1357. loff_t file_size;
  1358. loff_t end;
  1359. err = -EINVAL;
  1360. if (size < sizeof(outarg))
  1361. goto out_finish;
  1362. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1363. if (err)
  1364. goto out_finish;
  1365. err = -EINVAL;
  1366. if (size - sizeof(outarg) != outarg.size)
  1367. goto out_finish;
  1368. nodeid = outarg.nodeid;
  1369. down_read(&fc->killsb);
  1370. err = -ENOENT;
  1371. inode = fuse_ilookup(fc, nodeid, NULL);
  1372. if (!inode)
  1373. goto out_up_killsb;
  1374. mapping = inode->i_mapping;
  1375. index = outarg.offset >> PAGE_SHIFT;
  1376. offset = outarg.offset & ~PAGE_MASK;
  1377. file_size = i_size_read(inode);
  1378. end = outarg.offset + outarg.size;
  1379. if (end > file_size) {
  1380. file_size = end;
  1381. fuse_write_update_size(inode, file_size);
  1382. }
  1383. num = outarg.size;
  1384. while (num) {
  1385. struct page *page;
  1386. unsigned int this_num;
  1387. err = -ENOMEM;
  1388. page = find_or_create_page(mapping, index,
  1389. mapping_gfp_mask(mapping));
  1390. if (!page)
  1391. goto out_iput;
  1392. this_num = min_t(unsigned, num, PAGE_SIZE - offset);
  1393. err = fuse_copy_page(cs, &page, offset, this_num, 0);
  1394. if (!err && offset == 0 &&
  1395. (this_num == PAGE_SIZE || file_size == end))
  1396. SetPageUptodate(page);
  1397. unlock_page(page);
  1398. put_page(page);
  1399. if (err)
  1400. goto out_iput;
  1401. num -= this_num;
  1402. offset = 0;
  1403. index++;
  1404. }
  1405. err = 0;
  1406. out_iput:
  1407. iput(inode);
  1408. out_up_killsb:
  1409. up_read(&fc->killsb);
  1410. out_finish:
  1411. fuse_copy_finish(cs);
  1412. return err;
  1413. }
  1414. struct fuse_retrieve_args {
  1415. struct fuse_args_pages ap;
  1416. struct fuse_notify_retrieve_in inarg;
  1417. };
  1418. static void fuse_retrieve_end(struct fuse_mount *fm, struct fuse_args *args,
  1419. int error)
  1420. {
  1421. struct fuse_retrieve_args *ra =
  1422. container_of(args, typeof(*ra), ap.args);
  1423. release_pages(ra->ap.pages, ra->ap.num_pages);
  1424. kfree(ra);
  1425. }
  1426. static int fuse_retrieve(struct fuse_mount *fm, struct inode *inode,
  1427. struct fuse_notify_retrieve_out *outarg)
  1428. {
  1429. int err;
  1430. struct address_space *mapping = inode->i_mapping;
  1431. pgoff_t index;
  1432. loff_t file_size;
  1433. unsigned int num;
  1434. unsigned int offset;
  1435. size_t total_len = 0;
  1436. unsigned int num_pages;
  1437. struct fuse_conn *fc = fm->fc;
  1438. struct fuse_retrieve_args *ra;
  1439. size_t args_size = sizeof(*ra);
  1440. struct fuse_args_pages *ap;
  1441. struct fuse_args *args;
  1442. offset = outarg->offset & ~PAGE_MASK;
  1443. file_size = i_size_read(inode);
  1444. num = min(outarg->size, fc->max_write);
  1445. if (outarg->offset > file_size)
  1446. num = 0;
  1447. else if (outarg->offset + num > file_size)
  1448. num = file_size - outarg->offset;
  1449. num_pages = (num + offset + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1450. num_pages = min(num_pages, fc->max_pages);
  1451. args_size += num_pages * (sizeof(ap->pages[0]) + sizeof(ap->descs[0]));
  1452. ra = kzalloc(args_size, GFP_KERNEL);
  1453. if (!ra)
  1454. return -ENOMEM;
  1455. ap = &ra->ap;
  1456. ap->pages = (void *) (ra + 1);
  1457. ap->descs = (void *) (ap->pages + num_pages);
  1458. args = &ap->args;
  1459. args->nodeid = outarg->nodeid;
  1460. args->opcode = FUSE_NOTIFY_REPLY;
  1461. args->in_numargs = 2;
  1462. args->in_pages = true;
  1463. args->end = fuse_retrieve_end;
  1464. index = outarg->offset >> PAGE_SHIFT;
  1465. while (num && ap->num_pages < num_pages) {
  1466. struct page *page;
  1467. unsigned int this_num;
  1468. page = find_get_page(mapping, index);
  1469. if (!page)
  1470. break;
  1471. this_num = min_t(unsigned, num, PAGE_SIZE - offset);
  1472. ap->pages[ap->num_pages] = page;
  1473. ap->descs[ap->num_pages].offset = offset;
  1474. ap->descs[ap->num_pages].length = this_num;
  1475. ap->num_pages++;
  1476. offset = 0;
  1477. num -= this_num;
  1478. total_len += this_num;
  1479. index++;
  1480. }
  1481. ra->inarg.offset = outarg->offset;
  1482. ra->inarg.size = total_len;
  1483. args->in_args[0].size = sizeof(ra->inarg);
  1484. args->in_args[0].value = &ra->inarg;
  1485. args->in_args[1].size = total_len;
  1486. err = fuse_simple_notify_reply(fm, args, outarg->notify_unique);
  1487. if (err)
  1488. fuse_retrieve_end(fm, args, err);
  1489. return err;
  1490. }
  1491. static int fuse_notify_retrieve(struct fuse_conn *fc, unsigned int size,
  1492. struct fuse_copy_state *cs)
  1493. {
  1494. struct fuse_notify_retrieve_out outarg;
  1495. struct fuse_mount *fm;
  1496. struct inode *inode;
  1497. u64 nodeid;
  1498. int err;
  1499. err = -EINVAL;
  1500. if (size != sizeof(outarg))
  1501. goto copy_finish;
  1502. err = fuse_copy_one(cs, &outarg, sizeof(outarg));
  1503. if (err)
  1504. goto copy_finish;
  1505. fuse_copy_finish(cs);
  1506. down_read(&fc->killsb);
  1507. err = -ENOENT;
  1508. nodeid = outarg.nodeid;
  1509. inode = fuse_ilookup(fc, nodeid, &fm);
  1510. if (inode) {
  1511. err = fuse_retrieve(fm, inode, &outarg);
  1512. iput(inode);
  1513. }
  1514. up_read(&fc->killsb);
  1515. return err;
  1516. copy_finish:
  1517. fuse_copy_finish(cs);
  1518. return err;
  1519. }
  1520. static int fuse_notify(struct fuse_conn *fc, enum fuse_notify_code code,
  1521. unsigned int size, struct fuse_copy_state *cs)
  1522. {
  1523. /* Don't try to move pages (yet) */
  1524. cs->move_pages = 0;
  1525. switch (code) {
  1526. case FUSE_NOTIFY_POLL:
  1527. return fuse_notify_poll(fc, size, cs);
  1528. case FUSE_NOTIFY_INVAL_INODE:
  1529. return fuse_notify_inval_inode(fc, size, cs);
  1530. case FUSE_NOTIFY_INVAL_ENTRY:
  1531. return fuse_notify_inval_entry(fc, size, cs);
  1532. case FUSE_NOTIFY_STORE:
  1533. return fuse_notify_store(fc, size, cs);
  1534. case FUSE_NOTIFY_RETRIEVE:
  1535. return fuse_notify_retrieve(fc, size, cs);
  1536. case FUSE_NOTIFY_DELETE:
  1537. return fuse_notify_delete(fc, size, cs);
  1538. default:
  1539. fuse_copy_finish(cs);
  1540. return -EINVAL;
  1541. }
  1542. }
  1543. /* Look up request on processing list by unique ID */
  1544. static struct fuse_req *request_find(struct fuse_pqueue *fpq, u64 unique)
  1545. {
  1546. unsigned int hash = fuse_req_hash(unique);
  1547. struct fuse_req *req;
  1548. list_for_each_entry(req, &fpq->processing[hash], list) {
  1549. if (req->in.h.unique == unique)
  1550. return req;
  1551. }
  1552. return NULL;
  1553. }
  1554. static int copy_out_args(struct fuse_copy_state *cs, struct fuse_args *args,
  1555. unsigned nbytes)
  1556. {
  1557. unsigned reqsize = sizeof(struct fuse_out_header);
  1558. reqsize += fuse_len_args(args->out_numargs, args->out_args);
  1559. if (reqsize < nbytes || (reqsize > nbytes && !args->out_argvar))
  1560. return -EINVAL;
  1561. else if (reqsize > nbytes) {
  1562. struct fuse_arg *lastarg = &args->out_args[args->out_numargs-1];
  1563. unsigned diffsize = reqsize - nbytes;
  1564. if (diffsize > lastarg->size)
  1565. return -EINVAL;
  1566. lastarg->size -= diffsize;
  1567. }
  1568. return fuse_copy_args(cs, args->out_numargs, args->out_pages,
  1569. args->out_args, args->page_zeroing);
  1570. }
  1571. /*
  1572. * Write a single reply to a request. First the header is copied from
  1573. * the write buffer. The request is then searched on the processing
  1574. * list by the unique ID found in the header. If found, then remove
  1575. * it from the list and copy the rest of the buffer to the request.
  1576. * The request is finished by calling fuse_request_end().
  1577. */
  1578. static ssize_t fuse_dev_do_write(struct fuse_dev *fud,
  1579. struct fuse_copy_state *cs, size_t nbytes)
  1580. {
  1581. int err;
  1582. struct fuse_conn *fc = fud->fc;
  1583. struct fuse_pqueue *fpq = &fud->pq;
  1584. struct fuse_req *req;
  1585. struct fuse_out_header oh;
  1586. err = -EINVAL;
  1587. if (nbytes < sizeof(struct fuse_out_header))
  1588. goto out;
  1589. err = fuse_copy_one(cs, &oh, sizeof(oh));
  1590. if (err)
  1591. goto copy_finish;
  1592. err = -EINVAL;
  1593. if (oh.len != nbytes)
  1594. goto copy_finish;
  1595. /*
  1596. * Zero oh.unique indicates unsolicited notification message
  1597. * and error contains notification code.
  1598. */
  1599. if (!oh.unique) {
  1600. err = fuse_notify(fc, oh.error, nbytes - sizeof(oh), cs);
  1601. goto out;
  1602. }
  1603. err = -EINVAL;
  1604. if (oh.error <= -512 || oh.error > 0)
  1605. goto copy_finish;
  1606. spin_lock(&fpq->lock);
  1607. req = NULL;
  1608. if (fpq->connected)
  1609. req = request_find(fpq, oh.unique & ~FUSE_INT_REQ_BIT);
  1610. err = -ENOENT;
  1611. if (!req) {
  1612. spin_unlock(&fpq->lock);
  1613. goto copy_finish;
  1614. }
  1615. /* Is it an interrupt reply ID? */
  1616. if (oh.unique & FUSE_INT_REQ_BIT) {
  1617. __fuse_get_request(req);
  1618. spin_unlock(&fpq->lock);
  1619. err = 0;
  1620. if (nbytes != sizeof(struct fuse_out_header))
  1621. err = -EINVAL;
  1622. else if (oh.error == -ENOSYS)
  1623. fc->no_interrupt = 1;
  1624. else if (oh.error == -EAGAIN)
  1625. err = queue_interrupt(req);
  1626. fuse_put_request(req);
  1627. goto copy_finish;
  1628. }
  1629. clear_bit(FR_SENT, &req->flags);
  1630. list_move(&req->list, &fpq->io);
  1631. req->out.h = oh;
  1632. set_bit(FR_LOCKED, &req->flags);
  1633. spin_unlock(&fpq->lock);
  1634. cs->req = req;
  1635. if (!req->args->page_replace)
  1636. cs->move_pages = 0;
  1637. if (oh.error)
  1638. err = nbytes != sizeof(oh) ? -EINVAL : 0;
  1639. else
  1640. err = copy_out_args(cs, req->args, nbytes);
  1641. fuse_copy_finish(cs);
  1642. if (!err && req->in.h.opcode == FUSE_CANONICAL_PATH) {
  1643. char *path = (char *)req->args->out_args[0].value;
  1644. path[req->args->out_args[0].size - 1] = 0;
  1645. req->out.h.error =
  1646. kern_path(path, 0, req->args->canonical_path);
  1647. }
  1648. spin_lock(&fpq->lock);
  1649. clear_bit(FR_LOCKED, &req->flags);
  1650. if (!fpq->connected)
  1651. err = -ENOENT;
  1652. else if (err)
  1653. req->out.h.error = -EIO;
  1654. if (!test_bit(FR_PRIVATE, &req->flags))
  1655. list_del_init(&req->list);
  1656. spin_unlock(&fpq->lock);
  1657. fuse_request_end(req);
  1658. out:
  1659. return err ? err : nbytes;
  1660. copy_finish:
  1661. fuse_copy_finish(cs);
  1662. goto out;
  1663. }
  1664. static ssize_t fuse_dev_write(struct kiocb *iocb, struct iov_iter *from)
  1665. {
  1666. struct fuse_copy_state cs;
  1667. struct fuse_dev *fud = fuse_get_dev(iocb->ki_filp);
  1668. if (!fud)
  1669. return -EPERM;
  1670. if (!iter_is_iovec(from))
  1671. return -EINVAL;
  1672. fuse_copy_init(&cs, 0, from);
  1673. return fuse_dev_do_write(fud, &cs, iov_iter_count(from));
  1674. }
  1675. static ssize_t fuse_dev_splice_write(struct pipe_inode_info *pipe,
  1676. struct file *out, loff_t *ppos,
  1677. size_t len, unsigned int flags)
  1678. {
  1679. unsigned int head, tail, mask, count;
  1680. unsigned nbuf;
  1681. unsigned idx;
  1682. struct pipe_buffer *bufs;
  1683. struct fuse_copy_state cs;
  1684. struct fuse_dev *fud;
  1685. size_t rem;
  1686. ssize_t ret;
  1687. fud = fuse_get_dev(out);
  1688. if (!fud)
  1689. return -EPERM;
  1690. pipe_lock(pipe);
  1691. head = pipe->head;
  1692. tail = pipe->tail;
  1693. mask = pipe->ring_size - 1;
  1694. count = head - tail;
  1695. bufs = kvmalloc_array(count, sizeof(struct pipe_buffer), GFP_KERNEL);
  1696. if (!bufs) {
  1697. pipe_unlock(pipe);
  1698. return -ENOMEM;
  1699. }
  1700. nbuf = 0;
  1701. rem = 0;
  1702. for (idx = tail; idx != head && rem < len; idx++)
  1703. rem += pipe->bufs[idx & mask].len;
  1704. ret = -EINVAL;
  1705. if (rem < len)
  1706. goto out_free;
  1707. rem = len;
  1708. while (rem) {
  1709. struct pipe_buffer *ibuf;
  1710. struct pipe_buffer *obuf;
  1711. if (WARN_ON(nbuf >= count || tail == head))
  1712. goto out_free;
  1713. ibuf = &pipe->bufs[tail & mask];
  1714. obuf = &bufs[nbuf];
  1715. if (rem >= ibuf->len) {
  1716. *obuf = *ibuf;
  1717. ibuf->ops = NULL;
  1718. tail++;
  1719. pipe->tail = tail;
  1720. } else {
  1721. if (!pipe_buf_get(pipe, ibuf))
  1722. goto out_free;
  1723. *obuf = *ibuf;
  1724. obuf->flags &= ~PIPE_BUF_FLAG_GIFT;
  1725. obuf->len = rem;
  1726. ibuf->offset += obuf->len;
  1727. ibuf->len -= obuf->len;
  1728. }
  1729. nbuf++;
  1730. rem -= obuf->len;
  1731. }
  1732. pipe_unlock(pipe);
  1733. fuse_copy_init(&cs, 0, NULL);
  1734. cs.pipebufs = bufs;
  1735. cs.nr_segs = nbuf;
  1736. cs.pipe = pipe;
  1737. if (flags & SPLICE_F_MOVE)
  1738. cs.move_pages = 1;
  1739. ret = fuse_dev_do_write(fud, &cs, len);
  1740. pipe_lock(pipe);
  1741. out_free:
  1742. for (idx = 0; idx < nbuf; idx++) {
  1743. struct pipe_buffer *buf = &bufs[idx];
  1744. if (buf->ops)
  1745. pipe_buf_release(pipe, buf);
  1746. }
  1747. pipe_unlock(pipe);
  1748. kvfree(bufs);
  1749. return ret;
  1750. }
  1751. static __poll_t fuse_dev_poll(struct file *file, poll_table *wait)
  1752. {
  1753. __poll_t mask = EPOLLOUT | EPOLLWRNORM;
  1754. struct fuse_iqueue *fiq;
  1755. struct fuse_dev *fud = fuse_get_dev(file);
  1756. if (!fud)
  1757. return EPOLLERR;
  1758. fiq = &fud->fc->iq;
  1759. poll_wait(file, &fiq->waitq, wait);
  1760. spin_lock(&fiq->lock);
  1761. if (!fiq->connected)
  1762. mask = EPOLLERR;
  1763. else if (request_pending(fiq))
  1764. mask |= EPOLLIN | EPOLLRDNORM;
  1765. spin_unlock(&fiq->lock);
  1766. return mask;
  1767. }
  1768. /* Abort all requests on the given list (pending or processing) */
  1769. static void end_requests(struct list_head *head)
  1770. {
  1771. while (!list_empty(head)) {
  1772. struct fuse_req *req;
  1773. req = list_entry(head->next, struct fuse_req, list);
  1774. req->out.h.error = -ECONNABORTED;
  1775. clear_bit(FR_SENT, &req->flags);
  1776. list_del_init(&req->list);
  1777. fuse_request_end(req);
  1778. }
  1779. }
  1780. static void end_polls(struct fuse_conn *fc)
  1781. {
  1782. struct rb_node *p;
  1783. p = rb_first(&fc->polled_files);
  1784. while (p) {
  1785. struct fuse_file *ff;
  1786. ff = rb_entry(p, struct fuse_file, polled_node);
  1787. wake_up_interruptible_all(&ff->poll_wait);
  1788. p = rb_next(p);
  1789. }
  1790. }
  1791. /*
  1792. * Abort all requests.
  1793. *
  1794. * Emergency exit in case of a malicious or accidental deadlock, or just a hung
  1795. * filesystem.
  1796. *
  1797. * The same effect is usually achievable through killing the filesystem daemon
  1798. * and all users of the filesystem. The exception is the combination of an
  1799. * asynchronous request and the tricky deadlock (see
  1800. * Documentation/filesystems/fuse.rst).
  1801. *
  1802. * Aborting requests under I/O goes as follows: 1: Separate out unlocked
  1803. * requests, they should be finished off immediately. Locked requests will be
  1804. * finished after unlock; see unlock_request(). 2: Finish off the unlocked
  1805. * requests. It is possible that some request will finish before we can. This
  1806. * is OK, the request will in that case be removed from the list before we touch
  1807. * it.
  1808. */
  1809. void fuse_abort_conn(struct fuse_conn *fc)
  1810. {
  1811. struct fuse_iqueue *fiq = &fc->iq;
  1812. spin_lock(&fc->lock);
  1813. if (fc->connected) {
  1814. struct fuse_dev *fud;
  1815. struct fuse_req *req, *next;
  1816. LIST_HEAD(to_end);
  1817. unsigned int i;
  1818. /* Background queuing checks fc->connected under bg_lock */
  1819. spin_lock(&fc->bg_lock);
  1820. fc->connected = 0;
  1821. spin_unlock(&fc->bg_lock);
  1822. fuse_set_initialized(fc);
  1823. list_for_each_entry(fud, &fc->devices, entry) {
  1824. struct fuse_pqueue *fpq = &fud->pq;
  1825. spin_lock(&fpq->lock);
  1826. fpq->connected = 0;
  1827. list_for_each_entry_safe(req, next, &fpq->io, list) {
  1828. req->out.h.error = -ECONNABORTED;
  1829. spin_lock(&req->waitq.lock);
  1830. set_bit(FR_ABORTED, &req->flags);
  1831. if (!test_bit(FR_LOCKED, &req->flags)) {
  1832. set_bit(FR_PRIVATE, &req->flags);
  1833. __fuse_get_request(req);
  1834. list_move(&req->list, &to_end);
  1835. }
  1836. spin_unlock(&req->waitq.lock);
  1837. }
  1838. for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
  1839. list_splice_tail_init(&fpq->processing[i],
  1840. &to_end);
  1841. spin_unlock(&fpq->lock);
  1842. }
  1843. spin_lock(&fc->bg_lock);
  1844. fc->blocked = 0;
  1845. fc->max_background = UINT_MAX;
  1846. flush_bg_queue(fc);
  1847. spin_unlock(&fc->bg_lock);
  1848. spin_lock(&fiq->lock);
  1849. fiq->connected = 0;
  1850. list_for_each_entry(req, &fiq->pending, list)
  1851. clear_bit(FR_PENDING, &req->flags);
  1852. list_splice_tail_init(&fiq->pending, &to_end);
  1853. while (forget_pending(fiq))
  1854. kfree(fuse_dequeue_forget(fiq, 1, NULL));
  1855. wake_up_all(&fiq->waitq);
  1856. spin_unlock(&fiq->lock);
  1857. kill_fasync(&fiq->fasync, SIGIO, POLL_IN);
  1858. end_polls(fc);
  1859. wake_up_all(&fc->blocked_waitq);
  1860. spin_unlock(&fc->lock);
  1861. end_requests(&to_end);
  1862. } else {
  1863. spin_unlock(&fc->lock);
  1864. }
  1865. }
  1866. EXPORT_SYMBOL_GPL(fuse_abort_conn);
  1867. void fuse_wait_aborted(struct fuse_conn *fc)
  1868. {
  1869. /* matches implicit memory barrier in fuse_drop_waiting() */
  1870. smp_mb();
  1871. wait_event(fc->blocked_waitq, atomic_read(&fc->num_waiting) == 0);
  1872. }
  1873. int fuse_dev_release(struct inode *inode, struct file *file)
  1874. {
  1875. struct fuse_dev *fud = fuse_get_dev(file);
  1876. if (fud) {
  1877. struct fuse_conn *fc = fud->fc;
  1878. struct fuse_pqueue *fpq = &fud->pq;
  1879. LIST_HEAD(to_end);
  1880. unsigned int i;
  1881. spin_lock(&fpq->lock);
  1882. WARN_ON(!list_empty(&fpq->io));
  1883. for (i = 0; i < FUSE_PQ_HASH_SIZE; i++)
  1884. list_splice_init(&fpq->processing[i], &to_end);
  1885. spin_unlock(&fpq->lock);
  1886. end_requests(&to_end);
  1887. /* Are we the last open device? */
  1888. if (atomic_dec_and_test(&fc->dev_count)) {
  1889. WARN_ON(fc->iq.fasync != NULL);
  1890. fuse_abort_conn(fc);
  1891. }
  1892. fuse_dev_free(fud);
  1893. }
  1894. return 0;
  1895. }
  1896. EXPORT_SYMBOL_GPL(fuse_dev_release);
  1897. static int fuse_dev_fasync(int fd, struct file *file, int on)
  1898. {
  1899. struct fuse_dev *fud = fuse_get_dev(file);
  1900. if (!fud)
  1901. return -EPERM;
  1902. /* No locking - fasync_helper does its own locking */
  1903. return fasync_helper(fd, file, on, &fud->fc->iq.fasync);
  1904. }
  1905. static int fuse_device_clone(struct fuse_conn *fc, struct file *new)
  1906. {
  1907. struct fuse_dev *fud;
  1908. if (new->private_data)
  1909. return -EINVAL;
  1910. fud = fuse_dev_alloc_install(fc);
  1911. if (!fud)
  1912. return -ENOMEM;
  1913. new->private_data = fud;
  1914. atomic_inc(&fc->dev_count);
  1915. return 0;
  1916. }
  1917. static long fuse_dev_ioctl(struct file *file, unsigned int cmd,
  1918. unsigned long arg)
  1919. {
  1920. int res;
  1921. int oldfd;
  1922. struct fuse_dev *fud = NULL;
  1923. switch (cmd) {
  1924. case FUSE_DEV_IOC_CLONE:
  1925. res = -EFAULT;
  1926. if (!get_user(oldfd, (__u32 __user *)arg)) {
  1927. struct file *old = fget(oldfd);
  1928. res = -EINVAL;
  1929. if (old) {
  1930. /*
  1931. * Check against file->f_op because CUSE
  1932. * uses the same ioctl handler.
  1933. */
  1934. if (old->f_op == file->f_op &&
  1935. old->f_cred->user_ns ==
  1936. file->f_cred->user_ns)
  1937. fud = fuse_get_dev(old);
  1938. if (fud) {
  1939. mutex_lock(&fuse_mutex);
  1940. res = fuse_device_clone(fud->fc, file);
  1941. mutex_unlock(&fuse_mutex);
  1942. }
  1943. fput(old);
  1944. }
  1945. }
  1946. break;
  1947. case FUSE_DEV_IOC_PASSTHROUGH_OPEN:
  1948. res = -EFAULT;
  1949. if (!get_user(oldfd, (__u32 __user *)arg)) {
  1950. res = -EINVAL;
  1951. fud = fuse_get_dev(file);
  1952. if (fud)
  1953. res = fuse_passthrough_open(fud, oldfd);
  1954. }
  1955. break;
  1956. default:
  1957. res = -ENOTTY;
  1958. break;
  1959. }
  1960. return res;
  1961. }
  1962. const struct file_operations fuse_dev_operations = {
  1963. .owner = THIS_MODULE,
  1964. .open = fuse_dev_open,
  1965. .llseek = no_llseek,
  1966. .read_iter = fuse_dev_read,
  1967. .splice_read = fuse_dev_splice_read,
  1968. .write_iter = fuse_dev_write,
  1969. .splice_write = fuse_dev_splice_write,
  1970. .poll = fuse_dev_poll,
  1971. .release = fuse_dev_release,
  1972. .fasync = fuse_dev_fasync,
  1973. .unlocked_ioctl = fuse_dev_ioctl,
  1974. .compat_ioctl = compat_ptr_ioctl,
  1975. };
  1976. EXPORT_SYMBOL_GPL(fuse_dev_operations);
  1977. static struct miscdevice fuse_miscdevice = {
  1978. .minor = FUSE_MINOR,
  1979. .name = "fuse",
  1980. .fops = &fuse_dev_operations,
  1981. };
  1982. int __init fuse_dev_init(void)
  1983. {
  1984. int err = -ENOMEM;
  1985. fuse_req_cachep = kmem_cache_create("fuse_request",
  1986. sizeof(struct fuse_req),
  1987. 0, 0, NULL);
  1988. if (!fuse_req_cachep)
  1989. goto out;
  1990. err = misc_register(&fuse_miscdevice);
  1991. if (err)
  1992. goto out_cache_clean;
  1993. return 0;
  1994. out_cache_clean:
  1995. kmem_cache_destroy(fuse_req_cachep);
  1996. out:
  1997. return err;
  1998. }
  1999. void fuse_dev_cleanup(void)
  2000. {
  2001. misc_deregister(&fuse_miscdevice);
  2002. kmem_cache_destroy(fuse_req_cachep);
  2003. }