file.c 66 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516
  1. // SPDX-License-Identifier: GPL-2.0
  2. #include <linux/ceph/ceph_debug.h>
  3. #include <linux/ceph/striper.h>
  4. #include <linux/module.h>
  5. #include <linux/sched.h>
  6. #include <linux/slab.h>
  7. #include <linux/file.h>
  8. #include <linux/mount.h>
  9. #include <linux/namei.h>
  10. #include <linux/writeback.h>
  11. #include <linux/falloc.h>
  12. #include <linux/iversion.h>
  13. #include <linux/ktime.h>
  14. #include "super.h"
  15. #include "mds_client.h"
  16. #include "cache.h"
  17. #include "io.h"
  18. #include "metric.h"
  19. static __le32 ceph_flags_sys2wire(u32 flags)
  20. {
  21. u32 wire_flags = 0;
  22. switch (flags & O_ACCMODE) {
  23. case O_RDONLY:
  24. wire_flags |= CEPH_O_RDONLY;
  25. break;
  26. case O_WRONLY:
  27. wire_flags |= CEPH_O_WRONLY;
  28. break;
  29. case O_RDWR:
  30. wire_flags |= CEPH_O_RDWR;
  31. break;
  32. }
  33. flags &= ~O_ACCMODE;
  34. #define ceph_sys2wire(a) if (flags & a) { wire_flags |= CEPH_##a; flags &= ~a; }
  35. ceph_sys2wire(O_CREAT);
  36. ceph_sys2wire(O_EXCL);
  37. ceph_sys2wire(O_TRUNC);
  38. ceph_sys2wire(O_DIRECTORY);
  39. ceph_sys2wire(O_NOFOLLOW);
  40. #undef ceph_sys2wire
  41. if (flags)
  42. dout("unused open flags: %x\n", flags);
  43. return cpu_to_le32(wire_flags);
  44. }
  45. /*
  46. * Ceph file operations
  47. *
  48. * Implement basic open/close functionality, and implement
  49. * read/write.
  50. *
  51. * We implement three modes of file I/O:
  52. * - buffered uses the generic_file_aio_{read,write} helpers
  53. *
  54. * - synchronous is used when there is multi-client read/write
  55. * sharing, avoids the page cache, and synchronously waits for an
  56. * ack from the OSD.
  57. *
  58. * - direct io takes the variant of the sync path that references
  59. * user pages directly.
  60. *
  61. * fsync() flushes and waits on dirty pages, but just queues metadata
  62. * for writeback: since the MDS can recover size and mtime there is no
  63. * need to wait for MDS acknowledgement.
  64. */
  65. /*
  66. * How many pages to get in one call to iov_iter_get_pages(). This
  67. * determines the size of the on-stack array used as a buffer.
  68. */
  69. #define ITER_GET_BVECS_PAGES 64
  70. static ssize_t __iter_get_bvecs(struct iov_iter *iter, size_t maxsize,
  71. struct bio_vec *bvecs)
  72. {
  73. size_t size = 0;
  74. int bvec_idx = 0;
  75. if (maxsize > iov_iter_count(iter))
  76. maxsize = iov_iter_count(iter);
  77. while (size < maxsize) {
  78. struct page *pages[ITER_GET_BVECS_PAGES];
  79. ssize_t bytes;
  80. size_t start;
  81. int idx = 0;
  82. bytes = iov_iter_get_pages(iter, pages, maxsize - size,
  83. ITER_GET_BVECS_PAGES, &start);
  84. if (bytes < 0)
  85. return size ?: bytes;
  86. iov_iter_advance(iter, bytes);
  87. size += bytes;
  88. for ( ; bytes; idx++, bvec_idx++) {
  89. struct bio_vec bv = {
  90. .bv_page = pages[idx],
  91. .bv_len = min_t(int, bytes, PAGE_SIZE - start),
  92. .bv_offset = start,
  93. };
  94. bvecs[bvec_idx] = bv;
  95. bytes -= bv.bv_len;
  96. start = 0;
  97. }
  98. }
  99. return size;
  100. }
  101. /*
  102. * iov_iter_get_pages() only considers one iov_iter segment, no matter
  103. * what maxsize or maxpages are given. For ITER_BVEC that is a single
  104. * page.
  105. *
  106. * Attempt to get up to @maxsize bytes worth of pages from @iter.
  107. * Return the number of bytes in the created bio_vec array, or an error.
  108. */
  109. static ssize_t iter_get_bvecs_alloc(struct iov_iter *iter, size_t maxsize,
  110. struct bio_vec **bvecs, int *num_bvecs)
  111. {
  112. struct bio_vec *bv;
  113. size_t orig_count = iov_iter_count(iter);
  114. ssize_t bytes;
  115. int npages;
  116. iov_iter_truncate(iter, maxsize);
  117. npages = iov_iter_npages(iter, INT_MAX);
  118. iov_iter_reexpand(iter, orig_count);
  119. /*
  120. * __iter_get_bvecs() may populate only part of the array -- zero it
  121. * out.
  122. */
  123. bv = kvmalloc_array(npages, sizeof(*bv), GFP_KERNEL | __GFP_ZERO);
  124. if (!bv)
  125. return -ENOMEM;
  126. bytes = __iter_get_bvecs(iter, maxsize, bv);
  127. if (bytes < 0) {
  128. /*
  129. * No pages were pinned -- just free the array.
  130. */
  131. kvfree(bv);
  132. return bytes;
  133. }
  134. *bvecs = bv;
  135. *num_bvecs = npages;
  136. return bytes;
  137. }
  138. static void put_bvecs(struct bio_vec *bvecs, int num_bvecs, bool should_dirty)
  139. {
  140. int i;
  141. for (i = 0; i < num_bvecs; i++) {
  142. if (bvecs[i].bv_page) {
  143. if (should_dirty)
  144. set_page_dirty_lock(bvecs[i].bv_page);
  145. put_page(bvecs[i].bv_page);
  146. }
  147. }
  148. kvfree(bvecs);
  149. }
  150. /*
  151. * Prepare an open request. Preallocate ceph_cap to avoid an
  152. * inopportune ENOMEM later.
  153. */
  154. static struct ceph_mds_request *
  155. prepare_open_request(struct super_block *sb, int flags, int create_mode)
  156. {
  157. struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(sb);
  158. struct ceph_mds_request *req;
  159. int want_auth = USE_ANY_MDS;
  160. int op = (flags & O_CREAT) ? CEPH_MDS_OP_CREATE : CEPH_MDS_OP_OPEN;
  161. if (flags & (O_WRONLY|O_RDWR|O_CREAT|O_TRUNC))
  162. want_auth = USE_AUTH_MDS;
  163. req = ceph_mdsc_create_request(mdsc, op, want_auth);
  164. if (IS_ERR(req))
  165. goto out;
  166. req->r_fmode = ceph_flags_to_mode(flags);
  167. req->r_args.open.flags = ceph_flags_sys2wire(flags);
  168. req->r_args.open.mode = cpu_to_le32(create_mode);
  169. out:
  170. return req;
  171. }
  172. static int ceph_init_file_info(struct inode *inode, struct file *file,
  173. int fmode, bool isdir)
  174. {
  175. struct ceph_inode_info *ci = ceph_inode(inode);
  176. struct ceph_file_info *fi;
  177. dout("%s %p %p 0%o (%s)\n", __func__, inode, file,
  178. inode->i_mode, isdir ? "dir" : "regular");
  179. BUG_ON(inode->i_fop->release != ceph_release);
  180. if (isdir) {
  181. struct ceph_dir_file_info *dfi =
  182. kmem_cache_zalloc(ceph_dir_file_cachep, GFP_KERNEL);
  183. if (!dfi)
  184. return -ENOMEM;
  185. file->private_data = dfi;
  186. fi = &dfi->file_info;
  187. dfi->next_offset = 2;
  188. dfi->readdir_cache_idx = -1;
  189. } else {
  190. fi = kmem_cache_zalloc(ceph_file_cachep, GFP_KERNEL);
  191. if (!fi)
  192. return -ENOMEM;
  193. file->private_data = fi;
  194. }
  195. ceph_get_fmode(ci, fmode, 1);
  196. fi->fmode = fmode;
  197. spin_lock_init(&fi->rw_contexts_lock);
  198. INIT_LIST_HEAD(&fi->rw_contexts);
  199. fi->filp_gen = READ_ONCE(ceph_inode_to_client(inode)->filp_gen);
  200. return 0;
  201. }
  202. /*
  203. * initialize private struct file data.
  204. * if we fail, clean up by dropping fmode reference on the ceph_inode
  205. */
  206. static int ceph_init_file(struct inode *inode, struct file *file, int fmode)
  207. {
  208. int ret = 0;
  209. switch (inode->i_mode & S_IFMT) {
  210. case S_IFREG:
  211. ceph_fscache_register_inode_cookie(inode);
  212. ceph_fscache_file_set_cookie(inode, file);
  213. fallthrough;
  214. case S_IFDIR:
  215. ret = ceph_init_file_info(inode, file, fmode,
  216. S_ISDIR(inode->i_mode));
  217. break;
  218. case S_IFLNK:
  219. dout("init_file %p %p 0%o (symlink)\n", inode, file,
  220. inode->i_mode);
  221. break;
  222. default:
  223. dout("init_file %p %p 0%o (special)\n", inode, file,
  224. inode->i_mode);
  225. /*
  226. * we need to drop the open ref now, since we don't
  227. * have .release set to ceph_release.
  228. */
  229. BUG_ON(inode->i_fop->release == ceph_release);
  230. /* call the proper open fop */
  231. ret = inode->i_fop->open(inode, file);
  232. }
  233. return ret;
  234. }
  235. /*
  236. * try renew caps after session gets killed.
  237. */
  238. int ceph_renew_caps(struct inode *inode, int fmode)
  239. {
  240. struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(inode->i_sb);
  241. struct ceph_inode_info *ci = ceph_inode(inode);
  242. struct ceph_mds_request *req;
  243. int err, flags, wanted;
  244. spin_lock(&ci->i_ceph_lock);
  245. __ceph_touch_fmode(ci, mdsc, fmode);
  246. wanted = __ceph_caps_file_wanted(ci);
  247. if (__ceph_is_any_real_caps(ci) &&
  248. (!(wanted & CEPH_CAP_ANY_WR) || ci->i_auth_cap)) {
  249. int issued = __ceph_caps_issued(ci, NULL);
  250. spin_unlock(&ci->i_ceph_lock);
  251. dout("renew caps %p want %s issued %s updating mds_wanted\n",
  252. inode, ceph_cap_string(wanted), ceph_cap_string(issued));
  253. ceph_check_caps(ci, 0, NULL);
  254. return 0;
  255. }
  256. spin_unlock(&ci->i_ceph_lock);
  257. flags = 0;
  258. if ((wanted & CEPH_CAP_FILE_RD) && (wanted & CEPH_CAP_FILE_WR))
  259. flags = O_RDWR;
  260. else if (wanted & CEPH_CAP_FILE_RD)
  261. flags = O_RDONLY;
  262. else if (wanted & CEPH_CAP_FILE_WR)
  263. flags = O_WRONLY;
  264. #ifdef O_LAZY
  265. if (wanted & CEPH_CAP_FILE_LAZYIO)
  266. flags |= O_LAZY;
  267. #endif
  268. req = prepare_open_request(inode->i_sb, flags, 0);
  269. if (IS_ERR(req)) {
  270. err = PTR_ERR(req);
  271. goto out;
  272. }
  273. req->r_inode = inode;
  274. ihold(inode);
  275. req->r_num_caps = 1;
  276. err = ceph_mdsc_do_request(mdsc, NULL, req);
  277. ceph_mdsc_put_request(req);
  278. out:
  279. dout("renew caps %p open result=%d\n", inode, err);
  280. return err < 0 ? err : 0;
  281. }
  282. /*
  283. * If we already have the requisite capabilities, we can satisfy
  284. * the open request locally (no need to request new caps from the
  285. * MDS). We do, however, need to inform the MDS (asynchronously)
  286. * if our wanted caps set expands.
  287. */
  288. int ceph_open(struct inode *inode, struct file *file)
  289. {
  290. struct ceph_inode_info *ci = ceph_inode(inode);
  291. struct ceph_fs_client *fsc = ceph_sb_to_client(inode->i_sb);
  292. struct ceph_mds_client *mdsc = fsc->mdsc;
  293. struct ceph_mds_request *req;
  294. struct ceph_file_info *fi = file->private_data;
  295. int err;
  296. int flags, fmode, wanted;
  297. if (fi) {
  298. dout("open file %p is already opened\n", file);
  299. return 0;
  300. }
  301. /* filter out O_CREAT|O_EXCL; vfs did that already. yuck. */
  302. flags = file->f_flags & ~(O_CREAT|O_EXCL);
  303. if (S_ISDIR(inode->i_mode))
  304. flags = O_DIRECTORY; /* mds likes to know */
  305. dout("open inode %p ino %llx.%llx file %p flags %d (%d)\n", inode,
  306. ceph_vinop(inode), file, flags, file->f_flags);
  307. fmode = ceph_flags_to_mode(flags);
  308. wanted = ceph_caps_for_mode(fmode);
  309. /* snapped files are read-only */
  310. if (ceph_snap(inode) != CEPH_NOSNAP && (file->f_mode & FMODE_WRITE))
  311. return -EROFS;
  312. /* trivially open snapdir */
  313. if (ceph_snap(inode) == CEPH_SNAPDIR) {
  314. return ceph_init_file(inode, file, fmode);
  315. }
  316. /*
  317. * No need to block if we have caps on the auth MDS (for
  318. * write) or any MDS (for read). Update wanted set
  319. * asynchronously.
  320. */
  321. spin_lock(&ci->i_ceph_lock);
  322. if (__ceph_is_any_real_caps(ci) &&
  323. (((fmode & CEPH_FILE_MODE_WR) == 0) || ci->i_auth_cap)) {
  324. int mds_wanted = __ceph_caps_mds_wanted(ci, true);
  325. int issued = __ceph_caps_issued(ci, NULL);
  326. dout("open %p fmode %d want %s issued %s using existing\n",
  327. inode, fmode, ceph_cap_string(wanted),
  328. ceph_cap_string(issued));
  329. __ceph_touch_fmode(ci, mdsc, fmode);
  330. spin_unlock(&ci->i_ceph_lock);
  331. /* adjust wanted? */
  332. if ((issued & wanted) != wanted &&
  333. (mds_wanted & wanted) != wanted &&
  334. ceph_snap(inode) != CEPH_SNAPDIR)
  335. ceph_check_caps(ci, 0, NULL);
  336. return ceph_init_file(inode, file, fmode);
  337. } else if (ceph_snap(inode) != CEPH_NOSNAP &&
  338. (ci->i_snap_caps & wanted) == wanted) {
  339. __ceph_touch_fmode(ci, mdsc, fmode);
  340. spin_unlock(&ci->i_ceph_lock);
  341. return ceph_init_file(inode, file, fmode);
  342. }
  343. spin_unlock(&ci->i_ceph_lock);
  344. dout("open fmode %d wants %s\n", fmode, ceph_cap_string(wanted));
  345. req = prepare_open_request(inode->i_sb, flags, 0);
  346. if (IS_ERR(req)) {
  347. err = PTR_ERR(req);
  348. goto out;
  349. }
  350. req->r_inode = inode;
  351. ihold(inode);
  352. req->r_num_caps = 1;
  353. err = ceph_mdsc_do_request(mdsc, NULL, req);
  354. if (!err)
  355. err = ceph_init_file(inode, file, req->r_fmode);
  356. ceph_mdsc_put_request(req);
  357. dout("open result=%d on %llx.%llx\n", err, ceph_vinop(inode));
  358. out:
  359. return err;
  360. }
  361. /* Clone the layout from a synchronous create, if the dir now has Dc caps */
  362. static void
  363. cache_file_layout(struct inode *dst, struct inode *src)
  364. {
  365. struct ceph_inode_info *cdst = ceph_inode(dst);
  366. struct ceph_inode_info *csrc = ceph_inode(src);
  367. spin_lock(&cdst->i_ceph_lock);
  368. if ((__ceph_caps_issued(cdst, NULL) & CEPH_CAP_DIR_CREATE) &&
  369. !ceph_file_layout_is_valid(&cdst->i_cached_layout)) {
  370. memcpy(&cdst->i_cached_layout, &csrc->i_layout,
  371. sizeof(cdst->i_cached_layout));
  372. rcu_assign_pointer(cdst->i_cached_layout.pool_ns,
  373. ceph_try_get_string(csrc->i_layout.pool_ns));
  374. }
  375. spin_unlock(&cdst->i_ceph_lock);
  376. }
  377. /*
  378. * Try to set up an async create. We need caps, a file layout, and inode number,
  379. * and either a lease on the dentry or complete dir info. If any of those
  380. * criteria are not satisfied, then return false and the caller can go
  381. * synchronous.
  382. */
  383. static int try_prep_async_create(struct inode *dir, struct dentry *dentry,
  384. struct ceph_file_layout *lo, u64 *pino)
  385. {
  386. struct ceph_inode_info *ci = ceph_inode(dir);
  387. struct ceph_dentry_info *di = ceph_dentry(dentry);
  388. int got = 0, want = CEPH_CAP_FILE_EXCL | CEPH_CAP_DIR_CREATE;
  389. u64 ino;
  390. spin_lock(&ci->i_ceph_lock);
  391. /* No auth cap means no chance for Dc caps */
  392. if (!ci->i_auth_cap)
  393. goto no_async;
  394. /* Any delegated inos? */
  395. if (xa_empty(&ci->i_auth_cap->session->s_delegated_inos))
  396. goto no_async;
  397. if (!ceph_file_layout_is_valid(&ci->i_cached_layout))
  398. goto no_async;
  399. if ((__ceph_caps_issued(ci, NULL) & want) != want)
  400. goto no_async;
  401. if (d_in_lookup(dentry)) {
  402. if (!__ceph_dir_is_complete(ci))
  403. goto no_async;
  404. spin_lock(&dentry->d_lock);
  405. di->lease_shared_gen = atomic_read(&ci->i_shared_gen);
  406. spin_unlock(&dentry->d_lock);
  407. } else if (atomic_read(&ci->i_shared_gen) !=
  408. READ_ONCE(di->lease_shared_gen)) {
  409. goto no_async;
  410. }
  411. ino = ceph_get_deleg_ino(ci->i_auth_cap->session);
  412. if (!ino)
  413. goto no_async;
  414. *pino = ino;
  415. ceph_take_cap_refs(ci, want, false);
  416. memcpy(lo, &ci->i_cached_layout, sizeof(*lo));
  417. rcu_assign_pointer(lo->pool_ns,
  418. ceph_try_get_string(ci->i_cached_layout.pool_ns));
  419. got = want;
  420. no_async:
  421. spin_unlock(&ci->i_ceph_lock);
  422. return got;
  423. }
  424. static void restore_deleg_ino(struct inode *dir, u64 ino)
  425. {
  426. struct ceph_inode_info *ci = ceph_inode(dir);
  427. struct ceph_mds_session *s = NULL;
  428. spin_lock(&ci->i_ceph_lock);
  429. if (ci->i_auth_cap)
  430. s = ceph_get_mds_session(ci->i_auth_cap->session);
  431. spin_unlock(&ci->i_ceph_lock);
  432. if (s) {
  433. int err = ceph_restore_deleg_ino(s, ino);
  434. if (err)
  435. pr_warn("ceph: unable to restore delegated ino 0x%llx to session: %d\n",
  436. ino, err);
  437. ceph_put_mds_session(s);
  438. }
  439. }
  440. static void ceph_async_create_cb(struct ceph_mds_client *mdsc,
  441. struct ceph_mds_request *req)
  442. {
  443. int result = req->r_err ? req->r_err :
  444. le32_to_cpu(req->r_reply_info.head->result);
  445. if (result == -EJUKEBOX)
  446. goto out;
  447. mapping_set_error(req->r_parent->i_mapping, result);
  448. if (result) {
  449. struct dentry *dentry = req->r_dentry;
  450. int pathlen = 0;
  451. u64 base = 0;
  452. char *path = ceph_mdsc_build_path(req->r_dentry, &pathlen,
  453. &base, 0);
  454. ceph_dir_clear_complete(req->r_parent);
  455. if (!d_unhashed(dentry))
  456. d_drop(dentry);
  457. /* FIXME: start returning I/O errors on all accesses? */
  458. pr_warn("ceph: async create failure path=(%llx)%s result=%d!\n",
  459. base, IS_ERR(path) ? "<<bad>>" : path, result);
  460. ceph_mdsc_free_path(path, pathlen);
  461. }
  462. if (req->r_target_inode) {
  463. struct ceph_inode_info *ci = ceph_inode(req->r_target_inode);
  464. u64 ino = ceph_vino(req->r_target_inode).ino;
  465. if (req->r_deleg_ino != ino)
  466. pr_warn("%s: inode number mismatch! err=%d deleg_ino=0x%llx target=0x%llx\n",
  467. __func__, req->r_err, req->r_deleg_ino, ino);
  468. mapping_set_error(req->r_target_inode->i_mapping, result);
  469. spin_lock(&ci->i_ceph_lock);
  470. if (ci->i_ceph_flags & CEPH_I_ASYNC_CREATE) {
  471. ci->i_ceph_flags &= ~CEPH_I_ASYNC_CREATE;
  472. wake_up_bit(&ci->i_ceph_flags, CEPH_ASYNC_CREATE_BIT);
  473. }
  474. ceph_kick_flushing_inode_caps(req->r_session, ci);
  475. spin_unlock(&ci->i_ceph_lock);
  476. } else {
  477. pr_warn("%s: no req->r_target_inode for 0x%llx\n", __func__,
  478. req->r_deleg_ino);
  479. }
  480. out:
  481. ceph_mdsc_release_dir_caps(req);
  482. }
  483. static int ceph_finish_async_create(struct inode *dir, struct dentry *dentry,
  484. struct file *file, umode_t mode,
  485. struct ceph_mds_request *req,
  486. struct ceph_acl_sec_ctx *as_ctx,
  487. struct ceph_file_layout *lo)
  488. {
  489. int ret;
  490. char xattr_buf[4];
  491. struct ceph_mds_reply_inode in = { };
  492. struct ceph_mds_reply_info_in iinfo = { .in = &in };
  493. struct ceph_inode_info *ci = ceph_inode(dir);
  494. struct inode *inode;
  495. struct timespec64 now;
  496. struct ceph_string *pool_ns;
  497. struct ceph_mds_client *mdsc = ceph_sb_to_mdsc(dir->i_sb);
  498. struct ceph_vino vino = { .ino = req->r_deleg_ino,
  499. .snap = CEPH_NOSNAP };
  500. ktime_get_real_ts64(&now);
  501. inode = ceph_get_inode(dentry->d_sb, vino);
  502. if (IS_ERR(inode))
  503. return PTR_ERR(inode);
  504. iinfo.inline_version = CEPH_INLINE_NONE;
  505. iinfo.change_attr = 1;
  506. ceph_encode_timespec64(&iinfo.btime, &now);
  507. iinfo.xattr_len = ARRAY_SIZE(xattr_buf);
  508. iinfo.xattr_data = xattr_buf;
  509. memset(iinfo.xattr_data, 0, iinfo.xattr_len);
  510. in.ino = cpu_to_le64(vino.ino);
  511. in.snapid = cpu_to_le64(CEPH_NOSNAP);
  512. in.version = cpu_to_le64(1); // ???
  513. in.cap.caps = in.cap.wanted = cpu_to_le32(CEPH_CAP_ALL_FILE);
  514. in.cap.cap_id = cpu_to_le64(1);
  515. in.cap.realm = cpu_to_le64(ci->i_snap_realm->ino);
  516. in.cap.flags = CEPH_CAP_FLAG_AUTH;
  517. in.ctime = in.mtime = in.atime = iinfo.btime;
  518. in.truncate_seq = cpu_to_le32(1);
  519. in.truncate_size = cpu_to_le64(-1ULL);
  520. in.xattr_version = cpu_to_le64(1);
  521. in.uid = cpu_to_le32(from_kuid(&init_user_ns, current_fsuid()));
  522. if (dir->i_mode & S_ISGID) {
  523. in.gid = cpu_to_le32(from_kgid(&init_user_ns, dir->i_gid));
  524. /* Directories always inherit the setgid bit. */
  525. if (S_ISDIR(mode))
  526. mode |= S_ISGID;
  527. else if ((mode & (S_ISGID | S_IXGRP)) == (S_ISGID | S_IXGRP) &&
  528. !in_group_p(dir->i_gid) &&
  529. !capable_wrt_inode_uidgid(dir, CAP_FSETID))
  530. mode &= ~S_ISGID;
  531. } else {
  532. in.gid = cpu_to_le32(from_kgid(&init_user_ns, current_fsgid()));
  533. }
  534. in.mode = cpu_to_le32((u32)mode);
  535. in.nlink = cpu_to_le32(1);
  536. in.max_size = cpu_to_le64(lo->stripe_unit);
  537. ceph_file_layout_to_legacy(lo, &in.layout);
  538. /* lo is private, so pool_ns can't change */
  539. pool_ns = rcu_dereference_raw(lo->pool_ns);
  540. if (pool_ns) {
  541. iinfo.pool_ns_len = pool_ns->len;
  542. iinfo.pool_ns_data = pool_ns->str;
  543. }
  544. down_read(&mdsc->snap_rwsem);
  545. ret = ceph_fill_inode(inode, NULL, &iinfo, NULL, req->r_session,
  546. req->r_fmode, NULL);
  547. up_read(&mdsc->snap_rwsem);
  548. if (ret) {
  549. dout("%s failed to fill inode: %d\n", __func__, ret);
  550. ceph_dir_clear_complete(dir);
  551. if (!d_unhashed(dentry))
  552. d_drop(dentry);
  553. if (inode->i_state & I_NEW)
  554. discard_new_inode(inode);
  555. } else {
  556. struct dentry *dn;
  557. dout("%s d_adding new inode 0x%llx to 0x%llx/%s\n", __func__,
  558. vino.ino, ceph_ino(dir), dentry->d_name.name);
  559. ceph_dir_clear_ordered(dir);
  560. ceph_init_inode_acls(inode, as_ctx);
  561. if (inode->i_state & I_NEW) {
  562. /*
  563. * If it's not I_NEW, then someone created this before
  564. * we got here. Assume the server is aware of it at
  565. * that point and don't worry about setting
  566. * CEPH_I_ASYNC_CREATE.
  567. */
  568. ceph_inode(inode)->i_ceph_flags = CEPH_I_ASYNC_CREATE;
  569. unlock_new_inode(inode);
  570. }
  571. if (d_in_lookup(dentry) || d_really_is_negative(dentry)) {
  572. if (!d_unhashed(dentry))
  573. d_drop(dentry);
  574. dn = d_splice_alias(inode, dentry);
  575. WARN_ON_ONCE(dn && dn != dentry);
  576. }
  577. file->f_mode |= FMODE_CREATED;
  578. ret = finish_open(file, dentry, ceph_open);
  579. }
  580. return ret;
  581. }
  582. /*
  583. * Do a lookup + open with a single request. If we get a non-existent
  584. * file or symlink, return 1 so the VFS can retry.
  585. */
  586. int ceph_atomic_open(struct inode *dir, struct dentry *dentry,
  587. struct file *file, unsigned flags, umode_t mode)
  588. {
  589. struct ceph_fs_client *fsc = ceph_sb_to_client(dir->i_sb);
  590. struct ceph_mds_client *mdsc = fsc->mdsc;
  591. struct ceph_mds_request *req;
  592. struct dentry *dn;
  593. struct ceph_acl_sec_ctx as_ctx = {};
  594. bool try_async = ceph_test_mount_opt(fsc, ASYNC_DIROPS);
  595. int mask;
  596. int err;
  597. dout("atomic_open %p dentry %p '%pd' %s flags %d mode 0%o\n",
  598. dir, dentry, dentry,
  599. d_unhashed(dentry) ? "unhashed" : "hashed", flags, mode);
  600. if (dentry->d_name.len > NAME_MAX)
  601. return -ENAMETOOLONG;
  602. if (flags & O_CREAT) {
  603. if (ceph_quota_is_max_files_exceeded(dir))
  604. return -EDQUOT;
  605. err = ceph_pre_init_acls(dir, &mode, &as_ctx);
  606. if (err < 0)
  607. return err;
  608. err = ceph_security_init_secctx(dentry, mode, &as_ctx);
  609. if (err < 0)
  610. goto out_ctx;
  611. } else if (!d_in_lookup(dentry)) {
  612. /* If it's not being looked up, it's negative */
  613. return -ENOENT;
  614. }
  615. retry:
  616. /* do the open */
  617. req = prepare_open_request(dir->i_sb, flags, mode);
  618. if (IS_ERR(req)) {
  619. err = PTR_ERR(req);
  620. goto out_ctx;
  621. }
  622. req->r_dentry = dget(dentry);
  623. req->r_num_caps = 2;
  624. mask = CEPH_STAT_CAP_INODE | CEPH_CAP_AUTH_SHARED;
  625. if (ceph_security_xattr_wanted(dir))
  626. mask |= CEPH_CAP_XATTR_SHARED;
  627. req->r_args.open.mask = cpu_to_le32(mask);
  628. req->r_parent = dir;
  629. if (flags & O_CREAT) {
  630. struct ceph_file_layout lo;
  631. req->r_dentry_drop = CEPH_CAP_FILE_SHARED | CEPH_CAP_AUTH_EXCL;
  632. req->r_dentry_unless = CEPH_CAP_FILE_EXCL;
  633. if (as_ctx.pagelist) {
  634. req->r_pagelist = as_ctx.pagelist;
  635. as_ctx.pagelist = NULL;
  636. }
  637. if (try_async &&
  638. (req->r_dir_caps =
  639. try_prep_async_create(dir, dentry, &lo,
  640. &req->r_deleg_ino))) {
  641. set_bit(CEPH_MDS_R_ASYNC, &req->r_req_flags);
  642. req->r_args.open.flags |= cpu_to_le32(CEPH_O_EXCL);
  643. req->r_callback = ceph_async_create_cb;
  644. err = ceph_mdsc_submit_request(mdsc, dir, req);
  645. if (!err) {
  646. err = ceph_finish_async_create(dir, dentry,
  647. file, mode, req,
  648. &as_ctx, &lo);
  649. } else if (err == -EJUKEBOX) {
  650. restore_deleg_ino(dir, req->r_deleg_ino);
  651. ceph_mdsc_put_request(req);
  652. try_async = false;
  653. ceph_put_string(rcu_dereference_raw(lo.pool_ns));
  654. goto retry;
  655. }
  656. ceph_put_string(rcu_dereference_raw(lo.pool_ns));
  657. goto out_req;
  658. }
  659. }
  660. set_bit(CEPH_MDS_R_PARENT_LOCKED, &req->r_req_flags);
  661. err = ceph_mdsc_do_request(mdsc,
  662. (flags & (O_CREAT|O_TRUNC)) ? dir : NULL,
  663. req);
  664. err = ceph_handle_snapdir(req, dentry, err);
  665. if (err)
  666. goto out_req;
  667. if ((flags & O_CREAT) && !req->r_reply_info.head->is_dentry)
  668. err = ceph_handle_notrace_create(dir, dentry);
  669. if (d_in_lookup(dentry)) {
  670. dn = ceph_finish_lookup(req, dentry, err);
  671. if (IS_ERR(dn))
  672. err = PTR_ERR(dn);
  673. } else {
  674. /* we were given a hashed negative dentry */
  675. dn = NULL;
  676. }
  677. if (err)
  678. goto out_req;
  679. if (dn || d_really_is_negative(dentry) || d_is_symlink(dentry)) {
  680. /* make vfs retry on splice, ENOENT, or symlink */
  681. dout("atomic_open finish_no_open on dn %p\n", dn);
  682. err = finish_no_open(file, dn);
  683. } else {
  684. dout("atomic_open finish_open on dn %p\n", dn);
  685. if (req->r_op == CEPH_MDS_OP_CREATE && req->r_reply_info.has_create_ino) {
  686. struct inode *newino = d_inode(dentry);
  687. cache_file_layout(dir, newino);
  688. ceph_init_inode_acls(newino, &as_ctx);
  689. file->f_mode |= FMODE_CREATED;
  690. }
  691. err = finish_open(file, dentry, ceph_open);
  692. }
  693. out_req:
  694. ceph_mdsc_put_request(req);
  695. out_ctx:
  696. ceph_release_acl_sec_ctx(&as_ctx);
  697. dout("atomic_open result=%d\n", err);
  698. return err;
  699. }
  700. int ceph_release(struct inode *inode, struct file *file)
  701. {
  702. struct ceph_inode_info *ci = ceph_inode(inode);
  703. if (S_ISDIR(inode->i_mode)) {
  704. struct ceph_dir_file_info *dfi = file->private_data;
  705. dout("release inode %p dir file %p\n", inode, file);
  706. WARN_ON(!list_empty(&dfi->file_info.rw_contexts));
  707. ceph_put_fmode(ci, dfi->file_info.fmode, 1);
  708. if (dfi->last_readdir)
  709. ceph_mdsc_put_request(dfi->last_readdir);
  710. kfree(dfi->last_name);
  711. kfree(dfi->dir_info);
  712. kmem_cache_free(ceph_dir_file_cachep, dfi);
  713. } else {
  714. struct ceph_file_info *fi = file->private_data;
  715. dout("release inode %p regular file %p\n", inode, file);
  716. WARN_ON(!list_empty(&fi->rw_contexts));
  717. ceph_put_fmode(ci, fi->fmode, 1);
  718. kmem_cache_free(ceph_file_cachep, fi);
  719. }
  720. /* wake up anyone waiting for caps on this inode */
  721. wake_up_all(&ci->i_cap_wq);
  722. return 0;
  723. }
  724. enum {
  725. HAVE_RETRIED = 1,
  726. CHECK_EOF = 2,
  727. READ_INLINE = 3,
  728. };
  729. /*
  730. * Completely synchronous read and write methods. Direct from __user
  731. * buffer to osd, or directly to user pages (if O_DIRECT).
  732. *
  733. * If the read spans object boundary, just do multiple reads. (That's not
  734. * atomic, but good enough for now.)
  735. *
  736. * If we get a short result from the OSD, check against i_size; we need to
  737. * only return a short read to the caller if we hit EOF.
  738. */
  739. static ssize_t ceph_sync_read(struct kiocb *iocb, struct iov_iter *to,
  740. int *retry_op)
  741. {
  742. struct file *file = iocb->ki_filp;
  743. struct inode *inode = file_inode(file);
  744. struct ceph_inode_info *ci = ceph_inode(inode);
  745. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  746. struct ceph_osd_client *osdc = &fsc->client->osdc;
  747. ssize_t ret;
  748. u64 off = iocb->ki_pos;
  749. u64 len = iov_iter_count(to);
  750. dout("sync_read on file %p %llu~%u %s\n", file, off, (unsigned)len,
  751. (file->f_flags & O_DIRECT) ? "O_DIRECT" : "");
  752. if (!len)
  753. return 0;
  754. /*
  755. * flush any page cache pages in this range. this
  756. * will make concurrent normal and sync io slow,
  757. * but it will at least behave sensibly when they are
  758. * in sequence.
  759. */
  760. ret = filemap_write_and_wait_range(inode->i_mapping,
  761. off, off + len - 1);
  762. if (ret < 0)
  763. return ret;
  764. ret = 0;
  765. while ((len = iov_iter_count(to)) > 0) {
  766. struct ceph_osd_request *req;
  767. struct page **pages;
  768. int num_pages;
  769. size_t page_off;
  770. u64 i_size;
  771. bool more;
  772. int idx;
  773. size_t left;
  774. req = ceph_osdc_new_request(osdc, &ci->i_layout,
  775. ci->i_vino, off, &len, 0, 1,
  776. CEPH_OSD_OP_READ, CEPH_OSD_FLAG_READ,
  777. NULL, ci->i_truncate_seq,
  778. ci->i_truncate_size, false);
  779. if (IS_ERR(req)) {
  780. ret = PTR_ERR(req);
  781. break;
  782. }
  783. more = len < iov_iter_count(to);
  784. num_pages = calc_pages_for(off, len);
  785. page_off = off & ~PAGE_MASK;
  786. pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
  787. if (IS_ERR(pages)) {
  788. ceph_osdc_put_request(req);
  789. ret = PTR_ERR(pages);
  790. break;
  791. }
  792. osd_req_op_extent_osd_data_pages(req, 0, pages, len, page_off,
  793. false, false);
  794. ret = ceph_osdc_start_request(osdc, req, false);
  795. if (!ret)
  796. ret = ceph_osdc_wait_request(osdc, req);
  797. ceph_update_read_latency(&fsc->mdsc->metric,
  798. req->r_start_latency,
  799. req->r_end_latency,
  800. ret);
  801. ceph_osdc_put_request(req);
  802. i_size = i_size_read(inode);
  803. dout("sync_read %llu~%llu got %zd i_size %llu%s\n",
  804. off, len, ret, i_size, (more ? " MORE" : ""));
  805. if (ret == -ENOENT)
  806. ret = 0;
  807. if (ret >= 0 && ret < len && (off + ret < i_size)) {
  808. int zlen = min(len - ret, i_size - off - ret);
  809. int zoff = page_off + ret;
  810. dout("sync_read zero gap %llu~%llu\n",
  811. off + ret, off + ret + zlen);
  812. ceph_zero_page_vector_range(zoff, zlen, pages);
  813. ret += zlen;
  814. }
  815. idx = 0;
  816. left = ret > 0 ? ret : 0;
  817. while (left > 0) {
  818. size_t len, copied;
  819. page_off = off & ~PAGE_MASK;
  820. len = min_t(size_t, left, PAGE_SIZE - page_off);
  821. SetPageUptodate(pages[idx]);
  822. copied = copy_page_to_iter(pages[idx++],
  823. page_off, len, to);
  824. off += copied;
  825. left -= copied;
  826. if (copied < len) {
  827. ret = -EFAULT;
  828. break;
  829. }
  830. }
  831. ceph_release_page_vector(pages, num_pages);
  832. if (ret < 0) {
  833. if (ret == -EBLOCKLISTED)
  834. fsc->blocklisted = true;
  835. break;
  836. }
  837. if (off >= i_size || !more)
  838. break;
  839. }
  840. if (off > iocb->ki_pos) {
  841. if (ret >= 0 &&
  842. iov_iter_count(to) > 0 && off >= i_size_read(inode))
  843. *retry_op = CHECK_EOF;
  844. ret = off - iocb->ki_pos;
  845. iocb->ki_pos = off;
  846. }
  847. dout("sync_read result %zd retry_op %d\n", ret, *retry_op);
  848. return ret;
  849. }
  850. struct ceph_aio_request {
  851. struct kiocb *iocb;
  852. size_t total_len;
  853. bool write;
  854. bool should_dirty;
  855. int error;
  856. struct list_head osd_reqs;
  857. unsigned num_reqs;
  858. atomic_t pending_reqs;
  859. struct timespec64 mtime;
  860. struct ceph_cap_flush *prealloc_cf;
  861. };
  862. struct ceph_aio_work {
  863. struct work_struct work;
  864. struct ceph_osd_request *req;
  865. };
  866. static void ceph_aio_retry_work(struct work_struct *work);
  867. static void ceph_aio_complete(struct inode *inode,
  868. struct ceph_aio_request *aio_req)
  869. {
  870. struct ceph_inode_info *ci = ceph_inode(inode);
  871. int ret;
  872. if (!atomic_dec_and_test(&aio_req->pending_reqs))
  873. return;
  874. if (aio_req->iocb->ki_flags & IOCB_DIRECT)
  875. inode_dio_end(inode);
  876. ret = aio_req->error;
  877. if (!ret)
  878. ret = aio_req->total_len;
  879. dout("ceph_aio_complete %p rc %d\n", inode, ret);
  880. if (ret >= 0 && aio_req->write) {
  881. int dirty;
  882. loff_t endoff = aio_req->iocb->ki_pos + aio_req->total_len;
  883. if (endoff > i_size_read(inode)) {
  884. if (ceph_inode_set_size(inode, endoff))
  885. ceph_check_caps(ci, CHECK_CAPS_AUTHONLY, NULL);
  886. }
  887. spin_lock(&ci->i_ceph_lock);
  888. ci->i_inline_version = CEPH_INLINE_NONE;
  889. dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
  890. &aio_req->prealloc_cf);
  891. spin_unlock(&ci->i_ceph_lock);
  892. if (dirty)
  893. __mark_inode_dirty(inode, dirty);
  894. }
  895. ceph_put_cap_refs(ci, (aio_req->write ? CEPH_CAP_FILE_WR :
  896. CEPH_CAP_FILE_RD));
  897. aio_req->iocb->ki_complete(aio_req->iocb, ret, 0);
  898. ceph_free_cap_flush(aio_req->prealloc_cf);
  899. kfree(aio_req);
  900. }
  901. static void ceph_aio_complete_req(struct ceph_osd_request *req)
  902. {
  903. int rc = req->r_result;
  904. struct inode *inode = req->r_inode;
  905. struct ceph_aio_request *aio_req = req->r_priv;
  906. struct ceph_osd_data *osd_data = osd_req_op_extent_osd_data(req, 0);
  907. struct ceph_client_metric *metric = &ceph_sb_to_mdsc(inode->i_sb)->metric;
  908. BUG_ON(osd_data->type != CEPH_OSD_DATA_TYPE_BVECS);
  909. BUG_ON(!osd_data->num_bvecs);
  910. dout("ceph_aio_complete_req %p rc %d bytes %u\n",
  911. inode, rc, osd_data->bvec_pos.iter.bi_size);
  912. /* r_start_latency == 0 means the request was not submitted */
  913. if (req->r_start_latency) {
  914. if (aio_req->write)
  915. ceph_update_write_latency(metric, req->r_start_latency,
  916. req->r_end_latency, rc);
  917. else
  918. ceph_update_read_latency(metric, req->r_start_latency,
  919. req->r_end_latency, rc);
  920. }
  921. if (rc == -EOLDSNAPC) {
  922. struct ceph_aio_work *aio_work;
  923. BUG_ON(!aio_req->write);
  924. aio_work = kmalloc(sizeof(*aio_work), GFP_NOFS);
  925. if (aio_work) {
  926. INIT_WORK(&aio_work->work, ceph_aio_retry_work);
  927. aio_work->req = req;
  928. queue_work(ceph_inode_to_client(inode)->inode_wq,
  929. &aio_work->work);
  930. return;
  931. }
  932. rc = -ENOMEM;
  933. } else if (!aio_req->write) {
  934. if (rc == -ENOENT)
  935. rc = 0;
  936. if (rc >= 0 && osd_data->bvec_pos.iter.bi_size > rc) {
  937. struct iov_iter i;
  938. int zlen = osd_data->bvec_pos.iter.bi_size - rc;
  939. /*
  940. * If read is satisfied by single OSD request,
  941. * it can pass EOF. Otherwise read is within
  942. * i_size.
  943. */
  944. if (aio_req->num_reqs == 1) {
  945. loff_t i_size = i_size_read(inode);
  946. loff_t endoff = aio_req->iocb->ki_pos + rc;
  947. if (endoff < i_size)
  948. zlen = min_t(size_t, zlen,
  949. i_size - endoff);
  950. aio_req->total_len = rc + zlen;
  951. }
  952. iov_iter_bvec(&i, READ, osd_data->bvec_pos.bvecs,
  953. osd_data->num_bvecs,
  954. osd_data->bvec_pos.iter.bi_size);
  955. iov_iter_advance(&i, rc);
  956. iov_iter_zero(zlen, &i);
  957. }
  958. }
  959. put_bvecs(osd_data->bvec_pos.bvecs, osd_data->num_bvecs,
  960. aio_req->should_dirty);
  961. ceph_osdc_put_request(req);
  962. if (rc < 0)
  963. cmpxchg(&aio_req->error, 0, rc);
  964. ceph_aio_complete(inode, aio_req);
  965. return;
  966. }
  967. static void ceph_aio_retry_work(struct work_struct *work)
  968. {
  969. struct ceph_aio_work *aio_work =
  970. container_of(work, struct ceph_aio_work, work);
  971. struct ceph_osd_request *orig_req = aio_work->req;
  972. struct ceph_aio_request *aio_req = orig_req->r_priv;
  973. struct inode *inode = orig_req->r_inode;
  974. struct ceph_inode_info *ci = ceph_inode(inode);
  975. struct ceph_snap_context *snapc;
  976. struct ceph_osd_request *req;
  977. int ret;
  978. spin_lock(&ci->i_ceph_lock);
  979. if (__ceph_have_pending_cap_snap(ci)) {
  980. struct ceph_cap_snap *capsnap =
  981. list_last_entry(&ci->i_cap_snaps,
  982. struct ceph_cap_snap,
  983. ci_item);
  984. snapc = ceph_get_snap_context(capsnap->context);
  985. } else {
  986. BUG_ON(!ci->i_head_snapc);
  987. snapc = ceph_get_snap_context(ci->i_head_snapc);
  988. }
  989. spin_unlock(&ci->i_ceph_lock);
  990. req = ceph_osdc_alloc_request(orig_req->r_osdc, snapc, 1,
  991. false, GFP_NOFS);
  992. if (!req) {
  993. ret = -ENOMEM;
  994. req = orig_req;
  995. goto out;
  996. }
  997. req->r_flags = /* CEPH_OSD_FLAG_ORDERSNAP | */ CEPH_OSD_FLAG_WRITE;
  998. ceph_oloc_copy(&req->r_base_oloc, &orig_req->r_base_oloc);
  999. ceph_oid_copy(&req->r_base_oid, &orig_req->r_base_oid);
  1000. req->r_ops[0] = orig_req->r_ops[0];
  1001. req->r_mtime = aio_req->mtime;
  1002. req->r_data_offset = req->r_ops[0].extent.offset;
  1003. ret = ceph_osdc_alloc_messages(req, GFP_NOFS);
  1004. if (ret) {
  1005. ceph_osdc_put_request(req);
  1006. req = orig_req;
  1007. goto out;
  1008. }
  1009. ceph_osdc_put_request(orig_req);
  1010. req->r_callback = ceph_aio_complete_req;
  1011. req->r_inode = inode;
  1012. req->r_priv = aio_req;
  1013. ret = ceph_osdc_start_request(req->r_osdc, req, false);
  1014. out:
  1015. if (ret < 0) {
  1016. req->r_result = ret;
  1017. ceph_aio_complete_req(req);
  1018. }
  1019. ceph_put_snap_context(snapc);
  1020. kfree(aio_work);
  1021. }
  1022. static ssize_t
  1023. ceph_direct_read_write(struct kiocb *iocb, struct iov_iter *iter,
  1024. struct ceph_snap_context *snapc,
  1025. struct ceph_cap_flush **pcf)
  1026. {
  1027. struct file *file = iocb->ki_filp;
  1028. struct inode *inode = file_inode(file);
  1029. struct ceph_inode_info *ci = ceph_inode(inode);
  1030. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  1031. struct ceph_client_metric *metric = &fsc->mdsc->metric;
  1032. struct ceph_vino vino;
  1033. struct ceph_osd_request *req;
  1034. struct bio_vec *bvecs;
  1035. struct ceph_aio_request *aio_req = NULL;
  1036. int num_pages = 0;
  1037. int flags;
  1038. int ret = 0;
  1039. struct timespec64 mtime = current_time(inode);
  1040. size_t count = iov_iter_count(iter);
  1041. loff_t pos = iocb->ki_pos;
  1042. bool write = iov_iter_rw(iter) == WRITE;
  1043. bool should_dirty = !write && iter_is_iovec(iter);
  1044. if (write && ceph_snap(file_inode(file)) != CEPH_NOSNAP)
  1045. return -EROFS;
  1046. dout("sync_direct_%s on file %p %lld~%u snapc %p seq %lld\n",
  1047. (write ? "write" : "read"), file, pos, (unsigned)count,
  1048. snapc, snapc ? snapc->seq : 0);
  1049. if (write) {
  1050. int ret2 = invalidate_inode_pages2_range(inode->i_mapping,
  1051. pos >> PAGE_SHIFT,
  1052. (pos + count - 1) >> PAGE_SHIFT);
  1053. if (ret2 < 0)
  1054. dout("invalidate_inode_pages2_range returned %d\n", ret2);
  1055. flags = /* CEPH_OSD_FLAG_ORDERSNAP | */ CEPH_OSD_FLAG_WRITE;
  1056. } else {
  1057. flags = CEPH_OSD_FLAG_READ;
  1058. }
  1059. while (iov_iter_count(iter) > 0) {
  1060. u64 size = iov_iter_count(iter);
  1061. ssize_t len;
  1062. if (write)
  1063. size = min_t(u64, size, fsc->mount_options->wsize);
  1064. else
  1065. size = min_t(u64, size, fsc->mount_options->rsize);
  1066. vino = ceph_vino(inode);
  1067. req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
  1068. vino, pos, &size, 0,
  1069. 1,
  1070. write ? CEPH_OSD_OP_WRITE :
  1071. CEPH_OSD_OP_READ,
  1072. flags, snapc,
  1073. ci->i_truncate_seq,
  1074. ci->i_truncate_size,
  1075. false);
  1076. if (IS_ERR(req)) {
  1077. ret = PTR_ERR(req);
  1078. break;
  1079. }
  1080. len = iter_get_bvecs_alloc(iter, size, &bvecs, &num_pages);
  1081. if (len < 0) {
  1082. ceph_osdc_put_request(req);
  1083. ret = len;
  1084. break;
  1085. }
  1086. if (len != size)
  1087. osd_req_op_extent_update(req, 0, len);
  1088. /*
  1089. * To simplify error handling, allow AIO when IO within i_size
  1090. * or IO can be satisfied by single OSD request.
  1091. */
  1092. if (pos == iocb->ki_pos && !is_sync_kiocb(iocb) &&
  1093. (len == count || pos + count <= i_size_read(inode))) {
  1094. aio_req = kzalloc(sizeof(*aio_req), GFP_KERNEL);
  1095. if (aio_req) {
  1096. aio_req->iocb = iocb;
  1097. aio_req->write = write;
  1098. aio_req->should_dirty = should_dirty;
  1099. INIT_LIST_HEAD(&aio_req->osd_reqs);
  1100. if (write) {
  1101. aio_req->mtime = mtime;
  1102. swap(aio_req->prealloc_cf, *pcf);
  1103. }
  1104. }
  1105. /* ignore error */
  1106. }
  1107. if (write) {
  1108. /*
  1109. * throw out any page cache pages in this range. this
  1110. * may block.
  1111. */
  1112. truncate_inode_pages_range(inode->i_mapping, pos,
  1113. PAGE_ALIGN(pos + len) - 1);
  1114. req->r_mtime = mtime;
  1115. }
  1116. osd_req_op_extent_osd_data_bvecs(req, 0, bvecs, num_pages, len);
  1117. if (aio_req) {
  1118. aio_req->total_len += len;
  1119. aio_req->num_reqs++;
  1120. atomic_inc(&aio_req->pending_reqs);
  1121. req->r_callback = ceph_aio_complete_req;
  1122. req->r_inode = inode;
  1123. req->r_priv = aio_req;
  1124. list_add_tail(&req->r_private_item, &aio_req->osd_reqs);
  1125. pos += len;
  1126. continue;
  1127. }
  1128. ret = ceph_osdc_start_request(req->r_osdc, req, false);
  1129. if (!ret)
  1130. ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
  1131. if (write)
  1132. ceph_update_write_latency(metric, req->r_start_latency,
  1133. req->r_end_latency, ret);
  1134. else
  1135. ceph_update_read_latency(metric, req->r_start_latency,
  1136. req->r_end_latency, ret);
  1137. size = i_size_read(inode);
  1138. if (!write) {
  1139. if (ret == -ENOENT)
  1140. ret = 0;
  1141. if (ret >= 0 && ret < len && pos + ret < size) {
  1142. struct iov_iter i;
  1143. int zlen = min_t(size_t, len - ret,
  1144. size - pos - ret);
  1145. iov_iter_bvec(&i, READ, bvecs, num_pages, len);
  1146. iov_iter_advance(&i, ret);
  1147. iov_iter_zero(zlen, &i);
  1148. ret += zlen;
  1149. }
  1150. if (ret >= 0)
  1151. len = ret;
  1152. }
  1153. put_bvecs(bvecs, num_pages, should_dirty);
  1154. ceph_osdc_put_request(req);
  1155. if (ret < 0)
  1156. break;
  1157. pos += len;
  1158. if (!write && pos >= size)
  1159. break;
  1160. if (write && pos > size) {
  1161. if (ceph_inode_set_size(inode, pos))
  1162. ceph_check_caps(ceph_inode(inode),
  1163. CHECK_CAPS_AUTHONLY,
  1164. NULL);
  1165. }
  1166. }
  1167. if (aio_req) {
  1168. LIST_HEAD(osd_reqs);
  1169. if (aio_req->num_reqs == 0) {
  1170. kfree(aio_req);
  1171. return ret;
  1172. }
  1173. ceph_get_cap_refs(ci, write ? CEPH_CAP_FILE_WR :
  1174. CEPH_CAP_FILE_RD);
  1175. list_splice(&aio_req->osd_reqs, &osd_reqs);
  1176. inode_dio_begin(inode);
  1177. while (!list_empty(&osd_reqs)) {
  1178. req = list_first_entry(&osd_reqs,
  1179. struct ceph_osd_request,
  1180. r_private_item);
  1181. list_del_init(&req->r_private_item);
  1182. if (ret >= 0)
  1183. ret = ceph_osdc_start_request(req->r_osdc,
  1184. req, false);
  1185. if (ret < 0) {
  1186. req->r_result = ret;
  1187. ceph_aio_complete_req(req);
  1188. }
  1189. }
  1190. return -EIOCBQUEUED;
  1191. }
  1192. if (ret != -EOLDSNAPC && pos > iocb->ki_pos) {
  1193. ret = pos - iocb->ki_pos;
  1194. iocb->ki_pos = pos;
  1195. }
  1196. return ret;
  1197. }
  1198. /*
  1199. * Synchronous write, straight from __user pointer or user pages.
  1200. *
  1201. * If write spans object boundary, just do multiple writes. (For a
  1202. * correct atomic write, we should e.g. take write locks on all
  1203. * objects, rollback on failure, etc.)
  1204. */
  1205. static ssize_t
  1206. ceph_sync_write(struct kiocb *iocb, struct iov_iter *from, loff_t pos,
  1207. struct ceph_snap_context *snapc)
  1208. {
  1209. struct file *file = iocb->ki_filp;
  1210. struct inode *inode = file_inode(file);
  1211. struct ceph_inode_info *ci = ceph_inode(inode);
  1212. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  1213. struct ceph_vino vino;
  1214. struct ceph_osd_request *req;
  1215. struct page **pages;
  1216. u64 len;
  1217. int num_pages;
  1218. int written = 0;
  1219. int flags;
  1220. int ret;
  1221. bool check_caps = false;
  1222. struct timespec64 mtime = current_time(inode);
  1223. size_t count = iov_iter_count(from);
  1224. if (ceph_snap(file_inode(file)) != CEPH_NOSNAP)
  1225. return -EROFS;
  1226. dout("sync_write on file %p %lld~%u snapc %p seq %lld\n",
  1227. file, pos, (unsigned)count, snapc, snapc->seq);
  1228. ret = filemap_write_and_wait_range(inode->i_mapping,
  1229. pos, pos + count - 1);
  1230. if (ret < 0)
  1231. return ret;
  1232. ret = invalidate_inode_pages2_range(inode->i_mapping,
  1233. pos >> PAGE_SHIFT,
  1234. (pos + count - 1) >> PAGE_SHIFT);
  1235. if (ret < 0)
  1236. dout("invalidate_inode_pages2_range returned %d\n", ret);
  1237. flags = /* CEPH_OSD_FLAG_ORDERSNAP | */ CEPH_OSD_FLAG_WRITE;
  1238. while ((len = iov_iter_count(from)) > 0) {
  1239. size_t left;
  1240. int n;
  1241. vino = ceph_vino(inode);
  1242. req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
  1243. vino, pos, &len, 0, 1,
  1244. CEPH_OSD_OP_WRITE, flags, snapc,
  1245. ci->i_truncate_seq,
  1246. ci->i_truncate_size,
  1247. false);
  1248. if (IS_ERR(req)) {
  1249. ret = PTR_ERR(req);
  1250. break;
  1251. }
  1252. /*
  1253. * write from beginning of first page,
  1254. * regardless of io alignment
  1255. */
  1256. num_pages = (len + PAGE_SIZE - 1) >> PAGE_SHIFT;
  1257. pages = ceph_alloc_page_vector(num_pages, GFP_KERNEL);
  1258. if (IS_ERR(pages)) {
  1259. ret = PTR_ERR(pages);
  1260. goto out;
  1261. }
  1262. left = len;
  1263. for (n = 0; n < num_pages; n++) {
  1264. size_t plen = min_t(size_t, left, PAGE_SIZE);
  1265. ret = copy_page_from_iter(pages[n], 0, plen, from);
  1266. if (ret != plen) {
  1267. ret = -EFAULT;
  1268. break;
  1269. }
  1270. left -= ret;
  1271. }
  1272. if (ret < 0) {
  1273. ceph_release_page_vector(pages, num_pages);
  1274. goto out;
  1275. }
  1276. req->r_inode = inode;
  1277. osd_req_op_extent_osd_data_pages(req, 0, pages, len, 0,
  1278. false, true);
  1279. req->r_mtime = mtime;
  1280. ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
  1281. if (!ret)
  1282. ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
  1283. ceph_update_write_latency(&fsc->mdsc->metric, req->r_start_latency,
  1284. req->r_end_latency, ret);
  1285. out:
  1286. ceph_osdc_put_request(req);
  1287. if (ret != 0) {
  1288. ceph_set_error_write(ci);
  1289. break;
  1290. }
  1291. ceph_clear_error_write(ci);
  1292. pos += len;
  1293. written += len;
  1294. if (pos > i_size_read(inode)) {
  1295. check_caps = ceph_inode_set_size(inode, pos);
  1296. if (check_caps)
  1297. ceph_check_caps(ceph_inode(inode),
  1298. CHECK_CAPS_AUTHONLY,
  1299. NULL);
  1300. }
  1301. }
  1302. if (ret != -EOLDSNAPC && written > 0) {
  1303. ret = written;
  1304. iocb->ki_pos = pos;
  1305. }
  1306. return ret;
  1307. }
  1308. /*
  1309. * Wrap generic_file_aio_read with checks for cap bits on the inode.
  1310. * Atomically grab references, so that those bits are not released
  1311. * back to the MDS mid-read.
  1312. *
  1313. * Hmm, the sync read case isn't actually async... should it be?
  1314. */
  1315. static ssize_t ceph_read_iter(struct kiocb *iocb, struct iov_iter *to)
  1316. {
  1317. struct file *filp = iocb->ki_filp;
  1318. struct ceph_file_info *fi = filp->private_data;
  1319. size_t len = iov_iter_count(to);
  1320. struct inode *inode = file_inode(filp);
  1321. struct ceph_inode_info *ci = ceph_inode(inode);
  1322. struct page *pinned_page = NULL;
  1323. bool direct_lock = iocb->ki_flags & IOCB_DIRECT;
  1324. ssize_t ret;
  1325. int want, got = 0;
  1326. int retry_op = 0, read = 0;
  1327. again:
  1328. dout("aio_read %p %llx.%llx %llu~%u trying to get caps on %p\n",
  1329. inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len, inode);
  1330. if (direct_lock)
  1331. ceph_start_io_direct(inode);
  1332. else
  1333. ceph_start_io_read(inode);
  1334. if (fi->fmode & CEPH_FILE_MODE_LAZY)
  1335. want = CEPH_CAP_FILE_CACHE | CEPH_CAP_FILE_LAZYIO;
  1336. else
  1337. want = CEPH_CAP_FILE_CACHE;
  1338. ret = ceph_get_caps(filp, CEPH_CAP_FILE_RD, want, -1,
  1339. &got, &pinned_page);
  1340. if (ret < 0) {
  1341. if (iocb->ki_flags & IOCB_DIRECT)
  1342. ceph_end_io_direct(inode);
  1343. else
  1344. ceph_end_io_read(inode);
  1345. return ret;
  1346. }
  1347. if ((got & (CEPH_CAP_FILE_CACHE|CEPH_CAP_FILE_LAZYIO)) == 0 ||
  1348. (iocb->ki_flags & IOCB_DIRECT) ||
  1349. (fi->flags & CEPH_F_SYNC)) {
  1350. dout("aio_sync_read %p %llx.%llx %llu~%u got cap refs on %s\n",
  1351. inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len,
  1352. ceph_cap_string(got));
  1353. if (ci->i_inline_version == CEPH_INLINE_NONE) {
  1354. if (!retry_op && (iocb->ki_flags & IOCB_DIRECT)) {
  1355. ret = ceph_direct_read_write(iocb, to,
  1356. NULL, NULL);
  1357. if (ret >= 0 && ret < len)
  1358. retry_op = CHECK_EOF;
  1359. } else {
  1360. ret = ceph_sync_read(iocb, to, &retry_op);
  1361. }
  1362. } else {
  1363. retry_op = READ_INLINE;
  1364. }
  1365. } else {
  1366. CEPH_DEFINE_RW_CONTEXT(rw_ctx, got);
  1367. dout("aio_read %p %llx.%llx %llu~%u got cap refs on %s\n",
  1368. inode, ceph_vinop(inode), iocb->ki_pos, (unsigned)len,
  1369. ceph_cap_string(got));
  1370. ceph_add_rw_context(fi, &rw_ctx);
  1371. ret = generic_file_read_iter(iocb, to);
  1372. ceph_del_rw_context(fi, &rw_ctx);
  1373. }
  1374. dout("aio_read %p %llx.%llx dropping cap refs on %s = %d\n",
  1375. inode, ceph_vinop(inode), ceph_cap_string(got), (int)ret);
  1376. if (pinned_page) {
  1377. put_page(pinned_page);
  1378. pinned_page = NULL;
  1379. }
  1380. ceph_put_cap_refs(ci, got);
  1381. if (direct_lock)
  1382. ceph_end_io_direct(inode);
  1383. else
  1384. ceph_end_io_read(inode);
  1385. if (retry_op > HAVE_RETRIED && ret >= 0) {
  1386. int statret;
  1387. struct page *page = NULL;
  1388. loff_t i_size;
  1389. if (retry_op == READ_INLINE) {
  1390. page = __page_cache_alloc(GFP_KERNEL);
  1391. if (!page)
  1392. return -ENOMEM;
  1393. }
  1394. statret = __ceph_do_getattr(inode, page,
  1395. CEPH_STAT_CAP_INLINE_DATA, !!page);
  1396. if (statret < 0) {
  1397. if (page)
  1398. __free_page(page);
  1399. if (statret == -ENODATA) {
  1400. BUG_ON(retry_op != READ_INLINE);
  1401. goto again;
  1402. }
  1403. return statret;
  1404. }
  1405. i_size = i_size_read(inode);
  1406. if (retry_op == READ_INLINE) {
  1407. BUG_ON(ret > 0 || read > 0);
  1408. if (iocb->ki_pos < i_size &&
  1409. iocb->ki_pos < PAGE_SIZE) {
  1410. loff_t end = min_t(loff_t, i_size,
  1411. iocb->ki_pos + len);
  1412. end = min_t(loff_t, end, PAGE_SIZE);
  1413. if (statret < end)
  1414. zero_user_segment(page, statret, end);
  1415. ret = copy_page_to_iter(page,
  1416. iocb->ki_pos & ~PAGE_MASK,
  1417. end - iocb->ki_pos, to);
  1418. iocb->ki_pos += ret;
  1419. read += ret;
  1420. }
  1421. if (iocb->ki_pos < i_size && read < len) {
  1422. size_t zlen = min_t(size_t, len - read,
  1423. i_size - iocb->ki_pos);
  1424. ret = iov_iter_zero(zlen, to);
  1425. iocb->ki_pos += ret;
  1426. read += ret;
  1427. }
  1428. __free_pages(page, 0);
  1429. return read;
  1430. }
  1431. /* hit EOF or hole? */
  1432. if (retry_op == CHECK_EOF && iocb->ki_pos < i_size &&
  1433. ret < len) {
  1434. dout("sync_read hit hole, ppos %lld < size %lld"
  1435. ", reading more\n", iocb->ki_pos, i_size);
  1436. read += ret;
  1437. len -= ret;
  1438. retry_op = HAVE_RETRIED;
  1439. goto again;
  1440. }
  1441. }
  1442. if (ret >= 0)
  1443. ret += read;
  1444. return ret;
  1445. }
  1446. /*
  1447. * Take cap references to avoid releasing caps to MDS mid-write.
  1448. *
  1449. * If we are synchronous, and write with an old snap context, the OSD
  1450. * may return EOLDSNAPC. In that case, retry the write.. _after_
  1451. * dropping our cap refs and allowing the pending snap to logically
  1452. * complete _before_ this write occurs.
  1453. *
  1454. * If we are near ENOSPC, write synchronously.
  1455. */
  1456. static ssize_t ceph_write_iter(struct kiocb *iocb, struct iov_iter *from)
  1457. {
  1458. struct file *file = iocb->ki_filp;
  1459. struct ceph_file_info *fi = file->private_data;
  1460. struct inode *inode = file_inode(file);
  1461. struct ceph_inode_info *ci = ceph_inode(inode);
  1462. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  1463. struct ceph_osd_client *osdc = &fsc->client->osdc;
  1464. struct ceph_cap_flush *prealloc_cf;
  1465. ssize_t count, written = 0;
  1466. int err, want, got;
  1467. bool direct_lock = false;
  1468. u32 map_flags;
  1469. u64 pool_flags;
  1470. loff_t pos;
  1471. loff_t limit = max(i_size_read(inode), fsc->max_file_size);
  1472. if (ceph_snap(inode) != CEPH_NOSNAP)
  1473. return -EROFS;
  1474. prealloc_cf = ceph_alloc_cap_flush();
  1475. if (!prealloc_cf)
  1476. return -ENOMEM;
  1477. if ((iocb->ki_flags & (IOCB_DIRECT | IOCB_APPEND)) == IOCB_DIRECT)
  1478. direct_lock = true;
  1479. retry_snap:
  1480. if (direct_lock)
  1481. ceph_start_io_direct(inode);
  1482. else
  1483. ceph_start_io_write(inode);
  1484. /* We can write back this queue in page reclaim */
  1485. current->backing_dev_info = inode_to_bdi(inode);
  1486. if (iocb->ki_flags & IOCB_APPEND) {
  1487. err = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false);
  1488. if (err < 0)
  1489. goto out;
  1490. }
  1491. err = generic_write_checks(iocb, from);
  1492. if (err <= 0)
  1493. goto out;
  1494. pos = iocb->ki_pos;
  1495. if (unlikely(pos >= limit)) {
  1496. err = -EFBIG;
  1497. goto out;
  1498. } else {
  1499. iov_iter_truncate(from, limit - pos);
  1500. }
  1501. count = iov_iter_count(from);
  1502. if (ceph_quota_is_max_bytes_exceeded(inode, pos + count)) {
  1503. err = -EDQUOT;
  1504. goto out;
  1505. }
  1506. down_read(&osdc->lock);
  1507. map_flags = osdc->osdmap->flags;
  1508. pool_flags = ceph_pg_pool_flags(osdc->osdmap, ci->i_layout.pool_id);
  1509. up_read(&osdc->lock);
  1510. if ((map_flags & CEPH_OSDMAP_FULL) ||
  1511. (pool_flags & CEPH_POOL_FLAG_FULL)) {
  1512. err = -ENOSPC;
  1513. goto out;
  1514. }
  1515. err = file_remove_privs(file);
  1516. if (err)
  1517. goto out;
  1518. if (ci->i_inline_version != CEPH_INLINE_NONE) {
  1519. err = ceph_uninline_data(file, NULL);
  1520. if (err < 0)
  1521. goto out;
  1522. }
  1523. dout("aio_write %p %llx.%llx %llu~%zd getting caps. i_size %llu\n",
  1524. inode, ceph_vinop(inode), pos, count, i_size_read(inode));
  1525. if (fi->fmode & CEPH_FILE_MODE_LAZY)
  1526. want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO;
  1527. else
  1528. want = CEPH_CAP_FILE_BUFFER;
  1529. got = 0;
  1530. err = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, pos + count,
  1531. &got, NULL);
  1532. if (err < 0)
  1533. goto out;
  1534. err = file_update_time(file);
  1535. if (err)
  1536. goto out_caps;
  1537. inode_inc_iversion_raw(inode);
  1538. dout("aio_write %p %llx.%llx %llu~%zd got cap refs on %s\n",
  1539. inode, ceph_vinop(inode), pos, count, ceph_cap_string(got));
  1540. if ((got & (CEPH_CAP_FILE_BUFFER|CEPH_CAP_FILE_LAZYIO)) == 0 ||
  1541. (iocb->ki_flags & IOCB_DIRECT) || (fi->flags & CEPH_F_SYNC) ||
  1542. (ci->i_ceph_flags & CEPH_I_ERROR_WRITE)) {
  1543. struct ceph_snap_context *snapc;
  1544. struct iov_iter data;
  1545. spin_lock(&ci->i_ceph_lock);
  1546. if (__ceph_have_pending_cap_snap(ci)) {
  1547. struct ceph_cap_snap *capsnap =
  1548. list_last_entry(&ci->i_cap_snaps,
  1549. struct ceph_cap_snap,
  1550. ci_item);
  1551. snapc = ceph_get_snap_context(capsnap->context);
  1552. } else {
  1553. BUG_ON(!ci->i_head_snapc);
  1554. snapc = ceph_get_snap_context(ci->i_head_snapc);
  1555. }
  1556. spin_unlock(&ci->i_ceph_lock);
  1557. /* we might need to revert back to that point */
  1558. data = *from;
  1559. if (iocb->ki_flags & IOCB_DIRECT)
  1560. written = ceph_direct_read_write(iocb, &data, snapc,
  1561. &prealloc_cf);
  1562. else
  1563. written = ceph_sync_write(iocb, &data, pos, snapc);
  1564. if (direct_lock)
  1565. ceph_end_io_direct(inode);
  1566. else
  1567. ceph_end_io_write(inode);
  1568. if (written > 0)
  1569. iov_iter_advance(from, written);
  1570. ceph_put_snap_context(snapc);
  1571. } else {
  1572. /*
  1573. * No need to acquire the i_truncate_mutex. Because
  1574. * the MDS revokes Fwb caps before sending truncate
  1575. * message to us. We can't get Fwb cap while there
  1576. * are pending vmtruncate. So write and vmtruncate
  1577. * can not run at the same time
  1578. */
  1579. written = generic_perform_write(file, from, pos);
  1580. if (likely(written >= 0))
  1581. iocb->ki_pos = pos + written;
  1582. ceph_end_io_write(inode);
  1583. }
  1584. if (written >= 0) {
  1585. int dirty;
  1586. spin_lock(&ci->i_ceph_lock);
  1587. ci->i_inline_version = CEPH_INLINE_NONE;
  1588. dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
  1589. &prealloc_cf);
  1590. spin_unlock(&ci->i_ceph_lock);
  1591. if (dirty)
  1592. __mark_inode_dirty(inode, dirty);
  1593. if (ceph_quota_is_max_bytes_approaching(inode, iocb->ki_pos))
  1594. ceph_check_caps(ci, 0, NULL);
  1595. }
  1596. dout("aio_write %p %llx.%llx %llu~%u dropping cap refs on %s\n",
  1597. inode, ceph_vinop(inode), pos, (unsigned)count,
  1598. ceph_cap_string(got));
  1599. ceph_put_cap_refs(ci, got);
  1600. if (written == -EOLDSNAPC) {
  1601. dout("aio_write %p %llx.%llx %llu~%u" "got EOLDSNAPC, retrying\n",
  1602. inode, ceph_vinop(inode), pos, (unsigned)count);
  1603. goto retry_snap;
  1604. }
  1605. if (written >= 0) {
  1606. if ((map_flags & CEPH_OSDMAP_NEARFULL) ||
  1607. (pool_flags & CEPH_POOL_FLAG_NEARFULL))
  1608. iocb->ki_flags |= IOCB_DSYNC;
  1609. written = generic_write_sync(iocb, written);
  1610. }
  1611. goto out_unlocked;
  1612. out_caps:
  1613. ceph_put_cap_refs(ci, got);
  1614. out:
  1615. if (direct_lock)
  1616. ceph_end_io_direct(inode);
  1617. else
  1618. ceph_end_io_write(inode);
  1619. out_unlocked:
  1620. ceph_free_cap_flush(prealloc_cf);
  1621. current->backing_dev_info = NULL;
  1622. return written ? written : err;
  1623. }
  1624. /*
  1625. * llseek. be sure to verify file size on SEEK_END.
  1626. */
  1627. static loff_t ceph_llseek(struct file *file, loff_t offset, int whence)
  1628. {
  1629. struct inode *inode = file->f_mapping->host;
  1630. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  1631. loff_t i_size;
  1632. loff_t ret;
  1633. inode_lock(inode);
  1634. if (whence == SEEK_END || whence == SEEK_DATA || whence == SEEK_HOLE) {
  1635. ret = ceph_do_getattr(inode, CEPH_STAT_CAP_SIZE, false);
  1636. if (ret < 0)
  1637. goto out;
  1638. }
  1639. i_size = i_size_read(inode);
  1640. switch (whence) {
  1641. case SEEK_END:
  1642. offset += i_size;
  1643. break;
  1644. case SEEK_CUR:
  1645. /*
  1646. * Here we special-case the lseek(fd, 0, SEEK_CUR)
  1647. * position-querying operation. Avoid rewriting the "same"
  1648. * f_pos value back to the file because a concurrent read(),
  1649. * write() or lseek() might have altered it
  1650. */
  1651. if (offset == 0) {
  1652. ret = file->f_pos;
  1653. goto out;
  1654. }
  1655. offset += file->f_pos;
  1656. break;
  1657. case SEEK_DATA:
  1658. if (offset < 0 || offset >= i_size) {
  1659. ret = -ENXIO;
  1660. goto out;
  1661. }
  1662. break;
  1663. case SEEK_HOLE:
  1664. if (offset < 0 || offset >= i_size) {
  1665. ret = -ENXIO;
  1666. goto out;
  1667. }
  1668. offset = i_size;
  1669. break;
  1670. }
  1671. ret = vfs_setpos(file, offset, max(i_size, fsc->max_file_size));
  1672. out:
  1673. inode_unlock(inode);
  1674. return ret;
  1675. }
  1676. static inline void ceph_zero_partial_page(
  1677. struct inode *inode, loff_t offset, unsigned size)
  1678. {
  1679. struct page *page;
  1680. pgoff_t index = offset >> PAGE_SHIFT;
  1681. page = find_lock_page(inode->i_mapping, index);
  1682. if (page) {
  1683. wait_on_page_writeback(page);
  1684. zero_user(page, offset & (PAGE_SIZE - 1), size);
  1685. unlock_page(page);
  1686. put_page(page);
  1687. }
  1688. }
  1689. static void ceph_zero_pagecache_range(struct inode *inode, loff_t offset,
  1690. loff_t length)
  1691. {
  1692. loff_t nearly = round_up(offset, PAGE_SIZE);
  1693. if (offset < nearly) {
  1694. loff_t size = nearly - offset;
  1695. if (length < size)
  1696. size = length;
  1697. ceph_zero_partial_page(inode, offset, size);
  1698. offset += size;
  1699. length -= size;
  1700. }
  1701. if (length >= PAGE_SIZE) {
  1702. loff_t size = round_down(length, PAGE_SIZE);
  1703. truncate_pagecache_range(inode, offset, offset + size - 1);
  1704. offset += size;
  1705. length -= size;
  1706. }
  1707. if (length)
  1708. ceph_zero_partial_page(inode, offset, length);
  1709. }
  1710. static int ceph_zero_partial_object(struct inode *inode,
  1711. loff_t offset, loff_t *length)
  1712. {
  1713. struct ceph_inode_info *ci = ceph_inode(inode);
  1714. struct ceph_fs_client *fsc = ceph_inode_to_client(inode);
  1715. struct ceph_osd_request *req;
  1716. int ret = 0;
  1717. loff_t zero = 0;
  1718. int op;
  1719. if (!length) {
  1720. op = offset ? CEPH_OSD_OP_DELETE : CEPH_OSD_OP_TRUNCATE;
  1721. length = &zero;
  1722. } else {
  1723. op = CEPH_OSD_OP_ZERO;
  1724. }
  1725. req = ceph_osdc_new_request(&fsc->client->osdc, &ci->i_layout,
  1726. ceph_vino(inode),
  1727. offset, length,
  1728. 0, 1, op,
  1729. CEPH_OSD_FLAG_WRITE,
  1730. NULL, 0, 0, false);
  1731. if (IS_ERR(req)) {
  1732. ret = PTR_ERR(req);
  1733. goto out;
  1734. }
  1735. req->r_mtime = inode->i_mtime;
  1736. ret = ceph_osdc_start_request(&fsc->client->osdc, req, false);
  1737. if (!ret) {
  1738. ret = ceph_osdc_wait_request(&fsc->client->osdc, req);
  1739. if (ret == -ENOENT)
  1740. ret = 0;
  1741. }
  1742. ceph_osdc_put_request(req);
  1743. out:
  1744. return ret;
  1745. }
  1746. static int ceph_zero_objects(struct inode *inode, loff_t offset, loff_t length)
  1747. {
  1748. int ret = 0;
  1749. struct ceph_inode_info *ci = ceph_inode(inode);
  1750. s32 stripe_unit = ci->i_layout.stripe_unit;
  1751. s32 stripe_count = ci->i_layout.stripe_count;
  1752. s32 object_size = ci->i_layout.object_size;
  1753. u64 object_set_size = object_size * stripe_count;
  1754. u64 nearly, t;
  1755. /* round offset up to next period boundary */
  1756. nearly = offset + object_set_size - 1;
  1757. t = nearly;
  1758. nearly -= do_div(t, object_set_size);
  1759. while (length && offset < nearly) {
  1760. loff_t size = length;
  1761. ret = ceph_zero_partial_object(inode, offset, &size);
  1762. if (ret < 0)
  1763. return ret;
  1764. offset += size;
  1765. length -= size;
  1766. }
  1767. while (length >= object_set_size) {
  1768. int i;
  1769. loff_t pos = offset;
  1770. for (i = 0; i < stripe_count; ++i) {
  1771. ret = ceph_zero_partial_object(inode, pos, NULL);
  1772. if (ret < 0)
  1773. return ret;
  1774. pos += stripe_unit;
  1775. }
  1776. offset += object_set_size;
  1777. length -= object_set_size;
  1778. }
  1779. while (length) {
  1780. loff_t size = length;
  1781. ret = ceph_zero_partial_object(inode, offset, &size);
  1782. if (ret < 0)
  1783. return ret;
  1784. offset += size;
  1785. length -= size;
  1786. }
  1787. return ret;
  1788. }
  1789. static long ceph_fallocate(struct file *file, int mode,
  1790. loff_t offset, loff_t length)
  1791. {
  1792. struct ceph_file_info *fi = file->private_data;
  1793. struct inode *inode = file_inode(file);
  1794. struct ceph_inode_info *ci = ceph_inode(inode);
  1795. struct ceph_cap_flush *prealloc_cf;
  1796. int want, got = 0;
  1797. int dirty;
  1798. int ret = 0;
  1799. loff_t endoff = 0;
  1800. loff_t size;
  1801. if (mode != (FALLOC_FL_KEEP_SIZE | FALLOC_FL_PUNCH_HOLE))
  1802. return -EOPNOTSUPP;
  1803. if (!S_ISREG(inode->i_mode))
  1804. return -EOPNOTSUPP;
  1805. prealloc_cf = ceph_alloc_cap_flush();
  1806. if (!prealloc_cf)
  1807. return -ENOMEM;
  1808. inode_lock(inode);
  1809. if (ceph_snap(inode) != CEPH_NOSNAP) {
  1810. ret = -EROFS;
  1811. goto unlock;
  1812. }
  1813. if (ci->i_inline_version != CEPH_INLINE_NONE) {
  1814. ret = ceph_uninline_data(file, NULL);
  1815. if (ret < 0)
  1816. goto unlock;
  1817. }
  1818. size = i_size_read(inode);
  1819. /* Are we punching a hole beyond EOF? */
  1820. if (offset >= size)
  1821. goto unlock;
  1822. if ((offset + length) > size)
  1823. length = size - offset;
  1824. if (fi->fmode & CEPH_FILE_MODE_LAZY)
  1825. want = CEPH_CAP_FILE_BUFFER | CEPH_CAP_FILE_LAZYIO;
  1826. else
  1827. want = CEPH_CAP_FILE_BUFFER;
  1828. ret = ceph_get_caps(file, CEPH_CAP_FILE_WR, want, endoff, &got, NULL);
  1829. if (ret < 0)
  1830. goto unlock;
  1831. ceph_zero_pagecache_range(inode, offset, length);
  1832. ret = ceph_zero_objects(inode, offset, length);
  1833. if (!ret) {
  1834. spin_lock(&ci->i_ceph_lock);
  1835. ci->i_inline_version = CEPH_INLINE_NONE;
  1836. dirty = __ceph_mark_dirty_caps(ci, CEPH_CAP_FILE_WR,
  1837. &prealloc_cf);
  1838. spin_unlock(&ci->i_ceph_lock);
  1839. if (dirty)
  1840. __mark_inode_dirty(inode, dirty);
  1841. }
  1842. ceph_put_cap_refs(ci, got);
  1843. unlock:
  1844. inode_unlock(inode);
  1845. ceph_free_cap_flush(prealloc_cf);
  1846. return ret;
  1847. }
  1848. /*
  1849. * This function tries to get FILE_WR capabilities for dst_ci and FILE_RD for
  1850. * src_ci. Two attempts are made to obtain both caps, and an error is return if
  1851. * this fails; zero is returned on success.
  1852. */
  1853. static int get_rd_wr_caps(struct file *src_filp, int *src_got,
  1854. struct file *dst_filp,
  1855. loff_t dst_endoff, int *dst_got)
  1856. {
  1857. int ret = 0;
  1858. bool retrying = false;
  1859. retry_caps:
  1860. ret = ceph_get_caps(dst_filp, CEPH_CAP_FILE_WR, CEPH_CAP_FILE_BUFFER,
  1861. dst_endoff, dst_got, NULL);
  1862. if (ret < 0)
  1863. return ret;
  1864. /*
  1865. * Since we're already holding the FILE_WR capability for the dst file,
  1866. * we would risk a deadlock by using ceph_get_caps. Thus, we'll do some
  1867. * retry dance instead to try to get both capabilities.
  1868. */
  1869. ret = ceph_try_get_caps(file_inode(src_filp),
  1870. CEPH_CAP_FILE_RD, CEPH_CAP_FILE_SHARED,
  1871. false, src_got);
  1872. if (ret <= 0) {
  1873. /* Start by dropping dst_ci caps and getting src_ci caps */
  1874. ceph_put_cap_refs(ceph_inode(file_inode(dst_filp)), *dst_got);
  1875. if (retrying) {
  1876. if (!ret)
  1877. /* ceph_try_get_caps masks EAGAIN */
  1878. ret = -EAGAIN;
  1879. return ret;
  1880. }
  1881. ret = ceph_get_caps(src_filp, CEPH_CAP_FILE_RD,
  1882. CEPH_CAP_FILE_SHARED, -1, src_got, NULL);
  1883. if (ret < 0)
  1884. return ret;
  1885. /*... drop src_ci caps too, and retry */
  1886. ceph_put_cap_refs(ceph_inode(file_inode(src_filp)), *src_got);
  1887. retrying = true;
  1888. goto retry_caps;
  1889. }
  1890. return ret;
  1891. }
  1892. static void put_rd_wr_caps(struct ceph_inode_info *src_ci, int src_got,
  1893. struct ceph_inode_info *dst_ci, int dst_got)
  1894. {
  1895. ceph_put_cap_refs(src_ci, src_got);
  1896. ceph_put_cap_refs(dst_ci, dst_got);
  1897. }
  1898. /*
  1899. * This function does several size-related checks, returning an error if:
  1900. * - source file is smaller than off+len
  1901. * - destination file size is not OK (inode_newsize_ok())
  1902. * - max bytes quotas is exceeded
  1903. */
  1904. static int is_file_size_ok(struct inode *src_inode, struct inode *dst_inode,
  1905. loff_t src_off, loff_t dst_off, size_t len)
  1906. {
  1907. loff_t size, endoff;
  1908. size = i_size_read(src_inode);
  1909. /*
  1910. * Don't copy beyond source file EOF. Instead of simply setting length
  1911. * to (size - src_off), just drop to VFS default implementation, as the
  1912. * local i_size may be stale due to other clients writing to the source
  1913. * inode.
  1914. */
  1915. if (src_off + len > size) {
  1916. dout("Copy beyond EOF (%llu + %zu > %llu)\n",
  1917. src_off, len, size);
  1918. return -EOPNOTSUPP;
  1919. }
  1920. size = i_size_read(dst_inode);
  1921. endoff = dst_off + len;
  1922. if (inode_newsize_ok(dst_inode, endoff))
  1923. return -EOPNOTSUPP;
  1924. if (ceph_quota_is_max_bytes_exceeded(dst_inode, endoff))
  1925. return -EDQUOT;
  1926. return 0;
  1927. }
  1928. static ssize_t ceph_do_objects_copy(struct ceph_inode_info *src_ci, u64 *src_off,
  1929. struct ceph_inode_info *dst_ci, u64 *dst_off,
  1930. struct ceph_fs_client *fsc,
  1931. size_t len, unsigned int flags)
  1932. {
  1933. struct ceph_object_locator src_oloc, dst_oloc;
  1934. struct ceph_object_id src_oid, dst_oid;
  1935. size_t bytes = 0;
  1936. u64 src_objnum, src_objoff, dst_objnum, dst_objoff;
  1937. u32 src_objlen, dst_objlen;
  1938. u32 object_size = src_ci->i_layout.object_size;
  1939. int ret;
  1940. src_oloc.pool = src_ci->i_layout.pool_id;
  1941. src_oloc.pool_ns = ceph_try_get_string(src_ci->i_layout.pool_ns);
  1942. dst_oloc.pool = dst_ci->i_layout.pool_id;
  1943. dst_oloc.pool_ns = ceph_try_get_string(dst_ci->i_layout.pool_ns);
  1944. while (len >= object_size) {
  1945. ceph_calc_file_object_mapping(&src_ci->i_layout, *src_off,
  1946. object_size, &src_objnum,
  1947. &src_objoff, &src_objlen);
  1948. ceph_calc_file_object_mapping(&dst_ci->i_layout, *dst_off,
  1949. object_size, &dst_objnum,
  1950. &dst_objoff, &dst_objlen);
  1951. ceph_oid_init(&src_oid);
  1952. ceph_oid_printf(&src_oid, "%llx.%08llx",
  1953. src_ci->i_vino.ino, src_objnum);
  1954. ceph_oid_init(&dst_oid);
  1955. ceph_oid_printf(&dst_oid, "%llx.%08llx",
  1956. dst_ci->i_vino.ino, dst_objnum);
  1957. /* Do an object remote copy */
  1958. ret = ceph_osdc_copy_from(&fsc->client->osdc,
  1959. src_ci->i_vino.snap, 0,
  1960. &src_oid, &src_oloc,
  1961. CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL |
  1962. CEPH_OSD_OP_FLAG_FADVISE_NOCACHE,
  1963. &dst_oid, &dst_oloc,
  1964. CEPH_OSD_OP_FLAG_FADVISE_SEQUENTIAL |
  1965. CEPH_OSD_OP_FLAG_FADVISE_DONTNEED,
  1966. dst_ci->i_truncate_seq,
  1967. dst_ci->i_truncate_size,
  1968. CEPH_OSD_COPY_FROM_FLAG_TRUNCATE_SEQ);
  1969. if (ret) {
  1970. if (ret == -EOPNOTSUPP) {
  1971. fsc->have_copy_from2 = false;
  1972. pr_notice("OSDs don't support copy-from2; disabling copy offload\n");
  1973. }
  1974. dout("ceph_osdc_copy_from returned %d\n", ret);
  1975. if (!bytes)
  1976. bytes = ret;
  1977. goto out;
  1978. }
  1979. len -= object_size;
  1980. bytes += object_size;
  1981. *src_off += object_size;
  1982. *dst_off += object_size;
  1983. }
  1984. out:
  1985. ceph_oloc_destroy(&src_oloc);
  1986. ceph_oloc_destroy(&dst_oloc);
  1987. return bytes;
  1988. }
  1989. static ssize_t __ceph_copy_file_range(struct file *src_file, loff_t src_off,
  1990. struct file *dst_file, loff_t dst_off,
  1991. size_t len, unsigned int flags)
  1992. {
  1993. struct inode *src_inode = file_inode(src_file);
  1994. struct inode *dst_inode = file_inode(dst_file);
  1995. struct ceph_inode_info *src_ci = ceph_inode(src_inode);
  1996. struct ceph_inode_info *dst_ci = ceph_inode(dst_inode);
  1997. struct ceph_cap_flush *prealloc_cf;
  1998. struct ceph_fs_client *src_fsc = ceph_inode_to_client(src_inode);
  1999. loff_t size;
  2000. ssize_t ret = -EIO, bytes;
  2001. u64 src_objnum, dst_objnum, src_objoff, dst_objoff;
  2002. u32 src_objlen, dst_objlen;
  2003. int src_got = 0, dst_got = 0, err, dirty;
  2004. if (src_inode->i_sb != dst_inode->i_sb) {
  2005. struct ceph_fs_client *dst_fsc = ceph_inode_to_client(dst_inode);
  2006. if (ceph_fsid_compare(&src_fsc->client->fsid,
  2007. &dst_fsc->client->fsid)) {
  2008. dout("Copying files across clusters: src: %pU dst: %pU\n",
  2009. &src_fsc->client->fsid, &dst_fsc->client->fsid);
  2010. return -EXDEV;
  2011. }
  2012. }
  2013. if (ceph_snap(dst_inode) != CEPH_NOSNAP)
  2014. return -EROFS;
  2015. /*
  2016. * Some of the checks below will return -EOPNOTSUPP, which will force a
  2017. * fallback to the default VFS copy_file_range implementation. This is
  2018. * desirable in several cases (for ex, the 'len' is smaller than the
  2019. * size of the objects, or in cases where that would be more
  2020. * efficient).
  2021. */
  2022. if (ceph_test_mount_opt(src_fsc, NOCOPYFROM))
  2023. return -EOPNOTSUPP;
  2024. if (!src_fsc->have_copy_from2)
  2025. return -EOPNOTSUPP;
  2026. /*
  2027. * Striped file layouts require that we copy partial objects, but the
  2028. * OSD copy-from operation only supports full-object copies. Limit
  2029. * this to non-striped file layouts for now.
  2030. */
  2031. if ((src_ci->i_layout.stripe_unit != dst_ci->i_layout.stripe_unit) ||
  2032. (src_ci->i_layout.stripe_count != 1) ||
  2033. (dst_ci->i_layout.stripe_count != 1) ||
  2034. (src_ci->i_layout.object_size != dst_ci->i_layout.object_size)) {
  2035. dout("Invalid src/dst files layout\n");
  2036. return -EOPNOTSUPP;
  2037. }
  2038. if (len < src_ci->i_layout.object_size)
  2039. return -EOPNOTSUPP; /* no remote copy will be done */
  2040. prealloc_cf = ceph_alloc_cap_flush();
  2041. if (!prealloc_cf)
  2042. return -ENOMEM;
  2043. /* Start by sync'ing the source and destination files */
  2044. ret = file_write_and_wait_range(src_file, src_off, (src_off + len));
  2045. if (ret < 0) {
  2046. dout("failed to write src file (%zd)\n", ret);
  2047. goto out;
  2048. }
  2049. ret = file_write_and_wait_range(dst_file, dst_off, (dst_off + len));
  2050. if (ret < 0) {
  2051. dout("failed to write dst file (%zd)\n", ret);
  2052. goto out;
  2053. }
  2054. /*
  2055. * We need FILE_WR caps for dst_ci and FILE_RD for src_ci as other
  2056. * clients may have dirty data in their caches. And OSDs know nothing
  2057. * about caps, so they can't safely do the remote object copies.
  2058. */
  2059. err = get_rd_wr_caps(src_file, &src_got,
  2060. dst_file, (dst_off + len), &dst_got);
  2061. if (err < 0) {
  2062. dout("get_rd_wr_caps returned %d\n", err);
  2063. ret = -EOPNOTSUPP;
  2064. goto out;
  2065. }
  2066. ret = is_file_size_ok(src_inode, dst_inode, src_off, dst_off, len);
  2067. if (ret < 0)
  2068. goto out_caps;
  2069. /* Drop dst file cached pages */
  2070. ret = invalidate_inode_pages2_range(dst_inode->i_mapping,
  2071. dst_off >> PAGE_SHIFT,
  2072. (dst_off + len) >> PAGE_SHIFT);
  2073. if (ret < 0) {
  2074. dout("Failed to invalidate inode pages (%zd)\n", ret);
  2075. ret = 0; /* XXX */
  2076. }
  2077. ceph_calc_file_object_mapping(&src_ci->i_layout, src_off,
  2078. src_ci->i_layout.object_size,
  2079. &src_objnum, &src_objoff, &src_objlen);
  2080. ceph_calc_file_object_mapping(&dst_ci->i_layout, dst_off,
  2081. dst_ci->i_layout.object_size,
  2082. &dst_objnum, &dst_objoff, &dst_objlen);
  2083. /* object-level offsets need to the same */
  2084. if (src_objoff != dst_objoff) {
  2085. ret = -EOPNOTSUPP;
  2086. goto out_caps;
  2087. }
  2088. /*
  2089. * Do a manual copy if the object offset isn't object aligned.
  2090. * 'src_objlen' contains the bytes left until the end of the object,
  2091. * starting at the src_off
  2092. */
  2093. if (src_objoff) {
  2094. dout("Initial partial copy of %u bytes\n", src_objlen);
  2095. /*
  2096. * we need to temporarily drop all caps as we'll be calling
  2097. * {read,write}_iter, which will get caps again.
  2098. */
  2099. put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
  2100. ret = do_splice_direct(src_file, &src_off, dst_file,
  2101. &dst_off, src_objlen, flags);
  2102. /* Abort on short copies or on error */
  2103. if (ret < src_objlen) {
  2104. dout("Failed partial copy (%zd)\n", ret);
  2105. goto out;
  2106. }
  2107. len -= ret;
  2108. err = get_rd_wr_caps(src_file, &src_got,
  2109. dst_file, (dst_off + len), &dst_got);
  2110. if (err < 0)
  2111. goto out;
  2112. err = is_file_size_ok(src_inode, dst_inode,
  2113. src_off, dst_off, len);
  2114. if (err < 0)
  2115. goto out_caps;
  2116. }
  2117. size = i_size_read(dst_inode);
  2118. bytes = ceph_do_objects_copy(src_ci, &src_off, dst_ci, &dst_off,
  2119. src_fsc, len, flags);
  2120. if (bytes <= 0) {
  2121. if (!ret)
  2122. ret = bytes;
  2123. goto out_caps;
  2124. }
  2125. dout("Copied %zu bytes out of %zu\n", bytes, len);
  2126. len -= bytes;
  2127. ret += bytes;
  2128. file_update_time(dst_file);
  2129. inode_inc_iversion_raw(dst_inode);
  2130. if (dst_off > size) {
  2131. /* Let the MDS know about dst file size change */
  2132. if (ceph_inode_set_size(dst_inode, dst_off) ||
  2133. ceph_quota_is_max_bytes_approaching(dst_inode, dst_off))
  2134. ceph_check_caps(dst_ci, CHECK_CAPS_AUTHONLY, NULL);
  2135. }
  2136. /* Mark Fw dirty */
  2137. spin_lock(&dst_ci->i_ceph_lock);
  2138. dst_ci->i_inline_version = CEPH_INLINE_NONE;
  2139. dirty = __ceph_mark_dirty_caps(dst_ci, CEPH_CAP_FILE_WR, &prealloc_cf);
  2140. spin_unlock(&dst_ci->i_ceph_lock);
  2141. if (dirty)
  2142. __mark_inode_dirty(dst_inode, dirty);
  2143. out_caps:
  2144. put_rd_wr_caps(src_ci, src_got, dst_ci, dst_got);
  2145. /*
  2146. * Do the final manual copy if we still have some bytes left, unless
  2147. * there were errors in remote object copies (len >= object_size).
  2148. */
  2149. if (len && (len < src_ci->i_layout.object_size)) {
  2150. dout("Final partial copy of %zu bytes\n", len);
  2151. bytes = do_splice_direct(src_file, &src_off, dst_file,
  2152. &dst_off, len, flags);
  2153. if (bytes > 0)
  2154. ret += bytes;
  2155. else
  2156. dout("Failed partial copy (%zd)\n", bytes);
  2157. }
  2158. out:
  2159. ceph_free_cap_flush(prealloc_cf);
  2160. return ret;
  2161. }
  2162. static ssize_t ceph_copy_file_range(struct file *src_file, loff_t src_off,
  2163. struct file *dst_file, loff_t dst_off,
  2164. size_t len, unsigned int flags)
  2165. {
  2166. ssize_t ret;
  2167. ret = __ceph_copy_file_range(src_file, src_off, dst_file, dst_off,
  2168. len, flags);
  2169. if (ret == -EOPNOTSUPP || ret == -EXDEV)
  2170. ret = generic_copy_file_range(src_file, src_off, dst_file,
  2171. dst_off, len, flags);
  2172. return ret;
  2173. }
  2174. const struct file_operations ceph_file_fops = {
  2175. .open = ceph_open,
  2176. .release = ceph_release,
  2177. .llseek = ceph_llseek,
  2178. .read_iter = ceph_read_iter,
  2179. .write_iter = ceph_write_iter,
  2180. .mmap = ceph_mmap,
  2181. .fsync = ceph_fsync,
  2182. .lock = ceph_lock,
  2183. .setlease = simple_nosetlease,
  2184. .flock = ceph_flock,
  2185. .splice_read = generic_file_splice_read,
  2186. .splice_write = iter_file_splice_write,
  2187. .unlocked_ioctl = ceph_ioctl,
  2188. .compat_ioctl = compat_ptr_ioctl,
  2189. .fallocate = ceph_fallocate,
  2190. .copy_file_range = ceph_copy_file_range,
  2191. };