flexfilelayout.c 66 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320132113221323132413251326132713281329133013311332133313341335133613371338133913401341134213431344134513461347134813491350135113521353135413551356135713581359136013611362136313641365136613671368136913701371137213731374137513761377137813791380138113821383138413851386138713881389139013911392139313941395139613971398139914001401140214031404140514061407140814091410141114121413141414151416141714181419142014211422142314241425142614271428142914301431143214331434143514361437143814391440144114421443144414451446144714481449145014511452145314541455145614571458145914601461146214631464146514661467146814691470147114721473147414751476147714781479148014811482148314841485148614871488148914901491149214931494149514961497149814991500150115021503150415051506150715081509151015111512151315141515151615171518151915201521152215231524152515261527152815291530153115321533153415351536153715381539154015411542154315441545154615471548154915501551155215531554155515561557155815591560156115621563156415651566156715681569157015711572157315741575157615771578157915801581158215831584158515861587158815891590159115921593159415951596159715981599160016011602160316041605160616071608160916101611161216131614161516161617161816191620162116221623162416251626162716281629163016311632163316341635163616371638163916401641164216431644164516461647164816491650165116521653165416551656165716581659166016611662166316641665166616671668166916701671167216731674167516761677167816791680168116821683168416851686168716881689169016911692169316941695169616971698169917001701170217031704170517061707170817091710171117121713171417151716171717181719172017211722172317241725172617271728172917301731173217331734173517361737173817391740174117421743174417451746174717481749175017511752175317541755175617571758175917601761176217631764176517661767176817691770177117721773177417751776177717781779178017811782178317841785178617871788178917901791179217931794179517961797179817991800180118021803180418051806180718081809181018111812181318141815181618171818181918201821182218231824182518261827182818291830183118321833183418351836183718381839184018411842184318441845184618471848184918501851185218531854185518561857185818591860186118621863186418651866186718681869187018711872187318741875187618771878187918801881188218831884188518861887188818891890189118921893189418951896189718981899190019011902190319041905190619071908190919101911191219131914191519161917191819191920192119221923192419251926192719281929193019311932193319341935193619371938193919401941194219431944194519461947194819491950195119521953195419551956195719581959196019611962196319641965196619671968196919701971197219731974197519761977197819791980198119821983198419851986198719881989199019911992199319941995199619971998199920002001200220032004200520062007200820092010201120122013201420152016201720182019202020212022202320242025202620272028202920302031203220332034203520362037203820392040204120422043204420452046204720482049205020512052205320542055205620572058205920602061206220632064206520662067206820692070207120722073207420752076207720782079208020812082208320842085208620872088208920902091209220932094209520962097209820992100210121022103210421052106210721082109211021112112211321142115211621172118211921202121212221232124212521262127212821292130213121322133213421352136213721382139214021412142214321442145214621472148214921502151215221532154215521562157215821592160216121622163216421652166216721682169217021712172217321742175217621772178217921802181218221832184218521862187218821892190219121922193219421952196219721982199220022012202220322042205220622072208220922102211221222132214221522162217221822192220222122222223222422252226222722282229223022312232223322342235223622372238223922402241224222432244224522462247224822492250225122522253225422552256225722582259226022612262226322642265226622672268226922702271227222732274227522762277227822792280228122822283228422852286228722882289229022912292229322942295229622972298229923002301230223032304230523062307230823092310231123122313231423152316231723182319232023212322232323242325232623272328232923302331233223332334233523362337233823392340234123422343234423452346234723482349235023512352235323542355235623572358235923602361236223632364236523662367236823692370237123722373237423752376237723782379238023812382238323842385238623872388238923902391239223932394239523962397239823992400240124022403240424052406240724082409241024112412241324142415241624172418241924202421242224232424242524262427242824292430243124322433243424352436243724382439244024412442244324442445244624472448244924502451245224532454245524562457245824592460246124622463246424652466246724682469247024712472247324742475247624772478247924802481248224832484248524862487248824892490249124922493249424952496249724982499250025012502250325042505250625072508250925102511251225132514251525162517251825192520252125222523252425252526252725282529253025312532253325342535253625372538253925402541254225432544
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Module for pnfs flexfile layout driver.
  4. *
  5. * Copyright (c) 2014, Primary Data, Inc. All rights reserved.
  6. *
  7. * Tao Peng <bergwolf@primarydata.com>
  8. */
  9. #include <linux/nfs_fs.h>
  10. #include <linux/nfs_mount.h>
  11. #include <linux/nfs_page.h>
  12. #include <linux/module.h>
  13. #include <linux/sched/mm.h>
  14. #include <linux/sunrpc/metrics.h>
  15. #include "flexfilelayout.h"
  16. #include "../nfs4session.h"
  17. #include "../nfs4idmap.h"
  18. #include "../internal.h"
  19. #include "../delegation.h"
  20. #include "../nfs4trace.h"
  21. #include "../iostat.h"
  22. #include "../nfs.h"
  23. #include "../nfs42.h"
  24. #define NFSDBG_FACILITY NFSDBG_PNFS_LD
  25. #define FF_LAYOUT_POLL_RETRY_MAX (15*HZ)
  26. #define FF_LAYOUTRETURN_MAXERR 20
  27. static unsigned short io_maxretrans;
  28. static const struct pnfs_commit_ops ff_layout_commit_ops;
  29. static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
  30. struct nfs_pgio_header *hdr);
  31. static int ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
  32. struct nfs42_layoutstat_devinfo *devinfo,
  33. int dev_limit);
  34. static void ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
  35. const struct nfs42_layoutstat_devinfo *devinfo,
  36. struct nfs4_ff_layout_mirror *mirror);
  37. static struct pnfs_layout_hdr *
  38. ff_layout_alloc_layout_hdr(struct inode *inode, gfp_t gfp_flags)
  39. {
  40. struct nfs4_flexfile_layout *ffl;
  41. ffl = kzalloc(sizeof(*ffl), gfp_flags);
  42. if (ffl) {
  43. pnfs_init_ds_commit_info(&ffl->commit_info);
  44. INIT_LIST_HEAD(&ffl->error_list);
  45. INIT_LIST_HEAD(&ffl->mirrors);
  46. ffl->last_report_time = ktime_get();
  47. ffl->commit_info.ops = &ff_layout_commit_ops;
  48. return &ffl->generic_hdr;
  49. } else
  50. return NULL;
  51. }
  52. static void
  53. ff_layout_free_layout_hdr(struct pnfs_layout_hdr *lo)
  54. {
  55. struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(lo);
  56. struct nfs4_ff_layout_ds_err *err, *n;
  57. list_for_each_entry_safe(err, n, &ffl->error_list, list) {
  58. list_del(&err->list);
  59. kfree(err);
  60. }
  61. kfree_rcu(ffl, generic_hdr.plh_rcu);
  62. }
  63. static int decode_pnfs_stateid(struct xdr_stream *xdr, nfs4_stateid *stateid)
  64. {
  65. __be32 *p;
  66. p = xdr_inline_decode(xdr, NFS4_STATEID_SIZE);
  67. if (unlikely(p == NULL))
  68. return -ENOBUFS;
  69. stateid->type = NFS4_PNFS_DS_STATEID_TYPE;
  70. memcpy(stateid->data, p, NFS4_STATEID_SIZE);
  71. dprintk("%s: stateid id= [%x%x%x%x]\n", __func__,
  72. p[0], p[1], p[2], p[3]);
  73. return 0;
  74. }
  75. static int decode_deviceid(struct xdr_stream *xdr, struct nfs4_deviceid *devid)
  76. {
  77. __be32 *p;
  78. p = xdr_inline_decode(xdr, NFS4_DEVICEID4_SIZE);
  79. if (unlikely(!p))
  80. return -ENOBUFS;
  81. memcpy(devid, p, NFS4_DEVICEID4_SIZE);
  82. nfs4_print_deviceid(devid);
  83. return 0;
  84. }
  85. static int decode_nfs_fh(struct xdr_stream *xdr, struct nfs_fh *fh)
  86. {
  87. __be32 *p;
  88. p = xdr_inline_decode(xdr, 4);
  89. if (unlikely(!p))
  90. return -ENOBUFS;
  91. fh->size = be32_to_cpup(p++);
  92. if (fh->size > NFS_MAXFHSIZE) {
  93. printk(KERN_ERR "NFS flexfiles: Too big fh received %d\n",
  94. fh->size);
  95. return -EOVERFLOW;
  96. }
  97. /* fh.data */
  98. p = xdr_inline_decode(xdr, fh->size);
  99. if (unlikely(!p))
  100. return -ENOBUFS;
  101. memcpy(&fh->data, p, fh->size);
  102. dprintk("%s: fh len %d\n", __func__, fh->size);
  103. return 0;
  104. }
  105. /*
  106. * Currently only stringified uids and gids are accepted.
  107. * I.e., kerberos is not supported to the DSes, so no pricipals.
  108. *
  109. * That means that one common function will suffice, but when
  110. * principals are added, this should be split to accomodate
  111. * calls to both nfs_map_name_to_uid() and nfs_map_group_to_gid().
  112. */
  113. static int
  114. decode_name(struct xdr_stream *xdr, u32 *id)
  115. {
  116. __be32 *p;
  117. int len;
  118. /* opaque_length(4)*/
  119. p = xdr_inline_decode(xdr, 4);
  120. if (unlikely(!p))
  121. return -ENOBUFS;
  122. len = be32_to_cpup(p++);
  123. if (len < 0)
  124. return -EINVAL;
  125. dprintk("%s: len %u\n", __func__, len);
  126. /* opaque body */
  127. p = xdr_inline_decode(xdr, len);
  128. if (unlikely(!p))
  129. return -ENOBUFS;
  130. if (!nfs_map_string_to_numeric((char *)p, len, id))
  131. return -EINVAL;
  132. return 0;
  133. }
  134. static bool ff_mirror_match_fh(const struct nfs4_ff_layout_mirror *m1,
  135. const struct nfs4_ff_layout_mirror *m2)
  136. {
  137. int i, j;
  138. if (m1->fh_versions_cnt != m2->fh_versions_cnt)
  139. return false;
  140. for (i = 0; i < m1->fh_versions_cnt; i++) {
  141. bool found_fh = false;
  142. for (j = 0; j < m2->fh_versions_cnt; j++) {
  143. if (nfs_compare_fh(&m1->fh_versions[i],
  144. &m2->fh_versions[j]) == 0) {
  145. found_fh = true;
  146. break;
  147. }
  148. }
  149. if (!found_fh)
  150. return false;
  151. }
  152. return true;
  153. }
  154. static struct nfs4_ff_layout_mirror *
  155. ff_layout_add_mirror(struct pnfs_layout_hdr *lo,
  156. struct nfs4_ff_layout_mirror *mirror)
  157. {
  158. struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
  159. struct nfs4_ff_layout_mirror *pos;
  160. struct inode *inode = lo->plh_inode;
  161. spin_lock(&inode->i_lock);
  162. list_for_each_entry(pos, &ff_layout->mirrors, mirrors) {
  163. if (memcmp(&mirror->devid, &pos->devid, sizeof(pos->devid)) != 0)
  164. continue;
  165. if (!ff_mirror_match_fh(mirror, pos))
  166. continue;
  167. if (refcount_inc_not_zero(&pos->ref)) {
  168. spin_unlock(&inode->i_lock);
  169. return pos;
  170. }
  171. }
  172. list_add(&mirror->mirrors, &ff_layout->mirrors);
  173. mirror->layout = lo;
  174. spin_unlock(&inode->i_lock);
  175. return mirror;
  176. }
  177. static void
  178. ff_layout_remove_mirror(struct nfs4_ff_layout_mirror *mirror)
  179. {
  180. struct inode *inode;
  181. if (mirror->layout == NULL)
  182. return;
  183. inode = mirror->layout->plh_inode;
  184. spin_lock(&inode->i_lock);
  185. list_del(&mirror->mirrors);
  186. spin_unlock(&inode->i_lock);
  187. mirror->layout = NULL;
  188. }
  189. static struct nfs4_ff_layout_mirror *ff_layout_alloc_mirror(gfp_t gfp_flags)
  190. {
  191. struct nfs4_ff_layout_mirror *mirror;
  192. mirror = kzalloc(sizeof(*mirror), gfp_flags);
  193. if (mirror != NULL) {
  194. spin_lock_init(&mirror->lock);
  195. refcount_set(&mirror->ref, 1);
  196. INIT_LIST_HEAD(&mirror->mirrors);
  197. }
  198. return mirror;
  199. }
  200. static void ff_layout_free_mirror(struct nfs4_ff_layout_mirror *mirror)
  201. {
  202. const struct cred *cred;
  203. ff_layout_remove_mirror(mirror);
  204. kfree(mirror->fh_versions);
  205. cred = rcu_access_pointer(mirror->ro_cred);
  206. put_cred(cred);
  207. cred = rcu_access_pointer(mirror->rw_cred);
  208. put_cred(cred);
  209. nfs4_ff_layout_put_deviceid(mirror->mirror_ds);
  210. kfree(mirror);
  211. }
  212. static void ff_layout_put_mirror(struct nfs4_ff_layout_mirror *mirror)
  213. {
  214. if (mirror != NULL && refcount_dec_and_test(&mirror->ref))
  215. ff_layout_free_mirror(mirror);
  216. }
  217. static void ff_layout_free_mirror_array(struct nfs4_ff_layout_segment *fls)
  218. {
  219. u32 i;
  220. for (i = 0; i < fls->mirror_array_cnt; i++)
  221. ff_layout_put_mirror(fls->mirror_array[i]);
  222. }
  223. static void _ff_layout_free_lseg(struct nfs4_ff_layout_segment *fls)
  224. {
  225. if (fls) {
  226. ff_layout_free_mirror_array(fls);
  227. kfree(fls);
  228. }
  229. }
  230. static bool
  231. ff_lseg_match_mirrors(struct pnfs_layout_segment *l1,
  232. struct pnfs_layout_segment *l2)
  233. {
  234. const struct nfs4_ff_layout_segment *fl1 = FF_LAYOUT_LSEG(l1);
  235. const struct nfs4_ff_layout_segment *fl2 = FF_LAYOUT_LSEG(l1);
  236. u32 i;
  237. if (fl1->mirror_array_cnt != fl2->mirror_array_cnt)
  238. return false;
  239. for (i = 0; i < fl1->mirror_array_cnt; i++) {
  240. if (fl1->mirror_array[i] != fl2->mirror_array[i])
  241. return false;
  242. }
  243. return true;
  244. }
  245. static bool
  246. ff_lseg_range_is_after(const struct pnfs_layout_range *l1,
  247. const struct pnfs_layout_range *l2)
  248. {
  249. u64 end1, end2;
  250. if (l1->iomode != l2->iomode)
  251. return l1->iomode != IOMODE_READ;
  252. end1 = pnfs_calc_offset_end(l1->offset, l1->length);
  253. end2 = pnfs_calc_offset_end(l2->offset, l2->length);
  254. if (end1 < l2->offset)
  255. return false;
  256. if (end2 < l1->offset)
  257. return true;
  258. return l2->offset <= l1->offset;
  259. }
  260. static bool
  261. ff_lseg_merge(struct pnfs_layout_segment *new,
  262. struct pnfs_layout_segment *old)
  263. {
  264. u64 new_end, old_end;
  265. if (test_bit(NFS_LSEG_LAYOUTRETURN, &old->pls_flags))
  266. return false;
  267. if (new->pls_range.iomode != old->pls_range.iomode)
  268. return false;
  269. old_end = pnfs_calc_offset_end(old->pls_range.offset,
  270. old->pls_range.length);
  271. if (old_end < new->pls_range.offset)
  272. return false;
  273. new_end = pnfs_calc_offset_end(new->pls_range.offset,
  274. new->pls_range.length);
  275. if (new_end < old->pls_range.offset)
  276. return false;
  277. if (!ff_lseg_match_mirrors(new, old))
  278. return false;
  279. /* Mergeable: copy info from 'old' to 'new' */
  280. if (new_end < old_end)
  281. new_end = old_end;
  282. if (new->pls_range.offset < old->pls_range.offset)
  283. new->pls_range.offset = old->pls_range.offset;
  284. new->pls_range.length = pnfs_calc_offset_length(new->pls_range.offset,
  285. new_end);
  286. if (test_bit(NFS_LSEG_ROC, &old->pls_flags))
  287. set_bit(NFS_LSEG_ROC, &new->pls_flags);
  288. return true;
  289. }
  290. static void
  291. ff_layout_add_lseg(struct pnfs_layout_hdr *lo,
  292. struct pnfs_layout_segment *lseg,
  293. struct list_head *free_me)
  294. {
  295. pnfs_generic_layout_insert_lseg(lo, lseg,
  296. ff_lseg_range_is_after,
  297. ff_lseg_merge,
  298. free_me);
  299. }
  300. static void ff_layout_sort_mirrors(struct nfs4_ff_layout_segment *fls)
  301. {
  302. int i, j;
  303. for (i = 0; i < fls->mirror_array_cnt - 1; i++) {
  304. for (j = i + 1; j < fls->mirror_array_cnt; j++)
  305. if (fls->mirror_array[i]->efficiency <
  306. fls->mirror_array[j]->efficiency)
  307. swap(fls->mirror_array[i],
  308. fls->mirror_array[j]);
  309. }
  310. }
  311. static struct pnfs_layout_segment *
  312. ff_layout_alloc_lseg(struct pnfs_layout_hdr *lh,
  313. struct nfs4_layoutget_res *lgr,
  314. gfp_t gfp_flags)
  315. {
  316. struct pnfs_layout_segment *ret;
  317. struct nfs4_ff_layout_segment *fls = NULL;
  318. struct xdr_stream stream;
  319. struct xdr_buf buf;
  320. struct page *scratch;
  321. u64 stripe_unit;
  322. u32 mirror_array_cnt;
  323. __be32 *p;
  324. int i, rc;
  325. dprintk("--> %s\n", __func__);
  326. scratch = alloc_page(gfp_flags);
  327. if (!scratch)
  328. return ERR_PTR(-ENOMEM);
  329. xdr_init_decode_pages(&stream, &buf, lgr->layoutp->pages,
  330. lgr->layoutp->len);
  331. xdr_set_scratch_buffer(&stream, page_address(scratch), PAGE_SIZE);
  332. /* stripe unit and mirror_array_cnt */
  333. rc = -EIO;
  334. p = xdr_inline_decode(&stream, 8 + 4);
  335. if (!p)
  336. goto out_err_free;
  337. p = xdr_decode_hyper(p, &stripe_unit);
  338. mirror_array_cnt = be32_to_cpup(p++);
  339. dprintk("%s: stripe_unit=%llu mirror_array_cnt=%u\n", __func__,
  340. stripe_unit, mirror_array_cnt);
  341. if (mirror_array_cnt > NFS4_FLEXFILE_LAYOUT_MAX_MIRROR_CNT ||
  342. mirror_array_cnt == 0)
  343. goto out_err_free;
  344. rc = -ENOMEM;
  345. fls = kzalloc(struct_size(fls, mirror_array, mirror_array_cnt),
  346. gfp_flags);
  347. if (!fls)
  348. goto out_err_free;
  349. fls->mirror_array_cnt = mirror_array_cnt;
  350. fls->stripe_unit = stripe_unit;
  351. for (i = 0; i < fls->mirror_array_cnt; i++) {
  352. struct nfs4_ff_layout_mirror *mirror;
  353. struct cred *kcred;
  354. const struct cred __rcu *cred;
  355. kuid_t uid;
  356. kgid_t gid;
  357. u32 ds_count, fh_count, id;
  358. int j;
  359. rc = -EIO;
  360. p = xdr_inline_decode(&stream, 4);
  361. if (!p)
  362. goto out_err_free;
  363. ds_count = be32_to_cpup(p);
  364. /* FIXME: allow for striping? */
  365. if (ds_count != 1)
  366. goto out_err_free;
  367. fls->mirror_array[i] = ff_layout_alloc_mirror(gfp_flags);
  368. if (fls->mirror_array[i] == NULL) {
  369. rc = -ENOMEM;
  370. goto out_err_free;
  371. }
  372. fls->mirror_array[i]->ds_count = ds_count;
  373. /* deviceid */
  374. rc = decode_deviceid(&stream, &fls->mirror_array[i]->devid);
  375. if (rc)
  376. goto out_err_free;
  377. /* efficiency */
  378. rc = -EIO;
  379. p = xdr_inline_decode(&stream, 4);
  380. if (!p)
  381. goto out_err_free;
  382. fls->mirror_array[i]->efficiency = be32_to_cpup(p);
  383. /* stateid */
  384. rc = decode_pnfs_stateid(&stream, &fls->mirror_array[i]->stateid);
  385. if (rc)
  386. goto out_err_free;
  387. /* fh */
  388. rc = -EIO;
  389. p = xdr_inline_decode(&stream, 4);
  390. if (!p)
  391. goto out_err_free;
  392. fh_count = be32_to_cpup(p);
  393. fls->mirror_array[i]->fh_versions =
  394. kcalloc(fh_count, sizeof(struct nfs_fh),
  395. gfp_flags);
  396. if (fls->mirror_array[i]->fh_versions == NULL) {
  397. rc = -ENOMEM;
  398. goto out_err_free;
  399. }
  400. for (j = 0; j < fh_count; j++) {
  401. rc = decode_nfs_fh(&stream,
  402. &fls->mirror_array[i]->fh_versions[j]);
  403. if (rc)
  404. goto out_err_free;
  405. }
  406. fls->mirror_array[i]->fh_versions_cnt = fh_count;
  407. /* user */
  408. rc = decode_name(&stream, &id);
  409. if (rc)
  410. goto out_err_free;
  411. uid = make_kuid(&init_user_ns, id);
  412. /* group */
  413. rc = decode_name(&stream, &id);
  414. if (rc)
  415. goto out_err_free;
  416. gid = make_kgid(&init_user_ns, id);
  417. if (gfp_flags & __GFP_FS)
  418. kcred = prepare_kernel_cred(NULL);
  419. else {
  420. unsigned int nofs_flags = memalloc_nofs_save();
  421. kcred = prepare_kernel_cred(NULL);
  422. memalloc_nofs_restore(nofs_flags);
  423. }
  424. rc = -ENOMEM;
  425. if (!kcred)
  426. goto out_err_free;
  427. kcred->fsuid = uid;
  428. kcred->fsgid = gid;
  429. cred = RCU_INITIALIZER(kcred);
  430. if (lgr->range.iomode == IOMODE_READ)
  431. rcu_assign_pointer(fls->mirror_array[i]->ro_cred, cred);
  432. else
  433. rcu_assign_pointer(fls->mirror_array[i]->rw_cred, cred);
  434. mirror = ff_layout_add_mirror(lh, fls->mirror_array[i]);
  435. if (mirror != fls->mirror_array[i]) {
  436. /* swap cred ptrs so free_mirror will clean up old */
  437. if (lgr->range.iomode == IOMODE_READ) {
  438. cred = xchg(&mirror->ro_cred, cred);
  439. rcu_assign_pointer(fls->mirror_array[i]->ro_cred, cred);
  440. } else {
  441. cred = xchg(&mirror->rw_cred, cred);
  442. rcu_assign_pointer(fls->mirror_array[i]->rw_cred, cred);
  443. }
  444. ff_layout_free_mirror(fls->mirror_array[i]);
  445. fls->mirror_array[i] = mirror;
  446. }
  447. dprintk("%s: iomode %s uid %u gid %u\n", __func__,
  448. lgr->range.iomode == IOMODE_READ ? "READ" : "RW",
  449. from_kuid(&init_user_ns, uid),
  450. from_kgid(&init_user_ns, gid));
  451. }
  452. p = xdr_inline_decode(&stream, 4);
  453. if (!p)
  454. goto out_sort_mirrors;
  455. fls->flags = be32_to_cpup(p);
  456. p = xdr_inline_decode(&stream, 4);
  457. if (!p)
  458. goto out_sort_mirrors;
  459. for (i=0; i < fls->mirror_array_cnt; i++)
  460. fls->mirror_array[i]->report_interval = be32_to_cpup(p);
  461. out_sort_mirrors:
  462. ff_layout_sort_mirrors(fls);
  463. ret = &fls->generic_hdr;
  464. dprintk("<-- %s (success)\n", __func__);
  465. out_free_page:
  466. __free_page(scratch);
  467. return ret;
  468. out_err_free:
  469. _ff_layout_free_lseg(fls);
  470. ret = ERR_PTR(rc);
  471. dprintk("<-- %s (%d)\n", __func__, rc);
  472. goto out_free_page;
  473. }
  474. static void
  475. ff_layout_free_lseg(struct pnfs_layout_segment *lseg)
  476. {
  477. struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
  478. dprintk("--> %s\n", __func__);
  479. if (lseg->pls_range.iomode == IOMODE_RW) {
  480. struct nfs4_flexfile_layout *ffl;
  481. struct inode *inode;
  482. ffl = FF_LAYOUT_FROM_HDR(lseg->pls_layout);
  483. inode = ffl->generic_hdr.plh_inode;
  484. spin_lock(&inode->i_lock);
  485. pnfs_generic_ds_cinfo_release_lseg(&ffl->commit_info, lseg);
  486. spin_unlock(&inode->i_lock);
  487. }
  488. _ff_layout_free_lseg(fls);
  489. }
  490. static void
  491. nfs4_ff_start_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now)
  492. {
  493. /* first IO request? */
  494. if (atomic_inc_return(&timer->n_ops) == 1) {
  495. timer->start_time = now;
  496. }
  497. }
  498. static ktime_t
  499. nfs4_ff_end_busy_timer(struct nfs4_ff_busy_timer *timer, ktime_t now)
  500. {
  501. ktime_t start;
  502. if (atomic_dec_return(&timer->n_ops) < 0)
  503. WARN_ON_ONCE(1);
  504. start = timer->start_time;
  505. timer->start_time = now;
  506. return ktime_sub(now, start);
  507. }
  508. static bool
  509. nfs4_ff_layoutstat_start_io(struct nfs4_ff_layout_mirror *mirror,
  510. struct nfs4_ff_layoutstat *layoutstat,
  511. ktime_t now)
  512. {
  513. s64 report_interval = FF_LAYOUTSTATS_REPORT_INTERVAL;
  514. struct nfs4_flexfile_layout *ffl = FF_LAYOUT_FROM_HDR(mirror->layout);
  515. nfs4_ff_start_busy_timer(&layoutstat->busy_timer, now);
  516. if (!mirror->start_time)
  517. mirror->start_time = now;
  518. if (mirror->report_interval != 0)
  519. report_interval = (s64)mirror->report_interval * 1000LL;
  520. else if (layoutstats_timer != 0)
  521. report_interval = (s64)layoutstats_timer * 1000LL;
  522. if (ktime_to_ms(ktime_sub(now, ffl->last_report_time)) >=
  523. report_interval) {
  524. ffl->last_report_time = now;
  525. return true;
  526. }
  527. return false;
  528. }
  529. static void
  530. nfs4_ff_layout_stat_io_update_requested(struct nfs4_ff_layoutstat *layoutstat,
  531. __u64 requested)
  532. {
  533. struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat;
  534. iostat->ops_requested++;
  535. iostat->bytes_requested += requested;
  536. }
  537. static void
  538. nfs4_ff_layout_stat_io_update_completed(struct nfs4_ff_layoutstat *layoutstat,
  539. __u64 requested,
  540. __u64 completed,
  541. ktime_t time_completed,
  542. ktime_t time_started)
  543. {
  544. struct nfs4_ff_io_stat *iostat = &layoutstat->io_stat;
  545. ktime_t completion_time = ktime_sub(time_completed, time_started);
  546. ktime_t timer;
  547. iostat->ops_completed++;
  548. iostat->bytes_completed += completed;
  549. iostat->bytes_not_delivered += requested - completed;
  550. timer = nfs4_ff_end_busy_timer(&layoutstat->busy_timer, time_completed);
  551. iostat->total_busy_time =
  552. ktime_add(iostat->total_busy_time, timer);
  553. iostat->aggregate_completion_time =
  554. ktime_add(iostat->aggregate_completion_time,
  555. completion_time);
  556. }
  557. static void
  558. nfs4_ff_layout_stat_io_start_read(struct inode *inode,
  559. struct nfs4_ff_layout_mirror *mirror,
  560. __u64 requested, ktime_t now)
  561. {
  562. bool report;
  563. spin_lock(&mirror->lock);
  564. report = nfs4_ff_layoutstat_start_io(mirror, &mirror->read_stat, now);
  565. nfs4_ff_layout_stat_io_update_requested(&mirror->read_stat, requested);
  566. set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
  567. spin_unlock(&mirror->lock);
  568. if (report)
  569. pnfs_report_layoutstat(inode, GFP_KERNEL);
  570. }
  571. static void
  572. nfs4_ff_layout_stat_io_end_read(struct rpc_task *task,
  573. struct nfs4_ff_layout_mirror *mirror,
  574. __u64 requested,
  575. __u64 completed)
  576. {
  577. spin_lock(&mirror->lock);
  578. nfs4_ff_layout_stat_io_update_completed(&mirror->read_stat,
  579. requested, completed,
  580. ktime_get(), task->tk_start);
  581. set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
  582. spin_unlock(&mirror->lock);
  583. }
  584. static void
  585. nfs4_ff_layout_stat_io_start_write(struct inode *inode,
  586. struct nfs4_ff_layout_mirror *mirror,
  587. __u64 requested, ktime_t now)
  588. {
  589. bool report;
  590. spin_lock(&mirror->lock);
  591. report = nfs4_ff_layoutstat_start_io(mirror , &mirror->write_stat, now);
  592. nfs4_ff_layout_stat_io_update_requested(&mirror->write_stat, requested);
  593. set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
  594. spin_unlock(&mirror->lock);
  595. if (report)
  596. pnfs_report_layoutstat(inode, GFP_NOIO);
  597. }
  598. static void
  599. nfs4_ff_layout_stat_io_end_write(struct rpc_task *task,
  600. struct nfs4_ff_layout_mirror *mirror,
  601. __u64 requested,
  602. __u64 completed,
  603. enum nfs3_stable_how committed)
  604. {
  605. if (committed == NFS_UNSTABLE)
  606. requested = completed = 0;
  607. spin_lock(&mirror->lock);
  608. nfs4_ff_layout_stat_io_update_completed(&mirror->write_stat,
  609. requested, completed, ktime_get(), task->tk_start);
  610. set_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags);
  611. spin_unlock(&mirror->lock);
  612. }
  613. static void
  614. ff_layout_mark_ds_unreachable(struct pnfs_layout_segment *lseg, u32 idx)
  615. {
  616. struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
  617. if (devid)
  618. nfs4_mark_deviceid_unavailable(devid);
  619. }
  620. static void
  621. ff_layout_mark_ds_reachable(struct pnfs_layout_segment *lseg, u32 idx)
  622. {
  623. struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
  624. if (devid)
  625. nfs4_mark_deviceid_available(devid);
  626. }
  627. static struct nfs4_pnfs_ds *
  628. ff_layout_choose_ds_for_read(struct pnfs_layout_segment *lseg,
  629. u32 start_idx, u32 *best_idx,
  630. bool check_device)
  631. {
  632. struct nfs4_ff_layout_segment *fls = FF_LAYOUT_LSEG(lseg);
  633. struct nfs4_ff_layout_mirror *mirror;
  634. struct nfs4_pnfs_ds *ds;
  635. bool fail_return = false;
  636. u32 idx;
  637. /* mirrors are initially sorted by efficiency */
  638. for (idx = start_idx; idx < fls->mirror_array_cnt; idx++) {
  639. if (idx+1 == fls->mirror_array_cnt)
  640. fail_return = !check_device;
  641. mirror = FF_LAYOUT_COMP(lseg, idx);
  642. ds = nfs4_ff_layout_prepare_ds(lseg, mirror, fail_return);
  643. if (!ds)
  644. continue;
  645. if (check_device &&
  646. nfs4_test_deviceid_unavailable(&mirror->mirror_ds->id_node))
  647. continue;
  648. *best_idx = idx;
  649. return ds;
  650. }
  651. return NULL;
  652. }
  653. static struct nfs4_pnfs_ds *
  654. ff_layout_choose_any_ds_for_read(struct pnfs_layout_segment *lseg,
  655. u32 start_idx, u32 *best_idx)
  656. {
  657. return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, false);
  658. }
  659. static struct nfs4_pnfs_ds *
  660. ff_layout_choose_valid_ds_for_read(struct pnfs_layout_segment *lseg,
  661. u32 start_idx, u32 *best_idx)
  662. {
  663. return ff_layout_choose_ds_for_read(lseg, start_idx, best_idx, true);
  664. }
  665. static struct nfs4_pnfs_ds *
  666. ff_layout_choose_best_ds_for_read(struct pnfs_layout_segment *lseg,
  667. u32 start_idx, u32 *best_idx)
  668. {
  669. struct nfs4_pnfs_ds *ds;
  670. ds = ff_layout_choose_valid_ds_for_read(lseg, start_idx, best_idx);
  671. if (ds)
  672. return ds;
  673. return ff_layout_choose_any_ds_for_read(lseg, start_idx, best_idx);
  674. }
  675. static struct nfs4_pnfs_ds *
  676. ff_layout_get_ds_for_read(struct nfs_pageio_descriptor *pgio,
  677. u32 *best_idx)
  678. {
  679. struct pnfs_layout_segment *lseg = pgio->pg_lseg;
  680. struct nfs4_pnfs_ds *ds;
  681. ds = ff_layout_choose_best_ds_for_read(lseg, pgio->pg_mirror_idx,
  682. best_idx);
  683. if (ds || !pgio->pg_mirror_idx)
  684. return ds;
  685. return ff_layout_choose_best_ds_for_read(lseg, 0, best_idx);
  686. }
  687. static void
  688. ff_layout_pg_get_read(struct nfs_pageio_descriptor *pgio,
  689. struct nfs_page *req,
  690. bool strict_iomode)
  691. {
  692. pnfs_put_lseg(pgio->pg_lseg);
  693. pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
  694. nfs_req_openctx(req),
  695. req_offset(req),
  696. req->wb_bytes,
  697. IOMODE_READ,
  698. strict_iomode,
  699. GFP_KERNEL);
  700. if (IS_ERR(pgio->pg_lseg)) {
  701. pgio->pg_error = PTR_ERR(pgio->pg_lseg);
  702. pgio->pg_lseg = NULL;
  703. }
  704. }
  705. static void
  706. ff_layout_pg_check_layout(struct nfs_pageio_descriptor *pgio,
  707. struct nfs_page *req)
  708. {
  709. pnfs_generic_pg_check_layout(pgio);
  710. pnfs_generic_pg_check_range(pgio, req);
  711. }
  712. static void
  713. ff_layout_pg_init_read(struct nfs_pageio_descriptor *pgio,
  714. struct nfs_page *req)
  715. {
  716. struct nfs_pgio_mirror *pgm;
  717. struct nfs4_ff_layout_mirror *mirror;
  718. struct nfs4_pnfs_ds *ds;
  719. u32 ds_idx;
  720. retry:
  721. ff_layout_pg_check_layout(pgio, req);
  722. /* Use full layout for now */
  723. if (!pgio->pg_lseg) {
  724. ff_layout_pg_get_read(pgio, req, false);
  725. if (!pgio->pg_lseg)
  726. goto out_nolseg;
  727. }
  728. if (ff_layout_avoid_read_on_rw(pgio->pg_lseg)) {
  729. ff_layout_pg_get_read(pgio, req, true);
  730. if (!pgio->pg_lseg)
  731. goto out_nolseg;
  732. }
  733. ds = ff_layout_get_ds_for_read(pgio, &ds_idx);
  734. if (!ds) {
  735. if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg))
  736. goto out_mds;
  737. pnfs_generic_pg_cleanup(pgio);
  738. /* Sleep for 1 second before retrying */
  739. ssleep(1);
  740. goto retry;
  741. }
  742. mirror = FF_LAYOUT_COMP(pgio->pg_lseg, ds_idx);
  743. pgm = &pgio->pg_mirrors[0];
  744. pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].rsize;
  745. pgio->pg_mirror_idx = ds_idx;
  746. if (NFS_SERVER(pgio->pg_inode)->flags &
  747. (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR))
  748. pgio->pg_maxretrans = io_maxretrans;
  749. return;
  750. out_nolseg:
  751. if (pgio->pg_error < 0)
  752. return;
  753. out_mds:
  754. trace_pnfs_mds_fallback_pg_init_read(pgio->pg_inode,
  755. 0, NFS4_MAX_UINT64, IOMODE_READ,
  756. NFS_I(pgio->pg_inode)->layout,
  757. pgio->pg_lseg);
  758. pgio->pg_maxretrans = 0;
  759. nfs_pageio_reset_read_mds(pgio);
  760. }
  761. static void
  762. ff_layout_pg_init_write(struct nfs_pageio_descriptor *pgio,
  763. struct nfs_page *req)
  764. {
  765. struct nfs4_ff_layout_mirror *mirror;
  766. struct nfs_pgio_mirror *pgm;
  767. struct nfs4_pnfs_ds *ds;
  768. u32 i;
  769. retry:
  770. ff_layout_pg_check_layout(pgio, req);
  771. if (!pgio->pg_lseg) {
  772. pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
  773. nfs_req_openctx(req),
  774. req_offset(req),
  775. req->wb_bytes,
  776. IOMODE_RW,
  777. false,
  778. GFP_NOFS);
  779. if (IS_ERR(pgio->pg_lseg)) {
  780. pgio->pg_error = PTR_ERR(pgio->pg_lseg);
  781. pgio->pg_lseg = NULL;
  782. return;
  783. }
  784. }
  785. /* If no lseg, fall back to write through mds */
  786. if (pgio->pg_lseg == NULL)
  787. goto out_mds;
  788. /* Use a direct mapping of ds_idx to pgio mirror_idx */
  789. if (pgio->pg_mirror_count != FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg))
  790. goto out_eagain;
  791. for (i = 0; i < pgio->pg_mirror_count; i++) {
  792. mirror = FF_LAYOUT_COMP(pgio->pg_lseg, i);
  793. ds = nfs4_ff_layout_prepare_ds(pgio->pg_lseg, mirror, true);
  794. if (!ds) {
  795. if (!ff_layout_no_fallback_to_mds(pgio->pg_lseg))
  796. goto out_mds;
  797. pnfs_generic_pg_cleanup(pgio);
  798. /* Sleep for 1 second before retrying */
  799. ssleep(1);
  800. goto retry;
  801. }
  802. pgm = &pgio->pg_mirrors[i];
  803. pgm->pg_bsize = mirror->mirror_ds->ds_versions[0].wsize;
  804. }
  805. if (NFS_SERVER(pgio->pg_inode)->flags &
  806. (NFS_MOUNT_SOFT|NFS_MOUNT_SOFTERR))
  807. pgio->pg_maxretrans = io_maxretrans;
  808. return;
  809. out_eagain:
  810. pnfs_generic_pg_cleanup(pgio);
  811. pgio->pg_error = -EAGAIN;
  812. return;
  813. out_mds:
  814. trace_pnfs_mds_fallback_pg_init_write(pgio->pg_inode,
  815. 0, NFS4_MAX_UINT64, IOMODE_RW,
  816. NFS_I(pgio->pg_inode)->layout,
  817. pgio->pg_lseg);
  818. pgio->pg_maxretrans = 0;
  819. nfs_pageio_reset_write_mds(pgio);
  820. pgio->pg_error = -EAGAIN;
  821. }
  822. static unsigned int
  823. ff_layout_pg_get_mirror_count_write(struct nfs_pageio_descriptor *pgio,
  824. struct nfs_page *req)
  825. {
  826. if (!pgio->pg_lseg) {
  827. pgio->pg_lseg = pnfs_update_layout(pgio->pg_inode,
  828. nfs_req_openctx(req),
  829. req_offset(req),
  830. req->wb_bytes,
  831. IOMODE_RW,
  832. false,
  833. GFP_NOFS);
  834. if (IS_ERR(pgio->pg_lseg)) {
  835. pgio->pg_error = PTR_ERR(pgio->pg_lseg);
  836. pgio->pg_lseg = NULL;
  837. goto out;
  838. }
  839. }
  840. if (pgio->pg_lseg)
  841. return FF_LAYOUT_MIRROR_COUNT(pgio->pg_lseg);
  842. trace_pnfs_mds_fallback_pg_get_mirror_count(pgio->pg_inode,
  843. 0, NFS4_MAX_UINT64, IOMODE_RW,
  844. NFS_I(pgio->pg_inode)->layout,
  845. pgio->pg_lseg);
  846. /* no lseg means that pnfs is not in use, so no mirroring here */
  847. nfs_pageio_reset_write_mds(pgio);
  848. out:
  849. return 1;
  850. }
  851. static u32
  852. ff_layout_pg_set_mirror_write(struct nfs_pageio_descriptor *desc, u32 idx)
  853. {
  854. u32 old = desc->pg_mirror_idx;
  855. desc->pg_mirror_idx = idx;
  856. return old;
  857. }
  858. static struct nfs_pgio_mirror *
  859. ff_layout_pg_get_mirror_write(struct nfs_pageio_descriptor *desc, u32 idx)
  860. {
  861. return &desc->pg_mirrors[idx];
  862. }
  863. static const struct nfs_pageio_ops ff_layout_pg_read_ops = {
  864. .pg_init = ff_layout_pg_init_read,
  865. .pg_test = pnfs_generic_pg_test,
  866. .pg_doio = pnfs_generic_pg_readpages,
  867. .pg_cleanup = pnfs_generic_pg_cleanup,
  868. };
  869. static const struct nfs_pageio_ops ff_layout_pg_write_ops = {
  870. .pg_init = ff_layout_pg_init_write,
  871. .pg_test = pnfs_generic_pg_test,
  872. .pg_doio = pnfs_generic_pg_writepages,
  873. .pg_get_mirror_count = ff_layout_pg_get_mirror_count_write,
  874. .pg_cleanup = pnfs_generic_pg_cleanup,
  875. .pg_get_mirror = ff_layout_pg_get_mirror_write,
  876. .pg_set_mirror = ff_layout_pg_set_mirror_write,
  877. };
  878. static void ff_layout_reset_write(struct nfs_pgio_header *hdr, bool retry_pnfs)
  879. {
  880. struct rpc_task *task = &hdr->task;
  881. pnfs_layoutcommit_inode(hdr->inode, false);
  882. if (retry_pnfs) {
  883. dprintk("%s Reset task %5u for i/o through pNFS "
  884. "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
  885. hdr->task.tk_pid,
  886. hdr->inode->i_sb->s_id,
  887. (unsigned long long)NFS_FILEID(hdr->inode),
  888. hdr->args.count,
  889. (unsigned long long)hdr->args.offset);
  890. hdr->completion_ops->reschedule_io(hdr);
  891. return;
  892. }
  893. if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
  894. dprintk("%s Reset task %5u for i/o through MDS "
  895. "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
  896. hdr->task.tk_pid,
  897. hdr->inode->i_sb->s_id,
  898. (unsigned long long)NFS_FILEID(hdr->inode),
  899. hdr->args.count,
  900. (unsigned long long)hdr->args.offset);
  901. trace_pnfs_mds_fallback_write_done(hdr->inode,
  902. hdr->args.offset, hdr->args.count,
  903. IOMODE_RW, NFS_I(hdr->inode)->layout,
  904. hdr->lseg);
  905. task->tk_status = pnfs_write_done_resend_to_mds(hdr);
  906. }
  907. }
  908. static void ff_layout_resend_pnfs_read(struct nfs_pgio_header *hdr)
  909. {
  910. u32 idx = hdr->pgio_mirror_idx + 1;
  911. u32 new_idx = 0;
  912. if (ff_layout_choose_any_ds_for_read(hdr->lseg, idx, &new_idx))
  913. ff_layout_send_layouterror(hdr->lseg);
  914. else
  915. pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg);
  916. pnfs_read_resend_pnfs(hdr, new_idx);
  917. }
  918. static void ff_layout_reset_read(struct nfs_pgio_header *hdr)
  919. {
  920. struct rpc_task *task = &hdr->task;
  921. pnfs_layoutcommit_inode(hdr->inode, false);
  922. pnfs_error_mark_layout_for_return(hdr->inode, hdr->lseg);
  923. if (!test_and_set_bit(NFS_IOHDR_REDO, &hdr->flags)) {
  924. dprintk("%s Reset task %5u for i/o through MDS "
  925. "(req %s/%llu, %u bytes @ offset %llu)\n", __func__,
  926. hdr->task.tk_pid,
  927. hdr->inode->i_sb->s_id,
  928. (unsigned long long)NFS_FILEID(hdr->inode),
  929. hdr->args.count,
  930. (unsigned long long)hdr->args.offset);
  931. trace_pnfs_mds_fallback_read_done(hdr->inode,
  932. hdr->args.offset, hdr->args.count,
  933. IOMODE_READ, NFS_I(hdr->inode)->layout,
  934. hdr->lseg);
  935. task->tk_status = pnfs_read_done_resend_to_mds(hdr);
  936. }
  937. }
  938. static int ff_layout_async_handle_error_v4(struct rpc_task *task,
  939. struct nfs4_state *state,
  940. struct nfs_client *clp,
  941. struct pnfs_layout_segment *lseg,
  942. u32 idx)
  943. {
  944. struct pnfs_layout_hdr *lo = lseg->pls_layout;
  945. struct inode *inode = lo->plh_inode;
  946. struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
  947. struct nfs4_slot_table *tbl = &clp->cl_session->fc_slot_table;
  948. switch (task->tk_status) {
  949. case -NFS4ERR_BADSESSION:
  950. case -NFS4ERR_BADSLOT:
  951. case -NFS4ERR_BAD_HIGH_SLOT:
  952. case -NFS4ERR_DEADSESSION:
  953. case -NFS4ERR_CONN_NOT_BOUND_TO_SESSION:
  954. case -NFS4ERR_SEQ_FALSE_RETRY:
  955. case -NFS4ERR_SEQ_MISORDERED:
  956. dprintk("%s ERROR %d, Reset session. Exchangeid "
  957. "flags 0x%x\n", __func__, task->tk_status,
  958. clp->cl_exchange_flags);
  959. nfs4_schedule_session_recovery(clp->cl_session, task->tk_status);
  960. break;
  961. case -NFS4ERR_DELAY:
  962. case -NFS4ERR_GRACE:
  963. rpc_delay(task, FF_LAYOUT_POLL_RETRY_MAX);
  964. break;
  965. case -NFS4ERR_RETRY_UNCACHED_REP:
  966. break;
  967. /* Invalidate Layout errors */
  968. case -NFS4ERR_PNFS_NO_LAYOUT:
  969. case -ESTALE: /* mapped NFS4ERR_STALE */
  970. case -EBADHANDLE: /* mapped NFS4ERR_BADHANDLE */
  971. case -EISDIR: /* mapped NFS4ERR_ISDIR */
  972. case -NFS4ERR_FHEXPIRED:
  973. case -NFS4ERR_WRONG_TYPE:
  974. dprintk("%s Invalid layout error %d\n", __func__,
  975. task->tk_status);
  976. /*
  977. * Destroy layout so new i/o will get a new layout.
  978. * Layout will not be destroyed until all current lseg
  979. * references are put. Mark layout as invalid to resend failed
  980. * i/o and all i/o waiting on the slot table to the MDS until
  981. * layout is destroyed and a new valid layout is obtained.
  982. */
  983. pnfs_destroy_layout(NFS_I(inode));
  984. rpc_wake_up(&tbl->slot_tbl_waitq);
  985. goto reset;
  986. /* RPC connection errors */
  987. case -ECONNREFUSED:
  988. case -EHOSTDOWN:
  989. case -EHOSTUNREACH:
  990. case -ENETUNREACH:
  991. case -EIO:
  992. case -ETIMEDOUT:
  993. case -EPIPE:
  994. dprintk("%s DS connection error %d\n", __func__,
  995. task->tk_status);
  996. nfs4_delete_deviceid(devid->ld, devid->nfs_client,
  997. &devid->deviceid);
  998. rpc_wake_up(&tbl->slot_tbl_waitq);
  999. fallthrough;
  1000. default:
  1001. if (ff_layout_avoid_mds_available_ds(lseg))
  1002. return -NFS4ERR_RESET_TO_PNFS;
  1003. reset:
  1004. dprintk("%s Retry through MDS. Error %d\n", __func__,
  1005. task->tk_status);
  1006. return -NFS4ERR_RESET_TO_MDS;
  1007. }
  1008. task->tk_status = 0;
  1009. return -EAGAIN;
  1010. }
  1011. /* Retry all errors through either pNFS or MDS except for -EJUKEBOX */
  1012. static int ff_layout_async_handle_error_v3(struct rpc_task *task,
  1013. struct pnfs_layout_segment *lseg,
  1014. u32 idx)
  1015. {
  1016. struct nfs4_deviceid_node *devid = FF_LAYOUT_DEVID_NODE(lseg, idx);
  1017. switch (task->tk_status) {
  1018. /* File access problems. Don't mark the device as unavailable */
  1019. case -EACCES:
  1020. case -ESTALE:
  1021. case -EISDIR:
  1022. case -EBADHANDLE:
  1023. case -ELOOP:
  1024. case -ENOSPC:
  1025. break;
  1026. case -EJUKEBOX:
  1027. nfs_inc_stats(lseg->pls_layout->plh_inode, NFSIOS_DELAY);
  1028. goto out_retry;
  1029. default:
  1030. dprintk("%s DS connection error %d\n", __func__,
  1031. task->tk_status);
  1032. nfs4_delete_deviceid(devid->ld, devid->nfs_client,
  1033. &devid->deviceid);
  1034. }
  1035. /* FIXME: Need to prevent infinite looping here. */
  1036. return -NFS4ERR_RESET_TO_PNFS;
  1037. out_retry:
  1038. task->tk_status = 0;
  1039. rpc_restart_call_prepare(task);
  1040. rpc_delay(task, NFS_JUKEBOX_RETRY_TIME);
  1041. return -EAGAIN;
  1042. }
  1043. static int ff_layout_async_handle_error(struct rpc_task *task,
  1044. struct nfs4_state *state,
  1045. struct nfs_client *clp,
  1046. struct pnfs_layout_segment *lseg,
  1047. u32 idx)
  1048. {
  1049. int vers = clp->cl_nfs_mod->rpc_vers->number;
  1050. if (task->tk_status >= 0) {
  1051. ff_layout_mark_ds_reachable(lseg, idx);
  1052. return 0;
  1053. }
  1054. /* Handle the case of an invalid layout segment */
  1055. if (!pnfs_is_valid_lseg(lseg))
  1056. return -NFS4ERR_RESET_TO_PNFS;
  1057. switch (vers) {
  1058. case 3:
  1059. return ff_layout_async_handle_error_v3(task, lseg, idx);
  1060. case 4:
  1061. return ff_layout_async_handle_error_v4(task, state, clp,
  1062. lseg, idx);
  1063. default:
  1064. /* should never happen */
  1065. WARN_ON_ONCE(1);
  1066. return 0;
  1067. }
  1068. }
  1069. static void ff_layout_io_track_ds_error(struct pnfs_layout_segment *lseg,
  1070. u32 idx, u64 offset, u64 length,
  1071. u32 *op_status, int opnum, int error)
  1072. {
  1073. struct nfs4_ff_layout_mirror *mirror;
  1074. u32 status = *op_status;
  1075. int err;
  1076. if (status == 0) {
  1077. switch (error) {
  1078. case -ETIMEDOUT:
  1079. case -EPFNOSUPPORT:
  1080. case -EPROTONOSUPPORT:
  1081. case -EOPNOTSUPP:
  1082. case -ECONNREFUSED:
  1083. case -ECONNRESET:
  1084. case -EHOSTDOWN:
  1085. case -EHOSTUNREACH:
  1086. case -ENETUNREACH:
  1087. case -EADDRINUSE:
  1088. case -ENOBUFS:
  1089. case -EPIPE:
  1090. case -EPERM:
  1091. *op_status = status = NFS4ERR_NXIO;
  1092. break;
  1093. case -EACCES:
  1094. *op_status = status = NFS4ERR_ACCESS;
  1095. break;
  1096. default:
  1097. return;
  1098. }
  1099. }
  1100. mirror = FF_LAYOUT_COMP(lseg, idx);
  1101. err = ff_layout_track_ds_error(FF_LAYOUT_FROM_HDR(lseg->pls_layout),
  1102. mirror, offset, length, status, opnum,
  1103. GFP_NOIO);
  1104. switch (status) {
  1105. case NFS4ERR_DELAY:
  1106. case NFS4ERR_GRACE:
  1107. break;
  1108. case NFS4ERR_NXIO:
  1109. ff_layout_mark_ds_unreachable(lseg, idx);
  1110. /*
  1111. * Don't return the layout if this is a read and we still
  1112. * have layouts to try
  1113. */
  1114. if (opnum == OP_READ)
  1115. break;
  1116. fallthrough;
  1117. default:
  1118. pnfs_error_mark_layout_for_return(lseg->pls_layout->plh_inode,
  1119. lseg);
  1120. }
  1121. dprintk("%s: err %d op %d status %u\n", __func__, err, opnum, status);
  1122. }
  1123. /* NFS_PROTO call done callback routines */
  1124. static int ff_layout_read_done_cb(struct rpc_task *task,
  1125. struct nfs_pgio_header *hdr)
  1126. {
  1127. int err;
  1128. if (task->tk_status < 0) {
  1129. ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,
  1130. hdr->args.offset, hdr->args.count,
  1131. &hdr->res.op_status, OP_READ,
  1132. task->tk_status);
  1133. trace_ff_layout_read_error(hdr);
  1134. }
  1135. err = ff_layout_async_handle_error(task, hdr->args.context->state,
  1136. hdr->ds_clp, hdr->lseg,
  1137. hdr->pgio_mirror_idx);
  1138. trace_nfs4_pnfs_read(hdr, err);
  1139. clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
  1140. clear_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags);
  1141. switch (err) {
  1142. case -NFS4ERR_RESET_TO_PNFS:
  1143. set_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
  1144. return task->tk_status;
  1145. case -NFS4ERR_RESET_TO_MDS:
  1146. set_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags);
  1147. return task->tk_status;
  1148. case -EAGAIN:
  1149. goto out_eagain;
  1150. }
  1151. return 0;
  1152. out_eagain:
  1153. rpc_restart_call_prepare(task);
  1154. return -EAGAIN;
  1155. }
  1156. static bool
  1157. ff_layout_need_layoutcommit(struct pnfs_layout_segment *lseg)
  1158. {
  1159. return !(FF_LAYOUT_LSEG(lseg)->flags & FF_FLAGS_NO_LAYOUTCOMMIT);
  1160. }
  1161. /*
  1162. * We reference the rpc_cred of the first WRITE that triggers the need for
  1163. * a LAYOUTCOMMIT, and use it to send the layoutcommit compound.
  1164. * rfc5661 is not clear about which credential should be used.
  1165. *
  1166. * Flexlayout client should treat DS replied FILE_SYNC as DATA_SYNC, so
  1167. * to follow http://www.rfc-editor.org/errata_search.php?rfc=5661&eid=2751
  1168. * we always send layoutcommit after DS writes.
  1169. */
  1170. static void
  1171. ff_layout_set_layoutcommit(struct inode *inode,
  1172. struct pnfs_layout_segment *lseg,
  1173. loff_t end_offset)
  1174. {
  1175. if (!ff_layout_need_layoutcommit(lseg))
  1176. return;
  1177. pnfs_set_layoutcommit(inode, lseg, end_offset);
  1178. dprintk("%s inode %lu pls_end_pos %llu\n", __func__, inode->i_ino,
  1179. (unsigned long long) NFS_I(inode)->layout->plh_lwb);
  1180. }
  1181. static void ff_layout_read_record_layoutstats_start(struct rpc_task *task,
  1182. struct nfs_pgio_header *hdr)
  1183. {
  1184. if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags))
  1185. return;
  1186. nfs4_ff_layout_stat_io_start_read(hdr->inode,
  1187. FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),
  1188. hdr->args.count,
  1189. task->tk_start);
  1190. }
  1191. static void ff_layout_read_record_layoutstats_done(struct rpc_task *task,
  1192. struct nfs_pgio_header *hdr)
  1193. {
  1194. if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags))
  1195. return;
  1196. nfs4_ff_layout_stat_io_end_read(task,
  1197. FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),
  1198. hdr->args.count,
  1199. hdr->res.count);
  1200. set_bit(NFS_LSEG_LAYOUTRETURN, &hdr->lseg->pls_flags);
  1201. }
  1202. static int ff_layout_read_prepare_common(struct rpc_task *task,
  1203. struct nfs_pgio_header *hdr)
  1204. {
  1205. if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {
  1206. rpc_exit(task, -EIO);
  1207. return -EIO;
  1208. }
  1209. ff_layout_read_record_layoutstats_start(task, hdr);
  1210. return 0;
  1211. }
  1212. /*
  1213. * Call ops for the async read/write cases
  1214. * In the case of dense layouts, the offset needs to be reset to its
  1215. * original value.
  1216. */
  1217. static void ff_layout_read_prepare_v3(struct rpc_task *task, void *data)
  1218. {
  1219. struct nfs_pgio_header *hdr = data;
  1220. if (ff_layout_read_prepare_common(task, hdr))
  1221. return;
  1222. rpc_call_start(task);
  1223. }
  1224. static void ff_layout_read_prepare_v4(struct rpc_task *task, void *data)
  1225. {
  1226. struct nfs_pgio_header *hdr = data;
  1227. if (nfs4_setup_sequence(hdr->ds_clp,
  1228. &hdr->args.seq_args,
  1229. &hdr->res.seq_res,
  1230. task))
  1231. return;
  1232. ff_layout_read_prepare_common(task, hdr);
  1233. }
  1234. static void ff_layout_read_call_done(struct rpc_task *task, void *data)
  1235. {
  1236. struct nfs_pgio_header *hdr = data;
  1237. dprintk("--> %s task->tk_status %d\n", __func__, task->tk_status);
  1238. if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&
  1239. task->tk_status == 0) {
  1240. nfs4_sequence_done(task, &hdr->res.seq_res);
  1241. return;
  1242. }
  1243. /* Note this may cause RPC to be resent */
  1244. hdr->mds_ops->rpc_call_done(task, hdr);
  1245. }
  1246. static void ff_layout_read_count_stats(struct rpc_task *task, void *data)
  1247. {
  1248. struct nfs_pgio_header *hdr = data;
  1249. ff_layout_read_record_layoutstats_done(task, hdr);
  1250. rpc_count_iostats_metrics(task,
  1251. &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_READ]);
  1252. }
  1253. static void ff_layout_read_release(void *data)
  1254. {
  1255. struct nfs_pgio_header *hdr = data;
  1256. ff_layout_read_record_layoutstats_done(&hdr->task, hdr);
  1257. if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags))
  1258. ff_layout_resend_pnfs_read(hdr);
  1259. else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
  1260. ff_layout_reset_read(hdr);
  1261. pnfs_generic_rw_release(data);
  1262. }
  1263. static int ff_layout_write_done_cb(struct rpc_task *task,
  1264. struct nfs_pgio_header *hdr)
  1265. {
  1266. loff_t end_offs = 0;
  1267. int err;
  1268. if (task->tk_status < 0) {
  1269. ff_layout_io_track_ds_error(hdr->lseg, hdr->pgio_mirror_idx,
  1270. hdr->args.offset, hdr->args.count,
  1271. &hdr->res.op_status, OP_WRITE,
  1272. task->tk_status);
  1273. trace_ff_layout_write_error(hdr);
  1274. }
  1275. err = ff_layout_async_handle_error(task, hdr->args.context->state,
  1276. hdr->ds_clp, hdr->lseg,
  1277. hdr->pgio_mirror_idx);
  1278. trace_nfs4_pnfs_write(hdr, err);
  1279. clear_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
  1280. clear_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags);
  1281. switch (err) {
  1282. case -NFS4ERR_RESET_TO_PNFS:
  1283. set_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags);
  1284. return task->tk_status;
  1285. case -NFS4ERR_RESET_TO_MDS:
  1286. set_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags);
  1287. return task->tk_status;
  1288. case -EAGAIN:
  1289. return -EAGAIN;
  1290. }
  1291. if (hdr->res.verf->committed == NFS_FILE_SYNC ||
  1292. hdr->res.verf->committed == NFS_DATA_SYNC)
  1293. end_offs = hdr->mds_offset + (loff_t)hdr->res.count;
  1294. /* Note: if the write is unstable, don't set end_offs until commit */
  1295. ff_layout_set_layoutcommit(hdr->inode, hdr->lseg, end_offs);
  1296. /* zero out fattr since we don't care DS attr at all */
  1297. hdr->fattr.valid = 0;
  1298. if (task->tk_status >= 0)
  1299. nfs_writeback_update_inode(hdr);
  1300. return 0;
  1301. }
  1302. static int ff_layout_commit_done_cb(struct rpc_task *task,
  1303. struct nfs_commit_data *data)
  1304. {
  1305. int err;
  1306. if (task->tk_status < 0) {
  1307. ff_layout_io_track_ds_error(data->lseg, data->ds_commit_index,
  1308. data->args.offset, data->args.count,
  1309. &data->res.op_status, OP_COMMIT,
  1310. task->tk_status);
  1311. trace_ff_layout_commit_error(data);
  1312. }
  1313. err = ff_layout_async_handle_error(task, NULL, data->ds_clp,
  1314. data->lseg, data->ds_commit_index);
  1315. trace_nfs4_pnfs_commit_ds(data, err);
  1316. switch (err) {
  1317. case -NFS4ERR_RESET_TO_PNFS:
  1318. pnfs_generic_prepare_to_resend_writes(data);
  1319. return -EAGAIN;
  1320. case -NFS4ERR_RESET_TO_MDS:
  1321. pnfs_generic_prepare_to_resend_writes(data);
  1322. return -EAGAIN;
  1323. case -EAGAIN:
  1324. rpc_restart_call_prepare(task);
  1325. return -EAGAIN;
  1326. }
  1327. ff_layout_set_layoutcommit(data->inode, data->lseg, data->lwb);
  1328. return 0;
  1329. }
  1330. static void ff_layout_write_record_layoutstats_start(struct rpc_task *task,
  1331. struct nfs_pgio_header *hdr)
  1332. {
  1333. if (test_and_set_bit(NFS_IOHDR_STAT, &hdr->flags))
  1334. return;
  1335. nfs4_ff_layout_stat_io_start_write(hdr->inode,
  1336. FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),
  1337. hdr->args.count,
  1338. task->tk_start);
  1339. }
  1340. static void ff_layout_write_record_layoutstats_done(struct rpc_task *task,
  1341. struct nfs_pgio_header *hdr)
  1342. {
  1343. if (!test_and_clear_bit(NFS_IOHDR_STAT, &hdr->flags))
  1344. return;
  1345. nfs4_ff_layout_stat_io_end_write(task,
  1346. FF_LAYOUT_COMP(hdr->lseg, hdr->pgio_mirror_idx),
  1347. hdr->args.count, hdr->res.count,
  1348. hdr->res.verf->committed);
  1349. set_bit(NFS_LSEG_LAYOUTRETURN, &hdr->lseg->pls_flags);
  1350. }
  1351. static int ff_layout_write_prepare_common(struct rpc_task *task,
  1352. struct nfs_pgio_header *hdr)
  1353. {
  1354. if (unlikely(test_bit(NFS_CONTEXT_BAD, &hdr->args.context->flags))) {
  1355. rpc_exit(task, -EIO);
  1356. return -EIO;
  1357. }
  1358. ff_layout_write_record_layoutstats_start(task, hdr);
  1359. return 0;
  1360. }
  1361. static void ff_layout_write_prepare_v3(struct rpc_task *task, void *data)
  1362. {
  1363. struct nfs_pgio_header *hdr = data;
  1364. if (ff_layout_write_prepare_common(task, hdr))
  1365. return;
  1366. rpc_call_start(task);
  1367. }
  1368. static void ff_layout_write_prepare_v4(struct rpc_task *task, void *data)
  1369. {
  1370. struct nfs_pgio_header *hdr = data;
  1371. if (nfs4_setup_sequence(hdr->ds_clp,
  1372. &hdr->args.seq_args,
  1373. &hdr->res.seq_res,
  1374. task))
  1375. return;
  1376. ff_layout_write_prepare_common(task, hdr);
  1377. }
  1378. static void ff_layout_write_call_done(struct rpc_task *task, void *data)
  1379. {
  1380. struct nfs_pgio_header *hdr = data;
  1381. if (test_bit(NFS_IOHDR_REDO, &hdr->flags) &&
  1382. task->tk_status == 0) {
  1383. nfs4_sequence_done(task, &hdr->res.seq_res);
  1384. return;
  1385. }
  1386. /* Note this may cause RPC to be resent */
  1387. hdr->mds_ops->rpc_call_done(task, hdr);
  1388. }
  1389. static void ff_layout_write_count_stats(struct rpc_task *task, void *data)
  1390. {
  1391. struct nfs_pgio_header *hdr = data;
  1392. ff_layout_write_record_layoutstats_done(task, hdr);
  1393. rpc_count_iostats_metrics(task,
  1394. &NFS_CLIENT(hdr->inode)->cl_metrics[NFSPROC4_CLNT_WRITE]);
  1395. }
  1396. static void ff_layout_write_release(void *data)
  1397. {
  1398. struct nfs_pgio_header *hdr = data;
  1399. ff_layout_write_record_layoutstats_done(&hdr->task, hdr);
  1400. if (test_bit(NFS_IOHDR_RESEND_PNFS, &hdr->flags)) {
  1401. ff_layout_send_layouterror(hdr->lseg);
  1402. ff_layout_reset_write(hdr, true);
  1403. } else if (test_bit(NFS_IOHDR_RESEND_MDS, &hdr->flags))
  1404. ff_layout_reset_write(hdr, false);
  1405. pnfs_generic_rw_release(data);
  1406. }
  1407. static void ff_layout_commit_record_layoutstats_start(struct rpc_task *task,
  1408. struct nfs_commit_data *cdata)
  1409. {
  1410. if (test_and_set_bit(NFS_IOHDR_STAT, &cdata->flags))
  1411. return;
  1412. nfs4_ff_layout_stat_io_start_write(cdata->inode,
  1413. FF_LAYOUT_COMP(cdata->lseg, cdata->ds_commit_index),
  1414. 0, task->tk_start);
  1415. }
  1416. static void ff_layout_commit_record_layoutstats_done(struct rpc_task *task,
  1417. struct nfs_commit_data *cdata)
  1418. {
  1419. struct nfs_page *req;
  1420. __u64 count = 0;
  1421. if (!test_and_clear_bit(NFS_IOHDR_STAT, &cdata->flags))
  1422. return;
  1423. if (task->tk_status == 0) {
  1424. list_for_each_entry(req, &cdata->pages, wb_list)
  1425. count += req->wb_bytes;
  1426. }
  1427. nfs4_ff_layout_stat_io_end_write(task,
  1428. FF_LAYOUT_COMP(cdata->lseg, cdata->ds_commit_index),
  1429. count, count, NFS_FILE_SYNC);
  1430. set_bit(NFS_LSEG_LAYOUTRETURN, &cdata->lseg->pls_flags);
  1431. }
  1432. static void ff_layout_commit_prepare_common(struct rpc_task *task,
  1433. struct nfs_commit_data *cdata)
  1434. {
  1435. ff_layout_commit_record_layoutstats_start(task, cdata);
  1436. }
  1437. static void ff_layout_commit_prepare_v3(struct rpc_task *task, void *data)
  1438. {
  1439. ff_layout_commit_prepare_common(task, data);
  1440. rpc_call_start(task);
  1441. }
  1442. static void ff_layout_commit_prepare_v4(struct rpc_task *task, void *data)
  1443. {
  1444. struct nfs_commit_data *wdata = data;
  1445. if (nfs4_setup_sequence(wdata->ds_clp,
  1446. &wdata->args.seq_args,
  1447. &wdata->res.seq_res,
  1448. task))
  1449. return;
  1450. ff_layout_commit_prepare_common(task, data);
  1451. }
  1452. static void ff_layout_commit_done(struct rpc_task *task, void *data)
  1453. {
  1454. pnfs_generic_write_commit_done(task, data);
  1455. }
  1456. static void ff_layout_commit_count_stats(struct rpc_task *task, void *data)
  1457. {
  1458. struct nfs_commit_data *cdata = data;
  1459. ff_layout_commit_record_layoutstats_done(task, cdata);
  1460. rpc_count_iostats_metrics(task,
  1461. &NFS_CLIENT(cdata->inode)->cl_metrics[NFSPROC4_CLNT_COMMIT]);
  1462. }
  1463. static void ff_layout_commit_release(void *data)
  1464. {
  1465. struct nfs_commit_data *cdata = data;
  1466. ff_layout_commit_record_layoutstats_done(&cdata->task, cdata);
  1467. pnfs_generic_commit_release(data);
  1468. }
  1469. static const struct rpc_call_ops ff_layout_read_call_ops_v3 = {
  1470. .rpc_call_prepare = ff_layout_read_prepare_v3,
  1471. .rpc_call_done = ff_layout_read_call_done,
  1472. .rpc_count_stats = ff_layout_read_count_stats,
  1473. .rpc_release = ff_layout_read_release,
  1474. };
  1475. static const struct rpc_call_ops ff_layout_read_call_ops_v4 = {
  1476. .rpc_call_prepare = ff_layout_read_prepare_v4,
  1477. .rpc_call_done = ff_layout_read_call_done,
  1478. .rpc_count_stats = ff_layout_read_count_stats,
  1479. .rpc_release = ff_layout_read_release,
  1480. };
  1481. static const struct rpc_call_ops ff_layout_write_call_ops_v3 = {
  1482. .rpc_call_prepare = ff_layout_write_prepare_v3,
  1483. .rpc_call_done = ff_layout_write_call_done,
  1484. .rpc_count_stats = ff_layout_write_count_stats,
  1485. .rpc_release = ff_layout_write_release,
  1486. };
  1487. static const struct rpc_call_ops ff_layout_write_call_ops_v4 = {
  1488. .rpc_call_prepare = ff_layout_write_prepare_v4,
  1489. .rpc_call_done = ff_layout_write_call_done,
  1490. .rpc_count_stats = ff_layout_write_count_stats,
  1491. .rpc_release = ff_layout_write_release,
  1492. };
  1493. static const struct rpc_call_ops ff_layout_commit_call_ops_v3 = {
  1494. .rpc_call_prepare = ff_layout_commit_prepare_v3,
  1495. .rpc_call_done = ff_layout_commit_done,
  1496. .rpc_count_stats = ff_layout_commit_count_stats,
  1497. .rpc_release = ff_layout_commit_release,
  1498. };
  1499. static const struct rpc_call_ops ff_layout_commit_call_ops_v4 = {
  1500. .rpc_call_prepare = ff_layout_commit_prepare_v4,
  1501. .rpc_call_done = ff_layout_commit_done,
  1502. .rpc_count_stats = ff_layout_commit_count_stats,
  1503. .rpc_release = ff_layout_commit_release,
  1504. };
  1505. static enum pnfs_try_status
  1506. ff_layout_read_pagelist(struct nfs_pgio_header *hdr)
  1507. {
  1508. struct pnfs_layout_segment *lseg = hdr->lseg;
  1509. struct nfs4_pnfs_ds *ds;
  1510. struct rpc_clnt *ds_clnt;
  1511. struct nfs4_ff_layout_mirror *mirror;
  1512. const struct cred *ds_cred;
  1513. loff_t offset = hdr->args.offset;
  1514. u32 idx = hdr->pgio_mirror_idx;
  1515. int vers;
  1516. struct nfs_fh *fh;
  1517. dprintk("--> %s ino %lu pgbase %u req %zu@%llu\n",
  1518. __func__, hdr->inode->i_ino,
  1519. hdr->args.pgbase, (size_t)hdr->args.count, offset);
  1520. mirror = FF_LAYOUT_COMP(lseg, idx);
  1521. ds = nfs4_ff_layout_prepare_ds(lseg, mirror, false);
  1522. if (!ds)
  1523. goto out_failed;
  1524. ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
  1525. hdr->inode);
  1526. if (IS_ERR(ds_clnt))
  1527. goto out_failed;
  1528. ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred);
  1529. if (!ds_cred)
  1530. goto out_failed;
  1531. vers = nfs4_ff_layout_ds_version(mirror);
  1532. dprintk("%s USE DS: %s cl_count %d vers %d\n", __func__,
  1533. ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count), vers);
  1534. hdr->pgio_done_cb = ff_layout_read_done_cb;
  1535. refcount_inc(&ds->ds_clp->cl_count);
  1536. hdr->ds_clp = ds->ds_clp;
  1537. fh = nfs4_ff_layout_select_ds_fh(mirror);
  1538. if (fh)
  1539. hdr->args.fh = fh;
  1540. nfs4_ff_layout_select_ds_stateid(mirror, &hdr->args.stateid);
  1541. /*
  1542. * Note that if we ever decide to split across DSes,
  1543. * then we may need to handle dense-like offsets.
  1544. */
  1545. hdr->args.offset = offset;
  1546. hdr->mds_offset = offset;
  1547. /* Perform an asynchronous read to ds */
  1548. nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
  1549. vers == 3 ? &ff_layout_read_call_ops_v3 :
  1550. &ff_layout_read_call_ops_v4,
  1551. 0, RPC_TASK_SOFTCONN);
  1552. put_cred(ds_cred);
  1553. return PNFS_ATTEMPTED;
  1554. out_failed:
  1555. if (ff_layout_avoid_mds_available_ds(lseg))
  1556. return PNFS_TRY_AGAIN;
  1557. trace_pnfs_mds_fallback_read_pagelist(hdr->inode,
  1558. hdr->args.offset, hdr->args.count,
  1559. IOMODE_READ, NFS_I(hdr->inode)->layout, lseg);
  1560. return PNFS_NOT_ATTEMPTED;
  1561. }
  1562. /* Perform async writes. */
  1563. static enum pnfs_try_status
  1564. ff_layout_write_pagelist(struct nfs_pgio_header *hdr, int sync)
  1565. {
  1566. struct pnfs_layout_segment *lseg = hdr->lseg;
  1567. struct nfs4_pnfs_ds *ds;
  1568. struct rpc_clnt *ds_clnt;
  1569. struct nfs4_ff_layout_mirror *mirror;
  1570. const struct cred *ds_cred;
  1571. loff_t offset = hdr->args.offset;
  1572. int vers;
  1573. struct nfs_fh *fh;
  1574. u32 idx = hdr->pgio_mirror_idx;
  1575. mirror = FF_LAYOUT_COMP(lseg, idx);
  1576. ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true);
  1577. if (!ds)
  1578. goto out_failed;
  1579. ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
  1580. hdr->inode);
  1581. if (IS_ERR(ds_clnt))
  1582. goto out_failed;
  1583. ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, hdr->cred);
  1584. if (!ds_cred)
  1585. goto out_failed;
  1586. vers = nfs4_ff_layout_ds_version(mirror);
  1587. dprintk("%s ino %lu sync %d req %zu@%llu DS: %s cl_count %d vers %d\n",
  1588. __func__, hdr->inode->i_ino, sync, (size_t) hdr->args.count,
  1589. offset, ds->ds_remotestr, refcount_read(&ds->ds_clp->cl_count),
  1590. vers);
  1591. hdr->pgio_done_cb = ff_layout_write_done_cb;
  1592. refcount_inc(&ds->ds_clp->cl_count);
  1593. hdr->ds_clp = ds->ds_clp;
  1594. hdr->ds_commit_idx = idx;
  1595. fh = nfs4_ff_layout_select_ds_fh(mirror);
  1596. if (fh)
  1597. hdr->args.fh = fh;
  1598. nfs4_ff_layout_select_ds_stateid(mirror, &hdr->args.stateid);
  1599. /*
  1600. * Note that if we ever decide to split across DSes,
  1601. * then we may need to handle dense-like offsets.
  1602. */
  1603. hdr->args.offset = offset;
  1604. /* Perform an asynchronous write */
  1605. nfs_initiate_pgio(ds_clnt, hdr, ds_cred, ds->ds_clp->rpc_ops,
  1606. vers == 3 ? &ff_layout_write_call_ops_v3 :
  1607. &ff_layout_write_call_ops_v4,
  1608. sync, RPC_TASK_SOFTCONN);
  1609. put_cred(ds_cred);
  1610. return PNFS_ATTEMPTED;
  1611. out_failed:
  1612. if (ff_layout_avoid_mds_available_ds(lseg))
  1613. return PNFS_TRY_AGAIN;
  1614. trace_pnfs_mds_fallback_write_pagelist(hdr->inode,
  1615. hdr->args.offset, hdr->args.count,
  1616. IOMODE_RW, NFS_I(hdr->inode)->layout, lseg);
  1617. return PNFS_NOT_ATTEMPTED;
  1618. }
  1619. static u32 calc_ds_index_from_commit(struct pnfs_layout_segment *lseg, u32 i)
  1620. {
  1621. return i;
  1622. }
  1623. static struct nfs_fh *
  1624. select_ds_fh_from_commit(struct pnfs_layout_segment *lseg, u32 i)
  1625. {
  1626. struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg);
  1627. /* FIXME: Assume that there is only one NFS version available
  1628. * for the DS.
  1629. */
  1630. return &flseg->mirror_array[i]->fh_versions[0];
  1631. }
  1632. static int ff_layout_initiate_commit(struct nfs_commit_data *data, int how)
  1633. {
  1634. struct pnfs_layout_segment *lseg = data->lseg;
  1635. struct nfs4_pnfs_ds *ds;
  1636. struct rpc_clnt *ds_clnt;
  1637. struct nfs4_ff_layout_mirror *mirror;
  1638. const struct cred *ds_cred;
  1639. u32 idx;
  1640. int vers, ret;
  1641. struct nfs_fh *fh;
  1642. if (!lseg || !(pnfs_is_valid_lseg(lseg) ||
  1643. test_bit(NFS_LSEG_LAYOUTRETURN, &lseg->pls_flags)))
  1644. goto out_err;
  1645. idx = calc_ds_index_from_commit(lseg, data->ds_commit_index);
  1646. mirror = FF_LAYOUT_COMP(lseg, idx);
  1647. ds = nfs4_ff_layout_prepare_ds(lseg, mirror, true);
  1648. if (!ds)
  1649. goto out_err;
  1650. ds_clnt = nfs4_ff_find_or_create_ds_client(mirror, ds->ds_clp,
  1651. data->inode);
  1652. if (IS_ERR(ds_clnt))
  1653. goto out_err;
  1654. ds_cred = ff_layout_get_ds_cred(mirror, &lseg->pls_range, data->cred);
  1655. if (!ds_cred)
  1656. goto out_err;
  1657. vers = nfs4_ff_layout_ds_version(mirror);
  1658. dprintk("%s ino %lu, how %d cl_count %d vers %d\n", __func__,
  1659. data->inode->i_ino, how, refcount_read(&ds->ds_clp->cl_count),
  1660. vers);
  1661. data->commit_done_cb = ff_layout_commit_done_cb;
  1662. data->cred = ds_cred;
  1663. refcount_inc(&ds->ds_clp->cl_count);
  1664. data->ds_clp = ds->ds_clp;
  1665. fh = select_ds_fh_from_commit(lseg, data->ds_commit_index);
  1666. if (fh)
  1667. data->args.fh = fh;
  1668. ret = nfs_initiate_commit(ds_clnt, data, ds->ds_clp->rpc_ops,
  1669. vers == 3 ? &ff_layout_commit_call_ops_v3 :
  1670. &ff_layout_commit_call_ops_v4,
  1671. how, RPC_TASK_SOFTCONN);
  1672. put_cred(ds_cred);
  1673. return ret;
  1674. out_err:
  1675. pnfs_generic_prepare_to_resend_writes(data);
  1676. pnfs_generic_commit_release(data);
  1677. return -EAGAIN;
  1678. }
  1679. static int
  1680. ff_layout_commit_pagelist(struct inode *inode, struct list_head *mds_pages,
  1681. int how, struct nfs_commit_info *cinfo)
  1682. {
  1683. return pnfs_generic_commit_pagelist(inode, mds_pages, how, cinfo,
  1684. ff_layout_initiate_commit);
  1685. }
  1686. static struct pnfs_ds_commit_info *
  1687. ff_layout_get_ds_info(struct inode *inode)
  1688. {
  1689. struct pnfs_layout_hdr *layout = NFS_I(inode)->layout;
  1690. if (layout == NULL)
  1691. return NULL;
  1692. return &FF_LAYOUT_FROM_HDR(layout)->commit_info;
  1693. }
  1694. static void
  1695. ff_layout_setup_ds_info(struct pnfs_ds_commit_info *fl_cinfo,
  1696. struct pnfs_layout_segment *lseg)
  1697. {
  1698. struct nfs4_ff_layout_segment *flseg = FF_LAYOUT_LSEG(lseg);
  1699. struct inode *inode = lseg->pls_layout->plh_inode;
  1700. struct pnfs_commit_array *array, *new;
  1701. new = pnfs_alloc_commit_array(flseg->mirror_array_cnt, GFP_NOIO);
  1702. if (new) {
  1703. spin_lock(&inode->i_lock);
  1704. array = pnfs_add_commit_array(fl_cinfo, new, lseg);
  1705. spin_unlock(&inode->i_lock);
  1706. if (array != new)
  1707. pnfs_free_commit_array(new);
  1708. }
  1709. }
  1710. static void
  1711. ff_layout_release_ds_info(struct pnfs_ds_commit_info *fl_cinfo,
  1712. struct inode *inode)
  1713. {
  1714. spin_lock(&inode->i_lock);
  1715. pnfs_generic_ds_cinfo_destroy(fl_cinfo);
  1716. spin_unlock(&inode->i_lock);
  1717. }
  1718. static void
  1719. ff_layout_free_deviceid_node(struct nfs4_deviceid_node *d)
  1720. {
  1721. nfs4_ff_layout_free_deviceid(container_of(d, struct nfs4_ff_layout_ds,
  1722. id_node));
  1723. }
  1724. static int ff_layout_encode_ioerr(struct xdr_stream *xdr,
  1725. const struct nfs4_layoutreturn_args *args,
  1726. const struct nfs4_flexfile_layoutreturn_args *ff_args)
  1727. {
  1728. __be32 *start;
  1729. start = xdr_reserve_space(xdr, 4);
  1730. if (unlikely(!start))
  1731. return -E2BIG;
  1732. *start = cpu_to_be32(ff_args->num_errors);
  1733. /* This assume we always return _ALL_ layouts */
  1734. return ff_layout_encode_ds_ioerr(xdr, &ff_args->errors);
  1735. }
  1736. static void
  1737. encode_opaque_fixed(struct xdr_stream *xdr, const void *buf, size_t len)
  1738. {
  1739. WARN_ON_ONCE(xdr_stream_encode_opaque_fixed(xdr, buf, len) < 0);
  1740. }
  1741. static void
  1742. ff_layout_encode_ff_iostat_head(struct xdr_stream *xdr,
  1743. const nfs4_stateid *stateid,
  1744. const struct nfs42_layoutstat_devinfo *devinfo)
  1745. {
  1746. __be32 *p;
  1747. p = xdr_reserve_space(xdr, 8 + 8);
  1748. p = xdr_encode_hyper(p, devinfo->offset);
  1749. p = xdr_encode_hyper(p, devinfo->length);
  1750. encode_opaque_fixed(xdr, stateid->data, NFS4_STATEID_SIZE);
  1751. p = xdr_reserve_space(xdr, 4*8);
  1752. p = xdr_encode_hyper(p, devinfo->read_count);
  1753. p = xdr_encode_hyper(p, devinfo->read_bytes);
  1754. p = xdr_encode_hyper(p, devinfo->write_count);
  1755. p = xdr_encode_hyper(p, devinfo->write_bytes);
  1756. encode_opaque_fixed(xdr, devinfo->dev_id.data, NFS4_DEVICEID4_SIZE);
  1757. }
  1758. static void
  1759. ff_layout_encode_ff_iostat(struct xdr_stream *xdr,
  1760. const nfs4_stateid *stateid,
  1761. const struct nfs42_layoutstat_devinfo *devinfo)
  1762. {
  1763. ff_layout_encode_ff_iostat_head(xdr, stateid, devinfo);
  1764. ff_layout_encode_ff_layoutupdate(xdr, devinfo,
  1765. devinfo->ld_private.data);
  1766. }
  1767. /* report nothing for now */
  1768. static void ff_layout_encode_iostats_array(struct xdr_stream *xdr,
  1769. const struct nfs4_layoutreturn_args *args,
  1770. struct nfs4_flexfile_layoutreturn_args *ff_args)
  1771. {
  1772. __be32 *p;
  1773. int i;
  1774. p = xdr_reserve_space(xdr, 4);
  1775. *p = cpu_to_be32(ff_args->num_dev);
  1776. for (i = 0; i < ff_args->num_dev; i++)
  1777. ff_layout_encode_ff_iostat(xdr,
  1778. &args->layout->plh_stateid,
  1779. &ff_args->devinfo[i]);
  1780. }
  1781. static void
  1782. ff_layout_free_iostats_array(struct nfs42_layoutstat_devinfo *devinfo,
  1783. unsigned int num_entries)
  1784. {
  1785. unsigned int i;
  1786. for (i = 0; i < num_entries; i++) {
  1787. if (!devinfo[i].ld_private.ops)
  1788. continue;
  1789. if (!devinfo[i].ld_private.ops->free)
  1790. continue;
  1791. devinfo[i].ld_private.ops->free(&devinfo[i].ld_private);
  1792. }
  1793. }
  1794. static struct nfs4_deviceid_node *
  1795. ff_layout_alloc_deviceid_node(struct nfs_server *server,
  1796. struct pnfs_device *pdev, gfp_t gfp_flags)
  1797. {
  1798. struct nfs4_ff_layout_ds *dsaddr;
  1799. dsaddr = nfs4_ff_alloc_deviceid_node(server, pdev, gfp_flags);
  1800. if (!dsaddr)
  1801. return NULL;
  1802. return &dsaddr->id_node;
  1803. }
  1804. static void
  1805. ff_layout_encode_layoutreturn(struct xdr_stream *xdr,
  1806. const void *voidargs,
  1807. const struct nfs4_xdr_opaque_data *ff_opaque)
  1808. {
  1809. const struct nfs4_layoutreturn_args *args = voidargs;
  1810. struct nfs4_flexfile_layoutreturn_args *ff_args = ff_opaque->data;
  1811. struct xdr_buf tmp_buf = {
  1812. .head = {
  1813. [0] = {
  1814. .iov_base = page_address(ff_args->pages[0]),
  1815. },
  1816. },
  1817. .buflen = PAGE_SIZE,
  1818. };
  1819. struct xdr_stream tmp_xdr;
  1820. __be32 *start;
  1821. dprintk("%s: Begin\n", __func__);
  1822. xdr_init_encode(&tmp_xdr, &tmp_buf, NULL, NULL);
  1823. ff_layout_encode_ioerr(&tmp_xdr, args, ff_args);
  1824. ff_layout_encode_iostats_array(&tmp_xdr, args, ff_args);
  1825. start = xdr_reserve_space(xdr, 4);
  1826. *start = cpu_to_be32(tmp_buf.len);
  1827. xdr_write_pages(xdr, ff_args->pages, 0, tmp_buf.len);
  1828. dprintk("%s: Return\n", __func__);
  1829. }
  1830. static void
  1831. ff_layout_free_layoutreturn(struct nfs4_xdr_opaque_data *args)
  1832. {
  1833. struct nfs4_flexfile_layoutreturn_args *ff_args;
  1834. if (!args->data)
  1835. return;
  1836. ff_args = args->data;
  1837. args->data = NULL;
  1838. ff_layout_free_ds_ioerr(&ff_args->errors);
  1839. ff_layout_free_iostats_array(ff_args->devinfo, ff_args->num_dev);
  1840. put_page(ff_args->pages[0]);
  1841. kfree(ff_args);
  1842. }
  1843. static const struct nfs4_xdr_opaque_ops layoutreturn_ops = {
  1844. .encode = ff_layout_encode_layoutreturn,
  1845. .free = ff_layout_free_layoutreturn,
  1846. };
  1847. static int
  1848. ff_layout_prepare_layoutreturn(struct nfs4_layoutreturn_args *args)
  1849. {
  1850. struct nfs4_flexfile_layoutreturn_args *ff_args;
  1851. struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(args->layout);
  1852. ff_args = kmalloc(sizeof(*ff_args), GFP_KERNEL);
  1853. if (!ff_args)
  1854. goto out_nomem;
  1855. ff_args->pages[0] = alloc_page(GFP_KERNEL);
  1856. if (!ff_args->pages[0])
  1857. goto out_nomem_free;
  1858. INIT_LIST_HEAD(&ff_args->errors);
  1859. ff_args->num_errors = ff_layout_fetch_ds_ioerr(args->layout,
  1860. &args->range, &ff_args->errors,
  1861. FF_LAYOUTRETURN_MAXERR);
  1862. spin_lock(&args->inode->i_lock);
  1863. ff_args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
  1864. &ff_args->devinfo[0], ARRAY_SIZE(ff_args->devinfo));
  1865. spin_unlock(&args->inode->i_lock);
  1866. args->ld_private->ops = &layoutreturn_ops;
  1867. args->ld_private->data = ff_args;
  1868. return 0;
  1869. out_nomem_free:
  1870. kfree(ff_args);
  1871. out_nomem:
  1872. return -ENOMEM;
  1873. }
  1874. #ifdef CONFIG_NFS_V4_2
  1875. void
  1876. ff_layout_send_layouterror(struct pnfs_layout_segment *lseg)
  1877. {
  1878. struct pnfs_layout_hdr *lo = lseg->pls_layout;
  1879. struct nfs42_layout_error *errors;
  1880. LIST_HEAD(head);
  1881. if (!nfs_server_capable(lo->plh_inode, NFS_CAP_LAYOUTERROR))
  1882. return;
  1883. ff_layout_fetch_ds_ioerr(lo, &lseg->pls_range, &head, -1);
  1884. if (list_empty(&head))
  1885. return;
  1886. errors = kmalloc_array(NFS42_LAYOUTERROR_MAX,
  1887. sizeof(*errors), GFP_NOFS);
  1888. if (errors != NULL) {
  1889. const struct nfs4_ff_layout_ds_err *pos;
  1890. size_t n = 0;
  1891. list_for_each_entry(pos, &head, list) {
  1892. errors[n].offset = pos->offset;
  1893. errors[n].length = pos->length;
  1894. nfs4_stateid_copy(&errors[n].stateid, &pos->stateid);
  1895. errors[n].errors[0].dev_id = pos->deviceid;
  1896. errors[n].errors[0].status = pos->status;
  1897. errors[n].errors[0].opnum = pos->opnum;
  1898. n++;
  1899. if (!list_is_last(&pos->list, &head) &&
  1900. n < NFS42_LAYOUTERROR_MAX)
  1901. continue;
  1902. if (nfs42_proc_layouterror(lseg, errors, n) < 0)
  1903. break;
  1904. n = 0;
  1905. }
  1906. kfree(errors);
  1907. }
  1908. ff_layout_free_ds_ioerr(&head);
  1909. }
  1910. #else
  1911. void
  1912. ff_layout_send_layouterror(struct pnfs_layout_segment *lseg)
  1913. {
  1914. }
  1915. #endif
  1916. static int
  1917. ff_layout_ntop4(const struct sockaddr *sap, char *buf, const size_t buflen)
  1918. {
  1919. const struct sockaddr_in *sin = (struct sockaddr_in *)sap;
  1920. return snprintf(buf, buflen, "%pI4", &sin->sin_addr);
  1921. }
  1922. static size_t
  1923. ff_layout_ntop6_noscopeid(const struct sockaddr *sap, char *buf,
  1924. const int buflen)
  1925. {
  1926. const struct sockaddr_in6 *sin6 = (struct sockaddr_in6 *)sap;
  1927. const struct in6_addr *addr = &sin6->sin6_addr;
  1928. /*
  1929. * RFC 4291, Section 2.2.2
  1930. *
  1931. * Shorthanded ANY address
  1932. */
  1933. if (ipv6_addr_any(addr))
  1934. return snprintf(buf, buflen, "::");
  1935. /*
  1936. * RFC 4291, Section 2.2.2
  1937. *
  1938. * Shorthanded loopback address
  1939. */
  1940. if (ipv6_addr_loopback(addr))
  1941. return snprintf(buf, buflen, "::1");
  1942. /*
  1943. * RFC 4291, Section 2.2.3
  1944. *
  1945. * Special presentation address format for mapped v4
  1946. * addresses.
  1947. */
  1948. if (ipv6_addr_v4mapped(addr))
  1949. return snprintf(buf, buflen, "::ffff:%pI4",
  1950. &addr->s6_addr32[3]);
  1951. /*
  1952. * RFC 4291, Section 2.2.1
  1953. */
  1954. return snprintf(buf, buflen, "%pI6c", addr);
  1955. }
  1956. /* Derived from rpc_sockaddr2uaddr */
  1957. static void
  1958. ff_layout_encode_netaddr(struct xdr_stream *xdr, struct nfs4_pnfs_ds_addr *da)
  1959. {
  1960. struct sockaddr *sap = (struct sockaddr *)&da->da_addr;
  1961. char portbuf[RPCBIND_MAXUADDRPLEN];
  1962. char addrbuf[RPCBIND_MAXUADDRLEN];
  1963. char *netid;
  1964. unsigned short port;
  1965. int len, netid_len;
  1966. __be32 *p;
  1967. switch (sap->sa_family) {
  1968. case AF_INET:
  1969. if (ff_layout_ntop4(sap, addrbuf, sizeof(addrbuf)) == 0)
  1970. return;
  1971. port = ntohs(((struct sockaddr_in *)sap)->sin_port);
  1972. netid = "tcp";
  1973. netid_len = 3;
  1974. break;
  1975. case AF_INET6:
  1976. if (ff_layout_ntop6_noscopeid(sap, addrbuf, sizeof(addrbuf)) == 0)
  1977. return;
  1978. port = ntohs(((struct sockaddr_in6 *)sap)->sin6_port);
  1979. netid = "tcp6";
  1980. netid_len = 4;
  1981. break;
  1982. default:
  1983. /* we only support tcp and tcp6 */
  1984. WARN_ON_ONCE(1);
  1985. return;
  1986. }
  1987. snprintf(portbuf, sizeof(portbuf), ".%u.%u", port >> 8, port & 0xff);
  1988. len = strlcat(addrbuf, portbuf, sizeof(addrbuf));
  1989. p = xdr_reserve_space(xdr, 4 + netid_len);
  1990. xdr_encode_opaque(p, netid, netid_len);
  1991. p = xdr_reserve_space(xdr, 4 + len);
  1992. xdr_encode_opaque(p, addrbuf, len);
  1993. }
  1994. static void
  1995. ff_layout_encode_nfstime(struct xdr_stream *xdr,
  1996. ktime_t t)
  1997. {
  1998. struct timespec64 ts;
  1999. __be32 *p;
  2000. p = xdr_reserve_space(xdr, 12);
  2001. ts = ktime_to_timespec64(t);
  2002. p = xdr_encode_hyper(p, ts.tv_sec);
  2003. *p++ = cpu_to_be32(ts.tv_nsec);
  2004. }
  2005. static void
  2006. ff_layout_encode_io_latency(struct xdr_stream *xdr,
  2007. struct nfs4_ff_io_stat *stat)
  2008. {
  2009. __be32 *p;
  2010. p = xdr_reserve_space(xdr, 5 * 8);
  2011. p = xdr_encode_hyper(p, stat->ops_requested);
  2012. p = xdr_encode_hyper(p, stat->bytes_requested);
  2013. p = xdr_encode_hyper(p, stat->ops_completed);
  2014. p = xdr_encode_hyper(p, stat->bytes_completed);
  2015. p = xdr_encode_hyper(p, stat->bytes_not_delivered);
  2016. ff_layout_encode_nfstime(xdr, stat->total_busy_time);
  2017. ff_layout_encode_nfstime(xdr, stat->aggregate_completion_time);
  2018. }
  2019. static void
  2020. ff_layout_encode_ff_layoutupdate(struct xdr_stream *xdr,
  2021. const struct nfs42_layoutstat_devinfo *devinfo,
  2022. struct nfs4_ff_layout_mirror *mirror)
  2023. {
  2024. struct nfs4_pnfs_ds_addr *da;
  2025. struct nfs4_pnfs_ds *ds = mirror->mirror_ds->ds;
  2026. struct nfs_fh *fh = &mirror->fh_versions[0];
  2027. __be32 *p;
  2028. da = list_first_entry(&ds->ds_addrs, struct nfs4_pnfs_ds_addr, da_node);
  2029. dprintk("%s: DS %s: encoding address %s\n",
  2030. __func__, ds->ds_remotestr, da->da_remotestr);
  2031. /* netaddr4 */
  2032. ff_layout_encode_netaddr(xdr, da);
  2033. /* nfs_fh4 */
  2034. p = xdr_reserve_space(xdr, 4 + fh->size);
  2035. xdr_encode_opaque(p, fh->data, fh->size);
  2036. /* ff_io_latency4 read */
  2037. spin_lock(&mirror->lock);
  2038. ff_layout_encode_io_latency(xdr, &mirror->read_stat.io_stat);
  2039. /* ff_io_latency4 write */
  2040. ff_layout_encode_io_latency(xdr, &mirror->write_stat.io_stat);
  2041. spin_unlock(&mirror->lock);
  2042. /* nfstime4 */
  2043. ff_layout_encode_nfstime(xdr, ktime_sub(ktime_get(), mirror->start_time));
  2044. /* bool */
  2045. p = xdr_reserve_space(xdr, 4);
  2046. *p = cpu_to_be32(false);
  2047. }
  2048. static void
  2049. ff_layout_encode_layoutstats(struct xdr_stream *xdr, const void *args,
  2050. const struct nfs4_xdr_opaque_data *opaque)
  2051. {
  2052. struct nfs42_layoutstat_devinfo *devinfo = container_of(opaque,
  2053. struct nfs42_layoutstat_devinfo, ld_private);
  2054. __be32 *start;
  2055. /* layoutupdate length */
  2056. start = xdr_reserve_space(xdr, 4);
  2057. ff_layout_encode_ff_layoutupdate(xdr, devinfo, opaque->data);
  2058. *start = cpu_to_be32((xdr->p - start - 1) * 4);
  2059. }
  2060. static void
  2061. ff_layout_free_layoutstats(struct nfs4_xdr_opaque_data *opaque)
  2062. {
  2063. struct nfs4_ff_layout_mirror *mirror = opaque->data;
  2064. ff_layout_put_mirror(mirror);
  2065. }
  2066. static const struct nfs4_xdr_opaque_ops layoutstat_ops = {
  2067. .encode = ff_layout_encode_layoutstats,
  2068. .free = ff_layout_free_layoutstats,
  2069. };
  2070. static int
  2071. ff_layout_mirror_prepare_stats(struct pnfs_layout_hdr *lo,
  2072. struct nfs42_layoutstat_devinfo *devinfo,
  2073. int dev_limit)
  2074. {
  2075. struct nfs4_flexfile_layout *ff_layout = FF_LAYOUT_FROM_HDR(lo);
  2076. struct nfs4_ff_layout_mirror *mirror;
  2077. struct nfs4_deviceid_node *dev;
  2078. int i = 0;
  2079. list_for_each_entry(mirror, &ff_layout->mirrors, mirrors) {
  2080. if (i >= dev_limit)
  2081. break;
  2082. if (IS_ERR_OR_NULL(mirror->mirror_ds))
  2083. continue;
  2084. if (!test_and_clear_bit(NFS4_FF_MIRROR_STAT_AVAIL, &mirror->flags))
  2085. continue;
  2086. /* mirror refcount put in cleanup_layoutstats */
  2087. if (!refcount_inc_not_zero(&mirror->ref))
  2088. continue;
  2089. dev = &mirror->mirror_ds->id_node;
  2090. memcpy(&devinfo->dev_id, &dev->deviceid, NFS4_DEVICEID4_SIZE);
  2091. devinfo->offset = 0;
  2092. devinfo->length = NFS4_MAX_UINT64;
  2093. spin_lock(&mirror->lock);
  2094. devinfo->read_count = mirror->read_stat.io_stat.ops_completed;
  2095. devinfo->read_bytes = mirror->read_stat.io_stat.bytes_completed;
  2096. devinfo->write_count = mirror->write_stat.io_stat.ops_completed;
  2097. devinfo->write_bytes = mirror->write_stat.io_stat.bytes_completed;
  2098. spin_unlock(&mirror->lock);
  2099. devinfo->layout_type = LAYOUT_FLEX_FILES;
  2100. devinfo->ld_private.ops = &layoutstat_ops;
  2101. devinfo->ld_private.data = mirror;
  2102. devinfo++;
  2103. i++;
  2104. }
  2105. return i;
  2106. }
  2107. static int
  2108. ff_layout_prepare_layoutstats(struct nfs42_layoutstat_args *args)
  2109. {
  2110. struct nfs4_flexfile_layout *ff_layout;
  2111. const int dev_count = PNFS_LAYOUTSTATS_MAXDEV;
  2112. /* For now, send at most PNFS_LAYOUTSTATS_MAXDEV statistics */
  2113. args->devinfo = kmalloc_array(dev_count, sizeof(*args->devinfo), GFP_NOIO);
  2114. if (!args->devinfo)
  2115. return -ENOMEM;
  2116. spin_lock(&args->inode->i_lock);
  2117. ff_layout = FF_LAYOUT_FROM_HDR(NFS_I(args->inode)->layout);
  2118. args->num_dev = ff_layout_mirror_prepare_stats(&ff_layout->generic_hdr,
  2119. &args->devinfo[0], dev_count);
  2120. spin_unlock(&args->inode->i_lock);
  2121. if (!args->num_dev) {
  2122. kfree(args->devinfo);
  2123. args->devinfo = NULL;
  2124. return -ENOENT;
  2125. }
  2126. return 0;
  2127. }
  2128. static int
  2129. ff_layout_set_layoutdriver(struct nfs_server *server,
  2130. const struct nfs_fh *dummy)
  2131. {
  2132. #if IS_ENABLED(CONFIG_NFS_V4_2)
  2133. server->caps |= NFS_CAP_LAYOUTSTATS;
  2134. #endif
  2135. return 0;
  2136. }
  2137. static const struct pnfs_commit_ops ff_layout_commit_ops = {
  2138. .setup_ds_info = ff_layout_setup_ds_info,
  2139. .release_ds_info = ff_layout_release_ds_info,
  2140. .mark_request_commit = pnfs_layout_mark_request_commit,
  2141. .clear_request_commit = pnfs_generic_clear_request_commit,
  2142. .scan_commit_lists = pnfs_generic_scan_commit_lists,
  2143. .recover_commit_reqs = pnfs_generic_recover_commit_reqs,
  2144. .commit_pagelist = ff_layout_commit_pagelist,
  2145. };
  2146. static struct pnfs_layoutdriver_type flexfilelayout_type = {
  2147. .id = LAYOUT_FLEX_FILES,
  2148. .name = "LAYOUT_FLEX_FILES",
  2149. .owner = THIS_MODULE,
  2150. .flags = PNFS_LAYOUTGET_ON_OPEN,
  2151. .max_layoutget_response = 4096, /* 1 page or so... */
  2152. .set_layoutdriver = ff_layout_set_layoutdriver,
  2153. .alloc_layout_hdr = ff_layout_alloc_layout_hdr,
  2154. .free_layout_hdr = ff_layout_free_layout_hdr,
  2155. .alloc_lseg = ff_layout_alloc_lseg,
  2156. .free_lseg = ff_layout_free_lseg,
  2157. .add_lseg = ff_layout_add_lseg,
  2158. .pg_read_ops = &ff_layout_pg_read_ops,
  2159. .pg_write_ops = &ff_layout_pg_write_ops,
  2160. .get_ds_info = ff_layout_get_ds_info,
  2161. .free_deviceid_node = ff_layout_free_deviceid_node,
  2162. .read_pagelist = ff_layout_read_pagelist,
  2163. .write_pagelist = ff_layout_write_pagelist,
  2164. .alloc_deviceid_node = ff_layout_alloc_deviceid_node,
  2165. .prepare_layoutreturn = ff_layout_prepare_layoutreturn,
  2166. .sync = pnfs_nfs_generic_sync,
  2167. .prepare_layoutstats = ff_layout_prepare_layoutstats,
  2168. };
  2169. static int __init nfs4flexfilelayout_init(void)
  2170. {
  2171. printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Registering...\n",
  2172. __func__);
  2173. return pnfs_register_layoutdriver(&flexfilelayout_type);
  2174. }
  2175. static void __exit nfs4flexfilelayout_exit(void)
  2176. {
  2177. printk(KERN_INFO "%s: NFSv4 Flexfile Layout Driver Unregistering...\n",
  2178. __func__);
  2179. pnfs_unregister_layoutdriver(&flexfilelayout_type);
  2180. }
  2181. MODULE_ALIAS("nfs-layouttype4-4");
  2182. MODULE_LICENSE("GPL");
  2183. MODULE_DESCRIPTION("The NFSv4 flexfile layout driver");
  2184. module_init(nfs4flexfilelayout_init);
  2185. module_exit(nfs4flexfilelayout_exit);
  2186. module_param(io_maxretrans, ushort, 0644);
  2187. MODULE_PARM_DESC(io_maxretrans, "The number of times the NFSv4.1 client "
  2188. "retries an I/O request before returning an error. ");