rotate.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /* Handle fileserver selection and rotation.
  3. *
  4. * Copyright (C) 2017 Red Hat, Inc. All Rights Reserved.
  5. * Written by David Howells (dhowells@redhat.com)
  6. */
  7. #include <linux/kernel.h>
  8. #include <linux/slab.h>
  9. #include <linux/fs.h>
  10. #include <linux/sched.h>
  11. #include <linux/delay.h>
  12. #include <linux/sched/signal.h>
  13. #include "internal.h"
  14. #include "afs_fs.h"
  15. /*
  16. * Begin iteration through a server list, starting with the vnode's last used
  17. * server if possible, or the last recorded good server if not.
  18. */
  19. static bool afs_start_fs_iteration(struct afs_operation *op,
  20. struct afs_vnode *vnode)
  21. {
  22. struct afs_server *server;
  23. void *cb_server;
  24. int i;
  25. read_lock(&op->volume->servers_lock);
  26. op->server_list = afs_get_serverlist(
  27. rcu_dereference_protected(op->volume->servers,
  28. lockdep_is_held(&op->volume->servers_lock)));
  29. read_unlock(&op->volume->servers_lock);
  30. op->untried = (1UL << op->server_list->nr_servers) - 1;
  31. op->index = READ_ONCE(op->server_list->preferred);
  32. cb_server = vnode->cb_server;
  33. if (cb_server) {
  34. /* See if the vnode's preferred record is still available */
  35. for (i = 0; i < op->server_list->nr_servers; i++) {
  36. server = op->server_list->servers[i].server;
  37. if (server == cb_server) {
  38. op->index = i;
  39. goto found_interest;
  40. }
  41. }
  42. /* If we have a lock outstanding on a server that's no longer
  43. * serving this vnode, then we can't switch to another server
  44. * and have to return an error.
  45. */
  46. if (op->flags & AFS_OPERATION_CUR_ONLY) {
  47. op->error = -ESTALE;
  48. return false;
  49. }
  50. /* Note that the callback promise is effectively broken */
  51. write_seqlock(&vnode->cb_lock);
  52. ASSERTCMP(cb_server, ==, vnode->cb_server);
  53. vnode->cb_server = NULL;
  54. if (test_and_clear_bit(AFS_VNODE_CB_PROMISED, &vnode->flags))
  55. vnode->cb_break++;
  56. write_sequnlock(&vnode->cb_lock);
  57. }
  58. found_interest:
  59. return true;
  60. }
  61. /*
  62. * Post volume busy note.
  63. */
  64. static void afs_busy(struct afs_volume *volume, u32 abort_code)
  65. {
  66. const char *m;
  67. switch (abort_code) {
  68. case VOFFLINE: m = "offline"; break;
  69. case VRESTARTING: m = "restarting"; break;
  70. case VSALVAGING: m = "being salvaged"; break;
  71. default: m = "busy"; break;
  72. }
  73. pr_notice("kAFS: Volume %llu '%s' is %s\n", volume->vid, volume->name, m);
  74. }
  75. /*
  76. * Sleep and retry the operation to the same fileserver.
  77. */
  78. static bool afs_sleep_and_retry(struct afs_operation *op)
  79. {
  80. if (!(op->flags & AFS_OPERATION_UNINTR)) {
  81. msleep_interruptible(1000);
  82. if (signal_pending(current)) {
  83. op->error = -ERESTARTSYS;
  84. return false;
  85. }
  86. } else {
  87. msleep(1000);
  88. }
  89. return true;
  90. }
  91. /*
  92. * Select the fileserver to use. May be called multiple times to rotate
  93. * through the fileservers.
  94. */
  95. bool afs_select_fileserver(struct afs_operation *op)
  96. {
  97. struct afs_addr_list *alist;
  98. struct afs_server *server;
  99. struct afs_vnode *vnode = op->file[0].vnode;
  100. struct afs_error e;
  101. u32 rtt;
  102. int error = op->ac.error, i;
  103. _enter("%lx[%d],%lx[%d],%d,%d",
  104. op->untried, op->index,
  105. op->ac.tried, op->ac.index,
  106. error, op->ac.abort_code);
  107. if (op->flags & AFS_OPERATION_STOP) {
  108. _leave(" = f [stopped]");
  109. return false;
  110. }
  111. op->nr_iterations++;
  112. /* Evaluate the result of the previous operation, if there was one. */
  113. switch (error) {
  114. case SHRT_MAX:
  115. goto start;
  116. case 0:
  117. default:
  118. /* Success or local failure. Stop. */
  119. op->error = error;
  120. op->flags |= AFS_OPERATION_STOP;
  121. _leave(" = f [okay/local %d]", error);
  122. return false;
  123. case -ECONNABORTED:
  124. /* The far side rejected the operation on some grounds. This
  125. * might involve the server being busy or the volume having been moved.
  126. */
  127. switch (op->ac.abort_code) {
  128. case VNOVOL:
  129. /* This fileserver doesn't know about the volume.
  130. * - May indicate that the VL is wrong - retry once and compare
  131. * the results.
  132. * - May indicate that the fileserver couldn't attach to the vol.
  133. */
  134. if (op->flags & AFS_OPERATION_VNOVOL) {
  135. op->error = -EREMOTEIO;
  136. goto next_server;
  137. }
  138. write_lock(&op->volume->servers_lock);
  139. op->server_list->vnovol_mask |= 1 << op->index;
  140. write_unlock(&op->volume->servers_lock);
  141. set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags);
  142. error = afs_check_volume_status(op->volume, op);
  143. if (error < 0)
  144. goto failed_set_error;
  145. if (test_bit(AFS_VOLUME_DELETED, &op->volume->flags)) {
  146. op->error = -ENOMEDIUM;
  147. goto failed;
  148. }
  149. /* If the server list didn't change, then assume that
  150. * it's the fileserver having trouble.
  151. */
  152. if (rcu_access_pointer(op->volume->servers) == op->server_list) {
  153. op->error = -EREMOTEIO;
  154. goto next_server;
  155. }
  156. /* Try again */
  157. op->flags |= AFS_OPERATION_VNOVOL;
  158. _leave(" = t [vnovol]");
  159. return true;
  160. case VSALVAGE: /* TODO: Should this return an error or iterate? */
  161. case VVOLEXISTS:
  162. case VNOSERVICE:
  163. case VONLINE:
  164. case VDISKFULL:
  165. case VOVERQUOTA:
  166. op->error = afs_abort_to_error(op->ac.abort_code);
  167. goto next_server;
  168. case VOFFLINE:
  169. if (!test_and_set_bit(AFS_VOLUME_OFFLINE, &op->volume->flags)) {
  170. afs_busy(op->volume, op->ac.abort_code);
  171. clear_bit(AFS_VOLUME_BUSY, &op->volume->flags);
  172. }
  173. if (op->flags & AFS_OPERATION_NO_VSLEEP) {
  174. op->error = -EADV;
  175. goto failed;
  176. }
  177. if (op->flags & AFS_OPERATION_CUR_ONLY) {
  178. op->error = -ESTALE;
  179. goto failed;
  180. }
  181. goto busy;
  182. case VSALVAGING:
  183. case VRESTARTING:
  184. case VBUSY:
  185. /* Retry after going round all the servers unless we
  186. * have a file lock we need to maintain.
  187. */
  188. if (op->flags & AFS_OPERATION_NO_VSLEEP) {
  189. op->error = -EBUSY;
  190. goto failed;
  191. }
  192. if (!test_and_set_bit(AFS_VOLUME_BUSY, &op->volume->flags)) {
  193. afs_busy(op->volume, op->ac.abort_code);
  194. clear_bit(AFS_VOLUME_OFFLINE, &op->volume->flags);
  195. }
  196. busy:
  197. if (op->flags & AFS_OPERATION_CUR_ONLY) {
  198. if (!afs_sleep_and_retry(op))
  199. goto failed;
  200. /* Retry with same server & address */
  201. _leave(" = t [vbusy]");
  202. return true;
  203. }
  204. op->flags |= AFS_OPERATION_VBUSY;
  205. goto next_server;
  206. case VMOVED:
  207. /* The volume migrated to another server. We consider
  208. * consider all locks and callbacks broken and request
  209. * an update from the VLDB.
  210. *
  211. * We also limit the number of VMOVED hops we will
  212. * honour, just in case someone sets up a loop.
  213. */
  214. if (op->flags & AFS_OPERATION_VMOVED) {
  215. op->error = -EREMOTEIO;
  216. goto failed;
  217. }
  218. op->flags |= AFS_OPERATION_VMOVED;
  219. set_bit(AFS_VOLUME_WAIT, &op->volume->flags);
  220. set_bit(AFS_VOLUME_NEEDS_UPDATE, &op->volume->flags);
  221. error = afs_check_volume_status(op->volume, op);
  222. if (error < 0)
  223. goto failed_set_error;
  224. /* If the server list didn't change, then the VLDB is
  225. * out of sync with the fileservers. This is hopefully
  226. * a temporary condition, however, so we don't want to
  227. * permanently block access to the file.
  228. *
  229. * TODO: Try other fileservers if we can.
  230. *
  231. * TODO: Retry a few times with sleeps.
  232. */
  233. if (rcu_access_pointer(op->volume->servers) == op->server_list) {
  234. op->error = -ENOMEDIUM;
  235. goto failed;
  236. }
  237. goto restart_from_beginning;
  238. default:
  239. clear_bit(AFS_VOLUME_OFFLINE, &op->volume->flags);
  240. clear_bit(AFS_VOLUME_BUSY, &op->volume->flags);
  241. op->error = afs_abort_to_error(op->ac.abort_code);
  242. goto failed;
  243. }
  244. case -ETIMEDOUT:
  245. case -ETIME:
  246. if (op->error != -EDESTADDRREQ)
  247. goto iterate_address;
  248. fallthrough;
  249. case -ERFKILL:
  250. case -EADDRNOTAVAIL:
  251. case -ENETUNREACH:
  252. case -EHOSTUNREACH:
  253. case -EHOSTDOWN:
  254. case -ECONNREFUSED:
  255. _debug("no conn");
  256. op->error = error;
  257. goto iterate_address;
  258. case -ECONNRESET:
  259. _debug("call reset");
  260. op->error = error;
  261. goto failed;
  262. }
  263. restart_from_beginning:
  264. _debug("restart");
  265. afs_end_cursor(&op->ac);
  266. op->server = NULL;
  267. afs_put_serverlist(op->net, op->server_list);
  268. op->server_list = NULL;
  269. start:
  270. _debug("start");
  271. /* See if we need to do an update of the volume record. Note that the
  272. * volume may have moved or even have been deleted.
  273. */
  274. error = afs_check_volume_status(op->volume, op);
  275. if (error < 0)
  276. goto failed_set_error;
  277. if (!afs_start_fs_iteration(op, vnode))
  278. goto failed;
  279. _debug("__ VOL %llx __", op->volume->vid);
  280. pick_server:
  281. _debug("pick [%lx]", op->untried);
  282. error = afs_wait_for_fs_probes(op->server_list, op->untried);
  283. if (error < 0)
  284. goto failed_set_error;
  285. /* Pick the untried server with the lowest RTT. If we have outstanding
  286. * callbacks, we stick with the server we're already using if we can.
  287. */
  288. if (op->server) {
  289. _debug("server %u", op->index);
  290. if (test_bit(op->index, &op->untried))
  291. goto selected_server;
  292. op->server = NULL;
  293. _debug("no server");
  294. }
  295. op->index = -1;
  296. rtt = U32_MAX;
  297. for (i = 0; i < op->server_list->nr_servers; i++) {
  298. struct afs_server *s = op->server_list->servers[i].server;
  299. if (!test_bit(i, &op->untried) ||
  300. !test_bit(AFS_SERVER_FL_RESPONDING, &s->flags))
  301. continue;
  302. if (s->probe.rtt < rtt) {
  303. op->index = i;
  304. rtt = s->probe.rtt;
  305. }
  306. }
  307. if (op->index == -1)
  308. goto no_more_servers;
  309. selected_server:
  310. _debug("use %d", op->index);
  311. __clear_bit(op->index, &op->untried);
  312. /* We're starting on a different fileserver from the list. We need to
  313. * check it, create a callback intercept, find its address list and
  314. * probe its capabilities before we use it.
  315. */
  316. ASSERTCMP(op->ac.alist, ==, NULL);
  317. server = op->server_list->servers[op->index].server;
  318. if (!afs_check_server_record(op, server))
  319. goto failed;
  320. _debug("USING SERVER: %pU", &server->uuid);
  321. op->flags |= AFS_OPERATION_RETRY_SERVER;
  322. op->server = server;
  323. if (vnode->cb_server != server) {
  324. vnode->cb_server = server;
  325. vnode->cb_s_break = server->cb_s_break;
  326. vnode->cb_v_break = vnode->volume->cb_v_break;
  327. clear_bit(AFS_VNODE_CB_PROMISED, &vnode->flags);
  328. }
  329. read_lock(&server->fs_lock);
  330. alist = rcu_dereference_protected(server->addresses,
  331. lockdep_is_held(&server->fs_lock));
  332. afs_get_addrlist(alist);
  333. read_unlock(&server->fs_lock);
  334. retry_server:
  335. memset(&op->ac, 0, sizeof(op->ac));
  336. if (!op->ac.alist)
  337. op->ac.alist = alist;
  338. else
  339. afs_put_addrlist(alist);
  340. op->ac.index = -1;
  341. iterate_address:
  342. ASSERT(op->ac.alist);
  343. /* Iterate over the current server's address list to try and find an
  344. * address on which it will respond to us.
  345. */
  346. if (!afs_iterate_addresses(&op->ac))
  347. goto out_of_addresses;
  348. _debug("address [%u] %u/%u %pISp",
  349. op->index, op->ac.index, op->ac.alist->nr_addrs,
  350. &op->ac.alist->addrs[op->ac.index].transport);
  351. _leave(" = t");
  352. return true;
  353. out_of_addresses:
  354. /* We've now had a failure to respond on all of a server's addresses -
  355. * immediately probe them again and consider retrying the server.
  356. */
  357. afs_probe_fileserver(op->net, op->server);
  358. if (op->flags & AFS_OPERATION_RETRY_SERVER) {
  359. alist = op->ac.alist;
  360. error = afs_wait_for_one_fs_probe(
  361. op->server, !(op->flags & AFS_OPERATION_UNINTR));
  362. switch (error) {
  363. case 0:
  364. op->flags &= ~AFS_OPERATION_RETRY_SERVER;
  365. goto retry_server;
  366. case -ERESTARTSYS:
  367. goto failed_set_error;
  368. case -ETIME:
  369. case -EDESTADDRREQ:
  370. goto next_server;
  371. }
  372. }
  373. next_server:
  374. _debug("next");
  375. afs_end_cursor(&op->ac);
  376. goto pick_server;
  377. no_more_servers:
  378. /* That's all the servers poked to no good effect. Try again if some
  379. * of them were busy.
  380. */
  381. if (op->flags & AFS_OPERATION_VBUSY)
  382. goto restart_from_beginning;
  383. e.error = -EDESTADDRREQ;
  384. e.responded = false;
  385. for (i = 0; i < op->server_list->nr_servers; i++) {
  386. struct afs_server *s = op->server_list->servers[i].server;
  387. afs_prioritise_error(&e, READ_ONCE(s->probe.error),
  388. s->probe.abort_code);
  389. }
  390. error = e.error;
  391. failed_set_error:
  392. op->error = error;
  393. failed:
  394. op->flags |= AFS_OPERATION_STOP;
  395. afs_end_cursor(&op->ac);
  396. _leave(" = f [failed %d]", op->error);
  397. return false;
  398. }
  399. /*
  400. * Dump cursor state in the case of the error being EDESTADDRREQ.
  401. */
  402. void afs_dump_edestaddrreq(const struct afs_operation *op)
  403. {
  404. static int count;
  405. int i;
  406. if (!IS_ENABLED(CONFIG_AFS_DEBUG_CURSOR) || count > 3)
  407. return;
  408. count++;
  409. rcu_read_lock();
  410. pr_notice("EDESTADDR occurred\n");
  411. pr_notice("FC: cbb=%x cbb2=%x fl=%x err=%hd\n",
  412. op->file[0].cb_break_before,
  413. op->file[1].cb_break_before, op->flags, op->error);
  414. pr_notice("FC: ut=%lx ix=%d ni=%u\n",
  415. op->untried, op->index, op->nr_iterations);
  416. if (op->server_list) {
  417. const struct afs_server_list *sl = op->server_list;
  418. pr_notice("FC: SL nr=%u pr=%u vnov=%hx\n",
  419. sl->nr_servers, sl->preferred, sl->vnovol_mask);
  420. for (i = 0; i < sl->nr_servers; i++) {
  421. const struct afs_server *s = sl->servers[i].server;
  422. pr_notice("FC: server fl=%lx av=%u %pU\n",
  423. s->flags, s->addr_version, &s->uuid);
  424. if (s->addresses) {
  425. const struct afs_addr_list *a =
  426. rcu_dereference(s->addresses);
  427. pr_notice("FC: - av=%u nr=%u/%u/%u pr=%u\n",
  428. a->version,
  429. a->nr_ipv4, a->nr_addrs, a->max_addrs,
  430. a->preferred);
  431. pr_notice("FC: - R=%lx F=%lx\n",
  432. a->responded, a->failed);
  433. if (a == op->ac.alist)
  434. pr_notice("FC: - current\n");
  435. }
  436. }
  437. }
  438. pr_notice("AC: t=%lx ax=%u ac=%d er=%d r=%u ni=%u\n",
  439. op->ac.tried, op->ac.index, op->ac.abort_code, op->ac.error,
  440. op->ac.responded, op->ac.nr_iterations);
  441. rcu_read_unlock();
  442. }