xprtmultipath.c 13 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Multipath support for RPC
  4. *
  5. * Copyright (c) 2015, 2016, Primary Data, Inc. All rights reserved.
  6. *
  7. * Trond Myklebust <trond.myklebust@primarydata.com>
  8. *
  9. */
  10. #include <linux/types.h>
  11. #include <linux/kref.h>
  12. #include <linux/list.h>
  13. #include <linux/rcupdate.h>
  14. #include <linux/rculist.h>
  15. #include <linux/slab.h>
  16. #include <asm/cmpxchg.h>
  17. #include <linux/spinlock.h>
  18. #include <linux/sunrpc/xprt.h>
  19. #include <linux/sunrpc/addr.h>
  20. #include <linux/sunrpc/xprtmultipath.h>
  21. typedef struct rpc_xprt *(*xprt_switch_find_xprt_t)(struct rpc_xprt_switch *xps,
  22. const struct rpc_xprt *cur);
  23. static const struct rpc_xprt_iter_ops rpc_xprt_iter_singular;
  24. static const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin;
  25. static const struct rpc_xprt_iter_ops rpc_xprt_iter_listall;
  26. static void xprt_switch_add_xprt_locked(struct rpc_xprt_switch *xps,
  27. struct rpc_xprt *xprt)
  28. {
  29. if (unlikely(xprt_get(xprt) == NULL))
  30. return;
  31. list_add_tail_rcu(&xprt->xprt_switch, &xps->xps_xprt_list);
  32. smp_wmb();
  33. if (xps->xps_nxprts == 0)
  34. xps->xps_net = xprt->xprt_net;
  35. xps->xps_nxprts++;
  36. xps->xps_nactive++;
  37. }
  38. /**
  39. * rpc_xprt_switch_add_xprt - Add a new rpc_xprt to an rpc_xprt_switch
  40. * @xps: pointer to struct rpc_xprt_switch
  41. * @xprt: pointer to struct rpc_xprt
  42. *
  43. * Adds xprt to the end of the list of struct rpc_xprt in xps.
  44. */
  45. void rpc_xprt_switch_add_xprt(struct rpc_xprt_switch *xps,
  46. struct rpc_xprt *xprt)
  47. {
  48. if (xprt == NULL)
  49. return;
  50. spin_lock(&xps->xps_lock);
  51. if (xps->xps_net == xprt->xprt_net || xps->xps_net == NULL)
  52. xprt_switch_add_xprt_locked(xps, xprt);
  53. spin_unlock(&xps->xps_lock);
  54. }
  55. static void xprt_switch_remove_xprt_locked(struct rpc_xprt_switch *xps,
  56. struct rpc_xprt *xprt)
  57. {
  58. if (unlikely(xprt == NULL))
  59. return;
  60. xps->xps_nactive--;
  61. xps->xps_nxprts--;
  62. if (xps->xps_nxprts == 0)
  63. xps->xps_net = NULL;
  64. smp_wmb();
  65. list_del_rcu(&xprt->xprt_switch);
  66. }
  67. /**
  68. * rpc_xprt_switch_remove_xprt - Removes an rpc_xprt from a rpc_xprt_switch
  69. * @xps: pointer to struct rpc_xprt_switch
  70. * @xprt: pointer to struct rpc_xprt
  71. *
  72. * Removes xprt from the list of struct rpc_xprt in xps.
  73. */
  74. void rpc_xprt_switch_remove_xprt(struct rpc_xprt_switch *xps,
  75. struct rpc_xprt *xprt)
  76. {
  77. spin_lock(&xps->xps_lock);
  78. xprt_switch_remove_xprt_locked(xps, xprt);
  79. spin_unlock(&xps->xps_lock);
  80. xprt_put(xprt);
  81. }
  82. /**
  83. * xprt_switch_alloc - Allocate a new struct rpc_xprt_switch
  84. * @xprt: pointer to struct rpc_xprt
  85. * @gfp_flags: allocation flags
  86. *
  87. * On success, returns an initialised struct rpc_xprt_switch, containing
  88. * the entry xprt. Returns NULL on failure.
  89. */
  90. struct rpc_xprt_switch *xprt_switch_alloc(struct rpc_xprt *xprt,
  91. gfp_t gfp_flags)
  92. {
  93. struct rpc_xprt_switch *xps;
  94. xps = kmalloc(sizeof(*xps), gfp_flags);
  95. if (xps != NULL) {
  96. spin_lock_init(&xps->xps_lock);
  97. kref_init(&xps->xps_kref);
  98. xps->xps_nxprts = xps->xps_nactive = 0;
  99. atomic_long_set(&xps->xps_queuelen, 0);
  100. xps->xps_net = NULL;
  101. INIT_LIST_HEAD(&xps->xps_xprt_list);
  102. xps->xps_iter_ops = &rpc_xprt_iter_singular;
  103. xprt_switch_add_xprt_locked(xps, xprt);
  104. }
  105. return xps;
  106. }
  107. static void xprt_switch_free_entries(struct rpc_xprt_switch *xps)
  108. {
  109. spin_lock(&xps->xps_lock);
  110. while (!list_empty(&xps->xps_xprt_list)) {
  111. struct rpc_xprt *xprt;
  112. xprt = list_first_entry(&xps->xps_xprt_list,
  113. struct rpc_xprt, xprt_switch);
  114. xprt_switch_remove_xprt_locked(xps, xprt);
  115. spin_unlock(&xps->xps_lock);
  116. xprt_put(xprt);
  117. spin_lock(&xps->xps_lock);
  118. }
  119. spin_unlock(&xps->xps_lock);
  120. }
  121. static void xprt_switch_free(struct kref *kref)
  122. {
  123. struct rpc_xprt_switch *xps = container_of(kref,
  124. struct rpc_xprt_switch, xps_kref);
  125. xprt_switch_free_entries(xps);
  126. kfree_rcu(xps, xps_rcu);
  127. }
  128. /**
  129. * xprt_switch_get - Return a reference to a rpc_xprt_switch
  130. * @xps: pointer to struct rpc_xprt_switch
  131. *
  132. * Returns a reference to xps unless the refcount is already zero.
  133. */
  134. struct rpc_xprt_switch *xprt_switch_get(struct rpc_xprt_switch *xps)
  135. {
  136. if (xps != NULL && kref_get_unless_zero(&xps->xps_kref))
  137. return xps;
  138. return NULL;
  139. }
  140. /**
  141. * xprt_switch_put - Release a reference to a rpc_xprt_switch
  142. * @xps: pointer to struct rpc_xprt_switch
  143. *
  144. * Release the reference to xps, and free it once the refcount is zero.
  145. */
  146. void xprt_switch_put(struct rpc_xprt_switch *xps)
  147. {
  148. if (xps != NULL)
  149. kref_put(&xps->xps_kref, xprt_switch_free);
  150. }
  151. /**
  152. * rpc_xprt_switch_set_roundrobin - Set a round-robin policy on rpc_xprt_switch
  153. * @xps: pointer to struct rpc_xprt_switch
  154. *
  155. * Sets a round-robin default policy for iterators acting on xps.
  156. */
  157. void rpc_xprt_switch_set_roundrobin(struct rpc_xprt_switch *xps)
  158. {
  159. if (READ_ONCE(xps->xps_iter_ops) != &rpc_xprt_iter_roundrobin)
  160. WRITE_ONCE(xps->xps_iter_ops, &rpc_xprt_iter_roundrobin);
  161. }
  162. static
  163. const struct rpc_xprt_iter_ops *xprt_iter_ops(const struct rpc_xprt_iter *xpi)
  164. {
  165. if (xpi->xpi_ops != NULL)
  166. return xpi->xpi_ops;
  167. return rcu_dereference(xpi->xpi_xpswitch)->xps_iter_ops;
  168. }
  169. static
  170. void xprt_iter_no_rewind(struct rpc_xprt_iter *xpi)
  171. {
  172. }
  173. static
  174. void xprt_iter_default_rewind(struct rpc_xprt_iter *xpi)
  175. {
  176. WRITE_ONCE(xpi->xpi_cursor, NULL);
  177. }
  178. static
  179. bool xprt_is_active(const struct rpc_xprt *xprt)
  180. {
  181. return kref_read(&xprt->kref) != 0;
  182. }
  183. static
  184. struct rpc_xprt *xprt_switch_find_first_entry(struct list_head *head)
  185. {
  186. struct rpc_xprt *pos;
  187. list_for_each_entry_rcu(pos, head, xprt_switch) {
  188. if (xprt_is_active(pos))
  189. return pos;
  190. }
  191. return NULL;
  192. }
  193. static
  194. struct rpc_xprt *xprt_iter_first_entry(struct rpc_xprt_iter *xpi)
  195. {
  196. struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
  197. if (xps == NULL)
  198. return NULL;
  199. return xprt_switch_find_first_entry(&xps->xps_xprt_list);
  200. }
  201. static
  202. struct rpc_xprt *xprt_switch_find_current_entry(struct list_head *head,
  203. const struct rpc_xprt *cur)
  204. {
  205. struct rpc_xprt *pos;
  206. bool found = false;
  207. list_for_each_entry_rcu(pos, head, xprt_switch) {
  208. if (cur == pos)
  209. found = true;
  210. if (found && xprt_is_active(pos))
  211. return pos;
  212. }
  213. return NULL;
  214. }
  215. static
  216. struct rpc_xprt *xprt_iter_current_entry(struct rpc_xprt_iter *xpi)
  217. {
  218. struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
  219. struct list_head *head;
  220. if (xps == NULL)
  221. return NULL;
  222. head = &xps->xps_xprt_list;
  223. if (xpi->xpi_cursor == NULL || xps->xps_nxprts < 2)
  224. return xprt_switch_find_first_entry(head);
  225. return xprt_switch_find_current_entry(head, xpi->xpi_cursor);
  226. }
  227. bool rpc_xprt_switch_has_addr(struct rpc_xprt_switch *xps,
  228. const struct sockaddr *sap)
  229. {
  230. struct list_head *head;
  231. struct rpc_xprt *pos;
  232. if (xps == NULL || sap == NULL)
  233. return false;
  234. head = &xps->xps_xprt_list;
  235. list_for_each_entry_rcu(pos, head, xprt_switch) {
  236. if (rpc_cmp_addr_port(sap, (struct sockaddr *)&pos->addr)) {
  237. pr_info("RPC: addr %s already in xprt switch\n",
  238. pos->address_strings[RPC_DISPLAY_ADDR]);
  239. return true;
  240. }
  241. }
  242. return false;
  243. }
  244. static
  245. struct rpc_xprt *xprt_switch_find_next_entry(struct list_head *head,
  246. const struct rpc_xprt *cur)
  247. {
  248. struct rpc_xprt *pos, *prev = NULL;
  249. bool found = false;
  250. list_for_each_entry_rcu(pos, head, xprt_switch) {
  251. if (cur == prev)
  252. found = true;
  253. if (found && xprt_is_active(pos))
  254. return pos;
  255. prev = pos;
  256. }
  257. return NULL;
  258. }
  259. static
  260. struct rpc_xprt *xprt_switch_set_next_cursor(struct rpc_xprt_switch *xps,
  261. struct rpc_xprt **cursor,
  262. xprt_switch_find_xprt_t find_next)
  263. {
  264. struct rpc_xprt *pos, *old;
  265. old = smp_load_acquire(cursor);
  266. pos = find_next(xps, old);
  267. smp_store_release(cursor, pos);
  268. return pos;
  269. }
  270. static
  271. struct rpc_xprt *xprt_iter_next_entry_multiple(struct rpc_xprt_iter *xpi,
  272. xprt_switch_find_xprt_t find_next)
  273. {
  274. struct rpc_xprt_switch *xps = rcu_dereference(xpi->xpi_xpswitch);
  275. if (xps == NULL)
  276. return NULL;
  277. return xprt_switch_set_next_cursor(xps, &xpi->xpi_cursor, find_next);
  278. }
  279. static
  280. struct rpc_xprt *__xprt_switch_find_next_entry_roundrobin(struct list_head *head,
  281. const struct rpc_xprt *cur)
  282. {
  283. struct rpc_xprt *ret;
  284. ret = xprt_switch_find_next_entry(head, cur);
  285. if (ret != NULL)
  286. return ret;
  287. return xprt_switch_find_first_entry(head);
  288. }
  289. static
  290. struct rpc_xprt *xprt_switch_find_next_entry_roundrobin(struct rpc_xprt_switch *xps,
  291. const struct rpc_xprt *cur)
  292. {
  293. struct list_head *head = &xps->xps_xprt_list;
  294. struct rpc_xprt *xprt;
  295. unsigned int nactive;
  296. for (;;) {
  297. unsigned long xprt_queuelen, xps_queuelen;
  298. xprt = __xprt_switch_find_next_entry_roundrobin(head, cur);
  299. if (!xprt)
  300. break;
  301. xprt_queuelen = atomic_long_read(&xprt->queuelen);
  302. xps_queuelen = atomic_long_read(&xps->xps_queuelen);
  303. nactive = READ_ONCE(xps->xps_nactive);
  304. /* Exit loop if xprt_queuelen <= average queue length */
  305. if (xprt_queuelen * nactive <= xps_queuelen)
  306. break;
  307. cur = xprt;
  308. }
  309. return xprt;
  310. }
  311. static
  312. struct rpc_xprt *xprt_iter_next_entry_roundrobin(struct rpc_xprt_iter *xpi)
  313. {
  314. return xprt_iter_next_entry_multiple(xpi,
  315. xprt_switch_find_next_entry_roundrobin);
  316. }
  317. static
  318. struct rpc_xprt *xprt_switch_find_next_entry_all(struct rpc_xprt_switch *xps,
  319. const struct rpc_xprt *cur)
  320. {
  321. return xprt_switch_find_next_entry(&xps->xps_xprt_list, cur);
  322. }
  323. static
  324. struct rpc_xprt *xprt_iter_next_entry_all(struct rpc_xprt_iter *xpi)
  325. {
  326. return xprt_iter_next_entry_multiple(xpi,
  327. xprt_switch_find_next_entry_all);
  328. }
  329. /*
  330. * xprt_iter_rewind - Resets the xprt iterator
  331. * @xpi: pointer to rpc_xprt_iter
  332. *
  333. * Resets xpi to ensure that it points to the first entry in the list
  334. * of transports.
  335. */
  336. static
  337. void xprt_iter_rewind(struct rpc_xprt_iter *xpi)
  338. {
  339. rcu_read_lock();
  340. xprt_iter_ops(xpi)->xpi_rewind(xpi);
  341. rcu_read_unlock();
  342. }
  343. static void __xprt_iter_init(struct rpc_xprt_iter *xpi,
  344. struct rpc_xprt_switch *xps,
  345. const struct rpc_xprt_iter_ops *ops)
  346. {
  347. rcu_assign_pointer(xpi->xpi_xpswitch, xprt_switch_get(xps));
  348. xpi->xpi_cursor = NULL;
  349. xpi->xpi_ops = ops;
  350. }
  351. /**
  352. * xprt_iter_init - Initialise an xprt iterator
  353. * @xpi: pointer to rpc_xprt_iter
  354. * @xps: pointer to rpc_xprt_switch
  355. *
  356. * Initialises the iterator to use the default iterator ops
  357. * as set in xps. This function is mainly intended for internal
  358. * use in the rpc_client.
  359. */
  360. void xprt_iter_init(struct rpc_xprt_iter *xpi,
  361. struct rpc_xprt_switch *xps)
  362. {
  363. __xprt_iter_init(xpi, xps, NULL);
  364. }
  365. /**
  366. * xprt_iter_init_listall - Initialise an xprt iterator
  367. * @xpi: pointer to rpc_xprt_iter
  368. * @xps: pointer to rpc_xprt_switch
  369. *
  370. * Initialises the iterator to iterate once through the entire list
  371. * of entries in xps.
  372. */
  373. void xprt_iter_init_listall(struct rpc_xprt_iter *xpi,
  374. struct rpc_xprt_switch *xps)
  375. {
  376. __xprt_iter_init(xpi, xps, &rpc_xprt_iter_listall);
  377. }
  378. /**
  379. * xprt_iter_xchg_switch - Atomically swap out the rpc_xprt_switch
  380. * @xpi: pointer to rpc_xprt_iter
  381. * @newswitch: pointer to a new rpc_xprt_switch or NULL
  382. *
  383. * Swaps out the existing xpi->xpi_xpswitch with a new value.
  384. */
  385. struct rpc_xprt_switch *xprt_iter_xchg_switch(struct rpc_xprt_iter *xpi,
  386. struct rpc_xprt_switch *newswitch)
  387. {
  388. struct rpc_xprt_switch __rcu *oldswitch;
  389. /* Atomically swap out the old xpswitch */
  390. oldswitch = xchg(&xpi->xpi_xpswitch, RCU_INITIALIZER(newswitch));
  391. if (newswitch != NULL)
  392. xprt_iter_rewind(xpi);
  393. return rcu_dereference_protected(oldswitch, true);
  394. }
  395. /**
  396. * xprt_iter_destroy - Destroys the xprt iterator
  397. * @xpi: pointer to rpc_xprt_iter
  398. */
  399. void xprt_iter_destroy(struct rpc_xprt_iter *xpi)
  400. {
  401. xprt_switch_put(xprt_iter_xchg_switch(xpi, NULL));
  402. }
  403. /**
  404. * xprt_iter_xprt - Returns the rpc_xprt pointed to by the cursor
  405. * @xpi: pointer to rpc_xprt_iter
  406. *
  407. * Returns a pointer to the struct rpc_xprt that is currently
  408. * pointed to by the cursor.
  409. * Caller must be holding rcu_read_lock().
  410. */
  411. struct rpc_xprt *xprt_iter_xprt(struct rpc_xprt_iter *xpi)
  412. {
  413. WARN_ON_ONCE(!rcu_read_lock_held());
  414. return xprt_iter_ops(xpi)->xpi_xprt(xpi);
  415. }
  416. static
  417. struct rpc_xprt *xprt_iter_get_helper(struct rpc_xprt_iter *xpi,
  418. struct rpc_xprt *(*fn)(struct rpc_xprt_iter *))
  419. {
  420. struct rpc_xprt *ret;
  421. do {
  422. ret = fn(xpi);
  423. if (ret == NULL)
  424. break;
  425. ret = xprt_get(ret);
  426. } while (ret == NULL);
  427. return ret;
  428. }
  429. /**
  430. * xprt_iter_get_xprt - Returns the rpc_xprt pointed to by the cursor
  431. * @xpi: pointer to rpc_xprt_iter
  432. *
  433. * Returns a reference to the struct rpc_xprt that is currently
  434. * pointed to by the cursor.
  435. */
  436. struct rpc_xprt *xprt_iter_get_xprt(struct rpc_xprt_iter *xpi)
  437. {
  438. struct rpc_xprt *xprt;
  439. rcu_read_lock();
  440. xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_xprt);
  441. rcu_read_unlock();
  442. return xprt;
  443. }
  444. /**
  445. * xprt_iter_get_next - Returns the next rpc_xprt following the cursor
  446. * @xpi: pointer to rpc_xprt_iter
  447. *
  448. * Returns a reference to the struct rpc_xprt that immediately follows the
  449. * entry pointed to by the cursor.
  450. */
  451. struct rpc_xprt *xprt_iter_get_next(struct rpc_xprt_iter *xpi)
  452. {
  453. struct rpc_xprt *xprt;
  454. rcu_read_lock();
  455. xprt = xprt_iter_get_helper(xpi, xprt_iter_ops(xpi)->xpi_next);
  456. rcu_read_unlock();
  457. return xprt;
  458. }
  459. /* Policy for always returning the first entry in the rpc_xprt_switch */
  460. static
  461. const struct rpc_xprt_iter_ops rpc_xprt_iter_singular = {
  462. .xpi_rewind = xprt_iter_no_rewind,
  463. .xpi_xprt = xprt_iter_first_entry,
  464. .xpi_next = xprt_iter_first_entry,
  465. };
  466. /* Policy for round-robin iteration of entries in the rpc_xprt_switch */
  467. static
  468. const struct rpc_xprt_iter_ops rpc_xprt_iter_roundrobin = {
  469. .xpi_rewind = xprt_iter_default_rewind,
  470. .xpi_xprt = xprt_iter_current_entry,
  471. .xpi_next = xprt_iter_next_entry_roundrobin,
  472. };
  473. /* Policy for once-through iteration of entries in the rpc_xprt_switch */
  474. static
  475. const struct rpc_xprt_iter_ops rpc_xprt_iter_listall = {
  476. .xpi_rewind = xprt_iter_default_rewind,
  477. .xpi_xprt = xprt_iter_current_entry,
  478. .xpi_next = xprt_iter_next_entry_all,
  479. };