connection.c 26 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948
  1. /*
  2. * Copyright (c) 2006, 2018 Oracle and/or its affiliates. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/list.h>
  35. #include <linux/slab.h>
  36. #include <linux/export.h>
  37. #include <net/ipv6.h>
  38. #include <net/inet6_hashtables.h>
  39. #include <net/addrconf.h>
  40. #include "rds.h"
  41. #include "loop.h"
  42. #define RDS_CONNECTION_HASH_BITS 12
  43. #define RDS_CONNECTION_HASH_ENTRIES (1 << RDS_CONNECTION_HASH_BITS)
  44. #define RDS_CONNECTION_HASH_MASK (RDS_CONNECTION_HASH_ENTRIES - 1)
  45. /* converting this to RCU is a chore for another day.. */
  46. static DEFINE_SPINLOCK(rds_conn_lock);
  47. static unsigned long rds_conn_count;
  48. static struct hlist_head rds_conn_hash[RDS_CONNECTION_HASH_ENTRIES];
  49. static struct kmem_cache *rds_conn_slab;
  50. static struct hlist_head *rds_conn_bucket(const struct in6_addr *laddr,
  51. const struct in6_addr *faddr)
  52. {
  53. static u32 rds6_hash_secret __read_mostly;
  54. static u32 rds_hash_secret __read_mostly;
  55. u32 lhash, fhash, hash;
  56. net_get_random_once(&rds_hash_secret, sizeof(rds_hash_secret));
  57. net_get_random_once(&rds6_hash_secret, sizeof(rds6_hash_secret));
  58. lhash = (__force u32)laddr->s6_addr32[3];
  59. #if IS_ENABLED(CONFIG_IPV6)
  60. fhash = __ipv6_addr_jhash(faddr, rds6_hash_secret);
  61. #else
  62. fhash = (__force u32)faddr->s6_addr32[3];
  63. #endif
  64. hash = __inet_ehashfn(lhash, 0, fhash, 0, rds_hash_secret);
  65. return &rds_conn_hash[hash & RDS_CONNECTION_HASH_MASK];
  66. }
  67. #define rds_conn_info_set(var, test, suffix) do { \
  68. if (test) \
  69. var |= RDS_INFO_CONNECTION_FLAG_##suffix; \
  70. } while (0)
  71. /* rcu read lock must be held or the connection spinlock */
  72. static struct rds_connection *rds_conn_lookup(struct net *net,
  73. struct hlist_head *head,
  74. const struct in6_addr *laddr,
  75. const struct in6_addr *faddr,
  76. struct rds_transport *trans,
  77. u8 tos, int dev_if)
  78. {
  79. struct rds_connection *conn, *ret = NULL;
  80. hlist_for_each_entry_rcu(conn, head, c_hash_node) {
  81. if (ipv6_addr_equal(&conn->c_faddr, faddr) &&
  82. ipv6_addr_equal(&conn->c_laddr, laddr) &&
  83. conn->c_trans == trans &&
  84. conn->c_tos == tos &&
  85. net == rds_conn_net(conn) &&
  86. conn->c_dev_if == dev_if) {
  87. ret = conn;
  88. break;
  89. }
  90. }
  91. rdsdebug("returning conn %p for %pI6c -> %pI6c\n", ret,
  92. laddr, faddr);
  93. return ret;
  94. }
  95. /*
  96. * This is called by transports as they're bringing down a connection.
  97. * It clears partial message state so that the transport can start sending
  98. * and receiving over this connection again in the future. It is up to
  99. * the transport to have serialized this call with its send and recv.
  100. */
  101. static void rds_conn_path_reset(struct rds_conn_path *cp)
  102. {
  103. struct rds_connection *conn = cp->cp_conn;
  104. rdsdebug("connection %pI6c to %pI6c reset\n",
  105. &conn->c_laddr, &conn->c_faddr);
  106. rds_stats_inc(s_conn_reset);
  107. rds_send_path_reset(cp);
  108. cp->cp_flags = 0;
  109. /* Do not clear next_rx_seq here, else we cannot distinguish
  110. * retransmitted packets from new packets, and will hand all
  111. * of them to the application. That is not consistent with the
  112. * reliability guarantees of RDS. */
  113. }
  114. static void __rds_conn_path_init(struct rds_connection *conn,
  115. struct rds_conn_path *cp, bool is_outgoing)
  116. {
  117. spin_lock_init(&cp->cp_lock);
  118. cp->cp_next_tx_seq = 1;
  119. init_waitqueue_head(&cp->cp_waitq);
  120. INIT_LIST_HEAD(&cp->cp_send_queue);
  121. INIT_LIST_HEAD(&cp->cp_retrans);
  122. cp->cp_conn = conn;
  123. atomic_set(&cp->cp_state, RDS_CONN_DOWN);
  124. cp->cp_send_gen = 0;
  125. cp->cp_reconnect_jiffies = 0;
  126. cp->cp_conn->c_proposed_version = RDS_PROTOCOL_VERSION;
  127. INIT_DELAYED_WORK(&cp->cp_send_w, rds_send_worker);
  128. INIT_DELAYED_WORK(&cp->cp_recv_w, rds_recv_worker);
  129. INIT_DELAYED_WORK(&cp->cp_conn_w, rds_connect_worker);
  130. INIT_WORK(&cp->cp_down_w, rds_shutdown_worker);
  131. mutex_init(&cp->cp_cm_lock);
  132. cp->cp_flags = 0;
  133. }
  134. /*
  135. * There is only every one 'conn' for a given pair of addresses in the
  136. * system at a time. They contain messages to be retransmitted and so
  137. * span the lifetime of the actual underlying transport connections.
  138. *
  139. * For now they are not garbage collected once they're created. They
  140. * are torn down as the module is removed, if ever.
  141. */
  142. static struct rds_connection *__rds_conn_create(struct net *net,
  143. const struct in6_addr *laddr,
  144. const struct in6_addr *faddr,
  145. struct rds_transport *trans,
  146. gfp_t gfp, u8 tos,
  147. int is_outgoing,
  148. int dev_if)
  149. {
  150. struct rds_connection *conn, *parent = NULL;
  151. struct hlist_head *head = rds_conn_bucket(laddr, faddr);
  152. struct rds_transport *loop_trans;
  153. unsigned long flags;
  154. int ret, i;
  155. int npaths = (trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
  156. rcu_read_lock();
  157. conn = rds_conn_lookup(net, head, laddr, faddr, trans, tos, dev_if);
  158. if (conn &&
  159. conn->c_loopback &&
  160. conn->c_trans != &rds_loop_transport &&
  161. ipv6_addr_equal(laddr, faddr) &&
  162. !is_outgoing) {
  163. /* This is a looped back IB connection, and we're
  164. * called by the code handling the incoming connect.
  165. * We need a second connection object into which we
  166. * can stick the other QP. */
  167. parent = conn;
  168. conn = parent->c_passive;
  169. }
  170. rcu_read_unlock();
  171. if (conn)
  172. goto out;
  173. conn = kmem_cache_zalloc(rds_conn_slab, gfp);
  174. if (!conn) {
  175. conn = ERR_PTR(-ENOMEM);
  176. goto out;
  177. }
  178. conn->c_path = kcalloc(npaths, sizeof(struct rds_conn_path), gfp);
  179. if (!conn->c_path) {
  180. kmem_cache_free(rds_conn_slab, conn);
  181. conn = ERR_PTR(-ENOMEM);
  182. goto out;
  183. }
  184. INIT_HLIST_NODE(&conn->c_hash_node);
  185. conn->c_laddr = *laddr;
  186. conn->c_isv6 = !ipv6_addr_v4mapped(laddr);
  187. conn->c_faddr = *faddr;
  188. conn->c_dev_if = dev_if;
  189. conn->c_tos = tos;
  190. #if IS_ENABLED(CONFIG_IPV6)
  191. /* If the local address is link local, set c_bound_if to be the
  192. * index used for this connection. Otherwise, set it to 0 as
  193. * the socket is not bound to an interface. c_bound_if is used
  194. * to look up a socket when a packet is received
  195. */
  196. if (ipv6_addr_type(laddr) & IPV6_ADDR_LINKLOCAL)
  197. conn->c_bound_if = dev_if;
  198. else
  199. #endif
  200. conn->c_bound_if = 0;
  201. rds_conn_net_set(conn, net);
  202. ret = rds_cong_get_maps(conn);
  203. if (ret) {
  204. kfree(conn->c_path);
  205. kmem_cache_free(rds_conn_slab, conn);
  206. conn = ERR_PTR(ret);
  207. goto out;
  208. }
  209. /*
  210. * This is where a connection becomes loopback. If *any* RDS sockets
  211. * can bind to the destination address then we'd rather the messages
  212. * flow through loopback rather than either transport.
  213. */
  214. loop_trans = rds_trans_get_preferred(net, faddr, conn->c_dev_if);
  215. if (loop_trans) {
  216. rds_trans_put(loop_trans);
  217. conn->c_loopback = 1;
  218. if (trans->t_prefer_loopback) {
  219. if (likely(is_outgoing)) {
  220. /* "outgoing" connection to local address.
  221. * Protocol says it wants the connection
  222. * handled by the loopback transport.
  223. * This is what TCP does.
  224. */
  225. trans = &rds_loop_transport;
  226. } else {
  227. /* No transport currently in use
  228. * should end up here, but if it
  229. * does, reset/destroy the connection.
  230. */
  231. kfree(conn->c_path);
  232. kmem_cache_free(rds_conn_slab, conn);
  233. conn = ERR_PTR(-EOPNOTSUPP);
  234. goto out;
  235. }
  236. }
  237. }
  238. conn->c_trans = trans;
  239. init_waitqueue_head(&conn->c_hs_waitq);
  240. for (i = 0; i < npaths; i++) {
  241. __rds_conn_path_init(conn, &conn->c_path[i],
  242. is_outgoing);
  243. conn->c_path[i].cp_index = i;
  244. }
  245. rcu_read_lock();
  246. if (rds_destroy_pending(conn))
  247. ret = -ENETDOWN;
  248. else
  249. ret = trans->conn_alloc(conn, GFP_ATOMIC);
  250. if (ret) {
  251. rcu_read_unlock();
  252. kfree(conn->c_path);
  253. kmem_cache_free(rds_conn_slab, conn);
  254. conn = ERR_PTR(ret);
  255. goto out;
  256. }
  257. rdsdebug("allocated conn %p for %pI6c -> %pI6c over %s %s\n",
  258. conn, laddr, faddr,
  259. strnlen(trans->t_name, sizeof(trans->t_name)) ?
  260. trans->t_name : "[unknown]", is_outgoing ? "(outgoing)" : "");
  261. /*
  262. * Since we ran without holding the conn lock, someone could
  263. * have created the same conn (either normal or passive) in the
  264. * interim. We check while holding the lock. If we won, we complete
  265. * init and return our conn. If we lost, we rollback and return the
  266. * other one.
  267. */
  268. spin_lock_irqsave(&rds_conn_lock, flags);
  269. if (parent) {
  270. /* Creating passive conn */
  271. if (parent->c_passive) {
  272. trans->conn_free(conn->c_path[0].cp_transport_data);
  273. kfree(conn->c_path);
  274. kmem_cache_free(rds_conn_slab, conn);
  275. conn = parent->c_passive;
  276. } else {
  277. parent->c_passive = conn;
  278. rds_cong_add_conn(conn);
  279. rds_conn_count++;
  280. }
  281. } else {
  282. /* Creating normal conn */
  283. struct rds_connection *found;
  284. found = rds_conn_lookup(net, head, laddr, faddr, trans,
  285. tos, dev_if);
  286. if (found) {
  287. struct rds_conn_path *cp;
  288. int i;
  289. for (i = 0; i < npaths; i++) {
  290. cp = &conn->c_path[i];
  291. /* The ->conn_alloc invocation may have
  292. * allocated resource for all paths, so all
  293. * of them may have to be freed here.
  294. */
  295. if (cp->cp_transport_data)
  296. trans->conn_free(cp->cp_transport_data);
  297. }
  298. kfree(conn->c_path);
  299. kmem_cache_free(rds_conn_slab, conn);
  300. conn = found;
  301. } else {
  302. conn->c_my_gen_num = rds_gen_num;
  303. conn->c_peer_gen_num = 0;
  304. hlist_add_head_rcu(&conn->c_hash_node, head);
  305. rds_cong_add_conn(conn);
  306. rds_conn_count++;
  307. }
  308. }
  309. spin_unlock_irqrestore(&rds_conn_lock, flags);
  310. rcu_read_unlock();
  311. out:
  312. return conn;
  313. }
  314. struct rds_connection *rds_conn_create(struct net *net,
  315. const struct in6_addr *laddr,
  316. const struct in6_addr *faddr,
  317. struct rds_transport *trans, u8 tos,
  318. gfp_t gfp, int dev_if)
  319. {
  320. return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 0, dev_if);
  321. }
  322. EXPORT_SYMBOL_GPL(rds_conn_create);
  323. struct rds_connection *rds_conn_create_outgoing(struct net *net,
  324. const struct in6_addr *laddr,
  325. const struct in6_addr *faddr,
  326. struct rds_transport *trans,
  327. u8 tos, gfp_t gfp, int dev_if)
  328. {
  329. return __rds_conn_create(net, laddr, faddr, trans, gfp, tos, 1, dev_if);
  330. }
  331. EXPORT_SYMBOL_GPL(rds_conn_create_outgoing);
  332. void rds_conn_shutdown(struct rds_conn_path *cp)
  333. {
  334. struct rds_connection *conn = cp->cp_conn;
  335. /* shut it down unless it's down already */
  336. if (!rds_conn_path_transition(cp, RDS_CONN_DOWN, RDS_CONN_DOWN)) {
  337. /*
  338. * Quiesce the connection mgmt handlers before we start tearing
  339. * things down. We don't hold the mutex for the entire
  340. * duration of the shutdown operation, else we may be
  341. * deadlocking with the CM handler. Instead, the CM event
  342. * handler is supposed to check for state DISCONNECTING
  343. */
  344. mutex_lock(&cp->cp_cm_lock);
  345. if (!rds_conn_path_transition(cp, RDS_CONN_UP,
  346. RDS_CONN_DISCONNECTING) &&
  347. !rds_conn_path_transition(cp, RDS_CONN_ERROR,
  348. RDS_CONN_DISCONNECTING)) {
  349. rds_conn_path_error(cp,
  350. "shutdown called in state %d\n",
  351. atomic_read(&cp->cp_state));
  352. mutex_unlock(&cp->cp_cm_lock);
  353. return;
  354. }
  355. mutex_unlock(&cp->cp_cm_lock);
  356. wait_event(cp->cp_waitq,
  357. !test_bit(RDS_IN_XMIT, &cp->cp_flags));
  358. wait_event(cp->cp_waitq,
  359. !test_bit(RDS_RECV_REFILL, &cp->cp_flags));
  360. conn->c_trans->conn_path_shutdown(cp);
  361. rds_conn_path_reset(cp);
  362. if (!rds_conn_path_transition(cp, RDS_CONN_DISCONNECTING,
  363. RDS_CONN_DOWN) &&
  364. !rds_conn_path_transition(cp, RDS_CONN_ERROR,
  365. RDS_CONN_DOWN)) {
  366. /* This can happen - eg when we're in the middle of tearing
  367. * down the connection, and someone unloads the rds module.
  368. * Quite reproducible with loopback connections.
  369. * Mostly harmless.
  370. *
  371. * Note that this also happens with rds-tcp because
  372. * we could have triggered rds_conn_path_drop in irq
  373. * mode from rds_tcp_state change on the receipt of
  374. * a FIN, thus we need to recheck for RDS_CONN_ERROR
  375. * here.
  376. */
  377. rds_conn_path_error(cp, "%s: failed to transition "
  378. "to state DOWN, current state "
  379. "is %d\n", __func__,
  380. atomic_read(&cp->cp_state));
  381. return;
  382. }
  383. }
  384. /* Then reconnect if it's still live.
  385. * The passive side of an IB loopback connection is never added
  386. * to the conn hash, so we never trigger a reconnect on this
  387. * conn - the reconnect is always triggered by the active peer. */
  388. cancel_delayed_work_sync(&cp->cp_conn_w);
  389. rcu_read_lock();
  390. if (!hlist_unhashed(&conn->c_hash_node)) {
  391. rcu_read_unlock();
  392. rds_queue_reconnect(cp);
  393. } else {
  394. rcu_read_unlock();
  395. }
  396. }
  397. /* destroy a single rds_conn_path. rds_conn_destroy() iterates over
  398. * all paths using rds_conn_path_destroy()
  399. */
  400. static void rds_conn_path_destroy(struct rds_conn_path *cp)
  401. {
  402. struct rds_message *rm, *rtmp;
  403. if (!cp->cp_transport_data)
  404. return;
  405. /* make sure lingering queued work won't try to ref the conn */
  406. cancel_delayed_work_sync(&cp->cp_send_w);
  407. cancel_delayed_work_sync(&cp->cp_recv_w);
  408. rds_conn_path_drop(cp, true);
  409. flush_work(&cp->cp_down_w);
  410. /* tear down queued messages */
  411. list_for_each_entry_safe(rm, rtmp,
  412. &cp->cp_send_queue,
  413. m_conn_item) {
  414. list_del_init(&rm->m_conn_item);
  415. BUG_ON(!list_empty(&rm->m_sock_item));
  416. rds_message_put(rm);
  417. }
  418. if (cp->cp_xmit_rm)
  419. rds_message_put(cp->cp_xmit_rm);
  420. WARN_ON(delayed_work_pending(&cp->cp_send_w));
  421. WARN_ON(delayed_work_pending(&cp->cp_recv_w));
  422. WARN_ON(delayed_work_pending(&cp->cp_conn_w));
  423. WARN_ON(work_pending(&cp->cp_down_w));
  424. cp->cp_conn->c_trans->conn_free(cp->cp_transport_data);
  425. }
  426. /*
  427. * Stop and free a connection.
  428. *
  429. * This can only be used in very limited circumstances. It assumes that once
  430. * the conn has been shutdown that no one else is referencing the connection.
  431. * We can only ensure this in the rmmod path in the current code.
  432. */
  433. void rds_conn_destroy(struct rds_connection *conn)
  434. {
  435. unsigned long flags;
  436. int i;
  437. struct rds_conn_path *cp;
  438. int npaths = (conn->c_trans->t_mp_capable ? RDS_MPATH_WORKERS : 1);
  439. rdsdebug("freeing conn %p for %pI4 -> "
  440. "%pI4\n", conn, &conn->c_laddr,
  441. &conn->c_faddr);
  442. /* Ensure conn will not be scheduled for reconnect */
  443. spin_lock_irq(&rds_conn_lock);
  444. hlist_del_init_rcu(&conn->c_hash_node);
  445. spin_unlock_irq(&rds_conn_lock);
  446. synchronize_rcu();
  447. /* shut the connection down */
  448. for (i = 0; i < npaths; i++) {
  449. cp = &conn->c_path[i];
  450. rds_conn_path_destroy(cp);
  451. BUG_ON(!list_empty(&cp->cp_retrans));
  452. }
  453. /*
  454. * The congestion maps aren't freed up here. They're
  455. * freed by rds_cong_exit() after all the connections
  456. * have been freed.
  457. */
  458. rds_cong_remove_conn(conn);
  459. kfree(conn->c_path);
  460. kmem_cache_free(rds_conn_slab, conn);
  461. spin_lock_irqsave(&rds_conn_lock, flags);
  462. rds_conn_count--;
  463. spin_unlock_irqrestore(&rds_conn_lock, flags);
  464. }
  465. EXPORT_SYMBOL_GPL(rds_conn_destroy);
  466. static void __rds_inc_msg_cp(struct rds_incoming *inc,
  467. struct rds_info_iterator *iter,
  468. void *saddr, void *daddr, int flip, bool isv6)
  469. {
  470. #if IS_ENABLED(CONFIG_IPV6)
  471. if (isv6)
  472. rds6_inc_info_copy(inc, iter, saddr, daddr, flip);
  473. else
  474. #endif
  475. rds_inc_info_copy(inc, iter, *(__be32 *)saddr,
  476. *(__be32 *)daddr, flip);
  477. }
  478. static void rds_conn_message_info_cmn(struct socket *sock, unsigned int len,
  479. struct rds_info_iterator *iter,
  480. struct rds_info_lengths *lens,
  481. int want_send, bool isv6)
  482. {
  483. struct hlist_head *head;
  484. struct list_head *list;
  485. struct rds_connection *conn;
  486. struct rds_message *rm;
  487. unsigned int total = 0;
  488. unsigned long flags;
  489. size_t i;
  490. int j;
  491. if (isv6)
  492. len /= sizeof(struct rds6_info_message);
  493. else
  494. len /= sizeof(struct rds_info_message);
  495. rcu_read_lock();
  496. for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
  497. i++, head++) {
  498. hlist_for_each_entry_rcu(conn, head, c_hash_node) {
  499. struct rds_conn_path *cp;
  500. int npaths;
  501. if (!isv6 && conn->c_isv6)
  502. continue;
  503. npaths = (conn->c_trans->t_mp_capable ?
  504. RDS_MPATH_WORKERS : 1);
  505. for (j = 0; j < npaths; j++) {
  506. cp = &conn->c_path[j];
  507. if (want_send)
  508. list = &cp->cp_send_queue;
  509. else
  510. list = &cp->cp_retrans;
  511. spin_lock_irqsave(&cp->cp_lock, flags);
  512. /* XXX too lazy to maintain counts.. */
  513. list_for_each_entry(rm, list, m_conn_item) {
  514. total++;
  515. if (total <= len)
  516. __rds_inc_msg_cp(&rm->m_inc,
  517. iter,
  518. &conn->c_laddr,
  519. &conn->c_faddr,
  520. 0, isv6);
  521. }
  522. spin_unlock_irqrestore(&cp->cp_lock, flags);
  523. }
  524. }
  525. }
  526. rcu_read_unlock();
  527. lens->nr = total;
  528. if (isv6)
  529. lens->each = sizeof(struct rds6_info_message);
  530. else
  531. lens->each = sizeof(struct rds_info_message);
  532. }
  533. static void rds_conn_message_info(struct socket *sock, unsigned int len,
  534. struct rds_info_iterator *iter,
  535. struct rds_info_lengths *lens,
  536. int want_send)
  537. {
  538. rds_conn_message_info_cmn(sock, len, iter, lens, want_send, false);
  539. }
  540. #if IS_ENABLED(CONFIG_IPV6)
  541. static void rds6_conn_message_info(struct socket *sock, unsigned int len,
  542. struct rds_info_iterator *iter,
  543. struct rds_info_lengths *lens,
  544. int want_send)
  545. {
  546. rds_conn_message_info_cmn(sock, len, iter, lens, want_send, true);
  547. }
  548. #endif
  549. static void rds_conn_message_info_send(struct socket *sock, unsigned int len,
  550. struct rds_info_iterator *iter,
  551. struct rds_info_lengths *lens)
  552. {
  553. rds_conn_message_info(sock, len, iter, lens, 1);
  554. }
  555. #if IS_ENABLED(CONFIG_IPV6)
  556. static void rds6_conn_message_info_send(struct socket *sock, unsigned int len,
  557. struct rds_info_iterator *iter,
  558. struct rds_info_lengths *lens)
  559. {
  560. rds6_conn_message_info(sock, len, iter, lens, 1);
  561. }
  562. #endif
  563. static void rds_conn_message_info_retrans(struct socket *sock,
  564. unsigned int len,
  565. struct rds_info_iterator *iter,
  566. struct rds_info_lengths *lens)
  567. {
  568. rds_conn_message_info(sock, len, iter, lens, 0);
  569. }
  570. #if IS_ENABLED(CONFIG_IPV6)
  571. static void rds6_conn_message_info_retrans(struct socket *sock,
  572. unsigned int len,
  573. struct rds_info_iterator *iter,
  574. struct rds_info_lengths *lens)
  575. {
  576. rds6_conn_message_info(sock, len, iter, lens, 0);
  577. }
  578. #endif
  579. void rds_for_each_conn_info(struct socket *sock, unsigned int len,
  580. struct rds_info_iterator *iter,
  581. struct rds_info_lengths *lens,
  582. int (*visitor)(struct rds_connection *, void *),
  583. u64 *buffer,
  584. size_t item_len)
  585. {
  586. struct hlist_head *head;
  587. struct rds_connection *conn;
  588. size_t i;
  589. rcu_read_lock();
  590. lens->nr = 0;
  591. lens->each = item_len;
  592. for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
  593. i++, head++) {
  594. hlist_for_each_entry_rcu(conn, head, c_hash_node) {
  595. /* XXX no c_lock usage.. */
  596. if (!visitor(conn, buffer))
  597. continue;
  598. /* We copy as much as we can fit in the buffer,
  599. * but we count all items so that the caller
  600. * can resize the buffer. */
  601. if (len >= item_len) {
  602. rds_info_copy(iter, buffer, item_len);
  603. len -= item_len;
  604. }
  605. lens->nr++;
  606. }
  607. }
  608. rcu_read_unlock();
  609. }
  610. EXPORT_SYMBOL_GPL(rds_for_each_conn_info);
  611. static void rds_walk_conn_path_info(struct socket *sock, unsigned int len,
  612. struct rds_info_iterator *iter,
  613. struct rds_info_lengths *lens,
  614. int (*visitor)(struct rds_conn_path *, void *),
  615. u64 *buffer,
  616. size_t item_len)
  617. {
  618. struct hlist_head *head;
  619. struct rds_connection *conn;
  620. size_t i;
  621. rcu_read_lock();
  622. lens->nr = 0;
  623. lens->each = item_len;
  624. for (i = 0, head = rds_conn_hash; i < ARRAY_SIZE(rds_conn_hash);
  625. i++, head++) {
  626. hlist_for_each_entry_rcu(conn, head, c_hash_node) {
  627. struct rds_conn_path *cp;
  628. /* XXX We only copy the information from the first
  629. * path for now. The problem is that if there are
  630. * more than one underlying paths, we cannot report
  631. * information of all of them using the existing
  632. * API. For example, there is only one next_tx_seq,
  633. * which path's next_tx_seq should we report? It is
  634. * a bug in the design of MPRDS.
  635. */
  636. cp = conn->c_path;
  637. /* XXX no cp_lock usage.. */
  638. if (!visitor(cp, buffer))
  639. continue;
  640. /* We copy as much as we can fit in the buffer,
  641. * but we count all items so that the caller
  642. * can resize the buffer.
  643. */
  644. if (len >= item_len) {
  645. rds_info_copy(iter, buffer, item_len);
  646. len -= item_len;
  647. }
  648. lens->nr++;
  649. }
  650. }
  651. rcu_read_unlock();
  652. }
  653. static int rds_conn_info_visitor(struct rds_conn_path *cp, void *buffer)
  654. {
  655. struct rds_info_connection *cinfo = buffer;
  656. struct rds_connection *conn = cp->cp_conn;
  657. if (conn->c_isv6)
  658. return 0;
  659. cinfo->next_tx_seq = cp->cp_next_tx_seq;
  660. cinfo->next_rx_seq = cp->cp_next_rx_seq;
  661. cinfo->laddr = conn->c_laddr.s6_addr32[3];
  662. cinfo->faddr = conn->c_faddr.s6_addr32[3];
  663. cinfo->tos = conn->c_tos;
  664. strncpy(cinfo->transport, conn->c_trans->t_name,
  665. sizeof(cinfo->transport));
  666. cinfo->flags = 0;
  667. rds_conn_info_set(cinfo->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags),
  668. SENDING);
  669. /* XXX Future: return the state rather than these funky bits */
  670. rds_conn_info_set(cinfo->flags,
  671. atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING,
  672. CONNECTING);
  673. rds_conn_info_set(cinfo->flags,
  674. atomic_read(&cp->cp_state) == RDS_CONN_UP,
  675. CONNECTED);
  676. return 1;
  677. }
  678. #if IS_ENABLED(CONFIG_IPV6)
  679. static int rds6_conn_info_visitor(struct rds_conn_path *cp, void *buffer)
  680. {
  681. struct rds6_info_connection *cinfo6 = buffer;
  682. struct rds_connection *conn = cp->cp_conn;
  683. cinfo6->next_tx_seq = cp->cp_next_tx_seq;
  684. cinfo6->next_rx_seq = cp->cp_next_rx_seq;
  685. cinfo6->laddr = conn->c_laddr;
  686. cinfo6->faddr = conn->c_faddr;
  687. strncpy(cinfo6->transport, conn->c_trans->t_name,
  688. sizeof(cinfo6->transport));
  689. cinfo6->flags = 0;
  690. rds_conn_info_set(cinfo6->flags, test_bit(RDS_IN_XMIT, &cp->cp_flags),
  691. SENDING);
  692. /* XXX Future: return the state rather than these funky bits */
  693. rds_conn_info_set(cinfo6->flags,
  694. atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING,
  695. CONNECTING);
  696. rds_conn_info_set(cinfo6->flags,
  697. atomic_read(&cp->cp_state) == RDS_CONN_UP,
  698. CONNECTED);
  699. /* Just return 1 as there is no error case. This is a helper function
  700. * for rds_walk_conn_path_info() and it wants a return value.
  701. */
  702. return 1;
  703. }
  704. #endif
  705. static void rds_conn_info(struct socket *sock, unsigned int len,
  706. struct rds_info_iterator *iter,
  707. struct rds_info_lengths *lens)
  708. {
  709. u64 buffer[(sizeof(struct rds_info_connection) + 7) / 8];
  710. rds_walk_conn_path_info(sock, len, iter, lens,
  711. rds_conn_info_visitor,
  712. buffer,
  713. sizeof(struct rds_info_connection));
  714. }
  715. #if IS_ENABLED(CONFIG_IPV6)
  716. static void rds6_conn_info(struct socket *sock, unsigned int len,
  717. struct rds_info_iterator *iter,
  718. struct rds_info_lengths *lens)
  719. {
  720. u64 buffer[(sizeof(struct rds6_info_connection) + 7) / 8];
  721. rds_walk_conn_path_info(sock, len, iter, lens,
  722. rds6_conn_info_visitor,
  723. buffer,
  724. sizeof(struct rds6_info_connection));
  725. }
  726. #endif
  727. int rds_conn_init(void)
  728. {
  729. int ret;
  730. ret = rds_loop_net_init(); /* register pernet callback */
  731. if (ret)
  732. return ret;
  733. rds_conn_slab = kmem_cache_create("rds_connection",
  734. sizeof(struct rds_connection),
  735. 0, 0, NULL);
  736. if (!rds_conn_slab) {
  737. rds_loop_net_exit();
  738. return -ENOMEM;
  739. }
  740. rds_info_register_func(RDS_INFO_CONNECTIONS, rds_conn_info);
  741. rds_info_register_func(RDS_INFO_SEND_MESSAGES,
  742. rds_conn_message_info_send);
  743. rds_info_register_func(RDS_INFO_RETRANS_MESSAGES,
  744. rds_conn_message_info_retrans);
  745. #if IS_ENABLED(CONFIG_IPV6)
  746. rds_info_register_func(RDS6_INFO_CONNECTIONS, rds6_conn_info);
  747. rds_info_register_func(RDS6_INFO_SEND_MESSAGES,
  748. rds6_conn_message_info_send);
  749. rds_info_register_func(RDS6_INFO_RETRANS_MESSAGES,
  750. rds6_conn_message_info_retrans);
  751. #endif
  752. return 0;
  753. }
  754. void rds_conn_exit(void)
  755. {
  756. rds_loop_net_exit(); /* unregister pernet callback */
  757. rds_loop_exit();
  758. WARN_ON(!hlist_empty(rds_conn_hash));
  759. kmem_cache_destroy(rds_conn_slab);
  760. rds_info_deregister_func(RDS_INFO_CONNECTIONS, rds_conn_info);
  761. rds_info_deregister_func(RDS_INFO_SEND_MESSAGES,
  762. rds_conn_message_info_send);
  763. rds_info_deregister_func(RDS_INFO_RETRANS_MESSAGES,
  764. rds_conn_message_info_retrans);
  765. #if IS_ENABLED(CONFIG_IPV6)
  766. rds_info_deregister_func(RDS6_INFO_CONNECTIONS, rds6_conn_info);
  767. rds_info_deregister_func(RDS6_INFO_SEND_MESSAGES,
  768. rds6_conn_message_info_send);
  769. rds_info_deregister_func(RDS6_INFO_RETRANS_MESSAGES,
  770. rds6_conn_message_info_retrans);
  771. #endif
  772. }
  773. /*
  774. * Force a disconnect
  775. */
  776. void rds_conn_path_drop(struct rds_conn_path *cp, bool destroy)
  777. {
  778. atomic_set(&cp->cp_state, RDS_CONN_ERROR);
  779. rcu_read_lock();
  780. if (!destroy && rds_destroy_pending(cp->cp_conn)) {
  781. rcu_read_unlock();
  782. return;
  783. }
  784. queue_work(rds_wq, &cp->cp_down_w);
  785. rcu_read_unlock();
  786. }
  787. EXPORT_SYMBOL_GPL(rds_conn_path_drop);
  788. void rds_conn_drop(struct rds_connection *conn)
  789. {
  790. WARN_ON(conn->c_trans->t_mp_capable);
  791. rds_conn_path_drop(&conn->c_path[0], false);
  792. }
  793. EXPORT_SYMBOL_GPL(rds_conn_drop);
  794. /*
  795. * If the connection is down, trigger a connect. We may have scheduled a
  796. * delayed reconnect however - in this case we should not interfere.
  797. */
  798. void rds_conn_path_connect_if_down(struct rds_conn_path *cp)
  799. {
  800. rcu_read_lock();
  801. if (rds_destroy_pending(cp->cp_conn)) {
  802. rcu_read_unlock();
  803. return;
  804. }
  805. if (rds_conn_path_state(cp) == RDS_CONN_DOWN &&
  806. !test_and_set_bit(RDS_RECONNECT_PENDING, &cp->cp_flags))
  807. queue_delayed_work(rds_wq, &cp->cp_conn_w, 0);
  808. rcu_read_unlock();
  809. }
  810. EXPORT_SYMBOL_GPL(rds_conn_path_connect_if_down);
  811. /* Check connectivity of all paths
  812. */
  813. void rds_check_all_paths(struct rds_connection *conn)
  814. {
  815. int i = 0;
  816. do {
  817. rds_conn_path_connect_if_down(&conn->c_path[i]);
  818. } while (++i < conn->c_npaths);
  819. }
  820. void rds_conn_connect_if_down(struct rds_connection *conn)
  821. {
  822. WARN_ON(conn->c_trans->t_mp_capable);
  823. rds_conn_path_connect_if_down(&conn->c_path[0]);
  824. }
  825. EXPORT_SYMBOL_GPL(rds_conn_connect_if_down);
  826. void
  827. __rds_conn_path_error(struct rds_conn_path *cp, const char *fmt, ...)
  828. {
  829. va_list ap;
  830. va_start(ap, fmt);
  831. vprintk(fmt, ap);
  832. va_end(ap);
  833. rds_conn_path_drop(cp, false);
  834. }