ib_cm.c 37 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101110211031104110511061107110811091110111111121113111411151116111711181119112011211122112311241125112611271128112911301131113211331134113511361137113811391140114111421143114411451146114711481149115011511152115311541155115611571158115911601161116211631164116511661167116811691170117111721173117411751176117711781179118011811182118311841185118611871188118911901191119211931194119511961197119811991200120112021203120412051206120712081209121012111212121312141215121612171218121912201221122212231224122512261227122812291230123112321233123412351236123712381239124012411242124312441245124612471248124912501251125212531254125512561257125812591260126112621263126412651266126712681269127012711272127312741275127612771278127912801281128212831284128512861287128812891290129112921293129412951296129712981299130013011302130313041305130613071308130913101311131213131314131513161317131813191320
  1. /*
  2. * Copyright (c) 2006, 2019 Oracle and/or its affiliates. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/in.h>
  35. #include <linux/slab.h>
  36. #include <linux/vmalloc.h>
  37. #include <linux/ratelimit.h>
  38. #include <net/addrconf.h>
  39. #include <rdma/ib_cm.h>
  40. #include "rds_single_path.h"
  41. #include "rds.h"
  42. #include "ib.h"
  43. #include "ib_mr.h"
  44. /*
  45. * Set the selected protocol version
  46. */
  47. static void rds_ib_set_protocol(struct rds_connection *conn, unsigned int version)
  48. {
  49. conn->c_version = version;
  50. }
  51. /*
  52. * Set up flow control
  53. */
  54. static void rds_ib_set_flow_control(struct rds_connection *conn, u32 credits)
  55. {
  56. struct rds_ib_connection *ic = conn->c_transport_data;
  57. if (rds_ib_sysctl_flow_control && credits != 0) {
  58. /* We're doing flow control */
  59. ic->i_flowctl = 1;
  60. rds_ib_send_add_credits(conn, credits);
  61. } else {
  62. ic->i_flowctl = 0;
  63. }
  64. }
  65. /*
  66. * Tune RNR behavior. Without flow control, we use a rather
  67. * low timeout, but not the absolute minimum - this should
  68. * be tunable.
  69. *
  70. * We already set the RNR retry count to 7 (which is the
  71. * smallest infinite number :-) above.
  72. * If flow control is off, we want to change this back to 0
  73. * so that we learn quickly when our credit accounting is
  74. * buggy.
  75. *
  76. * Caller passes in a qp_attr pointer - don't waste stack spacv
  77. * by allocation this twice.
  78. */
  79. static void
  80. rds_ib_tune_rnr(struct rds_ib_connection *ic, struct ib_qp_attr *attr)
  81. {
  82. int ret;
  83. attr->min_rnr_timer = IB_RNR_TIMER_000_32;
  84. ret = ib_modify_qp(ic->i_cm_id->qp, attr, IB_QP_MIN_RNR_TIMER);
  85. if (ret)
  86. printk(KERN_NOTICE "ib_modify_qp(IB_QP_MIN_RNR_TIMER): err=%d\n", -ret);
  87. }
  88. /*
  89. * Connection established.
  90. * We get here for both outgoing and incoming connection.
  91. */
  92. void rds_ib_cm_connect_complete(struct rds_connection *conn, struct rdma_cm_event *event)
  93. {
  94. struct rds_ib_connection *ic = conn->c_transport_data;
  95. const union rds_ib_conn_priv *dp = NULL;
  96. struct ib_qp_attr qp_attr;
  97. __be64 ack_seq = 0;
  98. __be32 credit = 0;
  99. u8 major = 0;
  100. u8 minor = 0;
  101. int err;
  102. dp = event->param.conn.private_data;
  103. if (conn->c_isv6) {
  104. if (event->param.conn.private_data_len >=
  105. sizeof(struct rds6_ib_connect_private)) {
  106. major = dp->ricp_v6.dp_protocol_major;
  107. minor = dp->ricp_v6.dp_protocol_minor;
  108. credit = dp->ricp_v6.dp_credit;
  109. /* dp structure start is not guaranteed to be 8 bytes
  110. * aligned. Since dp_ack_seq is 64-bit extended load
  111. * operations can be used so go through get_unaligned
  112. * to avoid unaligned errors.
  113. */
  114. ack_seq = get_unaligned(&dp->ricp_v6.dp_ack_seq);
  115. }
  116. } else if (event->param.conn.private_data_len >=
  117. sizeof(struct rds_ib_connect_private)) {
  118. major = dp->ricp_v4.dp_protocol_major;
  119. minor = dp->ricp_v4.dp_protocol_minor;
  120. credit = dp->ricp_v4.dp_credit;
  121. ack_seq = get_unaligned(&dp->ricp_v4.dp_ack_seq);
  122. }
  123. /* make sure it isn't empty data */
  124. if (major) {
  125. rds_ib_set_protocol(conn, RDS_PROTOCOL(major, minor));
  126. rds_ib_set_flow_control(conn, be32_to_cpu(credit));
  127. }
  128. if (conn->c_version < RDS_PROTOCOL_VERSION) {
  129. if (conn->c_version != RDS_PROTOCOL_COMPAT_VERSION) {
  130. pr_notice("RDS/IB: Connection <%pI6c,%pI6c> version %u.%u no longer supported\n",
  131. &conn->c_laddr, &conn->c_faddr,
  132. RDS_PROTOCOL_MAJOR(conn->c_version),
  133. RDS_PROTOCOL_MINOR(conn->c_version));
  134. rds_conn_destroy(conn);
  135. return;
  136. }
  137. }
  138. pr_notice("RDS/IB: %s conn connected <%pI6c,%pI6c,%d> version %u.%u%s\n",
  139. ic->i_active_side ? "Active" : "Passive",
  140. &conn->c_laddr, &conn->c_faddr, conn->c_tos,
  141. RDS_PROTOCOL_MAJOR(conn->c_version),
  142. RDS_PROTOCOL_MINOR(conn->c_version),
  143. ic->i_flowctl ? ", flow control" : "");
  144. /* receive sl from the peer */
  145. ic->i_sl = ic->i_cm_id->route.path_rec->sl;
  146. atomic_set(&ic->i_cq_quiesce, 0);
  147. /* Init rings and fill recv. this needs to wait until protocol
  148. * negotiation is complete, since ring layout is different
  149. * from 3.1 to 4.1.
  150. */
  151. rds_ib_send_init_ring(ic);
  152. rds_ib_recv_init_ring(ic);
  153. /* Post receive buffers - as a side effect, this will update
  154. * the posted credit count. */
  155. rds_ib_recv_refill(conn, 1, GFP_KERNEL);
  156. /* Tune RNR behavior */
  157. rds_ib_tune_rnr(ic, &qp_attr);
  158. qp_attr.qp_state = IB_QPS_RTS;
  159. err = ib_modify_qp(ic->i_cm_id->qp, &qp_attr, IB_QP_STATE);
  160. if (err)
  161. printk(KERN_NOTICE "ib_modify_qp(IB_QP_STATE, RTS): err=%d\n", err);
  162. /* update ib_device with this local ipaddr */
  163. err = rds_ib_update_ipaddr(ic->rds_ibdev, &conn->c_laddr);
  164. if (err)
  165. printk(KERN_ERR "rds_ib_update_ipaddr failed (%d)\n",
  166. err);
  167. /* If the peer gave us the last packet it saw, process this as if
  168. * we had received a regular ACK. */
  169. if (dp) {
  170. if (ack_seq)
  171. rds_send_drop_acked(conn, be64_to_cpu(ack_seq),
  172. NULL);
  173. }
  174. conn->c_proposed_version = conn->c_version;
  175. rds_connect_complete(conn);
  176. }
  177. static void rds_ib_cm_fill_conn_param(struct rds_connection *conn,
  178. struct rdma_conn_param *conn_param,
  179. union rds_ib_conn_priv *dp,
  180. u32 protocol_version,
  181. u32 max_responder_resources,
  182. u32 max_initiator_depth,
  183. bool isv6)
  184. {
  185. struct rds_ib_connection *ic = conn->c_transport_data;
  186. struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
  187. memset(conn_param, 0, sizeof(struct rdma_conn_param));
  188. conn_param->responder_resources =
  189. min_t(u32, rds_ibdev->max_responder_resources, max_responder_resources);
  190. conn_param->initiator_depth =
  191. min_t(u32, rds_ibdev->max_initiator_depth, max_initiator_depth);
  192. conn_param->retry_count = min_t(unsigned int, rds_ib_retry_count, 7);
  193. conn_param->rnr_retry_count = 7;
  194. if (dp) {
  195. memset(dp, 0, sizeof(*dp));
  196. if (isv6) {
  197. dp->ricp_v6.dp_saddr = conn->c_laddr;
  198. dp->ricp_v6.dp_daddr = conn->c_faddr;
  199. dp->ricp_v6.dp_protocol_major =
  200. RDS_PROTOCOL_MAJOR(protocol_version);
  201. dp->ricp_v6.dp_protocol_minor =
  202. RDS_PROTOCOL_MINOR(protocol_version);
  203. dp->ricp_v6.dp_protocol_minor_mask =
  204. cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
  205. dp->ricp_v6.dp_ack_seq =
  206. cpu_to_be64(rds_ib_piggyb_ack(ic));
  207. dp->ricp_v6.dp_cmn.ricpc_dp_toss = conn->c_tos;
  208. conn_param->private_data = &dp->ricp_v6;
  209. conn_param->private_data_len = sizeof(dp->ricp_v6);
  210. } else {
  211. dp->ricp_v4.dp_saddr = conn->c_laddr.s6_addr32[3];
  212. dp->ricp_v4.dp_daddr = conn->c_faddr.s6_addr32[3];
  213. dp->ricp_v4.dp_protocol_major =
  214. RDS_PROTOCOL_MAJOR(protocol_version);
  215. dp->ricp_v4.dp_protocol_minor =
  216. RDS_PROTOCOL_MINOR(protocol_version);
  217. dp->ricp_v4.dp_protocol_minor_mask =
  218. cpu_to_be16(RDS_IB_SUPPORTED_PROTOCOLS);
  219. dp->ricp_v4.dp_ack_seq =
  220. cpu_to_be64(rds_ib_piggyb_ack(ic));
  221. dp->ricp_v4.dp_cmn.ricpc_dp_toss = conn->c_tos;
  222. conn_param->private_data = &dp->ricp_v4;
  223. conn_param->private_data_len = sizeof(dp->ricp_v4);
  224. }
  225. /* Advertise flow control */
  226. if (ic->i_flowctl) {
  227. unsigned int credits;
  228. credits = IB_GET_POST_CREDITS
  229. (atomic_read(&ic->i_credits));
  230. if (isv6)
  231. dp->ricp_v6.dp_credit = cpu_to_be32(credits);
  232. else
  233. dp->ricp_v4.dp_credit = cpu_to_be32(credits);
  234. atomic_sub(IB_SET_POST_CREDITS(credits),
  235. &ic->i_credits);
  236. }
  237. }
  238. }
  239. static void rds_ib_cq_event_handler(struct ib_event *event, void *data)
  240. {
  241. rdsdebug("event %u (%s) data %p\n",
  242. event->event, ib_event_msg(event->event), data);
  243. }
  244. /* Plucking the oldest entry from the ring can be done concurrently with
  245. * the thread refilling the ring. Each ring operation is protected by
  246. * spinlocks and the transient state of refilling doesn't change the
  247. * recording of which entry is oldest.
  248. *
  249. * This relies on IB only calling one cq comp_handler for each cq so that
  250. * there will only be one caller of rds_recv_incoming() per RDS connection.
  251. */
  252. static void rds_ib_cq_comp_handler_recv(struct ib_cq *cq, void *context)
  253. {
  254. struct rds_connection *conn = context;
  255. struct rds_ib_connection *ic = conn->c_transport_data;
  256. rdsdebug("conn %p cq %p\n", conn, cq);
  257. rds_ib_stats_inc(s_ib_evt_handler_call);
  258. tasklet_schedule(&ic->i_recv_tasklet);
  259. }
  260. static void poll_scq(struct rds_ib_connection *ic, struct ib_cq *cq,
  261. struct ib_wc *wcs)
  262. {
  263. int nr, i;
  264. struct ib_wc *wc;
  265. while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
  266. for (i = 0; i < nr; i++) {
  267. wc = wcs + i;
  268. rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
  269. (unsigned long long)wc->wr_id, wc->status,
  270. wc->byte_len, be32_to_cpu(wc->ex.imm_data));
  271. if (wc->wr_id <= ic->i_send_ring.w_nr ||
  272. wc->wr_id == RDS_IB_ACK_WR_ID)
  273. rds_ib_send_cqe_handler(ic, wc);
  274. else
  275. rds_ib_mr_cqe_handler(ic, wc);
  276. }
  277. }
  278. }
  279. static void rds_ib_tasklet_fn_send(unsigned long data)
  280. {
  281. struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
  282. struct rds_connection *conn = ic->conn;
  283. rds_ib_stats_inc(s_ib_tasklet_call);
  284. /* if cq has been already reaped, ignore incoming cq event */
  285. if (atomic_read(&ic->i_cq_quiesce))
  286. return;
  287. poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
  288. ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
  289. poll_scq(ic, ic->i_send_cq, ic->i_send_wc);
  290. if (rds_conn_up(conn) &&
  291. (!test_bit(RDS_LL_SEND_FULL, &conn->c_flags) ||
  292. test_bit(0, &conn->c_map_queued)))
  293. rds_send_xmit(&ic->conn->c_path[0]);
  294. }
  295. static void poll_rcq(struct rds_ib_connection *ic, struct ib_cq *cq,
  296. struct ib_wc *wcs,
  297. struct rds_ib_ack_state *ack_state)
  298. {
  299. int nr, i;
  300. struct ib_wc *wc;
  301. while ((nr = ib_poll_cq(cq, RDS_IB_WC_MAX, wcs)) > 0) {
  302. for (i = 0; i < nr; i++) {
  303. wc = wcs + i;
  304. rdsdebug("wc wr_id 0x%llx status %u byte_len %u imm_data %u\n",
  305. (unsigned long long)wc->wr_id, wc->status,
  306. wc->byte_len, be32_to_cpu(wc->ex.imm_data));
  307. rds_ib_recv_cqe_handler(ic, wc, ack_state);
  308. }
  309. }
  310. }
  311. static void rds_ib_tasklet_fn_recv(unsigned long data)
  312. {
  313. struct rds_ib_connection *ic = (struct rds_ib_connection *)data;
  314. struct rds_connection *conn = ic->conn;
  315. struct rds_ib_device *rds_ibdev = ic->rds_ibdev;
  316. struct rds_ib_ack_state state;
  317. if (!rds_ibdev)
  318. rds_conn_drop(conn);
  319. rds_ib_stats_inc(s_ib_tasklet_call);
  320. /* if cq has been already reaped, ignore incoming cq event */
  321. if (atomic_read(&ic->i_cq_quiesce))
  322. return;
  323. memset(&state, 0, sizeof(state));
  324. poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
  325. ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
  326. poll_rcq(ic, ic->i_recv_cq, ic->i_recv_wc, &state);
  327. if (state.ack_next_valid)
  328. rds_ib_set_ack(ic, state.ack_next, state.ack_required);
  329. if (state.ack_recv_valid && state.ack_recv > ic->i_ack_recv) {
  330. rds_send_drop_acked(conn, state.ack_recv, NULL);
  331. ic->i_ack_recv = state.ack_recv;
  332. }
  333. if (rds_conn_up(conn))
  334. rds_ib_attempt_ack(ic);
  335. }
  336. static void rds_ib_qp_event_handler(struct ib_event *event, void *data)
  337. {
  338. struct rds_connection *conn = data;
  339. struct rds_ib_connection *ic = conn->c_transport_data;
  340. rdsdebug("conn %p ic %p event %u (%s)\n", conn, ic, event->event,
  341. ib_event_msg(event->event));
  342. switch (event->event) {
  343. case IB_EVENT_COMM_EST:
  344. rdma_notify(ic->i_cm_id, IB_EVENT_COMM_EST);
  345. break;
  346. default:
  347. rdsdebug("Fatal QP Event %u (%s) - connection %pI6c->%pI6c, reconnecting\n",
  348. event->event, ib_event_msg(event->event),
  349. &conn->c_laddr, &conn->c_faddr);
  350. rds_conn_drop(conn);
  351. break;
  352. }
  353. }
  354. static void rds_ib_cq_comp_handler_send(struct ib_cq *cq, void *context)
  355. {
  356. struct rds_connection *conn = context;
  357. struct rds_ib_connection *ic = conn->c_transport_data;
  358. rdsdebug("conn %p cq %p\n", conn, cq);
  359. rds_ib_stats_inc(s_ib_evt_handler_call);
  360. tasklet_schedule(&ic->i_send_tasklet);
  361. }
  362. static inline int ibdev_get_unused_vector(struct rds_ib_device *rds_ibdev)
  363. {
  364. int min = rds_ibdev->vector_load[rds_ibdev->dev->num_comp_vectors - 1];
  365. int index = rds_ibdev->dev->num_comp_vectors - 1;
  366. int i;
  367. for (i = rds_ibdev->dev->num_comp_vectors - 1; i >= 0; i--) {
  368. if (rds_ibdev->vector_load[i] < min) {
  369. index = i;
  370. min = rds_ibdev->vector_load[i];
  371. }
  372. }
  373. rds_ibdev->vector_load[index]++;
  374. return index;
  375. }
  376. static inline void ibdev_put_vector(struct rds_ib_device *rds_ibdev, int index)
  377. {
  378. rds_ibdev->vector_load[index]--;
  379. }
  380. static void rds_dma_hdr_free(struct ib_device *dev, struct rds_header *hdr,
  381. dma_addr_t dma_addr, enum dma_data_direction dir)
  382. {
  383. ib_dma_unmap_single(dev, dma_addr, sizeof(*hdr), dir);
  384. kfree(hdr);
  385. }
  386. static struct rds_header *rds_dma_hdr_alloc(struct ib_device *dev,
  387. dma_addr_t *dma_addr, enum dma_data_direction dir)
  388. {
  389. struct rds_header *hdr;
  390. hdr = kzalloc_node(sizeof(*hdr), GFP_KERNEL, ibdev_to_node(dev));
  391. if (!hdr)
  392. return NULL;
  393. *dma_addr = ib_dma_map_single(dev, hdr, sizeof(*hdr),
  394. DMA_BIDIRECTIONAL);
  395. if (ib_dma_mapping_error(dev, *dma_addr)) {
  396. kfree(hdr);
  397. return NULL;
  398. }
  399. return hdr;
  400. }
  401. /* Free the DMA memory used to store struct rds_header.
  402. *
  403. * @dev: the RDS IB device
  404. * @hdrs: pointer to the array storing DMA memory pointers
  405. * @dma_addrs: pointer to the array storing DMA addresses
  406. * @num_hdars: number of headers to free.
  407. */
  408. static void rds_dma_hdrs_free(struct rds_ib_device *dev,
  409. struct rds_header **hdrs, dma_addr_t *dma_addrs, u32 num_hdrs,
  410. enum dma_data_direction dir)
  411. {
  412. u32 i;
  413. for (i = 0; i < num_hdrs; i++)
  414. rds_dma_hdr_free(dev->dev, hdrs[i], dma_addrs[i], dir);
  415. kvfree(hdrs);
  416. kvfree(dma_addrs);
  417. }
  418. /* Allocate DMA coherent memory to be used to store struct rds_header for
  419. * sending/receiving packets. The pointers to the DMA memory and the
  420. * associated DMA addresses are stored in two arrays.
  421. *
  422. * @dev: the RDS IB device
  423. * @dma_addrs: pointer to the array for storing DMA addresses
  424. * @num_hdrs: number of headers to allocate
  425. *
  426. * It returns the pointer to the array storing the DMA memory pointers. On
  427. * error, NULL pointer is returned.
  428. */
  429. static struct rds_header **rds_dma_hdrs_alloc(struct rds_ib_device *dev,
  430. dma_addr_t **dma_addrs, u32 num_hdrs,
  431. enum dma_data_direction dir)
  432. {
  433. struct rds_header **hdrs;
  434. dma_addr_t *hdr_daddrs;
  435. u32 i;
  436. hdrs = kvmalloc_node(sizeof(*hdrs) * num_hdrs, GFP_KERNEL,
  437. ibdev_to_node(dev->dev));
  438. if (!hdrs)
  439. return NULL;
  440. hdr_daddrs = kvmalloc_node(sizeof(*hdr_daddrs) * num_hdrs, GFP_KERNEL,
  441. ibdev_to_node(dev->dev));
  442. if (!hdr_daddrs) {
  443. kvfree(hdrs);
  444. return NULL;
  445. }
  446. for (i = 0; i < num_hdrs; i++) {
  447. hdrs[i] = rds_dma_hdr_alloc(dev->dev, &hdr_daddrs[i], dir);
  448. if (!hdrs[i]) {
  449. rds_dma_hdrs_free(dev, hdrs, hdr_daddrs, i, dir);
  450. return NULL;
  451. }
  452. }
  453. *dma_addrs = hdr_daddrs;
  454. return hdrs;
  455. }
  456. /*
  457. * This needs to be very careful to not leave IS_ERR pointers around for
  458. * cleanup to trip over.
  459. */
  460. static int rds_ib_setup_qp(struct rds_connection *conn)
  461. {
  462. struct rds_ib_connection *ic = conn->c_transport_data;
  463. struct ib_device *dev = ic->i_cm_id->device;
  464. struct ib_qp_init_attr attr;
  465. struct ib_cq_init_attr cq_attr = {};
  466. struct rds_ib_device *rds_ibdev;
  467. unsigned long max_wrs;
  468. int ret, fr_queue_space;
  469. /*
  470. * It's normal to see a null device if an incoming connection races
  471. * with device removal, so we don't print a warning.
  472. */
  473. rds_ibdev = rds_ib_get_client_data(dev);
  474. if (!rds_ibdev)
  475. return -EOPNOTSUPP;
  476. /* The fr_queue_space is currently set to 512, to add extra space on
  477. * completion queue and send queue. This extra space is used for FRWR
  478. * registration and invalidation work requests
  479. */
  480. fr_queue_space = RDS_IB_DEFAULT_FR_WR;
  481. /* add the conn now so that connection establishment has the dev */
  482. rds_ib_add_conn(rds_ibdev, conn);
  483. max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_send_wr + 1 ?
  484. rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_send_wr;
  485. if (ic->i_send_ring.w_nr != max_wrs)
  486. rds_ib_ring_resize(&ic->i_send_ring, max_wrs);
  487. max_wrs = rds_ibdev->max_wrs < rds_ib_sysctl_max_recv_wr + 1 ?
  488. rds_ibdev->max_wrs - 1 : rds_ib_sysctl_max_recv_wr;
  489. if (ic->i_recv_ring.w_nr != max_wrs)
  490. rds_ib_ring_resize(&ic->i_recv_ring, max_wrs);
  491. /* Protection domain and memory range */
  492. ic->i_pd = rds_ibdev->pd;
  493. ic->i_scq_vector = ibdev_get_unused_vector(rds_ibdev);
  494. cq_attr.cqe = ic->i_send_ring.w_nr + fr_queue_space + 1;
  495. cq_attr.comp_vector = ic->i_scq_vector;
  496. ic->i_send_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_send,
  497. rds_ib_cq_event_handler, conn,
  498. &cq_attr);
  499. if (IS_ERR(ic->i_send_cq)) {
  500. ret = PTR_ERR(ic->i_send_cq);
  501. ic->i_send_cq = NULL;
  502. ibdev_put_vector(rds_ibdev, ic->i_scq_vector);
  503. rdsdebug("ib_create_cq send failed: %d\n", ret);
  504. goto rds_ibdev_out;
  505. }
  506. ic->i_rcq_vector = ibdev_get_unused_vector(rds_ibdev);
  507. cq_attr.cqe = ic->i_recv_ring.w_nr;
  508. cq_attr.comp_vector = ic->i_rcq_vector;
  509. ic->i_recv_cq = ib_create_cq(dev, rds_ib_cq_comp_handler_recv,
  510. rds_ib_cq_event_handler, conn,
  511. &cq_attr);
  512. if (IS_ERR(ic->i_recv_cq)) {
  513. ret = PTR_ERR(ic->i_recv_cq);
  514. ic->i_recv_cq = NULL;
  515. ibdev_put_vector(rds_ibdev, ic->i_rcq_vector);
  516. rdsdebug("ib_create_cq recv failed: %d\n", ret);
  517. goto send_cq_out;
  518. }
  519. ret = ib_req_notify_cq(ic->i_send_cq, IB_CQ_NEXT_COMP);
  520. if (ret) {
  521. rdsdebug("ib_req_notify_cq send failed: %d\n", ret);
  522. goto recv_cq_out;
  523. }
  524. ret = ib_req_notify_cq(ic->i_recv_cq, IB_CQ_SOLICITED);
  525. if (ret) {
  526. rdsdebug("ib_req_notify_cq recv failed: %d\n", ret);
  527. goto recv_cq_out;
  528. }
  529. /* XXX negotiate max send/recv with remote? */
  530. memset(&attr, 0, sizeof(attr));
  531. attr.event_handler = rds_ib_qp_event_handler;
  532. attr.qp_context = conn;
  533. /* + 1 to allow for the single ack message */
  534. attr.cap.max_send_wr = ic->i_send_ring.w_nr + fr_queue_space + 1;
  535. attr.cap.max_recv_wr = ic->i_recv_ring.w_nr + 1;
  536. attr.cap.max_send_sge = rds_ibdev->max_sge;
  537. attr.cap.max_recv_sge = RDS_IB_RECV_SGE;
  538. attr.sq_sig_type = IB_SIGNAL_REQ_WR;
  539. attr.qp_type = IB_QPT_RC;
  540. attr.send_cq = ic->i_send_cq;
  541. attr.recv_cq = ic->i_recv_cq;
  542. /*
  543. * XXX this can fail if max_*_wr is too large? Are we supposed
  544. * to back off until we get a value that the hardware can support?
  545. */
  546. ret = rdma_create_qp(ic->i_cm_id, ic->i_pd, &attr);
  547. if (ret) {
  548. rdsdebug("rdma_create_qp failed: %d\n", ret);
  549. goto recv_cq_out;
  550. }
  551. ic->i_send_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_send_hdrs_dma,
  552. ic->i_send_ring.w_nr,
  553. DMA_TO_DEVICE);
  554. if (!ic->i_send_hdrs) {
  555. ret = -ENOMEM;
  556. rdsdebug("DMA send hdrs alloc failed\n");
  557. goto qp_out;
  558. }
  559. ic->i_recv_hdrs = rds_dma_hdrs_alloc(rds_ibdev, &ic->i_recv_hdrs_dma,
  560. ic->i_recv_ring.w_nr,
  561. DMA_FROM_DEVICE);
  562. if (!ic->i_recv_hdrs) {
  563. ret = -ENOMEM;
  564. rdsdebug("DMA recv hdrs alloc failed\n");
  565. goto send_hdrs_dma_out;
  566. }
  567. ic->i_ack = rds_dma_hdr_alloc(rds_ibdev->dev, &ic->i_ack_dma,
  568. DMA_TO_DEVICE);
  569. if (!ic->i_ack) {
  570. ret = -ENOMEM;
  571. rdsdebug("DMA ack header alloc failed\n");
  572. goto recv_hdrs_dma_out;
  573. }
  574. ic->i_sends = vzalloc_node(array_size(sizeof(struct rds_ib_send_work),
  575. ic->i_send_ring.w_nr),
  576. ibdev_to_node(dev));
  577. if (!ic->i_sends) {
  578. ret = -ENOMEM;
  579. rdsdebug("send allocation failed\n");
  580. goto ack_dma_out;
  581. }
  582. ic->i_recvs = vzalloc_node(array_size(sizeof(struct rds_ib_recv_work),
  583. ic->i_recv_ring.w_nr),
  584. ibdev_to_node(dev));
  585. if (!ic->i_recvs) {
  586. ret = -ENOMEM;
  587. rdsdebug("recv allocation failed\n");
  588. goto sends_out;
  589. }
  590. rds_ib_recv_init_ack(ic);
  591. rdsdebug("conn %p pd %p cq %p %p\n", conn, ic->i_pd,
  592. ic->i_send_cq, ic->i_recv_cq);
  593. goto out;
  594. sends_out:
  595. vfree(ic->i_sends);
  596. ack_dma_out:
  597. rds_dma_hdr_free(rds_ibdev->dev, ic->i_ack, ic->i_ack_dma,
  598. DMA_TO_DEVICE);
  599. ic->i_ack = NULL;
  600. recv_hdrs_dma_out:
  601. rds_dma_hdrs_free(rds_ibdev, ic->i_recv_hdrs, ic->i_recv_hdrs_dma,
  602. ic->i_recv_ring.w_nr, DMA_FROM_DEVICE);
  603. ic->i_recv_hdrs = NULL;
  604. ic->i_recv_hdrs_dma = NULL;
  605. send_hdrs_dma_out:
  606. rds_dma_hdrs_free(rds_ibdev, ic->i_send_hdrs, ic->i_send_hdrs_dma,
  607. ic->i_send_ring.w_nr, DMA_TO_DEVICE);
  608. ic->i_send_hdrs = NULL;
  609. ic->i_send_hdrs_dma = NULL;
  610. qp_out:
  611. rdma_destroy_qp(ic->i_cm_id);
  612. recv_cq_out:
  613. ib_destroy_cq(ic->i_recv_cq);
  614. ic->i_recv_cq = NULL;
  615. send_cq_out:
  616. ib_destroy_cq(ic->i_send_cq);
  617. ic->i_send_cq = NULL;
  618. rds_ibdev_out:
  619. rds_ib_remove_conn(rds_ibdev, conn);
  620. out:
  621. rds_ib_dev_put(rds_ibdev);
  622. return ret;
  623. }
  624. static u32 rds_ib_protocol_compatible(struct rdma_cm_event *event, bool isv6)
  625. {
  626. const union rds_ib_conn_priv *dp = event->param.conn.private_data;
  627. u8 data_len, major, minor;
  628. u32 version = 0;
  629. __be16 mask;
  630. u16 common;
  631. /*
  632. * rdma_cm private data is odd - when there is any private data in the
  633. * request, we will be given a pretty large buffer without telling us the
  634. * original size. The only way to tell the difference is by looking at
  635. * the contents, which are initialized to zero.
  636. * If the protocol version fields aren't set, this is a connection attempt
  637. * from an older version. This could be 3.0 or 2.0 - we can't tell.
  638. * We really should have changed this for OFED 1.3 :-(
  639. */
  640. /* Be paranoid. RDS always has privdata */
  641. if (!event->param.conn.private_data_len) {
  642. printk(KERN_NOTICE "RDS incoming connection has no private data, "
  643. "rejecting\n");
  644. return 0;
  645. }
  646. if (isv6) {
  647. data_len = sizeof(struct rds6_ib_connect_private);
  648. major = dp->ricp_v6.dp_protocol_major;
  649. minor = dp->ricp_v6.dp_protocol_minor;
  650. mask = dp->ricp_v6.dp_protocol_minor_mask;
  651. } else {
  652. data_len = sizeof(struct rds_ib_connect_private);
  653. major = dp->ricp_v4.dp_protocol_major;
  654. minor = dp->ricp_v4.dp_protocol_minor;
  655. mask = dp->ricp_v4.dp_protocol_minor_mask;
  656. }
  657. /* Even if len is crap *now* I still want to check it. -ASG */
  658. if (event->param.conn.private_data_len < data_len || major == 0)
  659. return RDS_PROTOCOL_4_0;
  660. common = be16_to_cpu(mask) & RDS_IB_SUPPORTED_PROTOCOLS;
  661. if (major == 4 && common) {
  662. version = RDS_PROTOCOL_4_0;
  663. while ((common >>= 1) != 0)
  664. version++;
  665. } else if (RDS_PROTOCOL_COMPAT_VERSION ==
  666. RDS_PROTOCOL(major, minor)) {
  667. version = RDS_PROTOCOL_COMPAT_VERSION;
  668. } else {
  669. if (isv6)
  670. printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI6c using incompatible protocol version %u.%u\n",
  671. &dp->ricp_v6.dp_saddr, major, minor);
  672. else
  673. printk_ratelimited(KERN_NOTICE "RDS: Connection from %pI4 using incompatible protocol version %u.%u\n",
  674. &dp->ricp_v4.dp_saddr, major, minor);
  675. }
  676. return version;
  677. }
  678. #if IS_ENABLED(CONFIG_IPV6)
  679. /* Given an IPv6 address, find the net_device which hosts that address and
  680. * return its index. This is used by the rds_ib_cm_handle_connect() code to
  681. * find the interface index of where an incoming request comes from when
  682. * the request is using a link local address.
  683. *
  684. * Note one problem in this search. It is possible that two interfaces have
  685. * the same link local address. Unfortunately, this cannot be solved unless
  686. * the underlying layer gives us the interface which an incoming RDMA connect
  687. * request comes from.
  688. */
  689. static u32 __rds_find_ifindex(struct net *net, const struct in6_addr *addr)
  690. {
  691. struct net_device *dev;
  692. int idx = 0;
  693. rcu_read_lock();
  694. for_each_netdev_rcu(net, dev) {
  695. if (ipv6_chk_addr(net, addr, dev, 1)) {
  696. idx = dev->ifindex;
  697. break;
  698. }
  699. }
  700. rcu_read_unlock();
  701. return idx;
  702. }
  703. #endif
  704. int rds_ib_cm_handle_connect(struct rdma_cm_id *cm_id,
  705. struct rdma_cm_event *event, bool isv6)
  706. {
  707. __be64 lguid = cm_id->route.path_rec->sgid.global.interface_id;
  708. __be64 fguid = cm_id->route.path_rec->dgid.global.interface_id;
  709. const struct rds_ib_conn_priv_cmn *dp_cmn;
  710. struct rds_connection *conn = NULL;
  711. struct rds_ib_connection *ic = NULL;
  712. struct rdma_conn_param conn_param;
  713. const union rds_ib_conn_priv *dp;
  714. union rds_ib_conn_priv dp_rep;
  715. struct in6_addr s_mapped_addr;
  716. struct in6_addr d_mapped_addr;
  717. const struct in6_addr *saddr6;
  718. const struct in6_addr *daddr6;
  719. int destroy = 1;
  720. u32 ifindex = 0;
  721. u32 version;
  722. int err = 1;
  723. /* Check whether the remote protocol version matches ours. */
  724. version = rds_ib_protocol_compatible(event, isv6);
  725. if (!version) {
  726. err = RDS_RDMA_REJ_INCOMPAT;
  727. goto out;
  728. }
  729. dp = event->param.conn.private_data;
  730. if (isv6) {
  731. #if IS_ENABLED(CONFIG_IPV6)
  732. dp_cmn = &dp->ricp_v6.dp_cmn;
  733. saddr6 = &dp->ricp_v6.dp_saddr;
  734. daddr6 = &dp->ricp_v6.dp_daddr;
  735. /* If either address is link local, need to find the
  736. * interface index in order to create a proper RDS
  737. * connection.
  738. */
  739. if (ipv6_addr_type(daddr6) & IPV6_ADDR_LINKLOCAL) {
  740. /* Using init_net for now .. */
  741. ifindex = __rds_find_ifindex(&init_net, daddr6);
  742. /* No index found... Need to bail out. */
  743. if (ifindex == 0) {
  744. err = -EOPNOTSUPP;
  745. goto out;
  746. }
  747. } else if (ipv6_addr_type(saddr6) & IPV6_ADDR_LINKLOCAL) {
  748. /* Use our address to find the correct index. */
  749. ifindex = __rds_find_ifindex(&init_net, daddr6);
  750. /* No index found... Need to bail out. */
  751. if (ifindex == 0) {
  752. err = -EOPNOTSUPP;
  753. goto out;
  754. }
  755. }
  756. #else
  757. err = -EOPNOTSUPP;
  758. goto out;
  759. #endif
  760. } else {
  761. dp_cmn = &dp->ricp_v4.dp_cmn;
  762. ipv6_addr_set_v4mapped(dp->ricp_v4.dp_saddr, &s_mapped_addr);
  763. ipv6_addr_set_v4mapped(dp->ricp_v4.dp_daddr, &d_mapped_addr);
  764. saddr6 = &s_mapped_addr;
  765. daddr6 = &d_mapped_addr;
  766. }
  767. rdsdebug("saddr %pI6c daddr %pI6c RDSv%u.%u lguid 0x%llx fguid 0x%llx, tos:%d\n",
  768. saddr6, daddr6, RDS_PROTOCOL_MAJOR(version),
  769. RDS_PROTOCOL_MINOR(version),
  770. (unsigned long long)be64_to_cpu(lguid),
  771. (unsigned long long)be64_to_cpu(fguid), dp_cmn->ricpc_dp_toss);
  772. /* RDS/IB is not currently netns aware, thus init_net */
  773. conn = rds_conn_create(&init_net, daddr6, saddr6,
  774. &rds_ib_transport, dp_cmn->ricpc_dp_toss,
  775. GFP_KERNEL, ifindex);
  776. if (IS_ERR(conn)) {
  777. rdsdebug("rds_conn_create failed (%ld)\n", PTR_ERR(conn));
  778. conn = NULL;
  779. goto out;
  780. }
  781. /*
  782. * The connection request may occur while the
  783. * previous connection exist, e.g. in case of failover.
  784. * But as connections may be initiated simultaneously
  785. * by both hosts, we have a random backoff mechanism -
  786. * see the comment above rds_queue_reconnect()
  787. */
  788. mutex_lock(&conn->c_cm_lock);
  789. if (!rds_conn_transition(conn, RDS_CONN_DOWN, RDS_CONN_CONNECTING)) {
  790. if (rds_conn_state(conn) == RDS_CONN_UP) {
  791. rdsdebug("incoming connect while connecting\n");
  792. rds_conn_drop(conn);
  793. rds_ib_stats_inc(s_ib_listen_closed_stale);
  794. } else
  795. if (rds_conn_state(conn) == RDS_CONN_CONNECTING) {
  796. /* Wait and see - our connect may still be succeeding */
  797. rds_ib_stats_inc(s_ib_connect_raced);
  798. }
  799. goto out;
  800. }
  801. ic = conn->c_transport_data;
  802. rds_ib_set_protocol(conn, version);
  803. rds_ib_set_flow_control(conn, be32_to_cpu(dp_cmn->ricpc_credit));
  804. /* If the peer gave us the last packet it saw, process this as if
  805. * we had received a regular ACK. */
  806. if (dp_cmn->ricpc_ack_seq)
  807. rds_send_drop_acked(conn, be64_to_cpu(dp_cmn->ricpc_ack_seq),
  808. NULL);
  809. BUG_ON(cm_id->context);
  810. BUG_ON(ic->i_cm_id);
  811. ic->i_cm_id = cm_id;
  812. cm_id->context = conn;
  813. /* We got halfway through setting up the ib_connection, if we
  814. * fail now, we have to take the long route out of this mess. */
  815. destroy = 0;
  816. err = rds_ib_setup_qp(conn);
  817. if (err) {
  818. rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", err);
  819. goto out;
  820. }
  821. rds_ib_cm_fill_conn_param(conn, &conn_param, &dp_rep, version,
  822. event->param.conn.responder_resources,
  823. event->param.conn.initiator_depth, isv6);
  824. /* rdma_accept() calls rdma_reject() internally if it fails */
  825. if (rdma_accept(cm_id, &conn_param))
  826. rds_ib_conn_error(conn, "rdma_accept failed\n");
  827. out:
  828. if (conn)
  829. mutex_unlock(&conn->c_cm_lock);
  830. if (err)
  831. rdma_reject(cm_id, &err, sizeof(int),
  832. IB_CM_REJ_CONSUMER_DEFINED);
  833. return destroy;
  834. }
  835. int rds_ib_cm_initiate_connect(struct rdma_cm_id *cm_id, bool isv6)
  836. {
  837. struct rds_connection *conn = cm_id->context;
  838. struct rds_ib_connection *ic = conn->c_transport_data;
  839. struct rdma_conn_param conn_param;
  840. union rds_ib_conn_priv dp;
  841. int ret;
  842. /* If the peer doesn't do protocol negotiation, we must
  843. * default to RDSv3.0 */
  844. rds_ib_set_protocol(conn, RDS_PROTOCOL_4_1);
  845. ic->i_flowctl = rds_ib_sysctl_flow_control; /* advertise flow control */
  846. ret = rds_ib_setup_qp(conn);
  847. if (ret) {
  848. rds_ib_conn_error(conn, "rds_ib_setup_qp failed (%d)\n", ret);
  849. goto out;
  850. }
  851. rds_ib_cm_fill_conn_param(conn, &conn_param, &dp,
  852. conn->c_proposed_version,
  853. UINT_MAX, UINT_MAX, isv6);
  854. ret = rdma_connect_locked(cm_id, &conn_param);
  855. if (ret)
  856. rds_ib_conn_error(conn, "rdma_connect_locked failed (%d)\n",
  857. ret);
  858. out:
  859. /* Beware - returning non-zero tells the rdma_cm to destroy
  860. * the cm_id. We should certainly not do it as long as we still
  861. * "own" the cm_id. */
  862. if (ret) {
  863. if (ic->i_cm_id == cm_id)
  864. ret = 0;
  865. }
  866. ic->i_active_side = true;
  867. return ret;
  868. }
  869. int rds_ib_conn_path_connect(struct rds_conn_path *cp)
  870. {
  871. struct rds_connection *conn = cp->cp_conn;
  872. struct sockaddr_storage src, dest;
  873. rdma_cm_event_handler handler;
  874. struct rds_ib_connection *ic;
  875. int ret;
  876. ic = conn->c_transport_data;
  877. /* XXX I wonder what affect the port space has */
  878. /* delegate cm event handler to rdma_transport */
  879. #if IS_ENABLED(CONFIG_IPV6)
  880. if (conn->c_isv6)
  881. handler = rds6_rdma_cm_event_handler;
  882. else
  883. #endif
  884. handler = rds_rdma_cm_event_handler;
  885. ic->i_cm_id = rdma_create_id(&init_net, handler, conn,
  886. RDMA_PS_TCP, IB_QPT_RC);
  887. if (IS_ERR(ic->i_cm_id)) {
  888. ret = PTR_ERR(ic->i_cm_id);
  889. ic->i_cm_id = NULL;
  890. rdsdebug("rdma_create_id() failed: %d\n", ret);
  891. goto out;
  892. }
  893. rdsdebug("created cm id %p for conn %p\n", ic->i_cm_id, conn);
  894. if (ipv6_addr_v4mapped(&conn->c_faddr)) {
  895. struct sockaddr_in *sin;
  896. sin = (struct sockaddr_in *)&src;
  897. sin->sin_family = AF_INET;
  898. sin->sin_addr.s_addr = conn->c_laddr.s6_addr32[3];
  899. sin->sin_port = 0;
  900. sin = (struct sockaddr_in *)&dest;
  901. sin->sin_family = AF_INET;
  902. sin->sin_addr.s_addr = conn->c_faddr.s6_addr32[3];
  903. sin->sin_port = htons(RDS_PORT);
  904. } else {
  905. struct sockaddr_in6 *sin6;
  906. sin6 = (struct sockaddr_in6 *)&src;
  907. sin6->sin6_family = AF_INET6;
  908. sin6->sin6_addr = conn->c_laddr;
  909. sin6->sin6_port = 0;
  910. sin6->sin6_scope_id = conn->c_dev_if;
  911. sin6 = (struct sockaddr_in6 *)&dest;
  912. sin6->sin6_family = AF_INET6;
  913. sin6->sin6_addr = conn->c_faddr;
  914. sin6->sin6_port = htons(RDS_CM_PORT);
  915. sin6->sin6_scope_id = conn->c_dev_if;
  916. }
  917. ret = rdma_resolve_addr(ic->i_cm_id, (struct sockaddr *)&src,
  918. (struct sockaddr *)&dest,
  919. RDS_RDMA_RESOLVE_TIMEOUT_MS);
  920. if (ret) {
  921. rdsdebug("addr resolve failed for cm id %p: %d\n", ic->i_cm_id,
  922. ret);
  923. rdma_destroy_id(ic->i_cm_id);
  924. ic->i_cm_id = NULL;
  925. }
  926. out:
  927. return ret;
  928. }
  929. /*
  930. * This is so careful about only cleaning up resources that were built up
  931. * so that it can be called at any point during startup. In fact it
  932. * can be called multiple times for a given connection.
  933. */
  934. void rds_ib_conn_path_shutdown(struct rds_conn_path *cp)
  935. {
  936. struct rds_connection *conn = cp->cp_conn;
  937. struct rds_ib_connection *ic = conn->c_transport_data;
  938. int err = 0;
  939. rdsdebug("cm %p pd %p cq %p %p qp %p\n", ic->i_cm_id,
  940. ic->i_pd, ic->i_send_cq, ic->i_recv_cq,
  941. ic->i_cm_id ? ic->i_cm_id->qp : NULL);
  942. if (ic->i_cm_id) {
  943. rdsdebug("disconnecting cm %p\n", ic->i_cm_id);
  944. err = rdma_disconnect(ic->i_cm_id);
  945. if (err) {
  946. /* Actually this may happen quite frequently, when
  947. * an outgoing connect raced with an incoming connect.
  948. */
  949. rdsdebug("failed to disconnect, cm: %p err %d\n",
  950. ic->i_cm_id, err);
  951. }
  952. /* kick off "flush_worker" for all pools in order to reap
  953. * all FRMR registrations that are still marked "FRMR_IS_INUSE"
  954. */
  955. rds_ib_flush_mrs();
  956. /*
  957. * We want to wait for tx and rx completion to finish
  958. * before we tear down the connection, but we have to be
  959. * careful not to get stuck waiting on a send ring that
  960. * only has unsignaled sends in it. We've shutdown new
  961. * sends before getting here so by waiting for signaled
  962. * sends to complete we're ensured that there will be no
  963. * more tx processing.
  964. */
  965. wait_event(rds_ib_ring_empty_wait,
  966. rds_ib_ring_empty(&ic->i_recv_ring) &&
  967. (atomic_read(&ic->i_signaled_sends) == 0) &&
  968. (atomic_read(&ic->i_fastreg_inuse_count) == 0) &&
  969. (atomic_read(&ic->i_fastreg_wrs) == RDS_IB_DEFAULT_FR_WR));
  970. tasklet_kill(&ic->i_send_tasklet);
  971. tasklet_kill(&ic->i_recv_tasklet);
  972. atomic_set(&ic->i_cq_quiesce, 1);
  973. /* first destroy the ib state that generates callbacks */
  974. if (ic->i_cm_id->qp)
  975. rdma_destroy_qp(ic->i_cm_id);
  976. if (ic->i_send_cq) {
  977. if (ic->rds_ibdev)
  978. ibdev_put_vector(ic->rds_ibdev, ic->i_scq_vector);
  979. ib_destroy_cq(ic->i_send_cq);
  980. }
  981. if (ic->i_recv_cq) {
  982. if (ic->rds_ibdev)
  983. ibdev_put_vector(ic->rds_ibdev, ic->i_rcq_vector);
  984. ib_destroy_cq(ic->i_recv_cq);
  985. }
  986. if (ic->rds_ibdev) {
  987. /* then free the resources that ib callbacks use */
  988. if (ic->i_send_hdrs) {
  989. rds_dma_hdrs_free(ic->rds_ibdev,
  990. ic->i_send_hdrs,
  991. ic->i_send_hdrs_dma,
  992. ic->i_send_ring.w_nr,
  993. DMA_TO_DEVICE);
  994. ic->i_send_hdrs = NULL;
  995. ic->i_send_hdrs_dma = NULL;
  996. }
  997. if (ic->i_recv_hdrs) {
  998. rds_dma_hdrs_free(ic->rds_ibdev,
  999. ic->i_recv_hdrs,
  1000. ic->i_recv_hdrs_dma,
  1001. ic->i_recv_ring.w_nr,
  1002. DMA_FROM_DEVICE);
  1003. ic->i_recv_hdrs = NULL;
  1004. ic->i_recv_hdrs_dma = NULL;
  1005. }
  1006. if (ic->i_ack) {
  1007. rds_dma_hdr_free(ic->rds_ibdev->dev, ic->i_ack,
  1008. ic->i_ack_dma, DMA_TO_DEVICE);
  1009. ic->i_ack = NULL;
  1010. }
  1011. } else {
  1012. WARN_ON(ic->i_send_hdrs);
  1013. WARN_ON(ic->i_send_hdrs_dma);
  1014. WARN_ON(ic->i_recv_hdrs);
  1015. WARN_ON(ic->i_recv_hdrs_dma);
  1016. WARN_ON(ic->i_ack);
  1017. }
  1018. if (ic->i_sends)
  1019. rds_ib_send_clear_ring(ic);
  1020. if (ic->i_recvs)
  1021. rds_ib_recv_clear_ring(ic);
  1022. rdma_destroy_id(ic->i_cm_id);
  1023. /*
  1024. * Move connection back to the nodev list.
  1025. */
  1026. if (ic->rds_ibdev)
  1027. rds_ib_remove_conn(ic->rds_ibdev, conn);
  1028. ic->i_cm_id = NULL;
  1029. ic->i_pd = NULL;
  1030. ic->i_send_cq = NULL;
  1031. ic->i_recv_cq = NULL;
  1032. }
  1033. BUG_ON(ic->rds_ibdev);
  1034. /* Clear pending transmit */
  1035. if (ic->i_data_op) {
  1036. struct rds_message *rm;
  1037. rm = container_of(ic->i_data_op, struct rds_message, data);
  1038. rds_message_put(rm);
  1039. ic->i_data_op = NULL;
  1040. }
  1041. /* Clear the ACK state */
  1042. clear_bit(IB_ACK_IN_FLIGHT, &ic->i_ack_flags);
  1043. #ifdef KERNEL_HAS_ATOMIC64
  1044. atomic64_set(&ic->i_ack_next, 0);
  1045. #else
  1046. ic->i_ack_next = 0;
  1047. #endif
  1048. ic->i_ack_recv = 0;
  1049. /* Clear flow control state */
  1050. ic->i_flowctl = 0;
  1051. atomic_set(&ic->i_credits, 0);
  1052. /* Re-init rings, but retain sizes. */
  1053. rds_ib_ring_init(&ic->i_send_ring, ic->i_send_ring.w_nr);
  1054. rds_ib_ring_init(&ic->i_recv_ring, ic->i_recv_ring.w_nr);
  1055. if (ic->i_ibinc) {
  1056. rds_inc_put(&ic->i_ibinc->ii_inc);
  1057. ic->i_ibinc = NULL;
  1058. }
  1059. vfree(ic->i_sends);
  1060. ic->i_sends = NULL;
  1061. vfree(ic->i_recvs);
  1062. ic->i_recvs = NULL;
  1063. ic->i_active_side = false;
  1064. }
  1065. int rds_ib_conn_alloc(struct rds_connection *conn, gfp_t gfp)
  1066. {
  1067. struct rds_ib_connection *ic;
  1068. unsigned long flags;
  1069. int ret;
  1070. /* XXX too lazy? */
  1071. ic = kzalloc(sizeof(struct rds_ib_connection), gfp);
  1072. if (!ic)
  1073. return -ENOMEM;
  1074. ret = rds_ib_recv_alloc_caches(ic, gfp);
  1075. if (ret) {
  1076. kfree(ic);
  1077. return ret;
  1078. }
  1079. INIT_LIST_HEAD(&ic->ib_node);
  1080. tasklet_init(&ic->i_send_tasklet, rds_ib_tasklet_fn_send,
  1081. (unsigned long)ic);
  1082. tasklet_init(&ic->i_recv_tasklet, rds_ib_tasklet_fn_recv,
  1083. (unsigned long)ic);
  1084. mutex_init(&ic->i_recv_mutex);
  1085. #ifndef KERNEL_HAS_ATOMIC64
  1086. spin_lock_init(&ic->i_ack_lock);
  1087. #endif
  1088. atomic_set(&ic->i_signaled_sends, 0);
  1089. atomic_set(&ic->i_fastreg_wrs, RDS_IB_DEFAULT_FR_WR);
  1090. /*
  1091. * rds_ib_conn_shutdown() waits for these to be emptied so they
  1092. * must be initialized before it can be called.
  1093. */
  1094. rds_ib_ring_init(&ic->i_send_ring, 0);
  1095. rds_ib_ring_init(&ic->i_recv_ring, 0);
  1096. ic->conn = conn;
  1097. conn->c_transport_data = ic;
  1098. spin_lock_irqsave(&ib_nodev_conns_lock, flags);
  1099. list_add_tail(&ic->ib_node, &ib_nodev_conns);
  1100. spin_unlock_irqrestore(&ib_nodev_conns_lock, flags);
  1101. rdsdebug("conn %p conn ic %p\n", conn, conn->c_transport_data);
  1102. return 0;
  1103. }
  1104. /*
  1105. * Free a connection. Connection must be shut down and not set for reconnect.
  1106. */
  1107. void rds_ib_conn_free(void *arg)
  1108. {
  1109. struct rds_ib_connection *ic = arg;
  1110. spinlock_t *lock_ptr;
  1111. rdsdebug("ic %p\n", ic);
  1112. /*
  1113. * Conn is either on a dev's list or on the nodev list.
  1114. * A race with shutdown() or connect() would cause problems
  1115. * (since rds_ibdev would change) but that should never happen.
  1116. */
  1117. lock_ptr = ic->rds_ibdev ? &ic->rds_ibdev->spinlock : &ib_nodev_conns_lock;
  1118. spin_lock_irq(lock_ptr);
  1119. list_del(&ic->ib_node);
  1120. spin_unlock_irq(lock_ptr);
  1121. rds_ib_recv_free_caches(ic);
  1122. kfree(ic);
  1123. }
  1124. /*
  1125. * An error occurred on the connection
  1126. */
  1127. void
  1128. __rds_ib_conn_error(struct rds_connection *conn, const char *fmt, ...)
  1129. {
  1130. va_list ap;
  1131. rds_conn_drop(conn);
  1132. va_start(ap, fmt);
  1133. vprintk(fmt, ap);
  1134. va_end(ap);
  1135. }