svc_rdma_transport.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614
  1. // SPDX-License-Identifier: GPL-2.0 OR BSD-3-Clause
  2. /*
  3. * Copyright (c) 2015-2018 Oracle. All rights reserved.
  4. * Copyright (c) 2014 Open Grid Computing, Inc. All rights reserved.
  5. * Copyright (c) 2005-2007 Network Appliance, Inc. All rights reserved.
  6. *
  7. * This software is available to you under a choice of one of two
  8. * licenses. You may choose to be licensed under the terms of the GNU
  9. * General Public License (GPL) Version 2, available from the file
  10. * COPYING in the main directory of this source tree, or the BSD-type
  11. * license below:
  12. *
  13. * Redistribution and use in source and binary forms, with or without
  14. * modification, are permitted provided that the following conditions
  15. * are met:
  16. *
  17. * Redistributions of source code must retain the above copyright
  18. * notice, this list of conditions and the following disclaimer.
  19. *
  20. * Redistributions in binary form must reproduce the above
  21. * copyright notice, this list of conditions and the following
  22. * disclaimer in the documentation and/or other materials provided
  23. * with the distribution.
  24. *
  25. * Neither the name of the Network Appliance, Inc. nor the names of
  26. * its contributors may be used to endorse or promote products
  27. * derived from this software without specific prior written
  28. * permission.
  29. *
  30. * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
  31. * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
  32. * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
  33. * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
  34. * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
  35. * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
  36. * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
  37. * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
  38. * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
  39. * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
  40. * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  41. *
  42. * Author: Tom Tucker <tom@opengridcomputing.com>
  43. */
  44. #include <linux/interrupt.h>
  45. #include <linux/sched.h>
  46. #include <linux/slab.h>
  47. #include <linux/spinlock.h>
  48. #include <linux/workqueue.h>
  49. #include <linux/export.h>
  50. #include <rdma/ib_verbs.h>
  51. #include <rdma/rdma_cm.h>
  52. #include <rdma/rw.h>
  53. #include <linux/sunrpc/addr.h>
  54. #include <linux/sunrpc/debug.h>
  55. #include <linux/sunrpc/svc_xprt.h>
  56. #include <linux/sunrpc/svc_rdma.h>
  57. #include "xprt_rdma.h"
  58. #include <trace/events/rpcrdma.h>
  59. #define RPCDBG_FACILITY RPCDBG_SVCXPRT
  60. static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
  61. struct net *net);
  62. static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
  63. struct net *net,
  64. struct sockaddr *sa, int salen,
  65. int flags);
  66. static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt);
  67. static void svc_rdma_detach(struct svc_xprt *xprt);
  68. static void svc_rdma_free(struct svc_xprt *xprt);
  69. static int svc_rdma_has_wspace(struct svc_xprt *xprt);
  70. static void svc_rdma_secure_port(struct svc_rqst *);
  71. static void svc_rdma_kill_temp_xprt(struct svc_xprt *);
  72. static const struct svc_xprt_ops svc_rdma_ops = {
  73. .xpo_create = svc_rdma_create,
  74. .xpo_recvfrom = svc_rdma_recvfrom,
  75. .xpo_sendto = svc_rdma_sendto,
  76. .xpo_read_payload = svc_rdma_read_payload,
  77. .xpo_release_rqst = svc_rdma_release_rqst,
  78. .xpo_detach = svc_rdma_detach,
  79. .xpo_free = svc_rdma_free,
  80. .xpo_has_wspace = svc_rdma_has_wspace,
  81. .xpo_accept = svc_rdma_accept,
  82. .xpo_secure_port = svc_rdma_secure_port,
  83. .xpo_kill_temp_xprt = svc_rdma_kill_temp_xprt,
  84. };
  85. struct svc_xprt_class svc_rdma_class = {
  86. .xcl_name = "rdma",
  87. .xcl_owner = THIS_MODULE,
  88. .xcl_ops = &svc_rdma_ops,
  89. .xcl_max_payload = RPCSVC_MAXPAYLOAD_RDMA,
  90. .xcl_ident = XPRT_TRANSPORT_RDMA,
  91. };
  92. /* QP event handler */
  93. static void qp_event_handler(struct ib_event *event, void *context)
  94. {
  95. struct svc_xprt *xprt = context;
  96. trace_svcrdma_qp_error(event, (struct sockaddr *)&xprt->xpt_remote);
  97. switch (event->event) {
  98. /* These are considered benign events */
  99. case IB_EVENT_PATH_MIG:
  100. case IB_EVENT_COMM_EST:
  101. case IB_EVENT_SQ_DRAINED:
  102. case IB_EVENT_QP_LAST_WQE_REACHED:
  103. break;
  104. /* These are considered fatal events */
  105. case IB_EVENT_PATH_MIG_ERR:
  106. case IB_EVENT_QP_FATAL:
  107. case IB_EVENT_QP_REQ_ERR:
  108. case IB_EVENT_QP_ACCESS_ERR:
  109. case IB_EVENT_DEVICE_FATAL:
  110. default:
  111. set_bit(XPT_CLOSE, &xprt->xpt_flags);
  112. svc_xprt_enqueue(xprt);
  113. break;
  114. }
  115. }
  116. static struct svcxprt_rdma *svc_rdma_create_xprt(struct svc_serv *serv,
  117. struct net *net)
  118. {
  119. struct svcxprt_rdma *cma_xprt = kzalloc(sizeof *cma_xprt, GFP_KERNEL);
  120. if (!cma_xprt) {
  121. dprintk("svcrdma: failed to create new transport\n");
  122. return NULL;
  123. }
  124. svc_xprt_init(net, &svc_rdma_class, &cma_xprt->sc_xprt, serv);
  125. INIT_LIST_HEAD(&cma_xprt->sc_accept_q);
  126. INIT_LIST_HEAD(&cma_xprt->sc_rq_dto_q);
  127. INIT_LIST_HEAD(&cma_xprt->sc_read_complete_q);
  128. INIT_LIST_HEAD(&cma_xprt->sc_send_ctxts);
  129. init_llist_head(&cma_xprt->sc_recv_ctxts);
  130. INIT_LIST_HEAD(&cma_xprt->sc_rw_ctxts);
  131. init_waitqueue_head(&cma_xprt->sc_send_wait);
  132. spin_lock_init(&cma_xprt->sc_lock);
  133. spin_lock_init(&cma_xprt->sc_rq_dto_lock);
  134. spin_lock_init(&cma_xprt->sc_send_lock);
  135. spin_lock_init(&cma_xprt->sc_rw_ctxt_lock);
  136. /*
  137. * Note that this implies that the underlying transport support
  138. * has some form of congestion control (see RFC 7530 section 3.1
  139. * paragraph 2). For now, we assume that all supported RDMA
  140. * transports are suitable here.
  141. */
  142. set_bit(XPT_CONG_CTRL, &cma_xprt->sc_xprt.xpt_flags);
  143. return cma_xprt;
  144. }
  145. static void
  146. svc_rdma_parse_connect_private(struct svcxprt_rdma *newxprt,
  147. struct rdma_conn_param *param)
  148. {
  149. const struct rpcrdma_connect_private *pmsg = param->private_data;
  150. if (pmsg &&
  151. pmsg->cp_magic == rpcrdma_cmp_magic &&
  152. pmsg->cp_version == RPCRDMA_CMP_VERSION) {
  153. newxprt->sc_snd_w_inv = pmsg->cp_flags &
  154. RPCRDMA_CMP_F_SND_W_INV_OK;
  155. dprintk("svcrdma: client send_size %u, recv_size %u "
  156. "remote inv %ssupported\n",
  157. rpcrdma_decode_buffer_size(pmsg->cp_send_size),
  158. rpcrdma_decode_buffer_size(pmsg->cp_recv_size),
  159. newxprt->sc_snd_w_inv ? "" : "un");
  160. }
  161. }
  162. /*
  163. * This function handles the CONNECT_REQUEST event on a listening
  164. * endpoint. It is passed the cma_id for the _new_ connection. The context in
  165. * this cma_id is inherited from the listening cma_id and is the svc_xprt
  166. * structure for the listening endpoint.
  167. *
  168. * This function creates a new xprt for the new connection and enqueues it on
  169. * the accept queue for the listent xprt. When the listen thread is kicked, it
  170. * will call the recvfrom method on the listen xprt which will accept the new
  171. * connection.
  172. */
  173. static void handle_connect_req(struct rdma_cm_id *new_cma_id,
  174. struct rdma_conn_param *param)
  175. {
  176. struct svcxprt_rdma *listen_xprt = new_cma_id->context;
  177. struct svcxprt_rdma *newxprt;
  178. struct sockaddr *sa;
  179. /* Create a new transport */
  180. newxprt = svc_rdma_create_xprt(listen_xprt->sc_xprt.xpt_server,
  181. listen_xprt->sc_xprt.xpt_net);
  182. if (!newxprt)
  183. return;
  184. newxprt->sc_cm_id = new_cma_id;
  185. new_cma_id->context = newxprt;
  186. svc_rdma_parse_connect_private(newxprt, param);
  187. /* Save client advertised inbound read limit for use later in accept. */
  188. newxprt->sc_ord = param->initiator_depth;
  189. sa = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr;
  190. newxprt->sc_xprt.xpt_remotelen = svc_addr_len(sa);
  191. memcpy(&newxprt->sc_xprt.xpt_remote, sa,
  192. newxprt->sc_xprt.xpt_remotelen);
  193. snprintf(newxprt->sc_xprt.xpt_remotebuf,
  194. sizeof(newxprt->sc_xprt.xpt_remotebuf) - 1, "%pISc", sa);
  195. /* The remote port is arbitrary and not under the control of the
  196. * client ULP. Set it to a fixed value so that the DRC continues
  197. * to be effective after a reconnect.
  198. */
  199. rpc_set_port((struct sockaddr *)&newxprt->sc_xprt.xpt_remote, 0);
  200. sa = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr;
  201. svc_xprt_set_local(&newxprt->sc_xprt, sa, svc_addr_len(sa));
  202. /*
  203. * Enqueue the new transport on the accept queue of the listening
  204. * transport
  205. */
  206. spin_lock(&listen_xprt->sc_lock);
  207. list_add_tail(&newxprt->sc_accept_q, &listen_xprt->sc_accept_q);
  208. spin_unlock(&listen_xprt->sc_lock);
  209. set_bit(XPT_CONN, &listen_xprt->sc_xprt.xpt_flags);
  210. svc_xprt_enqueue(&listen_xprt->sc_xprt);
  211. }
  212. /**
  213. * svc_rdma_listen_handler - Handle CM events generated on a listening endpoint
  214. * @cma_id: the server's listener rdma_cm_id
  215. * @event: details of the event
  216. *
  217. * Return values:
  218. * %0: Do not destroy @cma_id
  219. * %1: Destroy @cma_id (never returned here)
  220. *
  221. * NB: There is never a DEVICE_REMOVAL event for INADDR_ANY listeners.
  222. */
  223. static int svc_rdma_listen_handler(struct rdma_cm_id *cma_id,
  224. struct rdma_cm_event *event)
  225. {
  226. switch (event->event) {
  227. case RDMA_CM_EVENT_CONNECT_REQUEST:
  228. handle_connect_req(cma_id, &event->param.conn);
  229. break;
  230. default:
  231. break;
  232. }
  233. return 0;
  234. }
  235. /**
  236. * svc_rdma_cma_handler - Handle CM events on client connections
  237. * @cma_id: the server's listener rdma_cm_id
  238. * @event: details of the event
  239. *
  240. * Return values:
  241. * %0: Do not destroy @cma_id
  242. * %1: Destroy @cma_id (never returned here)
  243. */
  244. static int svc_rdma_cma_handler(struct rdma_cm_id *cma_id,
  245. struct rdma_cm_event *event)
  246. {
  247. struct svcxprt_rdma *rdma = cma_id->context;
  248. struct svc_xprt *xprt = &rdma->sc_xprt;
  249. switch (event->event) {
  250. case RDMA_CM_EVENT_ESTABLISHED:
  251. clear_bit(RDMAXPRT_CONN_PENDING, &rdma->sc_flags);
  252. svc_xprt_enqueue(xprt);
  253. break;
  254. case RDMA_CM_EVENT_DISCONNECTED:
  255. case RDMA_CM_EVENT_DEVICE_REMOVAL:
  256. set_bit(XPT_CLOSE, &xprt->xpt_flags);
  257. svc_xprt_enqueue(xprt);
  258. break;
  259. default:
  260. break;
  261. }
  262. return 0;
  263. }
  264. /*
  265. * Create a listening RDMA service endpoint.
  266. */
  267. static struct svc_xprt *svc_rdma_create(struct svc_serv *serv,
  268. struct net *net,
  269. struct sockaddr *sa, int salen,
  270. int flags)
  271. {
  272. struct rdma_cm_id *listen_id;
  273. struct svcxprt_rdma *cma_xprt;
  274. int ret;
  275. if (sa->sa_family != AF_INET && sa->sa_family != AF_INET6)
  276. return ERR_PTR(-EAFNOSUPPORT);
  277. cma_xprt = svc_rdma_create_xprt(serv, net);
  278. if (!cma_xprt)
  279. return ERR_PTR(-ENOMEM);
  280. set_bit(XPT_LISTENER, &cma_xprt->sc_xprt.xpt_flags);
  281. strcpy(cma_xprt->sc_xprt.xpt_remotebuf, "listener");
  282. listen_id = rdma_create_id(net, svc_rdma_listen_handler, cma_xprt,
  283. RDMA_PS_TCP, IB_QPT_RC);
  284. if (IS_ERR(listen_id)) {
  285. ret = PTR_ERR(listen_id);
  286. goto err0;
  287. }
  288. /* Allow both IPv4 and IPv6 sockets to bind a single port
  289. * at the same time.
  290. */
  291. #if IS_ENABLED(CONFIG_IPV6)
  292. ret = rdma_set_afonly(listen_id, 1);
  293. if (ret)
  294. goto err1;
  295. #endif
  296. ret = rdma_bind_addr(listen_id, sa);
  297. if (ret)
  298. goto err1;
  299. cma_xprt->sc_cm_id = listen_id;
  300. ret = rdma_listen(listen_id, RPCRDMA_LISTEN_BACKLOG);
  301. if (ret)
  302. goto err1;
  303. /*
  304. * We need to use the address from the cm_id in case the
  305. * caller specified 0 for the port number.
  306. */
  307. sa = (struct sockaddr *)&cma_xprt->sc_cm_id->route.addr.src_addr;
  308. svc_xprt_set_local(&cma_xprt->sc_xprt, sa, salen);
  309. return &cma_xprt->sc_xprt;
  310. err1:
  311. rdma_destroy_id(listen_id);
  312. err0:
  313. kfree(cma_xprt);
  314. return ERR_PTR(ret);
  315. }
  316. /*
  317. * This is the xpo_recvfrom function for listening endpoints. Its
  318. * purpose is to accept incoming connections. The CMA callback handler
  319. * has already created a new transport and attached it to the new CMA
  320. * ID.
  321. *
  322. * There is a queue of pending connections hung on the listening
  323. * transport. This queue contains the new svc_xprt structure. This
  324. * function takes svc_xprt structures off the accept_q and completes
  325. * the connection.
  326. */
  327. static struct svc_xprt *svc_rdma_accept(struct svc_xprt *xprt)
  328. {
  329. struct svcxprt_rdma *listen_rdma;
  330. struct svcxprt_rdma *newxprt = NULL;
  331. struct rdma_conn_param conn_param;
  332. struct rpcrdma_connect_private pmsg;
  333. struct ib_qp_init_attr qp_attr;
  334. unsigned int ctxts, rq_depth;
  335. struct ib_device *dev;
  336. int ret = 0;
  337. RPC_IFDEBUG(struct sockaddr *sap);
  338. listen_rdma = container_of(xprt, struct svcxprt_rdma, sc_xprt);
  339. clear_bit(XPT_CONN, &xprt->xpt_flags);
  340. /* Get the next entry off the accept list */
  341. spin_lock(&listen_rdma->sc_lock);
  342. if (!list_empty(&listen_rdma->sc_accept_q)) {
  343. newxprt = list_entry(listen_rdma->sc_accept_q.next,
  344. struct svcxprt_rdma, sc_accept_q);
  345. list_del_init(&newxprt->sc_accept_q);
  346. }
  347. if (!list_empty(&listen_rdma->sc_accept_q))
  348. set_bit(XPT_CONN, &listen_rdma->sc_xprt.xpt_flags);
  349. spin_unlock(&listen_rdma->sc_lock);
  350. if (!newxprt)
  351. return NULL;
  352. dev = newxprt->sc_cm_id->device;
  353. newxprt->sc_port_num = newxprt->sc_cm_id->port_num;
  354. /* Qualify the transport resource defaults with the
  355. * capabilities of this particular device */
  356. /* Transport header, head iovec, tail iovec */
  357. newxprt->sc_max_send_sges = 3;
  358. /* Add one SGE per page list entry */
  359. newxprt->sc_max_send_sges += (svcrdma_max_req_size / PAGE_SIZE) + 1;
  360. if (newxprt->sc_max_send_sges > dev->attrs.max_send_sge)
  361. newxprt->sc_max_send_sges = dev->attrs.max_send_sge;
  362. newxprt->sc_max_req_size = svcrdma_max_req_size;
  363. newxprt->sc_max_requests = svcrdma_max_requests;
  364. newxprt->sc_max_bc_requests = svcrdma_max_bc_requests;
  365. rq_depth = newxprt->sc_max_requests + newxprt->sc_max_bc_requests;
  366. if (rq_depth > dev->attrs.max_qp_wr) {
  367. pr_warn("svcrdma: reducing receive depth to %d\n",
  368. dev->attrs.max_qp_wr);
  369. rq_depth = dev->attrs.max_qp_wr;
  370. newxprt->sc_max_requests = rq_depth - 2;
  371. newxprt->sc_max_bc_requests = 2;
  372. }
  373. newxprt->sc_fc_credits = cpu_to_be32(newxprt->sc_max_requests);
  374. ctxts = rdma_rw_mr_factor(dev, newxprt->sc_port_num, RPCSVC_MAXPAGES);
  375. ctxts *= newxprt->sc_max_requests;
  376. newxprt->sc_sq_depth = rq_depth + ctxts;
  377. if (newxprt->sc_sq_depth > dev->attrs.max_qp_wr) {
  378. pr_warn("svcrdma: reducing send depth to %d\n",
  379. dev->attrs.max_qp_wr);
  380. newxprt->sc_sq_depth = dev->attrs.max_qp_wr;
  381. }
  382. atomic_set(&newxprt->sc_sq_avail, newxprt->sc_sq_depth);
  383. newxprt->sc_pd = ib_alloc_pd(dev, 0);
  384. if (IS_ERR(newxprt->sc_pd)) {
  385. trace_svcrdma_pd_err(newxprt, PTR_ERR(newxprt->sc_pd));
  386. goto errout;
  387. }
  388. newxprt->sc_sq_cq = ib_alloc_cq_any(dev, newxprt, newxprt->sc_sq_depth,
  389. IB_POLL_WORKQUEUE);
  390. if (IS_ERR(newxprt->sc_sq_cq))
  391. goto errout;
  392. newxprt->sc_rq_cq =
  393. ib_alloc_cq_any(dev, newxprt, rq_depth, IB_POLL_WORKQUEUE);
  394. if (IS_ERR(newxprt->sc_rq_cq))
  395. goto errout;
  396. memset(&qp_attr, 0, sizeof qp_attr);
  397. qp_attr.event_handler = qp_event_handler;
  398. qp_attr.qp_context = &newxprt->sc_xprt;
  399. qp_attr.port_num = newxprt->sc_port_num;
  400. qp_attr.cap.max_rdma_ctxs = ctxts;
  401. qp_attr.cap.max_send_wr = newxprt->sc_sq_depth - ctxts;
  402. qp_attr.cap.max_recv_wr = rq_depth;
  403. qp_attr.cap.max_send_sge = newxprt->sc_max_send_sges;
  404. qp_attr.cap.max_recv_sge = 1;
  405. qp_attr.sq_sig_type = IB_SIGNAL_REQ_WR;
  406. qp_attr.qp_type = IB_QPT_RC;
  407. qp_attr.send_cq = newxprt->sc_sq_cq;
  408. qp_attr.recv_cq = newxprt->sc_rq_cq;
  409. dprintk("svcrdma: newxprt->sc_cm_id=%p, newxprt->sc_pd=%p\n",
  410. newxprt->sc_cm_id, newxprt->sc_pd);
  411. dprintk(" cap.max_send_wr = %d, cap.max_recv_wr = %d\n",
  412. qp_attr.cap.max_send_wr, qp_attr.cap.max_recv_wr);
  413. dprintk(" cap.max_send_sge = %d, cap.max_recv_sge = %d\n",
  414. qp_attr.cap.max_send_sge, qp_attr.cap.max_recv_sge);
  415. ret = rdma_create_qp(newxprt->sc_cm_id, newxprt->sc_pd, &qp_attr);
  416. if (ret) {
  417. trace_svcrdma_qp_err(newxprt, ret);
  418. goto errout;
  419. }
  420. newxprt->sc_qp = newxprt->sc_cm_id->qp;
  421. if (!(dev->attrs.device_cap_flags & IB_DEVICE_MEM_MGT_EXTENSIONS))
  422. newxprt->sc_snd_w_inv = false;
  423. if (!rdma_protocol_iwarp(dev, newxprt->sc_port_num) &&
  424. !rdma_ib_or_roce(dev, newxprt->sc_port_num)) {
  425. trace_svcrdma_fabric_err(newxprt, -EINVAL);
  426. goto errout;
  427. }
  428. if (!svc_rdma_post_recvs(newxprt))
  429. goto errout;
  430. /* Construct RDMA-CM private message */
  431. pmsg.cp_magic = rpcrdma_cmp_magic;
  432. pmsg.cp_version = RPCRDMA_CMP_VERSION;
  433. pmsg.cp_flags = 0;
  434. pmsg.cp_send_size = pmsg.cp_recv_size =
  435. rpcrdma_encode_buffer_size(newxprt->sc_max_req_size);
  436. /* Accept Connection */
  437. set_bit(RDMAXPRT_CONN_PENDING, &newxprt->sc_flags);
  438. memset(&conn_param, 0, sizeof conn_param);
  439. conn_param.responder_resources = 0;
  440. conn_param.initiator_depth = min_t(int, newxprt->sc_ord,
  441. dev->attrs.max_qp_init_rd_atom);
  442. if (!conn_param.initiator_depth) {
  443. ret = -EINVAL;
  444. trace_svcrdma_initdepth_err(newxprt, ret);
  445. goto errout;
  446. }
  447. conn_param.private_data = &pmsg;
  448. conn_param.private_data_len = sizeof(pmsg);
  449. rdma_lock_handler(newxprt->sc_cm_id);
  450. newxprt->sc_cm_id->event_handler = svc_rdma_cma_handler;
  451. ret = rdma_accept(newxprt->sc_cm_id, &conn_param);
  452. rdma_unlock_handler(newxprt->sc_cm_id);
  453. if (ret) {
  454. trace_svcrdma_accept_err(newxprt, ret);
  455. goto errout;
  456. }
  457. #if IS_ENABLED(CONFIG_SUNRPC_DEBUG)
  458. dprintk("svcrdma: new connection %p accepted:\n", newxprt);
  459. sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.src_addr;
  460. dprintk(" local address : %pIS:%u\n", sap, rpc_get_port(sap));
  461. sap = (struct sockaddr *)&newxprt->sc_cm_id->route.addr.dst_addr;
  462. dprintk(" remote address : %pIS:%u\n", sap, rpc_get_port(sap));
  463. dprintk(" max_sge : %d\n", newxprt->sc_max_send_sges);
  464. dprintk(" sq_depth : %d\n", newxprt->sc_sq_depth);
  465. dprintk(" rdma_rw_ctxs : %d\n", ctxts);
  466. dprintk(" max_requests : %d\n", newxprt->sc_max_requests);
  467. dprintk(" ord : %d\n", conn_param.initiator_depth);
  468. #endif
  469. return &newxprt->sc_xprt;
  470. errout:
  471. /* Take a reference in case the DTO handler runs */
  472. svc_xprt_get(&newxprt->sc_xprt);
  473. if (newxprt->sc_qp && !IS_ERR(newxprt->sc_qp))
  474. ib_destroy_qp(newxprt->sc_qp);
  475. rdma_destroy_id(newxprt->sc_cm_id);
  476. /* This call to put will destroy the transport */
  477. svc_xprt_put(&newxprt->sc_xprt);
  478. return NULL;
  479. }
  480. static void svc_rdma_detach(struct svc_xprt *xprt)
  481. {
  482. struct svcxprt_rdma *rdma =
  483. container_of(xprt, struct svcxprt_rdma, sc_xprt);
  484. rdma_disconnect(rdma->sc_cm_id);
  485. }
  486. static void __svc_rdma_free(struct work_struct *work)
  487. {
  488. struct svcxprt_rdma *rdma =
  489. container_of(work, struct svcxprt_rdma, sc_work);
  490. struct svc_xprt *xprt = &rdma->sc_xprt;
  491. /* This blocks until the Completion Queues are empty */
  492. if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
  493. ib_drain_qp(rdma->sc_qp);
  494. svc_rdma_flush_recv_queues(rdma);
  495. /* Final put of backchannel client transport */
  496. if (xprt->xpt_bc_xprt) {
  497. xprt_put(xprt->xpt_bc_xprt);
  498. xprt->xpt_bc_xprt = NULL;
  499. }
  500. svc_rdma_destroy_rw_ctxts(rdma);
  501. svc_rdma_send_ctxts_destroy(rdma);
  502. svc_rdma_recv_ctxts_destroy(rdma);
  503. /* Destroy the QP if present (not a listener) */
  504. if (rdma->sc_qp && !IS_ERR(rdma->sc_qp))
  505. ib_destroy_qp(rdma->sc_qp);
  506. if (rdma->sc_sq_cq && !IS_ERR(rdma->sc_sq_cq))
  507. ib_free_cq(rdma->sc_sq_cq);
  508. if (rdma->sc_rq_cq && !IS_ERR(rdma->sc_rq_cq))
  509. ib_free_cq(rdma->sc_rq_cq);
  510. if (rdma->sc_pd && !IS_ERR(rdma->sc_pd))
  511. ib_dealloc_pd(rdma->sc_pd);
  512. /* Destroy the CM ID */
  513. rdma_destroy_id(rdma->sc_cm_id);
  514. kfree(rdma);
  515. }
  516. static void svc_rdma_free(struct svc_xprt *xprt)
  517. {
  518. struct svcxprt_rdma *rdma =
  519. container_of(xprt, struct svcxprt_rdma, sc_xprt);
  520. INIT_WORK(&rdma->sc_work, __svc_rdma_free);
  521. schedule_work(&rdma->sc_work);
  522. }
  523. static int svc_rdma_has_wspace(struct svc_xprt *xprt)
  524. {
  525. struct svcxprt_rdma *rdma =
  526. container_of(xprt, struct svcxprt_rdma, sc_xprt);
  527. /*
  528. * If there are already waiters on the SQ,
  529. * return false.
  530. */
  531. if (waitqueue_active(&rdma->sc_send_wait))
  532. return 0;
  533. /* Otherwise return true. */
  534. return 1;
  535. }
  536. static void svc_rdma_secure_port(struct svc_rqst *rqstp)
  537. {
  538. set_bit(RQ_SECURE, &rqstp->rq_flags);
  539. }
  540. static void svc_rdma_kill_temp_xprt(struct svc_xprt *xprt)
  541. {
  542. }