smc_tx.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * Shared Memory Communications over RDMA (SMC-R) and RoCE
  4. *
  5. * Manage send buffer.
  6. * Producer:
  7. * Copy user space data into send buffer, if send buffer space available.
  8. * Consumer:
  9. * Trigger RDMA write into RMBE of peer and send CDC, if RMBE space available.
  10. *
  11. * Copyright IBM Corp. 2016
  12. *
  13. * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
  14. */
  15. #include <linux/net.h>
  16. #include <linux/rcupdate.h>
  17. #include <linux/workqueue.h>
  18. #include <linux/sched/signal.h>
  19. #include <net/sock.h>
  20. #include <net/tcp.h>
  21. #include "smc.h"
  22. #include "smc_wr.h"
  23. #include "smc_cdc.h"
  24. #include "smc_close.h"
  25. #include "smc_ism.h"
  26. #include "smc_tx.h"
  27. #define SMC_TX_WORK_DELAY 0
  28. #define SMC_TX_CORK_DELAY (HZ >> 2) /* 250 ms */
  29. /***************************** sndbuf producer *******************************/
  30. /* callback implementation for sk.sk_write_space()
  31. * to wakeup sndbuf producers that blocked with smc_tx_wait().
  32. * called under sk_socket lock.
  33. */
  34. static void smc_tx_write_space(struct sock *sk)
  35. {
  36. struct socket *sock = sk->sk_socket;
  37. struct smc_sock *smc = smc_sk(sk);
  38. struct socket_wq *wq;
  39. /* similar to sk_stream_write_space */
  40. if (atomic_read(&smc->conn.sndbuf_space) && sock) {
  41. clear_bit(SOCK_NOSPACE, &sock->flags);
  42. rcu_read_lock();
  43. wq = rcu_dereference(sk->sk_wq);
  44. if (skwq_has_sleeper(wq))
  45. wake_up_interruptible_poll(&wq->wait,
  46. EPOLLOUT | EPOLLWRNORM |
  47. EPOLLWRBAND);
  48. if (wq && wq->fasync_list && !(sk->sk_shutdown & SEND_SHUTDOWN))
  49. sock_wake_async(wq, SOCK_WAKE_SPACE, POLL_OUT);
  50. rcu_read_unlock();
  51. }
  52. }
  53. /* Wakeup sndbuf producers that blocked with smc_tx_wait().
  54. * Cf. tcp_data_snd_check()=>tcp_check_space()=>tcp_new_space().
  55. */
  56. void smc_tx_sndbuf_nonfull(struct smc_sock *smc)
  57. {
  58. if (smc->sk.sk_socket &&
  59. test_bit(SOCK_NOSPACE, &smc->sk.sk_socket->flags))
  60. smc->sk.sk_write_space(&smc->sk);
  61. }
  62. /* blocks sndbuf producer until at least one byte of free space available
  63. * or urgent Byte was consumed
  64. */
  65. static int smc_tx_wait(struct smc_sock *smc, int flags)
  66. {
  67. DEFINE_WAIT_FUNC(wait, woken_wake_function);
  68. struct smc_connection *conn = &smc->conn;
  69. struct sock *sk = &smc->sk;
  70. long timeo;
  71. int rc = 0;
  72. /* similar to sk_stream_wait_memory */
  73. timeo = sock_sndtimeo(sk, flags & MSG_DONTWAIT);
  74. add_wait_queue(sk_sleep(sk), &wait);
  75. while (1) {
  76. sk_set_bit(SOCKWQ_ASYNC_NOSPACE, sk);
  77. if (sk->sk_err ||
  78. (sk->sk_shutdown & SEND_SHUTDOWN) ||
  79. conn->killed ||
  80. conn->local_tx_ctrl.conn_state_flags.peer_done_writing) {
  81. rc = -EPIPE;
  82. break;
  83. }
  84. if (smc_cdc_rxed_any_close(conn)) {
  85. rc = -ECONNRESET;
  86. break;
  87. }
  88. if (!timeo) {
  89. /* ensure EPOLLOUT is subsequently generated */
  90. set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  91. rc = -EAGAIN;
  92. break;
  93. }
  94. if (signal_pending(current)) {
  95. rc = sock_intr_errno(timeo);
  96. break;
  97. }
  98. sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
  99. if (atomic_read(&conn->sndbuf_space) && !conn->urg_tx_pend)
  100. break; /* at least 1 byte of free & no urgent data */
  101. set_bit(SOCK_NOSPACE, &sk->sk_socket->flags);
  102. sk_wait_event(sk, &timeo,
  103. sk->sk_err ||
  104. (sk->sk_shutdown & SEND_SHUTDOWN) ||
  105. smc_cdc_rxed_any_close(conn) ||
  106. (atomic_read(&conn->sndbuf_space) &&
  107. !conn->urg_tx_pend),
  108. &wait);
  109. }
  110. remove_wait_queue(sk_sleep(sk), &wait);
  111. return rc;
  112. }
  113. static bool smc_tx_is_corked(struct smc_sock *smc)
  114. {
  115. struct tcp_sock *tp = tcp_sk(smc->clcsock->sk);
  116. return (tp->nonagle & TCP_NAGLE_CORK) ? true : false;
  117. }
  118. /* sndbuf producer: main API called by socket layer.
  119. * called under sock lock.
  120. */
  121. int smc_tx_sendmsg(struct smc_sock *smc, struct msghdr *msg, size_t len)
  122. {
  123. size_t copylen, send_done = 0, send_remaining = len;
  124. size_t chunk_len, chunk_off, chunk_len_sum;
  125. struct smc_connection *conn = &smc->conn;
  126. union smc_host_cursor prep;
  127. struct sock *sk = &smc->sk;
  128. char *sndbuf_base;
  129. int tx_cnt_prep;
  130. int writespace;
  131. int rc, chunk;
  132. /* This should be in poll */
  133. sk_clear_bit(SOCKWQ_ASYNC_NOSPACE, sk);
  134. if (sk->sk_err || (sk->sk_shutdown & SEND_SHUTDOWN)) {
  135. rc = -EPIPE;
  136. goto out_err;
  137. }
  138. while (msg_data_left(msg)) {
  139. if (sk->sk_state == SMC_INIT)
  140. return -ENOTCONN;
  141. if (smc->sk.sk_shutdown & SEND_SHUTDOWN ||
  142. (smc->sk.sk_err == ECONNABORTED) ||
  143. conn->killed)
  144. return -EPIPE;
  145. if (smc_cdc_rxed_any_close(conn))
  146. return send_done ?: -ECONNRESET;
  147. if (msg->msg_flags & MSG_OOB)
  148. conn->local_tx_ctrl.prod_flags.urg_data_pending = 1;
  149. if (!atomic_read(&conn->sndbuf_space) || conn->urg_tx_pend) {
  150. if (send_done)
  151. return send_done;
  152. rc = smc_tx_wait(smc, msg->msg_flags);
  153. if (rc)
  154. goto out_err;
  155. continue;
  156. }
  157. /* initialize variables for 1st iteration of subsequent loop */
  158. /* could be just 1 byte, even after smc_tx_wait above */
  159. writespace = atomic_read(&conn->sndbuf_space);
  160. /* not more than what user space asked for */
  161. copylen = min_t(size_t, send_remaining, writespace);
  162. /* determine start of sndbuf */
  163. sndbuf_base = conn->sndbuf_desc->cpu_addr;
  164. smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
  165. tx_cnt_prep = prep.count;
  166. /* determine chunks where to write into sndbuf */
  167. /* either unwrapped case, or 1st chunk of wrapped case */
  168. chunk_len = min_t(size_t, copylen, conn->sndbuf_desc->len -
  169. tx_cnt_prep);
  170. chunk_len_sum = chunk_len;
  171. chunk_off = tx_cnt_prep;
  172. smc_sndbuf_sync_sg_for_cpu(conn);
  173. for (chunk = 0; chunk < 2; chunk++) {
  174. rc = memcpy_from_msg(sndbuf_base + chunk_off,
  175. msg, chunk_len);
  176. if (rc) {
  177. smc_sndbuf_sync_sg_for_device(conn);
  178. if (send_done)
  179. return send_done;
  180. goto out_err;
  181. }
  182. send_done += chunk_len;
  183. send_remaining -= chunk_len;
  184. if (chunk_len_sum == copylen)
  185. break; /* either on 1st or 2nd iteration */
  186. /* prepare next (== 2nd) iteration */
  187. chunk_len = copylen - chunk_len; /* remainder */
  188. chunk_len_sum += chunk_len;
  189. chunk_off = 0; /* modulo offset in send ring buffer */
  190. }
  191. smc_sndbuf_sync_sg_for_device(conn);
  192. /* update cursors */
  193. smc_curs_add(conn->sndbuf_desc->len, &prep, copylen);
  194. smc_curs_copy(&conn->tx_curs_prep, &prep, conn);
  195. /* increased in send tasklet smc_cdc_tx_handler() */
  196. smp_mb__before_atomic();
  197. atomic_sub(copylen, &conn->sndbuf_space);
  198. /* guarantee 0 <= sndbuf_space <= sndbuf_desc->len */
  199. smp_mb__after_atomic();
  200. /* since we just produced more new data into sndbuf,
  201. * trigger sndbuf consumer: RDMA write into peer RMBE and CDC
  202. */
  203. if ((msg->msg_flags & MSG_OOB) && !send_remaining)
  204. conn->urg_tx_pend = true;
  205. if ((msg->msg_flags & MSG_MORE || smc_tx_is_corked(smc)) &&
  206. (atomic_read(&conn->sndbuf_space) >
  207. (conn->sndbuf_desc->len >> 1)))
  208. /* for a corked socket defer the RDMA writes if there
  209. * is still sufficient sndbuf_space available
  210. */
  211. queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work,
  212. SMC_TX_CORK_DELAY);
  213. else
  214. smc_tx_sndbuf_nonempty(conn);
  215. } /* while (msg_data_left(msg)) */
  216. return send_done;
  217. out_err:
  218. rc = sk_stream_error(sk, msg->msg_flags, rc);
  219. /* make sure we wake any epoll edge trigger waiter */
  220. if (unlikely(rc == -EAGAIN))
  221. sk->sk_write_space(sk);
  222. return rc;
  223. }
  224. /***************************** sndbuf consumer *******************************/
  225. /* sndbuf consumer: actual data transfer of one target chunk with ISM write */
  226. int smcd_tx_ism_write(struct smc_connection *conn, void *data, size_t len,
  227. u32 offset, int signal)
  228. {
  229. struct smc_ism_position pos;
  230. int rc;
  231. memset(&pos, 0, sizeof(pos));
  232. pos.token = conn->peer_token;
  233. pos.index = conn->peer_rmbe_idx;
  234. pos.offset = conn->tx_off + offset;
  235. pos.signal = signal;
  236. rc = smc_ism_write(conn->lgr->smcd, &pos, data, len);
  237. if (rc)
  238. conn->local_tx_ctrl.conn_state_flags.peer_conn_abort = 1;
  239. return rc;
  240. }
  241. /* sndbuf consumer: actual data transfer of one target chunk with RDMA write */
  242. static int smc_tx_rdma_write(struct smc_connection *conn, int peer_rmbe_offset,
  243. int num_sges, struct ib_rdma_wr *rdma_wr)
  244. {
  245. struct smc_link_group *lgr = conn->lgr;
  246. struct smc_link *link = conn->lnk;
  247. int rc;
  248. rdma_wr->wr.wr_id = smc_wr_tx_get_next_wr_id(link);
  249. rdma_wr->wr.num_sge = num_sges;
  250. rdma_wr->remote_addr =
  251. lgr->rtokens[conn->rtoken_idx][link->link_idx].dma_addr +
  252. /* RMBE within RMB */
  253. conn->tx_off +
  254. /* offset within RMBE */
  255. peer_rmbe_offset;
  256. rdma_wr->rkey = lgr->rtokens[conn->rtoken_idx][link->link_idx].rkey;
  257. rc = ib_post_send(link->roce_qp, &rdma_wr->wr, NULL);
  258. if (rc)
  259. smcr_link_down_cond_sched(link);
  260. return rc;
  261. }
  262. /* sndbuf consumer */
  263. static inline void smc_tx_advance_cursors(struct smc_connection *conn,
  264. union smc_host_cursor *prod,
  265. union smc_host_cursor *sent,
  266. size_t len)
  267. {
  268. smc_curs_add(conn->peer_rmbe_size, prod, len);
  269. /* increased in recv tasklet smc_cdc_msg_rcv() */
  270. smp_mb__before_atomic();
  271. /* data in flight reduces usable snd_wnd */
  272. atomic_sub(len, &conn->peer_rmbe_space);
  273. /* guarantee 0 <= peer_rmbe_space <= peer_rmbe_size */
  274. smp_mb__after_atomic();
  275. smc_curs_add(conn->sndbuf_desc->len, sent, len);
  276. }
  277. /* SMC-R helper for smc_tx_rdma_writes() */
  278. static int smcr_tx_rdma_writes(struct smc_connection *conn, size_t len,
  279. size_t src_off, size_t src_len,
  280. size_t dst_off, size_t dst_len,
  281. struct smc_rdma_wr *wr_rdma_buf)
  282. {
  283. struct smc_link *link = conn->lnk;
  284. dma_addr_t dma_addr =
  285. sg_dma_address(conn->sndbuf_desc->sgt[link->link_idx].sgl);
  286. int src_len_sum = src_len, dst_len_sum = dst_len;
  287. int sent_count = src_off;
  288. int srcchunk, dstchunk;
  289. int num_sges;
  290. int rc;
  291. for (dstchunk = 0; dstchunk < 2; dstchunk++) {
  292. struct ib_sge *sge =
  293. wr_rdma_buf->wr_tx_rdma[dstchunk].wr.sg_list;
  294. num_sges = 0;
  295. for (srcchunk = 0; srcchunk < 2; srcchunk++) {
  296. sge[srcchunk].addr = dma_addr + src_off;
  297. sge[srcchunk].length = src_len;
  298. num_sges++;
  299. src_off += src_len;
  300. if (src_off >= conn->sndbuf_desc->len)
  301. src_off -= conn->sndbuf_desc->len;
  302. /* modulo in send ring */
  303. if (src_len_sum == dst_len)
  304. break; /* either on 1st or 2nd iteration */
  305. /* prepare next (== 2nd) iteration */
  306. src_len = dst_len - src_len; /* remainder */
  307. src_len_sum += src_len;
  308. }
  309. rc = smc_tx_rdma_write(conn, dst_off, num_sges,
  310. &wr_rdma_buf->wr_tx_rdma[dstchunk]);
  311. if (rc)
  312. return rc;
  313. if (dst_len_sum == len)
  314. break; /* either on 1st or 2nd iteration */
  315. /* prepare next (== 2nd) iteration */
  316. dst_off = 0; /* modulo offset in RMBE ring buffer */
  317. dst_len = len - dst_len; /* remainder */
  318. dst_len_sum += dst_len;
  319. src_len = min_t(int, dst_len, conn->sndbuf_desc->len -
  320. sent_count);
  321. src_len_sum = src_len;
  322. }
  323. return 0;
  324. }
  325. /* SMC-D helper for smc_tx_rdma_writes() */
  326. static int smcd_tx_rdma_writes(struct smc_connection *conn, size_t len,
  327. size_t src_off, size_t src_len,
  328. size_t dst_off, size_t dst_len)
  329. {
  330. int src_len_sum = src_len, dst_len_sum = dst_len;
  331. int srcchunk, dstchunk;
  332. int rc;
  333. for (dstchunk = 0; dstchunk < 2; dstchunk++) {
  334. for (srcchunk = 0; srcchunk < 2; srcchunk++) {
  335. void *data = conn->sndbuf_desc->cpu_addr + src_off;
  336. rc = smcd_tx_ism_write(conn, data, src_len, dst_off +
  337. sizeof(struct smcd_cdc_msg), 0);
  338. if (rc)
  339. return rc;
  340. dst_off += src_len;
  341. src_off += src_len;
  342. if (src_off >= conn->sndbuf_desc->len)
  343. src_off -= conn->sndbuf_desc->len;
  344. /* modulo in send ring */
  345. if (src_len_sum == dst_len)
  346. break; /* either on 1st or 2nd iteration */
  347. /* prepare next (== 2nd) iteration */
  348. src_len = dst_len - src_len; /* remainder */
  349. src_len_sum += src_len;
  350. }
  351. if (dst_len_sum == len)
  352. break; /* either on 1st or 2nd iteration */
  353. /* prepare next (== 2nd) iteration */
  354. dst_off = 0; /* modulo offset in RMBE ring buffer */
  355. dst_len = len - dst_len; /* remainder */
  356. dst_len_sum += dst_len;
  357. src_len = min_t(int, dst_len, conn->sndbuf_desc->len - src_off);
  358. src_len_sum = src_len;
  359. }
  360. return 0;
  361. }
  362. /* sndbuf consumer: prepare all necessary (src&dst) chunks of data transmit;
  363. * usable snd_wnd as max transmit
  364. */
  365. static int smc_tx_rdma_writes(struct smc_connection *conn,
  366. struct smc_rdma_wr *wr_rdma_buf)
  367. {
  368. size_t len, src_len, dst_off, dst_len; /* current chunk values */
  369. union smc_host_cursor sent, prep, prod, cons;
  370. struct smc_cdc_producer_flags *pflags;
  371. int to_send, rmbespace;
  372. int rc;
  373. /* source: sndbuf */
  374. smc_curs_copy(&sent, &conn->tx_curs_sent, conn);
  375. smc_curs_copy(&prep, &conn->tx_curs_prep, conn);
  376. /* cf. wmem_alloc - (snd_max - snd_una) */
  377. to_send = smc_curs_diff(conn->sndbuf_desc->len, &sent, &prep);
  378. if (to_send <= 0)
  379. return 0;
  380. /* destination: RMBE */
  381. /* cf. snd_wnd */
  382. rmbespace = atomic_read(&conn->peer_rmbe_space);
  383. if (rmbespace <= 0)
  384. return 0;
  385. smc_curs_copy(&prod, &conn->local_tx_ctrl.prod, conn);
  386. smc_curs_copy(&cons, &conn->local_rx_ctrl.cons, conn);
  387. /* if usable snd_wnd closes ask peer to advertise once it opens again */
  388. pflags = &conn->local_tx_ctrl.prod_flags;
  389. pflags->write_blocked = (to_send >= rmbespace);
  390. /* cf. usable snd_wnd */
  391. len = min(to_send, rmbespace);
  392. /* initialize variables for first iteration of subsequent nested loop */
  393. dst_off = prod.count;
  394. if (prod.wrap == cons.wrap) {
  395. /* the filled destination area is unwrapped,
  396. * hence the available free destination space is wrapped
  397. * and we need 2 destination chunks of sum len; start with 1st
  398. * which is limited by what's available in sndbuf
  399. */
  400. dst_len = min_t(size_t,
  401. conn->peer_rmbe_size - prod.count, len);
  402. } else {
  403. /* the filled destination area is wrapped,
  404. * hence the available free destination space is unwrapped
  405. * and we need a single destination chunk of entire len
  406. */
  407. dst_len = len;
  408. }
  409. /* dst_len determines the maximum src_len */
  410. if (sent.count + dst_len <= conn->sndbuf_desc->len) {
  411. /* unwrapped src case: single chunk of entire dst_len */
  412. src_len = dst_len;
  413. } else {
  414. /* wrapped src case: 2 chunks of sum dst_len; start with 1st: */
  415. src_len = conn->sndbuf_desc->len - sent.count;
  416. }
  417. if (conn->lgr->is_smcd)
  418. rc = smcd_tx_rdma_writes(conn, len, sent.count, src_len,
  419. dst_off, dst_len);
  420. else
  421. rc = smcr_tx_rdma_writes(conn, len, sent.count, src_len,
  422. dst_off, dst_len, wr_rdma_buf);
  423. if (rc)
  424. return rc;
  425. if (conn->urg_tx_pend && len == to_send)
  426. pflags->urg_data_present = 1;
  427. smc_tx_advance_cursors(conn, &prod, &sent, len);
  428. /* update connection's cursors with advanced local cursors */
  429. smc_curs_copy(&conn->local_tx_ctrl.prod, &prod, conn);
  430. /* dst: peer RMBE */
  431. smc_curs_copy(&conn->tx_curs_sent, &sent, conn);/* src: local sndbuf */
  432. return 0;
  433. }
  434. /* Wakeup sndbuf consumers from any context (IRQ or process)
  435. * since there is more data to transmit; usable snd_wnd as max transmit
  436. */
  437. static int smcr_tx_sndbuf_nonempty(struct smc_connection *conn)
  438. {
  439. struct smc_cdc_producer_flags *pflags = &conn->local_tx_ctrl.prod_flags;
  440. struct smc_link *link = conn->lnk;
  441. struct smc_rdma_wr *wr_rdma_buf;
  442. struct smc_cdc_tx_pend *pend;
  443. struct smc_wr_buf *wr_buf;
  444. int rc;
  445. if (!link || !smc_wr_tx_link_hold(link))
  446. return -ENOLINK;
  447. rc = smc_cdc_get_free_slot(conn, link, &wr_buf, &wr_rdma_buf, &pend);
  448. if (rc < 0) {
  449. smc_wr_tx_link_put(link);
  450. if (rc == -EBUSY) {
  451. struct smc_sock *smc =
  452. container_of(conn, struct smc_sock, conn);
  453. if (smc->sk.sk_err == ECONNABORTED)
  454. return sock_error(&smc->sk);
  455. if (conn->killed)
  456. return -EPIPE;
  457. rc = 0;
  458. mod_delayed_work(conn->lgr->tx_wq, &conn->tx_work,
  459. SMC_TX_WORK_DELAY);
  460. }
  461. return rc;
  462. }
  463. spin_lock_bh(&conn->send_lock);
  464. if (link != conn->lnk) {
  465. /* link of connection changed, tx_work will restart */
  466. smc_wr_tx_put_slot(link,
  467. (struct smc_wr_tx_pend_priv *)pend);
  468. rc = -ENOLINK;
  469. goto out_unlock;
  470. }
  471. if (!pflags->urg_data_present) {
  472. rc = smc_tx_rdma_writes(conn, wr_rdma_buf);
  473. if (rc) {
  474. smc_wr_tx_put_slot(link,
  475. (struct smc_wr_tx_pend_priv *)pend);
  476. goto out_unlock;
  477. }
  478. }
  479. rc = smc_cdc_msg_send(conn, wr_buf, pend);
  480. if (!rc && pflags->urg_data_present) {
  481. pflags->urg_data_pending = 0;
  482. pflags->urg_data_present = 0;
  483. }
  484. out_unlock:
  485. spin_unlock_bh(&conn->send_lock);
  486. smc_wr_tx_link_put(link);
  487. return rc;
  488. }
  489. static int smcd_tx_sndbuf_nonempty(struct smc_connection *conn)
  490. {
  491. struct smc_cdc_producer_flags *pflags = &conn->local_tx_ctrl.prod_flags;
  492. int rc = 0;
  493. spin_lock_bh(&conn->send_lock);
  494. if (!pflags->urg_data_present)
  495. rc = smc_tx_rdma_writes(conn, NULL);
  496. if (!rc)
  497. rc = smcd_cdc_msg_send(conn);
  498. if (!rc && pflags->urg_data_present) {
  499. pflags->urg_data_pending = 0;
  500. pflags->urg_data_present = 0;
  501. }
  502. spin_unlock_bh(&conn->send_lock);
  503. return rc;
  504. }
  505. int smc_tx_sndbuf_nonempty(struct smc_connection *conn)
  506. {
  507. int rc;
  508. if (conn->killed ||
  509. conn->local_rx_ctrl.conn_state_flags.peer_conn_abort)
  510. return -EPIPE; /* connection being aborted */
  511. if (conn->lgr->is_smcd)
  512. rc = smcd_tx_sndbuf_nonempty(conn);
  513. else
  514. rc = smcr_tx_sndbuf_nonempty(conn);
  515. if (!rc) {
  516. /* trigger socket release if connection is closing */
  517. struct smc_sock *smc = container_of(conn, struct smc_sock,
  518. conn);
  519. smc_close_wake_tx_prepared(smc);
  520. }
  521. return rc;
  522. }
  523. /* Wakeup sndbuf consumers from process context
  524. * since there is more data to transmit
  525. */
  526. void smc_tx_work(struct work_struct *work)
  527. {
  528. struct smc_connection *conn = container_of(to_delayed_work(work),
  529. struct smc_connection,
  530. tx_work);
  531. struct smc_sock *smc = container_of(conn, struct smc_sock, conn);
  532. int rc;
  533. lock_sock(&smc->sk);
  534. if (smc->sk.sk_err)
  535. goto out;
  536. rc = smc_tx_sndbuf_nonempty(conn);
  537. if (!rc && conn->local_rx_ctrl.prod_flags.write_blocked &&
  538. !atomic_read(&conn->bytes_to_rcv))
  539. conn->local_rx_ctrl.prod_flags.write_blocked = 0;
  540. out:
  541. release_sock(&smc->sk);
  542. }
  543. void smc_tx_consumer_update(struct smc_connection *conn, bool force)
  544. {
  545. union smc_host_cursor cfed, cons, prod;
  546. int sender_free = conn->rmb_desc->len;
  547. int to_confirm;
  548. smc_curs_copy(&cons, &conn->local_tx_ctrl.cons, conn);
  549. smc_curs_copy(&cfed, &conn->rx_curs_confirmed, conn);
  550. to_confirm = smc_curs_diff(conn->rmb_desc->len, &cfed, &cons);
  551. if (to_confirm > conn->rmbe_update_limit) {
  552. smc_curs_copy(&prod, &conn->local_rx_ctrl.prod, conn);
  553. sender_free = conn->rmb_desc->len -
  554. smc_curs_diff_large(conn->rmb_desc->len,
  555. &cfed, &prod);
  556. }
  557. if (conn->local_rx_ctrl.prod_flags.cons_curs_upd_req ||
  558. force ||
  559. ((to_confirm > conn->rmbe_update_limit) &&
  560. ((sender_free <= (conn->rmb_desc->len / 2)) ||
  561. conn->local_rx_ctrl.prod_flags.write_blocked))) {
  562. if (conn->killed ||
  563. conn->local_rx_ctrl.conn_state_flags.peer_conn_abort)
  564. return;
  565. if ((smc_cdc_get_slot_and_msg_send(conn) < 0) &&
  566. !conn->killed) {
  567. queue_delayed_work(conn->lgr->tx_wq, &conn->tx_work,
  568. SMC_TX_WORK_DELAY);
  569. return;
  570. }
  571. }
  572. if (conn->local_rx_ctrl.prod_flags.write_blocked &&
  573. !atomic_read(&conn->bytes_to_rcv))
  574. conn->local_rx_ctrl.prod_flags.write_blocked = 0;
  575. }
  576. /***************************** send initialize *******************************/
  577. /* Initialize send properties on connection establishment. NB: not __init! */
  578. void smc_tx_init(struct smc_sock *smc)
  579. {
  580. smc->sk.sk_write_space = smc_tx_write_space;
  581. }