smc_cdc.h 8.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. /*
  3. * Shared Memory Communications over RDMA (SMC-R) and RoCE
  4. *
  5. * Connection Data Control (CDC)
  6. *
  7. * Copyright IBM Corp. 2016
  8. *
  9. * Author(s): Ursula Braun <ubraun@linux.vnet.ibm.com>
  10. */
  11. #ifndef SMC_CDC_H
  12. #define SMC_CDC_H
  13. #include <linux/kernel.h> /* max_t */
  14. #include <linux/atomic.h>
  15. #include <linux/in.h>
  16. #include <linux/compiler.h>
  17. #include "smc.h"
  18. #include "smc_core.h"
  19. #include "smc_wr.h"
  20. #define SMC_CDC_MSG_TYPE 0xFE
  21. /* in network byte order */
  22. union smc_cdc_cursor { /* SMC cursor */
  23. struct {
  24. __be16 reserved;
  25. __be16 wrap;
  26. __be32 count;
  27. };
  28. #ifdef KERNEL_HAS_ATOMIC64
  29. atomic64_t acurs; /* for atomic processing */
  30. #else
  31. u64 acurs; /* for atomic processing */
  32. #endif
  33. } __aligned(8);
  34. /* in network byte order */
  35. struct smc_cdc_msg {
  36. struct smc_wr_rx_hdr common; /* .type = 0xFE */
  37. u8 len; /* 44 */
  38. __be16 seqno;
  39. __be32 token;
  40. union smc_cdc_cursor prod;
  41. union smc_cdc_cursor cons; /* piggy backed "ack" */
  42. struct smc_cdc_producer_flags prod_flags;
  43. struct smc_cdc_conn_state_flags conn_state_flags;
  44. u8 reserved[18];
  45. };
  46. /* SMC-D cursor format */
  47. union smcd_cdc_cursor {
  48. struct {
  49. u16 wrap;
  50. u32 count;
  51. struct smc_cdc_producer_flags prod_flags;
  52. struct smc_cdc_conn_state_flags conn_state_flags;
  53. } __packed;
  54. #ifdef KERNEL_HAS_ATOMIC64
  55. atomic64_t acurs; /* for atomic processing */
  56. #else
  57. u64 acurs; /* for atomic processing */
  58. #endif
  59. } __aligned(8);
  60. /* CDC message for SMC-D */
  61. struct smcd_cdc_msg {
  62. struct smc_wr_rx_hdr common; /* Type = 0xFE */
  63. u8 res1[7];
  64. union smcd_cdc_cursor prod;
  65. union smcd_cdc_cursor cons;
  66. u8 res3[8];
  67. } __aligned(8);
  68. static inline bool smc_cdc_rxed_any_close(struct smc_connection *conn)
  69. {
  70. return conn->local_rx_ctrl.conn_state_flags.peer_conn_abort ||
  71. conn->local_rx_ctrl.conn_state_flags.peer_conn_closed;
  72. }
  73. static inline bool smc_cdc_rxed_any_close_or_senddone(
  74. struct smc_connection *conn)
  75. {
  76. return smc_cdc_rxed_any_close(conn) ||
  77. conn->local_rx_ctrl.conn_state_flags.peer_done_writing;
  78. }
  79. static inline void smc_curs_add(int size, union smc_host_cursor *curs,
  80. int value)
  81. {
  82. curs->count += value;
  83. if (curs->count >= size) {
  84. curs->wrap++;
  85. curs->count -= size;
  86. }
  87. }
  88. /* Copy cursor src into tgt */
  89. static inline void smc_curs_copy(union smc_host_cursor *tgt,
  90. union smc_host_cursor *src,
  91. struct smc_connection *conn)
  92. {
  93. #ifndef KERNEL_HAS_ATOMIC64
  94. unsigned long flags;
  95. spin_lock_irqsave(&conn->acurs_lock, flags);
  96. tgt->acurs = src->acurs;
  97. spin_unlock_irqrestore(&conn->acurs_lock, flags);
  98. #else
  99. atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
  100. #endif
  101. }
  102. static inline void smc_curs_copy_net(union smc_cdc_cursor *tgt,
  103. union smc_cdc_cursor *src,
  104. struct smc_connection *conn)
  105. {
  106. #ifndef KERNEL_HAS_ATOMIC64
  107. unsigned long flags;
  108. spin_lock_irqsave(&conn->acurs_lock, flags);
  109. tgt->acurs = src->acurs;
  110. spin_unlock_irqrestore(&conn->acurs_lock, flags);
  111. #else
  112. atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
  113. #endif
  114. }
  115. static inline void smcd_curs_copy(union smcd_cdc_cursor *tgt,
  116. union smcd_cdc_cursor *src,
  117. struct smc_connection *conn)
  118. {
  119. #ifndef KERNEL_HAS_ATOMIC64
  120. unsigned long flags;
  121. spin_lock_irqsave(&conn->acurs_lock, flags);
  122. tgt->acurs = src->acurs;
  123. spin_unlock_irqrestore(&conn->acurs_lock, flags);
  124. #else
  125. atomic64_set(&tgt->acurs, atomic64_read(&src->acurs));
  126. #endif
  127. }
  128. /* calculate cursor difference between old and new, where old <= new and
  129. * difference cannot exceed size
  130. */
  131. static inline int smc_curs_diff(unsigned int size,
  132. union smc_host_cursor *old,
  133. union smc_host_cursor *new)
  134. {
  135. if (old->wrap != new->wrap)
  136. return max_t(int, 0,
  137. ((size - old->count) + new->count));
  138. return max_t(int, 0, (new->count - old->count));
  139. }
  140. /* calculate cursor difference between old and new - returns negative
  141. * value in case old > new
  142. */
  143. static inline int smc_curs_comp(unsigned int size,
  144. union smc_host_cursor *old,
  145. union smc_host_cursor *new)
  146. {
  147. if (old->wrap > new->wrap ||
  148. (old->wrap == new->wrap && old->count > new->count))
  149. return -smc_curs_diff(size, new, old);
  150. return smc_curs_diff(size, old, new);
  151. }
  152. /* calculate cursor difference between old and new, where old <= new and
  153. * difference may exceed size
  154. */
  155. static inline int smc_curs_diff_large(unsigned int size,
  156. union smc_host_cursor *old,
  157. union smc_host_cursor *new)
  158. {
  159. if (old->wrap < new->wrap)
  160. return min_t(int,
  161. (size - old->count) + new->count +
  162. (new->wrap - old->wrap - 1) * size,
  163. size);
  164. if (old->wrap > new->wrap) /* wrap has switched from 0xffff to 0x0000 */
  165. return min_t(int,
  166. (size - old->count) + new->count +
  167. (new->wrap + 0xffff - old->wrap) * size,
  168. size);
  169. return max_t(int, 0, (new->count - old->count));
  170. }
  171. static inline void smc_host_cursor_to_cdc(union smc_cdc_cursor *peer,
  172. union smc_host_cursor *local,
  173. union smc_host_cursor *save,
  174. struct smc_connection *conn)
  175. {
  176. smc_curs_copy(save, local, conn);
  177. peer->count = htonl(save->count);
  178. peer->wrap = htons(save->wrap);
  179. /* peer->reserved = htons(0); must be ensured by caller */
  180. }
  181. static inline void smc_host_msg_to_cdc(struct smc_cdc_msg *peer,
  182. struct smc_connection *conn,
  183. union smc_host_cursor *save)
  184. {
  185. struct smc_host_cdc_msg *local = &conn->local_tx_ctrl;
  186. peer->common.type = local->common.type;
  187. peer->len = local->len;
  188. peer->seqno = htons(local->seqno);
  189. peer->token = htonl(local->token);
  190. smc_host_cursor_to_cdc(&peer->prod, &local->prod, save, conn);
  191. smc_host_cursor_to_cdc(&peer->cons, &local->cons, save, conn);
  192. peer->prod_flags = local->prod_flags;
  193. peer->conn_state_flags = local->conn_state_flags;
  194. }
  195. static inline void smc_cdc_cursor_to_host(union smc_host_cursor *local,
  196. union smc_cdc_cursor *peer,
  197. struct smc_connection *conn)
  198. {
  199. union smc_host_cursor temp, old;
  200. union smc_cdc_cursor net;
  201. smc_curs_copy(&old, local, conn);
  202. smc_curs_copy_net(&net, peer, conn);
  203. temp.count = ntohl(net.count);
  204. temp.wrap = ntohs(net.wrap);
  205. if ((old.wrap > temp.wrap) && temp.wrap)
  206. return;
  207. if ((old.wrap == temp.wrap) &&
  208. (old.count > temp.count))
  209. return;
  210. smc_curs_copy(local, &temp, conn);
  211. }
  212. static inline void smcr_cdc_msg_to_host(struct smc_host_cdc_msg *local,
  213. struct smc_cdc_msg *peer,
  214. struct smc_connection *conn)
  215. {
  216. local->common.type = peer->common.type;
  217. local->len = peer->len;
  218. local->seqno = ntohs(peer->seqno);
  219. local->token = ntohl(peer->token);
  220. smc_cdc_cursor_to_host(&local->prod, &peer->prod, conn);
  221. smc_cdc_cursor_to_host(&local->cons, &peer->cons, conn);
  222. local->prod_flags = peer->prod_flags;
  223. local->conn_state_flags = peer->conn_state_flags;
  224. }
  225. static inline void smcd_cdc_msg_to_host(struct smc_host_cdc_msg *local,
  226. struct smcd_cdc_msg *peer,
  227. struct smc_connection *conn)
  228. {
  229. union smc_host_cursor temp;
  230. temp.wrap = peer->prod.wrap;
  231. temp.count = peer->prod.count;
  232. smc_curs_copy(&local->prod, &temp, conn);
  233. temp.wrap = peer->cons.wrap;
  234. temp.count = peer->cons.count;
  235. smc_curs_copy(&local->cons, &temp, conn);
  236. local->prod_flags = peer->cons.prod_flags;
  237. local->conn_state_flags = peer->cons.conn_state_flags;
  238. }
  239. static inline void smc_cdc_msg_to_host(struct smc_host_cdc_msg *local,
  240. struct smc_cdc_msg *peer,
  241. struct smc_connection *conn)
  242. {
  243. if (conn->lgr->is_smcd)
  244. smcd_cdc_msg_to_host(local, (struct smcd_cdc_msg *)peer, conn);
  245. else
  246. smcr_cdc_msg_to_host(local, peer, conn);
  247. }
  248. struct smc_cdc_tx_pend {
  249. struct smc_connection *conn; /* socket connection */
  250. union smc_host_cursor cursor; /* tx sndbuf cursor sent */
  251. union smc_host_cursor p_cursor; /* rx RMBE cursor produced */
  252. u16 ctrl_seq; /* conn. tx sequence # */
  253. };
  254. int smc_cdc_get_free_slot(struct smc_connection *conn,
  255. struct smc_link *link,
  256. struct smc_wr_buf **wr_buf,
  257. struct smc_rdma_wr **wr_rdma_buf,
  258. struct smc_cdc_tx_pend **pend);
  259. void smc_cdc_wait_pend_tx_wr(struct smc_connection *conn);
  260. int smc_cdc_msg_send(struct smc_connection *conn, struct smc_wr_buf *wr_buf,
  261. struct smc_cdc_tx_pend *pend);
  262. int smc_cdc_get_slot_and_msg_send(struct smc_connection *conn);
  263. int smcd_cdc_msg_send(struct smc_connection *conn);
  264. int smcr_cdc_msg_send_validation(struct smc_connection *conn,
  265. struct smc_cdc_tx_pend *pend,
  266. struct smc_wr_buf *wr_buf);
  267. int smc_cdc_init(void) __init;
  268. void smcd_cdc_rx_init(struct smc_connection *conn);
  269. #endif /* SMC_CDC_H */