rds.h 31 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019
  1. /* SPDX-License-Identifier: GPL-2.0 */
  2. #ifndef _RDS_RDS_H
  3. #define _RDS_RDS_H
  4. #include <net/sock.h>
  5. #include <linux/scatterlist.h>
  6. #include <linux/highmem.h>
  7. #include <rdma/rdma_cm.h>
  8. #include <linux/mutex.h>
  9. #include <linux/rds.h>
  10. #include <linux/rhashtable.h>
  11. #include <linux/refcount.h>
  12. #include <linux/in6.h>
  13. #include "info.h"
  14. /*
  15. * RDS Network protocol version
  16. */
  17. #define RDS_PROTOCOL_3_0 0x0300
  18. #define RDS_PROTOCOL_3_1 0x0301
  19. #define RDS_PROTOCOL_4_0 0x0400
  20. #define RDS_PROTOCOL_4_1 0x0401
  21. #define RDS_PROTOCOL_VERSION RDS_PROTOCOL_3_1
  22. #define RDS_PROTOCOL_MAJOR(v) ((v) >> 8)
  23. #define RDS_PROTOCOL_MINOR(v) ((v) & 255)
  24. #define RDS_PROTOCOL(maj, min) (((maj) << 8) | min)
  25. #define RDS_PROTOCOL_COMPAT_VERSION RDS_PROTOCOL_3_1
  26. /* The following ports, 16385, 18634, 18635, are registered with IANA as
  27. * the ports to be used for RDS over TCP and UDP. Currently, only RDS over
  28. * TCP and RDS over IB/RDMA are implemented. 18634 is the historical value
  29. * used for the RDMA_CM listener port. RDS/TCP uses port 16385. After
  30. * IPv6 work, RDMA_CM also uses 16385 as the listener port. 18634 is kept
  31. * to ensure compatibility with older RDS modules. Those ports are defined
  32. * in each transport's header file.
  33. */
  34. #define RDS_PORT 18634
  35. #ifdef ATOMIC64_INIT
  36. #define KERNEL_HAS_ATOMIC64
  37. #endif
  38. #ifdef RDS_DEBUG
  39. #define rdsdebug(fmt, args...) pr_debug("%s(): " fmt, __func__ , ##args)
  40. #else
  41. /* sigh, pr_debug() causes unused variable warnings */
  42. static inline __printf(1, 2)
  43. void rdsdebug(char *fmt, ...)
  44. {
  45. }
  46. #endif
  47. #define RDS_FRAG_SHIFT 12
  48. #define RDS_FRAG_SIZE ((unsigned int)(1 << RDS_FRAG_SHIFT))
  49. /* Used to limit both RDMA and non-RDMA RDS message to 1MB */
  50. #define RDS_MAX_MSG_SIZE ((unsigned int)(1 << 20))
  51. #define RDS_CONG_MAP_BYTES (65536 / 8)
  52. #define RDS_CONG_MAP_PAGES (PAGE_ALIGN(RDS_CONG_MAP_BYTES) / PAGE_SIZE)
  53. #define RDS_CONG_MAP_PAGE_BITS (PAGE_SIZE * 8)
  54. struct rds_cong_map {
  55. struct rb_node m_rb_node;
  56. struct in6_addr m_addr;
  57. wait_queue_head_t m_waitq;
  58. struct list_head m_conn_list;
  59. unsigned long m_page_addrs[RDS_CONG_MAP_PAGES];
  60. };
  61. /*
  62. * This is how we will track the connection state:
  63. * A connection is always in one of the following
  64. * states. Updates to the state are atomic and imply
  65. * a memory barrier.
  66. */
  67. enum {
  68. RDS_CONN_DOWN = 0,
  69. RDS_CONN_CONNECTING,
  70. RDS_CONN_DISCONNECTING,
  71. RDS_CONN_UP,
  72. RDS_CONN_RESETTING,
  73. RDS_CONN_ERROR,
  74. };
  75. /* Bits for c_flags */
  76. #define RDS_LL_SEND_FULL 0
  77. #define RDS_RECONNECT_PENDING 1
  78. #define RDS_IN_XMIT 2
  79. #define RDS_RECV_REFILL 3
  80. #define RDS_DESTROY_PENDING 4
  81. /* Max number of multipaths per RDS connection. Must be a power of 2 */
  82. #define RDS_MPATH_WORKERS 8
  83. #define RDS_MPATH_HASH(rs, n) (jhash_1word((rs)->rs_bound_port, \
  84. (rs)->rs_hash_initval) & ((n) - 1))
  85. #define IS_CANONICAL(laddr, faddr) (htonl(laddr) < htonl(faddr))
  86. /* Per mpath connection state */
  87. struct rds_conn_path {
  88. struct rds_connection *cp_conn;
  89. struct rds_message *cp_xmit_rm;
  90. unsigned long cp_xmit_sg;
  91. unsigned int cp_xmit_hdr_off;
  92. unsigned int cp_xmit_data_off;
  93. unsigned int cp_xmit_atomic_sent;
  94. unsigned int cp_xmit_rdma_sent;
  95. unsigned int cp_xmit_data_sent;
  96. spinlock_t cp_lock; /* protect msg queues */
  97. u64 cp_next_tx_seq;
  98. struct list_head cp_send_queue;
  99. struct list_head cp_retrans;
  100. u64 cp_next_rx_seq;
  101. void *cp_transport_data;
  102. atomic_t cp_state;
  103. unsigned long cp_send_gen;
  104. unsigned long cp_flags;
  105. unsigned long cp_reconnect_jiffies;
  106. struct delayed_work cp_send_w;
  107. struct delayed_work cp_recv_w;
  108. struct delayed_work cp_conn_w;
  109. struct work_struct cp_down_w;
  110. struct mutex cp_cm_lock; /* protect cp_state & cm */
  111. wait_queue_head_t cp_waitq;
  112. unsigned int cp_unacked_packets;
  113. unsigned int cp_unacked_bytes;
  114. unsigned int cp_index;
  115. };
  116. /* One rds_connection per RDS address pair */
  117. struct rds_connection {
  118. struct hlist_node c_hash_node;
  119. struct in6_addr c_laddr;
  120. struct in6_addr c_faddr;
  121. int c_dev_if; /* ifindex used for this conn */
  122. int c_bound_if; /* ifindex of c_laddr */
  123. unsigned int c_loopback:1,
  124. c_isv6:1,
  125. c_ping_triggered:1,
  126. c_pad_to_32:29;
  127. int c_npaths;
  128. struct rds_connection *c_passive;
  129. struct rds_transport *c_trans;
  130. struct rds_cong_map *c_lcong;
  131. struct rds_cong_map *c_fcong;
  132. /* Protocol version */
  133. unsigned int c_proposed_version;
  134. unsigned int c_version;
  135. possible_net_t c_net;
  136. /* TOS */
  137. u8 c_tos;
  138. struct list_head c_map_item;
  139. unsigned long c_map_queued;
  140. struct rds_conn_path *c_path;
  141. wait_queue_head_t c_hs_waitq; /* handshake waitq */
  142. u32 c_my_gen_num;
  143. u32 c_peer_gen_num;
  144. };
  145. static inline
  146. struct net *rds_conn_net(struct rds_connection *conn)
  147. {
  148. return read_pnet(&conn->c_net);
  149. }
  150. static inline
  151. void rds_conn_net_set(struct rds_connection *conn, struct net *net)
  152. {
  153. write_pnet(&conn->c_net, net);
  154. }
  155. #define RDS_FLAG_CONG_BITMAP 0x01
  156. #define RDS_FLAG_ACK_REQUIRED 0x02
  157. #define RDS_FLAG_RETRANSMITTED 0x04
  158. #define RDS_MAX_ADV_CREDIT 255
  159. /* RDS_FLAG_PROBE_PORT is the reserved sport used for sending a ping
  160. * probe to exchange control information before establishing a connection.
  161. * Currently the control information that is exchanged is the number of
  162. * supported paths. If the peer is a legacy (older kernel revision) peer,
  163. * it would return a pong message without additional control information
  164. * that would then alert the sender that the peer was an older rev.
  165. */
  166. #define RDS_FLAG_PROBE_PORT 1
  167. #define RDS_HS_PROBE(sport, dport) \
  168. ((sport == RDS_FLAG_PROBE_PORT && dport == 0) || \
  169. (sport == 0 && dport == RDS_FLAG_PROBE_PORT))
  170. /*
  171. * Maximum space available for extension headers.
  172. */
  173. #define RDS_HEADER_EXT_SPACE 16
  174. struct rds_header {
  175. __be64 h_sequence;
  176. __be64 h_ack;
  177. __be32 h_len;
  178. __be16 h_sport;
  179. __be16 h_dport;
  180. u8 h_flags;
  181. u8 h_credit;
  182. u8 h_padding[4];
  183. __sum16 h_csum;
  184. u8 h_exthdr[RDS_HEADER_EXT_SPACE];
  185. };
  186. /*
  187. * Reserved - indicates end of extensions
  188. */
  189. #define RDS_EXTHDR_NONE 0
  190. /*
  191. * This extension header is included in the very
  192. * first message that is sent on a new connection,
  193. * and identifies the protocol level. This will help
  194. * rolling updates if a future change requires breaking
  195. * the protocol.
  196. * NB: This is no longer true for IB, where we do a version
  197. * negotiation during the connection setup phase (protocol
  198. * version information is included in the RDMA CM private data).
  199. */
  200. #define RDS_EXTHDR_VERSION 1
  201. struct rds_ext_header_version {
  202. __be32 h_version;
  203. };
  204. /*
  205. * This extension header is included in the RDS message
  206. * chasing an RDMA operation.
  207. */
  208. #define RDS_EXTHDR_RDMA 2
  209. struct rds_ext_header_rdma {
  210. __be32 h_rdma_rkey;
  211. };
  212. /*
  213. * This extension header tells the peer about the
  214. * destination <R_Key,offset> of the requested RDMA
  215. * operation.
  216. */
  217. #define RDS_EXTHDR_RDMA_DEST 3
  218. struct rds_ext_header_rdma_dest {
  219. __be32 h_rdma_rkey;
  220. __be32 h_rdma_offset;
  221. };
  222. /* Extension header announcing number of paths.
  223. * Implicit length = 2 bytes.
  224. */
  225. #define RDS_EXTHDR_NPATHS 5
  226. #define RDS_EXTHDR_GEN_NUM 6
  227. #define __RDS_EXTHDR_MAX 16 /* for now */
  228. #define RDS_RX_MAX_TRACES (RDS_MSG_RX_DGRAM_TRACE_MAX + 1)
  229. #define RDS_MSG_RX_HDR 0
  230. #define RDS_MSG_RX_START 1
  231. #define RDS_MSG_RX_END 2
  232. #define RDS_MSG_RX_CMSG 3
  233. /* The following values are whitelisted for usercopy */
  234. struct rds_inc_usercopy {
  235. rds_rdma_cookie_t rdma_cookie;
  236. ktime_t rx_tstamp;
  237. };
  238. struct rds_incoming {
  239. refcount_t i_refcount;
  240. struct list_head i_item;
  241. struct rds_connection *i_conn;
  242. struct rds_conn_path *i_conn_path;
  243. struct rds_header i_hdr;
  244. unsigned long i_rx_jiffies;
  245. struct in6_addr i_saddr;
  246. struct rds_inc_usercopy i_usercopy;
  247. u64 i_rx_lat_trace[RDS_RX_MAX_TRACES];
  248. };
  249. struct rds_mr {
  250. struct rb_node r_rb_node;
  251. struct kref r_kref;
  252. u32 r_key;
  253. /* A copy of the creation flags */
  254. unsigned int r_use_once:1;
  255. unsigned int r_invalidate:1;
  256. unsigned int r_write:1;
  257. struct rds_sock *r_sock; /* back pointer to the socket that owns us */
  258. struct rds_transport *r_trans;
  259. void *r_trans_private;
  260. };
  261. static inline rds_rdma_cookie_t rds_rdma_make_cookie(u32 r_key, u32 offset)
  262. {
  263. return r_key | (((u64) offset) << 32);
  264. }
  265. static inline u32 rds_rdma_cookie_key(rds_rdma_cookie_t cookie)
  266. {
  267. return cookie;
  268. }
  269. static inline u32 rds_rdma_cookie_offset(rds_rdma_cookie_t cookie)
  270. {
  271. return cookie >> 32;
  272. }
  273. /* atomic operation types */
  274. #define RDS_ATOMIC_TYPE_CSWP 0
  275. #define RDS_ATOMIC_TYPE_FADD 1
  276. /*
  277. * m_sock_item and m_conn_item are on lists that are serialized under
  278. * conn->c_lock. m_sock_item has additional meaning in that once it is empty
  279. * the message will not be put back on the retransmit list after being sent.
  280. * messages that are canceled while being sent rely on this.
  281. *
  282. * m_inc is used by loopback so that it can pass an incoming message straight
  283. * back up into the rx path. It embeds a wire header which is also used by
  284. * the send path, which is kind of awkward.
  285. *
  286. * m_sock_item indicates the message's presence on a socket's send or receive
  287. * queue. m_rs will point to that socket.
  288. *
  289. * m_daddr is used by cancellation to prune messages to a given destination.
  290. *
  291. * The RDS_MSG_ON_SOCK and RDS_MSG_ON_CONN flags are used to avoid lock
  292. * nesting. As paths iterate over messages on a sock, or conn, they must
  293. * also lock the conn, or sock, to remove the message from those lists too.
  294. * Testing the flag to determine if the message is still on the lists lets
  295. * us avoid testing the list_head directly. That means each path can use
  296. * the message's list_head to keep it on a local list while juggling locks
  297. * without confusing the other path.
  298. *
  299. * m_ack_seq is an optional field set by transports who need a different
  300. * sequence number range to invalidate. They can use this in a callback
  301. * that they pass to rds_send_drop_acked() to see if each message has been
  302. * acked. The HAS_ACK_SEQ flag can be used to detect messages which haven't
  303. * had ack_seq set yet.
  304. */
  305. #define RDS_MSG_ON_SOCK 1
  306. #define RDS_MSG_ON_CONN 2
  307. #define RDS_MSG_HAS_ACK_SEQ 3
  308. #define RDS_MSG_ACK_REQUIRED 4
  309. #define RDS_MSG_RETRANSMITTED 5
  310. #define RDS_MSG_MAPPED 6
  311. #define RDS_MSG_PAGEVEC 7
  312. #define RDS_MSG_FLUSH 8
  313. struct rds_znotifier {
  314. struct mmpin z_mmp;
  315. u32 z_cookie;
  316. };
  317. struct rds_msg_zcopy_info {
  318. struct list_head rs_zcookie_next;
  319. union {
  320. struct rds_znotifier znotif;
  321. struct rds_zcopy_cookies zcookies;
  322. };
  323. };
  324. struct rds_msg_zcopy_queue {
  325. struct list_head zcookie_head;
  326. spinlock_t lock; /* protects zcookie_head queue */
  327. };
  328. static inline void rds_message_zcopy_queue_init(struct rds_msg_zcopy_queue *q)
  329. {
  330. spin_lock_init(&q->lock);
  331. INIT_LIST_HEAD(&q->zcookie_head);
  332. }
  333. struct rds_iov_vector {
  334. struct rds_iovec *iov;
  335. int len;
  336. };
  337. struct rds_iov_vector_arr {
  338. struct rds_iov_vector *vec;
  339. int len;
  340. int indx;
  341. int incr;
  342. };
  343. struct rds_message {
  344. refcount_t m_refcount;
  345. struct list_head m_sock_item;
  346. struct list_head m_conn_item;
  347. struct rds_incoming m_inc;
  348. u64 m_ack_seq;
  349. struct in6_addr m_daddr;
  350. unsigned long m_flags;
  351. /* Never access m_rs without holding m_rs_lock.
  352. * Lock nesting is
  353. * rm->m_rs_lock
  354. * -> rs->rs_lock
  355. */
  356. spinlock_t m_rs_lock;
  357. wait_queue_head_t m_flush_wait;
  358. struct rds_sock *m_rs;
  359. /* cookie to send to remote, in rds header */
  360. rds_rdma_cookie_t m_rdma_cookie;
  361. unsigned int m_used_sgs;
  362. unsigned int m_total_sgs;
  363. void *m_final_op;
  364. struct {
  365. struct rm_atomic_op {
  366. int op_type;
  367. union {
  368. struct {
  369. uint64_t compare;
  370. uint64_t swap;
  371. uint64_t compare_mask;
  372. uint64_t swap_mask;
  373. } op_m_cswp;
  374. struct {
  375. uint64_t add;
  376. uint64_t nocarry_mask;
  377. } op_m_fadd;
  378. };
  379. u32 op_rkey;
  380. u64 op_remote_addr;
  381. unsigned int op_notify:1;
  382. unsigned int op_recverr:1;
  383. unsigned int op_mapped:1;
  384. unsigned int op_silent:1;
  385. unsigned int op_active:1;
  386. struct scatterlist *op_sg;
  387. struct rds_notifier *op_notifier;
  388. struct rds_mr *op_rdma_mr;
  389. } atomic;
  390. struct rm_rdma_op {
  391. u32 op_rkey;
  392. u64 op_remote_addr;
  393. unsigned int op_write:1;
  394. unsigned int op_fence:1;
  395. unsigned int op_notify:1;
  396. unsigned int op_recverr:1;
  397. unsigned int op_mapped:1;
  398. unsigned int op_silent:1;
  399. unsigned int op_active:1;
  400. unsigned int op_bytes;
  401. unsigned int op_nents;
  402. unsigned int op_count;
  403. struct scatterlist *op_sg;
  404. struct rds_notifier *op_notifier;
  405. struct rds_mr *op_rdma_mr;
  406. u64 op_odp_addr;
  407. struct rds_mr *op_odp_mr;
  408. } rdma;
  409. struct rm_data_op {
  410. unsigned int op_active:1;
  411. unsigned int op_nents;
  412. unsigned int op_count;
  413. unsigned int op_dmasg;
  414. unsigned int op_dmaoff;
  415. struct rds_znotifier *op_mmp_znotifier;
  416. struct scatterlist *op_sg;
  417. } data;
  418. };
  419. struct rds_conn_path *m_conn_path;
  420. };
  421. /*
  422. * The RDS notifier is used (optionally) to tell the application about
  423. * completed RDMA operations. Rather than keeping the whole rds message
  424. * around on the queue, we allocate a small notifier that is put on the
  425. * socket's notifier_list. Notifications are delivered to the application
  426. * through control messages.
  427. */
  428. struct rds_notifier {
  429. struct list_head n_list;
  430. uint64_t n_user_token;
  431. int n_status;
  432. };
  433. /* Available as part of RDS core, so doesn't need to participate
  434. * in get_preferred transport etc
  435. */
  436. #define RDS_TRANS_LOOP 3
  437. /**
  438. * struct rds_transport - transport specific behavioural hooks
  439. *
  440. * @xmit: .xmit is called by rds_send_xmit() to tell the transport to send
  441. * part of a message. The caller serializes on the send_sem so this
  442. * doesn't need to be reentrant for a given conn. The header must be
  443. * sent before the data payload. .xmit must be prepared to send a
  444. * message with no data payload. .xmit should return the number of
  445. * bytes that were sent down the connection, including header bytes.
  446. * Returning 0 tells the caller that it doesn't need to perform any
  447. * additional work now. This is usually the case when the transport has
  448. * filled the sending queue for its connection and will handle
  449. * triggering the rds thread to continue the send when space becomes
  450. * available. Returning -EAGAIN tells the caller to retry the send
  451. * immediately. Returning -ENOMEM tells the caller to retry the send at
  452. * some point in the future.
  453. *
  454. * @conn_shutdown: conn_shutdown stops traffic on the given connection. Once
  455. * it returns the connection can not call rds_recv_incoming().
  456. * This will only be called once after conn_connect returns
  457. * non-zero success and will The caller serializes this with
  458. * the send and connecting paths (xmit_* and conn_*). The
  459. * transport is responsible for other serialization, including
  460. * rds_recv_incoming(). This is called in process context but
  461. * should try hard not to block.
  462. */
  463. struct rds_transport {
  464. char t_name[TRANSNAMSIZ];
  465. struct list_head t_item;
  466. struct module *t_owner;
  467. unsigned int t_prefer_loopback:1,
  468. t_mp_capable:1;
  469. unsigned int t_type;
  470. int (*laddr_check)(struct net *net, const struct in6_addr *addr,
  471. __u32 scope_id);
  472. int (*conn_alloc)(struct rds_connection *conn, gfp_t gfp);
  473. void (*conn_free)(void *data);
  474. int (*conn_path_connect)(struct rds_conn_path *cp);
  475. void (*conn_path_shutdown)(struct rds_conn_path *conn);
  476. void (*xmit_path_prepare)(struct rds_conn_path *cp);
  477. void (*xmit_path_complete)(struct rds_conn_path *cp);
  478. int (*xmit)(struct rds_connection *conn, struct rds_message *rm,
  479. unsigned int hdr_off, unsigned int sg, unsigned int off);
  480. int (*xmit_rdma)(struct rds_connection *conn, struct rm_rdma_op *op);
  481. int (*xmit_atomic)(struct rds_connection *conn, struct rm_atomic_op *op);
  482. int (*recv_path)(struct rds_conn_path *cp);
  483. int (*inc_copy_to_user)(struct rds_incoming *inc, struct iov_iter *to);
  484. void (*inc_free)(struct rds_incoming *inc);
  485. int (*cm_handle_connect)(struct rdma_cm_id *cm_id,
  486. struct rdma_cm_event *event, bool isv6);
  487. int (*cm_initiate_connect)(struct rdma_cm_id *cm_id, bool isv6);
  488. void (*cm_connect_complete)(struct rds_connection *conn,
  489. struct rdma_cm_event *event);
  490. unsigned int (*stats_info_copy)(struct rds_info_iterator *iter,
  491. unsigned int avail);
  492. void (*exit)(void);
  493. void *(*get_mr)(struct scatterlist *sg, unsigned long nr_sg,
  494. struct rds_sock *rs, u32 *key_ret,
  495. struct rds_connection *conn,
  496. u64 start, u64 length, int need_odp);
  497. void (*sync_mr)(void *trans_private, int direction);
  498. void (*free_mr)(void *trans_private, int invalidate);
  499. void (*flush_mrs)(void);
  500. bool (*t_unloading)(struct rds_connection *conn);
  501. u8 (*get_tos_map)(u8 tos);
  502. };
  503. /* Bind hash table key length. It is the sum of the size of a struct
  504. * in6_addr, a scope_id and a port.
  505. */
  506. #define RDS_BOUND_KEY_LEN \
  507. (sizeof(struct in6_addr) + sizeof(__u32) + sizeof(__be16))
  508. struct rds_sock {
  509. struct sock rs_sk;
  510. u64 rs_user_addr;
  511. u64 rs_user_bytes;
  512. /*
  513. * bound_addr used for both incoming and outgoing, no INADDR_ANY
  514. * support.
  515. */
  516. struct rhash_head rs_bound_node;
  517. u8 rs_bound_key[RDS_BOUND_KEY_LEN];
  518. struct sockaddr_in6 rs_bound_sin6;
  519. #define rs_bound_addr rs_bound_sin6.sin6_addr
  520. #define rs_bound_addr_v4 rs_bound_sin6.sin6_addr.s6_addr32[3]
  521. #define rs_bound_port rs_bound_sin6.sin6_port
  522. #define rs_bound_scope_id rs_bound_sin6.sin6_scope_id
  523. struct in6_addr rs_conn_addr;
  524. #define rs_conn_addr_v4 rs_conn_addr.s6_addr32[3]
  525. __be16 rs_conn_port;
  526. struct rds_transport *rs_transport;
  527. /*
  528. * rds_sendmsg caches the conn it used the last time around.
  529. * This helps avoid costly lookups.
  530. */
  531. struct rds_connection *rs_conn;
  532. /* flag indicating we were congested or not */
  533. int rs_congested;
  534. /* seen congestion (ENOBUFS) when sending? */
  535. int rs_seen_congestion;
  536. /* rs_lock protects all these adjacent members before the newline */
  537. spinlock_t rs_lock;
  538. struct list_head rs_send_queue;
  539. u32 rs_snd_bytes;
  540. int rs_rcv_bytes;
  541. struct list_head rs_notify_queue; /* currently used for failed RDMAs */
  542. /* Congestion wake_up. If rs_cong_monitor is set, we use cong_mask
  543. * to decide whether the application should be woken up.
  544. * If not set, we use rs_cong_track to find out whether a cong map
  545. * update arrived.
  546. */
  547. uint64_t rs_cong_mask;
  548. uint64_t rs_cong_notify;
  549. struct list_head rs_cong_list;
  550. unsigned long rs_cong_track;
  551. /*
  552. * rs_recv_lock protects the receive queue, and is
  553. * used to serialize with rds_release.
  554. */
  555. rwlock_t rs_recv_lock;
  556. struct list_head rs_recv_queue;
  557. /* just for stats reporting */
  558. struct list_head rs_item;
  559. /* these have their own lock */
  560. spinlock_t rs_rdma_lock;
  561. struct rb_root rs_rdma_keys;
  562. /* Socket options - in case there will be more */
  563. unsigned char rs_recverr,
  564. rs_cong_monitor;
  565. u32 rs_hash_initval;
  566. /* Socket receive path trace points*/
  567. u8 rs_rx_traces;
  568. u8 rs_rx_trace[RDS_MSG_RX_DGRAM_TRACE_MAX];
  569. struct rds_msg_zcopy_queue rs_zcookie_queue;
  570. u8 rs_tos;
  571. };
  572. static inline struct rds_sock *rds_sk_to_rs(const struct sock *sk)
  573. {
  574. return container_of(sk, struct rds_sock, rs_sk);
  575. }
  576. static inline struct sock *rds_rs_to_sk(struct rds_sock *rs)
  577. {
  578. return &rs->rs_sk;
  579. }
  580. /*
  581. * The stack assigns sk_sndbuf and sk_rcvbuf to twice the specified value
  582. * to account for overhead. We don't account for overhead, we just apply
  583. * the number of payload bytes to the specified value.
  584. */
  585. static inline int rds_sk_sndbuf(struct rds_sock *rs)
  586. {
  587. return rds_rs_to_sk(rs)->sk_sndbuf / 2;
  588. }
  589. static inline int rds_sk_rcvbuf(struct rds_sock *rs)
  590. {
  591. return rds_rs_to_sk(rs)->sk_rcvbuf / 2;
  592. }
  593. struct rds_statistics {
  594. uint64_t s_conn_reset;
  595. uint64_t s_recv_drop_bad_checksum;
  596. uint64_t s_recv_drop_old_seq;
  597. uint64_t s_recv_drop_no_sock;
  598. uint64_t s_recv_drop_dead_sock;
  599. uint64_t s_recv_deliver_raced;
  600. uint64_t s_recv_delivered;
  601. uint64_t s_recv_queued;
  602. uint64_t s_recv_immediate_retry;
  603. uint64_t s_recv_delayed_retry;
  604. uint64_t s_recv_ack_required;
  605. uint64_t s_recv_rdma_bytes;
  606. uint64_t s_recv_ping;
  607. uint64_t s_send_queue_empty;
  608. uint64_t s_send_queue_full;
  609. uint64_t s_send_lock_contention;
  610. uint64_t s_send_lock_queue_raced;
  611. uint64_t s_send_immediate_retry;
  612. uint64_t s_send_delayed_retry;
  613. uint64_t s_send_drop_acked;
  614. uint64_t s_send_ack_required;
  615. uint64_t s_send_queued;
  616. uint64_t s_send_rdma;
  617. uint64_t s_send_rdma_bytes;
  618. uint64_t s_send_pong;
  619. uint64_t s_page_remainder_hit;
  620. uint64_t s_page_remainder_miss;
  621. uint64_t s_copy_to_user;
  622. uint64_t s_copy_from_user;
  623. uint64_t s_cong_update_queued;
  624. uint64_t s_cong_update_received;
  625. uint64_t s_cong_send_error;
  626. uint64_t s_cong_send_blocked;
  627. uint64_t s_recv_bytes_added_to_socket;
  628. uint64_t s_recv_bytes_removed_from_socket;
  629. uint64_t s_send_stuck_rm;
  630. };
  631. /* af_rds.c */
  632. void rds_sock_addref(struct rds_sock *rs);
  633. void rds_sock_put(struct rds_sock *rs);
  634. void rds_wake_sk_sleep(struct rds_sock *rs);
  635. static inline void __rds_wake_sk_sleep(struct sock *sk)
  636. {
  637. wait_queue_head_t *waitq = sk_sleep(sk);
  638. if (!sock_flag(sk, SOCK_DEAD) && waitq)
  639. wake_up(waitq);
  640. }
  641. extern wait_queue_head_t rds_poll_waitq;
  642. /* bind.c */
  643. int rds_bind(struct socket *sock, struct sockaddr *uaddr, int addr_len);
  644. void rds_remove_bound(struct rds_sock *rs);
  645. struct rds_sock *rds_find_bound(const struct in6_addr *addr, __be16 port,
  646. __u32 scope_id);
  647. int rds_bind_lock_init(void);
  648. void rds_bind_lock_destroy(void);
  649. /* cong.c */
  650. int rds_cong_get_maps(struct rds_connection *conn);
  651. void rds_cong_add_conn(struct rds_connection *conn);
  652. void rds_cong_remove_conn(struct rds_connection *conn);
  653. void rds_cong_set_bit(struct rds_cong_map *map, __be16 port);
  654. void rds_cong_clear_bit(struct rds_cong_map *map, __be16 port);
  655. int rds_cong_wait(struct rds_cong_map *map, __be16 port, int nonblock, struct rds_sock *rs);
  656. void rds_cong_queue_updates(struct rds_cong_map *map);
  657. void rds_cong_map_updated(struct rds_cong_map *map, uint64_t);
  658. int rds_cong_updated_since(unsigned long *recent);
  659. void rds_cong_add_socket(struct rds_sock *);
  660. void rds_cong_remove_socket(struct rds_sock *);
  661. void rds_cong_exit(void);
  662. struct rds_message *rds_cong_update_alloc(struct rds_connection *conn);
  663. /* connection.c */
  664. extern u32 rds_gen_num;
  665. int rds_conn_init(void);
  666. void rds_conn_exit(void);
  667. struct rds_connection *rds_conn_create(struct net *net,
  668. const struct in6_addr *laddr,
  669. const struct in6_addr *faddr,
  670. struct rds_transport *trans,
  671. u8 tos, gfp_t gfp,
  672. int dev_if);
  673. struct rds_connection *rds_conn_create_outgoing(struct net *net,
  674. const struct in6_addr *laddr,
  675. const struct in6_addr *faddr,
  676. struct rds_transport *trans,
  677. u8 tos, gfp_t gfp, int dev_if);
  678. void rds_conn_shutdown(struct rds_conn_path *cpath);
  679. void rds_conn_destroy(struct rds_connection *conn);
  680. void rds_conn_drop(struct rds_connection *conn);
  681. void rds_conn_path_drop(struct rds_conn_path *cpath, bool destroy);
  682. void rds_conn_connect_if_down(struct rds_connection *conn);
  683. void rds_conn_path_connect_if_down(struct rds_conn_path *cp);
  684. void rds_check_all_paths(struct rds_connection *conn);
  685. void rds_for_each_conn_info(struct socket *sock, unsigned int len,
  686. struct rds_info_iterator *iter,
  687. struct rds_info_lengths *lens,
  688. int (*visitor)(struct rds_connection *, void *),
  689. u64 *buffer,
  690. size_t item_len);
  691. __printf(2, 3)
  692. void __rds_conn_path_error(struct rds_conn_path *cp, const char *, ...);
  693. #define rds_conn_path_error(cp, fmt...) \
  694. __rds_conn_path_error(cp, KERN_WARNING "RDS: " fmt)
  695. static inline int
  696. rds_conn_path_transition(struct rds_conn_path *cp, int old, int new)
  697. {
  698. return atomic_cmpxchg(&cp->cp_state, old, new) == old;
  699. }
  700. static inline int
  701. rds_conn_transition(struct rds_connection *conn, int old, int new)
  702. {
  703. WARN_ON(conn->c_trans->t_mp_capable);
  704. return rds_conn_path_transition(&conn->c_path[0], old, new);
  705. }
  706. static inline int
  707. rds_conn_path_state(struct rds_conn_path *cp)
  708. {
  709. return atomic_read(&cp->cp_state);
  710. }
  711. static inline int
  712. rds_conn_state(struct rds_connection *conn)
  713. {
  714. WARN_ON(conn->c_trans->t_mp_capable);
  715. return rds_conn_path_state(&conn->c_path[0]);
  716. }
  717. static inline int
  718. rds_conn_path_up(struct rds_conn_path *cp)
  719. {
  720. return atomic_read(&cp->cp_state) == RDS_CONN_UP;
  721. }
  722. static inline int
  723. rds_conn_path_down(struct rds_conn_path *cp)
  724. {
  725. return atomic_read(&cp->cp_state) == RDS_CONN_DOWN;
  726. }
  727. static inline int
  728. rds_conn_up(struct rds_connection *conn)
  729. {
  730. WARN_ON(conn->c_trans->t_mp_capable);
  731. return rds_conn_path_up(&conn->c_path[0]);
  732. }
  733. static inline int
  734. rds_conn_path_connecting(struct rds_conn_path *cp)
  735. {
  736. return atomic_read(&cp->cp_state) == RDS_CONN_CONNECTING;
  737. }
  738. static inline int
  739. rds_conn_connecting(struct rds_connection *conn)
  740. {
  741. WARN_ON(conn->c_trans->t_mp_capable);
  742. return rds_conn_path_connecting(&conn->c_path[0]);
  743. }
  744. /* message.c */
  745. struct rds_message *rds_message_alloc(unsigned int nents, gfp_t gfp);
  746. struct scatterlist *rds_message_alloc_sgs(struct rds_message *rm, int nents);
  747. int rds_message_copy_from_user(struct rds_message *rm, struct iov_iter *from,
  748. bool zcopy);
  749. struct rds_message *rds_message_map_pages(unsigned long *page_addrs, unsigned int total_len);
  750. void rds_message_populate_header(struct rds_header *hdr, __be16 sport,
  751. __be16 dport, u64 seq);
  752. int rds_message_add_extension(struct rds_header *hdr,
  753. unsigned int type, const void *data, unsigned int len);
  754. int rds_message_next_extension(struct rds_header *hdr,
  755. unsigned int *pos, void *buf, unsigned int *buflen);
  756. int rds_message_add_rdma_dest_extension(struct rds_header *hdr, u32 r_key, u32 offset);
  757. int rds_message_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to);
  758. void rds_message_inc_free(struct rds_incoming *inc);
  759. void rds_message_addref(struct rds_message *rm);
  760. void rds_message_put(struct rds_message *rm);
  761. void rds_message_wait(struct rds_message *rm);
  762. void rds_message_unmapped(struct rds_message *rm);
  763. void rds_notify_msg_zcopy_purge(struct rds_msg_zcopy_queue *info);
  764. static inline void rds_message_make_checksum(struct rds_header *hdr)
  765. {
  766. hdr->h_csum = 0;
  767. hdr->h_csum = ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2);
  768. }
  769. static inline int rds_message_verify_checksum(const struct rds_header *hdr)
  770. {
  771. return !hdr->h_csum || ip_fast_csum((void *) hdr, sizeof(*hdr) >> 2) == 0;
  772. }
  773. /* page.c */
  774. int rds_page_remainder_alloc(struct scatterlist *scat, unsigned long bytes,
  775. gfp_t gfp);
  776. void rds_page_exit(void);
  777. /* recv.c */
  778. void rds_inc_init(struct rds_incoming *inc, struct rds_connection *conn,
  779. struct in6_addr *saddr);
  780. void rds_inc_path_init(struct rds_incoming *inc, struct rds_conn_path *conn,
  781. struct in6_addr *saddr);
  782. void rds_inc_put(struct rds_incoming *inc);
  783. void rds_recv_incoming(struct rds_connection *conn, struct in6_addr *saddr,
  784. struct in6_addr *daddr,
  785. struct rds_incoming *inc, gfp_t gfp);
  786. int rds_recvmsg(struct socket *sock, struct msghdr *msg, size_t size,
  787. int msg_flags);
  788. void rds_clear_recv_queue(struct rds_sock *rs);
  789. int rds_notify_queue_get(struct rds_sock *rs, struct msghdr *msg);
  790. void rds_inc_info_copy(struct rds_incoming *inc,
  791. struct rds_info_iterator *iter,
  792. __be32 saddr, __be32 daddr, int flip);
  793. void rds6_inc_info_copy(struct rds_incoming *inc,
  794. struct rds_info_iterator *iter,
  795. struct in6_addr *saddr, struct in6_addr *daddr,
  796. int flip);
  797. /* send.c */
  798. int rds_sendmsg(struct socket *sock, struct msghdr *msg, size_t payload_len);
  799. void rds_send_path_reset(struct rds_conn_path *conn);
  800. int rds_send_xmit(struct rds_conn_path *cp);
  801. struct sockaddr_in;
  802. void rds_send_drop_to(struct rds_sock *rs, struct sockaddr_in6 *dest);
  803. typedef int (*is_acked_func)(struct rds_message *rm, uint64_t ack);
  804. void rds_send_drop_acked(struct rds_connection *conn, u64 ack,
  805. is_acked_func is_acked);
  806. void rds_send_path_drop_acked(struct rds_conn_path *cp, u64 ack,
  807. is_acked_func is_acked);
  808. void rds_send_ping(struct rds_connection *conn, int cp_index);
  809. int rds_send_pong(struct rds_conn_path *cp, __be16 dport);
  810. /* rdma.c */
  811. void rds_rdma_unuse(struct rds_sock *rs, u32 r_key, int force);
  812. int rds_get_mr(struct rds_sock *rs, sockptr_t optval, int optlen);
  813. int rds_get_mr_for_dest(struct rds_sock *rs, sockptr_t optval, int optlen);
  814. int rds_free_mr(struct rds_sock *rs, sockptr_t optval, int optlen);
  815. void rds_rdma_drop_keys(struct rds_sock *rs);
  816. int rds_rdma_extra_size(struct rds_rdma_args *args,
  817. struct rds_iov_vector *iov);
  818. int rds_cmsg_rdma_dest(struct rds_sock *rs, struct rds_message *rm,
  819. struct cmsghdr *cmsg);
  820. int rds_cmsg_rdma_args(struct rds_sock *rs, struct rds_message *rm,
  821. struct cmsghdr *cmsg,
  822. struct rds_iov_vector *vec);
  823. int rds_cmsg_rdma_map(struct rds_sock *rs, struct rds_message *rm,
  824. struct cmsghdr *cmsg);
  825. void rds_rdma_free_op(struct rm_rdma_op *ro);
  826. void rds_atomic_free_op(struct rm_atomic_op *ao);
  827. void rds_rdma_send_complete(struct rds_message *rm, int wc_status);
  828. void rds_atomic_send_complete(struct rds_message *rm, int wc_status);
  829. int rds_cmsg_atomic(struct rds_sock *rs, struct rds_message *rm,
  830. struct cmsghdr *cmsg);
  831. void __rds_put_mr_final(struct kref *kref);
  832. static inline bool rds_destroy_pending(struct rds_connection *conn)
  833. {
  834. return !check_net(rds_conn_net(conn)) ||
  835. (conn->c_trans->t_unloading && conn->c_trans->t_unloading(conn));
  836. }
  837. enum {
  838. ODP_NOT_NEEDED,
  839. ODP_ZEROBASED,
  840. ODP_VIRTUAL
  841. };
  842. /* stats.c */
  843. DECLARE_PER_CPU_SHARED_ALIGNED(struct rds_statistics, rds_stats);
  844. #define rds_stats_inc_which(which, member) do { \
  845. per_cpu(which, get_cpu()).member++; \
  846. put_cpu(); \
  847. } while (0)
  848. #define rds_stats_inc(member) rds_stats_inc_which(rds_stats, member)
  849. #define rds_stats_add_which(which, member, count) do { \
  850. per_cpu(which, get_cpu()).member += count; \
  851. put_cpu(); \
  852. } while (0)
  853. #define rds_stats_add(member, count) rds_stats_add_which(rds_stats, member, count)
  854. int rds_stats_init(void);
  855. void rds_stats_exit(void);
  856. void rds_stats_info_copy(struct rds_info_iterator *iter,
  857. uint64_t *values, const char *const *names,
  858. size_t nr);
  859. /* sysctl.c */
  860. int rds_sysctl_init(void);
  861. void rds_sysctl_exit(void);
  862. extern unsigned long rds_sysctl_sndbuf_min;
  863. extern unsigned long rds_sysctl_sndbuf_default;
  864. extern unsigned long rds_sysctl_sndbuf_max;
  865. extern unsigned long rds_sysctl_reconnect_min_jiffies;
  866. extern unsigned long rds_sysctl_reconnect_max_jiffies;
  867. extern unsigned int rds_sysctl_max_unacked_packets;
  868. extern unsigned int rds_sysctl_max_unacked_bytes;
  869. extern unsigned int rds_sysctl_ping_enable;
  870. extern unsigned long rds_sysctl_trace_flags;
  871. extern unsigned int rds_sysctl_trace_level;
  872. /* threads.c */
  873. int rds_threads_init(void);
  874. void rds_threads_exit(void);
  875. extern struct workqueue_struct *rds_wq;
  876. void rds_queue_reconnect(struct rds_conn_path *cp);
  877. void rds_connect_worker(struct work_struct *);
  878. void rds_shutdown_worker(struct work_struct *);
  879. void rds_send_worker(struct work_struct *);
  880. void rds_recv_worker(struct work_struct *);
  881. void rds_connect_path_complete(struct rds_conn_path *conn, int curr);
  882. void rds_connect_complete(struct rds_connection *conn);
  883. int rds_addr_cmp(const struct in6_addr *a1, const struct in6_addr *a2);
  884. /* transport.c */
  885. void rds_trans_register(struct rds_transport *trans);
  886. void rds_trans_unregister(struct rds_transport *trans);
  887. struct rds_transport *rds_trans_get_preferred(struct net *net,
  888. const struct in6_addr *addr,
  889. __u32 scope_id);
  890. void rds_trans_put(struct rds_transport *trans);
  891. unsigned int rds_trans_stats_info_copy(struct rds_info_iterator *iter,
  892. unsigned int avail);
  893. struct rds_transport *rds_trans_get(int t_type);
  894. int rds_trans_init(void);
  895. void rds_trans_exit(void);
  896. #endif