tcp_recv.c 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349
  1. /*
  2. * Copyright (c) 2006, 2017 Oracle and/or its affiliates. All rights reserved.
  3. *
  4. * This software is available to you under a choice of one of two
  5. * licenses. You may choose to be licensed under the terms of the GNU
  6. * General Public License (GPL) Version 2, available from the file
  7. * COPYING in the main directory of this source tree, or the
  8. * OpenIB.org BSD license below:
  9. *
  10. * Redistribution and use in source and binary forms, with or
  11. * without modification, are permitted provided that the following
  12. * conditions are met:
  13. *
  14. * - Redistributions of source code must retain the above
  15. * copyright notice, this list of conditions and the following
  16. * disclaimer.
  17. *
  18. * - Redistributions in binary form must reproduce the above
  19. * copyright notice, this list of conditions and the following
  20. * disclaimer in the documentation and/or other materials
  21. * provided with the distribution.
  22. *
  23. * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  24. * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  25. * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
  26. * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS
  27. * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN
  28. * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN
  29. * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
  30. * SOFTWARE.
  31. *
  32. */
  33. #include <linux/kernel.h>
  34. #include <linux/slab.h>
  35. #include <net/tcp.h>
  36. #include "rds.h"
  37. #include "tcp.h"
  38. static struct kmem_cache *rds_tcp_incoming_slab;
  39. static void rds_tcp_inc_purge(struct rds_incoming *inc)
  40. {
  41. struct rds_tcp_incoming *tinc;
  42. tinc = container_of(inc, struct rds_tcp_incoming, ti_inc);
  43. rdsdebug("purging tinc %p inc %p\n", tinc, inc);
  44. skb_queue_purge(&tinc->ti_skb_list);
  45. }
  46. void rds_tcp_inc_free(struct rds_incoming *inc)
  47. {
  48. struct rds_tcp_incoming *tinc;
  49. tinc = container_of(inc, struct rds_tcp_incoming, ti_inc);
  50. rds_tcp_inc_purge(inc);
  51. rdsdebug("freeing tinc %p inc %p\n", tinc, inc);
  52. kmem_cache_free(rds_tcp_incoming_slab, tinc);
  53. }
  54. /*
  55. * this is pretty lame, but, whatever.
  56. */
  57. int rds_tcp_inc_copy_to_user(struct rds_incoming *inc, struct iov_iter *to)
  58. {
  59. struct rds_tcp_incoming *tinc;
  60. struct sk_buff *skb;
  61. int ret = 0;
  62. if (!iov_iter_count(to))
  63. goto out;
  64. tinc = container_of(inc, struct rds_tcp_incoming, ti_inc);
  65. skb_queue_walk(&tinc->ti_skb_list, skb) {
  66. unsigned long to_copy, skb_off;
  67. for (skb_off = 0; skb_off < skb->len; skb_off += to_copy) {
  68. to_copy = iov_iter_count(to);
  69. to_copy = min(to_copy, skb->len - skb_off);
  70. if (skb_copy_datagram_iter(skb, skb_off, to, to_copy))
  71. return -EFAULT;
  72. rds_stats_add(s_copy_to_user, to_copy);
  73. ret += to_copy;
  74. if (!iov_iter_count(to))
  75. goto out;
  76. }
  77. }
  78. out:
  79. return ret;
  80. }
  81. /*
  82. * We have a series of skbs that have fragmented pieces of the congestion
  83. * bitmap. They must add up to the exact size of the congestion bitmap. We
  84. * use the skb helpers to copy those into the pages that make up the in-memory
  85. * congestion bitmap for the remote address of this connection. We then tell
  86. * the congestion core that the bitmap has been changed so that it can wake up
  87. * sleepers.
  88. *
  89. * This is racing with sending paths which are using test_bit to see if the
  90. * bitmap indicates that their recipient is congested.
  91. */
  92. static void rds_tcp_cong_recv(struct rds_connection *conn,
  93. struct rds_tcp_incoming *tinc)
  94. {
  95. struct sk_buff *skb;
  96. unsigned int to_copy, skb_off;
  97. unsigned int map_off;
  98. unsigned int map_page;
  99. struct rds_cong_map *map;
  100. int ret;
  101. /* catch completely corrupt packets */
  102. if (be32_to_cpu(tinc->ti_inc.i_hdr.h_len) != RDS_CONG_MAP_BYTES)
  103. return;
  104. map_page = 0;
  105. map_off = 0;
  106. map = conn->c_fcong;
  107. skb_queue_walk(&tinc->ti_skb_list, skb) {
  108. skb_off = 0;
  109. while (skb_off < skb->len) {
  110. to_copy = min_t(unsigned int, PAGE_SIZE - map_off,
  111. skb->len - skb_off);
  112. BUG_ON(map_page >= RDS_CONG_MAP_PAGES);
  113. /* only returns 0 or -error */
  114. ret = skb_copy_bits(skb, skb_off,
  115. (void *)map->m_page_addrs[map_page] + map_off,
  116. to_copy);
  117. BUG_ON(ret != 0);
  118. skb_off += to_copy;
  119. map_off += to_copy;
  120. if (map_off == PAGE_SIZE) {
  121. map_off = 0;
  122. map_page++;
  123. }
  124. }
  125. }
  126. rds_cong_map_updated(map, ~(u64) 0);
  127. }
  128. struct rds_tcp_desc_arg {
  129. struct rds_conn_path *conn_path;
  130. gfp_t gfp;
  131. };
  132. static int rds_tcp_data_recv(read_descriptor_t *desc, struct sk_buff *skb,
  133. unsigned int offset, size_t len)
  134. {
  135. struct rds_tcp_desc_arg *arg = desc->arg.data;
  136. struct rds_conn_path *cp = arg->conn_path;
  137. struct rds_tcp_connection *tc = cp->cp_transport_data;
  138. struct rds_tcp_incoming *tinc = tc->t_tinc;
  139. struct sk_buff *clone;
  140. size_t left = len, to_copy;
  141. rdsdebug("tcp data tc %p skb %p offset %u len %zu\n", tc, skb, offset,
  142. len);
  143. /*
  144. * tcp_read_sock() interprets partial progress as an indication to stop
  145. * processing.
  146. */
  147. while (left) {
  148. if (!tinc) {
  149. tinc = kmem_cache_alloc(rds_tcp_incoming_slab,
  150. arg->gfp);
  151. if (!tinc) {
  152. desc->error = -ENOMEM;
  153. goto out;
  154. }
  155. tc->t_tinc = tinc;
  156. rdsdebug("alloced tinc %p\n", tinc);
  157. rds_inc_path_init(&tinc->ti_inc, cp,
  158. &cp->cp_conn->c_faddr);
  159. tinc->ti_inc.i_rx_lat_trace[RDS_MSG_RX_HDR] =
  160. local_clock();
  161. /*
  162. * XXX * we might be able to use the __ variants when
  163. * we've already serialized at a higher level.
  164. */
  165. skb_queue_head_init(&tinc->ti_skb_list);
  166. }
  167. if (left && tc->t_tinc_hdr_rem) {
  168. to_copy = min(tc->t_tinc_hdr_rem, left);
  169. rdsdebug("copying %zu header from skb %p\n", to_copy,
  170. skb);
  171. skb_copy_bits(skb, offset,
  172. (char *)&tinc->ti_inc.i_hdr +
  173. sizeof(struct rds_header) -
  174. tc->t_tinc_hdr_rem,
  175. to_copy);
  176. tc->t_tinc_hdr_rem -= to_copy;
  177. left -= to_copy;
  178. offset += to_copy;
  179. if (tc->t_tinc_hdr_rem == 0) {
  180. /* could be 0 for a 0 len message */
  181. tc->t_tinc_data_rem =
  182. be32_to_cpu(tinc->ti_inc.i_hdr.h_len);
  183. tinc->ti_inc.i_rx_lat_trace[RDS_MSG_RX_START] =
  184. local_clock();
  185. }
  186. }
  187. if (left && tc->t_tinc_data_rem) {
  188. to_copy = min(tc->t_tinc_data_rem, left);
  189. clone = pskb_extract(skb, offset, to_copy, arg->gfp);
  190. if (!clone) {
  191. desc->error = -ENOMEM;
  192. goto out;
  193. }
  194. skb_queue_tail(&tinc->ti_skb_list, clone);
  195. rdsdebug("skb %p data %p len %d off %u to_copy %zu -> "
  196. "clone %p data %p len %d\n",
  197. skb, skb->data, skb->len, offset, to_copy,
  198. clone, clone->data, clone->len);
  199. tc->t_tinc_data_rem -= to_copy;
  200. left -= to_copy;
  201. offset += to_copy;
  202. }
  203. if (tc->t_tinc_hdr_rem == 0 && tc->t_tinc_data_rem == 0) {
  204. struct rds_connection *conn = cp->cp_conn;
  205. if (tinc->ti_inc.i_hdr.h_flags == RDS_FLAG_CONG_BITMAP)
  206. rds_tcp_cong_recv(conn, tinc);
  207. else
  208. rds_recv_incoming(conn, &conn->c_faddr,
  209. &conn->c_laddr,
  210. &tinc->ti_inc,
  211. arg->gfp);
  212. tc->t_tinc_hdr_rem = sizeof(struct rds_header);
  213. tc->t_tinc_data_rem = 0;
  214. tc->t_tinc = NULL;
  215. rds_inc_put(&tinc->ti_inc);
  216. tinc = NULL;
  217. }
  218. }
  219. out:
  220. rdsdebug("returning len %zu left %zu skb len %d rx queue depth %d\n",
  221. len, left, skb->len,
  222. skb_queue_len(&tc->t_sock->sk->sk_receive_queue));
  223. return len - left;
  224. }
  225. /* the caller has to hold the sock lock */
  226. static int rds_tcp_read_sock(struct rds_conn_path *cp, gfp_t gfp)
  227. {
  228. struct rds_tcp_connection *tc = cp->cp_transport_data;
  229. struct socket *sock = tc->t_sock;
  230. read_descriptor_t desc;
  231. struct rds_tcp_desc_arg arg;
  232. /* It's like glib in the kernel! */
  233. arg.conn_path = cp;
  234. arg.gfp = gfp;
  235. desc.arg.data = &arg;
  236. desc.error = 0;
  237. desc.count = 1; /* give more than one skb per call */
  238. tcp_read_sock(sock->sk, &desc, rds_tcp_data_recv);
  239. rdsdebug("tcp_read_sock for tc %p gfp 0x%x returned %d\n", tc, gfp,
  240. desc.error);
  241. return desc.error;
  242. }
  243. /*
  244. * We hold the sock lock to serialize our rds_tcp_recv->tcp_read_sock from
  245. * data_ready.
  246. *
  247. * if we fail to allocate we're in trouble.. blindly wait some time before
  248. * trying again to see if the VM can free up something for us.
  249. */
  250. int rds_tcp_recv_path(struct rds_conn_path *cp)
  251. {
  252. struct rds_tcp_connection *tc = cp->cp_transport_data;
  253. struct socket *sock = tc->t_sock;
  254. int ret = 0;
  255. rdsdebug("recv worker path [%d] tc %p sock %p\n",
  256. cp->cp_index, tc, sock);
  257. lock_sock(sock->sk);
  258. ret = rds_tcp_read_sock(cp, GFP_KERNEL);
  259. release_sock(sock->sk);
  260. return ret;
  261. }
  262. void rds_tcp_data_ready(struct sock *sk)
  263. {
  264. void (*ready)(struct sock *sk);
  265. struct rds_conn_path *cp;
  266. struct rds_tcp_connection *tc;
  267. rdsdebug("data ready sk %p\n", sk);
  268. read_lock_bh(&sk->sk_callback_lock);
  269. cp = sk->sk_user_data;
  270. if (!cp) { /* check for teardown race */
  271. ready = sk->sk_data_ready;
  272. goto out;
  273. }
  274. tc = cp->cp_transport_data;
  275. ready = tc->t_orig_data_ready;
  276. rds_tcp_stats_inc(s_tcp_data_ready_calls);
  277. if (rds_tcp_read_sock(cp, GFP_ATOMIC) == -ENOMEM) {
  278. rcu_read_lock();
  279. if (!rds_destroy_pending(cp->cp_conn))
  280. queue_delayed_work(rds_wq, &cp->cp_recv_w, 0);
  281. rcu_read_unlock();
  282. }
  283. out:
  284. read_unlock_bh(&sk->sk_callback_lock);
  285. ready(sk);
  286. }
  287. int rds_tcp_recv_init(void)
  288. {
  289. rds_tcp_incoming_slab = kmem_cache_create("rds_tcp_incoming",
  290. sizeof(struct rds_tcp_incoming),
  291. 0, 0, NULL);
  292. if (!rds_tcp_incoming_slab)
  293. return -ENOMEM;
  294. return 0;
  295. }
  296. void rds_tcp_recv_exit(void)
  297. {
  298. kmem_cache_destroy(rds_tcp_incoming_slab);
  299. }