inet_connection_sock.c 31 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * INET An implementation of the TCP/IP protocol suite for the LINUX
  4. * operating system. INET is implemented using the BSD Socket
  5. * interface as the means of communication with the user level.
  6. *
  7. * Support for INET connection oriented protocols.
  8. *
  9. * Authors: See the TCP sources
  10. */
  11. #include <linux/module.h>
  12. #include <linux/jhash.h>
  13. #include <net/inet_connection_sock.h>
  14. #include <net/inet_hashtables.h>
  15. #include <net/inet_timewait_sock.h>
  16. #include <net/ip.h>
  17. #include <net/route.h>
  18. #include <net/tcp_states.h>
  19. #include <net/xfrm.h>
  20. #include <net/tcp.h>
  21. #include <net/sock_reuseport.h>
  22. #include <net/addrconf.h>
  23. #if IS_ENABLED(CONFIG_IPV6)
  24. /* match_sk*_wildcard == true: IPV6_ADDR_ANY equals to any IPv6 addresses
  25. * if IPv6 only, and any IPv4 addresses
  26. * if not IPv6 only
  27. * match_sk*_wildcard == false: addresses must be exactly the same, i.e.
  28. * IPV6_ADDR_ANY only equals to IPV6_ADDR_ANY,
  29. * and 0.0.0.0 equals to 0.0.0.0 only
  30. */
  31. static bool ipv6_rcv_saddr_equal(const struct in6_addr *sk1_rcv_saddr6,
  32. const struct in6_addr *sk2_rcv_saddr6,
  33. __be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
  34. bool sk1_ipv6only, bool sk2_ipv6only,
  35. bool match_sk1_wildcard,
  36. bool match_sk2_wildcard)
  37. {
  38. int addr_type = ipv6_addr_type(sk1_rcv_saddr6);
  39. int addr_type2 = sk2_rcv_saddr6 ? ipv6_addr_type(sk2_rcv_saddr6) : IPV6_ADDR_MAPPED;
  40. /* if both are mapped, treat as IPv4 */
  41. if (addr_type == IPV6_ADDR_MAPPED && addr_type2 == IPV6_ADDR_MAPPED) {
  42. if (!sk2_ipv6only) {
  43. if (sk1_rcv_saddr == sk2_rcv_saddr)
  44. return true;
  45. return (match_sk1_wildcard && !sk1_rcv_saddr) ||
  46. (match_sk2_wildcard && !sk2_rcv_saddr);
  47. }
  48. return false;
  49. }
  50. if (addr_type == IPV6_ADDR_ANY && addr_type2 == IPV6_ADDR_ANY)
  51. return true;
  52. if (addr_type2 == IPV6_ADDR_ANY && match_sk2_wildcard &&
  53. !(sk2_ipv6only && addr_type == IPV6_ADDR_MAPPED))
  54. return true;
  55. if (addr_type == IPV6_ADDR_ANY && match_sk1_wildcard &&
  56. !(sk1_ipv6only && addr_type2 == IPV6_ADDR_MAPPED))
  57. return true;
  58. if (sk2_rcv_saddr6 &&
  59. ipv6_addr_equal(sk1_rcv_saddr6, sk2_rcv_saddr6))
  60. return true;
  61. return false;
  62. }
  63. #endif
  64. /* match_sk*_wildcard == true: 0.0.0.0 equals to any IPv4 addresses
  65. * match_sk*_wildcard == false: addresses must be exactly the same, i.e.
  66. * 0.0.0.0 only equals to 0.0.0.0
  67. */
  68. static bool ipv4_rcv_saddr_equal(__be32 sk1_rcv_saddr, __be32 sk2_rcv_saddr,
  69. bool sk2_ipv6only, bool match_sk1_wildcard,
  70. bool match_sk2_wildcard)
  71. {
  72. if (!sk2_ipv6only) {
  73. if (sk1_rcv_saddr == sk2_rcv_saddr)
  74. return true;
  75. return (match_sk1_wildcard && !sk1_rcv_saddr) ||
  76. (match_sk2_wildcard && !sk2_rcv_saddr);
  77. }
  78. return false;
  79. }
  80. bool inet_rcv_saddr_equal(const struct sock *sk, const struct sock *sk2,
  81. bool match_wildcard)
  82. {
  83. #if IS_ENABLED(CONFIG_IPV6)
  84. if (sk->sk_family == AF_INET6)
  85. return ipv6_rcv_saddr_equal(&sk->sk_v6_rcv_saddr,
  86. inet6_rcv_saddr(sk2),
  87. sk->sk_rcv_saddr,
  88. sk2->sk_rcv_saddr,
  89. ipv6_only_sock(sk),
  90. ipv6_only_sock(sk2),
  91. match_wildcard,
  92. match_wildcard);
  93. #endif
  94. return ipv4_rcv_saddr_equal(sk->sk_rcv_saddr, sk2->sk_rcv_saddr,
  95. ipv6_only_sock(sk2), match_wildcard,
  96. match_wildcard);
  97. }
  98. EXPORT_SYMBOL(inet_rcv_saddr_equal);
  99. bool inet_rcv_saddr_any(const struct sock *sk)
  100. {
  101. #if IS_ENABLED(CONFIG_IPV6)
  102. if (sk->sk_family == AF_INET6)
  103. return ipv6_addr_any(&sk->sk_v6_rcv_saddr);
  104. #endif
  105. return !sk->sk_rcv_saddr;
  106. }
  107. void inet_get_local_port_range(struct net *net, int *low, int *high)
  108. {
  109. unsigned int seq;
  110. do {
  111. seq = read_seqbegin(&net->ipv4.ip_local_ports.lock);
  112. *low = net->ipv4.ip_local_ports.range[0];
  113. *high = net->ipv4.ip_local_ports.range[1];
  114. } while (read_seqretry(&net->ipv4.ip_local_ports.lock, seq));
  115. }
  116. EXPORT_SYMBOL(inet_get_local_port_range);
  117. static int inet_csk_bind_conflict(const struct sock *sk,
  118. const struct inet_bind_bucket *tb,
  119. bool relax, bool reuseport_ok)
  120. {
  121. struct sock *sk2;
  122. bool reuse = sk->sk_reuse;
  123. bool reuseport = !!sk->sk_reuseport;
  124. kuid_t uid = sock_i_uid((struct sock *)sk);
  125. /*
  126. * Unlike other sk lookup places we do not check
  127. * for sk_net here, since _all_ the socks listed
  128. * in tb->owners list belong to the same net - the
  129. * one this bucket belongs to.
  130. */
  131. sk_for_each_bound(sk2, &tb->owners) {
  132. if (sk != sk2 &&
  133. (!sk->sk_bound_dev_if ||
  134. !sk2->sk_bound_dev_if ||
  135. sk->sk_bound_dev_if == sk2->sk_bound_dev_if)) {
  136. if (reuse && sk2->sk_reuse &&
  137. sk2->sk_state != TCP_LISTEN) {
  138. if ((!relax ||
  139. (!reuseport_ok &&
  140. reuseport && sk2->sk_reuseport &&
  141. !rcu_access_pointer(sk->sk_reuseport_cb) &&
  142. (sk2->sk_state == TCP_TIME_WAIT ||
  143. uid_eq(uid, sock_i_uid(sk2))))) &&
  144. inet_rcv_saddr_equal(sk, sk2, true))
  145. break;
  146. } else if (!reuseport_ok ||
  147. !reuseport || !sk2->sk_reuseport ||
  148. rcu_access_pointer(sk->sk_reuseport_cb) ||
  149. (sk2->sk_state != TCP_TIME_WAIT &&
  150. !uid_eq(uid, sock_i_uid(sk2)))) {
  151. if (inet_rcv_saddr_equal(sk, sk2, true))
  152. break;
  153. }
  154. }
  155. }
  156. return sk2 != NULL;
  157. }
  158. /*
  159. * Find an open port number for the socket. Returns with the
  160. * inet_bind_hashbucket lock held.
  161. */
  162. static struct inet_bind_hashbucket *
  163. inet_csk_find_open_port(struct sock *sk, struct inet_bind_bucket **tb_ret, int *port_ret)
  164. {
  165. struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
  166. int port = 0;
  167. struct inet_bind_hashbucket *head;
  168. struct net *net = sock_net(sk);
  169. bool relax = false;
  170. int i, low, high, attempt_half;
  171. struct inet_bind_bucket *tb;
  172. u32 remaining, offset;
  173. int l3mdev;
  174. l3mdev = inet_sk_bound_l3mdev(sk);
  175. ports_exhausted:
  176. attempt_half = (sk->sk_reuse == SK_CAN_REUSE) ? 1 : 0;
  177. other_half_scan:
  178. inet_get_local_port_range(net, &low, &high);
  179. high++; /* [32768, 60999] -> [32768, 61000[ */
  180. if (high - low < 4)
  181. attempt_half = 0;
  182. if (attempt_half) {
  183. int half = low + (((high - low) >> 2) << 1);
  184. if (attempt_half == 1)
  185. high = half;
  186. else
  187. low = half;
  188. }
  189. remaining = high - low;
  190. if (likely(remaining > 1))
  191. remaining &= ~1U;
  192. offset = prandom_u32() % remaining;
  193. /* __inet_hash_connect() favors ports having @low parity
  194. * We do the opposite to not pollute connect() users.
  195. */
  196. offset |= 1U;
  197. other_parity_scan:
  198. port = low + offset;
  199. for (i = 0; i < remaining; i += 2, port += 2) {
  200. if (unlikely(port >= high))
  201. port -= remaining;
  202. if (inet_is_local_reserved_port(net, port))
  203. continue;
  204. head = &hinfo->bhash[inet_bhashfn(net, port,
  205. hinfo->bhash_size)];
  206. spin_lock_bh(&head->lock);
  207. inet_bind_bucket_for_each(tb, &head->chain)
  208. if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
  209. tb->port == port) {
  210. if (!inet_csk_bind_conflict(sk, tb, relax, false))
  211. goto success;
  212. goto next_port;
  213. }
  214. tb = NULL;
  215. goto success;
  216. next_port:
  217. spin_unlock_bh(&head->lock);
  218. cond_resched();
  219. }
  220. offset--;
  221. if (!(offset & 1))
  222. goto other_parity_scan;
  223. if (attempt_half == 1) {
  224. /* OK we now try the upper half of the range */
  225. attempt_half = 2;
  226. goto other_half_scan;
  227. }
  228. if (net->ipv4.sysctl_ip_autobind_reuse && !relax) {
  229. /* We still have a chance to connect to different destinations */
  230. relax = true;
  231. goto ports_exhausted;
  232. }
  233. return NULL;
  234. success:
  235. *port_ret = port;
  236. *tb_ret = tb;
  237. return head;
  238. }
  239. static inline int sk_reuseport_match(struct inet_bind_bucket *tb,
  240. struct sock *sk)
  241. {
  242. kuid_t uid = sock_i_uid(sk);
  243. if (tb->fastreuseport <= 0)
  244. return 0;
  245. if (!sk->sk_reuseport)
  246. return 0;
  247. if (rcu_access_pointer(sk->sk_reuseport_cb))
  248. return 0;
  249. if (!uid_eq(tb->fastuid, uid))
  250. return 0;
  251. /* We only need to check the rcv_saddr if this tb was once marked
  252. * without fastreuseport and then was reset, as we can only know that
  253. * the fast_*rcv_saddr doesn't have any conflicts with the socks on the
  254. * owners list.
  255. */
  256. if (tb->fastreuseport == FASTREUSEPORT_ANY)
  257. return 1;
  258. #if IS_ENABLED(CONFIG_IPV6)
  259. if (tb->fast_sk_family == AF_INET6)
  260. return ipv6_rcv_saddr_equal(&tb->fast_v6_rcv_saddr,
  261. inet6_rcv_saddr(sk),
  262. tb->fast_rcv_saddr,
  263. sk->sk_rcv_saddr,
  264. tb->fast_ipv6_only,
  265. ipv6_only_sock(sk), true, false);
  266. #endif
  267. return ipv4_rcv_saddr_equal(tb->fast_rcv_saddr, sk->sk_rcv_saddr,
  268. ipv6_only_sock(sk), true, false);
  269. }
  270. void inet_csk_update_fastreuse(struct inet_bind_bucket *tb,
  271. struct sock *sk)
  272. {
  273. kuid_t uid = sock_i_uid(sk);
  274. bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
  275. if (hlist_empty(&tb->owners)) {
  276. tb->fastreuse = reuse;
  277. if (sk->sk_reuseport) {
  278. tb->fastreuseport = FASTREUSEPORT_ANY;
  279. tb->fastuid = uid;
  280. tb->fast_rcv_saddr = sk->sk_rcv_saddr;
  281. tb->fast_ipv6_only = ipv6_only_sock(sk);
  282. tb->fast_sk_family = sk->sk_family;
  283. #if IS_ENABLED(CONFIG_IPV6)
  284. tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
  285. #endif
  286. } else {
  287. tb->fastreuseport = 0;
  288. }
  289. } else {
  290. if (!reuse)
  291. tb->fastreuse = 0;
  292. if (sk->sk_reuseport) {
  293. /* We didn't match or we don't have fastreuseport set on
  294. * the tb, but we have sk_reuseport set on this socket
  295. * and we know that there are no bind conflicts with
  296. * this socket in this tb, so reset our tb's reuseport
  297. * settings so that any subsequent sockets that match
  298. * our current socket will be put on the fast path.
  299. *
  300. * If we reset we need to set FASTREUSEPORT_STRICT so we
  301. * do extra checking for all subsequent sk_reuseport
  302. * socks.
  303. */
  304. if (!sk_reuseport_match(tb, sk)) {
  305. tb->fastreuseport = FASTREUSEPORT_STRICT;
  306. tb->fastuid = uid;
  307. tb->fast_rcv_saddr = sk->sk_rcv_saddr;
  308. tb->fast_ipv6_only = ipv6_only_sock(sk);
  309. tb->fast_sk_family = sk->sk_family;
  310. #if IS_ENABLED(CONFIG_IPV6)
  311. tb->fast_v6_rcv_saddr = sk->sk_v6_rcv_saddr;
  312. #endif
  313. }
  314. } else {
  315. tb->fastreuseport = 0;
  316. }
  317. }
  318. }
  319. /* Obtain a reference to a local port for the given sock,
  320. * if snum is zero it means select any available local port.
  321. * We try to allocate an odd port (and leave even ports for connect())
  322. */
  323. int inet_csk_get_port(struct sock *sk, unsigned short snum)
  324. {
  325. bool reuse = sk->sk_reuse && sk->sk_state != TCP_LISTEN;
  326. struct inet_hashinfo *hinfo = sk->sk_prot->h.hashinfo;
  327. int ret = 1, port = snum;
  328. struct inet_bind_hashbucket *head;
  329. struct net *net = sock_net(sk);
  330. struct inet_bind_bucket *tb = NULL;
  331. int l3mdev;
  332. l3mdev = inet_sk_bound_l3mdev(sk);
  333. if (!port) {
  334. head = inet_csk_find_open_port(sk, &tb, &port);
  335. if (!head)
  336. return ret;
  337. if (!tb)
  338. goto tb_not_found;
  339. goto success;
  340. }
  341. head = &hinfo->bhash[inet_bhashfn(net, port,
  342. hinfo->bhash_size)];
  343. spin_lock_bh(&head->lock);
  344. inet_bind_bucket_for_each(tb, &head->chain)
  345. if (net_eq(ib_net(tb), net) && tb->l3mdev == l3mdev &&
  346. tb->port == port)
  347. goto tb_found;
  348. tb_not_found:
  349. tb = inet_bind_bucket_create(hinfo->bind_bucket_cachep,
  350. net, head, port, l3mdev);
  351. if (!tb)
  352. goto fail_unlock;
  353. tb_found:
  354. if (!hlist_empty(&tb->owners)) {
  355. if (sk->sk_reuse == SK_FORCE_REUSE)
  356. goto success;
  357. if ((tb->fastreuse > 0 && reuse) ||
  358. sk_reuseport_match(tb, sk))
  359. goto success;
  360. if (inet_csk_bind_conflict(sk, tb, true, true))
  361. goto fail_unlock;
  362. }
  363. success:
  364. inet_csk_update_fastreuse(tb, sk);
  365. if (!inet_csk(sk)->icsk_bind_hash)
  366. inet_bind_hash(sk, tb, port);
  367. WARN_ON(inet_csk(sk)->icsk_bind_hash != tb);
  368. ret = 0;
  369. fail_unlock:
  370. spin_unlock_bh(&head->lock);
  371. return ret;
  372. }
  373. EXPORT_SYMBOL_GPL(inet_csk_get_port);
  374. /*
  375. * Wait for an incoming connection, avoid race conditions. This must be called
  376. * with the socket locked.
  377. */
  378. static int inet_csk_wait_for_connect(struct sock *sk, long timeo)
  379. {
  380. struct inet_connection_sock *icsk = inet_csk(sk);
  381. DEFINE_WAIT(wait);
  382. int err;
  383. /*
  384. * True wake-one mechanism for incoming connections: only
  385. * one process gets woken up, not the 'whole herd'.
  386. * Since we do not 'race & poll' for established sockets
  387. * anymore, the common case will execute the loop only once.
  388. *
  389. * Subtle issue: "add_wait_queue_exclusive()" will be added
  390. * after any current non-exclusive waiters, and we know that
  391. * it will always _stay_ after any new non-exclusive waiters
  392. * because all non-exclusive waiters are added at the
  393. * beginning of the wait-queue. As such, it's ok to "drop"
  394. * our exclusiveness temporarily when we get woken up without
  395. * having to remove and re-insert us on the wait queue.
  396. */
  397. for (;;) {
  398. prepare_to_wait_exclusive(sk_sleep(sk), &wait,
  399. TASK_INTERRUPTIBLE);
  400. release_sock(sk);
  401. if (reqsk_queue_empty(&icsk->icsk_accept_queue))
  402. timeo = schedule_timeout(timeo);
  403. sched_annotate_sleep();
  404. lock_sock(sk);
  405. err = 0;
  406. if (!reqsk_queue_empty(&icsk->icsk_accept_queue))
  407. break;
  408. err = -EINVAL;
  409. if (sk->sk_state != TCP_LISTEN)
  410. break;
  411. err = sock_intr_errno(timeo);
  412. if (signal_pending(current))
  413. break;
  414. err = -EAGAIN;
  415. if (!timeo)
  416. break;
  417. }
  418. finish_wait(sk_sleep(sk), &wait);
  419. return err;
  420. }
  421. /*
  422. * This will accept the next outstanding connection.
  423. */
  424. struct sock *inet_csk_accept(struct sock *sk, int flags, int *err, bool kern)
  425. {
  426. struct inet_connection_sock *icsk = inet_csk(sk);
  427. struct request_sock_queue *queue = &icsk->icsk_accept_queue;
  428. struct request_sock *req;
  429. struct sock *newsk;
  430. int error;
  431. lock_sock(sk);
  432. /* We need to make sure that this socket is listening,
  433. * and that it has something pending.
  434. */
  435. error = -EINVAL;
  436. if (sk->sk_state != TCP_LISTEN)
  437. goto out_err;
  438. /* Find already established connection */
  439. if (reqsk_queue_empty(queue)) {
  440. long timeo = sock_rcvtimeo(sk, flags & O_NONBLOCK);
  441. /* If this is a non blocking socket don't sleep */
  442. error = -EAGAIN;
  443. if (!timeo)
  444. goto out_err;
  445. error = inet_csk_wait_for_connect(sk, timeo);
  446. if (error)
  447. goto out_err;
  448. }
  449. req = reqsk_queue_remove(queue, sk);
  450. newsk = req->sk;
  451. if (sk->sk_protocol == IPPROTO_TCP &&
  452. tcp_rsk(req)->tfo_listener) {
  453. spin_lock_bh(&queue->fastopenq.lock);
  454. if (tcp_rsk(req)->tfo_listener) {
  455. /* We are still waiting for the final ACK from 3WHS
  456. * so can't free req now. Instead, we set req->sk to
  457. * NULL to signify that the child socket is taken
  458. * so reqsk_fastopen_remove() will free the req
  459. * when 3WHS finishes (or is aborted).
  460. */
  461. req->sk = NULL;
  462. req = NULL;
  463. }
  464. spin_unlock_bh(&queue->fastopenq.lock);
  465. }
  466. out:
  467. release_sock(sk);
  468. if (newsk && mem_cgroup_sockets_enabled) {
  469. int amt;
  470. /* atomically get the memory usage, set and charge the
  471. * newsk->sk_memcg.
  472. */
  473. lock_sock(newsk);
  474. /* The socket has not been accepted yet, no need to look at
  475. * newsk->sk_wmem_queued.
  476. */
  477. amt = sk_mem_pages(newsk->sk_forward_alloc +
  478. atomic_read(&newsk->sk_rmem_alloc));
  479. mem_cgroup_sk_alloc(newsk);
  480. if (newsk->sk_memcg && amt)
  481. mem_cgroup_charge_skmem(newsk->sk_memcg, amt);
  482. release_sock(newsk);
  483. }
  484. if (req)
  485. reqsk_put(req);
  486. return newsk;
  487. out_err:
  488. newsk = NULL;
  489. req = NULL;
  490. *err = error;
  491. goto out;
  492. }
  493. EXPORT_SYMBOL(inet_csk_accept);
  494. /*
  495. * Using different timers for retransmit, delayed acks and probes
  496. * We may wish use just one timer maintaining a list of expire jiffies
  497. * to optimize.
  498. */
  499. void inet_csk_init_xmit_timers(struct sock *sk,
  500. void (*retransmit_handler)(struct timer_list *t),
  501. void (*delack_handler)(struct timer_list *t),
  502. void (*keepalive_handler)(struct timer_list *t))
  503. {
  504. struct inet_connection_sock *icsk = inet_csk(sk);
  505. timer_setup(&icsk->icsk_retransmit_timer, retransmit_handler, 0);
  506. timer_setup(&icsk->icsk_delack_timer, delack_handler, 0);
  507. timer_setup(&sk->sk_timer, keepalive_handler, 0);
  508. icsk->icsk_pending = icsk->icsk_ack.pending = 0;
  509. }
  510. EXPORT_SYMBOL(inet_csk_init_xmit_timers);
  511. void inet_csk_clear_xmit_timers(struct sock *sk)
  512. {
  513. struct inet_connection_sock *icsk = inet_csk(sk);
  514. icsk->icsk_pending = icsk->icsk_ack.pending = 0;
  515. sk_stop_timer(sk, &icsk->icsk_retransmit_timer);
  516. sk_stop_timer(sk, &icsk->icsk_delack_timer);
  517. sk_stop_timer(sk, &sk->sk_timer);
  518. }
  519. EXPORT_SYMBOL(inet_csk_clear_xmit_timers);
  520. void inet_csk_delete_keepalive_timer(struct sock *sk)
  521. {
  522. sk_stop_timer(sk, &sk->sk_timer);
  523. }
  524. EXPORT_SYMBOL(inet_csk_delete_keepalive_timer);
  525. void inet_csk_reset_keepalive_timer(struct sock *sk, unsigned long len)
  526. {
  527. sk_reset_timer(sk, &sk->sk_timer, jiffies + len);
  528. }
  529. EXPORT_SYMBOL(inet_csk_reset_keepalive_timer);
  530. struct dst_entry *inet_csk_route_req(const struct sock *sk,
  531. struct flowi4 *fl4,
  532. const struct request_sock *req)
  533. {
  534. const struct inet_request_sock *ireq = inet_rsk(req);
  535. struct net *net = read_pnet(&ireq->ireq_net);
  536. struct ip_options_rcu *opt;
  537. struct rtable *rt;
  538. rcu_read_lock();
  539. opt = rcu_dereference(ireq->ireq_opt);
  540. flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
  541. RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
  542. sk->sk_protocol, inet_sk_flowi_flags(sk),
  543. (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
  544. ireq->ir_loc_addr, ireq->ir_rmt_port,
  545. htons(ireq->ir_num), sk->sk_uid);
  546. security_req_classify_flow(req, flowi4_to_flowi(fl4));
  547. rt = ip_route_output_flow(net, fl4, sk);
  548. if (IS_ERR(rt))
  549. goto no_route;
  550. if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
  551. goto route_err;
  552. rcu_read_unlock();
  553. return &rt->dst;
  554. route_err:
  555. ip_rt_put(rt);
  556. no_route:
  557. rcu_read_unlock();
  558. __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
  559. return NULL;
  560. }
  561. EXPORT_SYMBOL_GPL(inet_csk_route_req);
  562. struct dst_entry *inet_csk_route_child_sock(const struct sock *sk,
  563. struct sock *newsk,
  564. const struct request_sock *req)
  565. {
  566. const struct inet_request_sock *ireq = inet_rsk(req);
  567. struct net *net = read_pnet(&ireq->ireq_net);
  568. struct inet_sock *newinet = inet_sk(newsk);
  569. struct ip_options_rcu *opt;
  570. struct flowi4 *fl4;
  571. struct rtable *rt;
  572. opt = rcu_dereference(ireq->ireq_opt);
  573. fl4 = &newinet->cork.fl.u.ip4;
  574. flowi4_init_output(fl4, ireq->ir_iif, ireq->ir_mark,
  575. RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
  576. sk->sk_protocol, inet_sk_flowi_flags(sk),
  577. (opt && opt->opt.srr) ? opt->opt.faddr : ireq->ir_rmt_addr,
  578. ireq->ir_loc_addr, ireq->ir_rmt_port,
  579. htons(ireq->ir_num), sk->sk_uid);
  580. security_req_classify_flow(req, flowi4_to_flowi(fl4));
  581. rt = ip_route_output_flow(net, fl4, sk);
  582. if (IS_ERR(rt))
  583. goto no_route;
  584. if (opt && opt->opt.is_strictroute && rt->rt_uses_gateway)
  585. goto route_err;
  586. return &rt->dst;
  587. route_err:
  588. ip_rt_put(rt);
  589. no_route:
  590. __IP_INC_STATS(net, IPSTATS_MIB_OUTNOROUTES);
  591. return NULL;
  592. }
  593. EXPORT_SYMBOL_GPL(inet_csk_route_child_sock);
  594. /* Decide when to expire the request and when to resend SYN-ACK */
  595. static void syn_ack_recalc(struct request_sock *req,
  596. const int max_syn_ack_retries,
  597. const u8 rskq_defer_accept,
  598. int *expire, int *resend)
  599. {
  600. if (!rskq_defer_accept) {
  601. *expire = req->num_timeout >= max_syn_ack_retries;
  602. *resend = 1;
  603. return;
  604. }
  605. *expire = req->num_timeout >= max_syn_ack_retries &&
  606. (!inet_rsk(req)->acked || req->num_timeout >= rskq_defer_accept);
  607. /* Do not resend while waiting for data after ACK,
  608. * start to resend on end of deferring period to give
  609. * last chance for data or ACK to create established socket.
  610. */
  611. *resend = !inet_rsk(req)->acked ||
  612. req->num_timeout >= rskq_defer_accept - 1;
  613. }
  614. int inet_rtx_syn_ack(const struct sock *parent, struct request_sock *req)
  615. {
  616. int err = req->rsk_ops->rtx_syn_ack(parent, req);
  617. if (!err)
  618. req->num_retrans++;
  619. return err;
  620. }
  621. EXPORT_SYMBOL(inet_rtx_syn_ack);
  622. /* return true if req was found in the ehash table */
  623. static bool reqsk_queue_unlink(struct request_sock *req)
  624. {
  625. struct inet_hashinfo *hashinfo = req_to_sk(req)->sk_prot->h.hashinfo;
  626. bool found = false;
  627. if (sk_hashed(req_to_sk(req))) {
  628. spinlock_t *lock = inet_ehash_lockp(hashinfo, req->rsk_hash);
  629. spin_lock(lock);
  630. found = __sk_nulls_del_node_init_rcu(req_to_sk(req));
  631. spin_unlock(lock);
  632. }
  633. if (timer_pending(&req->rsk_timer) && del_timer_sync(&req->rsk_timer))
  634. reqsk_put(req);
  635. return found;
  636. }
  637. bool inet_csk_reqsk_queue_drop(struct sock *sk, struct request_sock *req)
  638. {
  639. bool unlinked = reqsk_queue_unlink(req);
  640. if (unlinked) {
  641. reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
  642. reqsk_put(req);
  643. }
  644. return unlinked;
  645. }
  646. EXPORT_SYMBOL(inet_csk_reqsk_queue_drop);
  647. void inet_csk_reqsk_queue_drop_and_put(struct sock *sk, struct request_sock *req)
  648. {
  649. inet_csk_reqsk_queue_drop(sk, req);
  650. reqsk_put(req);
  651. }
  652. EXPORT_SYMBOL(inet_csk_reqsk_queue_drop_and_put);
  653. static void reqsk_timer_handler(struct timer_list *t)
  654. {
  655. struct request_sock *req = from_timer(req, t, rsk_timer);
  656. struct sock *sk_listener = req->rsk_listener;
  657. struct net *net = sock_net(sk_listener);
  658. struct inet_connection_sock *icsk = inet_csk(sk_listener);
  659. struct request_sock_queue *queue = &icsk->icsk_accept_queue;
  660. int max_syn_ack_retries, qlen, expire = 0, resend = 0;
  661. if (inet_sk_state_load(sk_listener) != TCP_LISTEN)
  662. goto drop;
  663. max_syn_ack_retries = icsk->icsk_syn_retries ? : net->ipv4.sysctl_tcp_synack_retries;
  664. /* Normally all the openreqs are young and become mature
  665. * (i.e. converted to established socket) for first timeout.
  666. * If synack was not acknowledged for 1 second, it means
  667. * one of the following things: synack was lost, ack was lost,
  668. * rtt is high or nobody planned to ack (i.e. synflood).
  669. * When server is a bit loaded, queue is populated with old
  670. * open requests, reducing effective size of queue.
  671. * When server is well loaded, queue size reduces to zero
  672. * after several minutes of work. It is not synflood,
  673. * it is normal operation. The solution is pruning
  674. * too old entries overriding normal timeout, when
  675. * situation becomes dangerous.
  676. *
  677. * Essentially, we reserve half of room for young
  678. * embrions; and abort old ones without pity, if old
  679. * ones are about to clog our table.
  680. */
  681. qlen = reqsk_queue_len(queue);
  682. if ((qlen << 1) > max(8U, READ_ONCE(sk_listener->sk_max_ack_backlog))) {
  683. int young = reqsk_queue_len_young(queue) << 1;
  684. while (max_syn_ack_retries > 2) {
  685. if (qlen < young)
  686. break;
  687. max_syn_ack_retries--;
  688. young <<= 1;
  689. }
  690. }
  691. syn_ack_recalc(req, max_syn_ack_retries, READ_ONCE(queue->rskq_defer_accept),
  692. &expire, &resend);
  693. req->rsk_ops->syn_ack_timeout(req);
  694. if (!expire &&
  695. (!resend ||
  696. !inet_rtx_syn_ack(sk_listener, req) ||
  697. inet_rsk(req)->acked)) {
  698. unsigned long timeo;
  699. if (req->num_timeout++ == 0)
  700. atomic_dec(&queue->young);
  701. timeo = min(TCP_TIMEOUT_INIT << req->num_timeout, TCP_RTO_MAX);
  702. mod_timer(&req->rsk_timer, jiffies + timeo);
  703. return;
  704. }
  705. drop:
  706. inet_csk_reqsk_queue_drop_and_put(sk_listener, req);
  707. }
  708. static void reqsk_queue_hash_req(struct request_sock *req,
  709. unsigned long timeout)
  710. {
  711. timer_setup(&req->rsk_timer, reqsk_timer_handler, TIMER_PINNED);
  712. mod_timer(&req->rsk_timer, jiffies + timeout);
  713. inet_ehash_insert(req_to_sk(req), NULL, NULL);
  714. /* before letting lookups find us, make sure all req fields
  715. * are committed to memory and refcnt initialized.
  716. */
  717. smp_wmb();
  718. refcount_set(&req->rsk_refcnt, 2 + 1);
  719. }
  720. void inet_csk_reqsk_queue_hash_add(struct sock *sk, struct request_sock *req,
  721. unsigned long timeout)
  722. {
  723. reqsk_queue_hash_req(req, timeout);
  724. inet_csk_reqsk_queue_added(sk);
  725. }
  726. EXPORT_SYMBOL_GPL(inet_csk_reqsk_queue_hash_add);
  727. static void inet_clone_ulp(const struct request_sock *req, struct sock *newsk,
  728. const gfp_t priority)
  729. {
  730. struct inet_connection_sock *icsk = inet_csk(newsk);
  731. if (!icsk->icsk_ulp_ops)
  732. return;
  733. if (icsk->icsk_ulp_ops->clone)
  734. icsk->icsk_ulp_ops->clone(req, newsk, priority);
  735. }
  736. /**
  737. * inet_csk_clone_lock - clone an inet socket, and lock its clone
  738. * @sk: the socket to clone
  739. * @req: request_sock
  740. * @priority: for allocation (%GFP_KERNEL, %GFP_ATOMIC, etc)
  741. *
  742. * Caller must unlock socket even in error path (bh_unlock_sock(newsk))
  743. */
  744. struct sock *inet_csk_clone_lock(const struct sock *sk,
  745. const struct request_sock *req,
  746. const gfp_t priority)
  747. {
  748. struct sock *newsk = sk_clone_lock(sk, priority);
  749. if (newsk) {
  750. struct inet_connection_sock *newicsk = inet_csk(newsk);
  751. inet_sk_set_state(newsk, TCP_SYN_RECV);
  752. newicsk->icsk_bind_hash = NULL;
  753. inet_sk(newsk)->inet_dport = inet_rsk(req)->ir_rmt_port;
  754. inet_sk(newsk)->inet_num = inet_rsk(req)->ir_num;
  755. inet_sk(newsk)->inet_sport = htons(inet_rsk(req)->ir_num);
  756. /* listeners have SOCK_RCU_FREE, not the children */
  757. sock_reset_flag(newsk, SOCK_RCU_FREE);
  758. inet_sk(newsk)->mc_list = NULL;
  759. newsk->sk_mark = inet_rsk(req)->ir_mark;
  760. atomic64_set(&newsk->sk_cookie,
  761. atomic64_read(&inet_rsk(req)->ir_cookie));
  762. newicsk->icsk_retransmits = 0;
  763. newicsk->icsk_backoff = 0;
  764. newicsk->icsk_probes_out = 0;
  765. newicsk->icsk_probes_tstamp = 0;
  766. /* Deinitialize accept_queue to trap illegal accesses. */
  767. memset(&newicsk->icsk_accept_queue, 0, sizeof(newicsk->icsk_accept_queue));
  768. inet_clone_ulp(req, newsk, priority);
  769. security_inet_csk_clone(newsk, req);
  770. }
  771. return newsk;
  772. }
  773. EXPORT_SYMBOL_GPL(inet_csk_clone_lock);
  774. /*
  775. * At this point, there should be no process reference to this
  776. * socket, and thus no user references at all. Therefore we
  777. * can assume the socket waitqueue is inactive and nobody will
  778. * try to jump onto it.
  779. */
  780. void inet_csk_destroy_sock(struct sock *sk)
  781. {
  782. WARN_ON(sk->sk_state != TCP_CLOSE);
  783. WARN_ON(!sock_flag(sk, SOCK_DEAD));
  784. /* It cannot be in hash table! */
  785. WARN_ON(!sk_unhashed(sk));
  786. /* If it has not 0 inet_sk(sk)->inet_num, it must be bound */
  787. WARN_ON(inet_sk(sk)->inet_num && !inet_csk(sk)->icsk_bind_hash);
  788. sk->sk_prot->destroy(sk);
  789. sk_stream_kill_queues(sk);
  790. xfrm_sk_free_policy(sk);
  791. sk_refcnt_debug_release(sk);
  792. percpu_counter_dec(sk->sk_prot->orphan_count);
  793. sock_put(sk);
  794. }
  795. EXPORT_SYMBOL(inet_csk_destroy_sock);
  796. /* This function allows to force a closure of a socket after the call to
  797. * tcp/dccp_create_openreq_child().
  798. */
  799. void inet_csk_prepare_forced_close(struct sock *sk)
  800. __releases(&sk->sk_lock.slock)
  801. {
  802. /* sk_clone_lock locked the socket and set refcnt to 2 */
  803. bh_unlock_sock(sk);
  804. sock_put(sk);
  805. inet_csk_prepare_for_destroy_sock(sk);
  806. inet_sk(sk)->inet_num = 0;
  807. }
  808. EXPORT_SYMBOL(inet_csk_prepare_forced_close);
  809. int inet_csk_listen_start(struct sock *sk, int backlog)
  810. {
  811. struct inet_connection_sock *icsk = inet_csk(sk);
  812. struct inet_sock *inet = inet_sk(sk);
  813. int err = -EADDRINUSE;
  814. reqsk_queue_alloc(&icsk->icsk_accept_queue);
  815. sk->sk_ack_backlog = 0;
  816. inet_csk_delack_init(sk);
  817. /* There is race window here: we announce ourselves listening,
  818. * but this transition is still not validated by get_port().
  819. * It is OK, because this socket enters to hash table only
  820. * after validation is complete.
  821. */
  822. inet_sk_state_store(sk, TCP_LISTEN);
  823. if (!sk->sk_prot->get_port(sk, inet->inet_num)) {
  824. inet->inet_sport = htons(inet->inet_num);
  825. sk_dst_reset(sk);
  826. err = sk->sk_prot->hash(sk);
  827. if (likely(!err))
  828. return 0;
  829. }
  830. inet_sk_set_state(sk, TCP_CLOSE);
  831. return err;
  832. }
  833. EXPORT_SYMBOL_GPL(inet_csk_listen_start);
  834. static void inet_child_forget(struct sock *sk, struct request_sock *req,
  835. struct sock *child)
  836. {
  837. sk->sk_prot->disconnect(child, O_NONBLOCK);
  838. sock_orphan(child);
  839. percpu_counter_inc(sk->sk_prot->orphan_count);
  840. if (sk->sk_protocol == IPPROTO_TCP && tcp_rsk(req)->tfo_listener) {
  841. BUG_ON(rcu_access_pointer(tcp_sk(child)->fastopen_rsk) != req);
  842. BUG_ON(sk != req->rsk_listener);
  843. /* Paranoid, to prevent race condition if
  844. * an inbound pkt destined for child is
  845. * blocked by sock lock in tcp_v4_rcv().
  846. * Also to satisfy an assertion in
  847. * tcp_v4_destroy_sock().
  848. */
  849. RCU_INIT_POINTER(tcp_sk(child)->fastopen_rsk, NULL);
  850. }
  851. inet_csk_destroy_sock(child);
  852. }
  853. struct sock *inet_csk_reqsk_queue_add(struct sock *sk,
  854. struct request_sock *req,
  855. struct sock *child)
  856. {
  857. struct request_sock_queue *queue = &inet_csk(sk)->icsk_accept_queue;
  858. spin_lock(&queue->rskq_lock);
  859. if (unlikely(sk->sk_state != TCP_LISTEN)) {
  860. inet_child_forget(sk, req, child);
  861. child = NULL;
  862. } else {
  863. req->sk = child;
  864. req->dl_next = NULL;
  865. if (queue->rskq_accept_head == NULL)
  866. WRITE_ONCE(queue->rskq_accept_head, req);
  867. else
  868. queue->rskq_accept_tail->dl_next = req;
  869. queue->rskq_accept_tail = req;
  870. sk_acceptq_added(sk);
  871. }
  872. spin_unlock(&queue->rskq_lock);
  873. return child;
  874. }
  875. EXPORT_SYMBOL(inet_csk_reqsk_queue_add);
  876. struct sock *inet_csk_complete_hashdance(struct sock *sk, struct sock *child,
  877. struct request_sock *req, bool own_req)
  878. {
  879. if (own_req) {
  880. inet_csk_reqsk_queue_drop(sk, req);
  881. reqsk_queue_removed(&inet_csk(sk)->icsk_accept_queue, req);
  882. if (inet_csk_reqsk_queue_add(sk, req, child))
  883. return child;
  884. }
  885. /* Too bad, another child took ownership of the request, undo. */
  886. bh_unlock_sock(child);
  887. sock_put(child);
  888. return NULL;
  889. }
  890. EXPORT_SYMBOL(inet_csk_complete_hashdance);
  891. /*
  892. * This routine closes sockets which have been at least partially
  893. * opened, but not yet accepted.
  894. */
  895. void inet_csk_listen_stop(struct sock *sk)
  896. {
  897. struct inet_connection_sock *icsk = inet_csk(sk);
  898. struct request_sock_queue *queue = &icsk->icsk_accept_queue;
  899. struct request_sock *next, *req;
  900. /* Following specs, it would be better either to send FIN
  901. * (and enter FIN-WAIT-1, it is normal close)
  902. * or to send active reset (abort).
  903. * Certainly, it is pretty dangerous while synflood, but it is
  904. * bad justification for our negligence 8)
  905. * To be honest, we are not able to make either
  906. * of the variants now. --ANK
  907. */
  908. while ((req = reqsk_queue_remove(queue, sk)) != NULL) {
  909. struct sock *child = req->sk;
  910. local_bh_disable();
  911. bh_lock_sock(child);
  912. WARN_ON(sock_owned_by_user(child));
  913. sock_hold(child);
  914. inet_child_forget(sk, req, child);
  915. reqsk_put(req);
  916. bh_unlock_sock(child);
  917. local_bh_enable();
  918. sock_put(child);
  919. cond_resched();
  920. }
  921. if (queue->fastopenq.rskq_rst_head) {
  922. /* Free all the reqs queued in rskq_rst_head. */
  923. spin_lock_bh(&queue->fastopenq.lock);
  924. req = queue->fastopenq.rskq_rst_head;
  925. queue->fastopenq.rskq_rst_head = NULL;
  926. spin_unlock_bh(&queue->fastopenq.lock);
  927. while (req != NULL) {
  928. next = req->dl_next;
  929. reqsk_put(req);
  930. req = next;
  931. }
  932. }
  933. WARN_ON_ONCE(sk->sk_ack_backlog);
  934. }
  935. EXPORT_SYMBOL_GPL(inet_csk_listen_stop);
  936. void inet_csk_addr2sockaddr(struct sock *sk, struct sockaddr *uaddr)
  937. {
  938. struct sockaddr_in *sin = (struct sockaddr_in *)uaddr;
  939. const struct inet_sock *inet = inet_sk(sk);
  940. sin->sin_family = AF_INET;
  941. sin->sin_addr.s_addr = inet->inet_daddr;
  942. sin->sin_port = inet->inet_dport;
  943. }
  944. EXPORT_SYMBOL_GPL(inet_csk_addr2sockaddr);
  945. static struct dst_entry *inet_csk_rebuild_route(struct sock *sk, struct flowi *fl)
  946. {
  947. const struct inet_sock *inet = inet_sk(sk);
  948. const struct ip_options_rcu *inet_opt;
  949. __be32 daddr = inet->inet_daddr;
  950. struct flowi4 *fl4;
  951. struct rtable *rt;
  952. rcu_read_lock();
  953. inet_opt = rcu_dereference(inet->inet_opt);
  954. if (inet_opt && inet_opt->opt.srr)
  955. daddr = inet_opt->opt.faddr;
  956. fl4 = &fl->u.ip4;
  957. rt = ip_route_output_ports(sock_net(sk), fl4, sk, daddr,
  958. inet->inet_saddr, inet->inet_dport,
  959. inet->inet_sport, sk->sk_protocol,
  960. RT_CONN_FLAGS(sk), sk->sk_bound_dev_if);
  961. if (IS_ERR(rt))
  962. rt = NULL;
  963. if (rt)
  964. sk_setup_caps(sk, &rt->dst);
  965. rcu_read_unlock();
  966. return &rt->dst;
  967. }
  968. struct dst_entry *inet_csk_update_pmtu(struct sock *sk, u32 mtu)
  969. {
  970. struct dst_entry *dst = __sk_dst_check(sk, 0);
  971. struct inet_sock *inet = inet_sk(sk);
  972. if (!dst) {
  973. dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
  974. if (!dst)
  975. goto out;
  976. }
  977. dst->ops->update_pmtu(dst, sk, NULL, mtu, true);
  978. dst = __sk_dst_check(sk, 0);
  979. if (!dst)
  980. dst = inet_csk_rebuild_route(sk, &inet->cork.fl);
  981. out:
  982. return dst;
  983. }
  984. EXPORT_SYMBOL_GPL(inet_csk_update_pmtu);