inet6_hashtables.c 9.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * INET An implementation of the TCP/IP protocol suite for the LINUX
  4. * operating system. INET is implemented using the BSD Socket
  5. * interface as the means of communication with the user level.
  6. *
  7. * Generic INET6 transport hashtables
  8. *
  9. * Authors: Lotsa people, from code originally in tcp, generalised here
  10. * by Arnaldo Carvalho de Melo <acme@mandriva.com>
  11. */
  12. #include <linux/module.h>
  13. #include <linux/random.h>
  14. #include <net/addrconf.h>
  15. #include <net/inet_connection_sock.h>
  16. #include <net/inet_hashtables.h>
  17. #include <net/inet6_hashtables.h>
  18. #include <net/secure_seq.h>
  19. #include <net/ip.h>
  20. #include <net/sock_reuseport.h>
  21. extern struct inet_hashinfo tcp_hashinfo;
  22. u32 inet6_ehashfn(const struct net *net,
  23. const struct in6_addr *laddr, const u16 lport,
  24. const struct in6_addr *faddr, const __be16 fport)
  25. {
  26. static u32 inet6_ehash_secret __read_mostly;
  27. static u32 ipv6_hash_secret __read_mostly;
  28. u32 lhash, fhash;
  29. net_get_random_once(&inet6_ehash_secret, sizeof(inet6_ehash_secret));
  30. net_get_random_once(&ipv6_hash_secret, sizeof(ipv6_hash_secret));
  31. lhash = (__force u32)laddr->s6_addr32[3];
  32. fhash = __ipv6_addr_jhash(faddr, ipv6_hash_secret);
  33. return __inet6_ehashfn(lhash, lport, fhash, fport,
  34. inet6_ehash_secret + net_hash_mix(net));
  35. }
  36. /*
  37. * Sockets in TCP_CLOSE state are _always_ taken out of the hash, so
  38. * we need not check it for TCP lookups anymore, thanks Alexey. -DaveM
  39. *
  40. * The sockhash lock must be held as a reader here.
  41. */
  42. struct sock *__inet6_lookup_established(struct net *net,
  43. struct inet_hashinfo *hashinfo,
  44. const struct in6_addr *saddr,
  45. const __be16 sport,
  46. const struct in6_addr *daddr,
  47. const u16 hnum,
  48. const int dif, const int sdif)
  49. {
  50. struct sock *sk;
  51. const struct hlist_nulls_node *node;
  52. const __portpair ports = INET_COMBINED_PORTS(sport, hnum);
  53. /* Optimize here for direct hit, only listening connections can
  54. * have wildcards anyways.
  55. */
  56. unsigned int hash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
  57. unsigned int slot = hash & hashinfo->ehash_mask;
  58. struct inet_ehash_bucket *head = &hashinfo->ehash[slot];
  59. begin:
  60. sk_nulls_for_each_rcu(sk, node, &head->chain) {
  61. if (sk->sk_hash != hash)
  62. continue;
  63. if (!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))
  64. continue;
  65. if (unlikely(!refcount_inc_not_zero(&sk->sk_refcnt)))
  66. goto out;
  67. if (unlikely(!INET6_MATCH(sk, net, saddr, daddr, ports, dif, sdif))) {
  68. sock_gen_put(sk);
  69. goto begin;
  70. }
  71. goto found;
  72. }
  73. if (get_nulls_value(node) != slot)
  74. goto begin;
  75. out:
  76. sk = NULL;
  77. found:
  78. return sk;
  79. }
  80. EXPORT_SYMBOL(__inet6_lookup_established);
  81. static inline int compute_score(struct sock *sk, struct net *net,
  82. const unsigned short hnum,
  83. const struct in6_addr *daddr,
  84. const int dif, const int sdif)
  85. {
  86. int score = -1;
  87. if (net_eq(sock_net(sk), net) && inet_sk(sk)->inet_num == hnum &&
  88. sk->sk_family == PF_INET6) {
  89. if (!ipv6_addr_equal(&sk->sk_v6_rcv_saddr, daddr))
  90. return -1;
  91. if (!inet_sk_bound_dev_eq(net, sk->sk_bound_dev_if, dif, sdif))
  92. return -1;
  93. score = sk->sk_bound_dev_if ? 2 : 1;
  94. if (READ_ONCE(sk->sk_incoming_cpu) == raw_smp_processor_id())
  95. score++;
  96. }
  97. return score;
  98. }
  99. static inline struct sock *lookup_reuseport(struct net *net, struct sock *sk,
  100. struct sk_buff *skb, int doff,
  101. const struct in6_addr *saddr,
  102. __be16 sport,
  103. const struct in6_addr *daddr,
  104. unsigned short hnum)
  105. {
  106. struct sock *reuse_sk = NULL;
  107. u32 phash;
  108. if (sk->sk_reuseport) {
  109. phash = inet6_ehashfn(net, daddr, hnum, saddr, sport);
  110. reuse_sk = reuseport_select_sock(sk, phash, skb, doff);
  111. }
  112. return reuse_sk;
  113. }
  114. /* called with rcu_read_lock() */
  115. static struct sock *inet6_lhash2_lookup(struct net *net,
  116. struct inet_listen_hashbucket *ilb2,
  117. struct sk_buff *skb, int doff,
  118. const struct in6_addr *saddr,
  119. const __be16 sport, const struct in6_addr *daddr,
  120. const unsigned short hnum, const int dif, const int sdif)
  121. {
  122. struct inet_connection_sock *icsk;
  123. struct sock *sk, *result = NULL;
  124. int score, hiscore = 0;
  125. inet_lhash2_for_each_icsk_rcu(icsk, &ilb2->head) {
  126. sk = (struct sock *)icsk;
  127. score = compute_score(sk, net, hnum, daddr, dif, sdif);
  128. if (score > hiscore) {
  129. result = lookup_reuseport(net, sk, skb, doff,
  130. saddr, sport, daddr, hnum);
  131. if (result)
  132. return result;
  133. result = sk;
  134. hiscore = score;
  135. }
  136. }
  137. return result;
  138. }
  139. static inline struct sock *inet6_lookup_run_bpf(struct net *net,
  140. struct inet_hashinfo *hashinfo,
  141. struct sk_buff *skb, int doff,
  142. const struct in6_addr *saddr,
  143. const __be16 sport,
  144. const struct in6_addr *daddr,
  145. const u16 hnum)
  146. {
  147. struct sock *sk, *reuse_sk;
  148. bool no_reuseport;
  149. if (hashinfo != &tcp_hashinfo)
  150. return NULL; /* only TCP is supported */
  151. no_reuseport = bpf_sk_lookup_run_v6(net, IPPROTO_TCP,
  152. saddr, sport, daddr, hnum, &sk);
  153. if (no_reuseport || IS_ERR_OR_NULL(sk))
  154. return sk;
  155. reuse_sk = lookup_reuseport(net, sk, skb, doff, saddr, sport, daddr, hnum);
  156. if (reuse_sk)
  157. sk = reuse_sk;
  158. return sk;
  159. }
  160. struct sock *inet6_lookup_listener(struct net *net,
  161. struct inet_hashinfo *hashinfo,
  162. struct sk_buff *skb, int doff,
  163. const struct in6_addr *saddr,
  164. const __be16 sport, const struct in6_addr *daddr,
  165. const unsigned short hnum, const int dif, const int sdif)
  166. {
  167. struct inet_listen_hashbucket *ilb2;
  168. struct sock *result = NULL;
  169. unsigned int hash2;
  170. /* Lookup redirect from BPF */
  171. if (static_branch_unlikely(&bpf_sk_lookup_enabled)) {
  172. result = inet6_lookup_run_bpf(net, hashinfo, skb, doff,
  173. saddr, sport, daddr, hnum);
  174. if (result)
  175. goto done;
  176. }
  177. hash2 = ipv6_portaddr_hash(net, daddr, hnum);
  178. ilb2 = inet_lhash2_bucket(hashinfo, hash2);
  179. result = inet6_lhash2_lookup(net, ilb2, skb, doff,
  180. saddr, sport, daddr, hnum,
  181. dif, sdif);
  182. if (result)
  183. goto done;
  184. /* Lookup lhash2 with in6addr_any */
  185. hash2 = ipv6_portaddr_hash(net, &in6addr_any, hnum);
  186. ilb2 = inet_lhash2_bucket(hashinfo, hash2);
  187. result = inet6_lhash2_lookup(net, ilb2, skb, doff,
  188. saddr, sport, &in6addr_any, hnum,
  189. dif, sdif);
  190. done:
  191. if (IS_ERR(result))
  192. return NULL;
  193. return result;
  194. }
  195. EXPORT_SYMBOL_GPL(inet6_lookup_listener);
  196. struct sock *inet6_lookup(struct net *net, struct inet_hashinfo *hashinfo,
  197. struct sk_buff *skb, int doff,
  198. const struct in6_addr *saddr, const __be16 sport,
  199. const struct in6_addr *daddr, const __be16 dport,
  200. const int dif)
  201. {
  202. struct sock *sk;
  203. bool refcounted;
  204. sk = __inet6_lookup(net, hashinfo, skb, doff, saddr, sport, daddr,
  205. ntohs(dport), dif, 0, &refcounted);
  206. if (sk && !refcounted && !refcount_inc_not_zero(&sk->sk_refcnt))
  207. sk = NULL;
  208. return sk;
  209. }
  210. EXPORT_SYMBOL_GPL(inet6_lookup);
  211. static int __inet6_check_established(struct inet_timewait_death_row *death_row,
  212. struct sock *sk, const __u16 lport,
  213. struct inet_timewait_sock **twp)
  214. {
  215. struct inet_hashinfo *hinfo = death_row->hashinfo;
  216. struct inet_sock *inet = inet_sk(sk);
  217. const struct in6_addr *daddr = &sk->sk_v6_rcv_saddr;
  218. const struct in6_addr *saddr = &sk->sk_v6_daddr;
  219. const int dif = sk->sk_bound_dev_if;
  220. struct net *net = sock_net(sk);
  221. const int sdif = l3mdev_master_ifindex_by_index(net, dif);
  222. const __portpair ports = INET_COMBINED_PORTS(inet->inet_dport, lport);
  223. const unsigned int hash = inet6_ehashfn(net, daddr, lport, saddr,
  224. inet->inet_dport);
  225. struct inet_ehash_bucket *head = inet_ehash_bucket(hinfo, hash);
  226. spinlock_t *lock = inet_ehash_lockp(hinfo, hash);
  227. struct sock *sk2;
  228. const struct hlist_nulls_node *node;
  229. struct inet_timewait_sock *tw = NULL;
  230. spin_lock(lock);
  231. sk_nulls_for_each(sk2, node, &head->chain) {
  232. if (sk2->sk_hash != hash)
  233. continue;
  234. if (likely(INET6_MATCH(sk2, net, saddr, daddr, ports,
  235. dif, sdif))) {
  236. if (sk2->sk_state == TCP_TIME_WAIT) {
  237. tw = inet_twsk(sk2);
  238. if (twsk_unique(sk, sk2, twp))
  239. break;
  240. }
  241. goto not_unique;
  242. }
  243. }
  244. /* Must record num and sport now. Otherwise we will see
  245. * in hash table socket with a funny identity.
  246. */
  247. inet->inet_num = lport;
  248. inet->inet_sport = htons(lport);
  249. sk->sk_hash = hash;
  250. WARN_ON(!sk_unhashed(sk));
  251. __sk_nulls_add_node_rcu(sk, &head->chain);
  252. if (tw) {
  253. sk_nulls_del_node_init_rcu((struct sock *)tw);
  254. __NET_INC_STATS(net, LINUX_MIB_TIMEWAITRECYCLED);
  255. }
  256. spin_unlock(lock);
  257. sock_prot_inuse_add(sock_net(sk), sk->sk_prot, 1);
  258. if (twp) {
  259. *twp = tw;
  260. } else if (tw) {
  261. /* Silly. Should hash-dance instead... */
  262. inet_twsk_deschedule_put(tw);
  263. }
  264. return 0;
  265. not_unique:
  266. spin_unlock(lock);
  267. return -EADDRNOTAVAIL;
  268. }
  269. static u32 inet6_sk_port_offset(const struct sock *sk)
  270. {
  271. const struct inet_sock *inet = inet_sk(sk);
  272. return secure_ipv6_port_ephemeral(sk->sk_v6_rcv_saddr.s6_addr32,
  273. sk->sk_v6_daddr.s6_addr32,
  274. inet->inet_dport);
  275. }
  276. int inet6_hash_connect(struct inet_timewait_death_row *death_row,
  277. struct sock *sk)
  278. {
  279. u32 port_offset = 0;
  280. if (!inet_sk(sk)->inet_num)
  281. port_offset = inet6_sk_port_offset(sk);
  282. return __inet_hash_connect(death_row, sk, port_offset,
  283. __inet6_check_established);
  284. }
  285. EXPORT_SYMBOL_GPL(inet6_hash_connect);
  286. int inet6_hash(struct sock *sk)
  287. {
  288. int err = 0;
  289. if (sk->sk_state != TCP_CLOSE)
  290. err = __inet_hash(sk, NULL);
  291. return err;
  292. }
  293. EXPORT_SYMBOL_GPL(inet6_hash);