udp_offload.c 5.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * IPV6 GSO/GRO offload support
  4. * Linux INET6 implementation
  5. *
  6. * UDPv6 GSO support
  7. */
  8. #include <linux/skbuff.h>
  9. #include <linux/netdevice.h>
  10. #include <linux/indirect_call_wrapper.h>
  11. #include <net/protocol.h>
  12. #include <net/ipv6.h>
  13. #include <net/udp.h>
  14. #include <net/ip6_checksum.h>
  15. #include "ip6_offload.h"
  16. static struct sk_buff *udp6_ufo_fragment(struct sk_buff *skb,
  17. netdev_features_t features)
  18. {
  19. struct sk_buff *segs = ERR_PTR(-EINVAL);
  20. unsigned int mss;
  21. unsigned int unfrag_ip6hlen, unfrag_len;
  22. struct frag_hdr *fptr;
  23. u8 *packet_start, *prevhdr;
  24. u8 nexthdr;
  25. u8 frag_hdr_sz = sizeof(struct frag_hdr);
  26. __wsum csum;
  27. int tnl_hlen;
  28. int err;
  29. mss = skb_shinfo(skb)->gso_size;
  30. if (unlikely(skb->len <= mss))
  31. goto out;
  32. if (skb->encapsulation && skb_shinfo(skb)->gso_type &
  33. (SKB_GSO_UDP_TUNNEL|SKB_GSO_UDP_TUNNEL_CSUM))
  34. segs = skb_udp_tunnel_segment(skb, features, true);
  35. else {
  36. const struct ipv6hdr *ipv6h;
  37. struct udphdr *uh;
  38. if (!(skb_shinfo(skb)->gso_type & (SKB_GSO_UDP | SKB_GSO_UDP_L4)))
  39. goto out;
  40. if (!pskb_may_pull(skb, sizeof(struct udphdr)))
  41. goto out;
  42. if (skb_shinfo(skb)->gso_type & SKB_GSO_UDP_L4)
  43. return __udp_gso_segment(skb, features, true);
  44. /* Do software UFO. Complete and fill in the UDP checksum as HW cannot
  45. * do checksum of UDP packets sent as multiple IP fragments.
  46. */
  47. uh = udp_hdr(skb);
  48. ipv6h = ipv6_hdr(skb);
  49. uh->check = 0;
  50. csum = skb_checksum(skb, 0, skb->len, 0);
  51. uh->check = udp_v6_check(skb->len, &ipv6h->saddr,
  52. &ipv6h->daddr, csum);
  53. if (uh->check == 0)
  54. uh->check = CSUM_MANGLED_0;
  55. skb->ip_summed = CHECKSUM_UNNECESSARY;
  56. /* If there is no outer header we can fake a checksum offload
  57. * due to the fact that we have already done the checksum in
  58. * software prior to segmenting the frame.
  59. */
  60. if (!skb->encap_hdr_csum)
  61. features |= NETIF_F_HW_CSUM;
  62. /* Check if there is enough headroom to insert fragment header. */
  63. tnl_hlen = skb_tnl_header_len(skb);
  64. if (skb->mac_header < (tnl_hlen + frag_hdr_sz)) {
  65. if (gso_pskb_expand_head(skb, tnl_hlen + frag_hdr_sz))
  66. goto out;
  67. }
  68. /* Find the unfragmentable header and shift it left by frag_hdr_sz
  69. * bytes to insert fragment header.
  70. */
  71. err = ip6_find_1stfragopt(skb, &prevhdr);
  72. if (err < 0)
  73. return ERR_PTR(err);
  74. unfrag_ip6hlen = err;
  75. nexthdr = *prevhdr;
  76. *prevhdr = NEXTHDR_FRAGMENT;
  77. unfrag_len = (skb_network_header(skb) - skb_mac_header(skb)) +
  78. unfrag_ip6hlen + tnl_hlen;
  79. packet_start = (u8 *) skb->head + SKB_GSO_CB(skb)->mac_offset;
  80. memmove(packet_start-frag_hdr_sz, packet_start, unfrag_len);
  81. SKB_GSO_CB(skb)->mac_offset -= frag_hdr_sz;
  82. skb->mac_header -= frag_hdr_sz;
  83. skb->network_header -= frag_hdr_sz;
  84. fptr = (struct frag_hdr *)(skb_network_header(skb) + unfrag_ip6hlen);
  85. fptr->nexthdr = nexthdr;
  86. fptr->reserved = 0;
  87. fptr->identification = ipv6_proxy_select_ident(dev_net(skb->dev), skb);
  88. /* Fragment the skb. ipv6 header and the remaining fields of the
  89. * fragment header are updated in ipv6_gso_segment()
  90. */
  91. segs = skb_segment(skb, features);
  92. }
  93. out:
  94. return segs;
  95. }
  96. static struct sock *udp6_gro_lookup_skb(struct sk_buff *skb, __be16 sport,
  97. __be16 dport)
  98. {
  99. const struct ipv6hdr *iph = skb_gro_network_header(skb);
  100. return __udp6_lib_lookup(dev_net(skb->dev), &iph->saddr, sport,
  101. &iph->daddr, dport, inet6_iif(skb),
  102. inet6_sdif(skb), &udp_table, NULL);
  103. }
  104. INDIRECT_CALLABLE_SCOPE
  105. struct sk_buff *udp6_gro_receive(struct list_head *head, struct sk_buff *skb)
  106. {
  107. struct udphdr *uh = udp_gro_udphdr(skb);
  108. struct sock *sk = NULL;
  109. struct sk_buff *pp;
  110. if (unlikely(!uh))
  111. goto flush;
  112. /* Don't bother verifying checksum if we're going to flush anyway. */
  113. if (NAPI_GRO_CB(skb)->flush)
  114. goto skip;
  115. if (skb_gro_checksum_validate_zero_check(skb, IPPROTO_UDP, uh->check,
  116. ip6_gro_compute_pseudo))
  117. goto flush;
  118. else if (uh->check)
  119. skb_gro_checksum_try_convert(skb, IPPROTO_UDP,
  120. ip6_gro_compute_pseudo);
  121. skip:
  122. NAPI_GRO_CB(skb)->is_ipv6 = 1;
  123. rcu_read_lock();
  124. if (static_branch_unlikely(&udpv6_encap_needed_key))
  125. sk = udp6_gro_lookup_skb(skb, uh->source, uh->dest);
  126. pp = udp_gro_receive(head, skb, uh, sk);
  127. rcu_read_unlock();
  128. return pp;
  129. flush:
  130. NAPI_GRO_CB(skb)->flush = 1;
  131. return NULL;
  132. }
  133. INDIRECT_CALLABLE_SCOPE int udp6_gro_complete(struct sk_buff *skb, int nhoff)
  134. {
  135. const struct ipv6hdr *ipv6h = ipv6_hdr(skb);
  136. struct udphdr *uh = (struct udphdr *)(skb->data + nhoff);
  137. if (NAPI_GRO_CB(skb)->is_flist) {
  138. uh->len = htons(skb->len - nhoff);
  139. skb_shinfo(skb)->gso_type |= (SKB_GSO_FRAGLIST|SKB_GSO_UDP_L4);
  140. skb_shinfo(skb)->gso_segs = NAPI_GRO_CB(skb)->count;
  141. if (skb->ip_summed == CHECKSUM_UNNECESSARY) {
  142. if (skb->csum_level < SKB_MAX_CSUM_LEVEL)
  143. skb->csum_level++;
  144. } else {
  145. skb->ip_summed = CHECKSUM_UNNECESSARY;
  146. skb->csum_level = 0;
  147. }
  148. return 0;
  149. }
  150. if (uh->check)
  151. uh->check = ~udp_v6_check(skb->len - nhoff, &ipv6h->saddr,
  152. &ipv6h->daddr, 0);
  153. return udp_gro_complete(skb, nhoff, udp6_lib_lookup_skb);
  154. }
  155. static const struct net_offload udpv6_offload = {
  156. .callbacks = {
  157. .gso_segment = udp6_ufo_fragment,
  158. .gro_receive = udp6_gro_receive,
  159. .gro_complete = udp6_gro_complete,
  160. },
  161. };
  162. int udpv6_offload_init(void)
  163. {
  164. return inet6_add_offload(&udpv6_offload, IPPROTO_UDP);
  165. }
  166. int udpv6_offload_exit(void)
  167. {
  168. return inet6_del_offload(&udpv6_offload, IPPROTO_UDP);
  169. }