ip6_offload.c 11 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * IPV6 GSO/GRO offload support
  4. * Linux INET6 implementation
  5. */
  6. #include <linux/kernel.h>
  7. #include <linux/socket.h>
  8. #include <linux/netdevice.h>
  9. #include <linux/skbuff.h>
  10. #include <linux/printk.h>
  11. #include <net/protocol.h>
  12. #include <net/ipv6.h>
  13. #include <net/inet_common.h>
  14. #include <net/tcp.h>
  15. #include <net/udp.h>
  16. #include "ip6_offload.h"
  17. /* All GRO functions are always builtin, except UDP over ipv6, which lays in
  18. * ipv6 module, as it depends on UDPv6 lookup function, so we need special care
  19. * when ipv6 is built as a module
  20. */
  21. #if IS_BUILTIN(CONFIG_IPV6)
  22. #define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_2(f, f2, f1, __VA_ARGS__)
  23. #else
  24. #define INDIRECT_CALL_L4(f, f2, f1, ...) INDIRECT_CALL_1(f, f2, __VA_ARGS__)
  25. #endif
  26. #define indirect_call_gro_receive_l4(f2, f1, cb, head, skb) \
  27. ({ \
  28. unlikely(gro_recursion_inc_test(skb)) ? \
  29. NAPI_GRO_CB(skb)->flush |= 1, NULL : \
  30. INDIRECT_CALL_L4(cb, f2, f1, head, skb); \
  31. })
  32. static int ipv6_gso_pull_exthdrs(struct sk_buff *skb, int proto)
  33. {
  34. const struct net_offload *ops = NULL;
  35. for (;;) {
  36. struct ipv6_opt_hdr *opth;
  37. int len;
  38. if (proto != NEXTHDR_HOP) {
  39. ops = rcu_dereference(inet6_offloads[proto]);
  40. if (unlikely(!ops))
  41. break;
  42. if (!(ops->flags & INET6_PROTO_GSO_EXTHDR))
  43. break;
  44. }
  45. if (unlikely(!pskb_may_pull(skb, 8)))
  46. break;
  47. opth = (void *)skb->data;
  48. len = ipv6_optlen(opth);
  49. if (unlikely(!pskb_may_pull(skb, len)))
  50. break;
  51. opth = (void *)skb->data;
  52. proto = opth->nexthdr;
  53. __skb_pull(skb, len);
  54. }
  55. return proto;
  56. }
  57. static struct sk_buff *ipv6_gso_segment(struct sk_buff *skb,
  58. netdev_features_t features)
  59. {
  60. struct sk_buff *segs = ERR_PTR(-EINVAL);
  61. struct ipv6hdr *ipv6h;
  62. const struct net_offload *ops;
  63. int proto;
  64. struct frag_hdr *fptr;
  65. unsigned int payload_len;
  66. u8 *prevhdr;
  67. int offset = 0;
  68. bool encap, udpfrag;
  69. int nhoff;
  70. bool gso_partial;
  71. skb_reset_network_header(skb);
  72. nhoff = skb_network_header(skb) - skb_mac_header(skb);
  73. if (unlikely(!pskb_may_pull(skb, sizeof(*ipv6h))))
  74. goto out;
  75. encap = SKB_GSO_CB(skb)->encap_level > 0;
  76. if (encap)
  77. features &= skb->dev->hw_enc_features;
  78. SKB_GSO_CB(skb)->encap_level += sizeof(*ipv6h);
  79. ipv6h = ipv6_hdr(skb);
  80. __skb_pull(skb, sizeof(*ipv6h));
  81. segs = ERR_PTR(-EPROTONOSUPPORT);
  82. proto = ipv6_gso_pull_exthdrs(skb, ipv6h->nexthdr);
  83. if (skb->encapsulation &&
  84. skb_shinfo(skb)->gso_type & (SKB_GSO_IPXIP4 | SKB_GSO_IPXIP6))
  85. udpfrag = proto == IPPROTO_UDP && encap &&
  86. (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
  87. else
  88. udpfrag = proto == IPPROTO_UDP && !skb->encapsulation &&
  89. (skb_shinfo(skb)->gso_type & SKB_GSO_UDP);
  90. ops = rcu_dereference(inet6_offloads[proto]);
  91. if (likely(ops && ops->callbacks.gso_segment)) {
  92. skb_reset_transport_header(skb);
  93. segs = ops->callbacks.gso_segment(skb, features);
  94. if (!segs)
  95. skb->network_header = skb_mac_header(skb) + nhoff - skb->head;
  96. }
  97. if (IS_ERR_OR_NULL(segs))
  98. goto out;
  99. gso_partial = !!(skb_shinfo(segs)->gso_type & SKB_GSO_PARTIAL);
  100. for (skb = segs; skb; skb = skb->next) {
  101. ipv6h = (struct ipv6hdr *)(skb_mac_header(skb) + nhoff);
  102. if (gso_partial && skb_is_gso(skb))
  103. payload_len = skb_shinfo(skb)->gso_size +
  104. SKB_GSO_CB(skb)->data_offset +
  105. skb->head - (unsigned char *)(ipv6h + 1);
  106. else
  107. payload_len = skb->len - nhoff - sizeof(*ipv6h);
  108. ipv6h->payload_len = htons(payload_len);
  109. skb->network_header = (u8 *)ipv6h - skb->head;
  110. skb_reset_mac_len(skb);
  111. if (udpfrag) {
  112. int err = ip6_find_1stfragopt(skb, &prevhdr);
  113. if (err < 0) {
  114. kfree_skb_list(segs);
  115. return ERR_PTR(err);
  116. }
  117. fptr = (struct frag_hdr *)((u8 *)ipv6h + err);
  118. fptr->frag_off = htons(offset);
  119. if (skb->next)
  120. fptr->frag_off |= htons(IP6_MF);
  121. offset += (ntohs(ipv6h->payload_len) -
  122. sizeof(struct frag_hdr));
  123. }
  124. if (encap)
  125. skb_reset_inner_headers(skb);
  126. }
  127. out:
  128. return segs;
  129. }
  130. /* Return the total length of all the extension hdrs, following the same
  131. * logic in ipv6_gso_pull_exthdrs() when parsing ext-hdrs.
  132. */
  133. static int ipv6_exthdrs_len(struct ipv6hdr *iph,
  134. const struct net_offload **opps)
  135. {
  136. struct ipv6_opt_hdr *opth = (void *)iph;
  137. int len = 0, proto, optlen = sizeof(*iph);
  138. proto = iph->nexthdr;
  139. for (;;) {
  140. if (proto != NEXTHDR_HOP) {
  141. *opps = rcu_dereference(inet6_offloads[proto]);
  142. if (unlikely(!(*opps)))
  143. break;
  144. if (!((*opps)->flags & INET6_PROTO_GSO_EXTHDR))
  145. break;
  146. }
  147. opth = (void *)opth + optlen;
  148. optlen = ipv6_optlen(opth);
  149. len += optlen;
  150. proto = opth->nexthdr;
  151. }
  152. return len;
  153. }
  154. INDIRECT_CALLABLE_SCOPE struct sk_buff *ipv6_gro_receive(struct list_head *head,
  155. struct sk_buff *skb)
  156. {
  157. const struct net_offload *ops;
  158. struct sk_buff *pp = NULL;
  159. struct sk_buff *p;
  160. struct ipv6hdr *iph;
  161. unsigned int nlen;
  162. unsigned int hlen;
  163. unsigned int off;
  164. u16 flush = 1;
  165. int proto;
  166. off = skb_gro_offset(skb);
  167. hlen = off + sizeof(*iph);
  168. iph = skb_gro_header_fast(skb, off);
  169. if (skb_gro_header_hard(skb, hlen)) {
  170. iph = skb_gro_header_slow(skb, hlen, off);
  171. if (unlikely(!iph))
  172. goto out;
  173. }
  174. skb_set_network_header(skb, off);
  175. skb_gro_pull(skb, sizeof(*iph));
  176. skb_set_transport_header(skb, skb_gro_offset(skb));
  177. flush += ntohs(iph->payload_len) != skb_gro_len(skb);
  178. rcu_read_lock();
  179. proto = iph->nexthdr;
  180. ops = rcu_dereference(inet6_offloads[proto]);
  181. if (!ops || !ops->callbacks.gro_receive) {
  182. __pskb_pull(skb, skb_gro_offset(skb));
  183. skb_gro_frag0_invalidate(skb);
  184. proto = ipv6_gso_pull_exthdrs(skb, proto);
  185. skb_gro_pull(skb, -skb_transport_offset(skb));
  186. skb_reset_transport_header(skb);
  187. __skb_push(skb, skb_gro_offset(skb));
  188. ops = rcu_dereference(inet6_offloads[proto]);
  189. if (!ops || !ops->callbacks.gro_receive)
  190. goto out_unlock;
  191. iph = ipv6_hdr(skb);
  192. }
  193. NAPI_GRO_CB(skb)->proto = proto;
  194. flush--;
  195. nlen = skb_network_header_len(skb);
  196. list_for_each_entry(p, head, list) {
  197. const struct ipv6hdr *iph2;
  198. __be32 first_word; /* <Version:4><Traffic_Class:8><Flow_Label:20> */
  199. if (!NAPI_GRO_CB(p)->same_flow)
  200. continue;
  201. iph2 = (struct ipv6hdr *)(p->data + off);
  202. first_word = *(__be32 *)iph ^ *(__be32 *)iph2;
  203. /* All fields must match except length and Traffic Class.
  204. * XXX skbs on the gro_list have all been parsed and pulled
  205. * already so we don't need to compare nlen
  206. * (nlen != (sizeof(*iph2) + ipv6_exthdrs_len(iph2, &ops)))
  207. * memcmp() alone below is sufficient, right?
  208. */
  209. if ((first_word & htonl(0xF00FFFFF)) ||
  210. !ipv6_addr_equal(&iph->saddr, &iph2->saddr) ||
  211. !ipv6_addr_equal(&iph->daddr, &iph2->daddr) ||
  212. *(u16 *)&iph->nexthdr != *(u16 *)&iph2->nexthdr) {
  213. not_same_flow:
  214. NAPI_GRO_CB(p)->same_flow = 0;
  215. continue;
  216. }
  217. if (unlikely(nlen > sizeof(struct ipv6hdr))) {
  218. if (memcmp(iph + 1, iph2 + 1,
  219. nlen - sizeof(struct ipv6hdr)))
  220. goto not_same_flow;
  221. }
  222. /* flush if Traffic Class fields are different */
  223. NAPI_GRO_CB(p)->flush |= !!(first_word & htonl(0x0FF00000));
  224. NAPI_GRO_CB(p)->flush |= flush;
  225. /* If the previous IP ID value was based on an atomic
  226. * datagram we can overwrite the value and ignore it.
  227. */
  228. if (NAPI_GRO_CB(skb)->is_atomic)
  229. NAPI_GRO_CB(p)->flush_id = 0;
  230. }
  231. NAPI_GRO_CB(skb)->is_atomic = true;
  232. NAPI_GRO_CB(skb)->flush |= flush;
  233. skb_gro_postpull_rcsum(skb, iph, nlen);
  234. pp = indirect_call_gro_receive_l4(tcp6_gro_receive, udp6_gro_receive,
  235. ops->callbacks.gro_receive, head, skb);
  236. out_unlock:
  237. rcu_read_unlock();
  238. out:
  239. skb_gro_flush_final(skb, pp, flush);
  240. return pp;
  241. }
  242. static struct sk_buff *sit_ip6ip6_gro_receive(struct list_head *head,
  243. struct sk_buff *skb)
  244. {
  245. /* Common GRO receive for SIT and IP6IP6 */
  246. if (NAPI_GRO_CB(skb)->encap_mark) {
  247. NAPI_GRO_CB(skb)->flush = 1;
  248. return NULL;
  249. }
  250. NAPI_GRO_CB(skb)->encap_mark = 1;
  251. return ipv6_gro_receive(head, skb);
  252. }
  253. static struct sk_buff *ip4ip6_gro_receive(struct list_head *head,
  254. struct sk_buff *skb)
  255. {
  256. /* Common GRO receive for SIT and IP6IP6 */
  257. if (NAPI_GRO_CB(skb)->encap_mark) {
  258. NAPI_GRO_CB(skb)->flush = 1;
  259. return NULL;
  260. }
  261. NAPI_GRO_CB(skb)->encap_mark = 1;
  262. return inet_gro_receive(head, skb);
  263. }
  264. INDIRECT_CALLABLE_SCOPE int ipv6_gro_complete(struct sk_buff *skb, int nhoff)
  265. {
  266. const struct net_offload *ops;
  267. struct ipv6hdr *iph = (struct ipv6hdr *)(skb->data + nhoff);
  268. int err = -ENOSYS;
  269. if (skb->encapsulation) {
  270. skb_set_inner_protocol(skb, cpu_to_be16(ETH_P_IPV6));
  271. skb_set_inner_network_header(skb, nhoff);
  272. }
  273. iph->payload_len = htons(skb->len - nhoff - sizeof(*iph));
  274. rcu_read_lock();
  275. nhoff += sizeof(*iph) + ipv6_exthdrs_len(iph, &ops);
  276. if (WARN_ON(!ops || !ops->callbacks.gro_complete))
  277. goto out_unlock;
  278. err = INDIRECT_CALL_L4(ops->callbacks.gro_complete, tcp6_gro_complete,
  279. udp6_gro_complete, skb, nhoff);
  280. out_unlock:
  281. rcu_read_unlock();
  282. return err;
  283. }
  284. static int sit_gro_complete(struct sk_buff *skb, int nhoff)
  285. {
  286. skb->encapsulation = 1;
  287. skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP4;
  288. return ipv6_gro_complete(skb, nhoff);
  289. }
  290. static int ip6ip6_gro_complete(struct sk_buff *skb, int nhoff)
  291. {
  292. skb->encapsulation = 1;
  293. skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
  294. return ipv6_gro_complete(skb, nhoff);
  295. }
  296. static int ip4ip6_gro_complete(struct sk_buff *skb, int nhoff)
  297. {
  298. skb->encapsulation = 1;
  299. skb_shinfo(skb)->gso_type |= SKB_GSO_IPXIP6;
  300. return inet_gro_complete(skb, nhoff);
  301. }
  302. static struct packet_offload ipv6_packet_offload __read_mostly = {
  303. .type = cpu_to_be16(ETH_P_IPV6),
  304. .callbacks = {
  305. .gso_segment = ipv6_gso_segment,
  306. .gro_receive = ipv6_gro_receive,
  307. .gro_complete = ipv6_gro_complete,
  308. },
  309. };
  310. static struct sk_buff *sit_gso_segment(struct sk_buff *skb,
  311. netdev_features_t features)
  312. {
  313. if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP4))
  314. return ERR_PTR(-EINVAL);
  315. return ipv6_gso_segment(skb, features);
  316. }
  317. static struct sk_buff *ip4ip6_gso_segment(struct sk_buff *skb,
  318. netdev_features_t features)
  319. {
  320. if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
  321. return ERR_PTR(-EINVAL);
  322. return inet_gso_segment(skb, features);
  323. }
  324. static struct sk_buff *ip6ip6_gso_segment(struct sk_buff *skb,
  325. netdev_features_t features)
  326. {
  327. if (!(skb_shinfo(skb)->gso_type & SKB_GSO_IPXIP6))
  328. return ERR_PTR(-EINVAL);
  329. return ipv6_gso_segment(skb, features);
  330. }
  331. static const struct net_offload sit_offload = {
  332. .callbacks = {
  333. .gso_segment = sit_gso_segment,
  334. .gro_receive = sit_ip6ip6_gro_receive,
  335. .gro_complete = sit_gro_complete,
  336. },
  337. };
  338. static const struct net_offload ip4ip6_offload = {
  339. .callbacks = {
  340. .gso_segment = ip4ip6_gso_segment,
  341. .gro_receive = ip4ip6_gro_receive,
  342. .gro_complete = ip4ip6_gro_complete,
  343. },
  344. };
  345. static const struct net_offload ip6ip6_offload = {
  346. .callbacks = {
  347. .gso_segment = ip6ip6_gso_segment,
  348. .gro_receive = sit_ip6ip6_gro_receive,
  349. .gro_complete = ip6ip6_gro_complete,
  350. },
  351. };
  352. static int __init ipv6_offload_init(void)
  353. {
  354. if (tcpv6_offload_init() < 0)
  355. pr_crit("%s: Cannot add TCP protocol offload\n", __func__);
  356. if (ipv6_exthdrs_offload_init() < 0)
  357. pr_crit("%s: Cannot add EXTHDRS protocol offload\n", __func__);
  358. dev_add_offload(&ipv6_packet_offload);
  359. inet_add_offload(&sit_offload, IPPROTO_IPV6);
  360. inet6_add_offload(&ip6ip6_offload, IPPROTO_IPV6);
  361. inet6_add_offload(&ip4ip6_offload, IPPROTO_IPIP);
  362. return 0;
  363. }
  364. fs_initcall(ipv6_offload_init);