xfrm_input.c 19 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824
  1. // SPDX-License-Identifier: GPL-2.0
  2. /*
  3. * xfrm_input.c
  4. *
  5. * Changes:
  6. * YOSHIFUJI Hideaki @USAGI
  7. * Split up af-specific portion
  8. *
  9. */
  10. #include <linux/bottom_half.h>
  11. #include <linux/cache.h>
  12. #include <linux/interrupt.h>
  13. #include <linux/slab.h>
  14. #include <linux/module.h>
  15. #include <linux/netdevice.h>
  16. #include <linux/percpu.h>
  17. #include <net/dst.h>
  18. #include <net/ip.h>
  19. #include <net/xfrm.h>
  20. #include <net/ip_tunnels.h>
  21. #include <net/ip6_tunnel.h>
  22. #include "xfrm_inout.h"
  23. struct xfrm_trans_tasklet {
  24. struct tasklet_struct tasklet;
  25. struct sk_buff_head queue;
  26. };
  27. struct xfrm_trans_cb {
  28. union {
  29. struct inet_skb_parm h4;
  30. #if IS_ENABLED(CONFIG_IPV6)
  31. struct inet6_skb_parm h6;
  32. #endif
  33. } header;
  34. int (*finish)(struct net *net, struct sock *sk, struct sk_buff *skb);
  35. struct net *net;
  36. };
  37. #define XFRM_TRANS_SKB_CB(__skb) ((struct xfrm_trans_cb *)&((__skb)->cb[0]))
  38. static DEFINE_SPINLOCK(xfrm_input_afinfo_lock);
  39. static struct xfrm_input_afinfo const __rcu *xfrm_input_afinfo[2][AF_INET6 + 1];
  40. static struct gro_cells gro_cells;
  41. static struct net_device xfrm_napi_dev;
  42. static DEFINE_PER_CPU(struct xfrm_trans_tasklet, xfrm_trans_tasklet);
  43. int xfrm_input_register_afinfo(const struct xfrm_input_afinfo *afinfo)
  44. {
  45. int err = 0;
  46. if (WARN_ON(afinfo->family > AF_INET6))
  47. return -EAFNOSUPPORT;
  48. spin_lock_bh(&xfrm_input_afinfo_lock);
  49. if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family]))
  50. err = -EEXIST;
  51. else
  52. rcu_assign_pointer(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], afinfo);
  53. spin_unlock_bh(&xfrm_input_afinfo_lock);
  54. return err;
  55. }
  56. EXPORT_SYMBOL(xfrm_input_register_afinfo);
  57. int xfrm_input_unregister_afinfo(const struct xfrm_input_afinfo *afinfo)
  58. {
  59. int err = 0;
  60. spin_lock_bh(&xfrm_input_afinfo_lock);
  61. if (likely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family])) {
  62. if (unlikely(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family] != afinfo))
  63. err = -EINVAL;
  64. else
  65. RCU_INIT_POINTER(xfrm_input_afinfo[afinfo->is_ipip][afinfo->family], NULL);
  66. }
  67. spin_unlock_bh(&xfrm_input_afinfo_lock);
  68. synchronize_rcu();
  69. return err;
  70. }
  71. EXPORT_SYMBOL(xfrm_input_unregister_afinfo);
  72. static const struct xfrm_input_afinfo *xfrm_input_get_afinfo(u8 family, bool is_ipip)
  73. {
  74. const struct xfrm_input_afinfo *afinfo;
  75. if (WARN_ON_ONCE(family > AF_INET6))
  76. return NULL;
  77. rcu_read_lock();
  78. afinfo = rcu_dereference(xfrm_input_afinfo[is_ipip][family]);
  79. if (unlikely(!afinfo))
  80. rcu_read_unlock();
  81. return afinfo;
  82. }
  83. static int xfrm_rcv_cb(struct sk_buff *skb, unsigned int family, u8 protocol,
  84. int err)
  85. {
  86. bool is_ipip = (protocol == IPPROTO_IPIP || protocol == IPPROTO_IPV6);
  87. const struct xfrm_input_afinfo *afinfo;
  88. int ret;
  89. afinfo = xfrm_input_get_afinfo(family, is_ipip);
  90. if (!afinfo)
  91. return -EAFNOSUPPORT;
  92. ret = afinfo->callback(skb, protocol, err);
  93. rcu_read_unlock();
  94. return ret;
  95. }
  96. struct sec_path *secpath_set(struct sk_buff *skb)
  97. {
  98. struct sec_path *sp, *tmp = skb_ext_find(skb, SKB_EXT_SEC_PATH);
  99. sp = skb_ext_add(skb, SKB_EXT_SEC_PATH);
  100. if (!sp)
  101. return NULL;
  102. if (tmp) /* reused existing one (was COW'd if needed) */
  103. return sp;
  104. /* allocated new secpath */
  105. memset(sp->ovec, 0, sizeof(sp->ovec));
  106. sp->olen = 0;
  107. sp->len = 0;
  108. return sp;
  109. }
  110. EXPORT_SYMBOL(secpath_set);
  111. /* Fetch spi and seq from ipsec header */
  112. int xfrm_parse_spi(struct sk_buff *skb, u8 nexthdr, __be32 *spi, __be32 *seq)
  113. {
  114. int offset, offset_seq;
  115. int hlen;
  116. switch (nexthdr) {
  117. case IPPROTO_AH:
  118. hlen = sizeof(struct ip_auth_hdr);
  119. offset = offsetof(struct ip_auth_hdr, spi);
  120. offset_seq = offsetof(struct ip_auth_hdr, seq_no);
  121. break;
  122. case IPPROTO_ESP:
  123. hlen = sizeof(struct ip_esp_hdr);
  124. offset = offsetof(struct ip_esp_hdr, spi);
  125. offset_seq = offsetof(struct ip_esp_hdr, seq_no);
  126. break;
  127. case IPPROTO_COMP:
  128. if (!pskb_may_pull(skb, sizeof(struct ip_comp_hdr)))
  129. return -EINVAL;
  130. *spi = htonl(ntohs(*(__be16 *)(skb_transport_header(skb) + 2)));
  131. *seq = 0;
  132. return 0;
  133. default:
  134. return 1;
  135. }
  136. if (!pskb_may_pull(skb, hlen))
  137. return -EINVAL;
  138. *spi = *(__be32 *)(skb_transport_header(skb) + offset);
  139. *seq = *(__be32 *)(skb_transport_header(skb) + offset_seq);
  140. return 0;
  141. }
  142. EXPORT_SYMBOL(xfrm_parse_spi);
  143. static int xfrm4_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
  144. {
  145. struct iphdr *iph;
  146. int optlen = 0;
  147. int err = -EINVAL;
  148. if (unlikely(XFRM_MODE_SKB_CB(skb)->protocol == IPPROTO_BEETPH)) {
  149. struct ip_beet_phdr *ph;
  150. int phlen;
  151. if (!pskb_may_pull(skb, sizeof(*ph)))
  152. goto out;
  153. ph = (struct ip_beet_phdr *)skb->data;
  154. phlen = sizeof(*ph) + ph->padlen;
  155. optlen = ph->hdrlen * 8 + (IPV4_BEET_PHMAXLEN - phlen);
  156. if (optlen < 0 || optlen & 3 || optlen > 250)
  157. goto out;
  158. XFRM_MODE_SKB_CB(skb)->protocol = ph->nexthdr;
  159. if (!pskb_may_pull(skb, phlen))
  160. goto out;
  161. __skb_pull(skb, phlen);
  162. }
  163. skb_push(skb, sizeof(*iph));
  164. skb_reset_network_header(skb);
  165. skb_mac_header_rebuild(skb);
  166. xfrm4_beet_make_header(skb);
  167. iph = ip_hdr(skb);
  168. iph->ihl += optlen / 4;
  169. iph->tot_len = htons(skb->len);
  170. iph->daddr = x->sel.daddr.a4;
  171. iph->saddr = x->sel.saddr.a4;
  172. iph->check = 0;
  173. iph->check = ip_fast_csum(skb_network_header(skb), iph->ihl);
  174. err = 0;
  175. out:
  176. return err;
  177. }
  178. static void ipip_ecn_decapsulate(struct sk_buff *skb)
  179. {
  180. struct iphdr *inner_iph = ipip_hdr(skb);
  181. if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
  182. IP_ECN_set_ce(inner_iph);
  183. }
  184. static int xfrm4_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
  185. {
  186. int err = -EINVAL;
  187. if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPIP)
  188. goto out;
  189. if (!pskb_may_pull(skb, sizeof(struct iphdr)))
  190. goto out;
  191. err = skb_unclone(skb, GFP_ATOMIC);
  192. if (err)
  193. goto out;
  194. if (x->props.flags & XFRM_STATE_DECAP_DSCP)
  195. ipv4_copy_dscp(XFRM_MODE_SKB_CB(skb)->tos, ipip_hdr(skb));
  196. if (!(x->props.flags & XFRM_STATE_NOECN))
  197. ipip_ecn_decapsulate(skb);
  198. skb_reset_network_header(skb);
  199. skb_mac_header_rebuild(skb);
  200. if (skb->mac_len)
  201. eth_hdr(skb)->h_proto = skb->protocol;
  202. err = 0;
  203. out:
  204. return err;
  205. }
  206. static void ipip6_ecn_decapsulate(struct sk_buff *skb)
  207. {
  208. struct ipv6hdr *inner_iph = ipipv6_hdr(skb);
  209. if (INET_ECN_is_ce(XFRM_MODE_SKB_CB(skb)->tos))
  210. IP6_ECN_set_ce(skb, inner_iph);
  211. }
  212. static int xfrm6_remove_tunnel_encap(struct xfrm_state *x, struct sk_buff *skb)
  213. {
  214. int err = -EINVAL;
  215. if (XFRM_MODE_SKB_CB(skb)->protocol != IPPROTO_IPV6)
  216. goto out;
  217. if (!pskb_may_pull(skb, sizeof(struct ipv6hdr)))
  218. goto out;
  219. err = skb_unclone(skb, GFP_ATOMIC);
  220. if (err)
  221. goto out;
  222. if (x->props.flags & XFRM_STATE_DECAP_DSCP)
  223. ipv6_copy_dscp(ipv6_get_dsfield(ipv6_hdr(skb)),
  224. ipipv6_hdr(skb));
  225. if (!(x->props.flags & XFRM_STATE_NOECN))
  226. ipip6_ecn_decapsulate(skb);
  227. skb_reset_network_header(skb);
  228. skb_mac_header_rebuild(skb);
  229. if (skb->mac_len)
  230. eth_hdr(skb)->h_proto = skb->protocol;
  231. err = 0;
  232. out:
  233. return err;
  234. }
  235. static int xfrm6_remove_beet_encap(struct xfrm_state *x, struct sk_buff *skb)
  236. {
  237. struct ipv6hdr *ip6h;
  238. int size = sizeof(struct ipv6hdr);
  239. int err;
  240. err = skb_cow_head(skb, size + skb->mac_len);
  241. if (err)
  242. goto out;
  243. __skb_push(skb, size);
  244. skb_reset_network_header(skb);
  245. skb_mac_header_rebuild(skb);
  246. xfrm6_beet_make_header(skb);
  247. ip6h = ipv6_hdr(skb);
  248. ip6h->payload_len = htons(skb->len - size);
  249. ip6h->daddr = x->sel.daddr.in6;
  250. ip6h->saddr = x->sel.saddr.in6;
  251. err = 0;
  252. out:
  253. return err;
  254. }
  255. /* Remove encapsulation header.
  256. *
  257. * The IP header will be moved over the top of the encapsulation
  258. * header.
  259. *
  260. * On entry, the transport header shall point to where the IP header
  261. * should be and the network header shall be set to where the IP
  262. * header currently is. skb->data shall point to the start of the
  263. * payload.
  264. */
  265. static int
  266. xfrm_inner_mode_encap_remove(struct xfrm_state *x,
  267. const struct xfrm_mode *inner_mode,
  268. struct sk_buff *skb)
  269. {
  270. switch (inner_mode->encap) {
  271. case XFRM_MODE_BEET:
  272. if (inner_mode->family == AF_INET)
  273. return xfrm4_remove_beet_encap(x, skb);
  274. if (inner_mode->family == AF_INET6)
  275. return xfrm6_remove_beet_encap(x, skb);
  276. break;
  277. case XFRM_MODE_TUNNEL:
  278. if (inner_mode->family == AF_INET)
  279. return xfrm4_remove_tunnel_encap(x, skb);
  280. if (inner_mode->family == AF_INET6)
  281. return xfrm6_remove_tunnel_encap(x, skb);
  282. break;
  283. }
  284. WARN_ON_ONCE(1);
  285. return -EOPNOTSUPP;
  286. }
  287. static int xfrm_prepare_input(struct xfrm_state *x, struct sk_buff *skb)
  288. {
  289. const struct xfrm_mode *inner_mode = &x->inner_mode;
  290. switch (x->outer_mode.family) {
  291. case AF_INET:
  292. xfrm4_extract_header(skb);
  293. break;
  294. case AF_INET6:
  295. xfrm6_extract_header(skb);
  296. break;
  297. default:
  298. WARN_ON_ONCE(1);
  299. return -EAFNOSUPPORT;
  300. }
  301. if (x->sel.family == AF_UNSPEC) {
  302. inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
  303. if (!inner_mode)
  304. return -EAFNOSUPPORT;
  305. }
  306. switch (inner_mode->family) {
  307. case AF_INET:
  308. skb->protocol = htons(ETH_P_IP);
  309. break;
  310. case AF_INET6:
  311. skb->protocol = htons(ETH_P_IPV6);
  312. break;
  313. default:
  314. WARN_ON_ONCE(1);
  315. break;
  316. }
  317. return xfrm_inner_mode_encap_remove(x, inner_mode, skb);
  318. }
  319. /* Remove encapsulation header.
  320. *
  321. * The IP header will be moved over the top of the encapsulation header.
  322. *
  323. * On entry, skb_transport_header() shall point to where the IP header
  324. * should be and skb_network_header() shall be set to where the IP header
  325. * currently is. skb->data shall point to the start of the payload.
  326. */
  327. static int xfrm4_transport_input(struct xfrm_state *x, struct sk_buff *skb)
  328. {
  329. int ihl = skb->data - skb_transport_header(skb);
  330. if (skb->transport_header != skb->network_header) {
  331. memmove(skb_transport_header(skb),
  332. skb_network_header(skb), ihl);
  333. skb->network_header = skb->transport_header;
  334. }
  335. ip_hdr(skb)->tot_len = htons(skb->len + ihl);
  336. skb_reset_transport_header(skb);
  337. return 0;
  338. }
  339. static int xfrm6_transport_input(struct xfrm_state *x, struct sk_buff *skb)
  340. {
  341. #if IS_ENABLED(CONFIG_IPV6)
  342. int ihl = skb->data - skb_transport_header(skb);
  343. if (skb->transport_header != skb->network_header) {
  344. memmove(skb_transport_header(skb),
  345. skb_network_header(skb), ihl);
  346. skb->network_header = skb->transport_header;
  347. }
  348. ipv6_hdr(skb)->payload_len = htons(skb->len + ihl -
  349. sizeof(struct ipv6hdr));
  350. skb_reset_transport_header(skb);
  351. return 0;
  352. #else
  353. WARN_ON_ONCE(1);
  354. return -EAFNOSUPPORT;
  355. #endif
  356. }
  357. static int xfrm_inner_mode_input(struct xfrm_state *x,
  358. const struct xfrm_mode *inner_mode,
  359. struct sk_buff *skb)
  360. {
  361. switch (inner_mode->encap) {
  362. case XFRM_MODE_BEET:
  363. case XFRM_MODE_TUNNEL:
  364. return xfrm_prepare_input(x, skb);
  365. case XFRM_MODE_TRANSPORT:
  366. if (inner_mode->family == AF_INET)
  367. return xfrm4_transport_input(x, skb);
  368. if (inner_mode->family == AF_INET6)
  369. return xfrm6_transport_input(x, skb);
  370. break;
  371. case XFRM_MODE_ROUTEOPTIMIZATION:
  372. WARN_ON_ONCE(1);
  373. break;
  374. default:
  375. WARN_ON_ONCE(1);
  376. break;
  377. }
  378. return -EOPNOTSUPP;
  379. }
  380. int xfrm_input(struct sk_buff *skb, int nexthdr, __be32 spi, int encap_type)
  381. {
  382. const struct xfrm_state_afinfo *afinfo;
  383. struct net *net = dev_net(skb->dev);
  384. const struct xfrm_mode *inner_mode;
  385. int err;
  386. __be32 seq;
  387. __be32 seq_hi;
  388. struct xfrm_state *x = NULL;
  389. xfrm_address_t *daddr;
  390. u32 mark = skb->mark;
  391. unsigned int family = AF_UNSPEC;
  392. int decaps = 0;
  393. int async = 0;
  394. bool xfrm_gro = false;
  395. bool crypto_done = false;
  396. struct xfrm_offload *xo = xfrm_offload(skb);
  397. struct sec_path *sp;
  398. if (encap_type < 0) {
  399. x = xfrm_input_state(skb);
  400. if (unlikely(x->km.state != XFRM_STATE_VALID)) {
  401. if (x->km.state == XFRM_STATE_ACQ)
  402. XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
  403. else
  404. XFRM_INC_STATS(net,
  405. LINUX_MIB_XFRMINSTATEINVALID);
  406. if (encap_type == -1)
  407. dev_put(skb->dev);
  408. goto drop;
  409. }
  410. family = x->outer_mode.family;
  411. /* An encap_type of -1 indicates async resumption. */
  412. if (encap_type == -1) {
  413. async = 1;
  414. seq = XFRM_SKB_CB(skb)->seq.input.low;
  415. goto resume;
  416. }
  417. /* encap_type < -1 indicates a GRO call. */
  418. encap_type = 0;
  419. seq = XFRM_SPI_SKB_CB(skb)->seq;
  420. if (xo && (xo->flags & CRYPTO_DONE)) {
  421. crypto_done = true;
  422. family = XFRM_SPI_SKB_CB(skb)->family;
  423. if (!(xo->status & CRYPTO_SUCCESS)) {
  424. if (xo->status &
  425. (CRYPTO_TRANSPORT_AH_AUTH_FAILED |
  426. CRYPTO_TRANSPORT_ESP_AUTH_FAILED |
  427. CRYPTO_TUNNEL_AH_AUTH_FAILED |
  428. CRYPTO_TUNNEL_ESP_AUTH_FAILED)) {
  429. xfrm_audit_state_icvfail(x, skb,
  430. x->type->proto);
  431. x->stats.integrity_failed++;
  432. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
  433. goto drop;
  434. }
  435. if (xo->status & CRYPTO_INVALID_PROTOCOL) {
  436. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
  437. goto drop;
  438. }
  439. XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
  440. goto drop;
  441. }
  442. if ((err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
  443. XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
  444. goto drop;
  445. }
  446. }
  447. goto lock;
  448. }
  449. family = XFRM_SPI_SKB_CB(skb)->family;
  450. /* if tunnel is present override skb->mark value with tunnel i_key */
  451. switch (family) {
  452. case AF_INET:
  453. if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4)
  454. mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip4->parms.i_key);
  455. break;
  456. case AF_INET6:
  457. if (XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6)
  458. mark = be32_to_cpu(XFRM_TUNNEL_SKB_CB(skb)->tunnel.ip6->parms.i_key);
  459. break;
  460. }
  461. sp = secpath_set(skb);
  462. if (!sp) {
  463. XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
  464. goto drop;
  465. }
  466. seq = 0;
  467. if (!spi && (err = xfrm_parse_spi(skb, nexthdr, &spi, &seq)) != 0) {
  468. secpath_reset(skb);
  469. XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
  470. goto drop;
  471. }
  472. daddr = (xfrm_address_t *)(skb_network_header(skb) +
  473. XFRM_SPI_SKB_CB(skb)->daddroff);
  474. do {
  475. sp = skb_sec_path(skb);
  476. if (sp->len == XFRM_MAX_DEPTH) {
  477. secpath_reset(skb);
  478. XFRM_INC_STATS(net, LINUX_MIB_XFRMINBUFFERERROR);
  479. goto drop;
  480. }
  481. x = xfrm_state_lookup(net, mark, daddr, spi, nexthdr, family);
  482. if (x == NULL) {
  483. secpath_reset(skb);
  484. XFRM_INC_STATS(net, LINUX_MIB_XFRMINNOSTATES);
  485. xfrm_audit_state_notfound(skb, family, spi, seq);
  486. goto drop;
  487. }
  488. skb->mark = xfrm_smark_get(skb->mark, x);
  489. sp->xvec[sp->len++] = x;
  490. skb_dst_force(skb);
  491. if (!skb_dst(skb)) {
  492. XFRM_INC_STATS(net, LINUX_MIB_XFRMINERROR);
  493. goto drop;
  494. }
  495. lock:
  496. spin_lock(&x->lock);
  497. if (unlikely(x->km.state != XFRM_STATE_VALID)) {
  498. if (x->km.state == XFRM_STATE_ACQ)
  499. XFRM_INC_STATS(net, LINUX_MIB_XFRMACQUIREERROR);
  500. else
  501. XFRM_INC_STATS(net,
  502. LINUX_MIB_XFRMINSTATEINVALID);
  503. goto drop_unlock;
  504. }
  505. if ((x->encap ? x->encap->encap_type : 0) != encap_type) {
  506. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMISMATCH);
  507. goto drop_unlock;
  508. }
  509. if (x->repl->check(x, skb, seq)) {
  510. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
  511. goto drop_unlock;
  512. }
  513. if (xfrm_state_check_expire(x)) {
  514. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEEXPIRED);
  515. goto drop_unlock;
  516. }
  517. spin_unlock(&x->lock);
  518. if (xfrm_tunnel_check(skb, x, family)) {
  519. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
  520. goto drop;
  521. }
  522. seq_hi = htonl(xfrm_replay_seqhi(x, seq));
  523. XFRM_SKB_CB(skb)->seq.input.low = seq;
  524. XFRM_SKB_CB(skb)->seq.input.hi = seq_hi;
  525. dev_hold(skb->dev);
  526. if (crypto_done)
  527. nexthdr = x->type_offload->input_tail(x, skb);
  528. else
  529. nexthdr = x->type->input(x, skb);
  530. if (nexthdr == -EINPROGRESS)
  531. return 0;
  532. resume:
  533. dev_put(skb->dev);
  534. spin_lock(&x->lock);
  535. if (nexthdr < 0) {
  536. if (nexthdr == -EBADMSG) {
  537. xfrm_audit_state_icvfail(x, skb,
  538. x->type->proto);
  539. x->stats.integrity_failed++;
  540. }
  541. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEPROTOERROR);
  542. goto drop_unlock;
  543. }
  544. /* only the first xfrm gets the encap type */
  545. encap_type = 0;
  546. if (x->repl->recheck(x, skb, seq)) {
  547. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATESEQERROR);
  548. goto drop_unlock;
  549. }
  550. x->repl->advance(x, seq);
  551. x->curlft.bytes += skb->len;
  552. x->curlft.packets++;
  553. spin_unlock(&x->lock);
  554. XFRM_MODE_SKB_CB(skb)->protocol = nexthdr;
  555. inner_mode = &x->inner_mode;
  556. if (x->sel.family == AF_UNSPEC) {
  557. inner_mode = xfrm_ip2inner_mode(x, XFRM_MODE_SKB_CB(skb)->protocol);
  558. if (inner_mode == NULL) {
  559. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
  560. goto drop;
  561. }
  562. }
  563. if (xfrm_inner_mode_input(x, inner_mode, skb)) {
  564. XFRM_INC_STATS(net, LINUX_MIB_XFRMINSTATEMODEERROR);
  565. goto drop;
  566. }
  567. if (x->outer_mode.flags & XFRM_MODE_FLAG_TUNNEL) {
  568. decaps = 1;
  569. break;
  570. }
  571. /*
  572. * We need the inner address. However, we only get here for
  573. * transport mode so the outer address is identical.
  574. */
  575. daddr = &x->id.daddr;
  576. family = x->outer_mode.family;
  577. err = xfrm_parse_spi(skb, nexthdr, &spi, &seq);
  578. if (err < 0) {
  579. XFRM_INC_STATS(net, LINUX_MIB_XFRMINHDRERROR);
  580. goto drop;
  581. }
  582. crypto_done = false;
  583. } while (!err);
  584. err = xfrm_rcv_cb(skb, family, x->type->proto, 0);
  585. if (err)
  586. goto drop;
  587. nf_reset_ct(skb);
  588. if (decaps) {
  589. sp = skb_sec_path(skb);
  590. if (sp)
  591. sp->olen = 0;
  592. skb_dst_drop(skb);
  593. gro_cells_receive(&gro_cells, skb);
  594. return 0;
  595. } else {
  596. xo = xfrm_offload(skb);
  597. if (xo)
  598. xfrm_gro = xo->flags & XFRM_GRO;
  599. err = -EAFNOSUPPORT;
  600. rcu_read_lock();
  601. afinfo = xfrm_state_afinfo_get_rcu(x->inner_mode.family);
  602. if (likely(afinfo))
  603. err = afinfo->transport_finish(skb, xfrm_gro || async);
  604. rcu_read_unlock();
  605. if (xfrm_gro) {
  606. sp = skb_sec_path(skb);
  607. if (sp)
  608. sp->olen = 0;
  609. skb_dst_drop(skb);
  610. gro_cells_receive(&gro_cells, skb);
  611. return err;
  612. }
  613. return err;
  614. }
  615. drop_unlock:
  616. spin_unlock(&x->lock);
  617. drop:
  618. xfrm_rcv_cb(skb, family, x && x->type ? x->type->proto : nexthdr, -1);
  619. kfree_skb(skb);
  620. return 0;
  621. }
  622. EXPORT_SYMBOL(xfrm_input);
  623. int xfrm_input_resume(struct sk_buff *skb, int nexthdr)
  624. {
  625. return xfrm_input(skb, nexthdr, 0, -1);
  626. }
  627. EXPORT_SYMBOL(xfrm_input_resume);
  628. static void xfrm_trans_reinject(unsigned long data)
  629. {
  630. struct xfrm_trans_tasklet *trans = (void *)data;
  631. struct sk_buff_head queue;
  632. struct sk_buff *skb;
  633. __skb_queue_head_init(&queue);
  634. skb_queue_splice_init(&trans->queue, &queue);
  635. while ((skb = __skb_dequeue(&queue)))
  636. XFRM_TRANS_SKB_CB(skb)->finish(XFRM_TRANS_SKB_CB(skb)->net,
  637. NULL, skb);
  638. }
  639. int xfrm_trans_queue_net(struct net *net, struct sk_buff *skb,
  640. int (*finish)(struct net *, struct sock *,
  641. struct sk_buff *))
  642. {
  643. struct xfrm_trans_tasklet *trans;
  644. trans = this_cpu_ptr(&xfrm_trans_tasklet);
  645. if (skb_queue_len(&trans->queue) >= netdev_max_backlog)
  646. return -ENOBUFS;
  647. BUILD_BUG_ON(sizeof(struct xfrm_trans_cb) > sizeof(skb->cb));
  648. XFRM_TRANS_SKB_CB(skb)->finish = finish;
  649. XFRM_TRANS_SKB_CB(skb)->net = net;
  650. __skb_queue_tail(&trans->queue, skb);
  651. tasklet_schedule(&trans->tasklet);
  652. return 0;
  653. }
  654. EXPORT_SYMBOL(xfrm_trans_queue_net);
  655. int xfrm_trans_queue(struct sk_buff *skb,
  656. int (*finish)(struct net *, struct sock *,
  657. struct sk_buff *))
  658. {
  659. return xfrm_trans_queue_net(dev_net(skb->dev), skb, finish);
  660. }
  661. EXPORT_SYMBOL(xfrm_trans_queue);
  662. void __init xfrm_input_init(void)
  663. {
  664. int err;
  665. int i;
  666. init_dummy_netdev(&xfrm_napi_dev);
  667. err = gro_cells_init(&gro_cells, &xfrm_napi_dev);
  668. if (err)
  669. gro_cells.cells = NULL;
  670. for_each_possible_cpu(i) {
  671. struct xfrm_trans_tasklet *trans;
  672. trans = &per_cpu(xfrm_trans_tasklet, i);
  673. __skb_queue_head_init(&trans->queue);
  674. tasklet_init(&trans->tasklet, xfrm_trans_reinject,
  675. (unsigned long)trans);
  676. }
  677. }