esp6.c 29 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * Copyright (C)2002 USAGI/WIDE Project
  4. *
  5. * Authors
  6. *
  7. * Mitsuru KANDA @USAGI : IPv6 Support
  8. * Kazunori MIYAZAWA @USAGI :
  9. * Kunihiro Ishiguro <kunihiro@ipinfusion.com>
  10. *
  11. * This file is derived from net/ipv4/esp.c
  12. */
  13. #define pr_fmt(fmt) "IPv6: " fmt
  14. #include <crypto/aead.h>
  15. #include <crypto/authenc.h>
  16. #include <linux/err.h>
  17. #include <linux/module.h>
  18. #include <net/ip.h>
  19. #include <net/xfrm.h>
  20. #include <net/esp.h>
  21. #include <linux/scatterlist.h>
  22. #include <linux/kernel.h>
  23. #include <linux/pfkeyv2.h>
  24. #include <linux/random.h>
  25. #include <linux/slab.h>
  26. #include <linux/spinlock.h>
  27. #include <net/ip6_checksum.h>
  28. #include <net/ip6_route.h>
  29. #include <net/icmp.h>
  30. #include <net/ipv6.h>
  31. #include <net/protocol.h>
  32. #include <net/udp.h>
  33. #include <linux/icmpv6.h>
  34. #include <net/tcp.h>
  35. #include <net/espintcp.h>
  36. #include <net/inet6_hashtables.h>
  37. #include <linux/highmem.h>
  38. struct esp_skb_cb {
  39. struct xfrm_skb_cb xfrm;
  40. void *tmp;
  41. };
  42. struct esp_output_extra {
  43. __be32 seqhi;
  44. u32 esphoff;
  45. };
  46. #define ESP_SKB_CB(__skb) ((struct esp_skb_cb *)&((__skb)->cb[0]))
  47. /*
  48. * Allocate an AEAD request structure with extra space for SG and IV.
  49. *
  50. * For alignment considerations the upper 32 bits of the sequence number are
  51. * placed at the front, if present. Followed by the IV, the request and finally
  52. * the SG list.
  53. *
  54. * TODO: Use spare space in skb for this where possible.
  55. */
  56. static void *esp_alloc_tmp(struct crypto_aead *aead, int nfrags, int seqihlen)
  57. {
  58. unsigned int len;
  59. len = seqihlen;
  60. len += crypto_aead_ivsize(aead);
  61. if (len) {
  62. len += crypto_aead_alignmask(aead) &
  63. ~(crypto_tfm_ctx_alignment() - 1);
  64. len = ALIGN(len, crypto_tfm_ctx_alignment());
  65. }
  66. len += sizeof(struct aead_request) + crypto_aead_reqsize(aead);
  67. len = ALIGN(len, __alignof__(struct scatterlist));
  68. len += sizeof(struct scatterlist) * nfrags;
  69. return kmalloc(len, GFP_ATOMIC);
  70. }
  71. static inline void *esp_tmp_extra(void *tmp)
  72. {
  73. return PTR_ALIGN(tmp, __alignof__(struct esp_output_extra));
  74. }
  75. static inline u8 *esp_tmp_iv(struct crypto_aead *aead, void *tmp, int seqhilen)
  76. {
  77. return crypto_aead_ivsize(aead) ?
  78. PTR_ALIGN((u8 *)tmp + seqhilen,
  79. crypto_aead_alignmask(aead) + 1) : tmp + seqhilen;
  80. }
  81. static inline struct aead_request *esp_tmp_req(struct crypto_aead *aead, u8 *iv)
  82. {
  83. struct aead_request *req;
  84. req = (void *)PTR_ALIGN(iv + crypto_aead_ivsize(aead),
  85. crypto_tfm_ctx_alignment());
  86. aead_request_set_tfm(req, aead);
  87. return req;
  88. }
  89. static inline struct scatterlist *esp_req_sg(struct crypto_aead *aead,
  90. struct aead_request *req)
  91. {
  92. return (void *)ALIGN((unsigned long)(req + 1) +
  93. crypto_aead_reqsize(aead),
  94. __alignof__(struct scatterlist));
  95. }
  96. static void esp_ssg_unref(struct xfrm_state *x, void *tmp)
  97. {
  98. struct esp_output_extra *extra = esp_tmp_extra(tmp);
  99. struct crypto_aead *aead = x->data;
  100. int extralen = 0;
  101. u8 *iv;
  102. struct aead_request *req;
  103. struct scatterlist *sg;
  104. if (x->props.flags & XFRM_STATE_ESN)
  105. extralen += sizeof(*extra);
  106. iv = esp_tmp_iv(aead, tmp, extralen);
  107. req = esp_tmp_req(aead, iv);
  108. /* Unref skb_frag_pages in the src scatterlist if necessary.
  109. * Skip the first sg which comes from skb->data.
  110. */
  111. if (req->src != req->dst)
  112. for (sg = sg_next(req->src); sg; sg = sg_next(sg))
  113. put_page(sg_page(sg));
  114. }
  115. #ifdef CONFIG_INET6_ESPINTCP
  116. struct esp_tcp_sk {
  117. struct sock *sk;
  118. struct rcu_head rcu;
  119. };
  120. static void esp_free_tcp_sk(struct rcu_head *head)
  121. {
  122. struct esp_tcp_sk *esk = container_of(head, struct esp_tcp_sk, rcu);
  123. sock_put(esk->sk);
  124. kfree(esk);
  125. }
  126. static struct sock *esp6_find_tcp_sk(struct xfrm_state *x)
  127. {
  128. struct xfrm_encap_tmpl *encap = x->encap;
  129. struct esp_tcp_sk *esk;
  130. __be16 sport, dport;
  131. struct sock *nsk;
  132. struct sock *sk;
  133. sk = rcu_dereference(x->encap_sk);
  134. if (sk && sk->sk_state == TCP_ESTABLISHED)
  135. return sk;
  136. spin_lock_bh(&x->lock);
  137. sport = encap->encap_sport;
  138. dport = encap->encap_dport;
  139. nsk = rcu_dereference_protected(x->encap_sk,
  140. lockdep_is_held(&x->lock));
  141. if (sk && sk == nsk) {
  142. esk = kmalloc(sizeof(*esk), GFP_ATOMIC);
  143. if (!esk) {
  144. spin_unlock_bh(&x->lock);
  145. return ERR_PTR(-ENOMEM);
  146. }
  147. RCU_INIT_POINTER(x->encap_sk, NULL);
  148. esk->sk = sk;
  149. call_rcu(&esk->rcu, esp_free_tcp_sk);
  150. }
  151. spin_unlock_bh(&x->lock);
  152. sk = __inet6_lookup_established(xs_net(x), &tcp_hashinfo, &x->id.daddr.in6,
  153. dport, &x->props.saddr.in6, ntohs(sport), 0, 0);
  154. if (!sk)
  155. return ERR_PTR(-ENOENT);
  156. if (!tcp_is_ulp_esp(sk)) {
  157. sock_put(sk);
  158. return ERR_PTR(-EINVAL);
  159. }
  160. spin_lock_bh(&x->lock);
  161. nsk = rcu_dereference_protected(x->encap_sk,
  162. lockdep_is_held(&x->lock));
  163. if (encap->encap_sport != sport ||
  164. encap->encap_dport != dport) {
  165. sock_put(sk);
  166. sk = nsk ?: ERR_PTR(-EREMCHG);
  167. } else if (sk == nsk) {
  168. sock_put(sk);
  169. } else {
  170. rcu_assign_pointer(x->encap_sk, sk);
  171. }
  172. spin_unlock_bh(&x->lock);
  173. return sk;
  174. }
  175. static int esp_output_tcp_finish(struct xfrm_state *x, struct sk_buff *skb)
  176. {
  177. struct sock *sk;
  178. int err;
  179. rcu_read_lock();
  180. sk = esp6_find_tcp_sk(x);
  181. err = PTR_ERR_OR_ZERO(sk);
  182. if (err)
  183. goto out;
  184. bh_lock_sock(sk);
  185. if (sock_owned_by_user(sk))
  186. err = espintcp_queue_out(sk, skb);
  187. else
  188. err = espintcp_push_skb(sk, skb);
  189. bh_unlock_sock(sk);
  190. out:
  191. rcu_read_unlock();
  192. return err;
  193. }
  194. static int esp_output_tcp_encap_cb(struct net *net, struct sock *sk,
  195. struct sk_buff *skb)
  196. {
  197. struct dst_entry *dst = skb_dst(skb);
  198. struct xfrm_state *x = dst->xfrm;
  199. return esp_output_tcp_finish(x, skb);
  200. }
  201. static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb)
  202. {
  203. int err;
  204. local_bh_disable();
  205. err = xfrm_trans_queue_net(xs_net(x), skb, esp_output_tcp_encap_cb);
  206. local_bh_enable();
  207. /* EINPROGRESS just happens to do the right thing. It
  208. * actually means that the skb has been consumed and
  209. * isn't coming back.
  210. */
  211. return err ?: -EINPROGRESS;
  212. }
  213. #else
  214. static int esp_output_tail_tcp(struct xfrm_state *x, struct sk_buff *skb)
  215. {
  216. kfree_skb(skb);
  217. return -EOPNOTSUPP;
  218. }
  219. #endif
  220. static void esp_output_encap_csum(struct sk_buff *skb)
  221. {
  222. /* UDP encap with IPv6 requires a valid checksum */
  223. if (*skb_mac_header(skb) == IPPROTO_UDP) {
  224. struct udphdr *uh = udp_hdr(skb);
  225. struct ipv6hdr *ip6h = ipv6_hdr(skb);
  226. int len = ntohs(uh->len);
  227. unsigned int offset = skb_transport_offset(skb);
  228. __wsum csum = skb_checksum(skb, offset, skb->len - offset, 0);
  229. uh->check = csum_ipv6_magic(&ip6h->saddr, &ip6h->daddr,
  230. len, IPPROTO_UDP, csum);
  231. if (uh->check == 0)
  232. uh->check = CSUM_MANGLED_0;
  233. }
  234. }
  235. static void esp_output_done(struct crypto_async_request *base, int err)
  236. {
  237. struct sk_buff *skb = base->data;
  238. struct xfrm_offload *xo = xfrm_offload(skb);
  239. void *tmp;
  240. struct xfrm_state *x;
  241. if (xo && (xo->flags & XFRM_DEV_RESUME)) {
  242. struct sec_path *sp = skb_sec_path(skb);
  243. x = sp->xvec[sp->len - 1];
  244. } else {
  245. x = skb_dst(skb)->xfrm;
  246. }
  247. tmp = ESP_SKB_CB(skb)->tmp;
  248. esp_ssg_unref(x, tmp);
  249. kfree(tmp);
  250. esp_output_encap_csum(skb);
  251. if (xo && (xo->flags & XFRM_DEV_RESUME)) {
  252. if (err) {
  253. XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
  254. kfree_skb(skb);
  255. return;
  256. }
  257. skb_push(skb, skb->data - skb_mac_header(skb));
  258. secpath_reset(skb);
  259. xfrm_dev_resume(skb);
  260. } else {
  261. if (!err &&
  262. x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP)
  263. esp_output_tail_tcp(x, skb);
  264. else
  265. xfrm_output_resume(skb, err);
  266. }
  267. }
  268. /* Move ESP header back into place. */
  269. static void esp_restore_header(struct sk_buff *skb, unsigned int offset)
  270. {
  271. struct ip_esp_hdr *esph = (void *)(skb->data + offset);
  272. void *tmp = ESP_SKB_CB(skb)->tmp;
  273. __be32 *seqhi = esp_tmp_extra(tmp);
  274. esph->seq_no = esph->spi;
  275. esph->spi = *seqhi;
  276. }
  277. static void esp_output_restore_header(struct sk_buff *skb)
  278. {
  279. void *tmp = ESP_SKB_CB(skb)->tmp;
  280. struct esp_output_extra *extra = esp_tmp_extra(tmp);
  281. esp_restore_header(skb, skb_transport_offset(skb) + extra->esphoff -
  282. sizeof(__be32));
  283. }
  284. static struct ip_esp_hdr *esp_output_set_esn(struct sk_buff *skb,
  285. struct xfrm_state *x,
  286. struct ip_esp_hdr *esph,
  287. struct esp_output_extra *extra)
  288. {
  289. /* For ESN we move the header forward by 4 bytes to
  290. * accomodate the high bits. We will move it back after
  291. * encryption.
  292. */
  293. if ((x->props.flags & XFRM_STATE_ESN)) {
  294. __u32 seqhi;
  295. struct xfrm_offload *xo = xfrm_offload(skb);
  296. if (xo)
  297. seqhi = xo->seq.hi;
  298. else
  299. seqhi = XFRM_SKB_CB(skb)->seq.output.hi;
  300. extra->esphoff = (unsigned char *)esph -
  301. skb_transport_header(skb);
  302. esph = (struct ip_esp_hdr *)((unsigned char *)esph - 4);
  303. extra->seqhi = esph->spi;
  304. esph->seq_no = htonl(seqhi);
  305. }
  306. esph->spi = x->id.spi;
  307. return esph;
  308. }
  309. static void esp_output_done_esn(struct crypto_async_request *base, int err)
  310. {
  311. struct sk_buff *skb = base->data;
  312. esp_output_restore_header(skb);
  313. esp_output_done(base, err);
  314. }
  315. static struct ip_esp_hdr *esp6_output_udp_encap(struct sk_buff *skb,
  316. int encap_type,
  317. struct esp_info *esp,
  318. __be16 sport,
  319. __be16 dport)
  320. {
  321. struct udphdr *uh;
  322. __be32 *udpdata32;
  323. unsigned int len;
  324. len = skb->len + esp->tailen - skb_transport_offset(skb);
  325. if (len > U16_MAX)
  326. return ERR_PTR(-EMSGSIZE);
  327. uh = (struct udphdr *)esp->esph;
  328. uh->source = sport;
  329. uh->dest = dport;
  330. uh->len = htons(len);
  331. uh->check = 0;
  332. *skb_mac_header(skb) = IPPROTO_UDP;
  333. if (encap_type == UDP_ENCAP_ESPINUDP_NON_IKE) {
  334. udpdata32 = (__be32 *)(uh + 1);
  335. udpdata32[0] = udpdata32[1] = 0;
  336. return (struct ip_esp_hdr *)(udpdata32 + 2);
  337. }
  338. return (struct ip_esp_hdr *)(uh + 1);
  339. }
  340. #ifdef CONFIG_INET6_ESPINTCP
  341. static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x,
  342. struct sk_buff *skb,
  343. struct esp_info *esp)
  344. {
  345. __be16 *lenp = (void *)esp->esph;
  346. struct ip_esp_hdr *esph;
  347. unsigned int len;
  348. struct sock *sk;
  349. len = skb->len + esp->tailen - skb_transport_offset(skb);
  350. if (len > IP_MAX_MTU)
  351. return ERR_PTR(-EMSGSIZE);
  352. rcu_read_lock();
  353. sk = esp6_find_tcp_sk(x);
  354. rcu_read_unlock();
  355. if (IS_ERR(sk))
  356. return ERR_CAST(sk);
  357. *lenp = htons(len);
  358. esph = (struct ip_esp_hdr *)(lenp + 1);
  359. return esph;
  360. }
  361. #else
  362. static struct ip_esp_hdr *esp6_output_tcp_encap(struct xfrm_state *x,
  363. struct sk_buff *skb,
  364. struct esp_info *esp)
  365. {
  366. return ERR_PTR(-EOPNOTSUPP);
  367. }
  368. #endif
  369. static int esp6_output_encap(struct xfrm_state *x, struct sk_buff *skb,
  370. struct esp_info *esp)
  371. {
  372. struct xfrm_encap_tmpl *encap = x->encap;
  373. struct ip_esp_hdr *esph;
  374. __be16 sport, dport;
  375. int encap_type;
  376. spin_lock_bh(&x->lock);
  377. sport = encap->encap_sport;
  378. dport = encap->encap_dport;
  379. encap_type = encap->encap_type;
  380. spin_unlock_bh(&x->lock);
  381. switch (encap_type) {
  382. default:
  383. case UDP_ENCAP_ESPINUDP:
  384. case UDP_ENCAP_ESPINUDP_NON_IKE:
  385. esph = esp6_output_udp_encap(skb, encap_type, esp, sport, dport);
  386. break;
  387. case TCP_ENCAP_ESPINTCP:
  388. esph = esp6_output_tcp_encap(x, skb, esp);
  389. break;
  390. }
  391. if (IS_ERR(esph))
  392. return PTR_ERR(esph);
  393. esp->esph = esph;
  394. return 0;
  395. }
  396. int esp6_output_head(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
  397. {
  398. u8 *tail;
  399. int nfrags;
  400. int esph_offset;
  401. struct page *page;
  402. struct sk_buff *trailer;
  403. int tailen = esp->tailen;
  404. if (x->encap) {
  405. int err = esp6_output_encap(x, skb, esp);
  406. if (err < 0)
  407. return err;
  408. }
  409. if (ALIGN(tailen, L1_CACHE_BYTES) > PAGE_SIZE ||
  410. ALIGN(skb->data_len, L1_CACHE_BYTES) > PAGE_SIZE)
  411. goto cow;
  412. if (!skb_cloned(skb)) {
  413. if (tailen <= skb_tailroom(skb)) {
  414. nfrags = 1;
  415. trailer = skb;
  416. tail = skb_tail_pointer(trailer);
  417. goto skip_cow;
  418. } else if ((skb_shinfo(skb)->nr_frags < MAX_SKB_FRAGS)
  419. && !skb_has_frag_list(skb)) {
  420. int allocsize;
  421. struct sock *sk = skb->sk;
  422. struct page_frag *pfrag = &x->xfrag;
  423. esp->inplace = false;
  424. allocsize = ALIGN(tailen, L1_CACHE_BYTES);
  425. spin_lock_bh(&x->lock);
  426. if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
  427. spin_unlock_bh(&x->lock);
  428. goto cow;
  429. }
  430. page = pfrag->page;
  431. get_page(page);
  432. tail = page_address(page) + pfrag->offset;
  433. esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
  434. nfrags = skb_shinfo(skb)->nr_frags;
  435. __skb_fill_page_desc(skb, nfrags, page, pfrag->offset,
  436. tailen);
  437. skb_shinfo(skb)->nr_frags = ++nfrags;
  438. pfrag->offset = pfrag->offset + allocsize;
  439. spin_unlock_bh(&x->lock);
  440. nfrags++;
  441. skb->len += tailen;
  442. skb->data_len += tailen;
  443. skb->truesize += tailen;
  444. if (sk && sk_fullsock(sk))
  445. refcount_add(tailen, &sk->sk_wmem_alloc);
  446. goto out;
  447. }
  448. }
  449. cow:
  450. esph_offset = (unsigned char *)esp->esph - skb_transport_header(skb);
  451. nfrags = skb_cow_data(skb, tailen, &trailer);
  452. if (nfrags < 0)
  453. goto out;
  454. tail = skb_tail_pointer(trailer);
  455. esp->esph = (struct ip_esp_hdr *)(skb_transport_header(skb) + esph_offset);
  456. skip_cow:
  457. esp_output_fill_trailer(tail, esp->tfclen, esp->plen, esp->proto);
  458. pskb_put(skb, trailer, tailen);
  459. out:
  460. return nfrags;
  461. }
  462. EXPORT_SYMBOL_GPL(esp6_output_head);
  463. int esp6_output_tail(struct xfrm_state *x, struct sk_buff *skb, struct esp_info *esp)
  464. {
  465. u8 *iv;
  466. int alen;
  467. void *tmp;
  468. int ivlen;
  469. int assoclen;
  470. int extralen;
  471. struct page *page;
  472. struct ip_esp_hdr *esph;
  473. struct aead_request *req;
  474. struct crypto_aead *aead;
  475. struct scatterlist *sg, *dsg;
  476. struct esp_output_extra *extra;
  477. int err = -ENOMEM;
  478. assoclen = sizeof(struct ip_esp_hdr);
  479. extralen = 0;
  480. if (x->props.flags & XFRM_STATE_ESN) {
  481. extralen += sizeof(*extra);
  482. assoclen += sizeof(__be32);
  483. }
  484. aead = x->data;
  485. alen = crypto_aead_authsize(aead);
  486. ivlen = crypto_aead_ivsize(aead);
  487. tmp = esp_alloc_tmp(aead, esp->nfrags + 2, extralen);
  488. if (!tmp)
  489. goto error;
  490. extra = esp_tmp_extra(tmp);
  491. iv = esp_tmp_iv(aead, tmp, extralen);
  492. req = esp_tmp_req(aead, iv);
  493. sg = esp_req_sg(aead, req);
  494. if (esp->inplace)
  495. dsg = sg;
  496. else
  497. dsg = &sg[esp->nfrags];
  498. esph = esp_output_set_esn(skb, x, esp->esph, extra);
  499. esp->esph = esph;
  500. sg_init_table(sg, esp->nfrags);
  501. err = skb_to_sgvec(skb, sg,
  502. (unsigned char *)esph - skb->data,
  503. assoclen + ivlen + esp->clen + alen);
  504. if (unlikely(err < 0))
  505. goto error_free;
  506. if (!esp->inplace) {
  507. int allocsize;
  508. struct page_frag *pfrag = &x->xfrag;
  509. allocsize = ALIGN(skb->data_len, L1_CACHE_BYTES);
  510. spin_lock_bh(&x->lock);
  511. if (unlikely(!skb_page_frag_refill(allocsize, pfrag, GFP_ATOMIC))) {
  512. spin_unlock_bh(&x->lock);
  513. goto error_free;
  514. }
  515. skb_shinfo(skb)->nr_frags = 1;
  516. page = pfrag->page;
  517. get_page(page);
  518. /* replace page frags in skb with new page */
  519. __skb_fill_page_desc(skb, 0, page, pfrag->offset, skb->data_len);
  520. pfrag->offset = pfrag->offset + allocsize;
  521. spin_unlock_bh(&x->lock);
  522. sg_init_table(dsg, skb_shinfo(skb)->nr_frags + 1);
  523. err = skb_to_sgvec(skb, dsg,
  524. (unsigned char *)esph - skb->data,
  525. assoclen + ivlen + esp->clen + alen);
  526. if (unlikely(err < 0))
  527. goto error_free;
  528. }
  529. if ((x->props.flags & XFRM_STATE_ESN))
  530. aead_request_set_callback(req, 0, esp_output_done_esn, skb);
  531. else
  532. aead_request_set_callback(req, 0, esp_output_done, skb);
  533. aead_request_set_crypt(req, sg, dsg, ivlen + esp->clen, iv);
  534. aead_request_set_ad(req, assoclen);
  535. memset(iv, 0, ivlen);
  536. memcpy(iv + ivlen - min(ivlen, 8), (u8 *)&esp->seqno + 8 - min(ivlen, 8),
  537. min(ivlen, 8));
  538. ESP_SKB_CB(skb)->tmp = tmp;
  539. err = crypto_aead_encrypt(req);
  540. switch (err) {
  541. case -EINPROGRESS:
  542. goto error;
  543. case -ENOSPC:
  544. err = NET_XMIT_DROP;
  545. break;
  546. case 0:
  547. if ((x->props.flags & XFRM_STATE_ESN))
  548. esp_output_restore_header(skb);
  549. esp_output_encap_csum(skb);
  550. }
  551. if (sg != dsg)
  552. esp_ssg_unref(x, tmp);
  553. if (!err && x->encap && x->encap->encap_type == TCP_ENCAP_ESPINTCP)
  554. err = esp_output_tail_tcp(x, skb);
  555. error_free:
  556. kfree(tmp);
  557. error:
  558. return err;
  559. }
  560. EXPORT_SYMBOL_GPL(esp6_output_tail);
  561. static int esp6_output(struct xfrm_state *x, struct sk_buff *skb)
  562. {
  563. int alen;
  564. int blksize;
  565. struct ip_esp_hdr *esph;
  566. struct crypto_aead *aead;
  567. struct esp_info esp;
  568. esp.inplace = true;
  569. esp.proto = *skb_mac_header(skb);
  570. *skb_mac_header(skb) = IPPROTO_ESP;
  571. /* skb is pure payload to encrypt */
  572. aead = x->data;
  573. alen = crypto_aead_authsize(aead);
  574. esp.tfclen = 0;
  575. if (x->tfcpad) {
  576. struct xfrm_dst *dst = (struct xfrm_dst *)skb_dst(skb);
  577. u32 padto;
  578. padto = min(x->tfcpad, xfrm_state_mtu(x, dst->child_mtu_cached));
  579. if (skb->len < padto)
  580. esp.tfclen = padto - skb->len;
  581. }
  582. blksize = ALIGN(crypto_aead_blocksize(aead), 4);
  583. esp.clen = ALIGN(skb->len + 2 + esp.tfclen, blksize);
  584. esp.plen = esp.clen - skb->len - esp.tfclen;
  585. esp.tailen = esp.tfclen + esp.plen + alen;
  586. esp.esph = ip_esp_hdr(skb);
  587. esp.nfrags = esp6_output_head(x, skb, &esp);
  588. if (esp.nfrags < 0)
  589. return esp.nfrags;
  590. esph = esp.esph;
  591. esph->spi = x->id.spi;
  592. esph->seq_no = htonl(XFRM_SKB_CB(skb)->seq.output.low);
  593. esp.seqno = cpu_to_be64(XFRM_SKB_CB(skb)->seq.output.low +
  594. ((u64)XFRM_SKB_CB(skb)->seq.output.hi << 32));
  595. skb_push(skb, -skb_network_offset(skb));
  596. return esp6_output_tail(x, skb, &esp);
  597. }
  598. static inline int esp_remove_trailer(struct sk_buff *skb)
  599. {
  600. struct xfrm_state *x = xfrm_input_state(skb);
  601. struct xfrm_offload *xo = xfrm_offload(skb);
  602. struct crypto_aead *aead = x->data;
  603. int alen, hlen, elen;
  604. int padlen, trimlen;
  605. __wsum csumdiff;
  606. u8 nexthdr[2];
  607. int ret;
  608. alen = crypto_aead_authsize(aead);
  609. hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
  610. elen = skb->len - hlen;
  611. if (xo && (xo->flags & XFRM_ESP_NO_TRAILER)) {
  612. ret = xo->proto;
  613. goto out;
  614. }
  615. ret = skb_copy_bits(skb, skb->len - alen - 2, nexthdr, 2);
  616. BUG_ON(ret);
  617. ret = -EINVAL;
  618. padlen = nexthdr[0];
  619. if (padlen + 2 + alen >= elen) {
  620. net_dbg_ratelimited("ipsec esp packet is garbage padlen=%d, elen=%d\n",
  621. padlen + 2, elen - alen);
  622. goto out;
  623. }
  624. trimlen = alen + padlen + 2;
  625. if (skb->ip_summed == CHECKSUM_COMPLETE) {
  626. csumdiff = skb_checksum(skb, skb->len - trimlen, trimlen, 0);
  627. skb->csum = csum_block_sub(skb->csum, csumdiff,
  628. skb->len - trimlen);
  629. }
  630. pskb_trim(skb, skb->len - trimlen);
  631. ret = nexthdr[1];
  632. out:
  633. return ret;
  634. }
  635. int esp6_input_done2(struct sk_buff *skb, int err)
  636. {
  637. struct xfrm_state *x = xfrm_input_state(skb);
  638. struct xfrm_offload *xo = xfrm_offload(skb);
  639. struct crypto_aead *aead = x->data;
  640. int hlen = sizeof(struct ip_esp_hdr) + crypto_aead_ivsize(aead);
  641. int hdr_len = skb_network_header_len(skb);
  642. if (!xo || (xo && !(xo->flags & CRYPTO_DONE)))
  643. kfree(ESP_SKB_CB(skb)->tmp);
  644. if (unlikely(err))
  645. goto out;
  646. err = esp_remove_trailer(skb);
  647. if (unlikely(err < 0))
  648. goto out;
  649. if (x->encap) {
  650. const struct ipv6hdr *ip6h = ipv6_hdr(skb);
  651. int offset = skb_network_offset(skb) + sizeof(*ip6h);
  652. struct xfrm_encap_tmpl *encap = x->encap;
  653. u8 nexthdr = ip6h->nexthdr;
  654. __be16 frag_off, source;
  655. struct udphdr *uh;
  656. struct tcphdr *th;
  657. offset = ipv6_skip_exthdr(skb, offset, &nexthdr, &frag_off);
  658. if (offset == -1) {
  659. err = -EINVAL;
  660. goto out;
  661. }
  662. uh = (void *)(skb->data + offset);
  663. th = (void *)(skb->data + offset);
  664. hdr_len += offset;
  665. switch (x->encap->encap_type) {
  666. case TCP_ENCAP_ESPINTCP:
  667. source = th->source;
  668. break;
  669. case UDP_ENCAP_ESPINUDP:
  670. case UDP_ENCAP_ESPINUDP_NON_IKE:
  671. source = uh->source;
  672. break;
  673. default:
  674. WARN_ON_ONCE(1);
  675. err = -EINVAL;
  676. goto out;
  677. }
  678. /*
  679. * 1) if the NAT-T peer's IP or port changed then
  680. * advertize the change to the keying daemon.
  681. * This is an inbound SA, so just compare
  682. * SRC ports.
  683. */
  684. if (!ipv6_addr_equal(&ip6h->saddr, &x->props.saddr.in6) ||
  685. source != encap->encap_sport) {
  686. xfrm_address_t ipaddr;
  687. memcpy(&ipaddr.a6, &ip6h->saddr.s6_addr, sizeof(ipaddr.a6));
  688. km_new_mapping(x, &ipaddr, source);
  689. /* XXX: perhaps add an extra
  690. * policy check here, to see
  691. * if we should allow or
  692. * reject a packet from a
  693. * different source
  694. * address/port.
  695. */
  696. }
  697. /*
  698. * 2) ignore UDP/TCP checksums in case
  699. * of NAT-T in Transport Mode, or
  700. * perform other post-processing fixes
  701. * as per draft-ietf-ipsec-udp-encaps-06,
  702. * section 3.1.2
  703. */
  704. if (x->props.mode == XFRM_MODE_TRANSPORT)
  705. skb->ip_summed = CHECKSUM_UNNECESSARY;
  706. }
  707. skb_postpull_rcsum(skb, skb_network_header(skb),
  708. skb_network_header_len(skb));
  709. skb_pull_rcsum(skb, hlen);
  710. if (x->props.mode == XFRM_MODE_TUNNEL)
  711. skb_reset_transport_header(skb);
  712. else
  713. skb_set_transport_header(skb, -hdr_len);
  714. /* RFC4303: Drop dummy packets without any error */
  715. if (err == IPPROTO_NONE)
  716. err = -EINVAL;
  717. out:
  718. return err;
  719. }
  720. EXPORT_SYMBOL_GPL(esp6_input_done2);
  721. static void esp_input_done(struct crypto_async_request *base, int err)
  722. {
  723. struct sk_buff *skb = base->data;
  724. xfrm_input_resume(skb, esp6_input_done2(skb, err));
  725. }
  726. static void esp_input_restore_header(struct sk_buff *skb)
  727. {
  728. esp_restore_header(skb, 0);
  729. __skb_pull(skb, 4);
  730. }
  731. static void esp_input_set_header(struct sk_buff *skb, __be32 *seqhi)
  732. {
  733. struct xfrm_state *x = xfrm_input_state(skb);
  734. /* For ESN we move the header forward by 4 bytes to
  735. * accomodate the high bits. We will move it back after
  736. * decryption.
  737. */
  738. if ((x->props.flags & XFRM_STATE_ESN)) {
  739. struct ip_esp_hdr *esph = skb_push(skb, 4);
  740. *seqhi = esph->spi;
  741. esph->spi = esph->seq_no;
  742. esph->seq_no = XFRM_SKB_CB(skb)->seq.input.hi;
  743. }
  744. }
  745. static void esp_input_done_esn(struct crypto_async_request *base, int err)
  746. {
  747. struct sk_buff *skb = base->data;
  748. esp_input_restore_header(skb);
  749. esp_input_done(base, err);
  750. }
  751. static int esp6_input(struct xfrm_state *x, struct sk_buff *skb)
  752. {
  753. struct crypto_aead *aead = x->data;
  754. struct aead_request *req;
  755. struct sk_buff *trailer;
  756. int ivlen = crypto_aead_ivsize(aead);
  757. int elen = skb->len - sizeof(struct ip_esp_hdr) - ivlen;
  758. int nfrags;
  759. int assoclen;
  760. int seqhilen;
  761. int ret = 0;
  762. void *tmp;
  763. __be32 *seqhi;
  764. u8 *iv;
  765. struct scatterlist *sg;
  766. if (!pskb_may_pull(skb, sizeof(struct ip_esp_hdr) + ivlen)) {
  767. ret = -EINVAL;
  768. goto out;
  769. }
  770. if (elen <= 0) {
  771. ret = -EINVAL;
  772. goto out;
  773. }
  774. assoclen = sizeof(struct ip_esp_hdr);
  775. seqhilen = 0;
  776. if (x->props.flags & XFRM_STATE_ESN) {
  777. seqhilen += sizeof(__be32);
  778. assoclen += seqhilen;
  779. }
  780. if (!skb_cloned(skb)) {
  781. if (!skb_is_nonlinear(skb)) {
  782. nfrags = 1;
  783. goto skip_cow;
  784. } else if (!skb_has_frag_list(skb)) {
  785. nfrags = skb_shinfo(skb)->nr_frags;
  786. nfrags++;
  787. goto skip_cow;
  788. }
  789. }
  790. nfrags = skb_cow_data(skb, 0, &trailer);
  791. if (nfrags < 0) {
  792. ret = -EINVAL;
  793. goto out;
  794. }
  795. skip_cow:
  796. ret = -ENOMEM;
  797. tmp = esp_alloc_tmp(aead, nfrags, seqhilen);
  798. if (!tmp)
  799. goto out;
  800. ESP_SKB_CB(skb)->tmp = tmp;
  801. seqhi = esp_tmp_extra(tmp);
  802. iv = esp_tmp_iv(aead, tmp, seqhilen);
  803. req = esp_tmp_req(aead, iv);
  804. sg = esp_req_sg(aead, req);
  805. esp_input_set_header(skb, seqhi);
  806. sg_init_table(sg, nfrags);
  807. ret = skb_to_sgvec(skb, sg, 0, skb->len);
  808. if (unlikely(ret < 0)) {
  809. kfree(tmp);
  810. goto out;
  811. }
  812. skb->ip_summed = CHECKSUM_NONE;
  813. if ((x->props.flags & XFRM_STATE_ESN))
  814. aead_request_set_callback(req, 0, esp_input_done_esn, skb);
  815. else
  816. aead_request_set_callback(req, 0, esp_input_done, skb);
  817. aead_request_set_crypt(req, sg, sg, elen + ivlen, iv);
  818. aead_request_set_ad(req, assoclen);
  819. ret = crypto_aead_decrypt(req);
  820. if (ret == -EINPROGRESS)
  821. goto out;
  822. if ((x->props.flags & XFRM_STATE_ESN))
  823. esp_input_restore_header(skb);
  824. ret = esp6_input_done2(skb, ret);
  825. out:
  826. return ret;
  827. }
  828. static int esp6_err(struct sk_buff *skb, struct inet6_skb_parm *opt,
  829. u8 type, u8 code, int offset, __be32 info)
  830. {
  831. struct net *net = dev_net(skb->dev);
  832. const struct ipv6hdr *iph = (const struct ipv6hdr *)skb->data;
  833. struct ip_esp_hdr *esph = (struct ip_esp_hdr *)(skb->data + offset);
  834. struct xfrm_state *x;
  835. if (type != ICMPV6_PKT_TOOBIG &&
  836. type != NDISC_REDIRECT)
  837. return 0;
  838. x = xfrm_state_lookup(net, skb->mark, (const xfrm_address_t *)&iph->daddr,
  839. esph->spi, IPPROTO_ESP, AF_INET6);
  840. if (!x)
  841. return 0;
  842. if (type == NDISC_REDIRECT)
  843. ip6_redirect(skb, net, skb->dev->ifindex, 0,
  844. sock_net_uid(net, NULL));
  845. else
  846. ip6_update_pmtu(skb, net, info, 0, 0, sock_net_uid(net, NULL));
  847. xfrm_state_put(x);
  848. return 0;
  849. }
  850. static void esp6_destroy(struct xfrm_state *x)
  851. {
  852. struct crypto_aead *aead = x->data;
  853. if (!aead)
  854. return;
  855. crypto_free_aead(aead);
  856. }
  857. static int esp_init_aead(struct xfrm_state *x)
  858. {
  859. char aead_name[CRYPTO_MAX_ALG_NAME];
  860. struct crypto_aead *aead;
  861. int err;
  862. err = -ENAMETOOLONG;
  863. if (snprintf(aead_name, CRYPTO_MAX_ALG_NAME, "%s(%s)",
  864. x->geniv, x->aead->alg_name) >= CRYPTO_MAX_ALG_NAME)
  865. goto error;
  866. aead = crypto_alloc_aead(aead_name, 0, 0);
  867. err = PTR_ERR(aead);
  868. if (IS_ERR(aead))
  869. goto error;
  870. x->data = aead;
  871. err = crypto_aead_setkey(aead, x->aead->alg_key,
  872. (x->aead->alg_key_len + 7) / 8);
  873. if (err)
  874. goto error;
  875. err = crypto_aead_setauthsize(aead, x->aead->alg_icv_len / 8);
  876. if (err)
  877. goto error;
  878. error:
  879. return err;
  880. }
  881. static int esp_init_authenc(struct xfrm_state *x)
  882. {
  883. struct crypto_aead *aead;
  884. struct crypto_authenc_key_param *param;
  885. struct rtattr *rta;
  886. char *key;
  887. char *p;
  888. char authenc_name[CRYPTO_MAX_ALG_NAME];
  889. unsigned int keylen;
  890. int err;
  891. err = -EINVAL;
  892. if (!x->ealg)
  893. goto error;
  894. err = -ENAMETOOLONG;
  895. if ((x->props.flags & XFRM_STATE_ESN)) {
  896. if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
  897. "%s%sauthencesn(%s,%s)%s",
  898. x->geniv ?: "", x->geniv ? "(" : "",
  899. x->aalg ? x->aalg->alg_name : "digest_null",
  900. x->ealg->alg_name,
  901. x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
  902. goto error;
  903. } else {
  904. if (snprintf(authenc_name, CRYPTO_MAX_ALG_NAME,
  905. "%s%sauthenc(%s,%s)%s",
  906. x->geniv ?: "", x->geniv ? "(" : "",
  907. x->aalg ? x->aalg->alg_name : "digest_null",
  908. x->ealg->alg_name,
  909. x->geniv ? ")" : "") >= CRYPTO_MAX_ALG_NAME)
  910. goto error;
  911. }
  912. aead = crypto_alloc_aead(authenc_name, 0, 0);
  913. err = PTR_ERR(aead);
  914. if (IS_ERR(aead))
  915. goto error;
  916. x->data = aead;
  917. keylen = (x->aalg ? (x->aalg->alg_key_len + 7) / 8 : 0) +
  918. (x->ealg->alg_key_len + 7) / 8 + RTA_SPACE(sizeof(*param));
  919. err = -ENOMEM;
  920. key = kmalloc(keylen, GFP_KERNEL);
  921. if (!key)
  922. goto error;
  923. p = key;
  924. rta = (void *)p;
  925. rta->rta_type = CRYPTO_AUTHENC_KEYA_PARAM;
  926. rta->rta_len = RTA_LENGTH(sizeof(*param));
  927. param = RTA_DATA(rta);
  928. p += RTA_SPACE(sizeof(*param));
  929. if (x->aalg) {
  930. struct xfrm_algo_desc *aalg_desc;
  931. memcpy(p, x->aalg->alg_key, (x->aalg->alg_key_len + 7) / 8);
  932. p += (x->aalg->alg_key_len + 7) / 8;
  933. aalg_desc = xfrm_aalg_get_byname(x->aalg->alg_name, 0);
  934. BUG_ON(!aalg_desc);
  935. err = -EINVAL;
  936. if (aalg_desc->uinfo.auth.icv_fullbits / 8 !=
  937. crypto_aead_authsize(aead)) {
  938. pr_info("ESP: %s digestsize %u != %hu\n",
  939. x->aalg->alg_name,
  940. crypto_aead_authsize(aead),
  941. aalg_desc->uinfo.auth.icv_fullbits / 8);
  942. goto free_key;
  943. }
  944. err = crypto_aead_setauthsize(
  945. aead, x->aalg->alg_trunc_len / 8);
  946. if (err)
  947. goto free_key;
  948. }
  949. param->enckeylen = cpu_to_be32((x->ealg->alg_key_len + 7) / 8);
  950. memcpy(p, x->ealg->alg_key, (x->ealg->alg_key_len + 7) / 8);
  951. err = crypto_aead_setkey(aead, key, keylen);
  952. free_key:
  953. kfree(key);
  954. error:
  955. return err;
  956. }
  957. static int esp6_init_state(struct xfrm_state *x)
  958. {
  959. struct crypto_aead *aead;
  960. u32 align;
  961. int err;
  962. x->data = NULL;
  963. if (x->aead)
  964. err = esp_init_aead(x);
  965. else
  966. err = esp_init_authenc(x);
  967. if (err)
  968. goto error;
  969. aead = x->data;
  970. x->props.header_len = sizeof(struct ip_esp_hdr) +
  971. crypto_aead_ivsize(aead);
  972. switch (x->props.mode) {
  973. case XFRM_MODE_BEET:
  974. if (x->sel.family != AF_INET6)
  975. x->props.header_len += IPV4_BEET_PHMAXLEN +
  976. (sizeof(struct ipv6hdr) - sizeof(struct iphdr));
  977. break;
  978. default:
  979. case XFRM_MODE_TRANSPORT:
  980. break;
  981. case XFRM_MODE_TUNNEL:
  982. x->props.header_len += sizeof(struct ipv6hdr);
  983. break;
  984. }
  985. if (x->encap) {
  986. struct xfrm_encap_tmpl *encap = x->encap;
  987. switch (encap->encap_type) {
  988. default:
  989. err = -EINVAL;
  990. goto error;
  991. case UDP_ENCAP_ESPINUDP:
  992. x->props.header_len += sizeof(struct udphdr);
  993. break;
  994. case UDP_ENCAP_ESPINUDP_NON_IKE:
  995. x->props.header_len += sizeof(struct udphdr) + 2 * sizeof(u32);
  996. break;
  997. #ifdef CONFIG_INET6_ESPINTCP
  998. case TCP_ENCAP_ESPINTCP:
  999. /* only the length field, TCP encap is done by
  1000. * the socket
  1001. */
  1002. x->props.header_len += 2;
  1003. break;
  1004. #endif
  1005. }
  1006. }
  1007. align = ALIGN(crypto_aead_blocksize(aead), 4);
  1008. x->props.trailer_len = align + 1 + crypto_aead_authsize(aead);
  1009. error:
  1010. return err;
  1011. }
  1012. static int esp6_rcv_cb(struct sk_buff *skb, int err)
  1013. {
  1014. return 0;
  1015. }
  1016. static const struct xfrm_type esp6_type = {
  1017. .description = "ESP6",
  1018. .owner = THIS_MODULE,
  1019. .proto = IPPROTO_ESP,
  1020. .flags = XFRM_TYPE_REPLAY_PROT,
  1021. .init_state = esp6_init_state,
  1022. .destructor = esp6_destroy,
  1023. .input = esp6_input,
  1024. .output = esp6_output,
  1025. .hdr_offset = xfrm6_find_1stfragopt,
  1026. };
  1027. static struct xfrm6_protocol esp6_protocol = {
  1028. .handler = xfrm6_rcv,
  1029. .input_handler = xfrm_input,
  1030. .cb_handler = esp6_rcv_cb,
  1031. .err_handler = esp6_err,
  1032. .priority = 0,
  1033. };
  1034. static int __init esp6_init(void)
  1035. {
  1036. if (xfrm_register_type(&esp6_type, AF_INET6) < 0) {
  1037. pr_info("%s: can't add xfrm type\n", __func__);
  1038. return -EAGAIN;
  1039. }
  1040. if (xfrm6_protocol_register(&esp6_protocol, IPPROTO_ESP) < 0) {
  1041. pr_info("%s: can't add protocol\n", __func__);
  1042. xfrm_unregister_type(&esp6_type, AF_INET6);
  1043. return -EAGAIN;
  1044. }
  1045. return 0;
  1046. }
  1047. static void __exit esp6_fini(void)
  1048. {
  1049. if (xfrm6_protocol_deregister(&esp6_protocol, IPPROTO_ESP) < 0)
  1050. pr_info("%s: can't remove protocol\n", __func__);
  1051. xfrm_unregister_type(&esp6_type, AF_INET6);
  1052. }
  1053. module_init(esp6_init);
  1054. module_exit(esp6_fini);
  1055. MODULE_LICENSE("GPL");
  1056. MODULE_ALIAS_XFRM_TYPE(AF_INET6, XFRM_PROTO_ESP);