actions.c 39 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2007-2017 Nicira, Inc.
  4. */
  5. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  6. #include <linux/skbuff.h>
  7. #include <linux/in.h>
  8. #include <linux/ip.h>
  9. #include <linux/openvswitch.h>
  10. #include <linux/sctp.h>
  11. #include <linux/tcp.h>
  12. #include <linux/udp.h>
  13. #include <linux/in6.h>
  14. #include <linux/if_arp.h>
  15. #include <linux/if_vlan.h>
  16. #include <net/dst.h>
  17. #include <net/ip.h>
  18. #include <net/ipv6.h>
  19. #include <net/ip6_fib.h>
  20. #include <net/checksum.h>
  21. #include <net/dsfield.h>
  22. #include <net/mpls.h>
  23. #include <net/sctp/checksum.h>
  24. #include "datapath.h"
  25. #include "flow.h"
  26. #include "conntrack.h"
  27. #include "vport.h"
  28. #include "flow_netlink.h"
  29. struct deferred_action {
  30. struct sk_buff *skb;
  31. const struct nlattr *actions;
  32. int actions_len;
  33. /* Store pkt_key clone when creating deferred action. */
  34. struct sw_flow_key pkt_key;
  35. };
  36. #define MAX_L2_LEN (VLAN_ETH_HLEN + 3 * MPLS_HLEN)
  37. struct ovs_frag_data {
  38. unsigned long dst;
  39. struct vport *vport;
  40. struct ovs_skb_cb cb;
  41. __be16 inner_protocol;
  42. u16 network_offset; /* valid only for MPLS */
  43. u16 vlan_tci;
  44. __be16 vlan_proto;
  45. unsigned int l2_len;
  46. u8 mac_proto;
  47. u8 l2_data[MAX_L2_LEN];
  48. };
  49. static DEFINE_PER_CPU(struct ovs_frag_data, ovs_frag_data_storage);
  50. #define DEFERRED_ACTION_FIFO_SIZE 10
  51. #define OVS_RECURSION_LIMIT 5
  52. #define OVS_DEFERRED_ACTION_THRESHOLD (OVS_RECURSION_LIMIT - 2)
  53. struct action_fifo {
  54. int head;
  55. int tail;
  56. /* Deferred action fifo queue storage. */
  57. struct deferred_action fifo[DEFERRED_ACTION_FIFO_SIZE];
  58. };
  59. struct action_flow_keys {
  60. struct sw_flow_key key[OVS_DEFERRED_ACTION_THRESHOLD];
  61. };
  62. static struct action_fifo __percpu *action_fifos;
  63. static struct action_flow_keys __percpu *flow_keys;
  64. static DEFINE_PER_CPU(int, exec_actions_level);
  65. /* Make a clone of the 'key', using the pre-allocated percpu 'flow_keys'
  66. * space. Return NULL if out of key spaces.
  67. */
  68. static struct sw_flow_key *clone_key(const struct sw_flow_key *key_)
  69. {
  70. struct action_flow_keys *keys = this_cpu_ptr(flow_keys);
  71. int level = this_cpu_read(exec_actions_level);
  72. struct sw_flow_key *key = NULL;
  73. if (level <= OVS_DEFERRED_ACTION_THRESHOLD) {
  74. key = &keys->key[level - 1];
  75. *key = *key_;
  76. }
  77. return key;
  78. }
  79. static void action_fifo_init(struct action_fifo *fifo)
  80. {
  81. fifo->head = 0;
  82. fifo->tail = 0;
  83. }
  84. static bool action_fifo_is_empty(const struct action_fifo *fifo)
  85. {
  86. return (fifo->head == fifo->tail);
  87. }
  88. static struct deferred_action *action_fifo_get(struct action_fifo *fifo)
  89. {
  90. if (action_fifo_is_empty(fifo))
  91. return NULL;
  92. return &fifo->fifo[fifo->tail++];
  93. }
  94. static struct deferred_action *action_fifo_put(struct action_fifo *fifo)
  95. {
  96. if (fifo->head >= DEFERRED_ACTION_FIFO_SIZE - 1)
  97. return NULL;
  98. return &fifo->fifo[fifo->head++];
  99. }
  100. /* Return true if fifo is not full */
  101. static struct deferred_action *add_deferred_actions(struct sk_buff *skb,
  102. const struct sw_flow_key *key,
  103. const struct nlattr *actions,
  104. const int actions_len)
  105. {
  106. struct action_fifo *fifo;
  107. struct deferred_action *da;
  108. fifo = this_cpu_ptr(action_fifos);
  109. da = action_fifo_put(fifo);
  110. if (da) {
  111. da->skb = skb;
  112. da->actions = actions;
  113. da->actions_len = actions_len;
  114. da->pkt_key = *key;
  115. }
  116. return da;
  117. }
  118. static void invalidate_flow_key(struct sw_flow_key *key)
  119. {
  120. key->mac_proto |= SW_FLOW_KEY_INVALID;
  121. }
  122. static bool is_flow_key_valid(const struct sw_flow_key *key)
  123. {
  124. return !(key->mac_proto & SW_FLOW_KEY_INVALID);
  125. }
  126. static int clone_execute(struct datapath *dp, struct sk_buff *skb,
  127. struct sw_flow_key *key,
  128. u32 recirc_id,
  129. const struct nlattr *actions, int len,
  130. bool last, bool clone_flow_key);
  131. static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
  132. struct sw_flow_key *key,
  133. const struct nlattr *attr, int len);
  134. static int push_mpls(struct sk_buff *skb, struct sw_flow_key *key,
  135. __be32 mpls_lse, __be16 mpls_ethertype, __u16 mac_len)
  136. {
  137. int err;
  138. err = skb_mpls_push(skb, mpls_lse, mpls_ethertype, mac_len, !!mac_len);
  139. if (err)
  140. return err;
  141. if (!mac_len)
  142. key->mac_proto = MAC_PROTO_NONE;
  143. invalidate_flow_key(key);
  144. return 0;
  145. }
  146. static int pop_mpls(struct sk_buff *skb, struct sw_flow_key *key,
  147. const __be16 ethertype)
  148. {
  149. int err;
  150. err = skb_mpls_pop(skb, ethertype, skb->mac_len,
  151. ovs_key_mac_proto(key) == MAC_PROTO_ETHERNET);
  152. if (err)
  153. return err;
  154. if (ethertype == htons(ETH_P_TEB))
  155. key->mac_proto = MAC_PROTO_ETHERNET;
  156. invalidate_flow_key(key);
  157. return 0;
  158. }
  159. static int set_mpls(struct sk_buff *skb, struct sw_flow_key *flow_key,
  160. const __be32 *mpls_lse, const __be32 *mask)
  161. {
  162. struct mpls_shim_hdr *stack;
  163. __be32 lse;
  164. int err;
  165. if (!pskb_may_pull(skb, skb_network_offset(skb) + MPLS_HLEN))
  166. return -ENOMEM;
  167. stack = mpls_hdr(skb);
  168. lse = OVS_MASKED(stack->label_stack_entry, *mpls_lse, *mask);
  169. err = skb_mpls_update_lse(skb, lse);
  170. if (err)
  171. return err;
  172. flow_key->mpls.lse[0] = lse;
  173. return 0;
  174. }
  175. static int pop_vlan(struct sk_buff *skb, struct sw_flow_key *key)
  176. {
  177. int err;
  178. err = skb_vlan_pop(skb);
  179. if (skb_vlan_tag_present(skb)) {
  180. invalidate_flow_key(key);
  181. } else {
  182. key->eth.vlan.tci = 0;
  183. key->eth.vlan.tpid = 0;
  184. }
  185. return err;
  186. }
  187. static int push_vlan(struct sk_buff *skb, struct sw_flow_key *key,
  188. const struct ovs_action_push_vlan *vlan)
  189. {
  190. if (skb_vlan_tag_present(skb)) {
  191. invalidate_flow_key(key);
  192. } else {
  193. key->eth.vlan.tci = vlan->vlan_tci;
  194. key->eth.vlan.tpid = vlan->vlan_tpid;
  195. }
  196. return skb_vlan_push(skb, vlan->vlan_tpid,
  197. ntohs(vlan->vlan_tci) & ~VLAN_CFI_MASK);
  198. }
  199. /* 'src' is already properly masked. */
  200. static void ether_addr_copy_masked(u8 *dst_, const u8 *src_, const u8 *mask_)
  201. {
  202. u16 *dst = (u16 *)dst_;
  203. const u16 *src = (const u16 *)src_;
  204. const u16 *mask = (const u16 *)mask_;
  205. OVS_SET_MASKED(dst[0], src[0], mask[0]);
  206. OVS_SET_MASKED(dst[1], src[1], mask[1]);
  207. OVS_SET_MASKED(dst[2], src[2], mask[2]);
  208. }
  209. static int set_eth_addr(struct sk_buff *skb, struct sw_flow_key *flow_key,
  210. const struct ovs_key_ethernet *key,
  211. const struct ovs_key_ethernet *mask)
  212. {
  213. int err;
  214. err = skb_ensure_writable(skb, ETH_HLEN);
  215. if (unlikely(err))
  216. return err;
  217. skb_postpull_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);
  218. ether_addr_copy_masked(eth_hdr(skb)->h_source, key->eth_src,
  219. mask->eth_src);
  220. ether_addr_copy_masked(eth_hdr(skb)->h_dest, key->eth_dst,
  221. mask->eth_dst);
  222. skb_postpush_rcsum(skb, eth_hdr(skb), ETH_ALEN * 2);
  223. ether_addr_copy(flow_key->eth.src, eth_hdr(skb)->h_source);
  224. ether_addr_copy(flow_key->eth.dst, eth_hdr(skb)->h_dest);
  225. return 0;
  226. }
  227. /* pop_eth does not support VLAN packets as this action is never called
  228. * for them.
  229. */
  230. static int pop_eth(struct sk_buff *skb, struct sw_flow_key *key)
  231. {
  232. int err;
  233. err = skb_eth_pop(skb);
  234. if (err)
  235. return err;
  236. /* safe right before invalidate_flow_key */
  237. key->mac_proto = MAC_PROTO_NONE;
  238. invalidate_flow_key(key);
  239. return 0;
  240. }
  241. static int push_eth(struct sk_buff *skb, struct sw_flow_key *key,
  242. const struct ovs_action_push_eth *ethh)
  243. {
  244. int err;
  245. err = skb_eth_push(skb, ethh->addresses.eth_dst,
  246. ethh->addresses.eth_src);
  247. if (err)
  248. return err;
  249. /* safe right before invalidate_flow_key */
  250. key->mac_proto = MAC_PROTO_ETHERNET;
  251. invalidate_flow_key(key);
  252. return 0;
  253. }
  254. static int push_nsh(struct sk_buff *skb, struct sw_flow_key *key,
  255. const struct nshhdr *nh)
  256. {
  257. int err;
  258. err = nsh_push(skb, nh);
  259. if (err)
  260. return err;
  261. /* safe right before invalidate_flow_key */
  262. key->mac_proto = MAC_PROTO_NONE;
  263. invalidate_flow_key(key);
  264. return 0;
  265. }
  266. static int pop_nsh(struct sk_buff *skb, struct sw_flow_key *key)
  267. {
  268. int err;
  269. err = nsh_pop(skb);
  270. if (err)
  271. return err;
  272. /* safe right before invalidate_flow_key */
  273. if (skb->protocol == htons(ETH_P_TEB))
  274. key->mac_proto = MAC_PROTO_ETHERNET;
  275. else
  276. key->mac_proto = MAC_PROTO_NONE;
  277. invalidate_flow_key(key);
  278. return 0;
  279. }
  280. static void update_ip_l4_checksum(struct sk_buff *skb, struct iphdr *nh,
  281. __be32 addr, __be32 new_addr)
  282. {
  283. int transport_len = skb->len - skb_transport_offset(skb);
  284. if (nh->frag_off & htons(IP_OFFSET))
  285. return;
  286. if (nh->protocol == IPPROTO_TCP) {
  287. if (likely(transport_len >= sizeof(struct tcphdr)))
  288. inet_proto_csum_replace4(&tcp_hdr(skb)->check, skb,
  289. addr, new_addr, true);
  290. } else if (nh->protocol == IPPROTO_UDP) {
  291. if (likely(transport_len >= sizeof(struct udphdr))) {
  292. struct udphdr *uh = udp_hdr(skb);
  293. if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) {
  294. inet_proto_csum_replace4(&uh->check, skb,
  295. addr, new_addr, true);
  296. if (!uh->check)
  297. uh->check = CSUM_MANGLED_0;
  298. }
  299. }
  300. }
  301. }
  302. static void set_ip_addr(struct sk_buff *skb, struct iphdr *nh,
  303. __be32 *addr, __be32 new_addr)
  304. {
  305. update_ip_l4_checksum(skb, nh, *addr, new_addr);
  306. csum_replace4(&nh->check, *addr, new_addr);
  307. skb_clear_hash(skb);
  308. *addr = new_addr;
  309. }
  310. static void update_ipv6_checksum(struct sk_buff *skb, u8 l4_proto,
  311. __be32 addr[4], const __be32 new_addr[4])
  312. {
  313. int transport_len = skb->len - skb_transport_offset(skb);
  314. if (l4_proto == NEXTHDR_TCP) {
  315. if (likely(transport_len >= sizeof(struct tcphdr)))
  316. inet_proto_csum_replace16(&tcp_hdr(skb)->check, skb,
  317. addr, new_addr, true);
  318. } else if (l4_proto == NEXTHDR_UDP) {
  319. if (likely(transport_len >= sizeof(struct udphdr))) {
  320. struct udphdr *uh = udp_hdr(skb);
  321. if (uh->check || skb->ip_summed == CHECKSUM_PARTIAL) {
  322. inet_proto_csum_replace16(&uh->check, skb,
  323. addr, new_addr, true);
  324. if (!uh->check)
  325. uh->check = CSUM_MANGLED_0;
  326. }
  327. }
  328. } else if (l4_proto == NEXTHDR_ICMP) {
  329. if (likely(transport_len >= sizeof(struct icmp6hdr)))
  330. inet_proto_csum_replace16(&icmp6_hdr(skb)->icmp6_cksum,
  331. skb, addr, new_addr, true);
  332. }
  333. }
  334. static void mask_ipv6_addr(const __be32 old[4], const __be32 addr[4],
  335. const __be32 mask[4], __be32 masked[4])
  336. {
  337. masked[0] = OVS_MASKED(old[0], addr[0], mask[0]);
  338. masked[1] = OVS_MASKED(old[1], addr[1], mask[1]);
  339. masked[2] = OVS_MASKED(old[2], addr[2], mask[2]);
  340. masked[3] = OVS_MASKED(old[3], addr[3], mask[3]);
  341. }
  342. static void set_ipv6_addr(struct sk_buff *skb, u8 l4_proto,
  343. __be32 addr[4], const __be32 new_addr[4],
  344. bool recalculate_csum)
  345. {
  346. if (recalculate_csum)
  347. update_ipv6_checksum(skb, l4_proto, addr, new_addr);
  348. skb_clear_hash(skb);
  349. memcpy(addr, new_addr, sizeof(__be32[4]));
  350. }
  351. static void set_ipv6_dsfield(struct sk_buff *skb, struct ipv6hdr *nh, u8 ipv6_tclass, u8 mask)
  352. {
  353. u8 old_ipv6_tclass = ipv6_get_dsfield(nh);
  354. ipv6_tclass = OVS_MASKED(old_ipv6_tclass, ipv6_tclass, mask);
  355. if (skb->ip_summed == CHECKSUM_COMPLETE)
  356. csum_replace(&skb->csum, (__force __wsum)(old_ipv6_tclass << 12),
  357. (__force __wsum)(ipv6_tclass << 12));
  358. ipv6_change_dsfield(nh, ~mask, ipv6_tclass);
  359. }
  360. static void set_ipv6_fl(struct sk_buff *skb, struct ipv6hdr *nh, u32 fl, u32 mask)
  361. {
  362. u32 ofl;
  363. ofl = nh->flow_lbl[0] << 16 | nh->flow_lbl[1] << 8 | nh->flow_lbl[2];
  364. fl = OVS_MASKED(ofl, fl, mask);
  365. /* Bits 21-24 are always unmasked, so this retains their values. */
  366. nh->flow_lbl[0] = (u8)(fl >> 16);
  367. nh->flow_lbl[1] = (u8)(fl >> 8);
  368. nh->flow_lbl[2] = (u8)fl;
  369. if (skb->ip_summed == CHECKSUM_COMPLETE)
  370. csum_replace(&skb->csum, (__force __wsum)htonl(ofl), (__force __wsum)htonl(fl));
  371. }
  372. static void set_ipv6_ttl(struct sk_buff *skb, struct ipv6hdr *nh, u8 new_ttl, u8 mask)
  373. {
  374. new_ttl = OVS_MASKED(nh->hop_limit, new_ttl, mask);
  375. if (skb->ip_summed == CHECKSUM_COMPLETE)
  376. csum_replace(&skb->csum, (__force __wsum)(nh->hop_limit << 8),
  377. (__force __wsum)(new_ttl << 8));
  378. nh->hop_limit = new_ttl;
  379. }
  380. static void set_ip_ttl(struct sk_buff *skb, struct iphdr *nh, u8 new_ttl,
  381. u8 mask)
  382. {
  383. new_ttl = OVS_MASKED(nh->ttl, new_ttl, mask);
  384. csum_replace2(&nh->check, htons(nh->ttl << 8), htons(new_ttl << 8));
  385. nh->ttl = new_ttl;
  386. }
  387. static int set_ipv4(struct sk_buff *skb, struct sw_flow_key *flow_key,
  388. const struct ovs_key_ipv4 *key,
  389. const struct ovs_key_ipv4 *mask)
  390. {
  391. struct iphdr *nh;
  392. __be32 new_addr;
  393. int err;
  394. err = skb_ensure_writable(skb, skb_network_offset(skb) +
  395. sizeof(struct iphdr));
  396. if (unlikely(err))
  397. return err;
  398. nh = ip_hdr(skb);
  399. /* Setting an IP addresses is typically only a side effect of
  400. * matching on them in the current userspace implementation, so it
  401. * makes sense to check if the value actually changed.
  402. */
  403. if (mask->ipv4_src) {
  404. new_addr = OVS_MASKED(nh->saddr, key->ipv4_src, mask->ipv4_src);
  405. if (unlikely(new_addr != nh->saddr)) {
  406. set_ip_addr(skb, nh, &nh->saddr, new_addr);
  407. flow_key->ipv4.addr.src = new_addr;
  408. }
  409. }
  410. if (mask->ipv4_dst) {
  411. new_addr = OVS_MASKED(nh->daddr, key->ipv4_dst, mask->ipv4_dst);
  412. if (unlikely(new_addr != nh->daddr)) {
  413. set_ip_addr(skb, nh, &nh->daddr, new_addr);
  414. flow_key->ipv4.addr.dst = new_addr;
  415. }
  416. }
  417. if (mask->ipv4_tos) {
  418. ipv4_change_dsfield(nh, ~mask->ipv4_tos, key->ipv4_tos);
  419. flow_key->ip.tos = nh->tos;
  420. }
  421. if (mask->ipv4_ttl) {
  422. set_ip_ttl(skb, nh, key->ipv4_ttl, mask->ipv4_ttl);
  423. flow_key->ip.ttl = nh->ttl;
  424. }
  425. return 0;
  426. }
  427. static bool is_ipv6_mask_nonzero(const __be32 addr[4])
  428. {
  429. return !!(addr[0] | addr[1] | addr[2] | addr[3]);
  430. }
  431. static int set_ipv6(struct sk_buff *skb, struct sw_flow_key *flow_key,
  432. const struct ovs_key_ipv6 *key,
  433. const struct ovs_key_ipv6 *mask)
  434. {
  435. struct ipv6hdr *nh;
  436. int err;
  437. err = skb_ensure_writable(skb, skb_network_offset(skb) +
  438. sizeof(struct ipv6hdr));
  439. if (unlikely(err))
  440. return err;
  441. nh = ipv6_hdr(skb);
  442. /* Setting an IP addresses is typically only a side effect of
  443. * matching on them in the current userspace implementation, so it
  444. * makes sense to check if the value actually changed.
  445. */
  446. if (is_ipv6_mask_nonzero(mask->ipv6_src)) {
  447. __be32 *saddr = (__be32 *)&nh->saddr;
  448. __be32 masked[4];
  449. mask_ipv6_addr(saddr, key->ipv6_src, mask->ipv6_src, masked);
  450. if (unlikely(memcmp(saddr, masked, sizeof(masked)))) {
  451. set_ipv6_addr(skb, flow_key->ip.proto, saddr, masked,
  452. true);
  453. memcpy(&flow_key->ipv6.addr.src, masked,
  454. sizeof(flow_key->ipv6.addr.src));
  455. }
  456. }
  457. if (is_ipv6_mask_nonzero(mask->ipv6_dst)) {
  458. unsigned int offset = 0;
  459. int flags = IP6_FH_F_SKIP_RH;
  460. bool recalc_csum = true;
  461. __be32 *daddr = (__be32 *)&nh->daddr;
  462. __be32 masked[4];
  463. mask_ipv6_addr(daddr, key->ipv6_dst, mask->ipv6_dst, masked);
  464. if (unlikely(memcmp(daddr, masked, sizeof(masked)))) {
  465. if (ipv6_ext_hdr(nh->nexthdr))
  466. recalc_csum = (ipv6_find_hdr(skb, &offset,
  467. NEXTHDR_ROUTING,
  468. NULL, &flags)
  469. != NEXTHDR_ROUTING);
  470. set_ipv6_addr(skb, flow_key->ip.proto, daddr, masked,
  471. recalc_csum);
  472. memcpy(&flow_key->ipv6.addr.dst, masked,
  473. sizeof(flow_key->ipv6.addr.dst));
  474. }
  475. }
  476. if (mask->ipv6_tclass) {
  477. set_ipv6_dsfield(skb, nh, key->ipv6_tclass, mask->ipv6_tclass);
  478. flow_key->ip.tos = ipv6_get_dsfield(nh);
  479. }
  480. if (mask->ipv6_label) {
  481. set_ipv6_fl(skb, nh, ntohl(key->ipv6_label),
  482. ntohl(mask->ipv6_label));
  483. flow_key->ipv6.label =
  484. *(__be32 *)nh & htonl(IPV6_FLOWINFO_FLOWLABEL);
  485. }
  486. if (mask->ipv6_hlimit) {
  487. set_ipv6_ttl(skb, nh, key->ipv6_hlimit, mask->ipv6_hlimit);
  488. flow_key->ip.ttl = nh->hop_limit;
  489. }
  490. return 0;
  491. }
  492. static int set_nsh(struct sk_buff *skb, struct sw_flow_key *flow_key,
  493. const struct nlattr *a)
  494. {
  495. struct nshhdr *nh;
  496. size_t length;
  497. int err;
  498. u8 flags;
  499. u8 ttl;
  500. int i;
  501. struct ovs_key_nsh key;
  502. struct ovs_key_nsh mask;
  503. err = nsh_key_from_nlattr(a, &key, &mask);
  504. if (err)
  505. return err;
  506. /* Make sure the NSH base header is there */
  507. if (!pskb_may_pull(skb, skb_network_offset(skb) + NSH_BASE_HDR_LEN))
  508. return -ENOMEM;
  509. nh = nsh_hdr(skb);
  510. length = nsh_hdr_len(nh);
  511. /* Make sure the whole NSH header is there */
  512. err = skb_ensure_writable(skb, skb_network_offset(skb) +
  513. length);
  514. if (unlikely(err))
  515. return err;
  516. nh = nsh_hdr(skb);
  517. skb_postpull_rcsum(skb, nh, length);
  518. flags = nsh_get_flags(nh);
  519. flags = OVS_MASKED(flags, key.base.flags, mask.base.flags);
  520. flow_key->nsh.base.flags = flags;
  521. ttl = nsh_get_ttl(nh);
  522. ttl = OVS_MASKED(ttl, key.base.ttl, mask.base.ttl);
  523. flow_key->nsh.base.ttl = ttl;
  524. nsh_set_flags_and_ttl(nh, flags, ttl);
  525. nh->path_hdr = OVS_MASKED(nh->path_hdr, key.base.path_hdr,
  526. mask.base.path_hdr);
  527. flow_key->nsh.base.path_hdr = nh->path_hdr;
  528. switch (nh->mdtype) {
  529. case NSH_M_TYPE1:
  530. for (i = 0; i < NSH_MD1_CONTEXT_SIZE; i++) {
  531. nh->md1.context[i] =
  532. OVS_MASKED(nh->md1.context[i], key.context[i],
  533. mask.context[i]);
  534. }
  535. memcpy(flow_key->nsh.context, nh->md1.context,
  536. sizeof(nh->md1.context));
  537. break;
  538. case NSH_M_TYPE2:
  539. memset(flow_key->nsh.context, 0,
  540. sizeof(flow_key->nsh.context));
  541. break;
  542. default:
  543. return -EINVAL;
  544. }
  545. skb_postpush_rcsum(skb, nh, length);
  546. return 0;
  547. }
  548. /* Must follow skb_ensure_writable() since that can move the skb data. */
  549. static void set_tp_port(struct sk_buff *skb, __be16 *port,
  550. __be16 new_port, __sum16 *check)
  551. {
  552. inet_proto_csum_replace2(check, skb, *port, new_port, false);
  553. *port = new_port;
  554. }
  555. static int set_udp(struct sk_buff *skb, struct sw_flow_key *flow_key,
  556. const struct ovs_key_udp *key,
  557. const struct ovs_key_udp *mask)
  558. {
  559. struct udphdr *uh;
  560. __be16 src, dst;
  561. int err;
  562. err = skb_ensure_writable(skb, skb_transport_offset(skb) +
  563. sizeof(struct udphdr));
  564. if (unlikely(err))
  565. return err;
  566. uh = udp_hdr(skb);
  567. /* Either of the masks is non-zero, so do not bother checking them. */
  568. src = OVS_MASKED(uh->source, key->udp_src, mask->udp_src);
  569. dst = OVS_MASKED(uh->dest, key->udp_dst, mask->udp_dst);
  570. if (uh->check && skb->ip_summed != CHECKSUM_PARTIAL) {
  571. if (likely(src != uh->source)) {
  572. set_tp_port(skb, &uh->source, src, &uh->check);
  573. flow_key->tp.src = src;
  574. }
  575. if (likely(dst != uh->dest)) {
  576. set_tp_port(skb, &uh->dest, dst, &uh->check);
  577. flow_key->tp.dst = dst;
  578. }
  579. if (unlikely(!uh->check))
  580. uh->check = CSUM_MANGLED_0;
  581. } else {
  582. uh->source = src;
  583. uh->dest = dst;
  584. flow_key->tp.src = src;
  585. flow_key->tp.dst = dst;
  586. }
  587. skb_clear_hash(skb);
  588. return 0;
  589. }
  590. static int set_tcp(struct sk_buff *skb, struct sw_flow_key *flow_key,
  591. const struct ovs_key_tcp *key,
  592. const struct ovs_key_tcp *mask)
  593. {
  594. struct tcphdr *th;
  595. __be16 src, dst;
  596. int err;
  597. err = skb_ensure_writable(skb, skb_transport_offset(skb) +
  598. sizeof(struct tcphdr));
  599. if (unlikely(err))
  600. return err;
  601. th = tcp_hdr(skb);
  602. src = OVS_MASKED(th->source, key->tcp_src, mask->tcp_src);
  603. if (likely(src != th->source)) {
  604. set_tp_port(skb, &th->source, src, &th->check);
  605. flow_key->tp.src = src;
  606. }
  607. dst = OVS_MASKED(th->dest, key->tcp_dst, mask->tcp_dst);
  608. if (likely(dst != th->dest)) {
  609. set_tp_port(skb, &th->dest, dst, &th->check);
  610. flow_key->tp.dst = dst;
  611. }
  612. skb_clear_hash(skb);
  613. return 0;
  614. }
  615. static int set_sctp(struct sk_buff *skb, struct sw_flow_key *flow_key,
  616. const struct ovs_key_sctp *key,
  617. const struct ovs_key_sctp *mask)
  618. {
  619. unsigned int sctphoff = skb_transport_offset(skb);
  620. struct sctphdr *sh;
  621. __le32 old_correct_csum, new_csum, old_csum;
  622. int err;
  623. err = skb_ensure_writable(skb, sctphoff + sizeof(struct sctphdr));
  624. if (unlikely(err))
  625. return err;
  626. sh = sctp_hdr(skb);
  627. old_csum = sh->checksum;
  628. old_correct_csum = sctp_compute_cksum(skb, sctphoff);
  629. sh->source = OVS_MASKED(sh->source, key->sctp_src, mask->sctp_src);
  630. sh->dest = OVS_MASKED(sh->dest, key->sctp_dst, mask->sctp_dst);
  631. new_csum = sctp_compute_cksum(skb, sctphoff);
  632. /* Carry any checksum errors through. */
  633. sh->checksum = old_csum ^ old_correct_csum ^ new_csum;
  634. skb_clear_hash(skb);
  635. flow_key->tp.src = sh->source;
  636. flow_key->tp.dst = sh->dest;
  637. return 0;
  638. }
  639. static int ovs_vport_output(struct net *net, struct sock *sk,
  640. struct sk_buff *skb)
  641. {
  642. struct ovs_frag_data *data = this_cpu_ptr(&ovs_frag_data_storage);
  643. struct vport *vport = data->vport;
  644. if (skb_cow_head(skb, data->l2_len) < 0) {
  645. kfree_skb(skb);
  646. return -ENOMEM;
  647. }
  648. __skb_dst_copy(skb, data->dst);
  649. *OVS_CB(skb) = data->cb;
  650. skb->inner_protocol = data->inner_protocol;
  651. if (data->vlan_tci & VLAN_CFI_MASK)
  652. __vlan_hwaccel_put_tag(skb, data->vlan_proto, data->vlan_tci & ~VLAN_CFI_MASK);
  653. else
  654. __vlan_hwaccel_clear_tag(skb);
  655. /* Reconstruct the MAC header. */
  656. skb_push(skb, data->l2_len);
  657. memcpy(skb->data, &data->l2_data, data->l2_len);
  658. skb_postpush_rcsum(skb, skb->data, data->l2_len);
  659. skb_reset_mac_header(skb);
  660. if (eth_p_mpls(skb->protocol)) {
  661. skb->inner_network_header = skb->network_header;
  662. skb_set_network_header(skb, data->network_offset);
  663. skb_reset_mac_len(skb);
  664. }
  665. ovs_vport_send(vport, skb, data->mac_proto);
  666. return 0;
  667. }
  668. static unsigned int
  669. ovs_dst_get_mtu(const struct dst_entry *dst)
  670. {
  671. return dst->dev->mtu;
  672. }
  673. static struct dst_ops ovs_dst_ops = {
  674. .family = AF_UNSPEC,
  675. .mtu = ovs_dst_get_mtu,
  676. };
  677. /* prepare_frag() is called once per (larger-than-MTU) frame; its inverse is
  678. * ovs_vport_output(), which is called once per fragmented packet.
  679. */
  680. static void prepare_frag(struct vport *vport, struct sk_buff *skb,
  681. u16 orig_network_offset, u8 mac_proto)
  682. {
  683. unsigned int hlen = skb_network_offset(skb);
  684. struct ovs_frag_data *data;
  685. data = this_cpu_ptr(&ovs_frag_data_storage);
  686. data->dst = skb->_skb_refdst;
  687. data->vport = vport;
  688. data->cb = *OVS_CB(skb);
  689. data->inner_protocol = skb->inner_protocol;
  690. data->network_offset = orig_network_offset;
  691. if (skb_vlan_tag_present(skb))
  692. data->vlan_tci = skb_vlan_tag_get(skb) | VLAN_CFI_MASK;
  693. else
  694. data->vlan_tci = 0;
  695. data->vlan_proto = skb->vlan_proto;
  696. data->mac_proto = mac_proto;
  697. data->l2_len = hlen;
  698. memcpy(&data->l2_data, skb->data, hlen);
  699. memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
  700. skb_pull(skb, hlen);
  701. }
  702. static void ovs_fragment(struct net *net, struct vport *vport,
  703. struct sk_buff *skb, u16 mru,
  704. struct sw_flow_key *key)
  705. {
  706. u16 orig_network_offset = 0;
  707. if (eth_p_mpls(skb->protocol)) {
  708. orig_network_offset = skb_network_offset(skb);
  709. skb->network_header = skb->inner_network_header;
  710. }
  711. if (skb_network_offset(skb) > MAX_L2_LEN) {
  712. OVS_NLERR(1, "L2 header too long to fragment");
  713. goto err;
  714. }
  715. if (key->eth.type == htons(ETH_P_IP)) {
  716. struct rtable ovs_rt = { 0 };
  717. unsigned long orig_dst;
  718. prepare_frag(vport, skb, orig_network_offset,
  719. ovs_key_mac_proto(key));
  720. dst_init(&ovs_rt.dst, &ovs_dst_ops, NULL, 1,
  721. DST_OBSOLETE_NONE, DST_NOCOUNT);
  722. ovs_rt.dst.dev = vport->dev;
  723. orig_dst = skb->_skb_refdst;
  724. skb_dst_set_noref(skb, &ovs_rt.dst);
  725. IPCB(skb)->frag_max_size = mru;
  726. ip_do_fragment(net, skb->sk, skb, ovs_vport_output);
  727. refdst_drop(orig_dst);
  728. } else if (key->eth.type == htons(ETH_P_IPV6)) {
  729. unsigned long orig_dst;
  730. struct rt6_info ovs_rt;
  731. prepare_frag(vport, skb, orig_network_offset,
  732. ovs_key_mac_proto(key));
  733. memset(&ovs_rt, 0, sizeof(ovs_rt));
  734. dst_init(&ovs_rt.dst, &ovs_dst_ops, NULL, 1,
  735. DST_OBSOLETE_NONE, DST_NOCOUNT);
  736. ovs_rt.dst.dev = vport->dev;
  737. orig_dst = skb->_skb_refdst;
  738. skb_dst_set_noref(skb, &ovs_rt.dst);
  739. IP6CB(skb)->frag_max_size = mru;
  740. ipv6_stub->ipv6_fragment(net, skb->sk, skb, ovs_vport_output);
  741. refdst_drop(orig_dst);
  742. } else {
  743. WARN_ONCE(1, "Failed fragment ->%s: eth=%04x, MRU=%d, MTU=%d.",
  744. ovs_vport_name(vport), ntohs(key->eth.type), mru,
  745. vport->dev->mtu);
  746. goto err;
  747. }
  748. return;
  749. err:
  750. kfree_skb(skb);
  751. }
  752. static void do_output(struct datapath *dp, struct sk_buff *skb, int out_port,
  753. struct sw_flow_key *key)
  754. {
  755. struct vport *vport = ovs_vport_rcu(dp, out_port);
  756. if (likely(vport)) {
  757. u16 mru = OVS_CB(skb)->mru;
  758. u32 cutlen = OVS_CB(skb)->cutlen;
  759. if (unlikely(cutlen > 0)) {
  760. if (skb->len - cutlen > ovs_mac_header_len(key))
  761. pskb_trim(skb, skb->len - cutlen);
  762. else
  763. pskb_trim(skb, ovs_mac_header_len(key));
  764. }
  765. if (likely(!mru ||
  766. (skb->len <= mru + vport->dev->hard_header_len))) {
  767. ovs_vport_send(vport, skb, ovs_key_mac_proto(key));
  768. } else if (mru <= vport->dev->mtu) {
  769. struct net *net = read_pnet(&dp->net);
  770. ovs_fragment(net, vport, skb, mru, key);
  771. } else {
  772. kfree_skb(skb);
  773. }
  774. } else {
  775. kfree_skb(skb);
  776. }
  777. }
  778. static int output_userspace(struct datapath *dp, struct sk_buff *skb,
  779. struct sw_flow_key *key, const struct nlattr *attr,
  780. const struct nlattr *actions, int actions_len,
  781. uint32_t cutlen)
  782. {
  783. struct dp_upcall_info upcall;
  784. const struct nlattr *a;
  785. int rem;
  786. memset(&upcall, 0, sizeof(upcall));
  787. upcall.cmd = OVS_PACKET_CMD_ACTION;
  788. upcall.mru = OVS_CB(skb)->mru;
  789. for (a = nla_data(attr), rem = nla_len(attr); rem > 0;
  790. a = nla_next(a, &rem)) {
  791. switch (nla_type(a)) {
  792. case OVS_USERSPACE_ATTR_USERDATA:
  793. upcall.userdata = a;
  794. break;
  795. case OVS_USERSPACE_ATTR_PID:
  796. upcall.portid = nla_get_u32(a);
  797. break;
  798. case OVS_USERSPACE_ATTR_EGRESS_TUN_PORT: {
  799. /* Get out tunnel info. */
  800. struct vport *vport;
  801. vport = ovs_vport_rcu(dp, nla_get_u32(a));
  802. if (vport) {
  803. int err;
  804. err = dev_fill_metadata_dst(vport->dev, skb);
  805. if (!err)
  806. upcall.egress_tun_info = skb_tunnel_info(skb);
  807. }
  808. break;
  809. }
  810. case OVS_USERSPACE_ATTR_ACTIONS: {
  811. /* Include actions. */
  812. upcall.actions = actions;
  813. upcall.actions_len = actions_len;
  814. break;
  815. }
  816. } /* End of switch. */
  817. }
  818. return ovs_dp_upcall(dp, skb, key, &upcall, cutlen);
  819. }
  820. static int dec_ttl_exception_handler(struct datapath *dp, struct sk_buff *skb,
  821. struct sw_flow_key *key,
  822. const struct nlattr *attr, bool last)
  823. {
  824. /* The first attribute is always 'OVS_DEC_TTL_ATTR_ACTION'. */
  825. struct nlattr *actions = nla_data(attr);
  826. if (nla_len(actions))
  827. return clone_execute(dp, skb, key, 0, nla_data(actions),
  828. nla_len(actions), last, false);
  829. consume_skb(skb);
  830. return 0;
  831. }
  832. /* When 'last' is true, sample() should always consume the 'skb'.
  833. * Otherwise, sample() should keep 'skb' intact regardless what
  834. * actions are executed within sample().
  835. */
  836. static int sample(struct datapath *dp, struct sk_buff *skb,
  837. struct sw_flow_key *key, const struct nlattr *attr,
  838. bool last)
  839. {
  840. struct nlattr *actions;
  841. struct nlattr *sample_arg;
  842. int rem = nla_len(attr);
  843. const struct sample_arg *arg;
  844. bool clone_flow_key;
  845. /* The first action is always 'OVS_SAMPLE_ATTR_ARG'. */
  846. sample_arg = nla_data(attr);
  847. arg = nla_data(sample_arg);
  848. actions = nla_next(sample_arg, &rem);
  849. if ((arg->probability != U32_MAX) &&
  850. (!arg->probability || prandom_u32() > arg->probability)) {
  851. if (last)
  852. consume_skb(skb);
  853. return 0;
  854. }
  855. clone_flow_key = !arg->exec;
  856. return clone_execute(dp, skb, key, 0, actions, rem, last,
  857. clone_flow_key);
  858. }
  859. /* When 'last' is true, clone() should always consume the 'skb'.
  860. * Otherwise, clone() should keep 'skb' intact regardless what
  861. * actions are executed within clone().
  862. */
  863. static int clone(struct datapath *dp, struct sk_buff *skb,
  864. struct sw_flow_key *key, const struct nlattr *attr,
  865. bool last)
  866. {
  867. struct nlattr *actions;
  868. struct nlattr *clone_arg;
  869. int rem = nla_len(attr);
  870. bool dont_clone_flow_key;
  871. /* The first action is always 'OVS_CLONE_ATTR_EXEC'. */
  872. clone_arg = nla_data(attr);
  873. dont_clone_flow_key = nla_get_u32(clone_arg);
  874. actions = nla_next(clone_arg, &rem);
  875. return clone_execute(dp, skb, key, 0, actions, rem, last,
  876. !dont_clone_flow_key);
  877. }
  878. static void execute_hash(struct sk_buff *skb, struct sw_flow_key *key,
  879. const struct nlattr *attr)
  880. {
  881. struct ovs_action_hash *hash_act = nla_data(attr);
  882. u32 hash = 0;
  883. /* OVS_HASH_ALG_L4 is the only possible hash algorithm. */
  884. hash = skb_get_hash(skb);
  885. hash = jhash_1word(hash, hash_act->hash_basis);
  886. if (!hash)
  887. hash = 0x1;
  888. key->ovs_flow_hash = hash;
  889. }
  890. static int execute_set_action(struct sk_buff *skb,
  891. struct sw_flow_key *flow_key,
  892. const struct nlattr *a)
  893. {
  894. /* Only tunnel set execution is supported without a mask. */
  895. if (nla_type(a) == OVS_KEY_ATTR_TUNNEL_INFO) {
  896. struct ovs_tunnel_info *tun = nla_data(a);
  897. skb_dst_drop(skb);
  898. dst_hold((struct dst_entry *)tun->tun_dst);
  899. skb_dst_set(skb, (struct dst_entry *)tun->tun_dst);
  900. return 0;
  901. }
  902. return -EINVAL;
  903. }
  904. /* Mask is at the midpoint of the data. */
  905. #define get_mask(a, type) ((const type)nla_data(a) + 1)
  906. static int execute_masked_set_action(struct sk_buff *skb,
  907. struct sw_flow_key *flow_key,
  908. const struct nlattr *a)
  909. {
  910. int err = 0;
  911. switch (nla_type(a)) {
  912. case OVS_KEY_ATTR_PRIORITY:
  913. OVS_SET_MASKED(skb->priority, nla_get_u32(a),
  914. *get_mask(a, u32 *));
  915. flow_key->phy.priority = skb->priority;
  916. break;
  917. case OVS_KEY_ATTR_SKB_MARK:
  918. OVS_SET_MASKED(skb->mark, nla_get_u32(a), *get_mask(a, u32 *));
  919. flow_key->phy.skb_mark = skb->mark;
  920. break;
  921. case OVS_KEY_ATTR_TUNNEL_INFO:
  922. /* Masked data not supported for tunnel. */
  923. err = -EINVAL;
  924. break;
  925. case OVS_KEY_ATTR_ETHERNET:
  926. err = set_eth_addr(skb, flow_key, nla_data(a),
  927. get_mask(a, struct ovs_key_ethernet *));
  928. break;
  929. case OVS_KEY_ATTR_NSH:
  930. err = set_nsh(skb, flow_key, a);
  931. break;
  932. case OVS_KEY_ATTR_IPV4:
  933. err = set_ipv4(skb, flow_key, nla_data(a),
  934. get_mask(a, struct ovs_key_ipv4 *));
  935. break;
  936. case OVS_KEY_ATTR_IPV6:
  937. err = set_ipv6(skb, flow_key, nla_data(a),
  938. get_mask(a, struct ovs_key_ipv6 *));
  939. break;
  940. case OVS_KEY_ATTR_TCP:
  941. err = set_tcp(skb, flow_key, nla_data(a),
  942. get_mask(a, struct ovs_key_tcp *));
  943. break;
  944. case OVS_KEY_ATTR_UDP:
  945. err = set_udp(skb, flow_key, nla_data(a),
  946. get_mask(a, struct ovs_key_udp *));
  947. break;
  948. case OVS_KEY_ATTR_SCTP:
  949. err = set_sctp(skb, flow_key, nla_data(a),
  950. get_mask(a, struct ovs_key_sctp *));
  951. break;
  952. case OVS_KEY_ATTR_MPLS:
  953. err = set_mpls(skb, flow_key, nla_data(a), get_mask(a,
  954. __be32 *));
  955. break;
  956. case OVS_KEY_ATTR_CT_STATE:
  957. case OVS_KEY_ATTR_CT_ZONE:
  958. case OVS_KEY_ATTR_CT_MARK:
  959. case OVS_KEY_ATTR_CT_LABELS:
  960. case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV4:
  961. case OVS_KEY_ATTR_CT_ORIG_TUPLE_IPV6:
  962. err = -EINVAL;
  963. break;
  964. }
  965. return err;
  966. }
  967. static int execute_recirc(struct datapath *dp, struct sk_buff *skb,
  968. struct sw_flow_key *key,
  969. const struct nlattr *a, bool last)
  970. {
  971. u32 recirc_id;
  972. if (!is_flow_key_valid(key)) {
  973. int err;
  974. err = ovs_flow_key_update(skb, key);
  975. if (err)
  976. return err;
  977. }
  978. BUG_ON(!is_flow_key_valid(key));
  979. recirc_id = nla_get_u32(a);
  980. return clone_execute(dp, skb, key, recirc_id, NULL, 0, last, true);
  981. }
  982. static int execute_check_pkt_len(struct datapath *dp, struct sk_buff *skb,
  983. struct sw_flow_key *key,
  984. const struct nlattr *attr, bool last)
  985. {
  986. struct ovs_skb_cb *ovs_cb = OVS_CB(skb);
  987. const struct nlattr *actions, *cpl_arg;
  988. int len, max_len, rem = nla_len(attr);
  989. const struct check_pkt_len_arg *arg;
  990. bool clone_flow_key;
  991. /* The first netlink attribute in 'attr' is always
  992. * 'OVS_CHECK_PKT_LEN_ATTR_ARG'.
  993. */
  994. cpl_arg = nla_data(attr);
  995. arg = nla_data(cpl_arg);
  996. len = ovs_cb->mru ? ovs_cb->mru + skb->mac_len : skb->len;
  997. max_len = arg->pkt_len;
  998. if ((skb_is_gso(skb) && skb_gso_validate_mac_len(skb, max_len)) ||
  999. len <= max_len) {
  1000. /* Second netlink attribute in 'attr' is always
  1001. * 'OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_LESS_EQUAL'.
  1002. */
  1003. actions = nla_next(cpl_arg, &rem);
  1004. clone_flow_key = !arg->exec_for_lesser_equal;
  1005. } else {
  1006. /* Third netlink attribute in 'attr' is always
  1007. * 'OVS_CHECK_PKT_LEN_ATTR_ACTIONS_IF_GREATER'.
  1008. */
  1009. actions = nla_next(cpl_arg, &rem);
  1010. actions = nla_next(actions, &rem);
  1011. clone_flow_key = !arg->exec_for_greater;
  1012. }
  1013. return clone_execute(dp, skb, key, 0, nla_data(actions),
  1014. nla_len(actions), last, clone_flow_key);
  1015. }
  1016. static int execute_dec_ttl(struct sk_buff *skb, struct sw_flow_key *key)
  1017. {
  1018. int err;
  1019. if (skb->protocol == htons(ETH_P_IPV6)) {
  1020. struct ipv6hdr *nh;
  1021. err = skb_ensure_writable(skb, skb_network_offset(skb) +
  1022. sizeof(*nh));
  1023. if (unlikely(err))
  1024. return err;
  1025. nh = ipv6_hdr(skb);
  1026. if (nh->hop_limit <= 1)
  1027. return -EHOSTUNREACH;
  1028. key->ip.ttl = --nh->hop_limit;
  1029. } else if (skb->protocol == htons(ETH_P_IP)) {
  1030. struct iphdr *nh;
  1031. u8 old_ttl;
  1032. err = skb_ensure_writable(skb, skb_network_offset(skb) +
  1033. sizeof(*nh));
  1034. if (unlikely(err))
  1035. return err;
  1036. nh = ip_hdr(skb);
  1037. if (nh->ttl <= 1)
  1038. return -EHOSTUNREACH;
  1039. old_ttl = nh->ttl--;
  1040. csum_replace2(&nh->check, htons(old_ttl << 8),
  1041. htons(nh->ttl << 8));
  1042. key->ip.ttl = nh->ttl;
  1043. }
  1044. return 0;
  1045. }
  1046. /* Execute a list of actions against 'skb'. */
  1047. static int do_execute_actions(struct datapath *dp, struct sk_buff *skb,
  1048. struct sw_flow_key *key,
  1049. const struct nlattr *attr, int len)
  1050. {
  1051. const struct nlattr *a;
  1052. int rem;
  1053. for (a = attr, rem = len; rem > 0;
  1054. a = nla_next(a, &rem)) {
  1055. int err = 0;
  1056. switch (nla_type(a)) {
  1057. case OVS_ACTION_ATTR_OUTPUT: {
  1058. int port = nla_get_u32(a);
  1059. struct sk_buff *clone;
  1060. /* Every output action needs a separate clone
  1061. * of 'skb', In case the output action is the
  1062. * last action, cloning can be avoided.
  1063. */
  1064. if (nla_is_last(a, rem)) {
  1065. do_output(dp, skb, port, key);
  1066. /* 'skb' has been used for output.
  1067. */
  1068. return 0;
  1069. }
  1070. clone = skb_clone(skb, GFP_ATOMIC);
  1071. if (clone)
  1072. do_output(dp, clone, port, key);
  1073. OVS_CB(skb)->cutlen = 0;
  1074. break;
  1075. }
  1076. case OVS_ACTION_ATTR_TRUNC: {
  1077. struct ovs_action_trunc *trunc = nla_data(a);
  1078. if (skb->len > trunc->max_len)
  1079. OVS_CB(skb)->cutlen = skb->len - trunc->max_len;
  1080. break;
  1081. }
  1082. case OVS_ACTION_ATTR_USERSPACE:
  1083. output_userspace(dp, skb, key, a, attr,
  1084. len, OVS_CB(skb)->cutlen);
  1085. OVS_CB(skb)->cutlen = 0;
  1086. break;
  1087. case OVS_ACTION_ATTR_HASH:
  1088. execute_hash(skb, key, a);
  1089. break;
  1090. case OVS_ACTION_ATTR_PUSH_MPLS: {
  1091. struct ovs_action_push_mpls *mpls = nla_data(a);
  1092. err = push_mpls(skb, key, mpls->mpls_lse,
  1093. mpls->mpls_ethertype, skb->mac_len);
  1094. break;
  1095. }
  1096. case OVS_ACTION_ATTR_ADD_MPLS: {
  1097. struct ovs_action_add_mpls *mpls = nla_data(a);
  1098. __u16 mac_len = 0;
  1099. if (mpls->tun_flags & OVS_MPLS_L3_TUNNEL_FLAG_MASK)
  1100. mac_len = skb->mac_len;
  1101. err = push_mpls(skb, key, mpls->mpls_lse,
  1102. mpls->mpls_ethertype, mac_len);
  1103. break;
  1104. }
  1105. case OVS_ACTION_ATTR_POP_MPLS:
  1106. err = pop_mpls(skb, key, nla_get_be16(a));
  1107. break;
  1108. case OVS_ACTION_ATTR_PUSH_VLAN:
  1109. err = push_vlan(skb, key, nla_data(a));
  1110. break;
  1111. case OVS_ACTION_ATTR_POP_VLAN:
  1112. err = pop_vlan(skb, key);
  1113. break;
  1114. case OVS_ACTION_ATTR_RECIRC: {
  1115. bool last = nla_is_last(a, rem);
  1116. err = execute_recirc(dp, skb, key, a, last);
  1117. if (last) {
  1118. /* If this is the last action, the skb has
  1119. * been consumed or freed.
  1120. * Return immediately.
  1121. */
  1122. return err;
  1123. }
  1124. break;
  1125. }
  1126. case OVS_ACTION_ATTR_SET:
  1127. err = execute_set_action(skb, key, nla_data(a));
  1128. break;
  1129. case OVS_ACTION_ATTR_SET_MASKED:
  1130. case OVS_ACTION_ATTR_SET_TO_MASKED:
  1131. err = execute_masked_set_action(skb, key, nla_data(a));
  1132. break;
  1133. case OVS_ACTION_ATTR_SAMPLE: {
  1134. bool last = nla_is_last(a, rem);
  1135. err = sample(dp, skb, key, a, last);
  1136. if (last)
  1137. return err;
  1138. break;
  1139. }
  1140. case OVS_ACTION_ATTR_CT:
  1141. if (!is_flow_key_valid(key)) {
  1142. err = ovs_flow_key_update(skb, key);
  1143. if (err)
  1144. return err;
  1145. }
  1146. err = ovs_ct_execute(ovs_dp_get_net(dp), skb, key,
  1147. nla_data(a));
  1148. /* Hide stolen IP fragments from user space. */
  1149. if (err)
  1150. return err == -EINPROGRESS ? 0 : err;
  1151. break;
  1152. case OVS_ACTION_ATTR_CT_CLEAR:
  1153. err = ovs_ct_clear(skb, key);
  1154. break;
  1155. case OVS_ACTION_ATTR_PUSH_ETH:
  1156. err = push_eth(skb, key, nla_data(a));
  1157. break;
  1158. case OVS_ACTION_ATTR_POP_ETH:
  1159. err = pop_eth(skb, key);
  1160. break;
  1161. case OVS_ACTION_ATTR_PUSH_NSH: {
  1162. u8 buffer[NSH_HDR_MAX_LEN];
  1163. struct nshhdr *nh = (struct nshhdr *)buffer;
  1164. err = nsh_hdr_from_nlattr(nla_data(a), nh,
  1165. NSH_HDR_MAX_LEN);
  1166. if (unlikely(err))
  1167. break;
  1168. err = push_nsh(skb, key, nh);
  1169. break;
  1170. }
  1171. case OVS_ACTION_ATTR_POP_NSH:
  1172. err = pop_nsh(skb, key);
  1173. break;
  1174. case OVS_ACTION_ATTR_METER:
  1175. if (ovs_meter_execute(dp, skb, key, nla_get_u32(a))) {
  1176. consume_skb(skb);
  1177. return 0;
  1178. }
  1179. break;
  1180. case OVS_ACTION_ATTR_CLONE: {
  1181. bool last = nla_is_last(a, rem);
  1182. err = clone(dp, skb, key, a, last);
  1183. if (last)
  1184. return err;
  1185. break;
  1186. }
  1187. case OVS_ACTION_ATTR_CHECK_PKT_LEN: {
  1188. bool last = nla_is_last(a, rem);
  1189. err = execute_check_pkt_len(dp, skb, key, a, last);
  1190. if (last)
  1191. return err;
  1192. break;
  1193. }
  1194. case OVS_ACTION_ATTR_DEC_TTL:
  1195. err = execute_dec_ttl(skb, key);
  1196. if (err == -EHOSTUNREACH) {
  1197. err = dec_ttl_exception_handler(dp, skb, key,
  1198. a, true);
  1199. return err;
  1200. }
  1201. break;
  1202. }
  1203. if (unlikely(err)) {
  1204. kfree_skb(skb);
  1205. return err;
  1206. }
  1207. }
  1208. consume_skb(skb);
  1209. return 0;
  1210. }
  1211. /* Execute the actions on the clone of the packet. The effect of the
  1212. * execution does not affect the original 'skb' nor the original 'key'.
  1213. *
  1214. * The execution may be deferred in case the actions can not be executed
  1215. * immediately.
  1216. */
  1217. static int clone_execute(struct datapath *dp, struct sk_buff *skb,
  1218. struct sw_flow_key *key, u32 recirc_id,
  1219. const struct nlattr *actions, int len,
  1220. bool last, bool clone_flow_key)
  1221. {
  1222. struct deferred_action *da;
  1223. struct sw_flow_key *clone;
  1224. skb = last ? skb : skb_clone(skb, GFP_ATOMIC);
  1225. if (!skb) {
  1226. /* Out of memory, skip this action.
  1227. */
  1228. return 0;
  1229. }
  1230. /* When clone_flow_key is false, the 'key' will not be change
  1231. * by the actions, then the 'key' can be used directly.
  1232. * Otherwise, try to clone key from the next recursion level of
  1233. * 'flow_keys'. If clone is successful, execute the actions
  1234. * without deferring.
  1235. */
  1236. clone = clone_flow_key ? clone_key(key) : key;
  1237. if (clone) {
  1238. int err = 0;
  1239. if (actions) { /* Sample action */
  1240. if (clone_flow_key)
  1241. __this_cpu_inc(exec_actions_level);
  1242. err = do_execute_actions(dp, skb, clone,
  1243. actions, len);
  1244. if (clone_flow_key)
  1245. __this_cpu_dec(exec_actions_level);
  1246. } else { /* Recirc action */
  1247. clone->recirc_id = recirc_id;
  1248. ovs_dp_process_packet(skb, clone);
  1249. }
  1250. return err;
  1251. }
  1252. /* Out of 'flow_keys' space. Defer actions */
  1253. da = add_deferred_actions(skb, key, actions, len);
  1254. if (da) {
  1255. if (!actions) { /* Recirc action */
  1256. key = &da->pkt_key;
  1257. key->recirc_id = recirc_id;
  1258. }
  1259. } else {
  1260. /* Out of per CPU action FIFO space. Drop the 'skb' and
  1261. * log an error.
  1262. */
  1263. kfree_skb(skb);
  1264. if (net_ratelimit()) {
  1265. if (actions) { /* Sample action */
  1266. pr_warn("%s: deferred action limit reached, drop sample action\n",
  1267. ovs_dp_name(dp));
  1268. } else { /* Recirc action */
  1269. pr_warn("%s: deferred action limit reached, drop recirc action\n",
  1270. ovs_dp_name(dp));
  1271. }
  1272. }
  1273. }
  1274. return 0;
  1275. }
  1276. static void process_deferred_actions(struct datapath *dp)
  1277. {
  1278. struct action_fifo *fifo = this_cpu_ptr(action_fifos);
  1279. /* Do not touch the FIFO in case there is no deferred actions. */
  1280. if (action_fifo_is_empty(fifo))
  1281. return;
  1282. /* Finishing executing all deferred actions. */
  1283. do {
  1284. struct deferred_action *da = action_fifo_get(fifo);
  1285. struct sk_buff *skb = da->skb;
  1286. struct sw_flow_key *key = &da->pkt_key;
  1287. const struct nlattr *actions = da->actions;
  1288. int actions_len = da->actions_len;
  1289. if (actions)
  1290. do_execute_actions(dp, skb, key, actions, actions_len);
  1291. else
  1292. ovs_dp_process_packet(skb, key);
  1293. } while (!action_fifo_is_empty(fifo));
  1294. /* Reset FIFO for the next packet. */
  1295. action_fifo_init(fifo);
  1296. }
  1297. /* Execute a list of actions against 'skb'. */
  1298. int ovs_execute_actions(struct datapath *dp, struct sk_buff *skb,
  1299. const struct sw_flow_actions *acts,
  1300. struct sw_flow_key *key)
  1301. {
  1302. int err, level;
  1303. level = __this_cpu_inc_return(exec_actions_level);
  1304. if (unlikely(level > OVS_RECURSION_LIMIT)) {
  1305. net_crit_ratelimited("ovs: recursion limit reached on datapath %s, probable configuration error\n",
  1306. ovs_dp_name(dp));
  1307. kfree_skb(skb);
  1308. err = -ENETDOWN;
  1309. goto out;
  1310. }
  1311. OVS_CB(skb)->acts_origlen = acts->orig_len;
  1312. err = do_execute_actions(dp, skb, key,
  1313. acts->actions, acts->actions_len);
  1314. if (level == 1)
  1315. process_deferred_actions(dp);
  1316. out:
  1317. __this_cpu_dec(exec_actions_level);
  1318. return err;
  1319. }
  1320. int action_fifos_init(void)
  1321. {
  1322. action_fifos = alloc_percpu(struct action_fifo);
  1323. if (!action_fifos)
  1324. return -ENOMEM;
  1325. flow_keys = alloc_percpu(struct action_flow_keys);
  1326. if (!flow_keys) {
  1327. free_percpu(action_fifos);
  1328. return -ENOMEM;
  1329. }
  1330. return 0;
  1331. }
  1332. void action_fifos_exit(void)
  1333. {
  1334. free_percpu(action_fifos);
  1335. free_percpu(flow_keys);
  1336. }