act_ct.c 38 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576
  1. // SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
  2. /* -
  3. * net/sched/act_ct.c Connection Tracking action
  4. *
  5. * Authors: Paul Blakey <paulb@mellanox.com>
  6. * Yossi Kuperman <yossiku@mellanox.com>
  7. * Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>
  8. */
  9. #include <linux/module.h>
  10. #include <linux/init.h>
  11. #include <linux/kernel.h>
  12. #include <linux/skbuff.h>
  13. #include <linux/rtnetlink.h>
  14. #include <linux/pkt_cls.h>
  15. #include <linux/ip.h>
  16. #include <linux/ipv6.h>
  17. #include <linux/rhashtable.h>
  18. #include <net/netlink.h>
  19. #include <net/pkt_sched.h>
  20. #include <net/pkt_cls.h>
  21. #include <net/act_api.h>
  22. #include <net/ip.h>
  23. #include <net/ipv6_frag.h>
  24. #include <uapi/linux/tc_act/tc_ct.h>
  25. #include <net/tc_act/tc_ct.h>
  26. #include <net/netfilter/nf_flow_table.h>
  27. #include <net/netfilter/nf_conntrack.h>
  28. #include <net/netfilter/nf_conntrack_core.h>
  29. #include <net/netfilter/nf_conntrack_zones.h>
  30. #include <net/netfilter/nf_conntrack_helper.h>
  31. #include <net/netfilter/nf_conntrack_acct.h>
  32. #include <net/netfilter/ipv6/nf_defrag_ipv6.h>
  33. #include <uapi/linux/netfilter/nf_nat.h>
  34. static struct workqueue_struct *act_ct_wq;
  35. static struct rhashtable zones_ht;
  36. static DEFINE_MUTEX(zones_mutex);
  37. struct tcf_ct_flow_table {
  38. struct rhash_head node; /* In zones tables */
  39. struct rcu_work rwork;
  40. struct nf_flowtable nf_ft;
  41. refcount_t ref;
  42. u16 zone;
  43. bool dying;
  44. };
  45. static const struct rhashtable_params zones_params = {
  46. .head_offset = offsetof(struct tcf_ct_flow_table, node),
  47. .key_offset = offsetof(struct tcf_ct_flow_table, zone),
  48. .key_len = sizeof_field(struct tcf_ct_flow_table, zone),
  49. .automatic_shrinking = true,
  50. };
  51. static struct flow_action_entry *
  52. tcf_ct_flow_table_flow_action_get_next(struct flow_action *flow_action)
  53. {
  54. int i = flow_action->num_entries++;
  55. return &flow_action->entries[i];
  56. }
  57. static void tcf_ct_add_mangle_action(struct flow_action *action,
  58. enum flow_action_mangle_base htype,
  59. u32 offset,
  60. u32 mask,
  61. u32 val)
  62. {
  63. struct flow_action_entry *entry;
  64. entry = tcf_ct_flow_table_flow_action_get_next(action);
  65. entry->id = FLOW_ACTION_MANGLE;
  66. entry->mangle.htype = htype;
  67. entry->mangle.mask = ~mask;
  68. entry->mangle.offset = offset;
  69. entry->mangle.val = val;
  70. }
  71. /* The following nat helper functions check if the inverted reverse tuple
  72. * (target) is different then the current dir tuple - meaning nat for ports
  73. * and/or ip is needed, and add the relevant mangle actions.
  74. */
  75. static void
  76. tcf_ct_flow_table_add_action_nat_ipv4(const struct nf_conntrack_tuple *tuple,
  77. struct nf_conntrack_tuple target,
  78. struct flow_action *action)
  79. {
  80. if (memcmp(&target.src.u3, &tuple->src.u3, sizeof(target.src.u3)))
  81. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP4,
  82. offsetof(struct iphdr, saddr),
  83. 0xFFFFFFFF,
  84. be32_to_cpu(target.src.u3.ip));
  85. if (memcmp(&target.dst.u3, &tuple->dst.u3, sizeof(target.dst.u3)))
  86. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP4,
  87. offsetof(struct iphdr, daddr),
  88. 0xFFFFFFFF,
  89. be32_to_cpu(target.dst.u3.ip));
  90. }
  91. static void
  92. tcf_ct_add_ipv6_addr_mangle_action(struct flow_action *action,
  93. union nf_inet_addr *addr,
  94. u32 offset)
  95. {
  96. int i;
  97. for (i = 0; i < sizeof(struct in6_addr) / sizeof(u32); i++)
  98. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_IP6,
  99. i * sizeof(u32) + offset,
  100. 0xFFFFFFFF, be32_to_cpu(addr->ip6[i]));
  101. }
  102. static void
  103. tcf_ct_flow_table_add_action_nat_ipv6(const struct nf_conntrack_tuple *tuple,
  104. struct nf_conntrack_tuple target,
  105. struct flow_action *action)
  106. {
  107. if (memcmp(&target.src.u3, &tuple->src.u3, sizeof(target.src.u3)))
  108. tcf_ct_add_ipv6_addr_mangle_action(action, &target.src.u3,
  109. offsetof(struct ipv6hdr,
  110. saddr));
  111. if (memcmp(&target.dst.u3, &tuple->dst.u3, sizeof(target.dst.u3)))
  112. tcf_ct_add_ipv6_addr_mangle_action(action, &target.dst.u3,
  113. offsetof(struct ipv6hdr,
  114. daddr));
  115. }
  116. static void
  117. tcf_ct_flow_table_add_action_nat_tcp(const struct nf_conntrack_tuple *tuple,
  118. struct nf_conntrack_tuple target,
  119. struct flow_action *action)
  120. {
  121. __be16 target_src = target.src.u.tcp.port;
  122. __be16 target_dst = target.dst.u.tcp.port;
  123. if (target_src != tuple->src.u.tcp.port)
  124. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_TCP,
  125. offsetof(struct tcphdr, source),
  126. 0xFFFF, be16_to_cpu(target_src));
  127. if (target_dst != tuple->dst.u.tcp.port)
  128. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_TCP,
  129. offsetof(struct tcphdr, dest),
  130. 0xFFFF, be16_to_cpu(target_dst));
  131. }
  132. static void
  133. tcf_ct_flow_table_add_action_nat_udp(const struct nf_conntrack_tuple *tuple,
  134. struct nf_conntrack_tuple target,
  135. struct flow_action *action)
  136. {
  137. __be16 target_src = target.src.u.udp.port;
  138. __be16 target_dst = target.dst.u.udp.port;
  139. if (target_src != tuple->src.u.udp.port)
  140. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_UDP,
  141. offsetof(struct udphdr, source),
  142. 0xFFFF, be16_to_cpu(target_src));
  143. if (target_dst != tuple->dst.u.udp.port)
  144. tcf_ct_add_mangle_action(action, FLOW_ACT_MANGLE_HDR_TYPE_UDP,
  145. offsetof(struct udphdr, dest),
  146. 0xFFFF, be16_to_cpu(target_dst));
  147. }
  148. static void tcf_ct_flow_table_add_action_meta(struct nf_conn *ct,
  149. enum ip_conntrack_dir dir,
  150. struct flow_action *action)
  151. {
  152. struct nf_conn_labels *ct_labels;
  153. struct flow_action_entry *entry;
  154. enum ip_conntrack_info ctinfo;
  155. u32 *act_ct_labels;
  156. entry = tcf_ct_flow_table_flow_action_get_next(action);
  157. entry->id = FLOW_ACTION_CT_METADATA;
  158. #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
  159. entry->ct_metadata.mark = ct->mark;
  160. #endif
  161. ctinfo = dir == IP_CT_DIR_ORIGINAL ? IP_CT_ESTABLISHED :
  162. IP_CT_ESTABLISHED_REPLY;
  163. /* aligns with the CT reference on the SKB nf_ct_set */
  164. entry->ct_metadata.cookie = (unsigned long)ct | ctinfo;
  165. act_ct_labels = entry->ct_metadata.labels;
  166. ct_labels = nf_ct_labels_find(ct);
  167. if (ct_labels)
  168. memcpy(act_ct_labels, ct_labels->bits, NF_CT_LABELS_MAX_SIZE);
  169. else
  170. memset(act_ct_labels, 0, NF_CT_LABELS_MAX_SIZE);
  171. }
  172. static int tcf_ct_flow_table_add_action_nat(struct net *net,
  173. struct nf_conn *ct,
  174. enum ip_conntrack_dir dir,
  175. struct flow_action *action)
  176. {
  177. const struct nf_conntrack_tuple *tuple = &ct->tuplehash[dir].tuple;
  178. struct nf_conntrack_tuple target;
  179. if (!(ct->status & IPS_NAT_MASK))
  180. return 0;
  181. nf_ct_invert_tuple(&target, &ct->tuplehash[!dir].tuple);
  182. switch (tuple->src.l3num) {
  183. case NFPROTO_IPV4:
  184. tcf_ct_flow_table_add_action_nat_ipv4(tuple, target,
  185. action);
  186. break;
  187. case NFPROTO_IPV6:
  188. tcf_ct_flow_table_add_action_nat_ipv6(tuple, target,
  189. action);
  190. break;
  191. default:
  192. return -EOPNOTSUPP;
  193. }
  194. switch (nf_ct_protonum(ct)) {
  195. case IPPROTO_TCP:
  196. tcf_ct_flow_table_add_action_nat_tcp(tuple, target, action);
  197. break;
  198. case IPPROTO_UDP:
  199. tcf_ct_flow_table_add_action_nat_udp(tuple, target, action);
  200. break;
  201. default:
  202. return -EOPNOTSUPP;
  203. }
  204. return 0;
  205. }
  206. static int tcf_ct_flow_table_fill_actions(struct net *net,
  207. const struct flow_offload *flow,
  208. enum flow_offload_tuple_dir tdir,
  209. struct nf_flow_rule *flow_rule)
  210. {
  211. struct flow_action *action = &flow_rule->rule->action;
  212. int num_entries = action->num_entries;
  213. struct nf_conn *ct = flow->ct;
  214. enum ip_conntrack_dir dir;
  215. int i, err;
  216. switch (tdir) {
  217. case FLOW_OFFLOAD_DIR_ORIGINAL:
  218. dir = IP_CT_DIR_ORIGINAL;
  219. break;
  220. case FLOW_OFFLOAD_DIR_REPLY:
  221. dir = IP_CT_DIR_REPLY;
  222. break;
  223. default:
  224. return -EOPNOTSUPP;
  225. }
  226. err = tcf_ct_flow_table_add_action_nat(net, ct, dir, action);
  227. if (err)
  228. goto err_nat;
  229. tcf_ct_flow_table_add_action_meta(ct, dir, action);
  230. return 0;
  231. err_nat:
  232. /* Clear filled actions */
  233. for (i = num_entries; i < action->num_entries; i++)
  234. memset(&action->entries[i], 0, sizeof(action->entries[i]));
  235. action->num_entries = num_entries;
  236. return err;
  237. }
  238. static struct nf_flowtable_type flowtable_ct = {
  239. .action = tcf_ct_flow_table_fill_actions,
  240. .owner = THIS_MODULE,
  241. };
  242. static int tcf_ct_flow_table_get(struct tcf_ct_params *params)
  243. {
  244. struct tcf_ct_flow_table *ct_ft;
  245. int err = -ENOMEM;
  246. mutex_lock(&zones_mutex);
  247. ct_ft = rhashtable_lookup_fast(&zones_ht, &params->zone, zones_params);
  248. if (ct_ft && refcount_inc_not_zero(&ct_ft->ref))
  249. goto out_unlock;
  250. ct_ft = kzalloc(sizeof(*ct_ft), GFP_KERNEL);
  251. if (!ct_ft)
  252. goto err_alloc;
  253. refcount_set(&ct_ft->ref, 1);
  254. ct_ft->zone = params->zone;
  255. err = rhashtable_insert_fast(&zones_ht, &ct_ft->node, zones_params);
  256. if (err)
  257. goto err_insert;
  258. ct_ft->nf_ft.type = &flowtable_ct;
  259. ct_ft->nf_ft.flags |= NF_FLOWTABLE_HW_OFFLOAD;
  260. err = nf_flow_table_init(&ct_ft->nf_ft);
  261. if (err)
  262. goto err_init;
  263. __module_get(THIS_MODULE);
  264. out_unlock:
  265. params->ct_ft = ct_ft;
  266. params->nf_ft = &ct_ft->nf_ft;
  267. mutex_unlock(&zones_mutex);
  268. return 0;
  269. err_init:
  270. rhashtable_remove_fast(&zones_ht, &ct_ft->node, zones_params);
  271. err_insert:
  272. kfree(ct_ft);
  273. err_alloc:
  274. mutex_unlock(&zones_mutex);
  275. return err;
  276. }
  277. static void tcf_ct_flow_table_cleanup_work(struct work_struct *work)
  278. {
  279. struct flow_block_cb *block_cb, *tmp_cb;
  280. struct tcf_ct_flow_table *ct_ft;
  281. struct flow_block *block;
  282. ct_ft = container_of(to_rcu_work(work), struct tcf_ct_flow_table,
  283. rwork);
  284. nf_flow_table_free(&ct_ft->nf_ft);
  285. /* Remove any remaining callbacks before cleanup */
  286. block = &ct_ft->nf_ft.flow_block;
  287. down_write(&ct_ft->nf_ft.flow_block_lock);
  288. list_for_each_entry_safe(block_cb, tmp_cb, &block->cb_list, list) {
  289. list_del(&block_cb->list);
  290. flow_block_cb_free(block_cb);
  291. }
  292. up_write(&ct_ft->nf_ft.flow_block_lock);
  293. kfree(ct_ft);
  294. module_put(THIS_MODULE);
  295. }
  296. static void tcf_ct_flow_table_put(struct tcf_ct_params *params)
  297. {
  298. struct tcf_ct_flow_table *ct_ft = params->ct_ft;
  299. if (refcount_dec_and_test(&params->ct_ft->ref)) {
  300. rhashtable_remove_fast(&zones_ht, &ct_ft->node, zones_params);
  301. INIT_RCU_WORK(&ct_ft->rwork, tcf_ct_flow_table_cleanup_work);
  302. queue_rcu_work(act_ct_wq, &ct_ft->rwork);
  303. }
  304. }
  305. static void tcf_ct_flow_table_add(struct tcf_ct_flow_table *ct_ft,
  306. struct nf_conn *ct,
  307. bool tcp)
  308. {
  309. struct flow_offload *entry;
  310. int err;
  311. if (test_and_set_bit(IPS_OFFLOAD_BIT, &ct->status))
  312. return;
  313. entry = flow_offload_alloc(ct);
  314. if (!entry) {
  315. WARN_ON_ONCE(1);
  316. goto err_alloc;
  317. }
  318. if (tcp) {
  319. ct->proto.tcp.seen[0].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
  320. ct->proto.tcp.seen[1].flags |= IP_CT_TCP_FLAG_BE_LIBERAL;
  321. }
  322. err = flow_offload_add(&ct_ft->nf_ft, entry);
  323. if (err)
  324. goto err_add;
  325. return;
  326. err_add:
  327. flow_offload_free(entry);
  328. err_alloc:
  329. clear_bit(IPS_OFFLOAD_BIT, &ct->status);
  330. }
  331. static void tcf_ct_flow_table_process_conn(struct tcf_ct_flow_table *ct_ft,
  332. struct nf_conn *ct,
  333. enum ip_conntrack_info ctinfo)
  334. {
  335. bool tcp = false;
  336. if (ctinfo != IP_CT_ESTABLISHED && ctinfo != IP_CT_ESTABLISHED_REPLY)
  337. return;
  338. switch (nf_ct_protonum(ct)) {
  339. case IPPROTO_TCP:
  340. tcp = true;
  341. if (ct->proto.tcp.state != TCP_CONNTRACK_ESTABLISHED)
  342. return;
  343. break;
  344. case IPPROTO_UDP:
  345. break;
  346. default:
  347. return;
  348. }
  349. if (nf_ct_ext_exist(ct, NF_CT_EXT_HELPER) ||
  350. ct->status & IPS_SEQ_ADJUST)
  351. return;
  352. tcf_ct_flow_table_add(ct_ft, ct, tcp);
  353. }
  354. static bool
  355. tcf_ct_flow_table_fill_tuple_ipv4(struct sk_buff *skb,
  356. struct flow_offload_tuple *tuple,
  357. struct tcphdr **tcph)
  358. {
  359. struct flow_ports *ports;
  360. unsigned int thoff;
  361. struct iphdr *iph;
  362. if (!pskb_network_may_pull(skb, sizeof(*iph)))
  363. return false;
  364. iph = ip_hdr(skb);
  365. thoff = iph->ihl * 4;
  366. if (ip_is_fragment(iph) ||
  367. unlikely(thoff != sizeof(struct iphdr)))
  368. return false;
  369. if (iph->protocol != IPPROTO_TCP &&
  370. iph->protocol != IPPROTO_UDP)
  371. return false;
  372. if (iph->ttl <= 1)
  373. return false;
  374. if (!pskb_network_may_pull(skb, iph->protocol == IPPROTO_TCP ?
  375. thoff + sizeof(struct tcphdr) :
  376. thoff + sizeof(*ports)))
  377. return false;
  378. iph = ip_hdr(skb);
  379. if (iph->protocol == IPPROTO_TCP)
  380. *tcph = (void *)(skb_network_header(skb) + thoff);
  381. ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
  382. tuple->src_v4.s_addr = iph->saddr;
  383. tuple->dst_v4.s_addr = iph->daddr;
  384. tuple->src_port = ports->source;
  385. tuple->dst_port = ports->dest;
  386. tuple->l3proto = AF_INET;
  387. tuple->l4proto = iph->protocol;
  388. return true;
  389. }
  390. static bool
  391. tcf_ct_flow_table_fill_tuple_ipv6(struct sk_buff *skb,
  392. struct flow_offload_tuple *tuple,
  393. struct tcphdr **tcph)
  394. {
  395. struct flow_ports *ports;
  396. struct ipv6hdr *ip6h;
  397. unsigned int thoff;
  398. if (!pskb_network_may_pull(skb, sizeof(*ip6h)))
  399. return false;
  400. ip6h = ipv6_hdr(skb);
  401. if (ip6h->nexthdr != IPPROTO_TCP &&
  402. ip6h->nexthdr != IPPROTO_UDP)
  403. return false;
  404. if (ip6h->hop_limit <= 1)
  405. return false;
  406. thoff = sizeof(*ip6h);
  407. if (!pskb_network_may_pull(skb, ip6h->nexthdr == IPPROTO_TCP ?
  408. thoff + sizeof(struct tcphdr) :
  409. thoff + sizeof(*ports)))
  410. return false;
  411. ip6h = ipv6_hdr(skb);
  412. if (ip6h->nexthdr == IPPROTO_TCP)
  413. *tcph = (void *)(skb_network_header(skb) + thoff);
  414. ports = (struct flow_ports *)(skb_network_header(skb) + thoff);
  415. tuple->src_v6 = ip6h->saddr;
  416. tuple->dst_v6 = ip6h->daddr;
  417. tuple->src_port = ports->source;
  418. tuple->dst_port = ports->dest;
  419. tuple->l3proto = AF_INET6;
  420. tuple->l4proto = ip6h->nexthdr;
  421. return true;
  422. }
  423. static bool tcf_ct_flow_table_lookup(struct tcf_ct_params *p,
  424. struct sk_buff *skb,
  425. u8 family)
  426. {
  427. struct nf_flowtable *nf_ft = &p->ct_ft->nf_ft;
  428. struct flow_offload_tuple_rhash *tuplehash;
  429. struct flow_offload_tuple tuple = {};
  430. enum ip_conntrack_info ctinfo;
  431. struct tcphdr *tcph = NULL;
  432. struct flow_offload *flow;
  433. struct nf_conn *ct;
  434. u8 dir;
  435. switch (family) {
  436. case NFPROTO_IPV4:
  437. if (!tcf_ct_flow_table_fill_tuple_ipv4(skb, &tuple, &tcph))
  438. return false;
  439. break;
  440. case NFPROTO_IPV6:
  441. if (!tcf_ct_flow_table_fill_tuple_ipv6(skb, &tuple, &tcph))
  442. return false;
  443. break;
  444. default:
  445. return false;
  446. }
  447. tuplehash = flow_offload_lookup(nf_ft, &tuple);
  448. if (!tuplehash)
  449. return false;
  450. dir = tuplehash->tuple.dir;
  451. flow = container_of(tuplehash, struct flow_offload, tuplehash[dir]);
  452. ct = flow->ct;
  453. if (tcph && (unlikely(tcph->fin || tcph->rst))) {
  454. flow_offload_teardown(flow);
  455. return false;
  456. }
  457. ctinfo = dir == FLOW_OFFLOAD_DIR_ORIGINAL ? IP_CT_ESTABLISHED :
  458. IP_CT_ESTABLISHED_REPLY;
  459. flow_offload_refresh(nf_ft, flow);
  460. nf_conntrack_get(&ct->ct_general);
  461. nf_ct_set(skb, ct, ctinfo);
  462. nf_ct_acct_update(ct, dir, skb->len);
  463. return true;
  464. }
  465. static int tcf_ct_flow_tables_init(void)
  466. {
  467. return rhashtable_init(&zones_ht, &zones_params);
  468. }
  469. static void tcf_ct_flow_tables_uninit(void)
  470. {
  471. rhashtable_destroy(&zones_ht);
  472. }
  473. static struct tc_action_ops act_ct_ops;
  474. static unsigned int ct_net_id;
  475. struct tc_ct_action_net {
  476. struct tc_action_net tn; /* Must be first */
  477. bool labels;
  478. };
  479. /* Determine whether skb->_nfct is equal to the result of conntrack lookup. */
  480. static bool tcf_ct_skb_nfct_cached(struct net *net, struct sk_buff *skb,
  481. u16 zone_id, bool force)
  482. {
  483. enum ip_conntrack_info ctinfo;
  484. struct nf_conn *ct;
  485. ct = nf_ct_get(skb, &ctinfo);
  486. if (!ct)
  487. return false;
  488. if (!net_eq(net, read_pnet(&ct->ct_net)))
  489. return false;
  490. if (nf_ct_zone(ct)->id != zone_id)
  491. return false;
  492. /* Force conntrack entry direction. */
  493. if (force && CTINFO2DIR(ctinfo) != IP_CT_DIR_ORIGINAL) {
  494. if (nf_ct_is_confirmed(ct))
  495. nf_ct_kill(ct);
  496. nf_conntrack_put(&ct->ct_general);
  497. nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
  498. return false;
  499. }
  500. return true;
  501. }
  502. /* Trim the skb to the length specified by the IP/IPv6 header,
  503. * removing any trailing lower-layer padding. This prepares the skb
  504. * for higher-layer processing that assumes skb->len excludes padding
  505. * (such as nf_ip_checksum). The caller needs to pull the skb to the
  506. * network header, and ensure ip_hdr/ipv6_hdr points to valid data.
  507. */
  508. static int tcf_ct_skb_network_trim(struct sk_buff *skb, int family)
  509. {
  510. unsigned int len;
  511. int err;
  512. switch (family) {
  513. case NFPROTO_IPV4:
  514. len = ntohs(ip_hdr(skb)->tot_len);
  515. break;
  516. case NFPROTO_IPV6:
  517. len = sizeof(struct ipv6hdr)
  518. + ntohs(ipv6_hdr(skb)->payload_len);
  519. break;
  520. default:
  521. len = skb->len;
  522. }
  523. err = pskb_trim_rcsum(skb, len);
  524. return err;
  525. }
  526. static u8 tcf_ct_skb_nf_family(struct sk_buff *skb)
  527. {
  528. u8 family = NFPROTO_UNSPEC;
  529. switch (skb_protocol(skb, true)) {
  530. case htons(ETH_P_IP):
  531. family = NFPROTO_IPV4;
  532. break;
  533. case htons(ETH_P_IPV6):
  534. family = NFPROTO_IPV6;
  535. break;
  536. default:
  537. break;
  538. }
  539. return family;
  540. }
  541. static int tcf_ct_ipv4_is_fragment(struct sk_buff *skb, bool *frag)
  542. {
  543. unsigned int len;
  544. len = skb_network_offset(skb) + sizeof(struct iphdr);
  545. if (unlikely(skb->len < len))
  546. return -EINVAL;
  547. if (unlikely(!pskb_may_pull(skb, len)))
  548. return -ENOMEM;
  549. *frag = ip_is_fragment(ip_hdr(skb));
  550. return 0;
  551. }
  552. static int tcf_ct_ipv6_is_fragment(struct sk_buff *skb, bool *frag)
  553. {
  554. unsigned int flags = 0, len, payload_ofs = 0;
  555. unsigned short frag_off;
  556. int nexthdr;
  557. len = skb_network_offset(skb) + sizeof(struct ipv6hdr);
  558. if (unlikely(skb->len < len))
  559. return -EINVAL;
  560. if (unlikely(!pskb_may_pull(skb, len)))
  561. return -ENOMEM;
  562. nexthdr = ipv6_find_hdr(skb, &payload_ofs, -1, &frag_off, &flags);
  563. if (unlikely(nexthdr < 0))
  564. return -EPROTO;
  565. *frag = flags & IP6_FH_F_FRAG;
  566. return 0;
  567. }
  568. static int tcf_ct_handle_fragments(struct net *net, struct sk_buff *skb,
  569. u8 family, u16 zone, bool *defrag)
  570. {
  571. enum ip_conntrack_info ctinfo;
  572. struct qdisc_skb_cb cb;
  573. struct nf_conn *ct;
  574. int err = 0;
  575. bool frag;
  576. /* Previously seen (loopback)? Ignore. */
  577. ct = nf_ct_get(skb, &ctinfo);
  578. if ((ct && !nf_ct_is_template(ct)) || ctinfo == IP_CT_UNTRACKED)
  579. return 0;
  580. if (family == NFPROTO_IPV4)
  581. err = tcf_ct_ipv4_is_fragment(skb, &frag);
  582. else
  583. err = tcf_ct_ipv6_is_fragment(skb, &frag);
  584. if (err || !frag)
  585. return err;
  586. skb_get(skb);
  587. cb = *qdisc_skb_cb(skb);
  588. if (family == NFPROTO_IPV4) {
  589. enum ip_defrag_users user = IP_DEFRAG_CONNTRACK_IN + zone;
  590. memset(IPCB(skb), 0, sizeof(struct inet_skb_parm));
  591. local_bh_disable();
  592. err = ip_defrag(net, skb, user);
  593. local_bh_enable();
  594. if (err && err != -EINPROGRESS)
  595. return err;
  596. if (!err) {
  597. *defrag = true;
  598. cb.mru = IPCB(skb)->frag_max_size;
  599. }
  600. } else { /* NFPROTO_IPV6 */
  601. #if IS_ENABLED(CONFIG_NF_DEFRAG_IPV6)
  602. enum ip6_defrag_users user = IP6_DEFRAG_CONNTRACK_IN + zone;
  603. memset(IP6CB(skb), 0, sizeof(struct inet6_skb_parm));
  604. err = nf_ct_frag6_gather(net, skb, user);
  605. if (err && err != -EINPROGRESS)
  606. goto out_free;
  607. if (!err) {
  608. *defrag = true;
  609. cb.mru = IP6CB(skb)->frag_max_size;
  610. }
  611. #else
  612. err = -EOPNOTSUPP;
  613. goto out_free;
  614. #endif
  615. }
  616. *qdisc_skb_cb(skb) = cb;
  617. skb_clear_hash(skb);
  618. skb->ignore_df = 1;
  619. return err;
  620. out_free:
  621. kfree_skb(skb);
  622. return err;
  623. }
  624. static void tcf_ct_params_free(struct rcu_head *head)
  625. {
  626. struct tcf_ct_params *params = container_of(head,
  627. struct tcf_ct_params, rcu);
  628. tcf_ct_flow_table_put(params);
  629. if (params->tmpl)
  630. nf_conntrack_put(&params->tmpl->ct_general);
  631. kfree(params);
  632. }
  633. #if IS_ENABLED(CONFIG_NF_NAT)
  634. /* Modelled after nf_nat_ipv[46]_fn().
  635. * range is only used for new, uninitialized NAT state.
  636. * Returns either NF_ACCEPT or NF_DROP.
  637. */
  638. static int ct_nat_execute(struct sk_buff *skb, struct nf_conn *ct,
  639. enum ip_conntrack_info ctinfo,
  640. const struct nf_nat_range2 *range,
  641. enum nf_nat_manip_type maniptype)
  642. {
  643. __be16 proto = skb_protocol(skb, true);
  644. int hooknum, err = NF_ACCEPT;
  645. /* See HOOK2MANIP(). */
  646. if (maniptype == NF_NAT_MANIP_SRC)
  647. hooknum = NF_INET_LOCAL_IN; /* Source NAT */
  648. else
  649. hooknum = NF_INET_LOCAL_OUT; /* Destination NAT */
  650. switch (ctinfo) {
  651. case IP_CT_RELATED:
  652. case IP_CT_RELATED_REPLY:
  653. if (proto == htons(ETH_P_IP) &&
  654. ip_hdr(skb)->protocol == IPPROTO_ICMP) {
  655. if (!nf_nat_icmp_reply_translation(skb, ct, ctinfo,
  656. hooknum))
  657. err = NF_DROP;
  658. goto out;
  659. } else if (IS_ENABLED(CONFIG_IPV6) && proto == htons(ETH_P_IPV6)) {
  660. __be16 frag_off;
  661. u8 nexthdr = ipv6_hdr(skb)->nexthdr;
  662. int hdrlen = ipv6_skip_exthdr(skb,
  663. sizeof(struct ipv6hdr),
  664. &nexthdr, &frag_off);
  665. if (hdrlen >= 0 && nexthdr == IPPROTO_ICMPV6) {
  666. if (!nf_nat_icmpv6_reply_translation(skb, ct,
  667. ctinfo,
  668. hooknum,
  669. hdrlen))
  670. err = NF_DROP;
  671. goto out;
  672. }
  673. }
  674. /* Non-ICMP, fall thru to initialize if needed. */
  675. fallthrough;
  676. case IP_CT_NEW:
  677. /* Seen it before? This can happen for loopback, retrans,
  678. * or local packets.
  679. */
  680. if (!nf_nat_initialized(ct, maniptype)) {
  681. /* Initialize according to the NAT action. */
  682. err = (range && range->flags & NF_NAT_RANGE_MAP_IPS)
  683. /* Action is set up to establish a new
  684. * mapping.
  685. */
  686. ? nf_nat_setup_info(ct, range, maniptype)
  687. : nf_nat_alloc_null_binding(ct, hooknum);
  688. if (err != NF_ACCEPT)
  689. goto out;
  690. }
  691. break;
  692. case IP_CT_ESTABLISHED:
  693. case IP_CT_ESTABLISHED_REPLY:
  694. break;
  695. default:
  696. err = NF_DROP;
  697. goto out;
  698. }
  699. err = nf_nat_packet(ct, ctinfo, hooknum, skb);
  700. out:
  701. return err;
  702. }
  703. #endif /* CONFIG_NF_NAT */
  704. static void tcf_ct_act_set_mark(struct nf_conn *ct, u32 mark, u32 mask)
  705. {
  706. #if IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)
  707. u32 new_mark;
  708. if (!mask)
  709. return;
  710. new_mark = mark | (ct->mark & ~(mask));
  711. if (ct->mark != new_mark) {
  712. ct->mark = new_mark;
  713. if (nf_ct_is_confirmed(ct))
  714. nf_conntrack_event_cache(IPCT_MARK, ct);
  715. }
  716. #endif
  717. }
  718. static void tcf_ct_act_set_labels(struct nf_conn *ct,
  719. u32 *labels,
  720. u32 *labels_m)
  721. {
  722. #if IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)
  723. size_t labels_sz = sizeof_field(struct tcf_ct_params, labels);
  724. if (!memchr_inv(labels_m, 0, labels_sz))
  725. return;
  726. nf_connlabels_replace(ct, labels, labels_m, 4);
  727. #endif
  728. }
  729. static int tcf_ct_act_nat(struct sk_buff *skb,
  730. struct nf_conn *ct,
  731. enum ip_conntrack_info ctinfo,
  732. int ct_action,
  733. struct nf_nat_range2 *range,
  734. bool commit)
  735. {
  736. #if IS_ENABLED(CONFIG_NF_NAT)
  737. int err;
  738. enum nf_nat_manip_type maniptype;
  739. if (!(ct_action & TCA_CT_ACT_NAT))
  740. return NF_ACCEPT;
  741. /* Add NAT extension if not confirmed yet. */
  742. if (!nf_ct_is_confirmed(ct) && !nf_ct_nat_ext_add(ct))
  743. return NF_DROP; /* Can't NAT. */
  744. if (ctinfo != IP_CT_NEW && (ct->status & IPS_NAT_MASK) &&
  745. (ctinfo != IP_CT_RELATED || commit)) {
  746. /* NAT an established or related connection like before. */
  747. if (CTINFO2DIR(ctinfo) == IP_CT_DIR_REPLY)
  748. /* This is the REPLY direction for a connection
  749. * for which NAT was applied in the forward
  750. * direction. Do the reverse NAT.
  751. */
  752. maniptype = ct->status & IPS_SRC_NAT
  753. ? NF_NAT_MANIP_DST : NF_NAT_MANIP_SRC;
  754. else
  755. maniptype = ct->status & IPS_SRC_NAT
  756. ? NF_NAT_MANIP_SRC : NF_NAT_MANIP_DST;
  757. } else if (ct_action & TCA_CT_ACT_NAT_SRC) {
  758. maniptype = NF_NAT_MANIP_SRC;
  759. } else if (ct_action & TCA_CT_ACT_NAT_DST) {
  760. maniptype = NF_NAT_MANIP_DST;
  761. } else {
  762. return NF_ACCEPT;
  763. }
  764. err = ct_nat_execute(skb, ct, ctinfo, range, maniptype);
  765. if (err == NF_ACCEPT && ct->status & IPS_DST_NAT) {
  766. if (ct->status & IPS_SRC_NAT) {
  767. if (maniptype == NF_NAT_MANIP_SRC)
  768. maniptype = NF_NAT_MANIP_DST;
  769. else
  770. maniptype = NF_NAT_MANIP_SRC;
  771. err = ct_nat_execute(skb, ct, ctinfo, range,
  772. maniptype);
  773. } else if (CTINFO2DIR(ctinfo) == IP_CT_DIR_ORIGINAL) {
  774. err = ct_nat_execute(skb, ct, ctinfo, NULL,
  775. NF_NAT_MANIP_SRC);
  776. }
  777. }
  778. return err;
  779. #else
  780. return NF_ACCEPT;
  781. #endif
  782. }
  783. static int tcf_ct_act(struct sk_buff *skb, const struct tc_action *a,
  784. struct tcf_result *res)
  785. {
  786. struct net *net = dev_net(skb->dev);
  787. bool cached, commit, clear, force;
  788. enum ip_conntrack_info ctinfo;
  789. struct tcf_ct *c = to_ct(a);
  790. struct nf_conn *tmpl = NULL;
  791. struct nf_hook_state state;
  792. int nh_ofs, err, retval;
  793. struct tcf_ct_params *p;
  794. bool skip_add = false;
  795. bool defrag = false;
  796. struct nf_conn *ct;
  797. u8 family;
  798. p = rcu_dereference_bh(c->params);
  799. retval = READ_ONCE(c->tcf_action);
  800. commit = p->ct_action & TCA_CT_ACT_COMMIT;
  801. clear = p->ct_action & TCA_CT_ACT_CLEAR;
  802. force = p->ct_action & TCA_CT_ACT_FORCE;
  803. tmpl = p->tmpl;
  804. tcf_lastuse_update(&c->tcf_tm);
  805. if (clear) {
  806. ct = nf_ct_get(skb, &ctinfo);
  807. if (ct) {
  808. nf_conntrack_put(&ct->ct_general);
  809. nf_ct_set(skb, NULL, IP_CT_UNTRACKED);
  810. }
  811. goto out;
  812. }
  813. family = tcf_ct_skb_nf_family(skb);
  814. if (family == NFPROTO_UNSPEC)
  815. goto drop;
  816. /* The conntrack module expects to be working at L3.
  817. * We also try to pull the IPv4/6 header to linear area
  818. */
  819. nh_ofs = skb_network_offset(skb);
  820. skb_pull_rcsum(skb, nh_ofs);
  821. err = tcf_ct_handle_fragments(net, skb, family, p->zone, &defrag);
  822. if (err == -EINPROGRESS) {
  823. retval = TC_ACT_STOLEN;
  824. goto out;
  825. }
  826. if (err)
  827. goto drop;
  828. err = tcf_ct_skb_network_trim(skb, family);
  829. if (err)
  830. goto drop;
  831. /* If we are recirculating packets to match on ct fields and
  832. * committing with a separate ct action, then we don't need to
  833. * actually run the packet through conntrack twice unless it's for a
  834. * different zone.
  835. */
  836. cached = tcf_ct_skb_nfct_cached(net, skb, p->zone, force);
  837. if (!cached) {
  838. if (tcf_ct_flow_table_lookup(p, skb, family)) {
  839. skip_add = true;
  840. goto do_nat;
  841. }
  842. /* Associate skb with specified zone. */
  843. if (tmpl) {
  844. ct = nf_ct_get(skb, &ctinfo);
  845. if (skb_nfct(skb))
  846. nf_conntrack_put(skb_nfct(skb));
  847. nf_conntrack_get(&tmpl->ct_general);
  848. nf_ct_set(skb, tmpl, IP_CT_NEW);
  849. }
  850. state.hook = NF_INET_PRE_ROUTING;
  851. state.net = net;
  852. state.pf = family;
  853. err = nf_conntrack_in(skb, &state);
  854. if (err != NF_ACCEPT)
  855. goto out_push;
  856. }
  857. do_nat:
  858. ct = nf_ct_get(skb, &ctinfo);
  859. if (!ct)
  860. goto out_push;
  861. nf_ct_deliver_cached_events(ct);
  862. err = tcf_ct_act_nat(skb, ct, ctinfo, p->ct_action, &p->range, commit);
  863. if (err != NF_ACCEPT)
  864. goto drop;
  865. if (commit) {
  866. tcf_ct_act_set_mark(ct, p->mark, p->mark_mask);
  867. tcf_ct_act_set_labels(ct, p->labels, p->labels_mask);
  868. /* This will take care of sending queued events
  869. * even if the connection is already confirmed.
  870. */
  871. if (nf_conntrack_confirm(skb) != NF_ACCEPT)
  872. goto drop;
  873. }
  874. if (!skip_add)
  875. tcf_ct_flow_table_process_conn(p->ct_ft, ct, ctinfo);
  876. out_push:
  877. skb_push_rcsum(skb, nh_ofs);
  878. out:
  879. tcf_action_update_bstats(&c->common, skb);
  880. if (defrag)
  881. qdisc_skb_cb(skb)->pkt_len = skb->len;
  882. return retval;
  883. drop:
  884. tcf_action_inc_drop_qstats(&c->common);
  885. return TC_ACT_SHOT;
  886. }
  887. static const struct nla_policy ct_policy[TCA_CT_MAX + 1] = {
  888. [TCA_CT_ACTION] = { .type = NLA_U16 },
  889. [TCA_CT_PARMS] = NLA_POLICY_EXACT_LEN(sizeof(struct tc_ct)),
  890. [TCA_CT_ZONE] = { .type = NLA_U16 },
  891. [TCA_CT_MARK] = { .type = NLA_U32 },
  892. [TCA_CT_MARK_MASK] = { .type = NLA_U32 },
  893. [TCA_CT_LABELS] = { .type = NLA_BINARY,
  894. .len = 128 / BITS_PER_BYTE },
  895. [TCA_CT_LABELS_MASK] = { .type = NLA_BINARY,
  896. .len = 128 / BITS_PER_BYTE },
  897. [TCA_CT_NAT_IPV4_MIN] = { .type = NLA_U32 },
  898. [TCA_CT_NAT_IPV4_MAX] = { .type = NLA_U32 },
  899. [TCA_CT_NAT_IPV6_MIN] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
  900. [TCA_CT_NAT_IPV6_MAX] = NLA_POLICY_EXACT_LEN(sizeof(struct in6_addr)),
  901. [TCA_CT_NAT_PORT_MIN] = { .type = NLA_U16 },
  902. [TCA_CT_NAT_PORT_MAX] = { .type = NLA_U16 },
  903. };
  904. static int tcf_ct_fill_params_nat(struct tcf_ct_params *p,
  905. struct tc_ct *parm,
  906. struct nlattr **tb,
  907. struct netlink_ext_ack *extack)
  908. {
  909. struct nf_nat_range2 *range;
  910. if (!(p->ct_action & TCA_CT_ACT_NAT))
  911. return 0;
  912. if (!IS_ENABLED(CONFIG_NF_NAT)) {
  913. NL_SET_ERR_MSG_MOD(extack, "Netfilter nat isn't enabled in kernel");
  914. return -EOPNOTSUPP;
  915. }
  916. if (!(p->ct_action & (TCA_CT_ACT_NAT_SRC | TCA_CT_ACT_NAT_DST)))
  917. return 0;
  918. if ((p->ct_action & TCA_CT_ACT_NAT_SRC) &&
  919. (p->ct_action & TCA_CT_ACT_NAT_DST)) {
  920. NL_SET_ERR_MSG_MOD(extack, "dnat and snat can't be enabled at the same time");
  921. return -EOPNOTSUPP;
  922. }
  923. range = &p->range;
  924. if (tb[TCA_CT_NAT_IPV4_MIN]) {
  925. struct nlattr *max_attr = tb[TCA_CT_NAT_IPV4_MAX];
  926. p->ipv4_range = true;
  927. range->flags |= NF_NAT_RANGE_MAP_IPS;
  928. range->min_addr.ip =
  929. nla_get_in_addr(tb[TCA_CT_NAT_IPV4_MIN]);
  930. range->max_addr.ip = max_attr ?
  931. nla_get_in_addr(max_attr) :
  932. range->min_addr.ip;
  933. } else if (tb[TCA_CT_NAT_IPV6_MIN]) {
  934. struct nlattr *max_attr = tb[TCA_CT_NAT_IPV6_MAX];
  935. p->ipv4_range = false;
  936. range->flags |= NF_NAT_RANGE_MAP_IPS;
  937. range->min_addr.in6 =
  938. nla_get_in6_addr(tb[TCA_CT_NAT_IPV6_MIN]);
  939. range->max_addr.in6 = max_attr ?
  940. nla_get_in6_addr(max_attr) :
  941. range->min_addr.in6;
  942. }
  943. if (tb[TCA_CT_NAT_PORT_MIN]) {
  944. range->flags |= NF_NAT_RANGE_PROTO_SPECIFIED;
  945. range->min_proto.all = nla_get_be16(tb[TCA_CT_NAT_PORT_MIN]);
  946. range->max_proto.all = tb[TCA_CT_NAT_PORT_MAX] ?
  947. nla_get_be16(tb[TCA_CT_NAT_PORT_MAX]) :
  948. range->min_proto.all;
  949. }
  950. return 0;
  951. }
  952. static void tcf_ct_set_key_val(struct nlattr **tb,
  953. void *val, int val_type,
  954. void *mask, int mask_type,
  955. int len)
  956. {
  957. if (!tb[val_type])
  958. return;
  959. nla_memcpy(val, tb[val_type], len);
  960. if (!mask)
  961. return;
  962. if (mask_type == TCA_CT_UNSPEC || !tb[mask_type])
  963. memset(mask, 0xff, len);
  964. else
  965. nla_memcpy(mask, tb[mask_type], len);
  966. }
  967. static int tcf_ct_fill_params(struct net *net,
  968. struct tcf_ct_params *p,
  969. struct tc_ct *parm,
  970. struct nlattr **tb,
  971. struct netlink_ext_ack *extack)
  972. {
  973. struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
  974. struct nf_conntrack_zone zone;
  975. struct nf_conn *tmpl;
  976. int err;
  977. p->zone = NF_CT_DEFAULT_ZONE_ID;
  978. tcf_ct_set_key_val(tb,
  979. &p->ct_action, TCA_CT_ACTION,
  980. NULL, TCA_CT_UNSPEC,
  981. sizeof(p->ct_action));
  982. if (p->ct_action & TCA_CT_ACT_CLEAR)
  983. return 0;
  984. err = tcf_ct_fill_params_nat(p, parm, tb, extack);
  985. if (err)
  986. return err;
  987. if (tb[TCA_CT_MARK]) {
  988. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) {
  989. NL_SET_ERR_MSG_MOD(extack, "Conntrack mark isn't enabled.");
  990. return -EOPNOTSUPP;
  991. }
  992. tcf_ct_set_key_val(tb,
  993. &p->mark, TCA_CT_MARK,
  994. &p->mark_mask, TCA_CT_MARK_MASK,
  995. sizeof(p->mark));
  996. }
  997. if (tb[TCA_CT_LABELS]) {
  998. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) {
  999. NL_SET_ERR_MSG_MOD(extack, "Conntrack labels isn't enabled.");
  1000. return -EOPNOTSUPP;
  1001. }
  1002. if (!tn->labels) {
  1003. NL_SET_ERR_MSG_MOD(extack, "Failed to set connlabel length");
  1004. return -EOPNOTSUPP;
  1005. }
  1006. tcf_ct_set_key_val(tb,
  1007. p->labels, TCA_CT_LABELS,
  1008. p->labels_mask, TCA_CT_LABELS_MASK,
  1009. sizeof(p->labels));
  1010. }
  1011. if (tb[TCA_CT_ZONE]) {
  1012. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) {
  1013. NL_SET_ERR_MSG_MOD(extack, "Conntrack zones isn't enabled.");
  1014. return -EOPNOTSUPP;
  1015. }
  1016. tcf_ct_set_key_val(tb,
  1017. &p->zone, TCA_CT_ZONE,
  1018. NULL, TCA_CT_UNSPEC,
  1019. sizeof(p->zone));
  1020. }
  1021. nf_ct_zone_init(&zone, p->zone, NF_CT_DEFAULT_ZONE_DIR, 0);
  1022. tmpl = nf_ct_tmpl_alloc(net, &zone, GFP_KERNEL);
  1023. if (!tmpl) {
  1024. NL_SET_ERR_MSG_MOD(extack, "Failed to allocate conntrack template");
  1025. return -ENOMEM;
  1026. }
  1027. __set_bit(IPS_CONFIRMED_BIT, &tmpl->status);
  1028. nf_conntrack_get(&tmpl->ct_general);
  1029. p->tmpl = tmpl;
  1030. return 0;
  1031. }
  1032. static int tcf_ct_init(struct net *net, struct nlattr *nla,
  1033. struct nlattr *est, struct tc_action **a,
  1034. int replace, int bind, bool rtnl_held,
  1035. struct tcf_proto *tp, u32 flags,
  1036. struct netlink_ext_ack *extack)
  1037. {
  1038. struct tc_action_net *tn = net_generic(net, ct_net_id);
  1039. struct tcf_ct_params *params = NULL;
  1040. struct nlattr *tb[TCA_CT_MAX + 1];
  1041. struct tcf_chain *goto_ch = NULL;
  1042. struct tc_ct *parm;
  1043. struct tcf_ct *c;
  1044. int err, res = 0;
  1045. u32 index;
  1046. if (!nla) {
  1047. NL_SET_ERR_MSG_MOD(extack, "Ct requires attributes to be passed");
  1048. return -EINVAL;
  1049. }
  1050. err = nla_parse_nested(tb, TCA_CT_MAX, nla, ct_policy, extack);
  1051. if (err < 0)
  1052. return err;
  1053. if (!tb[TCA_CT_PARMS]) {
  1054. NL_SET_ERR_MSG_MOD(extack, "Missing required ct parameters");
  1055. return -EINVAL;
  1056. }
  1057. parm = nla_data(tb[TCA_CT_PARMS]);
  1058. index = parm->index;
  1059. err = tcf_idr_check_alloc(tn, &index, a, bind);
  1060. if (err < 0)
  1061. return err;
  1062. if (!err) {
  1063. err = tcf_idr_create_from_flags(tn, index, est, a,
  1064. &act_ct_ops, bind, flags);
  1065. if (err) {
  1066. tcf_idr_cleanup(tn, index);
  1067. return err;
  1068. }
  1069. res = ACT_P_CREATED;
  1070. } else {
  1071. if (bind)
  1072. return 0;
  1073. if (!replace) {
  1074. tcf_idr_release(*a, bind);
  1075. return -EEXIST;
  1076. }
  1077. }
  1078. err = tcf_action_check_ctrlact(parm->action, tp, &goto_ch, extack);
  1079. if (err < 0)
  1080. goto cleanup;
  1081. c = to_ct(*a);
  1082. params = kzalloc(sizeof(*params), GFP_KERNEL);
  1083. if (unlikely(!params)) {
  1084. err = -ENOMEM;
  1085. goto cleanup;
  1086. }
  1087. err = tcf_ct_fill_params(net, params, parm, tb, extack);
  1088. if (err)
  1089. goto cleanup;
  1090. err = tcf_ct_flow_table_get(params);
  1091. if (err)
  1092. goto cleanup;
  1093. spin_lock_bh(&c->tcf_lock);
  1094. goto_ch = tcf_action_set_ctrlact(*a, parm->action, goto_ch);
  1095. params = rcu_replace_pointer(c->params, params,
  1096. lockdep_is_held(&c->tcf_lock));
  1097. spin_unlock_bh(&c->tcf_lock);
  1098. if (goto_ch)
  1099. tcf_chain_put_by_act(goto_ch);
  1100. if (params)
  1101. call_rcu(&params->rcu, tcf_ct_params_free);
  1102. return res;
  1103. cleanup:
  1104. if (goto_ch)
  1105. tcf_chain_put_by_act(goto_ch);
  1106. kfree(params);
  1107. tcf_idr_release(*a, bind);
  1108. return err;
  1109. }
  1110. static void tcf_ct_cleanup(struct tc_action *a)
  1111. {
  1112. struct tcf_ct_params *params;
  1113. struct tcf_ct *c = to_ct(a);
  1114. params = rcu_dereference_protected(c->params, 1);
  1115. if (params)
  1116. call_rcu(&params->rcu, tcf_ct_params_free);
  1117. }
  1118. static int tcf_ct_dump_key_val(struct sk_buff *skb,
  1119. void *val, int val_type,
  1120. void *mask, int mask_type,
  1121. int len)
  1122. {
  1123. int err;
  1124. if (mask && !memchr_inv(mask, 0, len))
  1125. return 0;
  1126. err = nla_put(skb, val_type, len, val);
  1127. if (err)
  1128. return err;
  1129. if (mask_type != TCA_CT_UNSPEC) {
  1130. err = nla_put(skb, mask_type, len, mask);
  1131. if (err)
  1132. return err;
  1133. }
  1134. return 0;
  1135. }
  1136. static int tcf_ct_dump_nat(struct sk_buff *skb, struct tcf_ct_params *p)
  1137. {
  1138. struct nf_nat_range2 *range = &p->range;
  1139. if (!(p->ct_action & TCA_CT_ACT_NAT))
  1140. return 0;
  1141. if (!(p->ct_action & (TCA_CT_ACT_NAT_SRC | TCA_CT_ACT_NAT_DST)))
  1142. return 0;
  1143. if (range->flags & NF_NAT_RANGE_MAP_IPS) {
  1144. if (p->ipv4_range) {
  1145. if (nla_put_in_addr(skb, TCA_CT_NAT_IPV4_MIN,
  1146. range->min_addr.ip))
  1147. return -1;
  1148. if (nla_put_in_addr(skb, TCA_CT_NAT_IPV4_MAX,
  1149. range->max_addr.ip))
  1150. return -1;
  1151. } else {
  1152. if (nla_put_in6_addr(skb, TCA_CT_NAT_IPV6_MIN,
  1153. &range->min_addr.in6))
  1154. return -1;
  1155. if (nla_put_in6_addr(skb, TCA_CT_NAT_IPV6_MAX,
  1156. &range->max_addr.in6))
  1157. return -1;
  1158. }
  1159. }
  1160. if (range->flags & NF_NAT_RANGE_PROTO_SPECIFIED) {
  1161. if (nla_put_be16(skb, TCA_CT_NAT_PORT_MIN,
  1162. range->min_proto.all))
  1163. return -1;
  1164. if (nla_put_be16(skb, TCA_CT_NAT_PORT_MAX,
  1165. range->max_proto.all))
  1166. return -1;
  1167. }
  1168. return 0;
  1169. }
  1170. static inline int tcf_ct_dump(struct sk_buff *skb, struct tc_action *a,
  1171. int bind, int ref)
  1172. {
  1173. unsigned char *b = skb_tail_pointer(skb);
  1174. struct tcf_ct *c = to_ct(a);
  1175. struct tcf_ct_params *p;
  1176. struct tc_ct opt = {
  1177. .index = c->tcf_index,
  1178. .refcnt = refcount_read(&c->tcf_refcnt) - ref,
  1179. .bindcnt = atomic_read(&c->tcf_bindcnt) - bind,
  1180. };
  1181. struct tcf_t t;
  1182. spin_lock_bh(&c->tcf_lock);
  1183. p = rcu_dereference_protected(c->params,
  1184. lockdep_is_held(&c->tcf_lock));
  1185. opt.action = c->tcf_action;
  1186. if (tcf_ct_dump_key_val(skb,
  1187. &p->ct_action, TCA_CT_ACTION,
  1188. NULL, TCA_CT_UNSPEC,
  1189. sizeof(p->ct_action)))
  1190. goto nla_put_failure;
  1191. if (p->ct_action & TCA_CT_ACT_CLEAR)
  1192. goto skip_dump;
  1193. if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
  1194. tcf_ct_dump_key_val(skb,
  1195. &p->mark, TCA_CT_MARK,
  1196. &p->mark_mask, TCA_CT_MARK_MASK,
  1197. sizeof(p->mark)))
  1198. goto nla_put_failure;
  1199. if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
  1200. tcf_ct_dump_key_val(skb,
  1201. p->labels, TCA_CT_LABELS,
  1202. p->labels_mask, TCA_CT_LABELS_MASK,
  1203. sizeof(p->labels)))
  1204. goto nla_put_failure;
  1205. if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
  1206. tcf_ct_dump_key_val(skb,
  1207. &p->zone, TCA_CT_ZONE,
  1208. NULL, TCA_CT_UNSPEC,
  1209. sizeof(p->zone)))
  1210. goto nla_put_failure;
  1211. if (tcf_ct_dump_nat(skb, p))
  1212. goto nla_put_failure;
  1213. skip_dump:
  1214. if (nla_put(skb, TCA_CT_PARMS, sizeof(opt), &opt))
  1215. goto nla_put_failure;
  1216. tcf_tm_dump(&t, &c->tcf_tm);
  1217. if (nla_put_64bit(skb, TCA_CT_TM, sizeof(t), &t, TCA_CT_PAD))
  1218. goto nla_put_failure;
  1219. spin_unlock_bh(&c->tcf_lock);
  1220. return skb->len;
  1221. nla_put_failure:
  1222. spin_unlock_bh(&c->tcf_lock);
  1223. nlmsg_trim(skb, b);
  1224. return -1;
  1225. }
  1226. static int tcf_ct_walker(struct net *net, struct sk_buff *skb,
  1227. struct netlink_callback *cb, int type,
  1228. const struct tc_action_ops *ops,
  1229. struct netlink_ext_ack *extack)
  1230. {
  1231. struct tc_action_net *tn = net_generic(net, ct_net_id);
  1232. return tcf_generic_walker(tn, skb, cb, type, ops, extack);
  1233. }
  1234. static int tcf_ct_search(struct net *net, struct tc_action **a, u32 index)
  1235. {
  1236. struct tc_action_net *tn = net_generic(net, ct_net_id);
  1237. return tcf_idr_search(tn, a, index);
  1238. }
  1239. static void tcf_stats_update(struct tc_action *a, u64 bytes, u64 packets,
  1240. u64 drops, u64 lastuse, bool hw)
  1241. {
  1242. struct tcf_ct *c = to_ct(a);
  1243. tcf_action_update_stats(a, bytes, packets, drops, hw);
  1244. c->tcf_tm.lastuse = max_t(u64, c->tcf_tm.lastuse, lastuse);
  1245. }
  1246. static struct tc_action_ops act_ct_ops = {
  1247. .kind = "ct",
  1248. .id = TCA_ID_CT,
  1249. .owner = THIS_MODULE,
  1250. .act = tcf_ct_act,
  1251. .dump = tcf_ct_dump,
  1252. .init = tcf_ct_init,
  1253. .cleanup = tcf_ct_cleanup,
  1254. .walk = tcf_ct_walker,
  1255. .lookup = tcf_ct_search,
  1256. .stats_update = tcf_stats_update,
  1257. .size = sizeof(struct tcf_ct),
  1258. };
  1259. static __net_init int ct_init_net(struct net *net)
  1260. {
  1261. unsigned int n_bits = sizeof_field(struct tcf_ct_params, labels) * 8;
  1262. struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
  1263. if (nf_connlabels_get(net, n_bits - 1)) {
  1264. tn->labels = false;
  1265. pr_err("act_ct: Failed to set connlabels length");
  1266. } else {
  1267. tn->labels = true;
  1268. }
  1269. return tc_action_net_init(net, &tn->tn, &act_ct_ops);
  1270. }
  1271. static void __net_exit ct_exit_net(struct list_head *net_list)
  1272. {
  1273. struct net *net;
  1274. rtnl_lock();
  1275. list_for_each_entry(net, net_list, exit_list) {
  1276. struct tc_ct_action_net *tn = net_generic(net, ct_net_id);
  1277. if (tn->labels)
  1278. nf_connlabels_put(net);
  1279. }
  1280. rtnl_unlock();
  1281. tc_action_net_exit(net_list, ct_net_id);
  1282. }
  1283. static struct pernet_operations ct_net_ops = {
  1284. .init = ct_init_net,
  1285. .exit_batch = ct_exit_net,
  1286. .id = &ct_net_id,
  1287. .size = sizeof(struct tc_ct_action_net),
  1288. };
  1289. static int __init ct_init_module(void)
  1290. {
  1291. int err;
  1292. act_ct_wq = alloc_ordered_workqueue("act_ct_workqueue", 0);
  1293. if (!act_ct_wq)
  1294. return -ENOMEM;
  1295. err = tcf_ct_flow_tables_init();
  1296. if (err)
  1297. goto err_tbl_init;
  1298. err = tcf_register_action(&act_ct_ops, &ct_net_ops);
  1299. if (err)
  1300. goto err_register;
  1301. return 0;
  1302. err_register:
  1303. tcf_ct_flow_tables_uninit();
  1304. err_tbl_init:
  1305. destroy_workqueue(act_ct_wq);
  1306. return err;
  1307. }
  1308. static void __exit ct_cleanup_module(void)
  1309. {
  1310. tcf_unregister_action(&act_ct_ops, &ct_net_ops);
  1311. tcf_ct_flow_tables_uninit();
  1312. destroy_workqueue(act_ct_wq);
  1313. }
  1314. module_init(ct_init_module);
  1315. module_exit(ct_cleanup_module);
  1316. MODULE_AUTHOR("Paul Blakey <paulb@mellanox.com>");
  1317. MODULE_AUTHOR("Yossi Kuperman <yossiku@mellanox.com>");
  1318. MODULE_AUTHOR("Marcelo Ricardo Leitner <marcelo.leitner@gmail.com>");
  1319. MODULE_DESCRIPTION("Connection tracking action");
  1320. MODULE_LICENSE("GPL v2");