nf_tables_core.c 7.3 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2008 Patrick McHardy <kaber@trash.net>
  4. *
  5. * Development of this code funded by Astaro AG (http://www.astaro.com/)
  6. */
  7. #include <linux/kernel.h>
  8. #include <linux/module.h>
  9. #include <linux/init.h>
  10. #include <linux/list.h>
  11. #include <linux/rculist.h>
  12. #include <linux/skbuff.h>
  13. #include <linux/netlink.h>
  14. #include <linux/netfilter.h>
  15. #include <linux/static_key.h>
  16. #include <linux/netfilter/nfnetlink.h>
  17. #include <linux/netfilter/nf_tables.h>
  18. #include <net/netfilter/nf_tables_core.h>
  19. #include <net/netfilter/nf_tables.h>
  20. #include <net/netfilter/nf_log.h>
  21. #include <net/netfilter/nft_meta.h>
  22. static noinline void __nft_trace_packet(struct nft_traceinfo *info,
  23. const struct nft_chain *chain,
  24. enum nft_trace_types type)
  25. {
  26. const struct nft_pktinfo *pkt = info->pkt;
  27. if (!info->trace || !pkt->skb->nf_trace)
  28. return;
  29. info->chain = chain;
  30. info->type = type;
  31. nft_trace_notify(info);
  32. }
  33. static inline void nft_trace_packet(struct nft_traceinfo *info,
  34. const struct nft_chain *chain,
  35. const struct nft_rule *rule,
  36. enum nft_trace_types type)
  37. {
  38. if (static_branch_unlikely(&nft_trace_enabled)) {
  39. info->rule = rule;
  40. __nft_trace_packet(info, chain, type);
  41. }
  42. }
  43. static void nft_bitwise_fast_eval(const struct nft_expr *expr,
  44. struct nft_regs *regs)
  45. {
  46. const struct nft_bitwise_fast_expr *priv = nft_expr_priv(expr);
  47. u32 *src = &regs->data[priv->sreg];
  48. u32 *dst = &regs->data[priv->dreg];
  49. *dst = (*src & priv->mask) ^ priv->xor;
  50. }
  51. static void nft_cmp_fast_eval(const struct nft_expr *expr,
  52. struct nft_regs *regs)
  53. {
  54. const struct nft_cmp_fast_expr *priv = nft_expr_priv(expr);
  55. if (((regs->data[priv->sreg] & priv->mask) == priv->data) ^ priv->inv)
  56. return;
  57. regs->verdict.code = NFT_BREAK;
  58. }
  59. static bool nft_payload_fast_eval(const struct nft_expr *expr,
  60. struct nft_regs *regs,
  61. const struct nft_pktinfo *pkt)
  62. {
  63. const struct nft_payload *priv = nft_expr_priv(expr);
  64. const struct sk_buff *skb = pkt->skb;
  65. u32 *dest = &regs->data[priv->dreg];
  66. unsigned char *ptr;
  67. if (priv->base == NFT_PAYLOAD_NETWORK_HEADER)
  68. ptr = skb_network_header(skb);
  69. else {
  70. if (!pkt->tprot_set)
  71. return false;
  72. ptr = skb_network_header(skb) + pkt->xt.thoff;
  73. }
  74. ptr += priv->offset;
  75. if (unlikely(ptr + priv->len > skb_tail_pointer(skb)))
  76. return false;
  77. *dest = 0;
  78. if (priv->len == 2)
  79. *(u16 *)dest = *(u16 *)ptr;
  80. else if (priv->len == 4)
  81. *(u32 *)dest = *(u32 *)ptr;
  82. else
  83. *(u8 *)dest = *(u8 *)ptr;
  84. return true;
  85. }
  86. DEFINE_STATIC_KEY_FALSE(nft_counters_enabled);
  87. static noinline void nft_update_chain_stats(const struct nft_chain *chain,
  88. const struct nft_pktinfo *pkt)
  89. {
  90. struct nft_base_chain *base_chain;
  91. struct nft_stats __percpu *pstats;
  92. struct nft_stats *stats;
  93. base_chain = nft_base_chain(chain);
  94. rcu_read_lock();
  95. pstats = READ_ONCE(base_chain->stats);
  96. if (pstats) {
  97. local_bh_disable();
  98. stats = this_cpu_ptr(pstats);
  99. u64_stats_update_begin(&stats->syncp);
  100. stats->pkts++;
  101. stats->bytes += pkt->skb->len;
  102. u64_stats_update_end(&stats->syncp);
  103. local_bh_enable();
  104. }
  105. rcu_read_unlock();
  106. }
  107. struct nft_jumpstack {
  108. const struct nft_chain *chain;
  109. struct nft_rule *const *rules;
  110. };
  111. static void expr_call_ops_eval(const struct nft_expr *expr,
  112. struct nft_regs *regs,
  113. struct nft_pktinfo *pkt)
  114. {
  115. #ifdef CONFIG_RETPOLINE
  116. unsigned long e = (unsigned long)expr->ops->eval;
  117. #define X(e, fun) \
  118. do { if ((e) == (unsigned long)(fun)) \
  119. return fun(expr, regs, pkt); } while (0)
  120. X(e, nft_payload_eval);
  121. X(e, nft_cmp_eval);
  122. X(e, nft_meta_get_eval);
  123. X(e, nft_lookup_eval);
  124. X(e, nft_range_eval);
  125. X(e, nft_immediate_eval);
  126. X(e, nft_byteorder_eval);
  127. X(e, nft_dynset_eval);
  128. X(e, nft_rt_get_eval);
  129. X(e, nft_bitwise_eval);
  130. #undef X
  131. #endif /* CONFIG_RETPOLINE */
  132. expr->ops->eval(expr, regs, pkt);
  133. }
  134. unsigned int
  135. nft_do_chain(struct nft_pktinfo *pkt, void *priv)
  136. {
  137. const struct nft_chain *chain = priv, *basechain = chain;
  138. const struct net *net = nft_net(pkt);
  139. struct nft_rule *const *rules;
  140. const struct nft_rule *rule;
  141. const struct nft_expr *expr, *last;
  142. struct nft_regs regs = {};
  143. unsigned int stackptr = 0;
  144. struct nft_jumpstack jumpstack[NFT_JUMP_STACK_SIZE];
  145. bool genbit = READ_ONCE(net->nft.gencursor);
  146. struct nft_traceinfo info;
  147. info.trace = false;
  148. if (static_branch_unlikely(&nft_trace_enabled))
  149. nft_trace_init(&info, pkt, &regs.verdict, basechain);
  150. do_chain:
  151. if (genbit)
  152. rules = rcu_dereference(chain->rules_gen_1);
  153. else
  154. rules = rcu_dereference(chain->rules_gen_0);
  155. next_rule:
  156. rule = *rules;
  157. regs.verdict.code = NFT_CONTINUE;
  158. for (; *rules ; rules++) {
  159. rule = *rules;
  160. nft_rule_for_each_expr(expr, last, rule) {
  161. if (expr->ops == &nft_cmp_fast_ops)
  162. nft_cmp_fast_eval(expr, &regs);
  163. else if (expr->ops == &nft_bitwise_fast_ops)
  164. nft_bitwise_fast_eval(expr, &regs);
  165. else if (expr->ops != &nft_payload_fast_ops ||
  166. !nft_payload_fast_eval(expr, &regs, pkt))
  167. expr_call_ops_eval(expr, &regs, pkt);
  168. if (regs.verdict.code != NFT_CONTINUE)
  169. break;
  170. }
  171. switch (regs.verdict.code) {
  172. case NFT_BREAK:
  173. regs.verdict.code = NFT_CONTINUE;
  174. continue;
  175. case NFT_CONTINUE:
  176. nft_trace_packet(&info, chain, rule,
  177. NFT_TRACETYPE_RULE);
  178. continue;
  179. }
  180. break;
  181. }
  182. switch (regs.verdict.code & NF_VERDICT_MASK) {
  183. case NF_ACCEPT:
  184. case NF_DROP:
  185. case NF_QUEUE:
  186. case NF_STOLEN:
  187. nft_trace_packet(&info, chain, rule,
  188. NFT_TRACETYPE_RULE);
  189. return regs.verdict.code;
  190. }
  191. switch (regs.verdict.code) {
  192. case NFT_JUMP:
  193. if (WARN_ON_ONCE(stackptr >= NFT_JUMP_STACK_SIZE))
  194. return NF_DROP;
  195. jumpstack[stackptr].chain = chain;
  196. jumpstack[stackptr].rules = rules + 1;
  197. stackptr++;
  198. fallthrough;
  199. case NFT_GOTO:
  200. nft_trace_packet(&info, chain, rule,
  201. NFT_TRACETYPE_RULE);
  202. chain = regs.verdict.chain;
  203. goto do_chain;
  204. case NFT_CONTINUE:
  205. case NFT_RETURN:
  206. nft_trace_packet(&info, chain, rule,
  207. NFT_TRACETYPE_RETURN);
  208. break;
  209. default:
  210. WARN_ON(1);
  211. }
  212. if (stackptr > 0) {
  213. stackptr--;
  214. chain = jumpstack[stackptr].chain;
  215. rules = jumpstack[stackptr].rules;
  216. goto next_rule;
  217. }
  218. nft_trace_packet(&info, basechain, NULL, NFT_TRACETYPE_POLICY);
  219. if (static_branch_unlikely(&nft_counters_enabled))
  220. nft_update_chain_stats(basechain, pkt);
  221. return nft_base_chain(basechain)->policy;
  222. }
  223. EXPORT_SYMBOL_GPL(nft_do_chain);
  224. static struct nft_expr_type *nft_basic_types[] = {
  225. &nft_imm_type,
  226. &nft_cmp_type,
  227. &nft_lookup_type,
  228. &nft_bitwise_type,
  229. &nft_byteorder_type,
  230. &nft_payload_type,
  231. &nft_dynset_type,
  232. &nft_range_type,
  233. &nft_meta_type,
  234. &nft_rt_type,
  235. &nft_exthdr_type,
  236. };
  237. static struct nft_object_type *nft_basic_objects[] = {
  238. #ifdef CONFIG_NETWORK_SECMARK
  239. &nft_secmark_obj_type,
  240. #endif
  241. };
  242. int __init nf_tables_core_module_init(void)
  243. {
  244. int err, i, j = 0;
  245. for (i = 0; i < ARRAY_SIZE(nft_basic_objects); i++) {
  246. err = nft_register_obj(nft_basic_objects[i]);
  247. if (err)
  248. goto err;
  249. }
  250. for (j = 0; j < ARRAY_SIZE(nft_basic_types); j++) {
  251. err = nft_register_expr(nft_basic_types[j]);
  252. if (err)
  253. goto err;
  254. }
  255. return 0;
  256. err:
  257. while (j-- > 0)
  258. nft_unregister_expr(nft_basic_types[j]);
  259. while (i-- > 0)
  260. nft_unregister_obj(nft_basic_objects[i]);
  261. return err;
  262. }
  263. void nf_tables_core_module_exit(void)
  264. {
  265. int i;
  266. i = ARRAY_SIZE(nft_basic_types);
  267. while (i-- > 0)
  268. nft_unregister_expr(nft_basic_types[i]);
  269. i = ARRAY_SIZE(nft_basic_objects);
  270. while (i-- > 0)
  271. nft_unregister_obj(nft_basic_objects[i]);
  272. }