cls_u32.c 33 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * net/sched/cls_u32.c Ugly (or Universal) 32bit key Packet Classifier.
  4. *
  5. * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6. *
  7. * The filters are packed to hash tables of key nodes
  8. * with a set of 32bit key/mask pairs at every node.
  9. * Nodes reference next level hash tables etc.
  10. *
  11. * This scheme is the best universal classifier I managed to
  12. * invent; it is not super-fast, but it is not slow (provided you
  13. * program it correctly), and general enough. And its relative
  14. * speed grows as the number of rules becomes larger.
  15. *
  16. * It seems that it represents the best middle point between
  17. * speed and manageability both by human and by machine.
  18. *
  19. * It is especially useful for link sharing combined with QoS;
  20. * pure RSVP doesn't need such a general approach and can use
  21. * much simpler (and faster) schemes, sort of cls_rsvp.c.
  22. *
  23. * nfmark match added by Catalin(ux aka Dino) BOIE <catab at umbrella.ro>
  24. */
  25. #include <linux/module.h>
  26. #include <linux/slab.h>
  27. #include <linux/types.h>
  28. #include <linux/kernel.h>
  29. #include <linux/string.h>
  30. #include <linux/errno.h>
  31. #include <linux/percpu.h>
  32. #include <linux/rtnetlink.h>
  33. #include <linux/skbuff.h>
  34. #include <linux/bitmap.h>
  35. #include <linux/netdevice.h>
  36. #include <linux/hash.h>
  37. #include <net/netlink.h>
  38. #include <net/act_api.h>
  39. #include <net/pkt_cls.h>
  40. #include <linux/idr.h>
  41. struct tc_u_knode {
  42. struct tc_u_knode __rcu *next;
  43. u32 handle;
  44. struct tc_u_hnode __rcu *ht_up;
  45. struct tcf_exts exts;
  46. int ifindex;
  47. u8 fshift;
  48. struct tcf_result res;
  49. struct tc_u_hnode __rcu *ht_down;
  50. #ifdef CONFIG_CLS_U32_PERF
  51. struct tc_u32_pcnt __percpu *pf;
  52. #endif
  53. u32 flags;
  54. unsigned int in_hw_count;
  55. #ifdef CONFIG_CLS_U32_MARK
  56. u32 val;
  57. u32 mask;
  58. u32 __percpu *pcpu_success;
  59. #endif
  60. struct rcu_work rwork;
  61. /* The 'sel' field MUST be the last field in structure to allow for
  62. * tc_u32_keys allocated at end of structure.
  63. */
  64. struct tc_u32_sel sel;
  65. };
  66. struct tc_u_hnode {
  67. struct tc_u_hnode __rcu *next;
  68. u32 handle;
  69. u32 prio;
  70. int refcnt;
  71. unsigned int divisor;
  72. struct idr handle_idr;
  73. bool is_root;
  74. struct rcu_head rcu;
  75. u32 flags;
  76. /* The 'ht' field MUST be the last field in structure to allow for
  77. * more entries allocated at end of structure.
  78. */
  79. struct tc_u_knode __rcu *ht[];
  80. };
  81. struct tc_u_common {
  82. struct tc_u_hnode __rcu *hlist;
  83. void *ptr;
  84. int refcnt;
  85. struct idr handle_idr;
  86. struct hlist_node hnode;
  87. long knodes;
  88. };
  89. static inline unsigned int u32_hash_fold(__be32 key,
  90. const struct tc_u32_sel *sel,
  91. u8 fshift)
  92. {
  93. unsigned int h = ntohl(key & sel->hmask) >> fshift;
  94. return h;
  95. }
  96. static int u32_classify(struct sk_buff *skb, const struct tcf_proto *tp,
  97. struct tcf_result *res)
  98. {
  99. struct {
  100. struct tc_u_knode *knode;
  101. unsigned int off;
  102. } stack[TC_U32_MAXDEPTH];
  103. struct tc_u_hnode *ht = rcu_dereference_bh(tp->root);
  104. unsigned int off = skb_network_offset(skb);
  105. struct tc_u_knode *n;
  106. int sdepth = 0;
  107. int off2 = 0;
  108. int sel = 0;
  109. #ifdef CONFIG_CLS_U32_PERF
  110. int j;
  111. #endif
  112. int i, r;
  113. next_ht:
  114. n = rcu_dereference_bh(ht->ht[sel]);
  115. next_knode:
  116. if (n) {
  117. struct tc_u32_key *key = n->sel.keys;
  118. #ifdef CONFIG_CLS_U32_PERF
  119. __this_cpu_inc(n->pf->rcnt);
  120. j = 0;
  121. #endif
  122. if (tc_skip_sw(n->flags)) {
  123. n = rcu_dereference_bh(n->next);
  124. goto next_knode;
  125. }
  126. #ifdef CONFIG_CLS_U32_MARK
  127. if ((skb->mark & n->mask) != n->val) {
  128. n = rcu_dereference_bh(n->next);
  129. goto next_knode;
  130. } else {
  131. __this_cpu_inc(*n->pcpu_success);
  132. }
  133. #endif
  134. for (i = n->sel.nkeys; i > 0; i--, key++) {
  135. int toff = off + key->off + (off2 & key->offmask);
  136. __be32 *data, hdata;
  137. if (skb_headroom(skb) + toff > INT_MAX)
  138. goto out;
  139. data = skb_header_pointer(skb, toff, 4, &hdata);
  140. if (!data)
  141. goto out;
  142. if ((*data ^ key->val) & key->mask) {
  143. n = rcu_dereference_bh(n->next);
  144. goto next_knode;
  145. }
  146. #ifdef CONFIG_CLS_U32_PERF
  147. __this_cpu_inc(n->pf->kcnts[j]);
  148. j++;
  149. #endif
  150. }
  151. ht = rcu_dereference_bh(n->ht_down);
  152. if (!ht) {
  153. check_terminal:
  154. if (n->sel.flags & TC_U32_TERMINAL) {
  155. *res = n->res;
  156. if (!tcf_match_indev(skb, n->ifindex)) {
  157. n = rcu_dereference_bh(n->next);
  158. goto next_knode;
  159. }
  160. #ifdef CONFIG_CLS_U32_PERF
  161. __this_cpu_inc(n->pf->rhit);
  162. #endif
  163. r = tcf_exts_exec(skb, &n->exts, res);
  164. if (r < 0) {
  165. n = rcu_dereference_bh(n->next);
  166. goto next_knode;
  167. }
  168. return r;
  169. }
  170. n = rcu_dereference_bh(n->next);
  171. goto next_knode;
  172. }
  173. /* PUSH */
  174. if (sdepth >= TC_U32_MAXDEPTH)
  175. goto deadloop;
  176. stack[sdepth].knode = n;
  177. stack[sdepth].off = off;
  178. sdepth++;
  179. ht = rcu_dereference_bh(n->ht_down);
  180. sel = 0;
  181. if (ht->divisor) {
  182. __be32 *data, hdata;
  183. data = skb_header_pointer(skb, off + n->sel.hoff, 4,
  184. &hdata);
  185. if (!data)
  186. goto out;
  187. sel = ht->divisor & u32_hash_fold(*data, &n->sel,
  188. n->fshift);
  189. }
  190. if (!(n->sel.flags & (TC_U32_VAROFFSET | TC_U32_OFFSET | TC_U32_EAT)))
  191. goto next_ht;
  192. if (n->sel.flags & (TC_U32_OFFSET | TC_U32_VAROFFSET)) {
  193. off2 = n->sel.off + 3;
  194. if (n->sel.flags & TC_U32_VAROFFSET) {
  195. __be16 *data, hdata;
  196. data = skb_header_pointer(skb,
  197. off + n->sel.offoff,
  198. 2, &hdata);
  199. if (!data)
  200. goto out;
  201. off2 += ntohs(n->sel.offmask & *data) >>
  202. n->sel.offshift;
  203. }
  204. off2 &= ~3;
  205. }
  206. if (n->sel.flags & TC_U32_EAT) {
  207. off += off2;
  208. off2 = 0;
  209. }
  210. if (off < skb->len)
  211. goto next_ht;
  212. }
  213. /* POP */
  214. if (sdepth--) {
  215. n = stack[sdepth].knode;
  216. ht = rcu_dereference_bh(n->ht_up);
  217. off = stack[sdepth].off;
  218. goto check_terminal;
  219. }
  220. out:
  221. return -1;
  222. deadloop:
  223. net_warn_ratelimited("cls_u32: dead loop\n");
  224. return -1;
  225. }
  226. static struct tc_u_hnode *u32_lookup_ht(struct tc_u_common *tp_c, u32 handle)
  227. {
  228. struct tc_u_hnode *ht;
  229. for (ht = rtnl_dereference(tp_c->hlist);
  230. ht;
  231. ht = rtnl_dereference(ht->next))
  232. if (ht->handle == handle)
  233. break;
  234. return ht;
  235. }
  236. static struct tc_u_knode *u32_lookup_key(struct tc_u_hnode *ht, u32 handle)
  237. {
  238. unsigned int sel;
  239. struct tc_u_knode *n = NULL;
  240. sel = TC_U32_HASH(handle);
  241. if (sel > ht->divisor)
  242. goto out;
  243. for (n = rtnl_dereference(ht->ht[sel]);
  244. n;
  245. n = rtnl_dereference(n->next))
  246. if (n->handle == handle)
  247. break;
  248. out:
  249. return n;
  250. }
  251. static void *u32_get(struct tcf_proto *tp, u32 handle)
  252. {
  253. struct tc_u_hnode *ht;
  254. struct tc_u_common *tp_c = tp->data;
  255. if (TC_U32_HTID(handle) == TC_U32_ROOT)
  256. ht = rtnl_dereference(tp->root);
  257. else
  258. ht = u32_lookup_ht(tp_c, TC_U32_HTID(handle));
  259. if (!ht)
  260. return NULL;
  261. if (TC_U32_KEY(handle) == 0)
  262. return ht;
  263. return u32_lookup_key(ht, handle);
  264. }
  265. /* Protected by rtnl lock */
  266. static u32 gen_new_htid(struct tc_u_common *tp_c, struct tc_u_hnode *ptr)
  267. {
  268. int id = idr_alloc_cyclic(&tp_c->handle_idr, ptr, 1, 0x7FF, GFP_KERNEL);
  269. if (id < 0)
  270. return 0;
  271. return (id | 0x800U) << 20;
  272. }
  273. static struct hlist_head *tc_u_common_hash;
  274. #define U32_HASH_SHIFT 10
  275. #define U32_HASH_SIZE (1 << U32_HASH_SHIFT)
  276. static void *tc_u_common_ptr(const struct tcf_proto *tp)
  277. {
  278. struct tcf_block *block = tp->chain->block;
  279. /* The block sharing is currently supported only
  280. * for classless qdiscs. In that case we use block
  281. * for tc_u_common identification. In case the
  282. * block is not shared, block->q is a valid pointer
  283. * and we can use that. That works for classful qdiscs.
  284. */
  285. if (tcf_block_shared(block))
  286. return block;
  287. else
  288. return block->q;
  289. }
  290. static struct hlist_head *tc_u_hash(void *key)
  291. {
  292. return tc_u_common_hash + hash_ptr(key, U32_HASH_SHIFT);
  293. }
  294. static struct tc_u_common *tc_u_common_find(void *key)
  295. {
  296. struct tc_u_common *tc;
  297. hlist_for_each_entry(tc, tc_u_hash(key), hnode) {
  298. if (tc->ptr == key)
  299. return tc;
  300. }
  301. return NULL;
  302. }
  303. static int u32_init(struct tcf_proto *tp)
  304. {
  305. struct tc_u_hnode *root_ht;
  306. void *key = tc_u_common_ptr(tp);
  307. struct tc_u_common *tp_c = tc_u_common_find(key);
  308. root_ht = kzalloc(struct_size(root_ht, ht, 1), GFP_KERNEL);
  309. if (root_ht == NULL)
  310. return -ENOBUFS;
  311. root_ht->refcnt++;
  312. root_ht->handle = tp_c ? gen_new_htid(tp_c, root_ht) : 0x80000000;
  313. root_ht->prio = tp->prio;
  314. root_ht->is_root = true;
  315. idr_init(&root_ht->handle_idr);
  316. if (tp_c == NULL) {
  317. tp_c = kzalloc(struct_size(tp_c, hlist->ht, 1), GFP_KERNEL);
  318. if (tp_c == NULL) {
  319. kfree(root_ht);
  320. return -ENOBUFS;
  321. }
  322. tp_c->ptr = key;
  323. INIT_HLIST_NODE(&tp_c->hnode);
  324. idr_init(&tp_c->handle_idr);
  325. hlist_add_head(&tp_c->hnode, tc_u_hash(key));
  326. }
  327. tp_c->refcnt++;
  328. RCU_INIT_POINTER(root_ht->next, tp_c->hlist);
  329. rcu_assign_pointer(tp_c->hlist, root_ht);
  330. root_ht->refcnt++;
  331. rcu_assign_pointer(tp->root, root_ht);
  332. tp->data = tp_c;
  333. return 0;
  334. }
  335. static void __u32_destroy_key(struct tc_u_knode *n)
  336. {
  337. struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
  338. tcf_exts_destroy(&n->exts);
  339. if (ht && --ht->refcnt == 0)
  340. kfree(ht);
  341. kfree(n);
  342. }
  343. static void u32_destroy_key(struct tc_u_knode *n, bool free_pf)
  344. {
  345. tcf_exts_put_net(&n->exts);
  346. #ifdef CONFIG_CLS_U32_PERF
  347. if (free_pf)
  348. free_percpu(n->pf);
  349. #endif
  350. #ifdef CONFIG_CLS_U32_MARK
  351. if (free_pf)
  352. free_percpu(n->pcpu_success);
  353. #endif
  354. __u32_destroy_key(n);
  355. }
  356. /* u32_delete_key_rcu should be called when free'ing a copied
  357. * version of a tc_u_knode obtained from u32_init_knode(). When
  358. * copies are obtained from u32_init_knode() the statistics are
  359. * shared between the old and new copies to allow readers to
  360. * continue to update the statistics during the copy. To support
  361. * this the u32_delete_key_rcu variant does not free the percpu
  362. * statistics.
  363. */
  364. static void u32_delete_key_work(struct work_struct *work)
  365. {
  366. struct tc_u_knode *key = container_of(to_rcu_work(work),
  367. struct tc_u_knode,
  368. rwork);
  369. rtnl_lock();
  370. u32_destroy_key(key, false);
  371. rtnl_unlock();
  372. }
  373. /* u32_delete_key_freepf_rcu is the rcu callback variant
  374. * that free's the entire structure including the statistics
  375. * percpu variables. Only use this if the key is not a copy
  376. * returned by u32_init_knode(). See u32_delete_key_rcu()
  377. * for the variant that should be used with keys return from
  378. * u32_init_knode()
  379. */
  380. static void u32_delete_key_freepf_work(struct work_struct *work)
  381. {
  382. struct tc_u_knode *key = container_of(to_rcu_work(work),
  383. struct tc_u_knode,
  384. rwork);
  385. rtnl_lock();
  386. u32_destroy_key(key, true);
  387. rtnl_unlock();
  388. }
  389. static int u32_delete_key(struct tcf_proto *tp, struct tc_u_knode *key)
  390. {
  391. struct tc_u_common *tp_c = tp->data;
  392. struct tc_u_knode __rcu **kp;
  393. struct tc_u_knode *pkp;
  394. struct tc_u_hnode *ht = rtnl_dereference(key->ht_up);
  395. if (ht) {
  396. kp = &ht->ht[TC_U32_HASH(key->handle)];
  397. for (pkp = rtnl_dereference(*kp); pkp;
  398. kp = &pkp->next, pkp = rtnl_dereference(*kp)) {
  399. if (pkp == key) {
  400. RCU_INIT_POINTER(*kp, key->next);
  401. tp_c->knodes--;
  402. tcf_unbind_filter(tp, &key->res);
  403. idr_remove(&ht->handle_idr, key->handle);
  404. tcf_exts_get_net(&key->exts);
  405. tcf_queue_work(&key->rwork, u32_delete_key_freepf_work);
  406. return 0;
  407. }
  408. }
  409. }
  410. WARN_ON(1);
  411. return 0;
  412. }
  413. static void u32_clear_hw_hnode(struct tcf_proto *tp, struct tc_u_hnode *h,
  414. struct netlink_ext_ack *extack)
  415. {
  416. struct tcf_block *block = tp->chain->block;
  417. struct tc_cls_u32_offload cls_u32 = {};
  418. tc_cls_common_offload_init(&cls_u32.common, tp, h->flags, extack);
  419. cls_u32.command = TC_CLSU32_DELETE_HNODE;
  420. cls_u32.hnode.divisor = h->divisor;
  421. cls_u32.hnode.handle = h->handle;
  422. cls_u32.hnode.prio = h->prio;
  423. tc_setup_cb_call(block, TC_SETUP_CLSU32, &cls_u32, false, true);
  424. }
  425. static int u32_replace_hw_hnode(struct tcf_proto *tp, struct tc_u_hnode *h,
  426. u32 flags, struct netlink_ext_ack *extack)
  427. {
  428. struct tcf_block *block = tp->chain->block;
  429. struct tc_cls_u32_offload cls_u32 = {};
  430. bool skip_sw = tc_skip_sw(flags);
  431. bool offloaded = false;
  432. int err;
  433. tc_cls_common_offload_init(&cls_u32.common, tp, flags, extack);
  434. cls_u32.command = TC_CLSU32_NEW_HNODE;
  435. cls_u32.hnode.divisor = h->divisor;
  436. cls_u32.hnode.handle = h->handle;
  437. cls_u32.hnode.prio = h->prio;
  438. err = tc_setup_cb_call(block, TC_SETUP_CLSU32, &cls_u32, skip_sw, true);
  439. if (err < 0) {
  440. u32_clear_hw_hnode(tp, h, NULL);
  441. return err;
  442. } else if (err > 0) {
  443. offloaded = true;
  444. }
  445. if (skip_sw && !offloaded)
  446. return -EINVAL;
  447. return 0;
  448. }
  449. static void u32_remove_hw_knode(struct tcf_proto *tp, struct tc_u_knode *n,
  450. struct netlink_ext_ack *extack)
  451. {
  452. struct tcf_block *block = tp->chain->block;
  453. struct tc_cls_u32_offload cls_u32 = {};
  454. tc_cls_common_offload_init(&cls_u32.common, tp, n->flags, extack);
  455. cls_u32.command = TC_CLSU32_DELETE_KNODE;
  456. cls_u32.knode.handle = n->handle;
  457. tc_setup_cb_destroy(block, tp, TC_SETUP_CLSU32, &cls_u32, false,
  458. &n->flags, &n->in_hw_count, true);
  459. }
  460. static int u32_replace_hw_knode(struct tcf_proto *tp, struct tc_u_knode *n,
  461. u32 flags, struct netlink_ext_ack *extack)
  462. {
  463. struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
  464. struct tcf_block *block = tp->chain->block;
  465. struct tc_cls_u32_offload cls_u32 = {};
  466. bool skip_sw = tc_skip_sw(flags);
  467. int err;
  468. tc_cls_common_offload_init(&cls_u32.common, tp, flags, extack);
  469. cls_u32.command = TC_CLSU32_REPLACE_KNODE;
  470. cls_u32.knode.handle = n->handle;
  471. cls_u32.knode.fshift = n->fshift;
  472. #ifdef CONFIG_CLS_U32_MARK
  473. cls_u32.knode.val = n->val;
  474. cls_u32.knode.mask = n->mask;
  475. #else
  476. cls_u32.knode.val = 0;
  477. cls_u32.knode.mask = 0;
  478. #endif
  479. cls_u32.knode.sel = &n->sel;
  480. cls_u32.knode.res = &n->res;
  481. cls_u32.knode.exts = &n->exts;
  482. if (n->ht_down)
  483. cls_u32.knode.link_handle = ht->handle;
  484. err = tc_setup_cb_add(block, tp, TC_SETUP_CLSU32, &cls_u32, skip_sw,
  485. &n->flags, &n->in_hw_count, true);
  486. if (err) {
  487. u32_remove_hw_knode(tp, n, NULL);
  488. return err;
  489. }
  490. if (skip_sw && !(n->flags & TCA_CLS_FLAGS_IN_HW))
  491. return -EINVAL;
  492. return 0;
  493. }
  494. static void u32_clear_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
  495. struct netlink_ext_ack *extack)
  496. {
  497. struct tc_u_common *tp_c = tp->data;
  498. struct tc_u_knode *n;
  499. unsigned int h;
  500. for (h = 0; h <= ht->divisor; h++) {
  501. while ((n = rtnl_dereference(ht->ht[h])) != NULL) {
  502. RCU_INIT_POINTER(ht->ht[h],
  503. rtnl_dereference(n->next));
  504. tp_c->knodes--;
  505. tcf_unbind_filter(tp, &n->res);
  506. u32_remove_hw_knode(tp, n, extack);
  507. idr_remove(&ht->handle_idr, n->handle);
  508. if (tcf_exts_get_net(&n->exts))
  509. tcf_queue_work(&n->rwork, u32_delete_key_freepf_work);
  510. else
  511. u32_destroy_key(n, true);
  512. }
  513. }
  514. }
  515. static int u32_destroy_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
  516. struct netlink_ext_ack *extack)
  517. {
  518. struct tc_u_common *tp_c = tp->data;
  519. struct tc_u_hnode __rcu **hn;
  520. struct tc_u_hnode *phn;
  521. WARN_ON(--ht->refcnt);
  522. u32_clear_hnode(tp, ht, extack);
  523. hn = &tp_c->hlist;
  524. for (phn = rtnl_dereference(*hn);
  525. phn;
  526. hn = &phn->next, phn = rtnl_dereference(*hn)) {
  527. if (phn == ht) {
  528. u32_clear_hw_hnode(tp, ht, extack);
  529. idr_destroy(&ht->handle_idr);
  530. idr_remove(&tp_c->handle_idr, ht->handle);
  531. RCU_INIT_POINTER(*hn, ht->next);
  532. kfree_rcu(ht, rcu);
  533. return 0;
  534. }
  535. }
  536. return -ENOENT;
  537. }
  538. static void u32_destroy(struct tcf_proto *tp, bool rtnl_held,
  539. struct netlink_ext_ack *extack)
  540. {
  541. struct tc_u_common *tp_c = tp->data;
  542. struct tc_u_hnode *root_ht = rtnl_dereference(tp->root);
  543. WARN_ON(root_ht == NULL);
  544. if (root_ht && --root_ht->refcnt == 1)
  545. u32_destroy_hnode(tp, root_ht, extack);
  546. if (--tp_c->refcnt == 0) {
  547. struct tc_u_hnode *ht;
  548. hlist_del(&tp_c->hnode);
  549. while ((ht = rtnl_dereference(tp_c->hlist)) != NULL) {
  550. u32_clear_hnode(tp, ht, extack);
  551. RCU_INIT_POINTER(tp_c->hlist, ht->next);
  552. /* u32_destroy_key() will later free ht for us, if it's
  553. * still referenced by some knode
  554. */
  555. if (--ht->refcnt == 0)
  556. kfree_rcu(ht, rcu);
  557. }
  558. idr_destroy(&tp_c->handle_idr);
  559. kfree(tp_c);
  560. }
  561. tp->data = NULL;
  562. }
  563. static int u32_delete(struct tcf_proto *tp, void *arg, bool *last,
  564. bool rtnl_held, struct netlink_ext_ack *extack)
  565. {
  566. struct tc_u_hnode *ht = arg;
  567. struct tc_u_common *tp_c = tp->data;
  568. int ret = 0;
  569. if (TC_U32_KEY(ht->handle)) {
  570. u32_remove_hw_knode(tp, (struct tc_u_knode *)ht, extack);
  571. ret = u32_delete_key(tp, (struct tc_u_knode *)ht);
  572. goto out;
  573. }
  574. if (ht->is_root) {
  575. NL_SET_ERR_MSG_MOD(extack, "Not allowed to delete root node");
  576. return -EINVAL;
  577. }
  578. if (ht->refcnt == 1) {
  579. u32_destroy_hnode(tp, ht, extack);
  580. } else {
  581. NL_SET_ERR_MSG_MOD(extack, "Can not delete in-use filter");
  582. return -EBUSY;
  583. }
  584. out:
  585. *last = tp_c->refcnt == 1 && tp_c->knodes == 0;
  586. return ret;
  587. }
  588. static u32 gen_new_kid(struct tc_u_hnode *ht, u32 htid)
  589. {
  590. u32 index = htid | 0x800;
  591. u32 max = htid | 0xFFF;
  592. if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max, GFP_KERNEL)) {
  593. index = htid + 1;
  594. if (idr_alloc_u32(&ht->handle_idr, NULL, &index, max,
  595. GFP_KERNEL))
  596. index = max;
  597. }
  598. return index;
  599. }
  600. static const struct nla_policy u32_policy[TCA_U32_MAX + 1] = {
  601. [TCA_U32_CLASSID] = { .type = NLA_U32 },
  602. [TCA_U32_HASH] = { .type = NLA_U32 },
  603. [TCA_U32_LINK] = { .type = NLA_U32 },
  604. [TCA_U32_DIVISOR] = { .type = NLA_U32 },
  605. [TCA_U32_SEL] = { .len = sizeof(struct tc_u32_sel) },
  606. [TCA_U32_INDEV] = { .type = NLA_STRING, .len = IFNAMSIZ },
  607. [TCA_U32_MARK] = { .len = sizeof(struct tc_u32_mark) },
  608. [TCA_U32_FLAGS] = { .type = NLA_U32 },
  609. };
  610. static int u32_set_parms(struct net *net, struct tcf_proto *tp,
  611. unsigned long base,
  612. struct tc_u_knode *n, struct nlattr **tb,
  613. struct nlattr *est, bool ovr,
  614. struct netlink_ext_ack *extack)
  615. {
  616. int err;
  617. err = tcf_exts_validate(net, tp, tb, est, &n->exts, ovr, true, extack);
  618. if (err < 0)
  619. return err;
  620. if (tb[TCA_U32_LINK]) {
  621. u32 handle = nla_get_u32(tb[TCA_U32_LINK]);
  622. struct tc_u_hnode *ht_down = NULL, *ht_old;
  623. if (TC_U32_KEY(handle)) {
  624. NL_SET_ERR_MSG_MOD(extack, "u32 Link handle must be a hash table");
  625. return -EINVAL;
  626. }
  627. if (handle) {
  628. ht_down = u32_lookup_ht(tp->data, handle);
  629. if (!ht_down) {
  630. NL_SET_ERR_MSG_MOD(extack, "Link hash table not found");
  631. return -EINVAL;
  632. }
  633. if (ht_down->is_root) {
  634. NL_SET_ERR_MSG_MOD(extack, "Not linking to root node");
  635. return -EINVAL;
  636. }
  637. ht_down->refcnt++;
  638. }
  639. ht_old = rtnl_dereference(n->ht_down);
  640. rcu_assign_pointer(n->ht_down, ht_down);
  641. if (ht_old)
  642. ht_old->refcnt--;
  643. }
  644. if (tb[TCA_U32_CLASSID]) {
  645. n->res.classid = nla_get_u32(tb[TCA_U32_CLASSID]);
  646. tcf_bind_filter(tp, &n->res, base);
  647. }
  648. if (tb[TCA_U32_INDEV]) {
  649. int ret;
  650. ret = tcf_change_indev(net, tb[TCA_U32_INDEV], extack);
  651. if (ret < 0)
  652. return -EINVAL;
  653. n->ifindex = ret;
  654. }
  655. return 0;
  656. }
  657. static void u32_replace_knode(struct tcf_proto *tp, struct tc_u_common *tp_c,
  658. struct tc_u_knode *n)
  659. {
  660. struct tc_u_knode __rcu **ins;
  661. struct tc_u_knode *pins;
  662. struct tc_u_hnode *ht;
  663. if (TC_U32_HTID(n->handle) == TC_U32_ROOT)
  664. ht = rtnl_dereference(tp->root);
  665. else
  666. ht = u32_lookup_ht(tp_c, TC_U32_HTID(n->handle));
  667. ins = &ht->ht[TC_U32_HASH(n->handle)];
  668. /* The node must always exist for it to be replaced if this is not the
  669. * case then something went very wrong elsewhere.
  670. */
  671. for (pins = rtnl_dereference(*ins); ;
  672. ins = &pins->next, pins = rtnl_dereference(*ins))
  673. if (pins->handle == n->handle)
  674. break;
  675. idr_replace(&ht->handle_idr, n, n->handle);
  676. RCU_INIT_POINTER(n->next, pins->next);
  677. rcu_assign_pointer(*ins, n);
  678. }
  679. static struct tc_u_knode *u32_init_knode(struct net *net, struct tcf_proto *tp,
  680. struct tc_u_knode *n)
  681. {
  682. struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
  683. struct tc_u32_sel *s = &n->sel;
  684. struct tc_u_knode *new;
  685. new = kzalloc(struct_size(new, sel.keys, s->nkeys), GFP_KERNEL);
  686. if (!new)
  687. return NULL;
  688. RCU_INIT_POINTER(new->next, n->next);
  689. new->handle = n->handle;
  690. RCU_INIT_POINTER(new->ht_up, n->ht_up);
  691. new->ifindex = n->ifindex;
  692. new->fshift = n->fshift;
  693. new->res = n->res;
  694. new->flags = n->flags;
  695. RCU_INIT_POINTER(new->ht_down, ht);
  696. #ifdef CONFIG_CLS_U32_PERF
  697. /* Statistics may be incremented by readers during update
  698. * so we must keep them in tact. When the node is later destroyed
  699. * a special destroy call must be made to not free the pf memory.
  700. */
  701. new->pf = n->pf;
  702. #endif
  703. #ifdef CONFIG_CLS_U32_MARK
  704. new->val = n->val;
  705. new->mask = n->mask;
  706. /* Similarly success statistics must be moved as pointers */
  707. new->pcpu_success = n->pcpu_success;
  708. #endif
  709. memcpy(&new->sel, s, struct_size(s, keys, s->nkeys));
  710. if (tcf_exts_init(&new->exts, net, TCA_U32_ACT, TCA_U32_POLICE)) {
  711. kfree(new);
  712. return NULL;
  713. }
  714. /* bump reference count as long as we hold pointer to structure */
  715. if (ht)
  716. ht->refcnt++;
  717. return new;
  718. }
  719. static int u32_change(struct net *net, struct sk_buff *in_skb,
  720. struct tcf_proto *tp, unsigned long base, u32 handle,
  721. struct nlattr **tca, void **arg, bool ovr, bool rtnl_held,
  722. struct netlink_ext_ack *extack)
  723. {
  724. struct tc_u_common *tp_c = tp->data;
  725. struct tc_u_hnode *ht;
  726. struct tc_u_knode *n;
  727. struct tc_u32_sel *s;
  728. struct nlattr *opt = tca[TCA_OPTIONS];
  729. struct nlattr *tb[TCA_U32_MAX + 1];
  730. u32 htid, flags = 0;
  731. size_t sel_size;
  732. int err;
  733. if (!opt) {
  734. if (handle) {
  735. NL_SET_ERR_MSG_MOD(extack, "Filter handle requires options");
  736. return -EINVAL;
  737. } else {
  738. return 0;
  739. }
  740. }
  741. err = nla_parse_nested_deprecated(tb, TCA_U32_MAX, opt, u32_policy,
  742. extack);
  743. if (err < 0)
  744. return err;
  745. if (tb[TCA_U32_FLAGS]) {
  746. flags = nla_get_u32(tb[TCA_U32_FLAGS]);
  747. if (!tc_flags_valid(flags)) {
  748. NL_SET_ERR_MSG_MOD(extack, "Invalid filter flags");
  749. return -EINVAL;
  750. }
  751. }
  752. n = *arg;
  753. if (n) {
  754. struct tc_u_knode *new;
  755. if (TC_U32_KEY(n->handle) == 0) {
  756. NL_SET_ERR_MSG_MOD(extack, "Key node id cannot be zero");
  757. return -EINVAL;
  758. }
  759. if ((n->flags ^ flags) &
  760. ~(TCA_CLS_FLAGS_IN_HW | TCA_CLS_FLAGS_NOT_IN_HW)) {
  761. NL_SET_ERR_MSG_MOD(extack, "Key node flags do not match passed flags");
  762. return -EINVAL;
  763. }
  764. new = u32_init_knode(net, tp, n);
  765. if (!new)
  766. return -ENOMEM;
  767. err = u32_set_parms(net, tp, base, new, tb,
  768. tca[TCA_RATE], ovr, extack);
  769. if (err) {
  770. __u32_destroy_key(new);
  771. return err;
  772. }
  773. err = u32_replace_hw_knode(tp, new, flags, extack);
  774. if (err) {
  775. __u32_destroy_key(new);
  776. return err;
  777. }
  778. if (!tc_in_hw(new->flags))
  779. new->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
  780. u32_replace_knode(tp, tp_c, new);
  781. tcf_unbind_filter(tp, &n->res);
  782. tcf_exts_get_net(&n->exts);
  783. tcf_queue_work(&n->rwork, u32_delete_key_work);
  784. return 0;
  785. }
  786. if (tb[TCA_U32_DIVISOR]) {
  787. unsigned int divisor = nla_get_u32(tb[TCA_U32_DIVISOR]);
  788. if (!is_power_of_2(divisor)) {
  789. NL_SET_ERR_MSG_MOD(extack, "Divisor is not a power of 2");
  790. return -EINVAL;
  791. }
  792. if (divisor-- > 0x100) {
  793. NL_SET_ERR_MSG_MOD(extack, "Exceeded maximum 256 hash buckets");
  794. return -EINVAL;
  795. }
  796. if (TC_U32_KEY(handle)) {
  797. NL_SET_ERR_MSG_MOD(extack, "Divisor can only be used on a hash table");
  798. return -EINVAL;
  799. }
  800. ht = kzalloc(struct_size(ht, ht, divisor + 1), GFP_KERNEL);
  801. if (ht == NULL)
  802. return -ENOBUFS;
  803. if (handle == 0) {
  804. handle = gen_new_htid(tp->data, ht);
  805. if (handle == 0) {
  806. kfree(ht);
  807. return -ENOMEM;
  808. }
  809. } else {
  810. err = idr_alloc_u32(&tp_c->handle_idr, ht, &handle,
  811. handle, GFP_KERNEL);
  812. if (err) {
  813. kfree(ht);
  814. return err;
  815. }
  816. }
  817. ht->refcnt = 1;
  818. ht->divisor = divisor;
  819. ht->handle = handle;
  820. ht->prio = tp->prio;
  821. idr_init(&ht->handle_idr);
  822. ht->flags = flags;
  823. err = u32_replace_hw_hnode(tp, ht, flags, extack);
  824. if (err) {
  825. idr_remove(&tp_c->handle_idr, handle);
  826. kfree(ht);
  827. return err;
  828. }
  829. RCU_INIT_POINTER(ht->next, tp_c->hlist);
  830. rcu_assign_pointer(tp_c->hlist, ht);
  831. *arg = ht;
  832. return 0;
  833. }
  834. if (tb[TCA_U32_HASH]) {
  835. htid = nla_get_u32(tb[TCA_U32_HASH]);
  836. if (TC_U32_HTID(htid) == TC_U32_ROOT) {
  837. ht = rtnl_dereference(tp->root);
  838. htid = ht->handle;
  839. } else {
  840. ht = u32_lookup_ht(tp->data, TC_U32_HTID(htid));
  841. if (!ht) {
  842. NL_SET_ERR_MSG_MOD(extack, "Specified hash table not found");
  843. return -EINVAL;
  844. }
  845. }
  846. } else {
  847. ht = rtnl_dereference(tp->root);
  848. htid = ht->handle;
  849. }
  850. if (ht->divisor < TC_U32_HASH(htid)) {
  851. NL_SET_ERR_MSG_MOD(extack, "Specified hash table buckets exceed configured value");
  852. return -EINVAL;
  853. }
  854. if (handle) {
  855. if (TC_U32_HTID(handle) && TC_U32_HTID(handle ^ htid)) {
  856. NL_SET_ERR_MSG_MOD(extack, "Handle specified hash table address mismatch");
  857. return -EINVAL;
  858. }
  859. handle = htid | TC_U32_NODE(handle);
  860. err = idr_alloc_u32(&ht->handle_idr, NULL, &handle, handle,
  861. GFP_KERNEL);
  862. if (err)
  863. return err;
  864. } else
  865. handle = gen_new_kid(ht, htid);
  866. if (tb[TCA_U32_SEL] == NULL) {
  867. NL_SET_ERR_MSG_MOD(extack, "Selector not specified");
  868. err = -EINVAL;
  869. goto erridr;
  870. }
  871. s = nla_data(tb[TCA_U32_SEL]);
  872. sel_size = struct_size(s, keys, s->nkeys);
  873. if (nla_len(tb[TCA_U32_SEL]) < sel_size) {
  874. err = -EINVAL;
  875. goto erridr;
  876. }
  877. n = kzalloc(struct_size(n, sel.keys, s->nkeys), GFP_KERNEL);
  878. if (n == NULL) {
  879. err = -ENOBUFS;
  880. goto erridr;
  881. }
  882. #ifdef CONFIG_CLS_U32_PERF
  883. n->pf = __alloc_percpu(struct_size(n->pf, kcnts, s->nkeys),
  884. __alignof__(struct tc_u32_pcnt));
  885. if (!n->pf) {
  886. err = -ENOBUFS;
  887. goto errfree;
  888. }
  889. #endif
  890. memcpy(&n->sel, s, sel_size);
  891. RCU_INIT_POINTER(n->ht_up, ht);
  892. n->handle = handle;
  893. n->fshift = s->hmask ? ffs(ntohl(s->hmask)) - 1 : 0;
  894. n->flags = flags;
  895. err = tcf_exts_init(&n->exts, net, TCA_U32_ACT, TCA_U32_POLICE);
  896. if (err < 0)
  897. goto errout;
  898. #ifdef CONFIG_CLS_U32_MARK
  899. n->pcpu_success = alloc_percpu(u32);
  900. if (!n->pcpu_success) {
  901. err = -ENOMEM;
  902. goto errout;
  903. }
  904. if (tb[TCA_U32_MARK]) {
  905. struct tc_u32_mark *mark;
  906. mark = nla_data(tb[TCA_U32_MARK]);
  907. n->val = mark->val;
  908. n->mask = mark->mask;
  909. }
  910. #endif
  911. err = u32_set_parms(net, tp, base, n, tb, tca[TCA_RATE], ovr,
  912. extack);
  913. if (err == 0) {
  914. struct tc_u_knode __rcu **ins;
  915. struct tc_u_knode *pins;
  916. err = u32_replace_hw_knode(tp, n, flags, extack);
  917. if (err)
  918. goto errhw;
  919. if (!tc_in_hw(n->flags))
  920. n->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
  921. ins = &ht->ht[TC_U32_HASH(handle)];
  922. for (pins = rtnl_dereference(*ins); pins;
  923. ins = &pins->next, pins = rtnl_dereference(*ins))
  924. if (TC_U32_NODE(handle) < TC_U32_NODE(pins->handle))
  925. break;
  926. RCU_INIT_POINTER(n->next, pins);
  927. rcu_assign_pointer(*ins, n);
  928. tp_c->knodes++;
  929. *arg = n;
  930. return 0;
  931. }
  932. errhw:
  933. #ifdef CONFIG_CLS_U32_MARK
  934. free_percpu(n->pcpu_success);
  935. #endif
  936. errout:
  937. tcf_exts_destroy(&n->exts);
  938. #ifdef CONFIG_CLS_U32_PERF
  939. errfree:
  940. free_percpu(n->pf);
  941. #endif
  942. kfree(n);
  943. erridr:
  944. idr_remove(&ht->handle_idr, handle);
  945. return err;
  946. }
  947. static void u32_walk(struct tcf_proto *tp, struct tcf_walker *arg,
  948. bool rtnl_held)
  949. {
  950. struct tc_u_common *tp_c = tp->data;
  951. struct tc_u_hnode *ht;
  952. struct tc_u_knode *n;
  953. unsigned int h;
  954. if (arg->stop)
  955. return;
  956. for (ht = rtnl_dereference(tp_c->hlist);
  957. ht;
  958. ht = rtnl_dereference(ht->next)) {
  959. if (ht->prio != tp->prio)
  960. continue;
  961. if (arg->count >= arg->skip) {
  962. if (arg->fn(tp, ht, arg) < 0) {
  963. arg->stop = 1;
  964. return;
  965. }
  966. }
  967. arg->count++;
  968. for (h = 0; h <= ht->divisor; h++) {
  969. for (n = rtnl_dereference(ht->ht[h]);
  970. n;
  971. n = rtnl_dereference(n->next)) {
  972. if (arg->count < arg->skip) {
  973. arg->count++;
  974. continue;
  975. }
  976. if (arg->fn(tp, n, arg) < 0) {
  977. arg->stop = 1;
  978. return;
  979. }
  980. arg->count++;
  981. }
  982. }
  983. }
  984. }
  985. static int u32_reoffload_hnode(struct tcf_proto *tp, struct tc_u_hnode *ht,
  986. bool add, flow_setup_cb_t *cb, void *cb_priv,
  987. struct netlink_ext_ack *extack)
  988. {
  989. struct tc_cls_u32_offload cls_u32 = {};
  990. int err;
  991. tc_cls_common_offload_init(&cls_u32.common, tp, ht->flags, extack);
  992. cls_u32.command = add ? TC_CLSU32_NEW_HNODE : TC_CLSU32_DELETE_HNODE;
  993. cls_u32.hnode.divisor = ht->divisor;
  994. cls_u32.hnode.handle = ht->handle;
  995. cls_u32.hnode.prio = ht->prio;
  996. err = cb(TC_SETUP_CLSU32, &cls_u32, cb_priv);
  997. if (err && add && tc_skip_sw(ht->flags))
  998. return err;
  999. return 0;
  1000. }
  1001. static int u32_reoffload_knode(struct tcf_proto *tp, struct tc_u_knode *n,
  1002. bool add, flow_setup_cb_t *cb, void *cb_priv,
  1003. struct netlink_ext_ack *extack)
  1004. {
  1005. struct tc_u_hnode *ht = rtnl_dereference(n->ht_down);
  1006. struct tcf_block *block = tp->chain->block;
  1007. struct tc_cls_u32_offload cls_u32 = {};
  1008. int err;
  1009. tc_cls_common_offload_init(&cls_u32.common, tp, n->flags, extack);
  1010. cls_u32.command = add ?
  1011. TC_CLSU32_REPLACE_KNODE : TC_CLSU32_DELETE_KNODE;
  1012. cls_u32.knode.handle = n->handle;
  1013. if (add) {
  1014. cls_u32.knode.fshift = n->fshift;
  1015. #ifdef CONFIG_CLS_U32_MARK
  1016. cls_u32.knode.val = n->val;
  1017. cls_u32.knode.mask = n->mask;
  1018. #else
  1019. cls_u32.knode.val = 0;
  1020. cls_u32.knode.mask = 0;
  1021. #endif
  1022. cls_u32.knode.sel = &n->sel;
  1023. cls_u32.knode.res = &n->res;
  1024. cls_u32.knode.exts = &n->exts;
  1025. if (n->ht_down)
  1026. cls_u32.knode.link_handle = ht->handle;
  1027. }
  1028. err = tc_setup_cb_reoffload(block, tp, add, cb, TC_SETUP_CLSU32,
  1029. &cls_u32, cb_priv, &n->flags,
  1030. &n->in_hw_count);
  1031. if (err)
  1032. return err;
  1033. return 0;
  1034. }
  1035. static int u32_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,
  1036. void *cb_priv, struct netlink_ext_ack *extack)
  1037. {
  1038. struct tc_u_common *tp_c = tp->data;
  1039. struct tc_u_hnode *ht;
  1040. struct tc_u_knode *n;
  1041. unsigned int h;
  1042. int err;
  1043. for (ht = rtnl_dereference(tp_c->hlist);
  1044. ht;
  1045. ht = rtnl_dereference(ht->next)) {
  1046. if (ht->prio != tp->prio)
  1047. continue;
  1048. /* When adding filters to a new dev, try to offload the
  1049. * hashtable first. When removing, do the filters before the
  1050. * hashtable.
  1051. */
  1052. if (add && !tc_skip_hw(ht->flags)) {
  1053. err = u32_reoffload_hnode(tp, ht, add, cb, cb_priv,
  1054. extack);
  1055. if (err)
  1056. return err;
  1057. }
  1058. for (h = 0; h <= ht->divisor; h++) {
  1059. for (n = rtnl_dereference(ht->ht[h]);
  1060. n;
  1061. n = rtnl_dereference(n->next)) {
  1062. if (tc_skip_hw(n->flags))
  1063. continue;
  1064. err = u32_reoffload_knode(tp, n, add, cb,
  1065. cb_priv, extack);
  1066. if (err)
  1067. return err;
  1068. }
  1069. }
  1070. if (!add && !tc_skip_hw(ht->flags))
  1071. u32_reoffload_hnode(tp, ht, add, cb, cb_priv, extack);
  1072. }
  1073. return 0;
  1074. }
  1075. static void u32_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
  1076. unsigned long base)
  1077. {
  1078. struct tc_u_knode *n = fh;
  1079. if (n && n->res.classid == classid) {
  1080. if (cl)
  1081. __tcf_bind_filter(q, &n->res, base);
  1082. else
  1083. __tcf_unbind_filter(q, &n->res);
  1084. }
  1085. }
  1086. static int u32_dump(struct net *net, struct tcf_proto *tp, void *fh,
  1087. struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
  1088. {
  1089. struct tc_u_knode *n = fh;
  1090. struct tc_u_hnode *ht_up, *ht_down;
  1091. struct nlattr *nest;
  1092. if (n == NULL)
  1093. return skb->len;
  1094. t->tcm_handle = n->handle;
  1095. nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
  1096. if (nest == NULL)
  1097. goto nla_put_failure;
  1098. if (TC_U32_KEY(n->handle) == 0) {
  1099. struct tc_u_hnode *ht = fh;
  1100. u32 divisor = ht->divisor + 1;
  1101. if (nla_put_u32(skb, TCA_U32_DIVISOR, divisor))
  1102. goto nla_put_failure;
  1103. } else {
  1104. #ifdef CONFIG_CLS_U32_PERF
  1105. struct tc_u32_pcnt *gpf;
  1106. int cpu;
  1107. #endif
  1108. if (nla_put(skb, TCA_U32_SEL, struct_size(&n->sel, keys, n->sel.nkeys),
  1109. &n->sel))
  1110. goto nla_put_failure;
  1111. ht_up = rtnl_dereference(n->ht_up);
  1112. if (ht_up) {
  1113. u32 htid = n->handle & 0xFFFFF000;
  1114. if (nla_put_u32(skb, TCA_U32_HASH, htid))
  1115. goto nla_put_failure;
  1116. }
  1117. if (n->res.classid &&
  1118. nla_put_u32(skb, TCA_U32_CLASSID, n->res.classid))
  1119. goto nla_put_failure;
  1120. ht_down = rtnl_dereference(n->ht_down);
  1121. if (ht_down &&
  1122. nla_put_u32(skb, TCA_U32_LINK, ht_down->handle))
  1123. goto nla_put_failure;
  1124. if (n->flags && nla_put_u32(skb, TCA_U32_FLAGS, n->flags))
  1125. goto nla_put_failure;
  1126. #ifdef CONFIG_CLS_U32_MARK
  1127. if ((n->val || n->mask)) {
  1128. struct tc_u32_mark mark = {.val = n->val,
  1129. .mask = n->mask,
  1130. .success = 0};
  1131. int cpum;
  1132. for_each_possible_cpu(cpum) {
  1133. __u32 cnt = *per_cpu_ptr(n->pcpu_success, cpum);
  1134. mark.success += cnt;
  1135. }
  1136. if (nla_put(skb, TCA_U32_MARK, sizeof(mark), &mark))
  1137. goto nla_put_failure;
  1138. }
  1139. #endif
  1140. if (tcf_exts_dump(skb, &n->exts) < 0)
  1141. goto nla_put_failure;
  1142. if (n->ifindex) {
  1143. struct net_device *dev;
  1144. dev = __dev_get_by_index(net, n->ifindex);
  1145. if (dev && nla_put_string(skb, TCA_U32_INDEV, dev->name))
  1146. goto nla_put_failure;
  1147. }
  1148. #ifdef CONFIG_CLS_U32_PERF
  1149. gpf = kzalloc(struct_size(gpf, kcnts, n->sel.nkeys), GFP_KERNEL);
  1150. if (!gpf)
  1151. goto nla_put_failure;
  1152. for_each_possible_cpu(cpu) {
  1153. int i;
  1154. struct tc_u32_pcnt *pf = per_cpu_ptr(n->pf, cpu);
  1155. gpf->rcnt += pf->rcnt;
  1156. gpf->rhit += pf->rhit;
  1157. for (i = 0; i < n->sel.nkeys; i++)
  1158. gpf->kcnts[i] += pf->kcnts[i];
  1159. }
  1160. if (nla_put_64bit(skb, TCA_U32_PCNT, struct_size(gpf, kcnts, n->sel.nkeys),
  1161. gpf, TCA_U32_PAD)) {
  1162. kfree(gpf);
  1163. goto nla_put_failure;
  1164. }
  1165. kfree(gpf);
  1166. #endif
  1167. }
  1168. nla_nest_end(skb, nest);
  1169. if (TC_U32_KEY(n->handle))
  1170. if (tcf_exts_dump_stats(skb, &n->exts) < 0)
  1171. goto nla_put_failure;
  1172. return skb->len;
  1173. nla_put_failure:
  1174. nla_nest_cancel(skb, nest);
  1175. return -1;
  1176. }
  1177. static struct tcf_proto_ops cls_u32_ops __read_mostly = {
  1178. .kind = "u32",
  1179. .classify = u32_classify,
  1180. .init = u32_init,
  1181. .destroy = u32_destroy,
  1182. .get = u32_get,
  1183. .change = u32_change,
  1184. .delete = u32_delete,
  1185. .walk = u32_walk,
  1186. .reoffload = u32_reoffload,
  1187. .dump = u32_dump,
  1188. .bind_class = u32_bind_class,
  1189. .owner = THIS_MODULE,
  1190. };
  1191. static int __init init_u32(void)
  1192. {
  1193. int i, ret;
  1194. pr_info("u32 classifier\n");
  1195. #ifdef CONFIG_CLS_U32_PERF
  1196. pr_info(" Performance counters on\n");
  1197. #endif
  1198. pr_info(" input device check on\n");
  1199. #ifdef CONFIG_NET_CLS_ACT
  1200. pr_info(" Actions configured\n");
  1201. #endif
  1202. tc_u_common_hash = kvmalloc_array(U32_HASH_SIZE,
  1203. sizeof(struct hlist_head),
  1204. GFP_KERNEL);
  1205. if (!tc_u_common_hash)
  1206. return -ENOMEM;
  1207. for (i = 0; i < U32_HASH_SIZE; i++)
  1208. INIT_HLIST_HEAD(&tc_u_common_hash[i]);
  1209. ret = register_tcf_proto_ops(&cls_u32_ops);
  1210. if (ret)
  1211. kvfree(tc_u_common_hash);
  1212. return ret;
  1213. }
  1214. static void __exit exit_u32(void)
  1215. {
  1216. unregister_tcf_proto_ops(&cls_u32_ops);
  1217. kvfree(tc_u_common_hash);
  1218. }
  1219. module_init(init_u32)
  1220. module_exit(exit_u32)
  1221. MODULE_LICENSE("GPL");