cls_flower.c 92 KB

1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859606162636465666768697071727374757677787980818283848586878889909192939495969798991001011021031041051061071081091101111121131141151161171181191201211221231241251261271281291301311321331341351361371381391401411421431441451461471481491501511521531541551561571581591601611621631641651661671681691701711721731741751761771781791801811821831841851861871881891901911921931941951961971981992002012022032042052062072082092102112122132142152162172182192202212222232242252262272282292302312322332342352362372382392402412422432442452462472482492502512522532542552562572582592602612622632642652662672682692702712722732742752762772782792802812822832842852862872882892902912922932942952962972982993003013023033043053063073083093103113123133143153163173183193203213223233243253263273283293303313323333343353363373383393403413423433443453463473483493503513523533543553563573583593603613623633643653663673683693703713723733743753763773783793803813823833843853863873883893903913923933943953963973983994004014024034044054064074084094104114124134144154164174184194204214224234244254264274284294304314324334344354364374384394404414424434444454464474484494504514524534544554564574584594604614624634644654664674684694704714724734744754764774784794804814824834844854864874884894904914924934944954964974984995005015025035045055065075085095105115125135145155165175185195205215225235245255265275285295305315325335345355365375385395405415425435445455465475485495505515525535545555565575585595605615625635645655665675685695705715725735745755765775785795805815825835845855865875885895905915925935945955965975985996006016026036046056066076086096106116126136146156166176186196206216226236246256266276286296306316326336346356366376386396406416426436446456466476486496506516526536546556566576586596606616626636646656666676686696706716726736746756766776786796806816826836846856866876886896906916926936946956966976986997007017027037047057067077087097107117127137147157167177187197207217227237247257267277287297307317327337347357367377387397407417427437447457467477487497507517527537547557567577587597607617627637647657667677687697707717727737747757767777787797807817827837847857867877887897907917927937947957967977987998008018028038048058068078088098108118128138148158168178188198208218228238248258268278288298308318328338348358368378388398408418428438448458468478488498508518528538548558568578588598608618628638648658668678688698708718728738748758768778788798808818828838848858868878888898908918928938948958968978988999009019029039049059069079089099109119129139149159169179189199209219229239249259269279289299309319329339349359369379389399409419429439449459469479489499509519529539549559569579589599609619629639649659669679689699709719729739749759769779789799809819829839849859869879889899909919929939949959969979989991000100110021003100410051006100710081009101010111012101310141015101610171018101910201021102210231024102510261027102810291030103110321033103410351036103710381039104010411042104310441045104610471048104910501051105210531054105510561057105810591060106110621063106410651066106710681069107010711072107310741075107610771078107910801081108210831084108510861087108810891090109110921093109410951096109710981099110011011102110311041105110611071108110911101111111211131114111511161117111811191120112111221123112411251126112711281129113011311132113311341135113611371138113911401141114211431144114511461147114811491150115111521153115411551156115711581159116011611162116311641165116611671168116911701171117211731174117511761177117811791180118111821183118411851186118711881189119011911192119311941195119611971198119912001201120212031204120512061207120812091210121112121213121412151216121712181219122012211222122312241225122612271228122912301231123212331234123512361237123812391240124112421243124412451246124712481249125012511252125312541255125612571258125912601261126212631264126512661267126812691270127112721273127412751276127712781279128012811282128312841285128612871288128912901291129212931294129512961297129812991300130113021303130413051306130713081309131013111312131313141315131613171318131913201321132213231324132513261327132813291330133113321333133413351336133713381339134013411342134313441345134613471348134913501351135213531354135513561357135813591360136113621363136413651366136713681369137013711372137313741375137613771378137913801381138213831384138513861387138813891390139113921393139413951396139713981399140014011402140314041405140614071408140914101411141214131414141514161417141814191420142114221423142414251426142714281429143014311432143314341435143614371438143914401441144214431444144514461447144814491450145114521453145414551456145714581459146014611462146314641465146614671468146914701471147214731474147514761477147814791480148114821483148414851486148714881489149014911492149314941495149614971498149915001501150215031504150515061507150815091510151115121513151415151516151715181519152015211522152315241525152615271528152915301531153215331534153515361537153815391540154115421543154415451546154715481549155015511552155315541555155615571558155915601561156215631564156515661567156815691570157115721573157415751576157715781579158015811582158315841585158615871588158915901591159215931594159515961597159815991600160116021603160416051606160716081609161016111612161316141615161616171618161916201621162216231624162516261627162816291630163116321633163416351636163716381639164016411642164316441645164616471648164916501651165216531654165516561657165816591660166116621663166416651666166716681669167016711672167316741675167616771678167916801681168216831684168516861687168816891690169116921693169416951696169716981699170017011702170317041705170617071708170917101711171217131714171517161717171817191720172117221723172417251726172717281729173017311732173317341735173617371738173917401741174217431744174517461747174817491750175117521753175417551756175717581759176017611762176317641765176617671768176917701771177217731774177517761777177817791780178117821783178417851786178717881789179017911792179317941795179617971798179918001801180218031804180518061807180818091810181118121813181418151816181718181819182018211822182318241825182618271828182918301831183218331834183518361837183818391840184118421843184418451846184718481849185018511852185318541855185618571858185918601861186218631864186518661867186818691870187118721873187418751876187718781879188018811882188318841885188618871888188918901891189218931894189518961897189818991900190119021903190419051906190719081909191019111912191319141915191619171918191919201921192219231924192519261927192819291930193119321933193419351936193719381939194019411942194319441945194619471948194919501951195219531954195519561957195819591960196119621963196419651966196719681969197019711972197319741975197619771978197919801981198219831984198519861987198819891990199119921993199419951996199719981999200020012002200320042005200620072008200920102011201220132014201520162017201820192020202120222023202420252026202720282029203020312032203320342035203620372038203920402041204220432044204520462047204820492050205120522053205420552056205720582059206020612062206320642065206620672068206920702071207220732074207520762077207820792080208120822083208420852086208720882089209020912092209320942095209620972098209921002101210221032104210521062107210821092110211121122113211421152116211721182119212021212122212321242125212621272128212921302131213221332134213521362137213821392140214121422143214421452146214721482149215021512152215321542155215621572158215921602161216221632164216521662167216821692170217121722173217421752176217721782179218021812182218321842185218621872188218921902191219221932194219521962197219821992200220122022203220422052206220722082209221022112212221322142215221622172218221922202221222222232224222522262227222822292230223122322233223422352236223722382239224022412242224322442245224622472248224922502251225222532254225522562257225822592260226122622263226422652266226722682269227022712272227322742275227622772278227922802281228222832284228522862287228822892290229122922293229422952296229722982299230023012302230323042305230623072308230923102311231223132314231523162317231823192320232123222323232423252326232723282329233023312332233323342335233623372338233923402341234223432344234523462347234823492350235123522353235423552356235723582359236023612362236323642365236623672368236923702371237223732374237523762377237823792380238123822383238423852386238723882389239023912392239323942395239623972398239924002401240224032404240524062407240824092410241124122413241424152416241724182419242024212422242324242425242624272428242924302431243224332434243524362437243824392440244124422443244424452446244724482449245024512452245324542455245624572458245924602461246224632464246524662467246824692470247124722473247424752476247724782479248024812482248324842485248624872488248924902491249224932494249524962497249824992500250125022503250425052506250725082509251025112512251325142515251625172518251925202521252225232524252525262527252825292530253125322533253425352536253725382539254025412542254325442545254625472548254925502551255225532554255525562557255825592560256125622563256425652566256725682569257025712572257325742575257625772578257925802581258225832584258525862587258825892590259125922593259425952596259725982599260026012602260326042605260626072608260926102611261226132614261526162617261826192620262126222623262426252626262726282629263026312632263326342635263626372638263926402641264226432644264526462647264826492650265126522653265426552656265726582659266026612662266326642665266626672668266926702671267226732674267526762677267826792680268126822683268426852686268726882689269026912692269326942695269626972698269927002701270227032704270527062707270827092710271127122713271427152716271727182719272027212722272327242725272627272728272927302731273227332734273527362737273827392740274127422743274427452746274727482749275027512752275327542755275627572758275927602761276227632764276527662767276827692770277127722773277427752776277727782779278027812782278327842785278627872788278927902791279227932794279527962797279827992800280128022803280428052806280728082809281028112812281328142815281628172818281928202821282228232824282528262827282828292830283128322833283428352836283728382839284028412842284328442845284628472848284928502851285228532854285528562857285828592860286128622863286428652866286728682869287028712872287328742875287628772878287928802881288228832884288528862887288828892890289128922893289428952896289728982899290029012902290329042905290629072908290929102911291229132914291529162917291829192920292129222923292429252926292729282929293029312932293329342935293629372938293929402941294229432944294529462947294829492950295129522953295429552956295729582959296029612962296329642965296629672968296929702971297229732974297529762977297829792980298129822983298429852986298729882989299029912992299329942995299629972998299930003001300230033004300530063007300830093010301130123013301430153016301730183019302030213022302330243025302630273028302930303031303230333034303530363037303830393040304130423043304430453046304730483049305030513052305330543055305630573058305930603061306230633064306530663067306830693070307130723073307430753076307730783079308030813082308330843085308630873088308930903091309230933094309530963097309830993100310131023103310431053106310731083109311031113112311331143115311631173118311931203121312231233124312531263127312831293130313131323133313431353136313731383139314031413142314331443145314631473148314931503151315231533154315531563157315831593160316131623163316431653166316731683169317031713172317331743175317631773178317931803181318231833184318531863187318831893190319131923193319431953196319731983199320032013202320332043205320632073208320932103211321232133214321532163217321832193220322132223223322432253226322732283229
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * net/sched/cls_flower.c Flower classifier
  4. *
  5. * Copyright (c) 2015 Jiri Pirko <jiri@resnulli.us>
  6. */
  7. #include <linux/kernel.h>
  8. #include <linux/init.h>
  9. #include <linux/module.h>
  10. #include <linux/rhashtable.h>
  11. #include <linux/workqueue.h>
  12. #include <linux/refcount.h>
  13. #include <linux/if_ether.h>
  14. #include <linux/in6.h>
  15. #include <linux/ip.h>
  16. #include <linux/mpls.h>
  17. #include <net/sch_generic.h>
  18. #include <net/pkt_cls.h>
  19. #include <net/ip.h>
  20. #include <net/flow_dissector.h>
  21. #include <net/geneve.h>
  22. #include <net/vxlan.h>
  23. #include <net/erspan.h>
  24. #include <net/dst.h>
  25. #include <net/dst_metadata.h>
  26. #include <uapi/linux/netfilter/nf_conntrack_common.h>
  27. #define TCA_FLOWER_KEY_CT_FLAGS_MAX \
  28. ((__TCA_FLOWER_KEY_CT_FLAGS_MAX - 1) << 1)
  29. #define TCA_FLOWER_KEY_CT_FLAGS_MASK \
  30. (TCA_FLOWER_KEY_CT_FLAGS_MAX - 1)
  31. struct fl_flow_key {
  32. struct flow_dissector_key_meta meta;
  33. struct flow_dissector_key_control control;
  34. struct flow_dissector_key_control enc_control;
  35. struct flow_dissector_key_basic basic;
  36. struct flow_dissector_key_eth_addrs eth;
  37. struct flow_dissector_key_vlan vlan;
  38. struct flow_dissector_key_vlan cvlan;
  39. union {
  40. struct flow_dissector_key_ipv4_addrs ipv4;
  41. struct flow_dissector_key_ipv6_addrs ipv6;
  42. };
  43. struct flow_dissector_key_ports tp;
  44. struct flow_dissector_key_icmp icmp;
  45. struct flow_dissector_key_arp arp;
  46. struct flow_dissector_key_keyid enc_key_id;
  47. union {
  48. struct flow_dissector_key_ipv4_addrs enc_ipv4;
  49. struct flow_dissector_key_ipv6_addrs enc_ipv6;
  50. };
  51. struct flow_dissector_key_ports enc_tp;
  52. struct flow_dissector_key_mpls mpls;
  53. struct flow_dissector_key_tcp tcp;
  54. struct flow_dissector_key_ip ip;
  55. struct flow_dissector_key_ip enc_ip;
  56. struct flow_dissector_key_enc_opts enc_opts;
  57. union {
  58. struct flow_dissector_key_ports tp;
  59. struct {
  60. struct flow_dissector_key_ports tp_min;
  61. struct flow_dissector_key_ports tp_max;
  62. };
  63. } tp_range;
  64. struct flow_dissector_key_ct ct;
  65. struct flow_dissector_key_hash hash;
  66. } __aligned(BITS_PER_LONG / 8); /* Ensure that we can do comparisons as longs. */
  67. struct fl_flow_mask_range {
  68. unsigned short int start;
  69. unsigned short int end;
  70. };
  71. struct fl_flow_mask {
  72. struct fl_flow_key key;
  73. struct fl_flow_mask_range range;
  74. u32 flags;
  75. struct rhash_head ht_node;
  76. struct rhashtable ht;
  77. struct rhashtable_params filter_ht_params;
  78. struct flow_dissector dissector;
  79. struct list_head filters;
  80. struct rcu_work rwork;
  81. struct list_head list;
  82. refcount_t refcnt;
  83. };
  84. struct fl_flow_tmplt {
  85. struct fl_flow_key dummy_key;
  86. struct fl_flow_key mask;
  87. struct flow_dissector dissector;
  88. struct tcf_chain *chain;
  89. };
  90. struct cls_fl_head {
  91. struct rhashtable ht;
  92. spinlock_t masks_lock; /* Protect masks list */
  93. struct list_head masks;
  94. struct list_head hw_filters;
  95. struct rcu_work rwork;
  96. struct idr handle_idr;
  97. };
  98. struct cls_fl_filter {
  99. struct fl_flow_mask *mask;
  100. struct rhash_head ht_node;
  101. struct fl_flow_key mkey;
  102. struct tcf_exts exts;
  103. struct tcf_result res;
  104. struct fl_flow_key key;
  105. struct list_head list;
  106. struct list_head hw_list;
  107. u32 handle;
  108. u32 flags;
  109. u32 in_hw_count;
  110. struct rcu_work rwork;
  111. struct net_device *hw_dev;
  112. /* Flower classifier is unlocked, which means that its reference counter
  113. * can be changed concurrently without any kind of external
  114. * synchronization. Use atomic reference counter to be concurrency-safe.
  115. */
  116. refcount_t refcnt;
  117. bool deleted;
  118. };
  119. static const struct rhashtable_params mask_ht_params = {
  120. .key_offset = offsetof(struct fl_flow_mask, key),
  121. .key_len = sizeof(struct fl_flow_key),
  122. .head_offset = offsetof(struct fl_flow_mask, ht_node),
  123. .automatic_shrinking = true,
  124. };
  125. static unsigned short int fl_mask_range(const struct fl_flow_mask *mask)
  126. {
  127. return mask->range.end - mask->range.start;
  128. }
  129. static void fl_mask_update_range(struct fl_flow_mask *mask)
  130. {
  131. const u8 *bytes = (const u8 *) &mask->key;
  132. size_t size = sizeof(mask->key);
  133. size_t i, first = 0, last;
  134. for (i = 0; i < size; i++) {
  135. if (bytes[i]) {
  136. first = i;
  137. break;
  138. }
  139. }
  140. last = first;
  141. for (i = size - 1; i != first; i--) {
  142. if (bytes[i]) {
  143. last = i;
  144. break;
  145. }
  146. }
  147. mask->range.start = rounddown(first, sizeof(long));
  148. mask->range.end = roundup(last + 1, sizeof(long));
  149. }
  150. static void *fl_key_get_start(struct fl_flow_key *key,
  151. const struct fl_flow_mask *mask)
  152. {
  153. return (u8 *) key + mask->range.start;
  154. }
  155. static void fl_set_masked_key(struct fl_flow_key *mkey, struct fl_flow_key *key,
  156. struct fl_flow_mask *mask)
  157. {
  158. const long *lkey = fl_key_get_start(key, mask);
  159. const long *lmask = fl_key_get_start(&mask->key, mask);
  160. long *lmkey = fl_key_get_start(mkey, mask);
  161. int i;
  162. for (i = 0; i < fl_mask_range(mask); i += sizeof(long))
  163. *lmkey++ = *lkey++ & *lmask++;
  164. }
  165. static bool fl_mask_fits_tmplt(struct fl_flow_tmplt *tmplt,
  166. struct fl_flow_mask *mask)
  167. {
  168. const long *lmask = fl_key_get_start(&mask->key, mask);
  169. const long *ltmplt;
  170. int i;
  171. if (!tmplt)
  172. return true;
  173. ltmplt = fl_key_get_start(&tmplt->mask, mask);
  174. for (i = 0; i < fl_mask_range(mask); i += sizeof(long)) {
  175. if (~*ltmplt++ & *lmask++)
  176. return false;
  177. }
  178. return true;
  179. }
  180. static void fl_clear_masked_range(struct fl_flow_key *key,
  181. struct fl_flow_mask *mask)
  182. {
  183. memset(fl_key_get_start(key, mask), 0, fl_mask_range(mask));
  184. }
  185. static bool fl_range_port_dst_cmp(struct cls_fl_filter *filter,
  186. struct fl_flow_key *key,
  187. struct fl_flow_key *mkey)
  188. {
  189. u16 min_mask, max_mask, min_val, max_val;
  190. min_mask = ntohs(filter->mask->key.tp_range.tp_min.dst);
  191. max_mask = ntohs(filter->mask->key.tp_range.tp_max.dst);
  192. min_val = ntohs(filter->key.tp_range.tp_min.dst);
  193. max_val = ntohs(filter->key.tp_range.tp_max.dst);
  194. if (min_mask && max_mask) {
  195. if (ntohs(key->tp_range.tp.dst) < min_val ||
  196. ntohs(key->tp_range.tp.dst) > max_val)
  197. return false;
  198. /* skb does not have min and max values */
  199. mkey->tp_range.tp_min.dst = filter->mkey.tp_range.tp_min.dst;
  200. mkey->tp_range.tp_max.dst = filter->mkey.tp_range.tp_max.dst;
  201. }
  202. return true;
  203. }
  204. static bool fl_range_port_src_cmp(struct cls_fl_filter *filter,
  205. struct fl_flow_key *key,
  206. struct fl_flow_key *mkey)
  207. {
  208. u16 min_mask, max_mask, min_val, max_val;
  209. min_mask = ntohs(filter->mask->key.tp_range.tp_min.src);
  210. max_mask = ntohs(filter->mask->key.tp_range.tp_max.src);
  211. min_val = ntohs(filter->key.tp_range.tp_min.src);
  212. max_val = ntohs(filter->key.tp_range.tp_max.src);
  213. if (min_mask && max_mask) {
  214. if (ntohs(key->tp_range.tp.src) < min_val ||
  215. ntohs(key->tp_range.tp.src) > max_val)
  216. return false;
  217. /* skb does not have min and max values */
  218. mkey->tp_range.tp_min.src = filter->mkey.tp_range.tp_min.src;
  219. mkey->tp_range.tp_max.src = filter->mkey.tp_range.tp_max.src;
  220. }
  221. return true;
  222. }
  223. static struct cls_fl_filter *__fl_lookup(struct fl_flow_mask *mask,
  224. struct fl_flow_key *mkey)
  225. {
  226. return rhashtable_lookup_fast(&mask->ht, fl_key_get_start(mkey, mask),
  227. mask->filter_ht_params);
  228. }
  229. static struct cls_fl_filter *fl_lookup_range(struct fl_flow_mask *mask,
  230. struct fl_flow_key *mkey,
  231. struct fl_flow_key *key)
  232. {
  233. struct cls_fl_filter *filter, *f;
  234. list_for_each_entry_rcu(filter, &mask->filters, list) {
  235. if (!fl_range_port_dst_cmp(filter, key, mkey))
  236. continue;
  237. if (!fl_range_port_src_cmp(filter, key, mkey))
  238. continue;
  239. f = __fl_lookup(mask, mkey);
  240. if (f)
  241. return f;
  242. }
  243. return NULL;
  244. }
  245. static noinline_for_stack
  246. struct cls_fl_filter *fl_mask_lookup(struct fl_flow_mask *mask, struct fl_flow_key *key)
  247. {
  248. struct fl_flow_key mkey;
  249. fl_set_masked_key(&mkey, key, mask);
  250. if ((mask->flags & TCA_FLOWER_MASK_FLAGS_RANGE))
  251. return fl_lookup_range(mask, &mkey, key);
  252. return __fl_lookup(mask, &mkey);
  253. }
  254. static u16 fl_ct_info_to_flower_map[] = {
  255. [IP_CT_ESTABLISHED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
  256. TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED,
  257. [IP_CT_RELATED] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
  258. TCA_FLOWER_KEY_CT_FLAGS_RELATED,
  259. [IP_CT_ESTABLISHED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
  260. TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED,
  261. [IP_CT_RELATED_REPLY] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
  262. TCA_FLOWER_KEY_CT_FLAGS_RELATED,
  263. [IP_CT_NEW] = TCA_FLOWER_KEY_CT_FLAGS_TRACKED |
  264. TCA_FLOWER_KEY_CT_FLAGS_NEW,
  265. };
  266. static int fl_classify(struct sk_buff *skb, const struct tcf_proto *tp,
  267. struct tcf_result *res)
  268. {
  269. struct cls_fl_head *head = rcu_dereference_bh(tp->root);
  270. struct fl_flow_key skb_key;
  271. struct fl_flow_mask *mask;
  272. struct cls_fl_filter *f;
  273. list_for_each_entry_rcu(mask, &head->masks, list) {
  274. flow_dissector_init_keys(&skb_key.control, &skb_key.basic);
  275. fl_clear_masked_range(&skb_key, mask);
  276. skb_flow_dissect_meta(skb, &mask->dissector, &skb_key);
  277. /* skb_flow_dissect() does not set n_proto in case an unknown
  278. * protocol, so do it rather here.
  279. */
  280. skb_key.basic.n_proto = skb_protocol(skb, false);
  281. skb_flow_dissect_tunnel_info(skb, &mask->dissector, &skb_key);
  282. skb_flow_dissect_ct(skb, &mask->dissector, &skb_key,
  283. fl_ct_info_to_flower_map,
  284. ARRAY_SIZE(fl_ct_info_to_flower_map));
  285. skb_flow_dissect_hash(skb, &mask->dissector, &skb_key);
  286. skb_flow_dissect(skb, &mask->dissector, &skb_key, 0);
  287. f = fl_mask_lookup(mask, &skb_key);
  288. if (f && !tc_skip_sw(f->flags)) {
  289. *res = f->res;
  290. return tcf_exts_exec(skb, &f->exts, res);
  291. }
  292. }
  293. return -1;
  294. }
  295. static int fl_init(struct tcf_proto *tp)
  296. {
  297. struct cls_fl_head *head;
  298. head = kzalloc(sizeof(*head), GFP_KERNEL);
  299. if (!head)
  300. return -ENOBUFS;
  301. spin_lock_init(&head->masks_lock);
  302. INIT_LIST_HEAD_RCU(&head->masks);
  303. INIT_LIST_HEAD(&head->hw_filters);
  304. rcu_assign_pointer(tp->root, head);
  305. idr_init(&head->handle_idr);
  306. return rhashtable_init(&head->ht, &mask_ht_params);
  307. }
  308. static void fl_mask_free(struct fl_flow_mask *mask, bool mask_init_done)
  309. {
  310. /* temporary masks don't have their filters list and ht initialized */
  311. if (mask_init_done) {
  312. WARN_ON(!list_empty(&mask->filters));
  313. rhashtable_destroy(&mask->ht);
  314. }
  315. kfree(mask);
  316. }
  317. static void fl_mask_free_work(struct work_struct *work)
  318. {
  319. struct fl_flow_mask *mask = container_of(to_rcu_work(work),
  320. struct fl_flow_mask, rwork);
  321. fl_mask_free(mask, true);
  322. }
  323. static void fl_uninit_mask_free_work(struct work_struct *work)
  324. {
  325. struct fl_flow_mask *mask = container_of(to_rcu_work(work),
  326. struct fl_flow_mask, rwork);
  327. fl_mask_free(mask, false);
  328. }
  329. static bool fl_mask_put(struct cls_fl_head *head, struct fl_flow_mask *mask)
  330. {
  331. if (!refcount_dec_and_test(&mask->refcnt))
  332. return false;
  333. rhashtable_remove_fast(&head->ht, &mask->ht_node, mask_ht_params);
  334. spin_lock(&head->masks_lock);
  335. list_del_rcu(&mask->list);
  336. spin_unlock(&head->masks_lock);
  337. tcf_queue_work(&mask->rwork, fl_mask_free_work);
  338. return true;
  339. }
  340. static struct cls_fl_head *fl_head_dereference(struct tcf_proto *tp)
  341. {
  342. /* Flower classifier only changes root pointer during init and destroy.
  343. * Users must obtain reference to tcf_proto instance before calling its
  344. * API, so tp->root pointer is protected from concurrent call to
  345. * fl_destroy() by reference counting.
  346. */
  347. return rcu_dereference_raw(tp->root);
  348. }
  349. static void __fl_destroy_filter(struct cls_fl_filter *f)
  350. {
  351. tcf_exts_destroy(&f->exts);
  352. tcf_exts_put_net(&f->exts);
  353. kfree(f);
  354. }
  355. static void fl_destroy_filter_work(struct work_struct *work)
  356. {
  357. struct cls_fl_filter *f = container_of(to_rcu_work(work),
  358. struct cls_fl_filter, rwork);
  359. __fl_destroy_filter(f);
  360. }
  361. static void fl_hw_destroy_filter(struct tcf_proto *tp, struct cls_fl_filter *f,
  362. bool rtnl_held, struct netlink_ext_ack *extack)
  363. {
  364. struct tcf_block *block = tp->chain->block;
  365. struct flow_cls_offload cls_flower = {};
  366. tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
  367. cls_flower.command = FLOW_CLS_DESTROY;
  368. cls_flower.cookie = (unsigned long) f;
  369. tc_setup_cb_destroy(block, tp, TC_SETUP_CLSFLOWER, &cls_flower, false,
  370. &f->flags, &f->in_hw_count, rtnl_held);
  371. }
  372. static int fl_hw_replace_filter(struct tcf_proto *tp,
  373. struct cls_fl_filter *f, bool rtnl_held,
  374. struct netlink_ext_ack *extack)
  375. {
  376. struct tcf_block *block = tp->chain->block;
  377. struct flow_cls_offload cls_flower = {};
  378. bool skip_sw = tc_skip_sw(f->flags);
  379. int err = 0;
  380. cls_flower.rule = flow_rule_alloc(tcf_exts_num_actions(&f->exts));
  381. if (!cls_flower.rule)
  382. return -ENOMEM;
  383. tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, extack);
  384. cls_flower.command = FLOW_CLS_REPLACE;
  385. cls_flower.cookie = (unsigned long) f;
  386. cls_flower.rule->match.dissector = &f->mask->dissector;
  387. cls_flower.rule->match.mask = &f->mask->key;
  388. cls_flower.rule->match.key = &f->mkey;
  389. cls_flower.classid = f->res.classid;
  390. err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts);
  391. if (err) {
  392. kfree(cls_flower.rule);
  393. if (skip_sw) {
  394. NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action");
  395. return err;
  396. }
  397. return 0;
  398. }
  399. err = tc_setup_cb_add(block, tp, TC_SETUP_CLSFLOWER, &cls_flower,
  400. skip_sw, &f->flags, &f->in_hw_count, rtnl_held);
  401. tc_cleanup_flow_action(&cls_flower.rule->action);
  402. kfree(cls_flower.rule);
  403. if (err) {
  404. fl_hw_destroy_filter(tp, f, rtnl_held, NULL);
  405. return err;
  406. }
  407. if (skip_sw && !(f->flags & TCA_CLS_FLAGS_IN_HW))
  408. return -EINVAL;
  409. return 0;
  410. }
  411. static void fl_hw_update_stats(struct tcf_proto *tp, struct cls_fl_filter *f,
  412. bool rtnl_held)
  413. {
  414. struct tcf_block *block = tp->chain->block;
  415. struct flow_cls_offload cls_flower = {};
  416. tc_cls_common_offload_init(&cls_flower.common, tp, f->flags, NULL);
  417. cls_flower.command = FLOW_CLS_STATS;
  418. cls_flower.cookie = (unsigned long) f;
  419. cls_flower.classid = f->res.classid;
  420. tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false,
  421. rtnl_held);
  422. tcf_exts_stats_update(&f->exts, cls_flower.stats.bytes,
  423. cls_flower.stats.pkts,
  424. cls_flower.stats.drops,
  425. cls_flower.stats.lastused,
  426. cls_flower.stats.used_hw_stats,
  427. cls_flower.stats.used_hw_stats_valid);
  428. }
  429. static void __fl_put(struct cls_fl_filter *f)
  430. {
  431. if (!refcount_dec_and_test(&f->refcnt))
  432. return;
  433. if (tcf_exts_get_net(&f->exts))
  434. tcf_queue_work(&f->rwork, fl_destroy_filter_work);
  435. else
  436. __fl_destroy_filter(f);
  437. }
  438. static struct cls_fl_filter *__fl_get(struct cls_fl_head *head, u32 handle)
  439. {
  440. struct cls_fl_filter *f;
  441. rcu_read_lock();
  442. f = idr_find(&head->handle_idr, handle);
  443. if (f && !refcount_inc_not_zero(&f->refcnt))
  444. f = NULL;
  445. rcu_read_unlock();
  446. return f;
  447. }
  448. static int __fl_delete(struct tcf_proto *tp, struct cls_fl_filter *f,
  449. bool *last, bool rtnl_held,
  450. struct netlink_ext_ack *extack)
  451. {
  452. struct cls_fl_head *head = fl_head_dereference(tp);
  453. *last = false;
  454. spin_lock(&tp->lock);
  455. if (f->deleted) {
  456. spin_unlock(&tp->lock);
  457. return -ENOENT;
  458. }
  459. f->deleted = true;
  460. rhashtable_remove_fast(&f->mask->ht, &f->ht_node,
  461. f->mask->filter_ht_params);
  462. idr_remove(&head->handle_idr, f->handle);
  463. list_del_rcu(&f->list);
  464. spin_unlock(&tp->lock);
  465. *last = fl_mask_put(head, f->mask);
  466. if (!tc_skip_hw(f->flags))
  467. fl_hw_destroy_filter(tp, f, rtnl_held, extack);
  468. tcf_unbind_filter(tp, &f->res);
  469. __fl_put(f);
  470. return 0;
  471. }
  472. static void fl_destroy_sleepable(struct work_struct *work)
  473. {
  474. struct cls_fl_head *head = container_of(to_rcu_work(work),
  475. struct cls_fl_head,
  476. rwork);
  477. rhashtable_destroy(&head->ht);
  478. kfree(head);
  479. module_put(THIS_MODULE);
  480. }
  481. static void fl_destroy(struct tcf_proto *tp, bool rtnl_held,
  482. struct netlink_ext_ack *extack)
  483. {
  484. struct cls_fl_head *head = fl_head_dereference(tp);
  485. struct fl_flow_mask *mask, *next_mask;
  486. struct cls_fl_filter *f, *next;
  487. bool last;
  488. list_for_each_entry_safe(mask, next_mask, &head->masks, list) {
  489. list_for_each_entry_safe(f, next, &mask->filters, list) {
  490. __fl_delete(tp, f, &last, rtnl_held, extack);
  491. if (last)
  492. break;
  493. }
  494. }
  495. idr_destroy(&head->handle_idr);
  496. __module_get(THIS_MODULE);
  497. tcf_queue_work(&head->rwork, fl_destroy_sleepable);
  498. }
  499. static void fl_put(struct tcf_proto *tp, void *arg)
  500. {
  501. struct cls_fl_filter *f = arg;
  502. __fl_put(f);
  503. }
  504. static void *fl_get(struct tcf_proto *tp, u32 handle)
  505. {
  506. struct cls_fl_head *head = fl_head_dereference(tp);
  507. return __fl_get(head, handle);
  508. }
  509. static const struct nla_policy fl_policy[TCA_FLOWER_MAX + 1] = {
  510. [TCA_FLOWER_UNSPEC] = { .type = NLA_UNSPEC },
  511. [TCA_FLOWER_CLASSID] = { .type = NLA_U32 },
  512. [TCA_FLOWER_INDEV] = { .type = NLA_STRING,
  513. .len = IFNAMSIZ },
  514. [TCA_FLOWER_KEY_ETH_DST] = { .len = ETH_ALEN },
  515. [TCA_FLOWER_KEY_ETH_DST_MASK] = { .len = ETH_ALEN },
  516. [TCA_FLOWER_KEY_ETH_SRC] = { .len = ETH_ALEN },
  517. [TCA_FLOWER_KEY_ETH_SRC_MASK] = { .len = ETH_ALEN },
  518. [TCA_FLOWER_KEY_ETH_TYPE] = { .type = NLA_U16 },
  519. [TCA_FLOWER_KEY_IP_PROTO] = { .type = NLA_U8 },
  520. [TCA_FLOWER_KEY_IPV4_SRC] = { .type = NLA_U32 },
  521. [TCA_FLOWER_KEY_IPV4_SRC_MASK] = { .type = NLA_U32 },
  522. [TCA_FLOWER_KEY_IPV4_DST] = { .type = NLA_U32 },
  523. [TCA_FLOWER_KEY_IPV4_DST_MASK] = { .type = NLA_U32 },
  524. [TCA_FLOWER_KEY_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
  525. [TCA_FLOWER_KEY_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
  526. [TCA_FLOWER_KEY_IPV6_DST] = { .len = sizeof(struct in6_addr) },
  527. [TCA_FLOWER_KEY_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
  528. [TCA_FLOWER_KEY_TCP_SRC] = { .type = NLA_U16 },
  529. [TCA_FLOWER_KEY_TCP_DST] = { .type = NLA_U16 },
  530. [TCA_FLOWER_KEY_UDP_SRC] = { .type = NLA_U16 },
  531. [TCA_FLOWER_KEY_UDP_DST] = { .type = NLA_U16 },
  532. [TCA_FLOWER_KEY_VLAN_ID] = { .type = NLA_U16 },
  533. [TCA_FLOWER_KEY_VLAN_PRIO] = { .type = NLA_U8 },
  534. [TCA_FLOWER_KEY_VLAN_ETH_TYPE] = { .type = NLA_U16 },
  535. [TCA_FLOWER_KEY_ENC_KEY_ID] = { .type = NLA_U32 },
  536. [TCA_FLOWER_KEY_ENC_IPV4_SRC] = { .type = NLA_U32 },
  537. [TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK] = { .type = NLA_U32 },
  538. [TCA_FLOWER_KEY_ENC_IPV4_DST] = { .type = NLA_U32 },
  539. [TCA_FLOWER_KEY_ENC_IPV4_DST_MASK] = { .type = NLA_U32 },
  540. [TCA_FLOWER_KEY_ENC_IPV6_SRC] = { .len = sizeof(struct in6_addr) },
  541. [TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK] = { .len = sizeof(struct in6_addr) },
  542. [TCA_FLOWER_KEY_ENC_IPV6_DST] = { .len = sizeof(struct in6_addr) },
  543. [TCA_FLOWER_KEY_ENC_IPV6_DST_MASK] = { .len = sizeof(struct in6_addr) },
  544. [TCA_FLOWER_KEY_TCP_SRC_MASK] = { .type = NLA_U16 },
  545. [TCA_FLOWER_KEY_TCP_DST_MASK] = { .type = NLA_U16 },
  546. [TCA_FLOWER_KEY_UDP_SRC_MASK] = { .type = NLA_U16 },
  547. [TCA_FLOWER_KEY_UDP_DST_MASK] = { .type = NLA_U16 },
  548. [TCA_FLOWER_KEY_SCTP_SRC_MASK] = { .type = NLA_U16 },
  549. [TCA_FLOWER_KEY_SCTP_DST_MASK] = { .type = NLA_U16 },
  550. [TCA_FLOWER_KEY_SCTP_SRC] = { .type = NLA_U16 },
  551. [TCA_FLOWER_KEY_SCTP_DST] = { .type = NLA_U16 },
  552. [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT] = { .type = NLA_U16 },
  553. [TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK] = { .type = NLA_U16 },
  554. [TCA_FLOWER_KEY_ENC_UDP_DST_PORT] = { .type = NLA_U16 },
  555. [TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK] = { .type = NLA_U16 },
  556. [TCA_FLOWER_KEY_FLAGS] = { .type = NLA_U32 },
  557. [TCA_FLOWER_KEY_FLAGS_MASK] = { .type = NLA_U32 },
  558. [TCA_FLOWER_KEY_ICMPV4_TYPE] = { .type = NLA_U8 },
  559. [TCA_FLOWER_KEY_ICMPV4_TYPE_MASK] = { .type = NLA_U8 },
  560. [TCA_FLOWER_KEY_ICMPV4_CODE] = { .type = NLA_U8 },
  561. [TCA_FLOWER_KEY_ICMPV4_CODE_MASK] = { .type = NLA_U8 },
  562. [TCA_FLOWER_KEY_ICMPV6_TYPE] = { .type = NLA_U8 },
  563. [TCA_FLOWER_KEY_ICMPV6_TYPE_MASK] = { .type = NLA_U8 },
  564. [TCA_FLOWER_KEY_ICMPV6_CODE] = { .type = NLA_U8 },
  565. [TCA_FLOWER_KEY_ICMPV6_CODE_MASK] = { .type = NLA_U8 },
  566. [TCA_FLOWER_KEY_ARP_SIP] = { .type = NLA_U32 },
  567. [TCA_FLOWER_KEY_ARP_SIP_MASK] = { .type = NLA_U32 },
  568. [TCA_FLOWER_KEY_ARP_TIP] = { .type = NLA_U32 },
  569. [TCA_FLOWER_KEY_ARP_TIP_MASK] = { .type = NLA_U32 },
  570. [TCA_FLOWER_KEY_ARP_OP] = { .type = NLA_U8 },
  571. [TCA_FLOWER_KEY_ARP_OP_MASK] = { .type = NLA_U8 },
  572. [TCA_FLOWER_KEY_ARP_SHA] = { .len = ETH_ALEN },
  573. [TCA_FLOWER_KEY_ARP_SHA_MASK] = { .len = ETH_ALEN },
  574. [TCA_FLOWER_KEY_ARP_THA] = { .len = ETH_ALEN },
  575. [TCA_FLOWER_KEY_ARP_THA_MASK] = { .len = ETH_ALEN },
  576. [TCA_FLOWER_KEY_MPLS_TTL] = { .type = NLA_U8 },
  577. [TCA_FLOWER_KEY_MPLS_BOS] = { .type = NLA_U8 },
  578. [TCA_FLOWER_KEY_MPLS_TC] = { .type = NLA_U8 },
  579. [TCA_FLOWER_KEY_MPLS_LABEL] = { .type = NLA_U32 },
  580. [TCA_FLOWER_KEY_MPLS_OPTS] = { .type = NLA_NESTED },
  581. [TCA_FLOWER_KEY_TCP_FLAGS] = { .type = NLA_U16 },
  582. [TCA_FLOWER_KEY_TCP_FLAGS_MASK] = { .type = NLA_U16 },
  583. [TCA_FLOWER_KEY_IP_TOS] = { .type = NLA_U8 },
  584. [TCA_FLOWER_KEY_IP_TOS_MASK] = { .type = NLA_U8 },
  585. [TCA_FLOWER_KEY_IP_TTL] = { .type = NLA_U8 },
  586. [TCA_FLOWER_KEY_IP_TTL_MASK] = { .type = NLA_U8 },
  587. [TCA_FLOWER_KEY_CVLAN_ID] = { .type = NLA_U16 },
  588. [TCA_FLOWER_KEY_CVLAN_PRIO] = { .type = NLA_U8 },
  589. [TCA_FLOWER_KEY_CVLAN_ETH_TYPE] = { .type = NLA_U16 },
  590. [TCA_FLOWER_KEY_ENC_IP_TOS] = { .type = NLA_U8 },
  591. [TCA_FLOWER_KEY_ENC_IP_TOS_MASK] = { .type = NLA_U8 },
  592. [TCA_FLOWER_KEY_ENC_IP_TTL] = { .type = NLA_U8 },
  593. [TCA_FLOWER_KEY_ENC_IP_TTL_MASK] = { .type = NLA_U8 },
  594. [TCA_FLOWER_KEY_ENC_OPTS] = { .type = NLA_NESTED },
  595. [TCA_FLOWER_KEY_ENC_OPTS_MASK] = { .type = NLA_NESTED },
  596. [TCA_FLOWER_KEY_CT_STATE] =
  597. NLA_POLICY_MASK(NLA_U16, TCA_FLOWER_KEY_CT_FLAGS_MASK),
  598. [TCA_FLOWER_KEY_CT_STATE_MASK] =
  599. NLA_POLICY_MASK(NLA_U16, TCA_FLOWER_KEY_CT_FLAGS_MASK),
  600. [TCA_FLOWER_KEY_CT_ZONE] = { .type = NLA_U16 },
  601. [TCA_FLOWER_KEY_CT_ZONE_MASK] = { .type = NLA_U16 },
  602. [TCA_FLOWER_KEY_CT_MARK] = { .type = NLA_U32 },
  603. [TCA_FLOWER_KEY_CT_MARK_MASK] = { .type = NLA_U32 },
  604. [TCA_FLOWER_KEY_CT_LABELS] = { .type = NLA_BINARY,
  605. .len = 128 / BITS_PER_BYTE },
  606. [TCA_FLOWER_KEY_CT_LABELS_MASK] = { .type = NLA_BINARY,
  607. .len = 128 / BITS_PER_BYTE },
  608. [TCA_FLOWER_FLAGS] = { .type = NLA_U32 },
  609. [TCA_FLOWER_KEY_HASH] = { .type = NLA_U32 },
  610. [TCA_FLOWER_KEY_HASH_MASK] = { .type = NLA_U32 },
  611. };
  612. static const struct nla_policy
  613. enc_opts_policy[TCA_FLOWER_KEY_ENC_OPTS_MAX + 1] = {
  614. [TCA_FLOWER_KEY_ENC_OPTS_UNSPEC] = {
  615. .strict_start_type = TCA_FLOWER_KEY_ENC_OPTS_VXLAN },
  616. [TCA_FLOWER_KEY_ENC_OPTS_GENEVE] = { .type = NLA_NESTED },
  617. [TCA_FLOWER_KEY_ENC_OPTS_VXLAN] = { .type = NLA_NESTED },
  618. [TCA_FLOWER_KEY_ENC_OPTS_ERSPAN] = { .type = NLA_NESTED },
  619. };
  620. static const struct nla_policy
  621. geneve_opt_policy[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1] = {
  622. [TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] = { .type = NLA_U16 },
  623. [TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] = { .type = NLA_U8 },
  624. [TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA] = { .type = NLA_BINARY,
  625. .len = 128 },
  626. };
  627. static const struct nla_policy
  628. vxlan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1] = {
  629. [TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP] = { .type = NLA_U32 },
  630. };
  631. static const struct nla_policy
  632. erspan_opt_policy[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1] = {
  633. [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER] = { .type = NLA_U8 },
  634. [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX] = { .type = NLA_U32 },
  635. [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] = { .type = NLA_U8 },
  636. [TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID] = { .type = NLA_U8 },
  637. };
  638. static const struct nla_policy
  639. mpls_stack_entry_policy[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1] = {
  640. [TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH] = { .type = NLA_U8 },
  641. [TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL] = { .type = NLA_U8 },
  642. [TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS] = { .type = NLA_U8 },
  643. [TCA_FLOWER_KEY_MPLS_OPT_LSE_TC] = { .type = NLA_U8 },
  644. [TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL] = { .type = NLA_U32 },
  645. };
  646. static void fl_set_key_val(struct nlattr **tb,
  647. void *val, int val_type,
  648. void *mask, int mask_type, int len)
  649. {
  650. if (!tb[val_type])
  651. return;
  652. nla_memcpy(val, tb[val_type], len);
  653. if (mask_type == TCA_FLOWER_UNSPEC || !tb[mask_type])
  654. memset(mask, 0xff, len);
  655. else
  656. nla_memcpy(mask, tb[mask_type], len);
  657. }
  658. static int fl_set_key_port_range(struct nlattr **tb, struct fl_flow_key *key,
  659. struct fl_flow_key *mask,
  660. struct netlink_ext_ack *extack)
  661. {
  662. fl_set_key_val(tb, &key->tp_range.tp_min.dst,
  663. TCA_FLOWER_KEY_PORT_DST_MIN, &mask->tp_range.tp_min.dst,
  664. TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.dst));
  665. fl_set_key_val(tb, &key->tp_range.tp_max.dst,
  666. TCA_FLOWER_KEY_PORT_DST_MAX, &mask->tp_range.tp_max.dst,
  667. TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.dst));
  668. fl_set_key_val(tb, &key->tp_range.tp_min.src,
  669. TCA_FLOWER_KEY_PORT_SRC_MIN, &mask->tp_range.tp_min.src,
  670. TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_min.src));
  671. fl_set_key_val(tb, &key->tp_range.tp_max.src,
  672. TCA_FLOWER_KEY_PORT_SRC_MAX, &mask->tp_range.tp_max.src,
  673. TCA_FLOWER_UNSPEC, sizeof(key->tp_range.tp_max.src));
  674. if (mask->tp_range.tp_min.dst && mask->tp_range.tp_max.dst &&
  675. ntohs(key->tp_range.tp_max.dst) <=
  676. ntohs(key->tp_range.tp_min.dst)) {
  677. NL_SET_ERR_MSG_ATTR(extack,
  678. tb[TCA_FLOWER_KEY_PORT_DST_MIN],
  679. "Invalid destination port range (min must be strictly smaller than max)");
  680. return -EINVAL;
  681. }
  682. if (mask->tp_range.tp_min.src && mask->tp_range.tp_max.src &&
  683. ntohs(key->tp_range.tp_max.src) <=
  684. ntohs(key->tp_range.tp_min.src)) {
  685. NL_SET_ERR_MSG_ATTR(extack,
  686. tb[TCA_FLOWER_KEY_PORT_SRC_MIN],
  687. "Invalid source port range (min must be strictly smaller than max)");
  688. return -EINVAL;
  689. }
  690. return 0;
  691. }
  692. static int fl_set_key_mpls_lse(const struct nlattr *nla_lse,
  693. struct flow_dissector_key_mpls *key_val,
  694. struct flow_dissector_key_mpls *key_mask,
  695. struct netlink_ext_ack *extack)
  696. {
  697. struct nlattr *tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX + 1];
  698. struct flow_dissector_mpls_lse *lse_mask;
  699. struct flow_dissector_mpls_lse *lse_val;
  700. u8 lse_index;
  701. u8 depth;
  702. int err;
  703. err = nla_parse_nested(tb, TCA_FLOWER_KEY_MPLS_OPT_LSE_MAX, nla_lse,
  704. mpls_stack_entry_policy, extack);
  705. if (err < 0)
  706. return err;
  707. if (!tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]) {
  708. NL_SET_ERR_MSG(extack, "Missing MPLS option \"depth\"");
  709. return -EINVAL;
  710. }
  711. depth = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH]);
  712. /* LSE depth starts at 1, for consistency with terminology used by
  713. * RFC 3031 (section 3.9), where depth 0 refers to unlabeled packets.
  714. */
  715. if (depth < 1 || depth > FLOW_DIS_MPLS_MAX) {
  716. NL_SET_ERR_MSG_ATTR(extack,
  717. tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH],
  718. "Invalid MPLS depth");
  719. return -EINVAL;
  720. }
  721. lse_index = depth - 1;
  722. dissector_set_mpls_lse(key_val, lse_index);
  723. dissector_set_mpls_lse(key_mask, lse_index);
  724. lse_val = &key_val->ls[lse_index];
  725. lse_mask = &key_mask->ls[lse_index];
  726. if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]) {
  727. lse_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL]);
  728. lse_mask->mpls_ttl = MPLS_TTL_MASK;
  729. }
  730. if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]) {
  731. u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS]);
  732. if (bos & ~MPLS_BOS_MASK) {
  733. NL_SET_ERR_MSG_ATTR(extack,
  734. tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS],
  735. "Bottom Of Stack (BOS) must be 0 or 1");
  736. return -EINVAL;
  737. }
  738. lse_val->mpls_bos = bos;
  739. lse_mask->mpls_bos = MPLS_BOS_MASK;
  740. }
  741. if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]) {
  742. u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC]);
  743. if (tc & ~MPLS_TC_MASK) {
  744. NL_SET_ERR_MSG_ATTR(extack,
  745. tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_TC],
  746. "Traffic Class (TC) must be between 0 and 7");
  747. return -EINVAL;
  748. }
  749. lse_val->mpls_tc = tc;
  750. lse_mask->mpls_tc = MPLS_TC_MASK;
  751. }
  752. if (tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]) {
  753. u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL]);
  754. if (label & ~MPLS_LABEL_MASK) {
  755. NL_SET_ERR_MSG_ATTR(extack,
  756. tb[TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL],
  757. "Label must be between 0 and 1048575");
  758. return -EINVAL;
  759. }
  760. lse_val->mpls_label = label;
  761. lse_mask->mpls_label = MPLS_LABEL_MASK;
  762. }
  763. return 0;
  764. }
  765. static int fl_set_key_mpls_opts(const struct nlattr *nla_mpls_opts,
  766. struct flow_dissector_key_mpls *key_val,
  767. struct flow_dissector_key_mpls *key_mask,
  768. struct netlink_ext_ack *extack)
  769. {
  770. struct nlattr *nla_lse;
  771. int rem;
  772. int err;
  773. if (!(nla_mpls_opts->nla_type & NLA_F_NESTED)) {
  774. NL_SET_ERR_MSG_ATTR(extack, nla_mpls_opts,
  775. "NLA_F_NESTED is missing");
  776. return -EINVAL;
  777. }
  778. nla_for_each_nested(nla_lse, nla_mpls_opts, rem) {
  779. if (nla_type(nla_lse) != TCA_FLOWER_KEY_MPLS_OPTS_LSE) {
  780. NL_SET_ERR_MSG_ATTR(extack, nla_lse,
  781. "Invalid MPLS option type");
  782. return -EINVAL;
  783. }
  784. err = fl_set_key_mpls_lse(nla_lse, key_val, key_mask, extack);
  785. if (err < 0)
  786. return err;
  787. }
  788. if (rem) {
  789. NL_SET_ERR_MSG(extack,
  790. "Bytes leftover after parsing MPLS options");
  791. return -EINVAL;
  792. }
  793. return 0;
  794. }
  795. static int fl_set_key_mpls(struct nlattr **tb,
  796. struct flow_dissector_key_mpls *key_val,
  797. struct flow_dissector_key_mpls *key_mask,
  798. struct netlink_ext_ack *extack)
  799. {
  800. struct flow_dissector_mpls_lse *lse_mask;
  801. struct flow_dissector_mpls_lse *lse_val;
  802. if (tb[TCA_FLOWER_KEY_MPLS_OPTS]) {
  803. if (tb[TCA_FLOWER_KEY_MPLS_TTL] ||
  804. tb[TCA_FLOWER_KEY_MPLS_BOS] ||
  805. tb[TCA_FLOWER_KEY_MPLS_TC] ||
  806. tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
  807. NL_SET_ERR_MSG_ATTR(extack,
  808. tb[TCA_FLOWER_KEY_MPLS_OPTS],
  809. "MPLS label, Traffic Class, Bottom Of Stack and Time To Live must be encapsulated in the MPLS options attribute");
  810. return -EBADMSG;
  811. }
  812. return fl_set_key_mpls_opts(tb[TCA_FLOWER_KEY_MPLS_OPTS],
  813. key_val, key_mask, extack);
  814. }
  815. lse_val = &key_val->ls[0];
  816. lse_mask = &key_mask->ls[0];
  817. if (tb[TCA_FLOWER_KEY_MPLS_TTL]) {
  818. lse_val->mpls_ttl = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TTL]);
  819. lse_mask->mpls_ttl = MPLS_TTL_MASK;
  820. dissector_set_mpls_lse(key_val, 0);
  821. dissector_set_mpls_lse(key_mask, 0);
  822. }
  823. if (tb[TCA_FLOWER_KEY_MPLS_BOS]) {
  824. u8 bos = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_BOS]);
  825. if (bos & ~MPLS_BOS_MASK) {
  826. NL_SET_ERR_MSG_ATTR(extack,
  827. tb[TCA_FLOWER_KEY_MPLS_BOS],
  828. "Bottom Of Stack (BOS) must be 0 or 1");
  829. return -EINVAL;
  830. }
  831. lse_val->mpls_bos = bos;
  832. lse_mask->mpls_bos = MPLS_BOS_MASK;
  833. dissector_set_mpls_lse(key_val, 0);
  834. dissector_set_mpls_lse(key_mask, 0);
  835. }
  836. if (tb[TCA_FLOWER_KEY_MPLS_TC]) {
  837. u8 tc = nla_get_u8(tb[TCA_FLOWER_KEY_MPLS_TC]);
  838. if (tc & ~MPLS_TC_MASK) {
  839. NL_SET_ERR_MSG_ATTR(extack,
  840. tb[TCA_FLOWER_KEY_MPLS_TC],
  841. "Traffic Class (TC) must be between 0 and 7");
  842. return -EINVAL;
  843. }
  844. lse_val->mpls_tc = tc;
  845. lse_mask->mpls_tc = MPLS_TC_MASK;
  846. dissector_set_mpls_lse(key_val, 0);
  847. dissector_set_mpls_lse(key_mask, 0);
  848. }
  849. if (tb[TCA_FLOWER_KEY_MPLS_LABEL]) {
  850. u32 label = nla_get_u32(tb[TCA_FLOWER_KEY_MPLS_LABEL]);
  851. if (label & ~MPLS_LABEL_MASK) {
  852. NL_SET_ERR_MSG_ATTR(extack,
  853. tb[TCA_FLOWER_KEY_MPLS_LABEL],
  854. "Label must be between 0 and 1048575");
  855. return -EINVAL;
  856. }
  857. lse_val->mpls_label = label;
  858. lse_mask->mpls_label = MPLS_LABEL_MASK;
  859. dissector_set_mpls_lse(key_val, 0);
  860. dissector_set_mpls_lse(key_mask, 0);
  861. }
  862. return 0;
  863. }
  864. static void fl_set_key_vlan(struct nlattr **tb,
  865. __be16 ethertype,
  866. int vlan_id_key, int vlan_prio_key,
  867. int vlan_next_eth_type_key,
  868. struct flow_dissector_key_vlan *key_val,
  869. struct flow_dissector_key_vlan *key_mask)
  870. {
  871. #define VLAN_PRIORITY_MASK 0x7
  872. if (tb[vlan_id_key]) {
  873. key_val->vlan_id =
  874. nla_get_u16(tb[vlan_id_key]) & VLAN_VID_MASK;
  875. key_mask->vlan_id = VLAN_VID_MASK;
  876. }
  877. if (tb[vlan_prio_key]) {
  878. key_val->vlan_priority =
  879. nla_get_u8(tb[vlan_prio_key]) &
  880. VLAN_PRIORITY_MASK;
  881. key_mask->vlan_priority = VLAN_PRIORITY_MASK;
  882. }
  883. key_val->vlan_tpid = ethertype;
  884. key_mask->vlan_tpid = cpu_to_be16(~0);
  885. if (tb[vlan_next_eth_type_key]) {
  886. key_val->vlan_eth_type =
  887. nla_get_be16(tb[vlan_next_eth_type_key]);
  888. key_mask->vlan_eth_type = cpu_to_be16(~0);
  889. }
  890. }
  891. static void fl_set_key_flag(u32 flower_key, u32 flower_mask,
  892. u32 *dissector_key, u32 *dissector_mask,
  893. u32 flower_flag_bit, u32 dissector_flag_bit)
  894. {
  895. if (flower_mask & flower_flag_bit) {
  896. *dissector_mask |= dissector_flag_bit;
  897. if (flower_key & flower_flag_bit)
  898. *dissector_key |= dissector_flag_bit;
  899. }
  900. }
  901. static int fl_set_key_flags(struct nlattr **tb, u32 *flags_key,
  902. u32 *flags_mask, struct netlink_ext_ack *extack)
  903. {
  904. u32 key, mask;
  905. /* mask is mandatory for flags */
  906. if (!tb[TCA_FLOWER_KEY_FLAGS_MASK]) {
  907. NL_SET_ERR_MSG(extack, "Missing flags mask");
  908. return -EINVAL;
  909. }
  910. key = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS]));
  911. mask = be32_to_cpu(nla_get_u32(tb[TCA_FLOWER_KEY_FLAGS_MASK]));
  912. *flags_key = 0;
  913. *flags_mask = 0;
  914. fl_set_key_flag(key, mask, flags_key, flags_mask,
  915. TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
  916. fl_set_key_flag(key, mask, flags_key, flags_mask,
  917. TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
  918. FLOW_DIS_FIRST_FRAG);
  919. return 0;
  920. }
  921. static void fl_set_key_ip(struct nlattr **tb, bool encap,
  922. struct flow_dissector_key_ip *key,
  923. struct flow_dissector_key_ip *mask)
  924. {
  925. int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
  926. int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
  927. int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
  928. int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
  929. fl_set_key_val(tb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos));
  930. fl_set_key_val(tb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl));
  931. }
  932. static int fl_set_geneve_opt(const struct nlattr *nla, struct fl_flow_key *key,
  933. int depth, int option_len,
  934. struct netlink_ext_ack *extack)
  935. {
  936. struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX + 1];
  937. struct nlattr *class = NULL, *type = NULL, *data = NULL;
  938. struct geneve_opt *opt;
  939. int err, data_len = 0;
  940. if (option_len > sizeof(struct geneve_opt))
  941. data_len = option_len - sizeof(struct geneve_opt);
  942. opt = (struct geneve_opt *)&key->enc_opts.data[key->enc_opts.len];
  943. memset(opt, 0xff, option_len);
  944. opt->length = data_len / 4;
  945. opt->r1 = 0;
  946. opt->r2 = 0;
  947. opt->r3 = 0;
  948. /* If no mask has been prodived we assume an exact match. */
  949. if (!depth)
  950. return sizeof(struct geneve_opt) + data_len;
  951. if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_GENEVE) {
  952. NL_SET_ERR_MSG(extack, "Non-geneve option type for mask");
  953. return -EINVAL;
  954. }
  955. err = nla_parse_nested_deprecated(tb,
  956. TCA_FLOWER_KEY_ENC_OPT_GENEVE_MAX,
  957. nla, geneve_opt_policy, extack);
  958. if (err < 0)
  959. return err;
  960. /* We are not allowed to omit any of CLASS, TYPE or DATA
  961. * fields from the key.
  962. */
  963. if (!option_len &&
  964. (!tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS] ||
  965. !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE] ||
  966. !tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA])) {
  967. NL_SET_ERR_MSG(extack, "Missing tunnel key geneve option class, type or data");
  968. return -EINVAL;
  969. }
  970. /* Omitting any of CLASS, TYPE or DATA fields is allowed
  971. * for the mask.
  972. */
  973. if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA]) {
  974. int new_len = key->enc_opts.len;
  975. data = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA];
  976. data_len = nla_len(data);
  977. if (data_len < 4) {
  978. NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is less than 4 bytes long");
  979. return -ERANGE;
  980. }
  981. if (data_len % 4) {
  982. NL_SET_ERR_MSG(extack, "Tunnel key geneve option data is not a multiple of 4 bytes long");
  983. return -ERANGE;
  984. }
  985. new_len += sizeof(struct geneve_opt) + data_len;
  986. BUILD_BUG_ON(FLOW_DIS_TUN_OPTS_MAX != IP_TUNNEL_OPTS_MAX);
  987. if (new_len > FLOW_DIS_TUN_OPTS_MAX) {
  988. NL_SET_ERR_MSG(extack, "Tunnel options exceeds max size");
  989. return -ERANGE;
  990. }
  991. opt->length = data_len / 4;
  992. memcpy(opt->opt_data, nla_data(data), data_len);
  993. }
  994. if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS]) {
  995. class = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS];
  996. opt->opt_class = nla_get_be16(class);
  997. }
  998. if (tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE]) {
  999. type = tb[TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE];
  1000. opt->type = nla_get_u8(type);
  1001. }
  1002. return sizeof(struct geneve_opt) + data_len;
  1003. }
  1004. static int fl_set_vxlan_opt(const struct nlattr *nla, struct fl_flow_key *key,
  1005. int depth, int option_len,
  1006. struct netlink_ext_ack *extack)
  1007. {
  1008. struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX + 1];
  1009. struct vxlan_metadata *md;
  1010. int err;
  1011. md = (struct vxlan_metadata *)&key->enc_opts.data[key->enc_opts.len];
  1012. memset(md, 0xff, sizeof(*md));
  1013. if (!depth)
  1014. return sizeof(*md);
  1015. if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_VXLAN) {
  1016. NL_SET_ERR_MSG(extack, "Non-vxlan option type for mask");
  1017. return -EINVAL;
  1018. }
  1019. err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_MAX, nla,
  1020. vxlan_opt_policy, extack);
  1021. if (err < 0)
  1022. return err;
  1023. if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) {
  1024. NL_SET_ERR_MSG(extack, "Missing tunnel key vxlan option gbp");
  1025. return -EINVAL;
  1026. }
  1027. if (tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]) {
  1028. md->gbp = nla_get_u32(tb[TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP]);
  1029. md->gbp &= VXLAN_GBP_MASK;
  1030. }
  1031. return sizeof(*md);
  1032. }
  1033. static int fl_set_erspan_opt(const struct nlattr *nla, struct fl_flow_key *key,
  1034. int depth, int option_len,
  1035. struct netlink_ext_ack *extack)
  1036. {
  1037. struct nlattr *tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX + 1];
  1038. struct erspan_metadata *md;
  1039. int err;
  1040. md = (struct erspan_metadata *)&key->enc_opts.data[key->enc_opts.len];
  1041. memset(md, 0xff, sizeof(*md));
  1042. md->version = 1;
  1043. if (!depth)
  1044. return sizeof(*md);
  1045. if (nla_type(nla) != TCA_FLOWER_KEY_ENC_OPTS_ERSPAN) {
  1046. NL_SET_ERR_MSG(extack, "Non-erspan option type for mask");
  1047. return -EINVAL;
  1048. }
  1049. err = nla_parse_nested(tb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_MAX, nla,
  1050. erspan_opt_policy, extack);
  1051. if (err < 0)
  1052. return err;
  1053. if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]) {
  1054. NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option ver");
  1055. return -EINVAL;
  1056. }
  1057. if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER])
  1058. md->version = nla_get_u8(tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER]);
  1059. if (md->version == 1) {
  1060. if (!option_len && !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) {
  1061. NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option index");
  1062. return -EINVAL;
  1063. }
  1064. if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX]) {
  1065. nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX];
  1066. memset(&md->u, 0x00, sizeof(md->u));
  1067. md->u.index = nla_get_be32(nla);
  1068. }
  1069. } else if (md->version == 2) {
  1070. if (!option_len && (!tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR] ||
  1071. !tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID])) {
  1072. NL_SET_ERR_MSG(extack, "Missing tunnel key erspan option dir or hwid");
  1073. return -EINVAL;
  1074. }
  1075. if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR]) {
  1076. nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR];
  1077. md->u.md2.dir = nla_get_u8(nla);
  1078. }
  1079. if (tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID]) {
  1080. nla = tb[TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID];
  1081. set_hwid(&md->u.md2, nla_get_u8(nla));
  1082. }
  1083. } else {
  1084. NL_SET_ERR_MSG(extack, "Tunnel key erspan option ver is incorrect");
  1085. return -EINVAL;
  1086. }
  1087. return sizeof(*md);
  1088. }
  1089. static int fl_set_enc_opt(struct nlattr **tb, struct fl_flow_key *key,
  1090. struct fl_flow_key *mask,
  1091. struct netlink_ext_ack *extack)
  1092. {
  1093. const struct nlattr *nla_enc_key, *nla_opt_key, *nla_opt_msk = NULL;
  1094. int err, option_len, key_depth, msk_depth = 0;
  1095. err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS],
  1096. TCA_FLOWER_KEY_ENC_OPTS_MAX,
  1097. enc_opts_policy, extack);
  1098. if (err)
  1099. return err;
  1100. nla_enc_key = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS]);
  1101. if (tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]) {
  1102. err = nla_validate_nested_deprecated(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK],
  1103. TCA_FLOWER_KEY_ENC_OPTS_MAX,
  1104. enc_opts_policy, extack);
  1105. if (err)
  1106. return err;
  1107. nla_opt_msk = nla_data(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
  1108. msk_depth = nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS_MASK]);
  1109. if (!nla_ok(nla_opt_msk, msk_depth)) {
  1110. NL_SET_ERR_MSG(extack, "Invalid nested attribute for masks");
  1111. return -EINVAL;
  1112. }
  1113. }
  1114. nla_for_each_attr(nla_opt_key, nla_enc_key,
  1115. nla_len(tb[TCA_FLOWER_KEY_ENC_OPTS]), key_depth) {
  1116. switch (nla_type(nla_opt_key)) {
  1117. case TCA_FLOWER_KEY_ENC_OPTS_GENEVE:
  1118. if (key->enc_opts.dst_opt_type &&
  1119. key->enc_opts.dst_opt_type != TUNNEL_GENEVE_OPT) {
  1120. NL_SET_ERR_MSG(extack, "Duplicate type for geneve options");
  1121. return -EINVAL;
  1122. }
  1123. option_len = 0;
  1124. key->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
  1125. option_len = fl_set_geneve_opt(nla_opt_key, key,
  1126. key_depth, option_len,
  1127. extack);
  1128. if (option_len < 0)
  1129. return option_len;
  1130. key->enc_opts.len += option_len;
  1131. /* At the same time we need to parse through the mask
  1132. * in order to verify exact and mask attribute lengths.
  1133. */
  1134. mask->enc_opts.dst_opt_type = TUNNEL_GENEVE_OPT;
  1135. option_len = fl_set_geneve_opt(nla_opt_msk, mask,
  1136. msk_depth, option_len,
  1137. extack);
  1138. if (option_len < 0)
  1139. return option_len;
  1140. mask->enc_opts.len += option_len;
  1141. if (key->enc_opts.len != mask->enc_opts.len) {
  1142. NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
  1143. return -EINVAL;
  1144. }
  1145. break;
  1146. case TCA_FLOWER_KEY_ENC_OPTS_VXLAN:
  1147. if (key->enc_opts.dst_opt_type) {
  1148. NL_SET_ERR_MSG(extack, "Duplicate type for vxlan options");
  1149. return -EINVAL;
  1150. }
  1151. option_len = 0;
  1152. key->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT;
  1153. option_len = fl_set_vxlan_opt(nla_opt_key, key,
  1154. key_depth, option_len,
  1155. extack);
  1156. if (option_len < 0)
  1157. return option_len;
  1158. key->enc_opts.len += option_len;
  1159. /* At the same time we need to parse through the mask
  1160. * in order to verify exact and mask attribute lengths.
  1161. */
  1162. mask->enc_opts.dst_opt_type = TUNNEL_VXLAN_OPT;
  1163. option_len = fl_set_vxlan_opt(nla_opt_msk, mask,
  1164. msk_depth, option_len,
  1165. extack);
  1166. if (option_len < 0)
  1167. return option_len;
  1168. mask->enc_opts.len += option_len;
  1169. if (key->enc_opts.len != mask->enc_opts.len) {
  1170. NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
  1171. return -EINVAL;
  1172. }
  1173. break;
  1174. case TCA_FLOWER_KEY_ENC_OPTS_ERSPAN:
  1175. if (key->enc_opts.dst_opt_type) {
  1176. NL_SET_ERR_MSG(extack, "Duplicate type for erspan options");
  1177. return -EINVAL;
  1178. }
  1179. option_len = 0;
  1180. key->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT;
  1181. option_len = fl_set_erspan_opt(nla_opt_key, key,
  1182. key_depth, option_len,
  1183. extack);
  1184. if (option_len < 0)
  1185. return option_len;
  1186. key->enc_opts.len += option_len;
  1187. /* At the same time we need to parse through the mask
  1188. * in order to verify exact and mask attribute lengths.
  1189. */
  1190. mask->enc_opts.dst_opt_type = TUNNEL_ERSPAN_OPT;
  1191. option_len = fl_set_erspan_opt(nla_opt_msk, mask,
  1192. msk_depth, option_len,
  1193. extack);
  1194. if (option_len < 0)
  1195. return option_len;
  1196. mask->enc_opts.len += option_len;
  1197. if (key->enc_opts.len != mask->enc_opts.len) {
  1198. NL_SET_ERR_MSG(extack, "Key and mask miss aligned");
  1199. return -EINVAL;
  1200. }
  1201. break;
  1202. default:
  1203. NL_SET_ERR_MSG(extack, "Unknown tunnel option type");
  1204. return -EINVAL;
  1205. }
  1206. if (!msk_depth)
  1207. continue;
  1208. if (!nla_ok(nla_opt_msk, msk_depth)) {
  1209. NL_SET_ERR_MSG(extack, "A mask attribute is invalid");
  1210. return -EINVAL;
  1211. }
  1212. nla_opt_msk = nla_next(nla_opt_msk, &msk_depth);
  1213. }
  1214. return 0;
  1215. }
  1216. static int fl_validate_ct_state(u16 state, struct nlattr *tb,
  1217. struct netlink_ext_ack *extack)
  1218. {
  1219. if (state && !(state & TCA_FLOWER_KEY_CT_FLAGS_TRACKED)) {
  1220. NL_SET_ERR_MSG_ATTR(extack, tb,
  1221. "no trk, so no other flag can be set");
  1222. return -EINVAL;
  1223. }
  1224. if (state & TCA_FLOWER_KEY_CT_FLAGS_NEW &&
  1225. state & TCA_FLOWER_KEY_CT_FLAGS_ESTABLISHED) {
  1226. NL_SET_ERR_MSG_ATTR(extack, tb,
  1227. "new and est are mutually exclusive");
  1228. return -EINVAL;
  1229. }
  1230. return 0;
  1231. }
  1232. static int fl_set_key_ct(struct nlattr **tb,
  1233. struct flow_dissector_key_ct *key,
  1234. struct flow_dissector_key_ct *mask,
  1235. struct netlink_ext_ack *extack)
  1236. {
  1237. if (tb[TCA_FLOWER_KEY_CT_STATE]) {
  1238. int err;
  1239. if (!IS_ENABLED(CONFIG_NF_CONNTRACK)) {
  1240. NL_SET_ERR_MSG(extack, "Conntrack isn't enabled");
  1241. return -EOPNOTSUPP;
  1242. }
  1243. fl_set_key_val(tb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE,
  1244. &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK,
  1245. sizeof(key->ct_state));
  1246. err = fl_validate_ct_state(key->ct_state & mask->ct_state,
  1247. tb[TCA_FLOWER_KEY_CT_STATE_MASK],
  1248. extack);
  1249. if (err)
  1250. return err;
  1251. }
  1252. if (tb[TCA_FLOWER_KEY_CT_ZONE]) {
  1253. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES)) {
  1254. NL_SET_ERR_MSG(extack, "Conntrack zones isn't enabled");
  1255. return -EOPNOTSUPP;
  1256. }
  1257. fl_set_key_val(tb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE,
  1258. &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK,
  1259. sizeof(key->ct_zone));
  1260. }
  1261. if (tb[TCA_FLOWER_KEY_CT_MARK]) {
  1262. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_MARK)) {
  1263. NL_SET_ERR_MSG(extack, "Conntrack mark isn't enabled");
  1264. return -EOPNOTSUPP;
  1265. }
  1266. fl_set_key_val(tb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK,
  1267. &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK,
  1268. sizeof(key->ct_mark));
  1269. }
  1270. if (tb[TCA_FLOWER_KEY_CT_LABELS]) {
  1271. if (!IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS)) {
  1272. NL_SET_ERR_MSG(extack, "Conntrack labels aren't enabled");
  1273. return -EOPNOTSUPP;
  1274. }
  1275. fl_set_key_val(tb, key->ct_labels, TCA_FLOWER_KEY_CT_LABELS,
  1276. mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK,
  1277. sizeof(key->ct_labels));
  1278. }
  1279. return 0;
  1280. }
  1281. static int fl_set_key(struct net *net, struct nlattr **tb,
  1282. struct fl_flow_key *key, struct fl_flow_key *mask,
  1283. struct netlink_ext_ack *extack)
  1284. {
  1285. __be16 ethertype;
  1286. int ret = 0;
  1287. if (tb[TCA_FLOWER_INDEV]) {
  1288. int err = tcf_change_indev(net, tb[TCA_FLOWER_INDEV], extack);
  1289. if (err < 0)
  1290. return err;
  1291. key->meta.ingress_ifindex = err;
  1292. mask->meta.ingress_ifindex = 0xffffffff;
  1293. }
  1294. fl_set_key_val(tb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
  1295. mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
  1296. sizeof(key->eth.dst));
  1297. fl_set_key_val(tb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
  1298. mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
  1299. sizeof(key->eth.src));
  1300. if (tb[TCA_FLOWER_KEY_ETH_TYPE]) {
  1301. ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_ETH_TYPE]);
  1302. if (eth_type_vlan(ethertype)) {
  1303. fl_set_key_vlan(tb, ethertype, TCA_FLOWER_KEY_VLAN_ID,
  1304. TCA_FLOWER_KEY_VLAN_PRIO,
  1305. TCA_FLOWER_KEY_VLAN_ETH_TYPE,
  1306. &key->vlan, &mask->vlan);
  1307. if (tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]) {
  1308. ethertype = nla_get_be16(tb[TCA_FLOWER_KEY_VLAN_ETH_TYPE]);
  1309. if (eth_type_vlan(ethertype)) {
  1310. fl_set_key_vlan(tb, ethertype,
  1311. TCA_FLOWER_KEY_CVLAN_ID,
  1312. TCA_FLOWER_KEY_CVLAN_PRIO,
  1313. TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
  1314. &key->cvlan, &mask->cvlan);
  1315. fl_set_key_val(tb, &key->basic.n_proto,
  1316. TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
  1317. &mask->basic.n_proto,
  1318. TCA_FLOWER_UNSPEC,
  1319. sizeof(key->basic.n_proto));
  1320. } else {
  1321. key->basic.n_proto = ethertype;
  1322. mask->basic.n_proto = cpu_to_be16(~0);
  1323. }
  1324. }
  1325. } else {
  1326. key->basic.n_proto = ethertype;
  1327. mask->basic.n_proto = cpu_to_be16(~0);
  1328. }
  1329. }
  1330. if (key->basic.n_proto == htons(ETH_P_IP) ||
  1331. key->basic.n_proto == htons(ETH_P_IPV6)) {
  1332. fl_set_key_val(tb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
  1333. &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
  1334. sizeof(key->basic.ip_proto));
  1335. fl_set_key_ip(tb, false, &key->ip, &mask->ip);
  1336. }
  1337. if (tb[TCA_FLOWER_KEY_IPV4_SRC] || tb[TCA_FLOWER_KEY_IPV4_DST]) {
  1338. key->control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
  1339. mask->control.addr_type = ~0;
  1340. fl_set_key_val(tb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
  1341. &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
  1342. sizeof(key->ipv4.src));
  1343. fl_set_key_val(tb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
  1344. &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
  1345. sizeof(key->ipv4.dst));
  1346. } else if (tb[TCA_FLOWER_KEY_IPV6_SRC] || tb[TCA_FLOWER_KEY_IPV6_DST]) {
  1347. key->control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
  1348. mask->control.addr_type = ~0;
  1349. fl_set_key_val(tb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
  1350. &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
  1351. sizeof(key->ipv6.src));
  1352. fl_set_key_val(tb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
  1353. &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
  1354. sizeof(key->ipv6.dst));
  1355. }
  1356. if (key->basic.ip_proto == IPPROTO_TCP) {
  1357. fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
  1358. &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
  1359. sizeof(key->tp.src));
  1360. fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
  1361. &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
  1362. sizeof(key->tp.dst));
  1363. fl_set_key_val(tb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
  1364. &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
  1365. sizeof(key->tcp.flags));
  1366. } else if (key->basic.ip_proto == IPPROTO_UDP) {
  1367. fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
  1368. &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
  1369. sizeof(key->tp.src));
  1370. fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
  1371. &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
  1372. sizeof(key->tp.dst));
  1373. } else if (key->basic.ip_proto == IPPROTO_SCTP) {
  1374. fl_set_key_val(tb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
  1375. &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
  1376. sizeof(key->tp.src));
  1377. fl_set_key_val(tb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
  1378. &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
  1379. sizeof(key->tp.dst));
  1380. } else if (key->basic.n_proto == htons(ETH_P_IP) &&
  1381. key->basic.ip_proto == IPPROTO_ICMP) {
  1382. fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV4_TYPE,
  1383. &mask->icmp.type,
  1384. TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
  1385. sizeof(key->icmp.type));
  1386. fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV4_CODE,
  1387. &mask->icmp.code,
  1388. TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
  1389. sizeof(key->icmp.code));
  1390. } else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
  1391. key->basic.ip_proto == IPPROTO_ICMPV6) {
  1392. fl_set_key_val(tb, &key->icmp.type, TCA_FLOWER_KEY_ICMPV6_TYPE,
  1393. &mask->icmp.type,
  1394. TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
  1395. sizeof(key->icmp.type));
  1396. fl_set_key_val(tb, &key->icmp.code, TCA_FLOWER_KEY_ICMPV6_CODE,
  1397. &mask->icmp.code,
  1398. TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
  1399. sizeof(key->icmp.code));
  1400. } else if (key->basic.n_proto == htons(ETH_P_MPLS_UC) ||
  1401. key->basic.n_proto == htons(ETH_P_MPLS_MC)) {
  1402. ret = fl_set_key_mpls(tb, &key->mpls, &mask->mpls, extack);
  1403. if (ret)
  1404. return ret;
  1405. } else if (key->basic.n_proto == htons(ETH_P_ARP) ||
  1406. key->basic.n_proto == htons(ETH_P_RARP)) {
  1407. fl_set_key_val(tb, &key->arp.sip, TCA_FLOWER_KEY_ARP_SIP,
  1408. &mask->arp.sip, TCA_FLOWER_KEY_ARP_SIP_MASK,
  1409. sizeof(key->arp.sip));
  1410. fl_set_key_val(tb, &key->arp.tip, TCA_FLOWER_KEY_ARP_TIP,
  1411. &mask->arp.tip, TCA_FLOWER_KEY_ARP_TIP_MASK,
  1412. sizeof(key->arp.tip));
  1413. fl_set_key_val(tb, &key->arp.op, TCA_FLOWER_KEY_ARP_OP,
  1414. &mask->arp.op, TCA_FLOWER_KEY_ARP_OP_MASK,
  1415. sizeof(key->arp.op));
  1416. fl_set_key_val(tb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
  1417. mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
  1418. sizeof(key->arp.sha));
  1419. fl_set_key_val(tb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
  1420. mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
  1421. sizeof(key->arp.tha));
  1422. }
  1423. if (key->basic.ip_proto == IPPROTO_TCP ||
  1424. key->basic.ip_proto == IPPROTO_UDP ||
  1425. key->basic.ip_proto == IPPROTO_SCTP) {
  1426. ret = fl_set_key_port_range(tb, key, mask, extack);
  1427. if (ret)
  1428. return ret;
  1429. }
  1430. if (tb[TCA_FLOWER_KEY_ENC_IPV4_SRC] ||
  1431. tb[TCA_FLOWER_KEY_ENC_IPV4_DST]) {
  1432. key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
  1433. mask->enc_control.addr_type = ~0;
  1434. fl_set_key_val(tb, &key->enc_ipv4.src,
  1435. TCA_FLOWER_KEY_ENC_IPV4_SRC,
  1436. &mask->enc_ipv4.src,
  1437. TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
  1438. sizeof(key->enc_ipv4.src));
  1439. fl_set_key_val(tb, &key->enc_ipv4.dst,
  1440. TCA_FLOWER_KEY_ENC_IPV4_DST,
  1441. &mask->enc_ipv4.dst,
  1442. TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
  1443. sizeof(key->enc_ipv4.dst));
  1444. }
  1445. if (tb[TCA_FLOWER_KEY_ENC_IPV6_SRC] ||
  1446. tb[TCA_FLOWER_KEY_ENC_IPV6_DST]) {
  1447. key->enc_control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
  1448. mask->enc_control.addr_type = ~0;
  1449. fl_set_key_val(tb, &key->enc_ipv6.src,
  1450. TCA_FLOWER_KEY_ENC_IPV6_SRC,
  1451. &mask->enc_ipv6.src,
  1452. TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
  1453. sizeof(key->enc_ipv6.src));
  1454. fl_set_key_val(tb, &key->enc_ipv6.dst,
  1455. TCA_FLOWER_KEY_ENC_IPV6_DST,
  1456. &mask->enc_ipv6.dst,
  1457. TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
  1458. sizeof(key->enc_ipv6.dst));
  1459. }
  1460. fl_set_key_val(tb, &key->enc_key_id.keyid, TCA_FLOWER_KEY_ENC_KEY_ID,
  1461. &mask->enc_key_id.keyid, TCA_FLOWER_UNSPEC,
  1462. sizeof(key->enc_key_id.keyid));
  1463. fl_set_key_val(tb, &key->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
  1464. &mask->enc_tp.src, TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
  1465. sizeof(key->enc_tp.src));
  1466. fl_set_key_val(tb, &key->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
  1467. &mask->enc_tp.dst, TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
  1468. sizeof(key->enc_tp.dst));
  1469. fl_set_key_ip(tb, true, &key->enc_ip, &mask->enc_ip);
  1470. fl_set_key_val(tb, &key->hash.hash, TCA_FLOWER_KEY_HASH,
  1471. &mask->hash.hash, TCA_FLOWER_KEY_HASH_MASK,
  1472. sizeof(key->hash.hash));
  1473. if (tb[TCA_FLOWER_KEY_ENC_OPTS]) {
  1474. ret = fl_set_enc_opt(tb, key, mask, extack);
  1475. if (ret)
  1476. return ret;
  1477. }
  1478. ret = fl_set_key_ct(tb, &key->ct, &mask->ct, extack);
  1479. if (ret)
  1480. return ret;
  1481. if (tb[TCA_FLOWER_KEY_FLAGS])
  1482. ret = fl_set_key_flags(tb, &key->control.flags,
  1483. &mask->control.flags, extack);
  1484. return ret;
  1485. }
  1486. static void fl_mask_copy(struct fl_flow_mask *dst,
  1487. struct fl_flow_mask *src)
  1488. {
  1489. const void *psrc = fl_key_get_start(&src->key, src);
  1490. void *pdst = fl_key_get_start(&dst->key, src);
  1491. memcpy(pdst, psrc, fl_mask_range(src));
  1492. dst->range = src->range;
  1493. }
  1494. static const struct rhashtable_params fl_ht_params = {
  1495. .key_offset = offsetof(struct cls_fl_filter, mkey), /* base offset */
  1496. .head_offset = offsetof(struct cls_fl_filter, ht_node),
  1497. .automatic_shrinking = true,
  1498. };
  1499. static int fl_init_mask_hashtable(struct fl_flow_mask *mask)
  1500. {
  1501. mask->filter_ht_params = fl_ht_params;
  1502. mask->filter_ht_params.key_len = fl_mask_range(mask);
  1503. mask->filter_ht_params.key_offset += mask->range.start;
  1504. return rhashtable_init(&mask->ht, &mask->filter_ht_params);
  1505. }
  1506. #define FL_KEY_MEMBER_OFFSET(member) offsetof(struct fl_flow_key, member)
  1507. #define FL_KEY_MEMBER_SIZE(member) sizeof_field(struct fl_flow_key, member)
  1508. #define FL_KEY_IS_MASKED(mask, member) \
  1509. memchr_inv(((char *)mask) + FL_KEY_MEMBER_OFFSET(member), \
  1510. 0, FL_KEY_MEMBER_SIZE(member)) \
  1511. #define FL_KEY_SET(keys, cnt, id, member) \
  1512. do { \
  1513. keys[cnt].key_id = id; \
  1514. keys[cnt].offset = FL_KEY_MEMBER_OFFSET(member); \
  1515. cnt++; \
  1516. } while(0);
  1517. #define FL_KEY_SET_IF_MASKED(mask, keys, cnt, id, member) \
  1518. do { \
  1519. if (FL_KEY_IS_MASKED(mask, member)) \
  1520. FL_KEY_SET(keys, cnt, id, member); \
  1521. } while(0);
  1522. static void fl_init_dissector(struct flow_dissector *dissector,
  1523. struct fl_flow_key *mask)
  1524. {
  1525. struct flow_dissector_key keys[FLOW_DISSECTOR_KEY_MAX];
  1526. size_t cnt = 0;
  1527. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1528. FLOW_DISSECTOR_KEY_META, meta);
  1529. FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_CONTROL, control);
  1530. FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_BASIC, basic);
  1531. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1532. FLOW_DISSECTOR_KEY_ETH_ADDRS, eth);
  1533. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1534. FLOW_DISSECTOR_KEY_IPV4_ADDRS, ipv4);
  1535. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1536. FLOW_DISSECTOR_KEY_IPV6_ADDRS, ipv6);
  1537. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1538. FLOW_DISSECTOR_KEY_PORTS, tp);
  1539. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1540. FLOW_DISSECTOR_KEY_PORTS_RANGE, tp_range);
  1541. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1542. FLOW_DISSECTOR_KEY_IP, ip);
  1543. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1544. FLOW_DISSECTOR_KEY_TCP, tcp);
  1545. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1546. FLOW_DISSECTOR_KEY_ICMP, icmp);
  1547. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1548. FLOW_DISSECTOR_KEY_ARP, arp);
  1549. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1550. FLOW_DISSECTOR_KEY_MPLS, mpls);
  1551. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1552. FLOW_DISSECTOR_KEY_VLAN, vlan);
  1553. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1554. FLOW_DISSECTOR_KEY_CVLAN, cvlan);
  1555. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1556. FLOW_DISSECTOR_KEY_ENC_KEYID, enc_key_id);
  1557. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1558. FLOW_DISSECTOR_KEY_ENC_IPV4_ADDRS, enc_ipv4);
  1559. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1560. FLOW_DISSECTOR_KEY_ENC_IPV6_ADDRS, enc_ipv6);
  1561. if (FL_KEY_IS_MASKED(mask, enc_ipv4) ||
  1562. FL_KEY_IS_MASKED(mask, enc_ipv6))
  1563. FL_KEY_SET(keys, cnt, FLOW_DISSECTOR_KEY_ENC_CONTROL,
  1564. enc_control);
  1565. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1566. FLOW_DISSECTOR_KEY_ENC_PORTS, enc_tp);
  1567. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1568. FLOW_DISSECTOR_KEY_ENC_IP, enc_ip);
  1569. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1570. FLOW_DISSECTOR_KEY_ENC_OPTS, enc_opts);
  1571. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1572. FLOW_DISSECTOR_KEY_CT, ct);
  1573. FL_KEY_SET_IF_MASKED(mask, keys, cnt,
  1574. FLOW_DISSECTOR_KEY_HASH, hash);
  1575. skb_flow_dissector_init(dissector, keys, cnt);
  1576. }
  1577. static struct fl_flow_mask *fl_create_new_mask(struct cls_fl_head *head,
  1578. struct fl_flow_mask *mask)
  1579. {
  1580. struct fl_flow_mask *newmask;
  1581. int err;
  1582. newmask = kzalloc(sizeof(*newmask), GFP_KERNEL);
  1583. if (!newmask)
  1584. return ERR_PTR(-ENOMEM);
  1585. fl_mask_copy(newmask, mask);
  1586. if ((newmask->key.tp_range.tp_min.dst &&
  1587. newmask->key.tp_range.tp_max.dst) ||
  1588. (newmask->key.tp_range.tp_min.src &&
  1589. newmask->key.tp_range.tp_max.src))
  1590. newmask->flags |= TCA_FLOWER_MASK_FLAGS_RANGE;
  1591. err = fl_init_mask_hashtable(newmask);
  1592. if (err)
  1593. goto errout_free;
  1594. fl_init_dissector(&newmask->dissector, &newmask->key);
  1595. INIT_LIST_HEAD_RCU(&newmask->filters);
  1596. refcount_set(&newmask->refcnt, 1);
  1597. err = rhashtable_replace_fast(&head->ht, &mask->ht_node,
  1598. &newmask->ht_node, mask_ht_params);
  1599. if (err)
  1600. goto errout_destroy;
  1601. spin_lock(&head->masks_lock);
  1602. list_add_tail_rcu(&newmask->list, &head->masks);
  1603. spin_unlock(&head->masks_lock);
  1604. return newmask;
  1605. errout_destroy:
  1606. rhashtable_destroy(&newmask->ht);
  1607. errout_free:
  1608. kfree(newmask);
  1609. return ERR_PTR(err);
  1610. }
  1611. static int fl_check_assign_mask(struct cls_fl_head *head,
  1612. struct cls_fl_filter *fnew,
  1613. struct cls_fl_filter *fold,
  1614. struct fl_flow_mask *mask)
  1615. {
  1616. struct fl_flow_mask *newmask;
  1617. int ret = 0;
  1618. rcu_read_lock();
  1619. /* Insert mask as temporary node to prevent concurrent creation of mask
  1620. * with same key. Any concurrent lookups with same key will return
  1621. * -EAGAIN because mask's refcnt is zero.
  1622. */
  1623. fnew->mask = rhashtable_lookup_get_insert_fast(&head->ht,
  1624. &mask->ht_node,
  1625. mask_ht_params);
  1626. if (!fnew->mask) {
  1627. rcu_read_unlock();
  1628. if (fold) {
  1629. ret = -EINVAL;
  1630. goto errout_cleanup;
  1631. }
  1632. newmask = fl_create_new_mask(head, mask);
  1633. if (IS_ERR(newmask)) {
  1634. ret = PTR_ERR(newmask);
  1635. goto errout_cleanup;
  1636. }
  1637. fnew->mask = newmask;
  1638. return 0;
  1639. } else if (IS_ERR(fnew->mask)) {
  1640. ret = PTR_ERR(fnew->mask);
  1641. } else if (fold && fold->mask != fnew->mask) {
  1642. ret = -EINVAL;
  1643. } else if (!refcount_inc_not_zero(&fnew->mask->refcnt)) {
  1644. /* Mask was deleted concurrently, try again */
  1645. ret = -EAGAIN;
  1646. }
  1647. rcu_read_unlock();
  1648. return ret;
  1649. errout_cleanup:
  1650. rhashtable_remove_fast(&head->ht, &mask->ht_node,
  1651. mask_ht_params);
  1652. return ret;
  1653. }
  1654. static int fl_set_parms(struct net *net, struct tcf_proto *tp,
  1655. struct cls_fl_filter *f, struct fl_flow_mask *mask,
  1656. unsigned long base, struct nlattr **tb,
  1657. struct nlattr *est, bool ovr,
  1658. struct fl_flow_tmplt *tmplt, bool rtnl_held,
  1659. struct netlink_ext_ack *extack)
  1660. {
  1661. int err;
  1662. err = tcf_exts_validate(net, tp, tb, est, &f->exts, ovr, rtnl_held,
  1663. extack);
  1664. if (err < 0)
  1665. return err;
  1666. if (tb[TCA_FLOWER_CLASSID]) {
  1667. f->res.classid = nla_get_u32(tb[TCA_FLOWER_CLASSID]);
  1668. if (!rtnl_held)
  1669. rtnl_lock();
  1670. tcf_bind_filter(tp, &f->res, base);
  1671. if (!rtnl_held)
  1672. rtnl_unlock();
  1673. }
  1674. err = fl_set_key(net, tb, &f->key, &mask->key, extack);
  1675. if (err)
  1676. return err;
  1677. fl_mask_update_range(mask);
  1678. fl_set_masked_key(&f->mkey, &f->key, mask);
  1679. if (!fl_mask_fits_tmplt(tmplt, mask)) {
  1680. NL_SET_ERR_MSG_MOD(extack, "Mask does not fit the template");
  1681. return -EINVAL;
  1682. }
  1683. return 0;
  1684. }
  1685. static int fl_ht_insert_unique(struct cls_fl_filter *fnew,
  1686. struct cls_fl_filter *fold,
  1687. bool *in_ht)
  1688. {
  1689. struct fl_flow_mask *mask = fnew->mask;
  1690. int err;
  1691. err = rhashtable_lookup_insert_fast(&mask->ht,
  1692. &fnew->ht_node,
  1693. mask->filter_ht_params);
  1694. if (err) {
  1695. *in_ht = false;
  1696. /* It is okay if filter with same key exists when
  1697. * overwriting.
  1698. */
  1699. return fold && err == -EEXIST ? 0 : err;
  1700. }
  1701. *in_ht = true;
  1702. return 0;
  1703. }
  1704. static int fl_change(struct net *net, struct sk_buff *in_skb,
  1705. struct tcf_proto *tp, unsigned long base,
  1706. u32 handle, struct nlattr **tca,
  1707. void **arg, bool ovr, bool rtnl_held,
  1708. struct netlink_ext_ack *extack)
  1709. {
  1710. struct cls_fl_head *head = fl_head_dereference(tp);
  1711. struct cls_fl_filter *fold = *arg;
  1712. struct cls_fl_filter *fnew;
  1713. struct fl_flow_mask *mask;
  1714. struct nlattr **tb;
  1715. bool in_ht;
  1716. int err;
  1717. if (!tca[TCA_OPTIONS]) {
  1718. err = -EINVAL;
  1719. goto errout_fold;
  1720. }
  1721. mask = kzalloc(sizeof(struct fl_flow_mask), GFP_KERNEL);
  1722. if (!mask) {
  1723. err = -ENOBUFS;
  1724. goto errout_fold;
  1725. }
  1726. tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
  1727. if (!tb) {
  1728. err = -ENOBUFS;
  1729. goto errout_mask_alloc;
  1730. }
  1731. err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX,
  1732. tca[TCA_OPTIONS], fl_policy, NULL);
  1733. if (err < 0)
  1734. goto errout_tb;
  1735. if (fold && handle && fold->handle != handle) {
  1736. err = -EINVAL;
  1737. goto errout_tb;
  1738. }
  1739. fnew = kzalloc(sizeof(*fnew), GFP_KERNEL);
  1740. if (!fnew) {
  1741. err = -ENOBUFS;
  1742. goto errout_tb;
  1743. }
  1744. INIT_LIST_HEAD(&fnew->hw_list);
  1745. refcount_set(&fnew->refcnt, 1);
  1746. err = tcf_exts_init(&fnew->exts, net, TCA_FLOWER_ACT, 0);
  1747. if (err < 0)
  1748. goto errout;
  1749. if (tb[TCA_FLOWER_FLAGS]) {
  1750. fnew->flags = nla_get_u32(tb[TCA_FLOWER_FLAGS]);
  1751. if (!tc_flags_valid(fnew->flags)) {
  1752. err = -EINVAL;
  1753. goto errout;
  1754. }
  1755. }
  1756. err = fl_set_parms(net, tp, fnew, mask, base, tb, tca[TCA_RATE], ovr,
  1757. tp->chain->tmplt_priv, rtnl_held, extack);
  1758. if (err)
  1759. goto errout;
  1760. err = fl_check_assign_mask(head, fnew, fold, mask);
  1761. if (err)
  1762. goto errout;
  1763. err = fl_ht_insert_unique(fnew, fold, &in_ht);
  1764. if (err)
  1765. goto errout_mask;
  1766. if (!tc_skip_hw(fnew->flags)) {
  1767. err = fl_hw_replace_filter(tp, fnew, rtnl_held, extack);
  1768. if (err)
  1769. goto errout_ht;
  1770. }
  1771. if (!tc_in_hw(fnew->flags))
  1772. fnew->flags |= TCA_CLS_FLAGS_NOT_IN_HW;
  1773. spin_lock(&tp->lock);
  1774. /* tp was deleted concurrently. -EAGAIN will cause caller to lookup
  1775. * proto again or create new one, if necessary.
  1776. */
  1777. if (tp->deleting) {
  1778. err = -EAGAIN;
  1779. goto errout_hw;
  1780. }
  1781. if (fold) {
  1782. /* Fold filter was deleted concurrently. Retry lookup. */
  1783. if (fold->deleted) {
  1784. err = -EAGAIN;
  1785. goto errout_hw;
  1786. }
  1787. fnew->handle = handle;
  1788. if (!in_ht) {
  1789. struct rhashtable_params params =
  1790. fnew->mask->filter_ht_params;
  1791. err = rhashtable_insert_fast(&fnew->mask->ht,
  1792. &fnew->ht_node,
  1793. params);
  1794. if (err)
  1795. goto errout_hw;
  1796. in_ht = true;
  1797. }
  1798. refcount_inc(&fnew->refcnt);
  1799. rhashtable_remove_fast(&fold->mask->ht,
  1800. &fold->ht_node,
  1801. fold->mask->filter_ht_params);
  1802. idr_replace(&head->handle_idr, fnew, fnew->handle);
  1803. list_replace_rcu(&fold->list, &fnew->list);
  1804. fold->deleted = true;
  1805. spin_unlock(&tp->lock);
  1806. fl_mask_put(head, fold->mask);
  1807. if (!tc_skip_hw(fold->flags))
  1808. fl_hw_destroy_filter(tp, fold, rtnl_held, NULL);
  1809. tcf_unbind_filter(tp, &fold->res);
  1810. /* Caller holds reference to fold, so refcnt is always > 0
  1811. * after this.
  1812. */
  1813. refcount_dec(&fold->refcnt);
  1814. __fl_put(fold);
  1815. } else {
  1816. if (handle) {
  1817. /* user specifies a handle and it doesn't exist */
  1818. err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
  1819. handle, GFP_ATOMIC);
  1820. /* Filter with specified handle was concurrently
  1821. * inserted after initial check in cls_api. This is not
  1822. * necessarily an error if NLM_F_EXCL is not set in
  1823. * message flags. Returning EAGAIN will cause cls_api to
  1824. * try to update concurrently inserted rule.
  1825. */
  1826. if (err == -ENOSPC)
  1827. err = -EAGAIN;
  1828. } else {
  1829. handle = 1;
  1830. err = idr_alloc_u32(&head->handle_idr, fnew, &handle,
  1831. INT_MAX, GFP_ATOMIC);
  1832. }
  1833. if (err)
  1834. goto errout_hw;
  1835. refcount_inc(&fnew->refcnt);
  1836. fnew->handle = handle;
  1837. list_add_tail_rcu(&fnew->list, &fnew->mask->filters);
  1838. spin_unlock(&tp->lock);
  1839. }
  1840. *arg = fnew;
  1841. kfree(tb);
  1842. tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work);
  1843. return 0;
  1844. errout_ht:
  1845. spin_lock(&tp->lock);
  1846. errout_hw:
  1847. fnew->deleted = true;
  1848. spin_unlock(&tp->lock);
  1849. if (!tc_skip_hw(fnew->flags))
  1850. fl_hw_destroy_filter(tp, fnew, rtnl_held, NULL);
  1851. if (in_ht)
  1852. rhashtable_remove_fast(&fnew->mask->ht, &fnew->ht_node,
  1853. fnew->mask->filter_ht_params);
  1854. errout_mask:
  1855. fl_mask_put(head, fnew->mask);
  1856. errout:
  1857. __fl_put(fnew);
  1858. errout_tb:
  1859. kfree(tb);
  1860. errout_mask_alloc:
  1861. tcf_queue_work(&mask->rwork, fl_uninit_mask_free_work);
  1862. errout_fold:
  1863. if (fold)
  1864. __fl_put(fold);
  1865. return err;
  1866. }
  1867. static int fl_delete(struct tcf_proto *tp, void *arg, bool *last,
  1868. bool rtnl_held, struct netlink_ext_ack *extack)
  1869. {
  1870. struct cls_fl_head *head = fl_head_dereference(tp);
  1871. struct cls_fl_filter *f = arg;
  1872. bool last_on_mask;
  1873. int err = 0;
  1874. err = __fl_delete(tp, f, &last_on_mask, rtnl_held, extack);
  1875. *last = list_empty(&head->masks);
  1876. __fl_put(f);
  1877. return err;
  1878. }
  1879. static void fl_walk(struct tcf_proto *tp, struct tcf_walker *arg,
  1880. bool rtnl_held)
  1881. {
  1882. struct cls_fl_head *head = fl_head_dereference(tp);
  1883. unsigned long id = arg->cookie, tmp;
  1884. struct cls_fl_filter *f;
  1885. arg->count = arg->skip;
  1886. rcu_read_lock();
  1887. idr_for_each_entry_continue_ul(&head->handle_idr, f, tmp, id) {
  1888. /* don't return filters that are being deleted */
  1889. if (!refcount_inc_not_zero(&f->refcnt))
  1890. continue;
  1891. rcu_read_unlock();
  1892. if (arg->fn(tp, f, arg) < 0) {
  1893. __fl_put(f);
  1894. arg->stop = 1;
  1895. rcu_read_lock();
  1896. break;
  1897. }
  1898. __fl_put(f);
  1899. arg->count++;
  1900. rcu_read_lock();
  1901. }
  1902. rcu_read_unlock();
  1903. arg->cookie = id;
  1904. }
  1905. static struct cls_fl_filter *
  1906. fl_get_next_hw_filter(struct tcf_proto *tp, struct cls_fl_filter *f, bool add)
  1907. {
  1908. struct cls_fl_head *head = fl_head_dereference(tp);
  1909. spin_lock(&tp->lock);
  1910. if (list_empty(&head->hw_filters)) {
  1911. spin_unlock(&tp->lock);
  1912. return NULL;
  1913. }
  1914. if (!f)
  1915. f = list_entry(&head->hw_filters, struct cls_fl_filter,
  1916. hw_list);
  1917. list_for_each_entry_continue(f, &head->hw_filters, hw_list) {
  1918. if (!(add && f->deleted) && refcount_inc_not_zero(&f->refcnt)) {
  1919. spin_unlock(&tp->lock);
  1920. return f;
  1921. }
  1922. }
  1923. spin_unlock(&tp->lock);
  1924. return NULL;
  1925. }
  1926. static int fl_reoffload(struct tcf_proto *tp, bool add, flow_setup_cb_t *cb,
  1927. void *cb_priv, struct netlink_ext_ack *extack)
  1928. {
  1929. struct tcf_block *block = tp->chain->block;
  1930. struct flow_cls_offload cls_flower = {};
  1931. struct cls_fl_filter *f = NULL;
  1932. int err;
  1933. /* hw_filters list can only be changed by hw offload functions after
  1934. * obtaining rtnl lock. Make sure it is not changed while reoffload is
  1935. * iterating it.
  1936. */
  1937. ASSERT_RTNL();
  1938. while ((f = fl_get_next_hw_filter(tp, f, add))) {
  1939. cls_flower.rule =
  1940. flow_rule_alloc(tcf_exts_num_actions(&f->exts));
  1941. if (!cls_flower.rule) {
  1942. __fl_put(f);
  1943. return -ENOMEM;
  1944. }
  1945. tc_cls_common_offload_init(&cls_flower.common, tp, f->flags,
  1946. extack);
  1947. cls_flower.command = add ?
  1948. FLOW_CLS_REPLACE : FLOW_CLS_DESTROY;
  1949. cls_flower.cookie = (unsigned long)f;
  1950. cls_flower.rule->match.dissector = &f->mask->dissector;
  1951. cls_flower.rule->match.mask = &f->mask->key;
  1952. cls_flower.rule->match.key = &f->mkey;
  1953. err = tc_setup_flow_action(&cls_flower.rule->action, &f->exts);
  1954. if (err) {
  1955. kfree(cls_flower.rule);
  1956. if (tc_skip_sw(f->flags)) {
  1957. NL_SET_ERR_MSG_MOD(extack, "Failed to setup flow action");
  1958. __fl_put(f);
  1959. return err;
  1960. }
  1961. goto next_flow;
  1962. }
  1963. cls_flower.classid = f->res.classid;
  1964. err = tc_setup_cb_reoffload(block, tp, add, cb,
  1965. TC_SETUP_CLSFLOWER, &cls_flower,
  1966. cb_priv, &f->flags,
  1967. &f->in_hw_count);
  1968. tc_cleanup_flow_action(&cls_flower.rule->action);
  1969. kfree(cls_flower.rule);
  1970. if (err) {
  1971. __fl_put(f);
  1972. return err;
  1973. }
  1974. next_flow:
  1975. __fl_put(f);
  1976. }
  1977. return 0;
  1978. }
  1979. static void fl_hw_add(struct tcf_proto *tp, void *type_data)
  1980. {
  1981. struct flow_cls_offload *cls_flower = type_data;
  1982. struct cls_fl_filter *f =
  1983. (struct cls_fl_filter *) cls_flower->cookie;
  1984. struct cls_fl_head *head = fl_head_dereference(tp);
  1985. spin_lock(&tp->lock);
  1986. list_add(&f->hw_list, &head->hw_filters);
  1987. spin_unlock(&tp->lock);
  1988. }
  1989. static void fl_hw_del(struct tcf_proto *tp, void *type_data)
  1990. {
  1991. struct flow_cls_offload *cls_flower = type_data;
  1992. struct cls_fl_filter *f =
  1993. (struct cls_fl_filter *) cls_flower->cookie;
  1994. spin_lock(&tp->lock);
  1995. if (!list_empty(&f->hw_list))
  1996. list_del_init(&f->hw_list);
  1997. spin_unlock(&tp->lock);
  1998. }
  1999. static int fl_hw_create_tmplt(struct tcf_chain *chain,
  2000. struct fl_flow_tmplt *tmplt)
  2001. {
  2002. struct flow_cls_offload cls_flower = {};
  2003. struct tcf_block *block = chain->block;
  2004. cls_flower.rule = flow_rule_alloc(0);
  2005. if (!cls_flower.rule)
  2006. return -ENOMEM;
  2007. cls_flower.common.chain_index = chain->index;
  2008. cls_flower.command = FLOW_CLS_TMPLT_CREATE;
  2009. cls_flower.cookie = (unsigned long) tmplt;
  2010. cls_flower.rule->match.dissector = &tmplt->dissector;
  2011. cls_flower.rule->match.mask = &tmplt->mask;
  2012. cls_flower.rule->match.key = &tmplt->dummy_key;
  2013. /* We don't care if driver (any of them) fails to handle this
  2014. * call. It serves just as a hint for it.
  2015. */
  2016. tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true);
  2017. kfree(cls_flower.rule);
  2018. return 0;
  2019. }
  2020. static void fl_hw_destroy_tmplt(struct tcf_chain *chain,
  2021. struct fl_flow_tmplt *tmplt)
  2022. {
  2023. struct flow_cls_offload cls_flower = {};
  2024. struct tcf_block *block = chain->block;
  2025. cls_flower.common.chain_index = chain->index;
  2026. cls_flower.command = FLOW_CLS_TMPLT_DESTROY;
  2027. cls_flower.cookie = (unsigned long) tmplt;
  2028. tc_setup_cb_call(block, TC_SETUP_CLSFLOWER, &cls_flower, false, true);
  2029. }
  2030. static void *fl_tmplt_create(struct net *net, struct tcf_chain *chain,
  2031. struct nlattr **tca,
  2032. struct netlink_ext_ack *extack)
  2033. {
  2034. struct fl_flow_tmplt *tmplt;
  2035. struct nlattr **tb;
  2036. int err;
  2037. if (!tca[TCA_OPTIONS])
  2038. return ERR_PTR(-EINVAL);
  2039. tb = kcalloc(TCA_FLOWER_MAX + 1, sizeof(struct nlattr *), GFP_KERNEL);
  2040. if (!tb)
  2041. return ERR_PTR(-ENOBUFS);
  2042. err = nla_parse_nested_deprecated(tb, TCA_FLOWER_MAX,
  2043. tca[TCA_OPTIONS], fl_policy, NULL);
  2044. if (err)
  2045. goto errout_tb;
  2046. tmplt = kzalloc(sizeof(*tmplt), GFP_KERNEL);
  2047. if (!tmplt) {
  2048. err = -ENOMEM;
  2049. goto errout_tb;
  2050. }
  2051. tmplt->chain = chain;
  2052. err = fl_set_key(net, tb, &tmplt->dummy_key, &tmplt->mask, extack);
  2053. if (err)
  2054. goto errout_tmplt;
  2055. fl_init_dissector(&tmplt->dissector, &tmplt->mask);
  2056. err = fl_hw_create_tmplt(chain, tmplt);
  2057. if (err)
  2058. goto errout_tmplt;
  2059. kfree(tb);
  2060. return tmplt;
  2061. errout_tmplt:
  2062. kfree(tmplt);
  2063. errout_tb:
  2064. kfree(tb);
  2065. return ERR_PTR(err);
  2066. }
  2067. static void fl_tmplt_destroy(void *tmplt_priv)
  2068. {
  2069. struct fl_flow_tmplt *tmplt = tmplt_priv;
  2070. fl_hw_destroy_tmplt(tmplt->chain, tmplt);
  2071. kfree(tmplt);
  2072. }
  2073. static int fl_dump_key_val(struct sk_buff *skb,
  2074. void *val, int val_type,
  2075. void *mask, int mask_type, int len)
  2076. {
  2077. int err;
  2078. if (!memchr_inv(mask, 0, len))
  2079. return 0;
  2080. err = nla_put(skb, val_type, len, val);
  2081. if (err)
  2082. return err;
  2083. if (mask_type != TCA_FLOWER_UNSPEC) {
  2084. err = nla_put(skb, mask_type, len, mask);
  2085. if (err)
  2086. return err;
  2087. }
  2088. return 0;
  2089. }
  2090. static int fl_dump_key_port_range(struct sk_buff *skb, struct fl_flow_key *key,
  2091. struct fl_flow_key *mask)
  2092. {
  2093. if (fl_dump_key_val(skb, &key->tp_range.tp_min.dst,
  2094. TCA_FLOWER_KEY_PORT_DST_MIN,
  2095. &mask->tp_range.tp_min.dst, TCA_FLOWER_UNSPEC,
  2096. sizeof(key->tp_range.tp_min.dst)) ||
  2097. fl_dump_key_val(skb, &key->tp_range.tp_max.dst,
  2098. TCA_FLOWER_KEY_PORT_DST_MAX,
  2099. &mask->tp_range.tp_max.dst, TCA_FLOWER_UNSPEC,
  2100. sizeof(key->tp_range.tp_max.dst)) ||
  2101. fl_dump_key_val(skb, &key->tp_range.tp_min.src,
  2102. TCA_FLOWER_KEY_PORT_SRC_MIN,
  2103. &mask->tp_range.tp_min.src, TCA_FLOWER_UNSPEC,
  2104. sizeof(key->tp_range.tp_min.src)) ||
  2105. fl_dump_key_val(skb, &key->tp_range.tp_max.src,
  2106. TCA_FLOWER_KEY_PORT_SRC_MAX,
  2107. &mask->tp_range.tp_max.src, TCA_FLOWER_UNSPEC,
  2108. sizeof(key->tp_range.tp_max.src)))
  2109. return -1;
  2110. return 0;
  2111. }
  2112. static int fl_dump_key_mpls_opt_lse(struct sk_buff *skb,
  2113. struct flow_dissector_key_mpls *mpls_key,
  2114. struct flow_dissector_key_mpls *mpls_mask,
  2115. u8 lse_index)
  2116. {
  2117. struct flow_dissector_mpls_lse *lse_mask = &mpls_mask->ls[lse_index];
  2118. struct flow_dissector_mpls_lse *lse_key = &mpls_key->ls[lse_index];
  2119. int err;
  2120. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_DEPTH,
  2121. lse_index + 1);
  2122. if (err)
  2123. return err;
  2124. if (lse_mask->mpls_ttl) {
  2125. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_TTL,
  2126. lse_key->mpls_ttl);
  2127. if (err)
  2128. return err;
  2129. }
  2130. if (lse_mask->mpls_bos) {
  2131. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_BOS,
  2132. lse_key->mpls_bos);
  2133. if (err)
  2134. return err;
  2135. }
  2136. if (lse_mask->mpls_tc) {
  2137. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_TC,
  2138. lse_key->mpls_tc);
  2139. if (err)
  2140. return err;
  2141. }
  2142. if (lse_mask->mpls_label) {
  2143. err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_OPT_LSE_LABEL,
  2144. lse_key->mpls_label);
  2145. if (err)
  2146. return err;
  2147. }
  2148. return 0;
  2149. }
  2150. static int fl_dump_key_mpls_opts(struct sk_buff *skb,
  2151. struct flow_dissector_key_mpls *mpls_key,
  2152. struct flow_dissector_key_mpls *mpls_mask)
  2153. {
  2154. struct nlattr *opts;
  2155. struct nlattr *lse;
  2156. u8 lse_index;
  2157. int err;
  2158. opts = nla_nest_start(skb, TCA_FLOWER_KEY_MPLS_OPTS);
  2159. if (!opts)
  2160. return -EMSGSIZE;
  2161. for (lse_index = 0; lse_index < FLOW_DIS_MPLS_MAX; lse_index++) {
  2162. if (!(mpls_mask->used_lses & 1 << lse_index))
  2163. continue;
  2164. lse = nla_nest_start(skb, TCA_FLOWER_KEY_MPLS_OPTS_LSE);
  2165. if (!lse) {
  2166. err = -EMSGSIZE;
  2167. goto err_opts;
  2168. }
  2169. err = fl_dump_key_mpls_opt_lse(skb, mpls_key, mpls_mask,
  2170. lse_index);
  2171. if (err)
  2172. goto err_opts_lse;
  2173. nla_nest_end(skb, lse);
  2174. }
  2175. nla_nest_end(skb, opts);
  2176. return 0;
  2177. err_opts_lse:
  2178. nla_nest_cancel(skb, lse);
  2179. err_opts:
  2180. nla_nest_cancel(skb, opts);
  2181. return err;
  2182. }
  2183. static int fl_dump_key_mpls(struct sk_buff *skb,
  2184. struct flow_dissector_key_mpls *mpls_key,
  2185. struct flow_dissector_key_mpls *mpls_mask)
  2186. {
  2187. struct flow_dissector_mpls_lse *lse_mask;
  2188. struct flow_dissector_mpls_lse *lse_key;
  2189. int err;
  2190. if (!mpls_mask->used_lses)
  2191. return 0;
  2192. lse_mask = &mpls_mask->ls[0];
  2193. lse_key = &mpls_key->ls[0];
  2194. /* For backward compatibility, don't use the MPLS nested attributes if
  2195. * the rule can be expressed using the old attributes.
  2196. */
  2197. if (mpls_mask->used_lses & ~1 ||
  2198. (!lse_mask->mpls_ttl && !lse_mask->mpls_bos &&
  2199. !lse_mask->mpls_tc && !lse_mask->mpls_label))
  2200. return fl_dump_key_mpls_opts(skb, mpls_key, mpls_mask);
  2201. if (lse_mask->mpls_ttl) {
  2202. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TTL,
  2203. lse_key->mpls_ttl);
  2204. if (err)
  2205. return err;
  2206. }
  2207. if (lse_mask->mpls_tc) {
  2208. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_TC,
  2209. lse_key->mpls_tc);
  2210. if (err)
  2211. return err;
  2212. }
  2213. if (lse_mask->mpls_label) {
  2214. err = nla_put_u32(skb, TCA_FLOWER_KEY_MPLS_LABEL,
  2215. lse_key->mpls_label);
  2216. if (err)
  2217. return err;
  2218. }
  2219. if (lse_mask->mpls_bos) {
  2220. err = nla_put_u8(skb, TCA_FLOWER_KEY_MPLS_BOS,
  2221. lse_key->mpls_bos);
  2222. if (err)
  2223. return err;
  2224. }
  2225. return 0;
  2226. }
  2227. static int fl_dump_key_ip(struct sk_buff *skb, bool encap,
  2228. struct flow_dissector_key_ip *key,
  2229. struct flow_dissector_key_ip *mask)
  2230. {
  2231. int tos_key = encap ? TCA_FLOWER_KEY_ENC_IP_TOS : TCA_FLOWER_KEY_IP_TOS;
  2232. int ttl_key = encap ? TCA_FLOWER_KEY_ENC_IP_TTL : TCA_FLOWER_KEY_IP_TTL;
  2233. int tos_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TOS_MASK : TCA_FLOWER_KEY_IP_TOS_MASK;
  2234. int ttl_mask = encap ? TCA_FLOWER_KEY_ENC_IP_TTL_MASK : TCA_FLOWER_KEY_IP_TTL_MASK;
  2235. if (fl_dump_key_val(skb, &key->tos, tos_key, &mask->tos, tos_mask, sizeof(key->tos)) ||
  2236. fl_dump_key_val(skb, &key->ttl, ttl_key, &mask->ttl, ttl_mask, sizeof(key->ttl)))
  2237. return -1;
  2238. return 0;
  2239. }
  2240. static int fl_dump_key_vlan(struct sk_buff *skb,
  2241. int vlan_id_key, int vlan_prio_key,
  2242. struct flow_dissector_key_vlan *vlan_key,
  2243. struct flow_dissector_key_vlan *vlan_mask)
  2244. {
  2245. int err;
  2246. if (!memchr_inv(vlan_mask, 0, sizeof(*vlan_mask)))
  2247. return 0;
  2248. if (vlan_mask->vlan_id) {
  2249. err = nla_put_u16(skb, vlan_id_key,
  2250. vlan_key->vlan_id);
  2251. if (err)
  2252. return err;
  2253. }
  2254. if (vlan_mask->vlan_priority) {
  2255. err = nla_put_u8(skb, vlan_prio_key,
  2256. vlan_key->vlan_priority);
  2257. if (err)
  2258. return err;
  2259. }
  2260. return 0;
  2261. }
  2262. static void fl_get_key_flag(u32 dissector_key, u32 dissector_mask,
  2263. u32 *flower_key, u32 *flower_mask,
  2264. u32 flower_flag_bit, u32 dissector_flag_bit)
  2265. {
  2266. if (dissector_mask & dissector_flag_bit) {
  2267. *flower_mask |= flower_flag_bit;
  2268. if (dissector_key & dissector_flag_bit)
  2269. *flower_key |= flower_flag_bit;
  2270. }
  2271. }
  2272. static int fl_dump_key_flags(struct sk_buff *skb, u32 flags_key, u32 flags_mask)
  2273. {
  2274. u32 key, mask;
  2275. __be32 _key, _mask;
  2276. int err;
  2277. if (!memchr_inv(&flags_mask, 0, sizeof(flags_mask)))
  2278. return 0;
  2279. key = 0;
  2280. mask = 0;
  2281. fl_get_key_flag(flags_key, flags_mask, &key, &mask,
  2282. TCA_FLOWER_KEY_FLAGS_IS_FRAGMENT, FLOW_DIS_IS_FRAGMENT);
  2283. fl_get_key_flag(flags_key, flags_mask, &key, &mask,
  2284. TCA_FLOWER_KEY_FLAGS_FRAG_IS_FIRST,
  2285. FLOW_DIS_FIRST_FRAG);
  2286. _key = cpu_to_be32(key);
  2287. _mask = cpu_to_be32(mask);
  2288. err = nla_put(skb, TCA_FLOWER_KEY_FLAGS, 4, &_key);
  2289. if (err)
  2290. return err;
  2291. return nla_put(skb, TCA_FLOWER_KEY_FLAGS_MASK, 4, &_mask);
  2292. }
  2293. static int fl_dump_key_geneve_opt(struct sk_buff *skb,
  2294. struct flow_dissector_key_enc_opts *enc_opts)
  2295. {
  2296. struct geneve_opt *opt;
  2297. struct nlattr *nest;
  2298. int opt_off = 0;
  2299. nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_GENEVE);
  2300. if (!nest)
  2301. goto nla_put_failure;
  2302. while (enc_opts->len > opt_off) {
  2303. opt = (struct geneve_opt *)&enc_opts->data[opt_off];
  2304. if (nla_put_be16(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_CLASS,
  2305. opt->opt_class))
  2306. goto nla_put_failure;
  2307. if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_TYPE,
  2308. opt->type))
  2309. goto nla_put_failure;
  2310. if (nla_put(skb, TCA_FLOWER_KEY_ENC_OPT_GENEVE_DATA,
  2311. opt->length * 4, opt->opt_data))
  2312. goto nla_put_failure;
  2313. opt_off += sizeof(struct geneve_opt) + opt->length * 4;
  2314. }
  2315. nla_nest_end(skb, nest);
  2316. return 0;
  2317. nla_put_failure:
  2318. nla_nest_cancel(skb, nest);
  2319. return -EMSGSIZE;
  2320. }
  2321. static int fl_dump_key_vxlan_opt(struct sk_buff *skb,
  2322. struct flow_dissector_key_enc_opts *enc_opts)
  2323. {
  2324. struct vxlan_metadata *md;
  2325. struct nlattr *nest;
  2326. nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_VXLAN);
  2327. if (!nest)
  2328. goto nla_put_failure;
  2329. md = (struct vxlan_metadata *)&enc_opts->data[0];
  2330. if (nla_put_u32(skb, TCA_FLOWER_KEY_ENC_OPT_VXLAN_GBP, md->gbp))
  2331. goto nla_put_failure;
  2332. nla_nest_end(skb, nest);
  2333. return 0;
  2334. nla_put_failure:
  2335. nla_nest_cancel(skb, nest);
  2336. return -EMSGSIZE;
  2337. }
  2338. static int fl_dump_key_erspan_opt(struct sk_buff *skb,
  2339. struct flow_dissector_key_enc_opts *enc_opts)
  2340. {
  2341. struct erspan_metadata *md;
  2342. struct nlattr *nest;
  2343. nest = nla_nest_start_noflag(skb, TCA_FLOWER_KEY_ENC_OPTS_ERSPAN);
  2344. if (!nest)
  2345. goto nla_put_failure;
  2346. md = (struct erspan_metadata *)&enc_opts->data[0];
  2347. if (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_VER, md->version))
  2348. goto nla_put_failure;
  2349. if (md->version == 1 &&
  2350. nla_put_be32(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_INDEX, md->u.index))
  2351. goto nla_put_failure;
  2352. if (md->version == 2 &&
  2353. (nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_DIR,
  2354. md->u.md2.dir) ||
  2355. nla_put_u8(skb, TCA_FLOWER_KEY_ENC_OPT_ERSPAN_HWID,
  2356. get_hwid(&md->u.md2))))
  2357. goto nla_put_failure;
  2358. nla_nest_end(skb, nest);
  2359. return 0;
  2360. nla_put_failure:
  2361. nla_nest_cancel(skb, nest);
  2362. return -EMSGSIZE;
  2363. }
  2364. static int fl_dump_key_ct(struct sk_buff *skb,
  2365. struct flow_dissector_key_ct *key,
  2366. struct flow_dissector_key_ct *mask)
  2367. {
  2368. if (IS_ENABLED(CONFIG_NF_CONNTRACK) &&
  2369. fl_dump_key_val(skb, &key->ct_state, TCA_FLOWER_KEY_CT_STATE,
  2370. &mask->ct_state, TCA_FLOWER_KEY_CT_STATE_MASK,
  2371. sizeof(key->ct_state)))
  2372. goto nla_put_failure;
  2373. if (IS_ENABLED(CONFIG_NF_CONNTRACK_ZONES) &&
  2374. fl_dump_key_val(skb, &key->ct_zone, TCA_FLOWER_KEY_CT_ZONE,
  2375. &mask->ct_zone, TCA_FLOWER_KEY_CT_ZONE_MASK,
  2376. sizeof(key->ct_zone)))
  2377. goto nla_put_failure;
  2378. if (IS_ENABLED(CONFIG_NF_CONNTRACK_MARK) &&
  2379. fl_dump_key_val(skb, &key->ct_mark, TCA_FLOWER_KEY_CT_MARK,
  2380. &mask->ct_mark, TCA_FLOWER_KEY_CT_MARK_MASK,
  2381. sizeof(key->ct_mark)))
  2382. goto nla_put_failure;
  2383. if (IS_ENABLED(CONFIG_NF_CONNTRACK_LABELS) &&
  2384. fl_dump_key_val(skb, &key->ct_labels, TCA_FLOWER_KEY_CT_LABELS,
  2385. &mask->ct_labels, TCA_FLOWER_KEY_CT_LABELS_MASK,
  2386. sizeof(key->ct_labels)))
  2387. goto nla_put_failure;
  2388. return 0;
  2389. nla_put_failure:
  2390. return -EMSGSIZE;
  2391. }
  2392. static int fl_dump_key_options(struct sk_buff *skb, int enc_opt_type,
  2393. struct flow_dissector_key_enc_opts *enc_opts)
  2394. {
  2395. struct nlattr *nest;
  2396. int err;
  2397. if (!enc_opts->len)
  2398. return 0;
  2399. nest = nla_nest_start_noflag(skb, enc_opt_type);
  2400. if (!nest)
  2401. goto nla_put_failure;
  2402. switch (enc_opts->dst_opt_type) {
  2403. case TUNNEL_GENEVE_OPT:
  2404. err = fl_dump_key_geneve_opt(skb, enc_opts);
  2405. if (err)
  2406. goto nla_put_failure;
  2407. break;
  2408. case TUNNEL_VXLAN_OPT:
  2409. err = fl_dump_key_vxlan_opt(skb, enc_opts);
  2410. if (err)
  2411. goto nla_put_failure;
  2412. break;
  2413. case TUNNEL_ERSPAN_OPT:
  2414. err = fl_dump_key_erspan_opt(skb, enc_opts);
  2415. if (err)
  2416. goto nla_put_failure;
  2417. break;
  2418. default:
  2419. goto nla_put_failure;
  2420. }
  2421. nla_nest_end(skb, nest);
  2422. return 0;
  2423. nla_put_failure:
  2424. nla_nest_cancel(skb, nest);
  2425. return -EMSGSIZE;
  2426. }
  2427. static int fl_dump_key_enc_opt(struct sk_buff *skb,
  2428. struct flow_dissector_key_enc_opts *key_opts,
  2429. struct flow_dissector_key_enc_opts *msk_opts)
  2430. {
  2431. int err;
  2432. err = fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS, key_opts);
  2433. if (err)
  2434. return err;
  2435. return fl_dump_key_options(skb, TCA_FLOWER_KEY_ENC_OPTS_MASK, msk_opts);
  2436. }
  2437. static int fl_dump_key(struct sk_buff *skb, struct net *net,
  2438. struct fl_flow_key *key, struct fl_flow_key *mask)
  2439. {
  2440. if (mask->meta.ingress_ifindex) {
  2441. struct net_device *dev;
  2442. dev = __dev_get_by_index(net, key->meta.ingress_ifindex);
  2443. if (dev && nla_put_string(skb, TCA_FLOWER_INDEV, dev->name))
  2444. goto nla_put_failure;
  2445. }
  2446. if (fl_dump_key_val(skb, key->eth.dst, TCA_FLOWER_KEY_ETH_DST,
  2447. mask->eth.dst, TCA_FLOWER_KEY_ETH_DST_MASK,
  2448. sizeof(key->eth.dst)) ||
  2449. fl_dump_key_val(skb, key->eth.src, TCA_FLOWER_KEY_ETH_SRC,
  2450. mask->eth.src, TCA_FLOWER_KEY_ETH_SRC_MASK,
  2451. sizeof(key->eth.src)) ||
  2452. fl_dump_key_val(skb, &key->basic.n_proto, TCA_FLOWER_KEY_ETH_TYPE,
  2453. &mask->basic.n_proto, TCA_FLOWER_UNSPEC,
  2454. sizeof(key->basic.n_proto)))
  2455. goto nla_put_failure;
  2456. if (fl_dump_key_mpls(skb, &key->mpls, &mask->mpls))
  2457. goto nla_put_failure;
  2458. if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_VLAN_ID,
  2459. TCA_FLOWER_KEY_VLAN_PRIO, &key->vlan, &mask->vlan))
  2460. goto nla_put_failure;
  2461. if (fl_dump_key_vlan(skb, TCA_FLOWER_KEY_CVLAN_ID,
  2462. TCA_FLOWER_KEY_CVLAN_PRIO,
  2463. &key->cvlan, &mask->cvlan) ||
  2464. (mask->cvlan.vlan_tpid &&
  2465. nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
  2466. key->cvlan.vlan_tpid)))
  2467. goto nla_put_failure;
  2468. if (mask->basic.n_proto) {
  2469. if (mask->cvlan.vlan_eth_type) {
  2470. if (nla_put_be16(skb, TCA_FLOWER_KEY_CVLAN_ETH_TYPE,
  2471. key->basic.n_proto))
  2472. goto nla_put_failure;
  2473. } else if (mask->vlan.vlan_eth_type) {
  2474. if (nla_put_be16(skb, TCA_FLOWER_KEY_VLAN_ETH_TYPE,
  2475. key->vlan.vlan_eth_type))
  2476. goto nla_put_failure;
  2477. }
  2478. }
  2479. if ((key->basic.n_proto == htons(ETH_P_IP) ||
  2480. key->basic.n_proto == htons(ETH_P_IPV6)) &&
  2481. (fl_dump_key_val(skb, &key->basic.ip_proto, TCA_FLOWER_KEY_IP_PROTO,
  2482. &mask->basic.ip_proto, TCA_FLOWER_UNSPEC,
  2483. sizeof(key->basic.ip_proto)) ||
  2484. fl_dump_key_ip(skb, false, &key->ip, &mask->ip)))
  2485. goto nla_put_failure;
  2486. if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
  2487. (fl_dump_key_val(skb, &key->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC,
  2488. &mask->ipv4.src, TCA_FLOWER_KEY_IPV4_SRC_MASK,
  2489. sizeof(key->ipv4.src)) ||
  2490. fl_dump_key_val(skb, &key->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST,
  2491. &mask->ipv4.dst, TCA_FLOWER_KEY_IPV4_DST_MASK,
  2492. sizeof(key->ipv4.dst))))
  2493. goto nla_put_failure;
  2494. else if (key->control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
  2495. (fl_dump_key_val(skb, &key->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC,
  2496. &mask->ipv6.src, TCA_FLOWER_KEY_IPV6_SRC_MASK,
  2497. sizeof(key->ipv6.src)) ||
  2498. fl_dump_key_val(skb, &key->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST,
  2499. &mask->ipv6.dst, TCA_FLOWER_KEY_IPV6_DST_MASK,
  2500. sizeof(key->ipv6.dst))))
  2501. goto nla_put_failure;
  2502. if (key->basic.ip_proto == IPPROTO_TCP &&
  2503. (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_TCP_SRC,
  2504. &mask->tp.src, TCA_FLOWER_KEY_TCP_SRC_MASK,
  2505. sizeof(key->tp.src)) ||
  2506. fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_TCP_DST,
  2507. &mask->tp.dst, TCA_FLOWER_KEY_TCP_DST_MASK,
  2508. sizeof(key->tp.dst)) ||
  2509. fl_dump_key_val(skb, &key->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS,
  2510. &mask->tcp.flags, TCA_FLOWER_KEY_TCP_FLAGS_MASK,
  2511. sizeof(key->tcp.flags))))
  2512. goto nla_put_failure;
  2513. else if (key->basic.ip_proto == IPPROTO_UDP &&
  2514. (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_UDP_SRC,
  2515. &mask->tp.src, TCA_FLOWER_KEY_UDP_SRC_MASK,
  2516. sizeof(key->tp.src)) ||
  2517. fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_UDP_DST,
  2518. &mask->tp.dst, TCA_FLOWER_KEY_UDP_DST_MASK,
  2519. sizeof(key->tp.dst))))
  2520. goto nla_put_failure;
  2521. else if (key->basic.ip_proto == IPPROTO_SCTP &&
  2522. (fl_dump_key_val(skb, &key->tp.src, TCA_FLOWER_KEY_SCTP_SRC,
  2523. &mask->tp.src, TCA_FLOWER_KEY_SCTP_SRC_MASK,
  2524. sizeof(key->tp.src)) ||
  2525. fl_dump_key_val(skb, &key->tp.dst, TCA_FLOWER_KEY_SCTP_DST,
  2526. &mask->tp.dst, TCA_FLOWER_KEY_SCTP_DST_MASK,
  2527. sizeof(key->tp.dst))))
  2528. goto nla_put_failure;
  2529. else if (key->basic.n_proto == htons(ETH_P_IP) &&
  2530. key->basic.ip_proto == IPPROTO_ICMP &&
  2531. (fl_dump_key_val(skb, &key->icmp.type,
  2532. TCA_FLOWER_KEY_ICMPV4_TYPE, &mask->icmp.type,
  2533. TCA_FLOWER_KEY_ICMPV4_TYPE_MASK,
  2534. sizeof(key->icmp.type)) ||
  2535. fl_dump_key_val(skb, &key->icmp.code,
  2536. TCA_FLOWER_KEY_ICMPV4_CODE, &mask->icmp.code,
  2537. TCA_FLOWER_KEY_ICMPV4_CODE_MASK,
  2538. sizeof(key->icmp.code))))
  2539. goto nla_put_failure;
  2540. else if (key->basic.n_proto == htons(ETH_P_IPV6) &&
  2541. key->basic.ip_proto == IPPROTO_ICMPV6 &&
  2542. (fl_dump_key_val(skb, &key->icmp.type,
  2543. TCA_FLOWER_KEY_ICMPV6_TYPE, &mask->icmp.type,
  2544. TCA_FLOWER_KEY_ICMPV6_TYPE_MASK,
  2545. sizeof(key->icmp.type)) ||
  2546. fl_dump_key_val(skb, &key->icmp.code,
  2547. TCA_FLOWER_KEY_ICMPV6_CODE, &mask->icmp.code,
  2548. TCA_FLOWER_KEY_ICMPV6_CODE_MASK,
  2549. sizeof(key->icmp.code))))
  2550. goto nla_put_failure;
  2551. else if ((key->basic.n_proto == htons(ETH_P_ARP) ||
  2552. key->basic.n_proto == htons(ETH_P_RARP)) &&
  2553. (fl_dump_key_val(skb, &key->arp.sip,
  2554. TCA_FLOWER_KEY_ARP_SIP, &mask->arp.sip,
  2555. TCA_FLOWER_KEY_ARP_SIP_MASK,
  2556. sizeof(key->arp.sip)) ||
  2557. fl_dump_key_val(skb, &key->arp.tip,
  2558. TCA_FLOWER_KEY_ARP_TIP, &mask->arp.tip,
  2559. TCA_FLOWER_KEY_ARP_TIP_MASK,
  2560. sizeof(key->arp.tip)) ||
  2561. fl_dump_key_val(skb, &key->arp.op,
  2562. TCA_FLOWER_KEY_ARP_OP, &mask->arp.op,
  2563. TCA_FLOWER_KEY_ARP_OP_MASK,
  2564. sizeof(key->arp.op)) ||
  2565. fl_dump_key_val(skb, key->arp.sha, TCA_FLOWER_KEY_ARP_SHA,
  2566. mask->arp.sha, TCA_FLOWER_KEY_ARP_SHA_MASK,
  2567. sizeof(key->arp.sha)) ||
  2568. fl_dump_key_val(skb, key->arp.tha, TCA_FLOWER_KEY_ARP_THA,
  2569. mask->arp.tha, TCA_FLOWER_KEY_ARP_THA_MASK,
  2570. sizeof(key->arp.tha))))
  2571. goto nla_put_failure;
  2572. if ((key->basic.ip_proto == IPPROTO_TCP ||
  2573. key->basic.ip_proto == IPPROTO_UDP ||
  2574. key->basic.ip_proto == IPPROTO_SCTP) &&
  2575. fl_dump_key_port_range(skb, key, mask))
  2576. goto nla_put_failure;
  2577. if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS &&
  2578. (fl_dump_key_val(skb, &key->enc_ipv4.src,
  2579. TCA_FLOWER_KEY_ENC_IPV4_SRC, &mask->enc_ipv4.src,
  2580. TCA_FLOWER_KEY_ENC_IPV4_SRC_MASK,
  2581. sizeof(key->enc_ipv4.src)) ||
  2582. fl_dump_key_val(skb, &key->enc_ipv4.dst,
  2583. TCA_FLOWER_KEY_ENC_IPV4_DST, &mask->enc_ipv4.dst,
  2584. TCA_FLOWER_KEY_ENC_IPV4_DST_MASK,
  2585. sizeof(key->enc_ipv4.dst))))
  2586. goto nla_put_failure;
  2587. else if (key->enc_control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS &&
  2588. (fl_dump_key_val(skb, &key->enc_ipv6.src,
  2589. TCA_FLOWER_KEY_ENC_IPV6_SRC, &mask->enc_ipv6.src,
  2590. TCA_FLOWER_KEY_ENC_IPV6_SRC_MASK,
  2591. sizeof(key->enc_ipv6.src)) ||
  2592. fl_dump_key_val(skb, &key->enc_ipv6.dst,
  2593. TCA_FLOWER_KEY_ENC_IPV6_DST,
  2594. &mask->enc_ipv6.dst,
  2595. TCA_FLOWER_KEY_ENC_IPV6_DST_MASK,
  2596. sizeof(key->enc_ipv6.dst))))
  2597. goto nla_put_failure;
  2598. if (fl_dump_key_val(skb, &key->enc_key_id, TCA_FLOWER_KEY_ENC_KEY_ID,
  2599. &mask->enc_key_id, TCA_FLOWER_UNSPEC,
  2600. sizeof(key->enc_key_id)) ||
  2601. fl_dump_key_val(skb, &key->enc_tp.src,
  2602. TCA_FLOWER_KEY_ENC_UDP_SRC_PORT,
  2603. &mask->enc_tp.src,
  2604. TCA_FLOWER_KEY_ENC_UDP_SRC_PORT_MASK,
  2605. sizeof(key->enc_tp.src)) ||
  2606. fl_dump_key_val(skb, &key->enc_tp.dst,
  2607. TCA_FLOWER_KEY_ENC_UDP_DST_PORT,
  2608. &mask->enc_tp.dst,
  2609. TCA_FLOWER_KEY_ENC_UDP_DST_PORT_MASK,
  2610. sizeof(key->enc_tp.dst)) ||
  2611. fl_dump_key_ip(skb, true, &key->enc_ip, &mask->enc_ip) ||
  2612. fl_dump_key_enc_opt(skb, &key->enc_opts, &mask->enc_opts))
  2613. goto nla_put_failure;
  2614. if (fl_dump_key_ct(skb, &key->ct, &mask->ct))
  2615. goto nla_put_failure;
  2616. if (fl_dump_key_flags(skb, key->control.flags, mask->control.flags))
  2617. goto nla_put_failure;
  2618. if (fl_dump_key_val(skb, &key->hash.hash, TCA_FLOWER_KEY_HASH,
  2619. &mask->hash.hash, TCA_FLOWER_KEY_HASH_MASK,
  2620. sizeof(key->hash.hash)))
  2621. goto nla_put_failure;
  2622. return 0;
  2623. nla_put_failure:
  2624. return -EMSGSIZE;
  2625. }
  2626. static int fl_dump(struct net *net, struct tcf_proto *tp, void *fh,
  2627. struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
  2628. {
  2629. struct cls_fl_filter *f = fh;
  2630. struct nlattr *nest;
  2631. struct fl_flow_key *key, *mask;
  2632. bool skip_hw;
  2633. if (!f)
  2634. return skb->len;
  2635. t->tcm_handle = f->handle;
  2636. nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
  2637. if (!nest)
  2638. goto nla_put_failure;
  2639. spin_lock(&tp->lock);
  2640. if (f->res.classid &&
  2641. nla_put_u32(skb, TCA_FLOWER_CLASSID, f->res.classid))
  2642. goto nla_put_failure_locked;
  2643. key = &f->key;
  2644. mask = &f->mask->key;
  2645. skip_hw = tc_skip_hw(f->flags);
  2646. if (fl_dump_key(skb, net, key, mask))
  2647. goto nla_put_failure_locked;
  2648. if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags))
  2649. goto nla_put_failure_locked;
  2650. spin_unlock(&tp->lock);
  2651. if (!skip_hw)
  2652. fl_hw_update_stats(tp, f, rtnl_held);
  2653. if (nla_put_u32(skb, TCA_FLOWER_IN_HW_COUNT, f->in_hw_count))
  2654. goto nla_put_failure;
  2655. if (tcf_exts_dump(skb, &f->exts))
  2656. goto nla_put_failure;
  2657. nla_nest_end(skb, nest);
  2658. if (tcf_exts_dump_stats(skb, &f->exts) < 0)
  2659. goto nla_put_failure;
  2660. return skb->len;
  2661. nla_put_failure_locked:
  2662. spin_unlock(&tp->lock);
  2663. nla_put_failure:
  2664. nla_nest_cancel(skb, nest);
  2665. return -1;
  2666. }
  2667. static int fl_terse_dump(struct net *net, struct tcf_proto *tp, void *fh,
  2668. struct sk_buff *skb, struct tcmsg *t, bool rtnl_held)
  2669. {
  2670. struct cls_fl_filter *f = fh;
  2671. struct nlattr *nest;
  2672. bool skip_hw;
  2673. if (!f)
  2674. return skb->len;
  2675. t->tcm_handle = f->handle;
  2676. nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
  2677. if (!nest)
  2678. goto nla_put_failure;
  2679. spin_lock(&tp->lock);
  2680. skip_hw = tc_skip_hw(f->flags);
  2681. if (f->flags && nla_put_u32(skb, TCA_FLOWER_FLAGS, f->flags))
  2682. goto nla_put_failure_locked;
  2683. spin_unlock(&tp->lock);
  2684. if (!skip_hw)
  2685. fl_hw_update_stats(tp, f, rtnl_held);
  2686. if (tcf_exts_terse_dump(skb, &f->exts))
  2687. goto nla_put_failure;
  2688. nla_nest_end(skb, nest);
  2689. return skb->len;
  2690. nla_put_failure_locked:
  2691. spin_unlock(&tp->lock);
  2692. nla_put_failure:
  2693. nla_nest_cancel(skb, nest);
  2694. return -1;
  2695. }
  2696. static int fl_tmplt_dump(struct sk_buff *skb, struct net *net, void *tmplt_priv)
  2697. {
  2698. struct fl_flow_tmplt *tmplt = tmplt_priv;
  2699. struct fl_flow_key *key, *mask;
  2700. struct nlattr *nest;
  2701. nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
  2702. if (!nest)
  2703. goto nla_put_failure;
  2704. key = &tmplt->dummy_key;
  2705. mask = &tmplt->mask;
  2706. if (fl_dump_key(skb, net, key, mask))
  2707. goto nla_put_failure;
  2708. nla_nest_end(skb, nest);
  2709. return skb->len;
  2710. nla_put_failure:
  2711. nla_nest_cancel(skb, nest);
  2712. return -EMSGSIZE;
  2713. }
  2714. static void fl_bind_class(void *fh, u32 classid, unsigned long cl, void *q,
  2715. unsigned long base)
  2716. {
  2717. struct cls_fl_filter *f = fh;
  2718. if (f && f->res.classid == classid) {
  2719. if (cl)
  2720. __tcf_bind_filter(q, &f->res, base);
  2721. else
  2722. __tcf_unbind_filter(q, &f->res);
  2723. }
  2724. }
  2725. static bool fl_delete_empty(struct tcf_proto *tp)
  2726. {
  2727. struct cls_fl_head *head = fl_head_dereference(tp);
  2728. spin_lock(&tp->lock);
  2729. tp->deleting = idr_is_empty(&head->handle_idr);
  2730. spin_unlock(&tp->lock);
  2731. return tp->deleting;
  2732. }
  2733. static struct tcf_proto_ops cls_fl_ops __read_mostly = {
  2734. .kind = "flower",
  2735. .classify = fl_classify,
  2736. .init = fl_init,
  2737. .destroy = fl_destroy,
  2738. .get = fl_get,
  2739. .put = fl_put,
  2740. .change = fl_change,
  2741. .delete = fl_delete,
  2742. .delete_empty = fl_delete_empty,
  2743. .walk = fl_walk,
  2744. .reoffload = fl_reoffload,
  2745. .hw_add = fl_hw_add,
  2746. .hw_del = fl_hw_del,
  2747. .dump = fl_dump,
  2748. .terse_dump = fl_terse_dump,
  2749. .bind_class = fl_bind_class,
  2750. .tmplt_create = fl_tmplt_create,
  2751. .tmplt_destroy = fl_tmplt_destroy,
  2752. .tmplt_dump = fl_tmplt_dump,
  2753. .owner = THIS_MODULE,
  2754. .flags = TCF_PROTO_OPS_DOIT_UNLOCKED,
  2755. };
  2756. static int __init cls_fl_init(void)
  2757. {
  2758. return register_tcf_proto_ops(&cls_fl_ops);
  2759. }
  2760. static void __exit cls_fl_exit(void)
  2761. {
  2762. unregister_tcf_proto_ops(&cls_fl_ops);
  2763. }
  2764. module_init(cls_fl_init);
  2765. module_exit(cls_fl_exit);
  2766. MODULE_AUTHOR("Jiri Pirko <jiri@resnulli.us>");
  2767. MODULE_DESCRIPTION("Flower classifier");
  2768. MODULE_LICENSE("GPL v2");