datapath.c 64 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472147314741475147614771478147914801481148214831484148514861487148814891490149114921493149414951496149714981499150015011502150315041505150615071508150915101511151215131514151515161517151815191520152115221523152415251526152715281529153015311532153315341535153615371538153915401541154215431544154515461547154815491550155115521553155415551556155715581559156015611562156315641565156615671568156915701571157215731574157515761577157815791580158115821583158415851586158715881589159015911592159315941595159615971598159916001601160216031604160516061607160816091610161116121613161416151616161716181619162016211622162316241625162616271628162916301631163216331634163516361637163816391640164116421643164416451646164716481649165016511652165316541655165616571658165916601661166216631664166516661667166816691670167116721673167416751676167716781679168016811682168316841685168616871688168916901691169216931694169516961697169816991700170117021703170417051706170717081709171017111712171317141715171617171718171917201721172217231724172517261727172817291730173117321733173417351736173717381739174017411742174317441745174617471748174917501751175217531754175517561757175817591760176117621763176417651766176717681769177017711772177317741775177617771778177917801781178217831784178517861787178817891790179117921793179417951796179717981799180018011802180318041805180618071808180918101811181218131814181518161817181818191820182118221823182418251826182718281829183018311832183318341835183618371838183918401841184218431844184518461847184818491850185118521853185418551856185718581859186018611862186318641865186618671868186918701871187218731874187518761877187818791880188118821883188418851886188718881889189018911892189318941895189618971898189919001901190219031904190519061907190819091910191119121913191419151916191719181919192019211922192319241925192619271928192919301931193219331934193519361937193819391940194119421943194419451946194719481949195019511952195319541955195619571958195919601961196219631964196519661967196819691970197119721973197419751976197719781979198019811982198319841985198619871988198919901991199219931994199519961997199819992000200120022003200420052006200720082009201020112012201320142015201620172018201920202021202220232024202520262027202820292030203120322033203420352036203720382039204020412042204320442045204620472048204920502051205220532054205520562057205820592060206120622063206420652066206720682069207020712072207320742075207620772078207920802081208220832084208520862087208820892090209120922093209420952096209720982099210021012102210321042105210621072108210921102111211221132114211521162117211821192120212121222123212421252126212721282129213021312132213321342135213621372138213921402141214221432144214521462147214821492150215121522153215421552156215721582159216021612162216321642165216621672168216921702171217221732174217521762177217821792180218121822183218421852186218721882189219021912192219321942195219621972198219922002201220222032204220522062207220822092210221122122213221422152216221722182219222022212222222322242225222622272228222922302231223222332234223522362237223822392240224122422243224422452246224722482249225022512252225322542255225622572258225922602261226222632264226522662267226822692270227122722273227422752276227722782279228022812282228322842285228622872288228922902291229222932294229522962297229822992300230123022303230423052306230723082309231023112312231323142315231623172318231923202321232223232324232523262327232823292330233123322333233423352336233723382339234023412342234323442345234623472348234923502351235223532354235523562357235823592360236123622363236423652366236723682369237023712372237323742375237623772378237923802381238223832384238523862387238823892390239123922393239423952396239723982399240024012402240324042405240624072408240924102411241224132414241524162417241824192420242124222423242424252426242724282429243024312432243324342435243624372438243924402441244224432444244524462447244824492450245124522453245424552456245724582459246024612462246324642465246624672468246924702471247224732474247524762477247824792480248124822483248424852486248724882489249024912492249324942495249624972498249925002501250225032504250525062507250825092510251125122513251425152516251725182519252025212522252325242525252625272528252925302531253225332534253525362537253825392540254125422543254425452546254725482549255025512552255325542555255625572558255925602561256225632564256525662567256825692570257125722573257425752576257725782579258025812582258325842585258625872588258925902591259225932594259525962597259825992600260126022603260426052606260726082609261026112612261326142615261626172618261926202621262226232624262526262627262826292630263126322633263426352636263726382639264026412642264326442645
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * Copyright (c) 2007-2014 Nicira, Inc.
  4. */
  5. #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
  6. #include <linux/init.h>
  7. #include <linux/module.h>
  8. #include <linux/if_arp.h>
  9. #include <linux/if_vlan.h>
  10. #include <linux/in.h>
  11. #include <linux/ip.h>
  12. #include <linux/jhash.h>
  13. #include <linux/delay.h>
  14. #include <linux/time.h>
  15. #include <linux/etherdevice.h>
  16. #include <linux/genetlink.h>
  17. #include <linux/kernel.h>
  18. #include <linux/kthread.h>
  19. #include <linux/mutex.h>
  20. #include <linux/percpu.h>
  21. #include <linux/rcupdate.h>
  22. #include <linux/tcp.h>
  23. #include <linux/udp.h>
  24. #include <linux/ethtool.h>
  25. #include <linux/wait.h>
  26. #include <asm/div64.h>
  27. #include <linux/highmem.h>
  28. #include <linux/netfilter_bridge.h>
  29. #include <linux/netfilter_ipv4.h>
  30. #include <linux/inetdevice.h>
  31. #include <linux/list.h>
  32. #include <linux/openvswitch.h>
  33. #include <linux/rculist.h>
  34. #include <linux/dmi.h>
  35. #include <net/genetlink.h>
  36. #include <net/net_namespace.h>
  37. #include <net/netns/generic.h>
  38. #include "datapath.h"
  39. #include "flow.h"
  40. #include "flow_table.h"
  41. #include "flow_netlink.h"
  42. #include "meter.h"
  43. #include "vport-internal_dev.h"
  44. #include "vport-netdev.h"
  45. unsigned int ovs_net_id __read_mostly;
  46. static struct genl_family dp_packet_genl_family;
  47. static struct genl_family dp_flow_genl_family;
  48. static struct genl_family dp_datapath_genl_family;
  49. static const struct nla_policy flow_policy[];
  50. static const struct genl_multicast_group ovs_dp_flow_multicast_group = {
  51. .name = OVS_FLOW_MCGROUP,
  52. };
  53. static const struct genl_multicast_group ovs_dp_datapath_multicast_group = {
  54. .name = OVS_DATAPATH_MCGROUP,
  55. };
  56. static const struct genl_multicast_group ovs_dp_vport_multicast_group = {
  57. .name = OVS_VPORT_MCGROUP,
  58. };
  59. /* Check if need to build a reply message.
  60. * OVS userspace sets the NLM_F_ECHO flag if it needs the reply. */
  61. static bool ovs_must_notify(struct genl_family *family, struct genl_info *info,
  62. unsigned int group)
  63. {
  64. return info->nlhdr->nlmsg_flags & NLM_F_ECHO ||
  65. genl_has_listeners(family, genl_info_net(info), group);
  66. }
  67. static void ovs_notify(struct genl_family *family,
  68. struct sk_buff *skb, struct genl_info *info)
  69. {
  70. genl_notify(family, skb, info, 0, GFP_KERNEL);
  71. }
  72. /**
  73. * DOC: Locking:
  74. *
  75. * All writes e.g. Writes to device state (add/remove datapath, port, set
  76. * operations on vports, etc.), Writes to other state (flow table
  77. * modifications, set miscellaneous datapath parameters, etc.) are protected
  78. * by ovs_lock.
  79. *
  80. * Reads are protected by RCU.
  81. *
  82. * There are a few special cases (mostly stats) that have their own
  83. * synchronization but they nest under all of above and don't interact with
  84. * each other.
  85. *
  86. * The RTNL lock nests inside ovs_mutex.
  87. */
  88. static DEFINE_MUTEX(ovs_mutex);
  89. void ovs_lock(void)
  90. {
  91. mutex_lock(&ovs_mutex);
  92. }
  93. void ovs_unlock(void)
  94. {
  95. mutex_unlock(&ovs_mutex);
  96. }
  97. #ifdef CONFIG_LOCKDEP
  98. int lockdep_ovsl_is_held(void)
  99. {
  100. if (debug_locks)
  101. return lockdep_is_held(&ovs_mutex);
  102. else
  103. return 1;
  104. }
  105. #endif
  106. static struct vport *new_vport(const struct vport_parms *);
  107. static int queue_gso_packets(struct datapath *dp, struct sk_buff *,
  108. const struct sw_flow_key *,
  109. const struct dp_upcall_info *,
  110. uint32_t cutlen);
  111. static int queue_userspace_packet(struct datapath *dp, struct sk_buff *,
  112. const struct sw_flow_key *,
  113. const struct dp_upcall_info *,
  114. uint32_t cutlen);
  115. static void ovs_dp_masks_rebalance(struct work_struct *work);
  116. /* Must be called with rcu_read_lock or ovs_mutex. */
  117. const char *ovs_dp_name(const struct datapath *dp)
  118. {
  119. struct vport *vport = ovs_vport_ovsl_rcu(dp, OVSP_LOCAL);
  120. return ovs_vport_name(vport);
  121. }
  122. static int get_dpifindex(const struct datapath *dp)
  123. {
  124. struct vport *local;
  125. int ifindex;
  126. rcu_read_lock();
  127. local = ovs_vport_rcu(dp, OVSP_LOCAL);
  128. if (local)
  129. ifindex = local->dev->ifindex;
  130. else
  131. ifindex = 0;
  132. rcu_read_unlock();
  133. return ifindex;
  134. }
  135. static void destroy_dp_rcu(struct rcu_head *rcu)
  136. {
  137. struct datapath *dp = container_of(rcu, struct datapath, rcu);
  138. ovs_flow_tbl_destroy(&dp->table);
  139. free_percpu(dp->stats_percpu);
  140. kfree(dp->ports);
  141. ovs_meters_exit(dp);
  142. kfree(dp);
  143. }
  144. static struct hlist_head *vport_hash_bucket(const struct datapath *dp,
  145. u16 port_no)
  146. {
  147. return &dp->ports[port_no & (DP_VPORT_HASH_BUCKETS - 1)];
  148. }
  149. /* Called with ovs_mutex or RCU read lock. */
  150. struct vport *ovs_lookup_vport(const struct datapath *dp, u16 port_no)
  151. {
  152. struct vport *vport;
  153. struct hlist_head *head;
  154. head = vport_hash_bucket(dp, port_no);
  155. hlist_for_each_entry_rcu(vport, head, dp_hash_node,
  156. lockdep_ovsl_is_held()) {
  157. if (vport->port_no == port_no)
  158. return vport;
  159. }
  160. return NULL;
  161. }
  162. /* Called with ovs_mutex. */
  163. static struct vport *new_vport(const struct vport_parms *parms)
  164. {
  165. struct vport *vport;
  166. vport = ovs_vport_add(parms);
  167. if (!IS_ERR(vport)) {
  168. struct datapath *dp = parms->dp;
  169. struct hlist_head *head = vport_hash_bucket(dp, vport->port_no);
  170. hlist_add_head_rcu(&vport->dp_hash_node, head);
  171. }
  172. return vport;
  173. }
  174. void ovs_dp_detach_port(struct vport *p)
  175. {
  176. ASSERT_OVSL();
  177. /* First drop references to device. */
  178. hlist_del_rcu(&p->dp_hash_node);
  179. /* Then destroy it. */
  180. ovs_vport_del(p);
  181. }
  182. /* Must be called with rcu_read_lock. */
  183. void ovs_dp_process_packet(struct sk_buff *skb, struct sw_flow_key *key)
  184. {
  185. const struct vport *p = OVS_CB(skb)->input_vport;
  186. struct datapath *dp = p->dp;
  187. struct sw_flow *flow;
  188. struct sw_flow_actions *sf_acts;
  189. struct dp_stats_percpu *stats;
  190. u64 *stats_counter;
  191. u32 n_mask_hit;
  192. u32 n_cache_hit;
  193. int error;
  194. stats = this_cpu_ptr(dp->stats_percpu);
  195. /* Look up flow. */
  196. flow = ovs_flow_tbl_lookup_stats(&dp->table, key, skb_get_hash(skb),
  197. &n_mask_hit, &n_cache_hit);
  198. if (unlikely(!flow)) {
  199. struct dp_upcall_info upcall;
  200. memset(&upcall, 0, sizeof(upcall));
  201. upcall.cmd = OVS_PACKET_CMD_MISS;
  202. upcall.portid = ovs_vport_find_upcall_portid(p, skb);
  203. upcall.mru = OVS_CB(skb)->mru;
  204. error = ovs_dp_upcall(dp, skb, key, &upcall, 0);
  205. if (unlikely(error))
  206. kfree_skb(skb);
  207. else
  208. consume_skb(skb);
  209. stats_counter = &stats->n_missed;
  210. goto out;
  211. }
  212. ovs_flow_stats_update(flow, key->tp.flags, skb);
  213. sf_acts = rcu_dereference(flow->sf_acts);
  214. error = ovs_execute_actions(dp, skb, sf_acts, key);
  215. if (unlikely(error))
  216. net_dbg_ratelimited("ovs: action execution error on datapath %s: %d\n",
  217. ovs_dp_name(dp), error);
  218. stats_counter = &stats->n_hit;
  219. out:
  220. /* Update datapath statistics. */
  221. u64_stats_update_begin(&stats->syncp);
  222. (*stats_counter)++;
  223. stats->n_mask_hit += n_mask_hit;
  224. stats->n_cache_hit += n_cache_hit;
  225. u64_stats_update_end(&stats->syncp);
  226. }
  227. int ovs_dp_upcall(struct datapath *dp, struct sk_buff *skb,
  228. const struct sw_flow_key *key,
  229. const struct dp_upcall_info *upcall_info,
  230. uint32_t cutlen)
  231. {
  232. struct dp_stats_percpu *stats;
  233. int err;
  234. if (upcall_info->portid == 0) {
  235. err = -ENOTCONN;
  236. goto err;
  237. }
  238. if (!skb_is_gso(skb))
  239. err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
  240. else
  241. err = queue_gso_packets(dp, skb, key, upcall_info, cutlen);
  242. if (err)
  243. goto err;
  244. return 0;
  245. err:
  246. stats = this_cpu_ptr(dp->stats_percpu);
  247. u64_stats_update_begin(&stats->syncp);
  248. stats->n_lost++;
  249. u64_stats_update_end(&stats->syncp);
  250. return err;
  251. }
  252. static int queue_gso_packets(struct datapath *dp, struct sk_buff *skb,
  253. const struct sw_flow_key *key,
  254. const struct dp_upcall_info *upcall_info,
  255. uint32_t cutlen)
  256. {
  257. unsigned int gso_type = skb_shinfo(skb)->gso_type;
  258. struct sw_flow_key later_key;
  259. struct sk_buff *segs, *nskb;
  260. int err;
  261. BUILD_BUG_ON(sizeof(*OVS_CB(skb)) > SKB_GSO_CB_OFFSET);
  262. segs = __skb_gso_segment(skb, NETIF_F_SG, false);
  263. if (IS_ERR(segs))
  264. return PTR_ERR(segs);
  265. if (segs == NULL)
  266. return -EINVAL;
  267. if (gso_type & SKB_GSO_UDP) {
  268. /* The initial flow key extracted by ovs_flow_key_extract()
  269. * in this case is for a first fragment, so we need to
  270. * properly mark later fragments.
  271. */
  272. later_key = *key;
  273. later_key.ip.frag = OVS_FRAG_TYPE_LATER;
  274. }
  275. /* Queue all of the segments. */
  276. skb_list_walk_safe(segs, skb, nskb) {
  277. if (gso_type & SKB_GSO_UDP && skb != segs)
  278. key = &later_key;
  279. err = queue_userspace_packet(dp, skb, key, upcall_info, cutlen);
  280. if (err)
  281. break;
  282. }
  283. /* Free all of the segments. */
  284. skb_list_walk_safe(segs, skb, nskb) {
  285. if (err)
  286. kfree_skb(skb);
  287. else
  288. consume_skb(skb);
  289. }
  290. return err;
  291. }
  292. static size_t upcall_msg_size(const struct dp_upcall_info *upcall_info,
  293. unsigned int hdrlen, int actions_attrlen)
  294. {
  295. size_t size = NLMSG_ALIGN(sizeof(struct ovs_header))
  296. + nla_total_size(hdrlen) /* OVS_PACKET_ATTR_PACKET */
  297. + nla_total_size(ovs_key_attr_size()) /* OVS_PACKET_ATTR_KEY */
  298. + nla_total_size(sizeof(unsigned int)) /* OVS_PACKET_ATTR_LEN */
  299. + nla_total_size(sizeof(u64)); /* OVS_PACKET_ATTR_HASH */
  300. /* OVS_PACKET_ATTR_USERDATA */
  301. if (upcall_info->userdata)
  302. size += NLA_ALIGN(upcall_info->userdata->nla_len);
  303. /* OVS_PACKET_ATTR_EGRESS_TUN_KEY */
  304. if (upcall_info->egress_tun_info)
  305. size += nla_total_size(ovs_tun_key_attr_size());
  306. /* OVS_PACKET_ATTR_ACTIONS */
  307. if (upcall_info->actions_len)
  308. size += nla_total_size(actions_attrlen);
  309. /* OVS_PACKET_ATTR_MRU */
  310. if (upcall_info->mru)
  311. size += nla_total_size(sizeof(upcall_info->mru));
  312. return size;
  313. }
  314. static void pad_packet(struct datapath *dp, struct sk_buff *skb)
  315. {
  316. if (!(dp->user_features & OVS_DP_F_UNALIGNED)) {
  317. size_t plen = NLA_ALIGN(skb->len) - skb->len;
  318. if (plen > 0)
  319. skb_put_zero(skb, plen);
  320. }
  321. }
  322. static int queue_userspace_packet(struct datapath *dp, struct sk_buff *skb,
  323. const struct sw_flow_key *key,
  324. const struct dp_upcall_info *upcall_info,
  325. uint32_t cutlen)
  326. {
  327. struct ovs_header *upcall;
  328. struct sk_buff *nskb = NULL;
  329. struct sk_buff *user_skb = NULL; /* to be queued to userspace */
  330. struct nlattr *nla;
  331. size_t len;
  332. unsigned int hlen;
  333. int err, dp_ifindex;
  334. u64 hash;
  335. dp_ifindex = get_dpifindex(dp);
  336. if (!dp_ifindex)
  337. return -ENODEV;
  338. if (skb_vlan_tag_present(skb)) {
  339. nskb = skb_clone(skb, GFP_ATOMIC);
  340. if (!nskb)
  341. return -ENOMEM;
  342. nskb = __vlan_hwaccel_push_inside(nskb);
  343. if (!nskb)
  344. return -ENOMEM;
  345. skb = nskb;
  346. }
  347. if (nla_attr_size(skb->len) > USHRT_MAX) {
  348. err = -EFBIG;
  349. goto out;
  350. }
  351. /* Complete checksum if needed */
  352. if (skb->ip_summed == CHECKSUM_PARTIAL &&
  353. (err = skb_csum_hwoffload_help(skb, 0)))
  354. goto out;
  355. /* Older versions of OVS user space enforce alignment of the last
  356. * Netlink attribute to NLA_ALIGNTO which would require extensive
  357. * padding logic. Only perform zerocopy if padding is not required.
  358. */
  359. if (dp->user_features & OVS_DP_F_UNALIGNED)
  360. hlen = skb_zerocopy_headlen(skb);
  361. else
  362. hlen = skb->len;
  363. len = upcall_msg_size(upcall_info, hlen - cutlen,
  364. OVS_CB(skb)->acts_origlen);
  365. user_skb = genlmsg_new(len, GFP_ATOMIC);
  366. if (!user_skb) {
  367. err = -ENOMEM;
  368. goto out;
  369. }
  370. upcall = genlmsg_put(user_skb, 0, 0, &dp_packet_genl_family,
  371. 0, upcall_info->cmd);
  372. if (!upcall) {
  373. err = -EINVAL;
  374. goto out;
  375. }
  376. upcall->dp_ifindex = dp_ifindex;
  377. err = ovs_nla_put_key(key, key, OVS_PACKET_ATTR_KEY, false, user_skb);
  378. if (err)
  379. goto out;
  380. if (upcall_info->userdata)
  381. __nla_put(user_skb, OVS_PACKET_ATTR_USERDATA,
  382. nla_len(upcall_info->userdata),
  383. nla_data(upcall_info->userdata));
  384. if (upcall_info->egress_tun_info) {
  385. nla = nla_nest_start_noflag(user_skb,
  386. OVS_PACKET_ATTR_EGRESS_TUN_KEY);
  387. if (!nla) {
  388. err = -EMSGSIZE;
  389. goto out;
  390. }
  391. err = ovs_nla_put_tunnel_info(user_skb,
  392. upcall_info->egress_tun_info);
  393. if (err)
  394. goto out;
  395. nla_nest_end(user_skb, nla);
  396. }
  397. if (upcall_info->actions_len) {
  398. nla = nla_nest_start_noflag(user_skb, OVS_PACKET_ATTR_ACTIONS);
  399. if (!nla) {
  400. err = -EMSGSIZE;
  401. goto out;
  402. }
  403. err = ovs_nla_put_actions(upcall_info->actions,
  404. upcall_info->actions_len,
  405. user_skb);
  406. if (!err)
  407. nla_nest_end(user_skb, nla);
  408. else
  409. nla_nest_cancel(user_skb, nla);
  410. }
  411. /* Add OVS_PACKET_ATTR_MRU */
  412. if (upcall_info->mru &&
  413. nla_put_u16(user_skb, OVS_PACKET_ATTR_MRU, upcall_info->mru)) {
  414. err = -ENOBUFS;
  415. goto out;
  416. }
  417. /* Add OVS_PACKET_ATTR_LEN when packet is truncated */
  418. if (cutlen > 0 &&
  419. nla_put_u32(user_skb, OVS_PACKET_ATTR_LEN, skb->len)) {
  420. err = -ENOBUFS;
  421. goto out;
  422. }
  423. /* Add OVS_PACKET_ATTR_HASH */
  424. hash = skb_get_hash_raw(skb);
  425. if (skb->sw_hash)
  426. hash |= OVS_PACKET_HASH_SW_BIT;
  427. if (skb->l4_hash)
  428. hash |= OVS_PACKET_HASH_L4_BIT;
  429. if (nla_put(user_skb, OVS_PACKET_ATTR_HASH, sizeof (u64), &hash)) {
  430. err = -ENOBUFS;
  431. goto out;
  432. }
  433. /* Only reserve room for attribute header, packet data is added
  434. * in skb_zerocopy() */
  435. if (!(nla = nla_reserve(user_skb, OVS_PACKET_ATTR_PACKET, 0))) {
  436. err = -ENOBUFS;
  437. goto out;
  438. }
  439. nla->nla_len = nla_attr_size(skb->len - cutlen);
  440. err = skb_zerocopy(user_skb, skb, skb->len - cutlen, hlen);
  441. if (err)
  442. goto out;
  443. /* Pad OVS_PACKET_ATTR_PACKET if linear copy was performed */
  444. pad_packet(dp, user_skb);
  445. ((struct nlmsghdr *) user_skb->data)->nlmsg_len = user_skb->len;
  446. err = genlmsg_unicast(ovs_dp_get_net(dp), user_skb, upcall_info->portid);
  447. user_skb = NULL;
  448. out:
  449. if (err)
  450. skb_tx_error(skb);
  451. kfree_skb(user_skb);
  452. kfree_skb(nskb);
  453. return err;
  454. }
  455. static int ovs_packet_cmd_execute(struct sk_buff *skb, struct genl_info *info)
  456. {
  457. struct ovs_header *ovs_header = info->userhdr;
  458. struct net *net = sock_net(skb->sk);
  459. struct nlattr **a = info->attrs;
  460. struct sw_flow_actions *acts;
  461. struct sk_buff *packet;
  462. struct sw_flow *flow;
  463. struct sw_flow_actions *sf_acts;
  464. struct datapath *dp;
  465. struct vport *input_vport;
  466. u16 mru = 0;
  467. u64 hash;
  468. int len;
  469. int err;
  470. bool log = !a[OVS_PACKET_ATTR_PROBE];
  471. err = -EINVAL;
  472. if (!a[OVS_PACKET_ATTR_PACKET] || !a[OVS_PACKET_ATTR_KEY] ||
  473. !a[OVS_PACKET_ATTR_ACTIONS])
  474. goto err;
  475. len = nla_len(a[OVS_PACKET_ATTR_PACKET]);
  476. packet = __dev_alloc_skb(NET_IP_ALIGN + len, GFP_KERNEL);
  477. err = -ENOMEM;
  478. if (!packet)
  479. goto err;
  480. skb_reserve(packet, NET_IP_ALIGN);
  481. nla_memcpy(__skb_put(packet, len), a[OVS_PACKET_ATTR_PACKET], len);
  482. /* Set packet's mru */
  483. if (a[OVS_PACKET_ATTR_MRU]) {
  484. mru = nla_get_u16(a[OVS_PACKET_ATTR_MRU]);
  485. packet->ignore_df = 1;
  486. }
  487. OVS_CB(packet)->mru = mru;
  488. if (a[OVS_PACKET_ATTR_HASH]) {
  489. hash = nla_get_u64(a[OVS_PACKET_ATTR_HASH]);
  490. __skb_set_hash(packet, hash & 0xFFFFFFFFULL,
  491. !!(hash & OVS_PACKET_HASH_SW_BIT),
  492. !!(hash & OVS_PACKET_HASH_L4_BIT));
  493. }
  494. /* Build an sw_flow for sending this packet. */
  495. flow = ovs_flow_alloc();
  496. err = PTR_ERR(flow);
  497. if (IS_ERR(flow))
  498. goto err_kfree_skb;
  499. err = ovs_flow_key_extract_userspace(net, a[OVS_PACKET_ATTR_KEY],
  500. packet, &flow->key, log);
  501. if (err)
  502. goto err_flow_free;
  503. err = ovs_nla_copy_actions(net, a[OVS_PACKET_ATTR_ACTIONS],
  504. &flow->key, &acts, log);
  505. if (err)
  506. goto err_flow_free;
  507. rcu_assign_pointer(flow->sf_acts, acts);
  508. packet->priority = flow->key.phy.priority;
  509. packet->mark = flow->key.phy.skb_mark;
  510. rcu_read_lock();
  511. dp = get_dp_rcu(net, ovs_header->dp_ifindex);
  512. err = -ENODEV;
  513. if (!dp)
  514. goto err_unlock;
  515. input_vport = ovs_vport_rcu(dp, flow->key.phy.in_port);
  516. if (!input_vport)
  517. input_vport = ovs_vport_rcu(dp, OVSP_LOCAL);
  518. if (!input_vport)
  519. goto err_unlock;
  520. packet->dev = input_vport->dev;
  521. OVS_CB(packet)->input_vport = input_vport;
  522. sf_acts = rcu_dereference(flow->sf_acts);
  523. local_bh_disable();
  524. err = ovs_execute_actions(dp, packet, sf_acts, &flow->key);
  525. local_bh_enable();
  526. rcu_read_unlock();
  527. ovs_flow_free(flow, false);
  528. return err;
  529. err_unlock:
  530. rcu_read_unlock();
  531. err_flow_free:
  532. ovs_flow_free(flow, false);
  533. err_kfree_skb:
  534. kfree_skb(packet);
  535. err:
  536. return err;
  537. }
  538. static const struct nla_policy packet_policy[OVS_PACKET_ATTR_MAX + 1] = {
  539. [OVS_PACKET_ATTR_PACKET] = { .len = ETH_HLEN },
  540. [OVS_PACKET_ATTR_KEY] = { .type = NLA_NESTED },
  541. [OVS_PACKET_ATTR_ACTIONS] = { .type = NLA_NESTED },
  542. [OVS_PACKET_ATTR_PROBE] = { .type = NLA_FLAG },
  543. [OVS_PACKET_ATTR_MRU] = { .type = NLA_U16 },
  544. [OVS_PACKET_ATTR_HASH] = { .type = NLA_U64 },
  545. };
  546. static const struct genl_small_ops dp_packet_genl_ops[] = {
  547. { .cmd = OVS_PACKET_CMD_EXECUTE,
  548. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  549. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  550. .doit = ovs_packet_cmd_execute
  551. }
  552. };
  553. static struct genl_family dp_packet_genl_family __ro_after_init = {
  554. .hdrsize = sizeof(struct ovs_header),
  555. .name = OVS_PACKET_FAMILY,
  556. .version = OVS_PACKET_VERSION,
  557. .maxattr = OVS_PACKET_ATTR_MAX,
  558. .policy = packet_policy,
  559. .netnsok = true,
  560. .parallel_ops = true,
  561. .small_ops = dp_packet_genl_ops,
  562. .n_small_ops = ARRAY_SIZE(dp_packet_genl_ops),
  563. .module = THIS_MODULE,
  564. };
  565. static void get_dp_stats(const struct datapath *dp, struct ovs_dp_stats *stats,
  566. struct ovs_dp_megaflow_stats *mega_stats)
  567. {
  568. int i;
  569. memset(mega_stats, 0, sizeof(*mega_stats));
  570. stats->n_flows = ovs_flow_tbl_count(&dp->table);
  571. mega_stats->n_masks = ovs_flow_tbl_num_masks(&dp->table);
  572. stats->n_hit = stats->n_missed = stats->n_lost = 0;
  573. for_each_possible_cpu(i) {
  574. const struct dp_stats_percpu *percpu_stats;
  575. struct dp_stats_percpu local_stats;
  576. unsigned int start;
  577. percpu_stats = per_cpu_ptr(dp->stats_percpu, i);
  578. do {
  579. start = u64_stats_fetch_begin_irq(&percpu_stats->syncp);
  580. local_stats = *percpu_stats;
  581. } while (u64_stats_fetch_retry_irq(&percpu_stats->syncp, start));
  582. stats->n_hit += local_stats.n_hit;
  583. stats->n_missed += local_stats.n_missed;
  584. stats->n_lost += local_stats.n_lost;
  585. mega_stats->n_mask_hit += local_stats.n_mask_hit;
  586. mega_stats->n_cache_hit += local_stats.n_cache_hit;
  587. }
  588. }
  589. static bool should_fill_key(const struct sw_flow_id *sfid, uint32_t ufid_flags)
  590. {
  591. return ovs_identifier_is_ufid(sfid) &&
  592. !(ufid_flags & OVS_UFID_F_OMIT_KEY);
  593. }
  594. static bool should_fill_mask(uint32_t ufid_flags)
  595. {
  596. return !(ufid_flags & OVS_UFID_F_OMIT_MASK);
  597. }
  598. static bool should_fill_actions(uint32_t ufid_flags)
  599. {
  600. return !(ufid_flags & OVS_UFID_F_OMIT_ACTIONS);
  601. }
  602. static size_t ovs_flow_cmd_msg_size(const struct sw_flow_actions *acts,
  603. const struct sw_flow_id *sfid,
  604. uint32_t ufid_flags)
  605. {
  606. size_t len = NLMSG_ALIGN(sizeof(struct ovs_header));
  607. /* OVS_FLOW_ATTR_UFID, or unmasked flow key as fallback
  608. * see ovs_nla_put_identifier()
  609. */
  610. if (sfid && ovs_identifier_is_ufid(sfid))
  611. len += nla_total_size(sfid->ufid_len);
  612. else
  613. len += nla_total_size(ovs_key_attr_size());
  614. /* OVS_FLOW_ATTR_KEY */
  615. if (!sfid || should_fill_key(sfid, ufid_flags))
  616. len += nla_total_size(ovs_key_attr_size());
  617. /* OVS_FLOW_ATTR_MASK */
  618. if (should_fill_mask(ufid_flags))
  619. len += nla_total_size(ovs_key_attr_size());
  620. /* OVS_FLOW_ATTR_ACTIONS */
  621. if (should_fill_actions(ufid_flags))
  622. len += nla_total_size(acts->orig_len);
  623. return len
  624. + nla_total_size_64bit(sizeof(struct ovs_flow_stats)) /* OVS_FLOW_ATTR_STATS */
  625. + nla_total_size(1) /* OVS_FLOW_ATTR_TCP_FLAGS */
  626. + nla_total_size_64bit(8); /* OVS_FLOW_ATTR_USED */
  627. }
  628. /* Called with ovs_mutex or RCU read lock. */
  629. static int ovs_flow_cmd_fill_stats(const struct sw_flow *flow,
  630. struct sk_buff *skb)
  631. {
  632. struct ovs_flow_stats stats;
  633. __be16 tcp_flags;
  634. unsigned long used;
  635. ovs_flow_stats_get(flow, &stats, &used, &tcp_flags);
  636. if (used &&
  637. nla_put_u64_64bit(skb, OVS_FLOW_ATTR_USED, ovs_flow_used_time(used),
  638. OVS_FLOW_ATTR_PAD))
  639. return -EMSGSIZE;
  640. if (stats.n_packets &&
  641. nla_put_64bit(skb, OVS_FLOW_ATTR_STATS,
  642. sizeof(struct ovs_flow_stats), &stats,
  643. OVS_FLOW_ATTR_PAD))
  644. return -EMSGSIZE;
  645. if ((u8)ntohs(tcp_flags) &&
  646. nla_put_u8(skb, OVS_FLOW_ATTR_TCP_FLAGS, (u8)ntohs(tcp_flags)))
  647. return -EMSGSIZE;
  648. return 0;
  649. }
  650. /* Called with ovs_mutex or RCU read lock. */
  651. static int ovs_flow_cmd_fill_actions(const struct sw_flow *flow,
  652. struct sk_buff *skb, int skb_orig_len)
  653. {
  654. struct nlattr *start;
  655. int err;
  656. /* If OVS_FLOW_ATTR_ACTIONS doesn't fit, skip dumping the actions if
  657. * this is the first flow to be dumped into 'skb'. This is unusual for
  658. * Netlink but individual action lists can be longer than
  659. * NLMSG_GOODSIZE and thus entirely undumpable if we didn't do this.
  660. * The userspace caller can always fetch the actions separately if it
  661. * really wants them. (Most userspace callers in fact don't care.)
  662. *
  663. * This can only fail for dump operations because the skb is always
  664. * properly sized for single flows.
  665. */
  666. start = nla_nest_start_noflag(skb, OVS_FLOW_ATTR_ACTIONS);
  667. if (start) {
  668. const struct sw_flow_actions *sf_acts;
  669. sf_acts = rcu_dereference_ovsl(flow->sf_acts);
  670. err = ovs_nla_put_actions(sf_acts->actions,
  671. sf_acts->actions_len, skb);
  672. if (!err)
  673. nla_nest_end(skb, start);
  674. else {
  675. if (skb_orig_len)
  676. return err;
  677. nla_nest_cancel(skb, start);
  678. }
  679. } else if (skb_orig_len) {
  680. return -EMSGSIZE;
  681. }
  682. return 0;
  683. }
  684. /* Called with ovs_mutex or RCU read lock. */
  685. static int ovs_flow_cmd_fill_info(const struct sw_flow *flow, int dp_ifindex,
  686. struct sk_buff *skb, u32 portid,
  687. u32 seq, u32 flags, u8 cmd, u32 ufid_flags)
  688. {
  689. const int skb_orig_len = skb->len;
  690. struct ovs_header *ovs_header;
  691. int err;
  692. ovs_header = genlmsg_put(skb, portid, seq, &dp_flow_genl_family,
  693. flags, cmd);
  694. if (!ovs_header)
  695. return -EMSGSIZE;
  696. ovs_header->dp_ifindex = dp_ifindex;
  697. err = ovs_nla_put_identifier(flow, skb);
  698. if (err)
  699. goto error;
  700. if (should_fill_key(&flow->id, ufid_flags)) {
  701. err = ovs_nla_put_masked_key(flow, skb);
  702. if (err)
  703. goto error;
  704. }
  705. if (should_fill_mask(ufid_flags)) {
  706. err = ovs_nla_put_mask(flow, skb);
  707. if (err)
  708. goto error;
  709. }
  710. err = ovs_flow_cmd_fill_stats(flow, skb);
  711. if (err)
  712. goto error;
  713. if (should_fill_actions(ufid_flags)) {
  714. err = ovs_flow_cmd_fill_actions(flow, skb, skb_orig_len);
  715. if (err)
  716. goto error;
  717. }
  718. genlmsg_end(skb, ovs_header);
  719. return 0;
  720. error:
  721. genlmsg_cancel(skb, ovs_header);
  722. return err;
  723. }
  724. /* May not be called with RCU read lock. */
  725. static struct sk_buff *ovs_flow_cmd_alloc_info(const struct sw_flow_actions *acts,
  726. const struct sw_flow_id *sfid,
  727. struct genl_info *info,
  728. bool always,
  729. uint32_t ufid_flags)
  730. {
  731. struct sk_buff *skb;
  732. size_t len;
  733. if (!always && !ovs_must_notify(&dp_flow_genl_family, info, 0))
  734. return NULL;
  735. len = ovs_flow_cmd_msg_size(acts, sfid, ufid_flags);
  736. skb = genlmsg_new(len, GFP_KERNEL);
  737. if (!skb)
  738. return ERR_PTR(-ENOMEM);
  739. return skb;
  740. }
  741. /* Called with ovs_mutex. */
  742. static struct sk_buff *ovs_flow_cmd_build_info(const struct sw_flow *flow,
  743. int dp_ifindex,
  744. struct genl_info *info, u8 cmd,
  745. bool always, u32 ufid_flags)
  746. {
  747. struct sk_buff *skb;
  748. int retval;
  749. skb = ovs_flow_cmd_alloc_info(ovsl_dereference(flow->sf_acts),
  750. &flow->id, info, always, ufid_flags);
  751. if (IS_ERR_OR_NULL(skb))
  752. return skb;
  753. retval = ovs_flow_cmd_fill_info(flow, dp_ifindex, skb,
  754. info->snd_portid, info->snd_seq, 0,
  755. cmd, ufid_flags);
  756. if (WARN_ON_ONCE(retval < 0)) {
  757. kfree_skb(skb);
  758. skb = ERR_PTR(retval);
  759. }
  760. return skb;
  761. }
  762. static int ovs_flow_cmd_new(struct sk_buff *skb, struct genl_info *info)
  763. {
  764. struct net *net = sock_net(skb->sk);
  765. struct nlattr **a = info->attrs;
  766. struct ovs_header *ovs_header = info->userhdr;
  767. struct sw_flow *flow = NULL, *new_flow;
  768. struct sw_flow_mask mask;
  769. struct sk_buff *reply;
  770. struct datapath *dp;
  771. struct sw_flow_actions *acts;
  772. struct sw_flow_match match;
  773. u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
  774. int error;
  775. bool log = !a[OVS_FLOW_ATTR_PROBE];
  776. /* Must have key and actions. */
  777. error = -EINVAL;
  778. if (!a[OVS_FLOW_ATTR_KEY]) {
  779. OVS_NLERR(log, "Flow key attr not present in new flow.");
  780. goto error;
  781. }
  782. if (!a[OVS_FLOW_ATTR_ACTIONS]) {
  783. OVS_NLERR(log, "Flow actions attr not present in new flow.");
  784. goto error;
  785. }
  786. /* Most of the time we need to allocate a new flow, do it before
  787. * locking.
  788. */
  789. new_flow = ovs_flow_alloc();
  790. if (IS_ERR(new_flow)) {
  791. error = PTR_ERR(new_flow);
  792. goto error;
  793. }
  794. /* Extract key. */
  795. ovs_match_init(&match, &new_flow->key, false, &mask);
  796. error = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
  797. a[OVS_FLOW_ATTR_MASK], log);
  798. if (error)
  799. goto err_kfree_flow;
  800. /* Extract flow identifier. */
  801. error = ovs_nla_get_identifier(&new_flow->id, a[OVS_FLOW_ATTR_UFID],
  802. &new_flow->key, log);
  803. if (error)
  804. goto err_kfree_flow;
  805. /* unmasked key is needed to match when ufid is not used. */
  806. if (ovs_identifier_is_key(&new_flow->id))
  807. match.key = new_flow->id.unmasked_key;
  808. ovs_flow_mask_key(&new_flow->key, &new_flow->key, true, &mask);
  809. /* Validate actions. */
  810. error = ovs_nla_copy_actions(net, a[OVS_FLOW_ATTR_ACTIONS],
  811. &new_flow->key, &acts, log);
  812. if (error) {
  813. OVS_NLERR(log, "Flow actions may not be safe on all matching packets.");
  814. goto err_kfree_flow;
  815. }
  816. reply = ovs_flow_cmd_alloc_info(acts, &new_flow->id, info, false,
  817. ufid_flags);
  818. if (IS_ERR(reply)) {
  819. error = PTR_ERR(reply);
  820. goto err_kfree_acts;
  821. }
  822. ovs_lock();
  823. dp = get_dp(net, ovs_header->dp_ifindex);
  824. if (unlikely(!dp)) {
  825. error = -ENODEV;
  826. goto err_unlock_ovs;
  827. }
  828. /* Check if this is a duplicate flow */
  829. if (ovs_identifier_is_ufid(&new_flow->id))
  830. flow = ovs_flow_tbl_lookup_ufid(&dp->table, &new_flow->id);
  831. if (!flow)
  832. flow = ovs_flow_tbl_lookup(&dp->table, &new_flow->key);
  833. if (likely(!flow)) {
  834. rcu_assign_pointer(new_flow->sf_acts, acts);
  835. /* Put flow in bucket. */
  836. error = ovs_flow_tbl_insert(&dp->table, new_flow, &mask);
  837. if (unlikely(error)) {
  838. acts = NULL;
  839. goto err_unlock_ovs;
  840. }
  841. if (unlikely(reply)) {
  842. error = ovs_flow_cmd_fill_info(new_flow,
  843. ovs_header->dp_ifindex,
  844. reply, info->snd_portid,
  845. info->snd_seq, 0,
  846. OVS_FLOW_CMD_NEW,
  847. ufid_flags);
  848. BUG_ON(error < 0);
  849. }
  850. ovs_unlock();
  851. } else {
  852. struct sw_flow_actions *old_acts;
  853. /* Bail out if we're not allowed to modify an existing flow.
  854. * We accept NLM_F_CREATE in place of the intended NLM_F_EXCL
  855. * because Generic Netlink treats the latter as a dump
  856. * request. We also accept NLM_F_EXCL in case that bug ever
  857. * gets fixed.
  858. */
  859. if (unlikely(info->nlhdr->nlmsg_flags & (NLM_F_CREATE
  860. | NLM_F_EXCL))) {
  861. error = -EEXIST;
  862. goto err_unlock_ovs;
  863. }
  864. /* The flow identifier has to be the same for flow updates.
  865. * Look for any overlapping flow.
  866. */
  867. if (unlikely(!ovs_flow_cmp(flow, &match))) {
  868. if (ovs_identifier_is_key(&flow->id))
  869. flow = ovs_flow_tbl_lookup_exact(&dp->table,
  870. &match);
  871. else /* UFID matches but key is different */
  872. flow = NULL;
  873. if (!flow) {
  874. error = -ENOENT;
  875. goto err_unlock_ovs;
  876. }
  877. }
  878. /* Update actions. */
  879. old_acts = ovsl_dereference(flow->sf_acts);
  880. rcu_assign_pointer(flow->sf_acts, acts);
  881. if (unlikely(reply)) {
  882. error = ovs_flow_cmd_fill_info(flow,
  883. ovs_header->dp_ifindex,
  884. reply, info->snd_portid,
  885. info->snd_seq, 0,
  886. OVS_FLOW_CMD_NEW,
  887. ufid_flags);
  888. BUG_ON(error < 0);
  889. }
  890. ovs_unlock();
  891. ovs_nla_free_flow_actions_rcu(old_acts);
  892. ovs_flow_free(new_flow, false);
  893. }
  894. if (reply)
  895. ovs_notify(&dp_flow_genl_family, reply, info);
  896. return 0;
  897. err_unlock_ovs:
  898. ovs_unlock();
  899. kfree_skb(reply);
  900. err_kfree_acts:
  901. ovs_nla_free_flow_actions(acts);
  902. err_kfree_flow:
  903. ovs_flow_free(new_flow, false);
  904. error:
  905. return error;
  906. }
  907. /* Factor out action copy to avoid "Wframe-larger-than=1024" warning. */
  908. static noinline_for_stack
  909. struct sw_flow_actions *get_flow_actions(struct net *net,
  910. const struct nlattr *a,
  911. const struct sw_flow_key *key,
  912. const struct sw_flow_mask *mask,
  913. bool log)
  914. {
  915. struct sw_flow_actions *acts;
  916. struct sw_flow_key masked_key;
  917. int error;
  918. ovs_flow_mask_key(&masked_key, key, true, mask);
  919. error = ovs_nla_copy_actions(net, a, &masked_key, &acts, log);
  920. if (error) {
  921. OVS_NLERR(log,
  922. "Actions may not be safe on all matching packets");
  923. return ERR_PTR(error);
  924. }
  925. return acts;
  926. }
  927. /* Factor out match-init and action-copy to avoid
  928. * "Wframe-larger-than=1024" warning. Because mask is only
  929. * used to get actions, we new a function to save some
  930. * stack space.
  931. *
  932. * If there are not key and action attrs, we return 0
  933. * directly. In the case, the caller will also not use the
  934. * match as before. If there is action attr, we try to get
  935. * actions and save them to *acts. Before returning from
  936. * the function, we reset the match->mask pointer. Because
  937. * we should not to return match object with dangling reference
  938. * to mask.
  939. * */
  940. static noinline_for_stack int
  941. ovs_nla_init_match_and_action(struct net *net,
  942. struct sw_flow_match *match,
  943. struct sw_flow_key *key,
  944. struct nlattr **a,
  945. struct sw_flow_actions **acts,
  946. bool log)
  947. {
  948. struct sw_flow_mask mask;
  949. int error = 0;
  950. if (a[OVS_FLOW_ATTR_KEY]) {
  951. ovs_match_init(match, key, true, &mask);
  952. error = ovs_nla_get_match(net, match, a[OVS_FLOW_ATTR_KEY],
  953. a[OVS_FLOW_ATTR_MASK], log);
  954. if (error)
  955. goto error;
  956. }
  957. if (a[OVS_FLOW_ATTR_ACTIONS]) {
  958. if (!a[OVS_FLOW_ATTR_KEY]) {
  959. OVS_NLERR(log,
  960. "Flow key attribute not present in set flow.");
  961. error = -EINVAL;
  962. goto error;
  963. }
  964. *acts = get_flow_actions(net, a[OVS_FLOW_ATTR_ACTIONS], key,
  965. &mask, log);
  966. if (IS_ERR(*acts)) {
  967. error = PTR_ERR(*acts);
  968. goto error;
  969. }
  970. }
  971. /* On success, error is 0. */
  972. error:
  973. match->mask = NULL;
  974. return error;
  975. }
  976. static int ovs_flow_cmd_set(struct sk_buff *skb, struct genl_info *info)
  977. {
  978. struct net *net = sock_net(skb->sk);
  979. struct nlattr **a = info->attrs;
  980. struct ovs_header *ovs_header = info->userhdr;
  981. struct sw_flow_key key;
  982. struct sw_flow *flow;
  983. struct sk_buff *reply = NULL;
  984. struct datapath *dp;
  985. struct sw_flow_actions *old_acts = NULL, *acts = NULL;
  986. struct sw_flow_match match;
  987. struct sw_flow_id sfid;
  988. u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
  989. int error = 0;
  990. bool log = !a[OVS_FLOW_ATTR_PROBE];
  991. bool ufid_present;
  992. ufid_present = ovs_nla_get_ufid(&sfid, a[OVS_FLOW_ATTR_UFID], log);
  993. if (!a[OVS_FLOW_ATTR_KEY] && !ufid_present) {
  994. OVS_NLERR(log,
  995. "Flow set message rejected, Key attribute missing.");
  996. return -EINVAL;
  997. }
  998. error = ovs_nla_init_match_and_action(net, &match, &key, a,
  999. &acts, log);
  1000. if (error)
  1001. goto error;
  1002. if (acts) {
  1003. /* Can allocate before locking if have acts. */
  1004. reply = ovs_flow_cmd_alloc_info(acts, &sfid, info, false,
  1005. ufid_flags);
  1006. if (IS_ERR(reply)) {
  1007. error = PTR_ERR(reply);
  1008. goto err_kfree_acts;
  1009. }
  1010. }
  1011. ovs_lock();
  1012. dp = get_dp(net, ovs_header->dp_ifindex);
  1013. if (unlikely(!dp)) {
  1014. error = -ENODEV;
  1015. goto err_unlock_ovs;
  1016. }
  1017. /* Check that the flow exists. */
  1018. if (ufid_present)
  1019. flow = ovs_flow_tbl_lookup_ufid(&dp->table, &sfid);
  1020. else
  1021. flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
  1022. if (unlikely(!flow)) {
  1023. error = -ENOENT;
  1024. goto err_unlock_ovs;
  1025. }
  1026. /* Update actions, if present. */
  1027. if (likely(acts)) {
  1028. old_acts = ovsl_dereference(flow->sf_acts);
  1029. rcu_assign_pointer(flow->sf_acts, acts);
  1030. if (unlikely(reply)) {
  1031. error = ovs_flow_cmd_fill_info(flow,
  1032. ovs_header->dp_ifindex,
  1033. reply, info->snd_portid,
  1034. info->snd_seq, 0,
  1035. OVS_FLOW_CMD_SET,
  1036. ufid_flags);
  1037. BUG_ON(error < 0);
  1038. }
  1039. } else {
  1040. /* Could not alloc without acts before locking. */
  1041. reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex,
  1042. info, OVS_FLOW_CMD_SET, false,
  1043. ufid_flags);
  1044. if (IS_ERR(reply)) {
  1045. error = PTR_ERR(reply);
  1046. goto err_unlock_ovs;
  1047. }
  1048. }
  1049. /* Clear stats. */
  1050. if (a[OVS_FLOW_ATTR_CLEAR])
  1051. ovs_flow_stats_clear(flow);
  1052. ovs_unlock();
  1053. if (reply)
  1054. ovs_notify(&dp_flow_genl_family, reply, info);
  1055. if (old_acts)
  1056. ovs_nla_free_flow_actions_rcu(old_acts);
  1057. return 0;
  1058. err_unlock_ovs:
  1059. ovs_unlock();
  1060. kfree_skb(reply);
  1061. err_kfree_acts:
  1062. ovs_nla_free_flow_actions(acts);
  1063. error:
  1064. return error;
  1065. }
  1066. static int ovs_flow_cmd_get(struct sk_buff *skb, struct genl_info *info)
  1067. {
  1068. struct nlattr **a = info->attrs;
  1069. struct ovs_header *ovs_header = info->userhdr;
  1070. struct net *net = sock_net(skb->sk);
  1071. struct sw_flow_key key;
  1072. struct sk_buff *reply;
  1073. struct sw_flow *flow;
  1074. struct datapath *dp;
  1075. struct sw_flow_match match;
  1076. struct sw_flow_id ufid;
  1077. u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
  1078. int err = 0;
  1079. bool log = !a[OVS_FLOW_ATTR_PROBE];
  1080. bool ufid_present;
  1081. ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
  1082. if (a[OVS_FLOW_ATTR_KEY]) {
  1083. ovs_match_init(&match, &key, true, NULL);
  1084. err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY], NULL,
  1085. log);
  1086. } else if (!ufid_present) {
  1087. OVS_NLERR(log,
  1088. "Flow get message rejected, Key attribute missing.");
  1089. err = -EINVAL;
  1090. }
  1091. if (err)
  1092. return err;
  1093. ovs_lock();
  1094. dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
  1095. if (!dp) {
  1096. err = -ENODEV;
  1097. goto unlock;
  1098. }
  1099. if (ufid_present)
  1100. flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
  1101. else
  1102. flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
  1103. if (!flow) {
  1104. err = -ENOENT;
  1105. goto unlock;
  1106. }
  1107. reply = ovs_flow_cmd_build_info(flow, ovs_header->dp_ifindex, info,
  1108. OVS_FLOW_CMD_GET, true, ufid_flags);
  1109. if (IS_ERR(reply)) {
  1110. err = PTR_ERR(reply);
  1111. goto unlock;
  1112. }
  1113. ovs_unlock();
  1114. return genlmsg_reply(reply, info);
  1115. unlock:
  1116. ovs_unlock();
  1117. return err;
  1118. }
  1119. static int ovs_flow_cmd_del(struct sk_buff *skb, struct genl_info *info)
  1120. {
  1121. struct nlattr **a = info->attrs;
  1122. struct ovs_header *ovs_header = info->userhdr;
  1123. struct net *net = sock_net(skb->sk);
  1124. struct sw_flow_key key;
  1125. struct sk_buff *reply;
  1126. struct sw_flow *flow = NULL;
  1127. struct datapath *dp;
  1128. struct sw_flow_match match;
  1129. struct sw_flow_id ufid;
  1130. u32 ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
  1131. int err;
  1132. bool log = !a[OVS_FLOW_ATTR_PROBE];
  1133. bool ufid_present;
  1134. ufid_present = ovs_nla_get_ufid(&ufid, a[OVS_FLOW_ATTR_UFID], log);
  1135. if (a[OVS_FLOW_ATTR_KEY]) {
  1136. ovs_match_init(&match, &key, true, NULL);
  1137. err = ovs_nla_get_match(net, &match, a[OVS_FLOW_ATTR_KEY],
  1138. NULL, log);
  1139. if (unlikely(err))
  1140. return err;
  1141. }
  1142. ovs_lock();
  1143. dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
  1144. if (unlikely(!dp)) {
  1145. err = -ENODEV;
  1146. goto unlock;
  1147. }
  1148. if (unlikely(!a[OVS_FLOW_ATTR_KEY] && !ufid_present)) {
  1149. err = ovs_flow_tbl_flush(&dp->table);
  1150. goto unlock;
  1151. }
  1152. if (ufid_present)
  1153. flow = ovs_flow_tbl_lookup_ufid(&dp->table, &ufid);
  1154. else
  1155. flow = ovs_flow_tbl_lookup_exact(&dp->table, &match);
  1156. if (unlikely(!flow)) {
  1157. err = -ENOENT;
  1158. goto unlock;
  1159. }
  1160. ovs_flow_tbl_remove(&dp->table, flow);
  1161. ovs_unlock();
  1162. reply = ovs_flow_cmd_alloc_info((const struct sw_flow_actions __force *) flow->sf_acts,
  1163. &flow->id, info, false, ufid_flags);
  1164. if (likely(reply)) {
  1165. if (!IS_ERR(reply)) {
  1166. rcu_read_lock(); /*To keep RCU checker happy. */
  1167. err = ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex,
  1168. reply, info->snd_portid,
  1169. info->snd_seq, 0,
  1170. OVS_FLOW_CMD_DEL,
  1171. ufid_flags);
  1172. rcu_read_unlock();
  1173. if (WARN_ON_ONCE(err < 0)) {
  1174. kfree_skb(reply);
  1175. goto out_free;
  1176. }
  1177. ovs_notify(&dp_flow_genl_family, reply, info);
  1178. } else {
  1179. netlink_set_err(sock_net(skb->sk)->genl_sock, 0, 0,
  1180. PTR_ERR(reply));
  1181. }
  1182. }
  1183. out_free:
  1184. ovs_flow_free(flow, true);
  1185. return 0;
  1186. unlock:
  1187. ovs_unlock();
  1188. return err;
  1189. }
  1190. static int ovs_flow_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
  1191. {
  1192. struct nlattr *a[__OVS_FLOW_ATTR_MAX];
  1193. struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
  1194. struct table_instance *ti;
  1195. struct datapath *dp;
  1196. u32 ufid_flags;
  1197. int err;
  1198. err = genlmsg_parse_deprecated(cb->nlh, &dp_flow_genl_family, a,
  1199. OVS_FLOW_ATTR_MAX, flow_policy, NULL);
  1200. if (err)
  1201. return err;
  1202. ufid_flags = ovs_nla_get_ufid_flags(a[OVS_FLOW_ATTR_UFID_FLAGS]);
  1203. rcu_read_lock();
  1204. dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
  1205. if (!dp) {
  1206. rcu_read_unlock();
  1207. return -ENODEV;
  1208. }
  1209. ti = rcu_dereference(dp->table.ti);
  1210. for (;;) {
  1211. struct sw_flow *flow;
  1212. u32 bucket, obj;
  1213. bucket = cb->args[0];
  1214. obj = cb->args[1];
  1215. flow = ovs_flow_tbl_dump_next(ti, &bucket, &obj);
  1216. if (!flow)
  1217. break;
  1218. if (ovs_flow_cmd_fill_info(flow, ovs_header->dp_ifindex, skb,
  1219. NETLINK_CB(cb->skb).portid,
  1220. cb->nlh->nlmsg_seq, NLM_F_MULTI,
  1221. OVS_FLOW_CMD_GET, ufid_flags) < 0)
  1222. break;
  1223. cb->args[0] = bucket;
  1224. cb->args[1] = obj;
  1225. }
  1226. rcu_read_unlock();
  1227. return skb->len;
  1228. }
  1229. static const struct nla_policy flow_policy[OVS_FLOW_ATTR_MAX + 1] = {
  1230. [OVS_FLOW_ATTR_KEY] = { .type = NLA_NESTED },
  1231. [OVS_FLOW_ATTR_MASK] = { .type = NLA_NESTED },
  1232. [OVS_FLOW_ATTR_ACTIONS] = { .type = NLA_NESTED },
  1233. [OVS_FLOW_ATTR_CLEAR] = { .type = NLA_FLAG },
  1234. [OVS_FLOW_ATTR_PROBE] = { .type = NLA_FLAG },
  1235. [OVS_FLOW_ATTR_UFID] = { .type = NLA_UNSPEC, .len = 1 },
  1236. [OVS_FLOW_ATTR_UFID_FLAGS] = { .type = NLA_U32 },
  1237. };
  1238. static const struct genl_small_ops dp_flow_genl_ops[] = {
  1239. { .cmd = OVS_FLOW_CMD_NEW,
  1240. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1241. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1242. .doit = ovs_flow_cmd_new
  1243. },
  1244. { .cmd = OVS_FLOW_CMD_DEL,
  1245. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1246. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1247. .doit = ovs_flow_cmd_del
  1248. },
  1249. { .cmd = OVS_FLOW_CMD_GET,
  1250. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1251. .flags = 0, /* OK for unprivileged users. */
  1252. .doit = ovs_flow_cmd_get,
  1253. .dumpit = ovs_flow_cmd_dump
  1254. },
  1255. { .cmd = OVS_FLOW_CMD_SET,
  1256. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1257. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1258. .doit = ovs_flow_cmd_set,
  1259. },
  1260. };
  1261. static struct genl_family dp_flow_genl_family __ro_after_init = {
  1262. .hdrsize = sizeof(struct ovs_header),
  1263. .name = OVS_FLOW_FAMILY,
  1264. .version = OVS_FLOW_VERSION,
  1265. .maxattr = OVS_FLOW_ATTR_MAX,
  1266. .policy = flow_policy,
  1267. .netnsok = true,
  1268. .parallel_ops = true,
  1269. .small_ops = dp_flow_genl_ops,
  1270. .n_small_ops = ARRAY_SIZE(dp_flow_genl_ops),
  1271. .mcgrps = &ovs_dp_flow_multicast_group,
  1272. .n_mcgrps = 1,
  1273. .module = THIS_MODULE,
  1274. };
  1275. static size_t ovs_dp_cmd_msg_size(void)
  1276. {
  1277. size_t msgsize = NLMSG_ALIGN(sizeof(struct ovs_header));
  1278. msgsize += nla_total_size(IFNAMSIZ);
  1279. msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_stats));
  1280. msgsize += nla_total_size_64bit(sizeof(struct ovs_dp_megaflow_stats));
  1281. msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_USER_FEATURES */
  1282. msgsize += nla_total_size(sizeof(u32)); /* OVS_DP_ATTR_MASKS_CACHE_SIZE */
  1283. return msgsize;
  1284. }
  1285. /* Called with ovs_mutex. */
  1286. static int ovs_dp_cmd_fill_info(struct datapath *dp, struct sk_buff *skb,
  1287. u32 portid, u32 seq, u32 flags, u8 cmd)
  1288. {
  1289. struct ovs_header *ovs_header;
  1290. struct ovs_dp_stats dp_stats;
  1291. struct ovs_dp_megaflow_stats dp_megaflow_stats;
  1292. int err;
  1293. ovs_header = genlmsg_put(skb, portid, seq, &dp_datapath_genl_family,
  1294. flags, cmd);
  1295. if (!ovs_header)
  1296. goto error;
  1297. ovs_header->dp_ifindex = get_dpifindex(dp);
  1298. err = nla_put_string(skb, OVS_DP_ATTR_NAME, ovs_dp_name(dp));
  1299. if (err)
  1300. goto nla_put_failure;
  1301. get_dp_stats(dp, &dp_stats, &dp_megaflow_stats);
  1302. if (nla_put_64bit(skb, OVS_DP_ATTR_STATS, sizeof(struct ovs_dp_stats),
  1303. &dp_stats, OVS_DP_ATTR_PAD))
  1304. goto nla_put_failure;
  1305. if (nla_put_64bit(skb, OVS_DP_ATTR_MEGAFLOW_STATS,
  1306. sizeof(struct ovs_dp_megaflow_stats),
  1307. &dp_megaflow_stats, OVS_DP_ATTR_PAD))
  1308. goto nla_put_failure;
  1309. if (nla_put_u32(skb, OVS_DP_ATTR_USER_FEATURES, dp->user_features))
  1310. goto nla_put_failure;
  1311. if (nla_put_u32(skb, OVS_DP_ATTR_MASKS_CACHE_SIZE,
  1312. ovs_flow_tbl_masks_cache_size(&dp->table)))
  1313. goto nla_put_failure;
  1314. genlmsg_end(skb, ovs_header);
  1315. return 0;
  1316. nla_put_failure:
  1317. genlmsg_cancel(skb, ovs_header);
  1318. error:
  1319. return -EMSGSIZE;
  1320. }
  1321. static struct sk_buff *ovs_dp_cmd_alloc_info(void)
  1322. {
  1323. return genlmsg_new(ovs_dp_cmd_msg_size(), GFP_KERNEL);
  1324. }
  1325. /* Called with rcu_read_lock or ovs_mutex. */
  1326. static struct datapath *lookup_datapath(struct net *net,
  1327. const struct ovs_header *ovs_header,
  1328. struct nlattr *a[OVS_DP_ATTR_MAX + 1])
  1329. {
  1330. struct datapath *dp;
  1331. if (!a[OVS_DP_ATTR_NAME])
  1332. dp = get_dp(net, ovs_header->dp_ifindex);
  1333. else {
  1334. struct vport *vport;
  1335. vport = ovs_vport_locate(net, nla_data(a[OVS_DP_ATTR_NAME]));
  1336. dp = vport && vport->port_no == OVSP_LOCAL ? vport->dp : NULL;
  1337. }
  1338. return dp ? dp : ERR_PTR(-ENODEV);
  1339. }
  1340. static void ovs_dp_reset_user_features(struct sk_buff *skb,
  1341. struct genl_info *info)
  1342. {
  1343. struct datapath *dp;
  1344. dp = lookup_datapath(sock_net(skb->sk), info->userhdr,
  1345. info->attrs);
  1346. if (IS_ERR(dp))
  1347. return;
  1348. WARN(dp->user_features, "Dropping previously announced user features\n");
  1349. dp->user_features = 0;
  1350. }
  1351. DEFINE_STATIC_KEY_FALSE(tc_recirc_sharing_support);
  1352. static int ovs_dp_change(struct datapath *dp, struct nlattr *a[])
  1353. {
  1354. u32 user_features = 0;
  1355. if (a[OVS_DP_ATTR_USER_FEATURES]) {
  1356. user_features = nla_get_u32(a[OVS_DP_ATTR_USER_FEATURES]);
  1357. if (user_features & ~(OVS_DP_F_VPORT_PIDS |
  1358. OVS_DP_F_UNALIGNED |
  1359. OVS_DP_F_TC_RECIRC_SHARING))
  1360. return -EOPNOTSUPP;
  1361. #if !IS_ENABLED(CONFIG_NET_TC_SKB_EXT)
  1362. if (user_features & OVS_DP_F_TC_RECIRC_SHARING)
  1363. return -EOPNOTSUPP;
  1364. #endif
  1365. }
  1366. if (a[OVS_DP_ATTR_MASKS_CACHE_SIZE]) {
  1367. int err;
  1368. u32 cache_size;
  1369. cache_size = nla_get_u32(a[OVS_DP_ATTR_MASKS_CACHE_SIZE]);
  1370. err = ovs_flow_tbl_masks_cache_resize(&dp->table, cache_size);
  1371. if (err)
  1372. return err;
  1373. }
  1374. dp->user_features = user_features;
  1375. if (dp->user_features & OVS_DP_F_TC_RECIRC_SHARING)
  1376. static_branch_enable(&tc_recirc_sharing_support);
  1377. else
  1378. static_branch_disable(&tc_recirc_sharing_support);
  1379. return 0;
  1380. }
  1381. static int ovs_dp_stats_init(struct datapath *dp)
  1382. {
  1383. dp->stats_percpu = netdev_alloc_pcpu_stats(struct dp_stats_percpu);
  1384. if (!dp->stats_percpu)
  1385. return -ENOMEM;
  1386. return 0;
  1387. }
  1388. static int ovs_dp_vport_init(struct datapath *dp)
  1389. {
  1390. int i;
  1391. dp->ports = kmalloc_array(DP_VPORT_HASH_BUCKETS,
  1392. sizeof(struct hlist_head),
  1393. GFP_KERNEL);
  1394. if (!dp->ports)
  1395. return -ENOMEM;
  1396. for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++)
  1397. INIT_HLIST_HEAD(&dp->ports[i]);
  1398. return 0;
  1399. }
  1400. static int ovs_dp_cmd_new(struct sk_buff *skb, struct genl_info *info)
  1401. {
  1402. struct nlattr **a = info->attrs;
  1403. struct vport_parms parms;
  1404. struct sk_buff *reply;
  1405. struct datapath *dp;
  1406. struct vport *vport;
  1407. struct ovs_net *ovs_net;
  1408. int err;
  1409. err = -EINVAL;
  1410. if (!a[OVS_DP_ATTR_NAME] || !a[OVS_DP_ATTR_UPCALL_PID])
  1411. goto err;
  1412. reply = ovs_dp_cmd_alloc_info();
  1413. if (!reply)
  1414. return -ENOMEM;
  1415. err = -ENOMEM;
  1416. dp = kzalloc(sizeof(*dp), GFP_KERNEL);
  1417. if (dp == NULL)
  1418. goto err_destroy_reply;
  1419. ovs_dp_set_net(dp, sock_net(skb->sk));
  1420. /* Allocate table. */
  1421. err = ovs_flow_tbl_init(&dp->table);
  1422. if (err)
  1423. goto err_destroy_dp;
  1424. err = ovs_dp_stats_init(dp);
  1425. if (err)
  1426. goto err_destroy_table;
  1427. err = ovs_dp_vport_init(dp);
  1428. if (err)
  1429. goto err_destroy_stats;
  1430. err = ovs_meters_init(dp);
  1431. if (err)
  1432. goto err_destroy_ports;
  1433. /* Set up our datapath device. */
  1434. parms.name = nla_data(a[OVS_DP_ATTR_NAME]);
  1435. parms.type = OVS_VPORT_TYPE_INTERNAL;
  1436. parms.options = NULL;
  1437. parms.dp = dp;
  1438. parms.port_no = OVSP_LOCAL;
  1439. parms.upcall_portids = a[OVS_DP_ATTR_UPCALL_PID];
  1440. /* So far only local changes have been made, now need the lock. */
  1441. ovs_lock();
  1442. err = ovs_dp_change(dp, a);
  1443. if (err)
  1444. goto err_unlock_and_destroy_meters;
  1445. vport = new_vport(&parms);
  1446. if (IS_ERR(vport)) {
  1447. err = PTR_ERR(vport);
  1448. if (err == -EBUSY)
  1449. err = -EEXIST;
  1450. if (err == -EEXIST) {
  1451. /* An outdated user space instance that does not understand
  1452. * the concept of user_features has attempted to create a new
  1453. * datapath and is likely to reuse it. Drop all user features.
  1454. */
  1455. if (info->genlhdr->version < OVS_DP_VER_FEATURES)
  1456. ovs_dp_reset_user_features(skb, info);
  1457. }
  1458. goto err_unlock_and_destroy_meters;
  1459. }
  1460. err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
  1461. info->snd_seq, 0, OVS_DP_CMD_NEW);
  1462. BUG_ON(err < 0);
  1463. ovs_net = net_generic(ovs_dp_get_net(dp), ovs_net_id);
  1464. list_add_tail_rcu(&dp->list_node, &ovs_net->dps);
  1465. ovs_unlock();
  1466. ovs_notify(&dp_datapath_genl_family, reply, info);
  1467. return 0;
  1468. err_unlock_and_destroy_meters:
  1469. ovs_unlock();
  1470. ovs_meters_exit(dp);
  1471. err_destroy_ports:
  1472. kfree(dp->ports);
  1473. err_destroy_stats:
  1474. free_percpu(dp->stats_percpu);
  1475. err_destroy_table:
  1476. ovs_flow_tbl_destroy(&dp->table);
  1477. err_destroy_dp:
  1478. kfree(dp);
  1479. err_destroy_reply:
  1480. kfree_skb(reply);
  1481. err:
  1482. return err;
  1483. }
  1484. /* Called with ovs_mutex. */
  1485. static void __dp_destroy(struct datapath *dp)
  1486. {
  1487. struct flow_table *table = &dp->table;
  1488. int i;
  1489. for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
  1490. struct vport *vport;
  1491. struct hlist_node *n;
  1492. hlist_for_each_entry_safe(vport, n, &dp->ports[i], dp_hash_node)
  1493. if (vport->port_no != OVSP_LOCAL)
  1494. ovs_dp_detach_port(vport);
  1495. }
  1496. list_del_rcu(&dp->list_node);
  1497. /* OVSP_LOCAL is datapath internal port. We need to make sure that
  1498. * all ports in datapath are destroyed first before freeing datapath.
  1499. */
  1500. ovs_dp_detach_port(ovs_vport_ovsl(dp, OVSP_LOCAL));
  1501. /* Flush sw_flow in the tables. RCU cb only releases resource
  1502. * such as dp, ports and tables. That may avoid some issues
  1503. * such as RCU usage warning.
  1504. */
  1505. table_instance_flow_flush(table, ovsl_dereference(table->ti),
  1506. ovsl_dereference(table->ufid_ti));
  1507. /* RCU destroy the ports, meters and flow tables. */
  1508. call_rcu(&dp->rcu, destroy_dp_rcu);
  1509. }
  1510. static int ovs_dp_cmd_del(struct sk_buff *skb, struct genl_info *info)
  1511. {
  1512. struct sk_buff *reply;
  1513. struct datapath *dp;
  1514. int err;
  1515. reply = ovs_dp_cmd_alloc_info();
  1516. if (!reply)
  1517. return -ENOMEM;
  1518. ovs_lock();
  1519. dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
  1520. err = PTR_ERR(dp);
  1521. if (IS_ERR(dp))
  1522. goto err_unlock_free;
  1523. err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
  1524. info->snd_seq, 0, OVS_DP_CMD_DEL);
  1525. BUG_ON(err < 0);
  1526. __dp_destroy(dp);
  1527. ovs_unlock();
  1528. ovs_notify(&dp_datapath_genl_family, reply, info);
  1529. return 0;
  1530. err_unlock_free:
  1531. ovs_unlock();
  1532. kfree_skb(reply);
  1533. return err;
  1534. }
  1535. static int ovs_dp_cmd_set(struct sk_buff *skb, struct genl_info *info)
  1536. {
  1537. struct sk_buff *reply;
  1538. struct datapath *dp;
  1539. int err;
  1540. reply = ovs_dp_cmd_alloc_info();
  1541. if (!reply)
  1542. return -ENOMEM;
  1543. ovs_lock();
  1544. dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
  1545. err = PTR_ERR(dp);
  1546. if (IS_ERR(dp))
  1547. goto err_unlock_free;
  1548. err = ovs_dp_change(dp, info->attrs);
  1549. if (err)
  1550. goto err_unlock_free;
  1551. err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
  1552. info->snd_seq, 0, OVS_DP_CMD_SET);
  1553. BUG_ON(err < 0);
  1554. ovs_unlock();
  1555. ovs_notify(&dp_datapath_genl_family, reply, info);
  1556. return 0;
  1557. err_unlock_free:
  1558. ovs_unlock();
  1559. kfree_skb(reply);
  1560. return err;
  1561. }
  1562. static int ovs_dp_cmd_get(struct sk_buff *skb, struct genl_info *info)
  1563. {
  1564. struct sk_buff *reply;
  1565. struct datapath *dp;
  1566. int err;
  1567. reply = ovs_dp_cmd_alloc_info();
  1568. if (!reply)
  1569. return -ENOMEM;
  1570. ovs_lock();
  1571. dp = lookup_datapath(sock_net(skb->sk), info->userhdr, info->attrs);
  1572. if (IS_ERR(dp)) {
  1573. err = PTR_ERR(dp);
  1574. goto err_unlock_free;
  1575. }
  1576. err = ovs_dp_cmd_fill_info(dp, reply, info->snd_portid,
  1577. info->snd_seq, 0, OVS_DP_CMD_GET);
  1578. BUG_ON(err < 0);
  1579. ovs_unlock();
  1580. return genlmsg_reply(reply, info);
  1581. err_unlock_free:
  1582. ovs_unlock();
  1583. kfree_skb(reply);
  1584. return err;
  1585. }
  1586. static int ovs_dp_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
  1587. {
  1588. struct ovs_net *ovs_net = net_generic(sock_net(skb->sk), ovs_net_id);
  1589. struct datapath *dp;
  1590. int skip = cb->args[0];
  1591. int i = 0;
  1592. ovs_lock();
  1593. list_for_each_entry(dp, &ovs_net->dps, list_node) {
  1594. if (i >= skip &&
  1595. ovs_dp_cmd_fill_info(dp, skb, NETLINK_CB(cb->skb).portid,
  1596. cb->nlh->nlmsg_seq, NLM_F_MULTI,
  1597. OVS_DP_CMD_GET) < 0)
  1598. break;
  1599. i++;
  1600. }
  1601. ovs_unlock();
  1602. cb->args[0] = i;
  1603. return skb->len;
  1604. }
  1605. static const struct nla_policy datapath_policy[OVS_DP_ATTR_MAX + 1] = {
  1606. [OVS_DP_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
  1607. [OVS_DP_ATTR_UPCALL_PID] = { .type = NLA_U32 },
  1608. [OVS_DP_ATTR_USER_FEATURES] = { .type = NLA_U32 },
  1609. [OVS_DP_ATTR_MASKS_CACHE_SIZE] = NLA_POLICY_RANGE(NLA_U32, 0,
  1610. PCPU_MIN_UNIT_SIZE / sizeof(struct mask_cache_entry)),
  1611. };
  1612. static const struct genl_small_ops dp_datapath_genl_ops[] = {
  1613. { .cmd = OVS_DP_CMD_NEW,
  1614. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1615. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1616. .doit = ovs_dp_cmd_new
  1617. },
  1618. { .cmd = OVS_DP_CMD_DEL,
  1619. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1620. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1621. .doit = ovs_dp_cmd_del
  1622. },
  1623. { .cmd = OVS_DP_CMD_GET,
  1624. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1625. .flags = 0, /* OK for unprivileged users. */
  1626. .doit = ovs_dp_cmd_get,
  1627. .dumpit = ovs_dp_cmd_dump
  1628. },
  1629. { .cmd = OVS_DP_CMD_SET,
  1630. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  1631. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  1632. .doit = ovs_dp_cmd_set,
  1633. },
  1634. };
  1635. static struct genl_family dp_datapath_genl_family __ro_after_init = {
  1636. .hdrsize = sizeof(struct ovs_header),
  1637. .name = OVS_DATAPATH_FAMILY,
  1638. .version = OVS_DATAPATH_VERSION,
  1639. .maxattr = OVS_DP_ATTR_MAX,
  1640. .policy = datapath_policy,
  1641. .netnsok = true,
  1642. .parallel_ops = true,
  1643. .small_ops = dp_datapath_genl_ops,
  1644. .n_small_ops = ARRAY_SIZE(dp_datapath_genl_ops),
  1645. .mcgrps = &ovs_dp_datapath_multicast_group,
  1646. .n_mcgrps = 1,
  1647. .module = THIS_MODULE,
  1648. };
  1649. /* Called with ovs_mutex or RCU read lock. */
  1650. static int ovs_vport_cmd_fill_info(struct vport *vport, struct sk_buff *skb,
  1651. struct net *net, u32 portid, u32 seq,
  1652. u32 flags, u8 cmd, gfp_t gfp)
  1653. {
  1654. struct ovs_header *ovs_header;
  1655. struct ovs_vport_stats vport_stats;
  1656. int err;
  1657. ovs_header = genlmsg_put(skb, portid, seq, &dp_vport_genl_family,
  1658. flags, cmd);
  1659. if (!ovs_header)
  1660. return -EMSGSIZE;
  1661. ovs_header->dp_ifindex = get_dpifindex(vport->dp);
  1662. if (nla_put_u32(skb, OVS_VPORT_ATTR_PORT_NO, vport->port_no) ||
  1663. nla_put_u32(skb, OVS_VPORT_ATTR_TYPE, vport->ops->type) ||
  1664. nla_put_string(skb, OVS_VPORT_ATTR_NAME,
  1665. ovs_vport_name(vport)) ||
  1666. nla_put_u32(skb, OVS_VPORT_ATTR_IFINDEX, vport->dev->ifindex))
  1667. goto nla_put_failure;
  1668. if (!net_eq(net, dev_net(vport->dev))) {
  1669. int id = peernet2id_alloc(net, dev_net(vport->dev), gfp);
  1670. if (nla_put_s32(skb, OVS_VPORT_ATTR_NETNSID, id))
  1671. goto nla_put_failure;
  1672. }
  1673. ovs_vport_get_stats(vport, &vport_stats);
  1674. if (nla_put_64bit(skb, OVS_VPORT_ATTR_STATS,
  1675. sizeof(struct ovs_vport_stats), &vport_stats,
  1676. OVS_VPORT_ATTR_PAD))
  1677. goto nla_put_failure;
  1678. if (ovs_vport_get_upcall_portids(vport, skb))
  1679. goto nla_put_failure;
  1680. err = ovs_vport_get_options(vport, skb);
  1681. if (err == -EMSGSIZE)
  1682. goto error;
  1683. genlmsg_end(skb, ovs_header);
  1684. return 0;
  1685. nla_put_failure:
  1686. err = -EMSGSIZE;
  1687. error:
  1688. genlmsg_cancel(skb, ovs_header);
  1689. return err;
  1690. }
  1691. static struct sk_buff *ovs_vport_cmd_alloc_info(void)
  1692. {
  1693. return nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
  1694. }
  1695. /* Called with ovs_mutex, only via ovs_dp_notify_wq(). */
  1696. struct sk_buff *ovs_vport_cmd_build_info(struct vport *vport, struct net *net,
  1697. u32 portid, u32 seq, u8 cmd)
  1698. {
  1699. struct sk_buff *skb;
  1700. int retval;
  1701. skb = nlmsg_new(NLMSG_DEFAULT_SIZE, GFP_KERNEL);
  1702. if (!skb)
  1703. return ERR_PTR(-ENOMEM);
  1704. retval = ovs_vport_cmd_fill_info(vport, skb, net, portid, seq, 0, cmd,
  1705. GFP_KERNEL);
  1706. BUG_ON(retval < 0);
  1707. return skb;
  1708. }
  1709. /* Called with ovs_mutex or RCU read lock. */
  1710. static struct vport *lookup_vport(struct net *net,
  1711. const struct ovs_header *ovs_header,
  1712. struct nlattr *a[OVS_VPORT_ATTR_MAX + 1])
  1713. {
  1714. struct datapath *dp;
  1715. struct vport *vport;
  1716. if (a[OVS_VPORT_ATTR_IFINDEX])
  1717. return ERR_PTR(-EOPNOTSUPP);
  1718. if (a[OVS_VPORT_ATTR_NAME]) {
  1719. vport = ovs_vport_locate(net, nla_data(a[OVS_VPORT_ATTR_NAME]));
  1720. if (!vport)
  1721. return ERR_PTR(-ENODEV);
  1722. if (ovs_header->dp_ifindex &&
  1723. ovs_header->dp_ifindex != get_dpifindex(vport->dp))
  1724. return ERR_PTR(-ENODEV);
  1725. return vport;
  1726. } else if (a[OVS_VPORT_ATTR_PORT_NO]) {
  1727. u32 port_no = nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]);
  1728. if (port_no >= DP_MAX_PORTS)
  1729. return ERR_PTR(-EFBIG);
  1730. dp = get_dp(net, ovs_header->dp_ifindex);
  1731. if (!dp)
  1732. return ERR_PTR(-ENODEV);
  1733. vport = ovs_vport_ovsl_rcu(dp, port_no);
  1734. if (!vport)
  1735. return ERR_PTR(-ENODEV);
  1736. return vport;
  1737. } else
  1738. return ERR_PTR(-EINVAL);
  1739. }
  1740. static unsigned int ovs_get_max_headroom(struct datapath *dp)
  1741. {
  1742. unsigned int dev_headroom, max_headroom = 0;
  1743. struct net_device *dev;
  1744. struct vport *vport;
  1745. int i;
  1746. for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
  1747. hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
  1748. lockdep_ovsl_is_held()) {
  1749. dev = vport->dev;
  1750. dev_headroom = netdev_get_fwd_headroom(dev);
  1751. if (dev_headroom > max_headroom)
  1752. max_headroom = dev_headroom;
  1753. }
  1754. }
  1755. return max_headroom;
  1756. }
  1757. /* Called with ovs_mutex */
  1758. static void ovs_update_headroom(struct datapath *dp, unsigned int new_headroom)
  1759. {
  1760. struct vport *vport;
  1761. int i;
  1762. dp->max_headroom = new_headroom;
  1763. for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
  1764. hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node,
  1765. lockdep_ovsl_is_held())
  1766. netdev_set_rx_headroom(vport->dev, new_headroom);
  1767. }
  1768. }
  1769. static int ovs_vport_cmd_new(struct sk_buff *skb, struct genl_info *info)
  1770. {
  1771. struct nlattr **a = info->attrs;
  1772. struct ovs_header *ovs_header = info->userhdr;
  1773. struct vport_parms parms;
  1774. struct sk_buff *reply;
  1775. struct vport *vport;
  1776. struct datapath *dp;
  1777. unsigned int new_headroom;
  1778. u32 port_no;
  1779. int err;
  1780. if (!a[OVS_VPORT_ATTR_NAME] || !a[OVS_VPORT_ATTR_TYPE] ||
  1781. !a[OVS_VPORT_ATTR_UPCALL_PID])
  1782. return -EINVAL;
  1783. if (a[OVS_VPORT_ATTR_IFINDEX])
  1784. return -EOPNOTSUPP;
  1785. port_no = a[OVS_VPORT_ATTR_PORT_NO]
  1786. ? nla_get_u32(a[OVS_VPORT_ATTR_PORT_NO]) : 0;
  1787. if (port_no >= DP_MAX_PORTS)
  1788. return -EFBIG;
  1789. reply = ovs_vport_cmd_alloc_info();
  1790. if (!reply)
  1791. return -ENOMEM;
  1792. ovs_lock();
  1793. restart:
  1794. dp = get_dp(sock_net(skb->sk), ovs_header->dp_ifindex);
  1795. err = -ENODEV;
  1796. if (!dp)
  1797. goto exit_unlock_free;
  1798. if (port_no) {
  1799. vport = ovs_vport_ovsl(dp, port_no);
  1800. err = -EBUSY;
  1801. if (vport)
  1802. goto exit_unlock_free;
  1803. } else {
  1804. for (port_no = 1; ; port_no++) {
  1805. if (port_no >= DP_MAX_PORTS) {
  1806. err = -EFBIG;
  1807. goto exit_unlock_free;
  1808. }
  1809. vport = ovs_vport_ovsl(dp, port_no);
  1810. if (!vport)
  1811. break;
  1812. }
  1813. }
  1814. parms.name = nla_data(a[OVS_VPORT_ATTR_NAME]);
  1815. parms.type = nla_get_u32(a[OVS_VPORT_ATTR_TYPE]);
  1816. parms.options = a[OVS_VPORT_ATTR_OPTIONS];
  1817. parms.dp = dp;
  1818. parms.port_no = port_no;
  1819. parms.upcall_portids = a[OVS_VPORT_ATTR_UPCALL_PID];
  1820. vport = new_vport(&parms);
  1821. err = PTR_ERR(vport);
  1822. if (IS_ERR(vport)) {
  1823. if (err == -EAGAIN)
  1824. goto restart;
  1825. goto exit_unlock_free;
  1826. }
  1827. err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
  1828. info->snd_portid, info->snd_seq, 0,
  1829. OVS_VPORT_CMD_NEW, GFP_KERNEL);
  1830. new_headroom = netdev_get_fwd_headroom(vport->dev);
  1831. if (new_headroom > dp->max_headroom)
  1832. ovs_update_headroom(dp, new_headroom);
  1833. else
  1834. netdev_set_rx_headroom(vport->dev, dp->max_headroom);
  1835. BUG_ON(err < 0);
  1836. ovs_unlock();
  1837. ovs_notify(&dp_vport_genl_family, reply, info);
  1838. return 0;
  1839. exit_unlock_free:
  1840. ovs_unlock();
  1841. kfree_skb(reply);
  1842. return err;
  1843. }
  1844. static int ovs_vport_cmd_set(struct sk_buff *skb, struct genl_info *info)
  1845. {
  1846. struct nlattr **a = info->attrs;
  1847. struct sk_buff *reply;
  1848. struct vport *vport;
  1849. int err;
  1850. reply = ovs_vport_cmd_alloc_info();
  1851. if (!reply)
  1852. return -ENOMEM;
  1853. ovs_lock();
  1854. vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
  1855. err = PTR_ERR(vport);
  1856. if (IS_ERR(vport))
  1857. goto exit_unlock_free;
  1858. if (a[OVS_VPORT_ATTR_TYPE] &&
  1859. nla_get_u32(a[OVS_VPORT_ATTR_TYPE]) != vport->ops->type) {
  1860. err = -EINVAL;
  1861. goto exit_unlock_free;
  1862. }
  1863. if (a[OVS_VPORT_ATTR_OPTIONS]) {
  1864. err = ovs_vport_set_options(vport, a[OVS_VPORT_ATTR_OPTIONS]);
  1865. if (err)
  1866. goto exit_unlock_free;
  1867. }
  1868. if (a[OVS_VPORT_ATTR_UPCALL_PID]) {
  1869. struct nlattr *ids = a[OVS_VPORT_ATTR_UPCALL_PID];
  1870. err = ovs_vport_set_upcall_portids(vport, ids);
  1871. if (err)
  1872. goto exit_unlock_free;
  1873. }
  1874. err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
  1875. info->snd_portid, info->snd_seq, 0,
  1876. OVS_VPORT_CMD_SET, GFP_KERNEL);
  1877. BUG_ON(err < 0);
  1878. ovs_unlock();
  1879. ovs_notify(&dp_vport_genl_family, reply, info);
  1880. return 0;
  1881. exit_unlock_free:
  1882. ovs_unlock();
  1883. kfree_skb(reply);
  1884. return err;
  1885. }
  1886. static int ovs_vport_cmd_del(struct sk_buff *skb, struct genl_info *info)
  1887. {
  1888. bool update_headroom = false;
  1889. struct nlattr **a = info->attrs;
  1890. struct sk_buff *reply;
  1891. struct datapath *dp;
  1892. struct vport *vport;
  1893. unsigned int new_headroom;
  1894. int err;
  1895. reply = ovs_vport_cmd_alloc_info();
  1896. if (!reply)
  1897. return -ENOMEM;
  1898. ovs_lock();
  1899. vport = lookup_vport(sock_net(skb->sk), info->userhdr, a);
  1900. err = PTR_ERR(vport);
  1901. if (IS_ERR(vport))
  1902. goto exit_unlock_free;
  1903. if (vport->port_no == OVSP_LOCAL) {
  1904. err = -EINVAL;
  1905. goto exit_unlock_free;
  1906. }
  1907. err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
  1908. info->snd_portid, info->snd_seq, 0,
  1909. OVS_VPORT_CMD_DEL, GFP_KERNEL);
  1910. BUG_ON(err < 0);
  1911. /* the vport deletion may trigger dp headroom update */
  1912. dp = vport->dp;
  1913. if (netdev_get_fwd_headroom(vport->dev) == dp->max_headroom)
  1914. update_headroom = true;
  1915. netdev_reset_rx_headroom(vport->dev);
  1916. ovs_dp_detach_port(vport);
  1917. if (update_headroom) {
  1918. new_headroom = ovs_get_max_headroom(dp);
  1919. if (new_headroom < dp->max_headroom)
  1920. ovs_update_headroom(dp, new_headroom);
  1921. }
  1922. ovs_unlock();
  1923. ovs_notify(&dp_vport_genl_family, reply, info);
  1924. return 0;
  1925. exit_unlock_free:
  1926. ovs_unlock();
  1927. kfree_skb(reply);
  1928. return err;
  1929. }
  1930. static int ovs_vport_cmd_get(struct sk_buff *skb, struct genl_info *info)
  1931. {
  1932. struct nlattr **a = info->attrs;
  1933. struct ovs_header *ovs_header = info->userhdr;
  1934. struct sk_buff *reply;
  1935. struct vport *vport;
  1936. int err;
  1937. reply = ovs_vport_cmd_alloc_info();
  1938. if (!reply)
  1939. return -ENOMEM;
  1940. rcu_read_lock();
  1941. vport = lookup_vport(sock_net(skb->sk), ovs_header, a);
  1942. err = PTR_ERR(vport);
  1943. if (IS_ERR(vport))
  1944. goto exit_unlock_free;
  1945. err = ovs_vport_cmd_fill_info(vport, reply, genl_info_net(info),
  1946. info->snd_portid, info->snd_seq, 0,
  1947. OVS_VPORT_CMD_GET, GFP_ATOMIC);
  1948. BUG_ON(err < 0);
  1949. rcu_read_unlock();
  1950. return genlmsg_reply(reply, info);
  1951. exit_unlock_free:
  1952. rcu_read_unlock();
  1953. kfree_skb(reply);
  1954. return err;
  1955. }
  1956. static int ovs_vport_cmd_dump(struct sk_buff *skb, struct netlink_callback *cb)
  1957. {
  1958. struct ovs_header *ovs_header = genlmsg_data(nlmsg_data(cb->nlh));
  1959. struct datapath *dp;
  1960. int bucket = cb->args[0], skip = cb->args[1];
  1961. int i, j = 0;
  1962. rcu_read_lock();
  1963. dp = get_dp_rcu(sock_net(skb->sk), ovs_header->dp_ifindex);
  1964. if (!dp) {
  1965. rcu_read_unlock();
  1966. return -ENODEV;
  1967. }
  1968. for (i = bucket; i < DP_VPORT_HASH_BUCKETS; i++) {
  1969. struct vport *vport;
  1970. j = 0;
  1971. hlist_for_each_entry_rcu(vport, &dp->ports[i], dp_hash_node) {
  1972. if (j >= skip &&
  1973. ovs_vport_cmd_fill_info(vport, skb,
  1974. sock_net(skb->sk),
  1975. NETLINK_CB(cb->skb).portid,
  1976. cb->nlh->nlmsg_seq,
  1977. NLM_F_MULTI,
  1978. OVS_VPORT_CMD_GET,
  1979. GFP_ATOMIC) < 0)
  1980. goto out;
  1981. j++;
  1982. }
  1983. skip = 0;
  1984. }
  1985. out:
  1986. rcu_read_unlock();
  1987. cb->args[0] = i;
  1988. cb->args[1] = j;
  1989. return skb->len;
  1990. }
  1991. static void ovs_dp_masks_rebalance(struct work_struct *work)
  1992. {
  1993. struct ovs_net *ovs_net = container_of(work, struct ovs_net,
  1994. masks_rebalance.work);
  1995. struct datapath *dp;
  1996. ovs_lock();
  1997. list_for_each_entry(dp, &ovs_net->dps, list_node)
  1998. ovs_flow_masks_rebalance(&dp->table);
  1999. ovs_unlock();
  2000. schedule_delayed_work(&ovs_net->masks_rebalance,
  2001. msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
  2002. }
  2003. static const struct nla_policy vport_policy[OVS_VPORT_ATTR_MAX + 1] = {
  2004. [OVS_VPORT_ATTR_NAME] = { .type = NLA_NUL_STRING, .len = IFNAMSIZ - 1 },
  2005. [OVS_VPORT_ATTR_STATS] = { .len = sizeof(struct ovs_vport_stats) },
  2006. [OVS_VPORT_ATTR_PORT_NO] = { .type = NLA_U32 },
  2007. [OVS_VPORT_ATTR_TYPE] = { .type = NLA_U32 },
  2008. [OVS_VPORT_ATTR_UPCALL_PID] = { .type = NLA_UNSPEC },
  2009. [OVS_VPORT_ATTR_OPTIONS] = { .type = NLA_NESTED },
  2010. [OVS_VPORT_ATTR_IFINDEX] = { .type = NLA_U32 },
  2011. [OVS_VPORT_ATTR_NETNSID] = { .type = NLA_S32 },
  2012. };
  2013. static const struct genl_small_ops dp_vport_genl_ops[] = {
  2014. { .cmd = OVS_VPORT_CMD_NEW,
  2015. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  2016. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  2017. .doit = ovs_vport_cmd_new
  2018. },
  2019. { .cmd = OVS_VPORT_CMD_DEL,
  2020. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  2021. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  2022. .doit = ovs_vport_cmd_del
  2023. },
  2024. { .cmd = OVS_VPORT_CMD_GET,
  2025. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  2026. .flags = 0, /* OK for unprivileged users. */
  2027. .doit = ovs_vport_cmd_get,
  2028. .dumpit = ovs_vport_cmd_dump
  2029. },
  2030. { .cmd = OVS_VPORT_CMD_SET,
  2031. .validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
  2032. .flags = GENL_UNS_ADMIN_PERM, /* Requires CAP_NET_ADMIN privilege. */
  2033. .doit = ovs_vport_cmd_set,
  2034. },
  2035. };
  2036. struct genl_family dp_vport_genl_family __ro_after_init = {
  2037. .hdrsize = sizeof(struct ovs_header),
  2038. .name = OVS_VPORT_FAMILY,
  2039. .version = OVS_VPORT_VERSION,
  2040. .maxattr = OVS_VPORT_ATTR_MAX,
  2041. .policy = vport_policy,
  2042. .netnsok = true,
  2043. .parallel_ops = true,
  2044. .small_ops = dp_vport_genl_ops,
  2045. .n_small_ops = ARRAY_SIZE(dp_vport_genl_ops),
  2046. .mcgrps = &ovs_dp_vport_multicast_group,
  2047. .n_mcgrps = 1,
  2048. .module = THIS_MODULE,
  2049. };
  2050. static struct genl_family * const dp_genl_families[] = {
  2051. &dp_datapath_genl_family,
  2052. &dp_vport_genl_family,
  2053. &dp_flow_genl_family,
  2054. &dp_packet_genl_family,
  2055. &dp_meter_genl_family,
  2056. #if IS_ENABLED(CONFIG_NETFILTER_CONNCOUNT)
  2057. &dp_ct_limit_genl_family,
  2058. #endif
  2059. };
  2060. static void dp_unregister_genl(int n_families)
  2061. {
  2062. int i;
  2063. for (i = 0; i < n_families; i++)
  2064. genl_unregister_family(dp_genl_families[i]);
  2065. }
  2066. static int __init dp_register_genl(void)
  2067. {
  2068. int err;
  2069. int i;
  2070. for (i = 0; i < ARRAY_SIZE(dp_genl_families); i++) {
  2071. err = genl_register_family(dp_genl_families[i]);
  2072. if (err)
  2073. goto error;
  2074. }
  2075. return 0;
  2076. error:
  2077. dp_unregister_genl(i);
  2078. return err;
  2079. }
  2080. static int __net_init ovs_init_net(struct net *net)
  2081. {
  2082. struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
  2083. int err;
  2084. INIT_LIST_HEAD(&ovs_net->dps);
  2085. INIT_WORK(&ovs_net->dp_notify_work, ovs_dp_notify_wq);
  2086. INIT_DELAYED_WORK(&ovs_net->masks_rebalance, ovs_dp_masks_rebalance);
  2087. err = ovs_ct_init(net);
  2088. if (err)
  2089. return err;
  2090. schedule_delayed_work(&ovs_net->masks_rebalance,
  2091. msecs_to_jiffies(DP_MASKS_REBALANCE_INTERVAL));
  2092. return 0;
  2093. }
  2094. static void __net_exit list_vports_from_net(struct net *net, struct net *dnet,
  2095. struct list_head *head)
  2096. {
  2097. struct ovs_net *ovs_net = net_generic(net, ovs_net_id);
  2098. struct datapath *dp;
  2099. list_for_each_entry(dp, &ovs_net->dps, list_node) {
  2100. int i;
  2101. for (i = 0; i < DP_VPORT_HASH_BUCKETS; i++) {
  2102. struct vport *vport;
  2103. hlist_for_each_entry(vport, &dp->ports[i], dp_hash_node) {
  2104. if (vport->ops->type != OVS_VPORT_TYPE_INTERNAL)
  2105. continue;
  2106. if (dev_net(vport->dev) == dnet)
  2107. list_add(&vport->detach_list, head);
  2108. }
  2109. }
  2110. }
  2111. }
  2112. static void __net_exit ovs_exit_net(struct net *dnet)
  2113. {
  2114. struct datapath *dp, *dp_next;
  2115. struct ovs_net *ovs_net = net_generic(dnet, ovs_net_id);
  2116. struct vport *vport, *vport_next;
  2117. struct net *net;
  2118. LIST_HEAD(head);
  2119. ovs_lock();
  2120. ovs_ct_exit(dnet);
  2121. list_for_each_entry_safe(dp, dp_next, &ovs_net->dps, list_node)
  2122. __dp_destroy(dp);
  2123. down_read(&net_rwsem);
  2124. for_each_net(net)
  2125. list_vports_from_net(net, dnet, &head);
  2126. up_read(&net_rwsem);
  2127. /* Detach all vports from given namespace. */
  2128. list_for_each_entry_safe(vport, vport_next, &head, detach_list) {
  2129. list_del(&vport->detach_list);
  2130. ovs_dp_detach_port(vport);
  2131. }
  2132. ovs_unlock();
  2133. cancel_delayed_work_sync(&ovs_net->masks_rebalance);
  2134. cancel_work_sync(&ovs_net->dp_notify_work);
  2135. }
  2136. static struct pernet_operations ovs_net_ops = {
  2137. .init = ovs_init_net,
  2138. .exit = ovs_exit_net,
  2139. .id = &ovs_net_id,
  2140. .size = sizeof(struct ovs_net),
  2141. };
  2142. static int __init dp_init(void)
  2143. {
  2144. int err;
  2145. BUILD_BUG_ON(sizeof(struct ovs_skb_cb) >
  2146. sizeof_field(struct sk_buff, cb));
  2147. pr_info("Open vSwitch switching datapath\n");
  2148. err = action_fifos_init();
  2149. if (err)
  2150. goto error;
  2151. err = ovs_internal_dev_rtnl_link_register();
  2152. if (err)
  2153. goto error_action_fifos_exit;
  2154. err = ovs_flow_init();
  2155. if (err)
  2156. goto error_unreg_rtnl_link;
  2157. err = ovs_vport_init();
  2158. if (err)
  2159. goto error_flow_exit;
  2160. err = register_pernet_device(&ovs_net_ops);
  2161. if (err)
  2162. goto error_vport_exit;
  2163. err = register_netdevice_notifier(&ovs_dp_device_notifier);
  2164. if (err)
  2165. goto error_netns_exit;
  2166. err = ovs_netdev_init();
  2167. if (err)
  2168. goto error_unreg_notifier;
  2169. err = dp_register_genl();
  2170. if (err < 0)
  2171. goto error_unreg_netdev;
  2172. return 0;
  2173. error_unreg_netdev:
  2174. ovs_netdev_exit();
  2175. error_unreg_notifier:
  2176. unregister_netdevice_notifier(&ovs_dp_device_notifier);
  2177. error_netns_exit:
  2178. unregister_pernet_device(&ovs_net_ops);
  2179. error_vport_exit:
  2180. ovs_vport_exit();
  2181. error_flow_exit:
  2182. ovs_flow_exit();
  2183. error_unreg_rtnl_link:
  2184. ovs_internal_dev_rtnl_link_unregister();
  2185. error_action_fifos_exit:
  2186. action_fifos_exit();
  2187. error:
  2188. return err;
  2189. }
  2190. static void dp_cleanup(void)
  2191. {
  2192. dp_unregister_genl(ARRAY_SIZE(dp_genl_families));
  2193. ovs_netdev_exit();
  2194. unregister_netdevice_notifier(&ovs_dp_device_notifier);
  2195. unregister_pernet_device(&ovs_net_ops);
  2196. rcu_barrier();
  2197. ovs_vport_exit();
  2198. ovs_flow_exit();
  2199. ovs_internal_dev_rtnl_link_unregister();
  2200. action_fifos_exit();
  2201. }
  2202. module_init(dp_init);
  2203. module_exit(dp_cleanup);
  2204. MODULE_DESCRIPTION("Open vSwitch switching datapath");
  2205. MODULE_LICENSE("GPL");
  2206. MODULE_ALIAS_GENL_FAMILY(OVS_DATAPATH_FAMILY);
  2207. MODULE_ALIAS_GENL_FAMILY(OVS_VPORT_FAMILY);
  2208. MODULE_ALIAS_GENL_FAMILY(OVS_FLOW_FAMILY);
  2209. MODULE_ALIAS_GENL_FAMILY(OVS_PACKET_FAMILY);
  2210. MODULE_ALIAS_GENL_FAMILY(OVS_METER_FAMILY);
  2211. MODULE_ALIAS_GENL_FAMILY(OVS_CT_LIMIT_FAMILY);