sch_ets.c 20 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837
  1. // SPDX-License-Identifier: GPL-2.0-only
  2. /*
  3. * net/sched/sch_ets.c Enhanced Transmission Selection scheduler
  4. *
  5. * Description
  6. * -----------
  7. *
  8. * The Enhanced Transmission Selection scheduler is a classful queuing
  9. * discipline that merges functionality of PRIO and DRR qdiscs in one scheduler.
  10. * ETS makes it easy to configure a set of strict and bandwidth-sharing bands to
  11. * implement the transmission selection described in 802.1Qaz.
  12. *
  13. * Although ETS is technically classful, it's not possible to add and remove
  14. * classes at will. Instead one specifies number of classes, how many are
  15. * PRIO-like and how many DRR-like, and quanta for the latter.
  16. *
  17. * Algorithm
  18. * ---------
  19. *
  20. * The strict classes, if any, are tried for traffic first: first band 0, if it
  21. * has no traffic then band 1, etc.
  22. *
  23. * When there is no traffic in any of the strict queues, the bandwidth-sharing
  24. * ones are tried next. Each band is assigned a deficit counter, initialized to
  25. * "quantum" of that band. ETS maintains a list of active bandwidth-sharing
  26. * bands whose qdiscs are non-empty. A packet is dequeued from the band at the
  27. * head of the list if the packet size is smaller or equal to the deficit
  28. * counter. If the counter is too small, it is increased by "quantum" and the
  29. * scheduler moves on to the next band in the active list.
  30. */
  31. #include <linux/module.h>
  32. #include <net/gen_stats.h>
  33. #include <net/netlink.h>
  34. #include <net/pkt_cls.h>
  35. #include <net/pkt_sched.h>
  36. #include <net/sch_generic.h>
  37. struct ets_class {
  38. struct list_head alist; /* In struct ets_sched.active. */
  39. struct Qdisc *qdisc;
  40. u32 quantum;
  41. u32 deficit;
  42. struct gnet_stats_basic_packed bstats;
  43. struct gnet_stats_queue qstats;
  44. };
  45. struct ets_sched {
  46. struct list_head active;
  47. struct tcf_proto __rcu *filter_list;
  48. struct tcf_block *block;
  49. unsigned int nbands;
  50. unsigned int nstrict;
  51. u8 prio2band[TC_PRIO_MAX + 1];
  52. struct ets_class classes[TCQ_ETS_MAX_BANDS];
  53. };
  54. static const struct nla_policy ets_policy[TCA_ETS_MAX + 1] = {
  55. [TCA_ETS_NBANDS] = { .type = NLA_U8 },
  56. [TCA_ETS_NSTRICT] = { .type = NLA_U8 },
  57. [TCA_ETS_QUANTA] = { .type = NLA_NESTED },
  58. [TCA_ETS_PRIOMAP] = { .type = NLA_NESTED },
  59. };
  60. static const struct nla_policy ets_priomap_policy[TCA_ETS_MAX + 1] = {
  61. [TCA_ETS_PRIOMAP_BAND] = { .type = NLA_U8 },
  62. };
  63. static const struct nla_policy ets_quanta_policy[TCA_ETS_MAX + 1] = {
  64. [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 },
  65. };
  66. static const struct nla_policy ets_class_policy[TCA_ETS_MAX + 1] = {
  67. [TCA_ETS_QUANTA_BAND] = { .type = NLA_U32 },
  68. };
  69. static int ets_quantum_parse(struct Qdisc *sch, const struct nlattr *attr,
  70. unsigned int *quantum,
  71. struct netlink_ext_ack *extack)
  72. {
  73. *quantum = nla_get_u32(attr);
  74. if (!*quantum) {
  75. NL_SET_ERR_MSG(extack, "ETS quantum cannot be zero");
  76. return -EINVAL;
  77. }
  78. return 0;
  79. }
  80. static struct ets_class *
  81. ets_class_from_arg(struct Qdisc *sch, unsigned long arg)
  82. {
  83. struct ets_sched *q = qdisc_priv(sch);
  84. return &q->classes[arg - 1];
  85. }
  86. static u32 ets_class_id(struct Qdisc *sch, const struct ets_class *cl)
  87. {
  88. struct ets_sched *q = qdisc_priv(sch);
  89. int band = cl - q->classes;
  90. return TC_H_MAKE(sch->handle, band + 1);
  91. }
  92. static void ets_offload_change(struct Qdisc *sch)
  93. {
  94. struct net_device *dev = qdisc_dev(sch);
  95. struct ets_sched *q = qdisc_priv(sch);
  96. struct tc_ets_qopt_offload qopt;
  97. unsigned int w_psum_prev = 0;
  98. unsigned int q_psum = 0;
  99. unsigned int q_sum = 0;
  100. unsigned int quantum;
  101. unsigned int w_psum;
  102. unsigned int weight;
  103. unsigned int i;
  104. if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
  105. return;
  106. qopt.command = TC_ETS_REPLACE;
  107. qopt.handle = sch->handle;
  108. qopt.parent = sch->parent;
  109. qopt.replace_params.bands = q->nbands;
  110. qopt.replace_params.qstats = &sch->qstats;
  111. memcpy(&qopt.replace_params.priomap,
  112. q->prio2band, sizeof(q->prio2band));
  113. for (i = 0; i < q->nbands; i++)
  114. q_sum += q->classes[i].quantum;
  115. for (i = 0; i < q->nbands; i++) {
  116. quantum = q->classes[i].quantum;
  117. q_psum += quantum;
  118. w_psum = quantum ? q_psum * 100 / q_sum : 0;
  119. weight = w_psum - w_psum_prev;
  120. w_psum_prev = w_psum;
  121. qopt.replace_params.quanta[i] = quantum;
  122. qopt.replace_params.weights[i] = weight;
  123. }
  124. dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt);
  125. }
  126. static void ets_offload_destroy(struct Qdisc *sch)
  127. {
  128. struct net_device *dev = qdisc_dev(sch);
  129. struct tc_ets_qopt_offload qopt;
  130. if (!tc_can_offload(dev) || !dev->netdev_ops->ndo_setup_tc)
  131. return;
  132. qopt.command = TC_ETS_DESTROY;
  133. qopt.handle = sch->handle;
  134. qopt.parent = sch->parent;
  135. dev->netdev_ops->ndo_setup_tc(dev, TC_SETUP_QDISC_ETS, &qopt);
  136. }
  137. static void ets_offload_graft(struct Qdisc *sch, struct Qdisc *new,
  138. struct Qdisc *old, unsigned long arg,
  139. struct netlink_ext_ack *extack)
  140. {
  141. struct net_device *dev = qdisc_dev(sch);
  142. struct tc_ets_qopt_offload qopt;
  143. qopt.command = TC_ETS_GRAFT;
  144. qopt.handle = sch->handle;
  145. qopt.parent = sch->parent;
  146. qopt.graft_params.band = arg - 1;
  147. qopt.graft_params.child_handle = new->handle;
  148. qdisc_offload_graft_helper(dev, sch, new, old, TC_SETUP_QDISC_ETS,
  149. &qopt, extack);
  150. }
  151. static int ets_offload_dump(struct Qdisc *sch)
  152. {
  153. struct tc_ets_qopt_offload qopt;
  154. qopt.command = TC_ETS_STATS;
  155. qopt.handle = sch->handle;
  156. qopt.parent = sch->parent;
  157. qopt.stats.bstats = &sch->bstats;
  158. qopt.stats.qstats = &sch->qstats;
  159. return qdisc_offload_dump_helper(sch, TC_SETUP_QDISC_ETS, &qopt);
  160. }
  161. static bool ets_class_is_strict(struct ets_sched *q, const struct ets_class *cl)
  162. {
  163. unsigned int band = cl - q->classes;
  164. return band < q->nstrict;
  165. }
  166. static int ets_class_change(struct Qdisc *sch, u32 classid, u32 parentid,
  167. struct nlattr **tca, unsigned long *arg,
  168. struct netlink_ext_ack *extack)
  169. {
  170. struct ets_class *cl = ets_class_from_arg(sch, *arg);
  171. struct ets_sched *q = qdisc_priv(sch);
  172. struct nlattr *opt = tca[TCA_OPTIONS];
  173. struct nlattr *tb[TCA_ETS_MAX + 1];
  174. unsigned int quantum;
  175. int err;
  176. /* Classes can be added and removed only through Qdisc_ops.change
  177. * interface.
  178. */
  179. if (!cl) {
  180. NL_SET_ERR_MSG(extack, "Fine-grained class addition and removal is not supported");
  181. return -EOPNOTSUPP;
  182. }
  183. if (!opt) {
  184. NL_SET_ERR_MSG(extack, "ETS options are required for this operation");
  185. return -EINVAL;
  186. }
  187. err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_class_policy, extack);
  188. if (err < 0)
  189. return err;
  190. if (!tb[TCA_ETS_QUANTA_BAND])
  191. /* Nothing to configure. */
  192. return 0;
  193. if (ets_class_is_strict(q, cl)) {
  194. NL_SET_ERR_MSG(extack, "Strict bands do not have a configurable quantum");
  195. return -EINVAL;
  196. }
  197. err = ets_quantum_parse(sch, tb[TCA_ETS_QUANTA_BAND], &quantum,
  198. extack);
  199. if (err)
  200. return err;
  201. sch_tree_lock(sch);
  202. cl->quantum = quantum;
  203. sch_tree_unlock(sch);
  204. ets_offload_change(sch);
  205. return 0;
  206. }
  207. static int ets_class_graft(struct Qdisc *sch, unsigned long arg,
  208. struct Qdisc *new, struct Qdisc **old,
  209. struct netlink_ext_ack *extack)
  210. {
  211. struct ets_class *cl = ets_class_from_arg(sch, arg);
  212. if (!new) {
  213. new = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
  214. ets_class_id(sch, cl), NULL);
  215. if (!new)
  216. new = &noop_qdisc;
  217. else
  218. qdisc_hash_add(new, true);
  219. }
  220. *old = qdisc_replace(sch, new, &cl->qdisc);
  221. ets_offload_graft(sch, new, *old, arg, extack);
  222. return 0;
  223. }
  224. static struct Qdisc *ets_class_leaf(struct Qdisc *sch, unsigned long arg)
  225. {
  226. struct ets_class *cl = ets_class_from_arg(sch, arg);
  227. return cl->qdisc;
  228. }
  229. static unsigned long ets_class_find(struct Qdisc *sch, u32 classid)
  230. {
  231. unsigned long band = TC_H_MIN(classid);
  232. struct ets_sched *q = qdisc_priv(sch);
  233. if (band - 1 >= q->nbands)
  234. return 0;
  235. return band;
  236. }
  237. static void ets_class_qlen_notify(struct Qdisc *sch, unsigned long arg)
  238. {
  239. struct ets_class *cl = ets_class_from_arg(sch, arg);
  240. struct ets_sched *q = qdisc_priv(sch);
  241. /* We get notified about zero-length child Qdiscs as well if they are
  242. * offloaded. Those aren't on the active list though, so don't attempt
  243. * to remove them.
  244. */
  245. if (!ets_class_is_strict(q, cl) && sch->q.qlen)
  246. list_del(&cl->alist);
  247. }
  248. static int ets_class_dump(struct Qdisc *sch, unsigned long arg,
  249. struct sk_buff *skb, struct tcmsg *tcm)
  250. {
  251. struct ets_class *cl = ets_class_from_arg(sch, arg);
  252. struct ets_sched *q = qdisc_priv(sch);
  253. struct nlattr *nest;
  254. tcm->tcm_parent = TC_H_ROOT;
  255. tcm->tcm_handle = ets_class_id(sch, cl);
  256. tcm->tcm_info = cl->qdisc->handle;
  257. nest = nla_nest_start_noflag(skb, TCA_OPTIONS);
  258. if (!nest)
  259. goto nla_put_failure;
  260. if (!ets_class_is_strict(q, cl)) {
  261. if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND, cl->quantum))
  262. goto nla_put_failure;
  263. }
  264. return nla_nest_end(skb, nest);
  265. nla_put_failure:
  266. nla_nest_cancel(skb, nest);
  267. return -EMSGSIZE;
  268. }
  269. static int ets_class_dump_stats(struct Qdisc *sch, unsigned long arg,
  270. struct gnet_dump *d)
  271. {
  272. struct ets_class *cl = ets_class_from_arg(sch, arg);
  273. struct Qdisc *cl_q = cl->qdisc;
  274. if (gnet_stats_copy_basic(qdisc_root_sleeping_running(sch),
  275. d, NULL, &cl_q->bstats) < 0 ||
  276. qdisc_qstats_copy(d, cl_q) < 0)
  277. return -1;
  278. return 0;
  279. }
  280. static void ets_qdisc_walk(struct Qdisc *sch, struct qdisc_walker *arg)
  281. {
  282. struct ets_sched *q = qdisc_priv(sch);
  283. int i;
  284. if (arg->stop)
  285. return;
  286. for (i = 0; i < q->nbands; i++) {
  287. if (arg->count < arg->skip) {
  288. arg->count++;
  289. continue;
  290. }
  291. if (arg->fn(sch, i + 1, arg) < 0) {
  292. arg->stop = 1;
  293. break;
  294. }
  295. arg->count++;
  296. }
  297. }
  298. static struct tcf_block *
  299. ets_qdisc_tcf_block(struct Qdisc *sch, unsigned long cl,
  300. struct netlink_ext_ack *extack)
  301. {
  302. struct ets_sched *q = qdisc_priv(sch);
  303. if (cl) {
  304. NL_SET_ERR_MSG(extack, "ETS classid must be zero");
  305. return NULL;
  306. }
  307. return q->block;
  308. }
  309. static unsigned long ets_qdisc_bind_tcf(struct Qdisc *sch, unsigned long parent,
  310. u32 classid)
  311. {
  312. return ets_class_find(sch, classid);
  313. }
  314. static void ets_qdisc_unbind_tcf(struct Qdisc *sch, unsigned long arg)
  315. {
  316. }
  317. static struct ets_class *ets_classify(struct sk_buff *skb, struct Qdisc *sch,
  318. int *qerr)
  319. {
  320. struct ets_sched *q = qdisc_priv(sch);
  321. u32 band = skb->priority;
  322. struct tcf_result res;
  323. struct tcf_proto *fl;
  324. int err;
  325. *qerr = NET_XMIT_SUCCESS | __NET_XMIT_BYPASS;
  326. if (TC_H_MAJ(skb->priority) != sch->handle) {
  327. fl = rcu_dereference_bh(q->filter_list);
  328. err = tcf_classify(skb, fl, &res, false);
  329. #ifdef CONFIG_NET_CLS_ACT
  330. switch (err) {
  331. case TC_ACT_STOLEN:
  332. case TC_ACT_QUEUED:
  333. case TC_ACT_TRAP:
  334. *qerr = NET_XMIT_SUCCESS | __NET_XMIT_STOLEN;
  335. fallthrough;
  336. case TC_ACT_SHOT:
  337. return NULL;
  338. }
  339. #endif
  340. if (!fl || err < 0) {
  341. if (TC_H_MAJ(band))
  342. band = 0;
  343. return &q->classes[q->prio2band[band & TC_PRIO_MAX]];
  344. }
  345. band = res.classid;
  346. }
  347. band = TC_H_MIN(band) - 1;
  348. if (band >= q->nbands)
  349. return &q->classes[q->prio2band[0]];
  350. return &q->classes[band];
  351. }
  352. static int ets_qdisc_enqueue(struct sk_buff *skb, struct Qdisc *sch,
  353. struct sk_buff **to_free)
  354. {
  355. unsigned int len = qdisc_pkt_len(skb);
  356. struct ets_sched *q = qdisc_priv(sch);
  357. struct ets_class *cl;
  358. int err = 0;
  359. bool first;
  360. cl = ets_classify(skb, sch, &err);
  361. if (!cl) {
  362. if (err & __NET_XMIT_BYPASS)
  363. qdisc_qstats_drop(sch);
  364. __qdisc_drop(skb, to_free);
  365. return err;
  366. }
  367. first = !cl->qdisc->q.qlen;
  368. err = qdisc_enqueue(skb, cl->qdisc, to_free);
  369. if (unlikely(err != NET_XMIT_SUCCESS)) {
  370. if (net_xmit_drop_count(err)) {
  371. cl->qstats.drops++;
  372. qdisc_qstats_drop(sch);
  373. }
  374. return err;
  375. }
  376. if (first && !ets_class_is_strict(q, cl)) {
  377. list_add_tail(&cl->alist, &q->active);
  378. cl->deficit = cl->quantum;
  379. }
  380. sch->qstats.backlog += len;
  381. sch->q.qlen++;
  382. return err;
  383. }
  384. static struct sk_buff *
  385. ets_qdisc_dequeue_skb(struct Qdisc *sch, struct sk_buff *skb)
  386. {
  387. qdisc_bstats_update(sch, skb);
  388. qdisc_qstats_backlog_dec(sch, skb);
  389. sch->q.qlen--;
  390. return skb;
  391. }
  392. static struct sk_buff *ets_qdisc_dequeue(struct Qdisc *sch)
  393. {
  394. struct ets_sched *q = qdisc_priv(sch);
  395. struct ets_class *cl;
  396. struct sk_buff *skb;
  397. unsigned int band;
  398. unsigned int len;
  399. while (1) {
  400. for (band = 0; band < q->nstrict; band++) {
  401. cl = &q->classes[band];
  402. skb = qdisc_dequeue_peeked(cl->qdisc);
  403. if (skb)
  404. return ets_qdisc_dequeue_skb(sch, skb);
  405. }
  406. if (list_empty(&q->active))
  407. goto out;
  408. cl = list_first_entry(&q->active, struct ets_class, alist);
  409. skb = cl->qdisc->ops->peek(cl->qdisc);
  410. if (!skb) {
  411. qdisc_warn_nonwc(__func__, cl->qdisc);
  412. goto out;
  413. }
  414. len = qdisc_pkt_len(skb);
  415. if (len <= cl->deficit) {
  416. cl->deficit -= len;
  417. skb = qdisc_dequeue_peeked(cl->qdisc);
  418. if (unlikely(!skb))
  419. goto out;
  420. if (cl->qdisc->q.qlen == 0)
  421. list_del(&cl->alist);
  422. return ets_qdisc_dequeue_skb(sch, skb);
  423. }
  424. cl->deficit += cl->quantum;
  425. list_move_tail(&cl->alist, &q->active);
  426. }
  427. out:
  428. return NULL;
  429. }
  430. static int ets_qdisc_priomap_parse(struct nlattr *priomap_attr,
  431. unsigned int nbands, u8 *priomap,
  432. struct netlink_ext_ack *extack)
  433. {
  434. const struct nlattr *attr;
  435. int prio = 0;
  436. u8 band;
  437. int rem;
  438. int err;
  439. err = __nla_validate_nested(priomap_attr, TCA_ETS_MAX,
  440. ets_priomap_policy, NL_VALIDATE_STRICT,
  441. extack);
  442. if (err)
  443. return err;
  444. nla_for_each_nested(attr, priomap_attr, rem) {
  445. switch (nla_type(attr)) {
  446. case TCA_ETS_PRIOMAP_BAND:
  447. if (prio > TC_PRIO_MAX) {
  448. NL_SET_ERR_MSG_MOD(extack, "Too many priorities in ETS priomap");
  449. return -EINVAL;
  450. }
  451. band = nla_get_u8(attr);
  452. if (band >= nbands) {
  453. NL_SET_ERR_MSG_MOD(extack, "Invalid band number in ETS priomap");
  454. return -EINVAL;
  455. }
  456. priomap[prio++] = band;
  457. break;
  458. default:
  459. WARN_ON_ONCE(1); /* Validate should have caught this. */
  460. return -EINVAL;
  461. }
  462. }
  463. return 0;
  464. }
  465. static int ets_qdisc_quanta_parse(struct Qdisc *sch, struct nlattr *quanta_attr,
  466. unsigned int nbands, unsigned int nstrict,
  467. unsigned int *quanta,
  468. struct netlink_ext_ack *extack)
  469. {
  470. const struct nlattr *attr;
  471. int band = nstrict;
  472. int rem;
  473. int err;
  474. err = __nla_validate_nested(quanta_attr, TCA_ETS_MAX,
  475. ets_quanta_policy, NL_VALIDATE_STRICT,
  476. extack);
  477. if (err < 0)
  478. return err;
  479. nla_for_each_nested(attr, quanta_attr, rem) {
  480. switch (nla_type(attr)) {
  481. case TCA_ETS_QUANTA_BAND:
  482. if (band >= nbands) {
  483. NL_SET_ERR_MSG_MOD(extack, "ETS quanta has more values than bands");
  484. return -EINVAL;
  485. }
  486. err = ets_quantum_parse(sch, attr, &quanta[band++],
  487. extack);
  488. if (err)
  489. return err;
  490. break;
  491. default:
  492. WARN_ON_ONCE(1); /* Validate should have caught this. */
  493. return -EINVAL;
  494. }
  495. }
  496. return 0;
  497. }
  498. static int ets_qdisc_change(struct Qdisc *sch, struct nlattr *opt,
  499. struct netlink_ext_ack *extack)
  500. {
  501. unsigned int quanta[TCQ_ETS_MAX_BANDS] = {0};
  502. struct Qdisc *queues[TCQ_ETS_MAX_BANDS];
  503. struct ets_sched *q = qdisc_priv(sch);
  504. struct nlattr *tb[TCA_ETS_MAX + 1];
  505. unsigned int oldbands = q->nbands;
  506. u8 priomap[TC_PRIO_MAX + 1];
  507. unsigned int nstrict = 0;
  508. unsigned int nbands;
  509. unsigned int i;
  510. int err;
  511. if (!opt) {
  512. NL_SET_ERR_MSG(extack, "ETS options are required for this operation");
  513. return -EINVAL;
  514. }
  515. err = nla_parse_nested(tb, TCA_ETS_MAX, opt, ets_policy, extack);
  516. if (err < 0)
  517. return err;
  518. if (!tb[TCA_ETS_NBANDS]) {
  519. NL_SET_ERR_MSG_MOD(extack, "Number of bands is a required argument");
  520. return -EINVAL;
  521. }
  522. nbands = nla_get_u8(tb[TCA_ETS_NBANDS]);
  523. if (nbands < 1 || nbands > TCQ_ETS_MAX_BANDS) {
  524. NL_SET_ERR_MSG_MOD(extack, "Invalid number of bands");
  525. return -EINVAL;
  526. }
  527. /* Unless overridden, traffic goes to the last band. */
  528. memset(priomap, nbands - 1, sizeof(priomap));
  529. if (tb[TCA_ETS_NSTRICT]) {
  530. nstrict = nla_get_u8(tb[TCA_ETS_NSTRICT]);
  531. if (nstrict > nbands) {
  532. NL_SET_ERR_MSG_MOD(extack, "Invalid number of strict bands");
  533. return -EINVAL;
  534. }
  535. }
  536. if (tb[TCA_ETS_PRIOMAP]) {
  537. err = ets_qdisc_priomap_parse(tb[TCA_ETS_PRIOMAP],
  538. nbands, priomap, extack);
  539. if (err)
  540. return err;
  541. }
  542. if (tb[TCA_ETS_QUANTA]) {
  543. err = ets_qdisc_quanta_parse(sch, tb[TCA_ETS_QUANTA],
  544. nbands, nstrict, quanta, extack);
  545. if (err)
  546. return err;
  547. }
  548. /* If there are more bands than strict + quanta provided, the remaining
  549. * ones are ETS with quantum of MTU. Initialize the missing values here.
  550. */
  551. for (i = nstrict; i < nbands; i++) {
  552. if (!quanta[i])
  553. quanta[i] = psched_mtu(qdisc_dev(sch));
  554. }
  555. /* Before commit, make sure we can allocate all new qdiscs */
  556. for (i = oldbands; i < nbands; i++) {
  557. queues[i] = qdisc_create_dflt(sch->dev_queue, &pfifo_qdisc_ops,
  558. ets_class_id(sch, &q->classes[i]),
  559. extack);
  560. if (!queues[i]) {
  561. while (i > oldbands)
  562. qdisc_put(queues[--i]);
  563. return -ENOMEM;
  564. }
  565. }
  566. sch_tree_lock(sch);
  567. q->nbands = nbands;
  568. for (i = nstrict; i < q->nstrict; i++) {
  569. INIT_LIST_HEAD(&q->classes[i].alist);
  570. if (q->classes[i].qdisc->q.qlen) {
  571. list_add_tail(&q->classes[i].alist, &q->active);
  572. q->classes[i].deficit = quanta[i];
  573. }
  574. }
  575. for (i = q->nbands; i < oldbands; i++) {
  576. if (i >= q->nstrict && q->classes[i].qdisc->q.qlen)
  577. list_del(&q->classes[i].alist);
  578. qdisc_tree_flush_backlog(q->classes[i].qdisc);
  579. }
  580. q->nstrict = nstrict;
  581. memcpy(q->prio2band, priomap, sizeof(priomap));
  582. for (i = 0; i < q->nbands; i++)
  583. q->classes[i].quantum = quanta[i];
  584. for (i = oldbands; i < q->nbands; i++) {
  585. q->classes[i].qdisc = queues[i];
  586. if (q->classes[i].qdisc != &noop_qdisc)
  587. qdisc_hash_add(q->classes[i].qdisc, true);
  588. }
  589. sch_tree_unlock(sch);
  590. ets_offload_change(sch);
  591. for (i = q->nbands; i < oldbands; i++) {
  592. qdisc_put(q->classes[i].qdisc);
  593. memset(&q->classes[i], 0, sizeof(q->classes[i]));
  594. }
  595. return 0;
  596. }
  597. static int ets_qdisc_init(struct Qdisc *sch, struct nlattr *opt,
  598. struct netlink_ext_ack *extack)
  599. {
  600. struct ets_sched *q = qdisc_priv(sch);
  601. int err;
  602. if (!opt)
  603. return -EINVAL;
  604. err = tcf_block_get(&q->block, &q->filter_list, sch, extack);
  605. if (err)
  606. return err;
  607. INIT_LIST_HEAD(&q->active);
  608. return ets_qdisc_change(sch, opt, extack);
  609. }
  610. static void ets_qdisc_reset(struct Qdisc *sch)
  611. {
  612. struct ets_sched *q = qdisc_priv(sch);
  613. int band;
  614. for (band = q->nstrict; band < q->nbands; band++) {
  615. if (q->classes[band].qdisc->q.qlen)
  616. list_del(&q->classes[band].alist);
  617. }
  618. for (band = 0; band < q->nbands; band++)
  619. qdisc_reset(q->classes[band].qdisc);
  620. sch->qstats.backlog = 0;
  621. sch->q.qlen = 0;
  622. }
  623. static void ets_qdisc_destroy(struct Qdisc *sch)
  624. {
  625. struct ets_sched *q = qdisc_priv(sch);
  626. int band;
  627. ets_offload_destroy(sch);
  628. tcf_block_put(q->block);
  629. for (band = 0; band < q->nbands; band++)
  630. qdisc_put(q->classes[band].qdisc);
  631. }
  632. static int ets_qdisc_dump(struct Qdisc *sch, struct sk_buff *skb)
  633. {
  634. struct ets_sched *q = qdisc_priv(sch);
  635. struct nlattr *opts;
  636. struct nlattr *nest;
  637. int band;
  638. int prio;
  639. int err;
  640. err = ets_offload_dump(sch);
  641. if (err)
  642. return err;
  643. opts = nla_nest_start_noflag(skb, TCA_OPTIONS);
  644. if (!opts)
  645. goto nla_err;
  646. if (nla_put_u8(skb, TCA_ETS_NBANDS, q->nbands))
  647. goto nla_err;
  648. if (q->nstrict &&
  649. nla_put_u8(skb, TCA_ETS_NSTRICT, q->nstrict))
  650. goto nla_err;
  651. if (q->nbands > q->nstrict) {
  652. nest = nla_nest_start(skb, TCA_ETS_QUANTA);
  653. if (!nest)
  654. goto nla_err;
  655. for (band = q->nstrict; band < q->nbands; band++) {
  656. if (nla_put_u32(skb, TCA_ETS_QUANTA_BAND,
  657. q->classes[band].quantum))
  658. goto nla_err;
  659. }
  660. nla_nest_end(skb, nest);
  661. }
  662. nest = nla_nest_start(skb, TCA_ETS_PRIOMAP);
  663. if (!nest)
  664. goto nla_err;
  665. for (prio = 0; prio <= TC_PRIO_MAX; prio++) {
  666. if (nla_put_u8(skb, TCA_ETS_PRIOMAP_BAND, q->prio2band[prio]))
  667. goto nla_err;
  668. }
  669. nla_nest_end(skb, nest);
  670. return nla_nest_end(skb, opts);
  671. nla_err:
  672. nla_nest_cancel(skb, opts);
  673. return -EMSGSIZE;
  674. }
  675. static const struct Qdisc_class_ops ets_class_ops = {
  676. .change = ets_class_change,
  677. .graft = ets_class_graft,
  678. .leaf = ets_class_leaf,
  679. .find = ets_class_find,
  680. .qlen_notify = ets_class_qlen_notify,
  681. .dump = ets_class_dump,
  682. .dump_stats = ets_class_dump_stats,
  683. .walk = ets_qdisc_walk,
  684. .tcf_block = ets_qdisc_tcf_block,
  685. .bind_tcf = ets_qdisc_bind_tcf,
  686. .unbind_tcf = ets_qdisc_unbind_tcf,
  687. };
  688. static struct Qdisc_ops ets_qdisc_ops __read_mostly = {
  689. .cl_ops = &ets_class_ops,
  690. .id = "ets",
  691. .priv_size = sizeof(struct ets_sched),
  692. .enqueue = ets_qdisc_enqueue,
  693. .dequeue = ets_qdisc_dequeue,
  694. .peek = qdisc_peek_dequeued,
  695. .change = ets_qdisc_change,
  696. .init = ets_qdisc_init,
  697. .reset = ets_qdisc_reset,
  698. .destroy = ets_qdisc_destroy,
  699. .dump = ets_qdisc_dump,
  700. .owner = THIS_MODULE,
  701. };
  702. static int __init ets_init(void)
  703. {
  704. return register_qdisc(&ets_qdisc_ops);
  705. }
  706. static void __exit ets_exit(void)
  707. {
  708. unregister_qdisc(&ets_qdisc_ops);
  709. }
  710. module_init(ets_init);
  711. module_exit(ets_exit);
  712. MODULE_LICENSE("GPL");