sch_generic.c 34 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455565758596061626364656667686970717273747576777879808182838485868788899091929394959697989910010110210310410510610710810911011111211311411511611711811912012112212312412512612712812913013113213313413513613713813914014114214314414514614714814915015115215315415515615715815916016116216316416516616716816917017117217317417517617717817918018118218318418518618718818919019119219319419519619719819920020120220320420520620720820921021121221321421521621721821922022122222322422522622722822923023123223323423523623723823924024124224324424524624724824925025125225325425525625725825926026126226326426526626726826927027127227327427527627727827928028128228328428528628728828929029129229329429529629729829930030130230330430530630730830931031131231331431531631731831932032132232332432532632732832933033133233333433533633733833934034134234334434534634734834935035135235335435535635735835936036136236336436536636736836937037137237337437537637737837938038138238338438538638738838939039139239339439539639739839940040140240340440540640740840941041141241341441541641741841942042142242342442542642742842943043143243343443543643743843944044144244344444544644744844945045145245345445545645745845946046146246346446546646746846947047147247347447547647747847948048148248348448548648748848949049149249349449549649749849950050150250350450550650750850951051151251351451551651751851952052152252352452552652752852953053153253353453553653753853954054154254354454554654754854955055155255355455555655755855956056156256356456556656756856957057157257357457557657757857958058158258358458558658758858959059159259359459559659759859960060160260360460560660760860961061161261361461561661761861962062162262362462562662762862963063163263363463563663763863964064164264364464564664764864965065165265365465565665765865966066166266366466566666766866967067167267367467567667767867968068168268368468568668768868969069169269369469569669769869970070170270370470570670770870971071171271371471571671771871972072172272372472572672772872973073173273373473573673773873974074174274374474574674774874975075175275375475575675775875976076176276376476576676776876977077177277377477577677777877978078178278378478578678778878979079179279379479579679779879980080180280380480580680780880981081181281381481581681781881982082182282382482582682782882983083183283383483583683783883984084184284384484584684784884985085185285385485585685785885986086186286386486586686786886987087187287387487587687787887988088188288388488588688788888989089189289389489589689789889990090190290390490590690790890991091191291391491591691791891992092192292392492592692792892993093193293393493593693793893994094194294394494594694794894995095195295395495595695795895996096196296396496596696796896997097197297397497597697797897998098198298398498598698798898999099199299399499599699799899910001001100210031004100510061007100810091010101110121013101410151016101710181019102010211022102310241025102610271028102910301031103210331034103510361037103810391040104110421043104410451046104710481049105010511052105310541055105610571058105910601061106210631064106510661067106810691070107110721073107410751076107710781079108010811082108310841085108610871088108910901091109210931094109510961097109810991100110111021103110411051106110711081109111011111112111311141115111611171118111911201121112211231124112511261127112811291130113111321133113411351136113711381139114011411142114311441145114611471148114911501151115211531154115511561157115811591160116111621163116411651166116711681169117011711172117311741175117611771178117911801181118211831184118511861187118811891190119111921193119411951196119711981199120012011202120312041205120612071208120912101211121212131214121512161217121812191220122112221223122412251226122712281229123012311232123312341235123612371238123912401241124212431244124512461247124812491250125112521253125412551256125712581259126012611262126312641265126612671268126912701271127212731274127512761277127812791280128112821283128412851286128712881289129012911292129312941295129612971298129913001301130213031304130513061307130813091310131113121313131413151316131713181319132013211322132313241325132613271328132913301331133213331334133513361337133813391340134113421343134413451346134713481349135013511352135313541355135613571358135913601361136213631364136513661367136813691370137113721373137413751376137713781379138013811382138313841385138613871388138913901391139213931394139513961397139813991400140114021403140414051406140714081409141014111412141314141415141614171418141914201421142214231424142514261427142814291430143114321433143414351436143714381439144014411442144314441445144614471448144914501451145214531454145514561457145814591460146114621463146414651466146714681469147014711472
  1. // SPDX-License-Identifier: GPL-2.0-or-later
  2. /*
  3. * net/sched/sch_generic.c Generic packet scheduler routines.
  4. *
  5. * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
  6. * Jamal Hadi Salim, <hadi@cyberus.ca> 990601
  7. * - Ingress support
  8. */
  9. #include <linux/bitops.h>
  10. #include <linux/module.h>
  11. #include <linux/types.h>
  12. #include <linux/kernel.h>
  13. #include <linux/sched.h>
  14. #include <linux/string.h>
  15. #include <linux/errno.h>
  16. #include <linux/netdevice.h>
  17. #include <linux/skbuff.h>
  18. #include <linux/rtnetlink.h>
  19. #include <linux/init.h>
  20. #include <linux/rcupdate.h>
  21. #include <linux/list.h>
  22. #include <linux/slab.h>
  23. #include <linux/if_vlan.h>
  24. #include <linux/skb_array.h>
  25. #include <linux/if_macvlan.h>
  26. #include <net/sch_generic.h>
  27. #include <net/pkt_sched.h>
  28. #include <net/dst.h>
  29. #include <trace/events/qdisc.h>
  30. #include <trace/events/net.h>
  31. #include <net/xfrm.h>
  32. /* Qdisc to use by default */
  33. const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
  34. EXPORT_SYMBOL(default_qdisc_ops);
  35. static void qdisc_maybe_clear_missed(struct Qdisc *q,
  36. const struct netdev_queue *txq)
  37. {
  38. clear_bit(__QDISC_STATE_MISSED, &q->state);
  39. /* Make sure the below netif_xmit_frozen_or_stopped()
  40. * checking happens after clearing STATE_MISSED.
  41. */
  42. smp_mb__after_atomic();
  43. /* Checking netif_xmit_frozen_or_stopped() again to
  44. * make sure STATE_MISSED is set if the STATE_MISSED
  45. * set by netif_tx_wake_queue()'s rescheduling of
  46. * net_tx_action() is cleared by the above clear_bit().
  47. */
  48. if (!netif_xmit_frozen_or_stopped(txq))
  49. set_bit(__QDISC_STATE_MISSED, &q->state);
  50. }
  51. /* Main transmission queue. */
  52. /* Modifications to data participating in scheduling must be protected with
  53. * qdisc_lock(qdisc) spinlock.
  54. *
  55. * The idea is the following:
  56. * - enqueue, dequeue are serialized via qdisc root lock
  57. * - ingress filtering is also serialized via qdisc root lock
  58. * - updates to tree and tree walking are only done under the rtnl mutex.
  59. */
  60. #define SKB_XOFF_MAGIC ((struct sk_buff *)1UL)
  61. static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
  62. {
  63. const struct netdev_queue *txq = q->dev_queue;
  64. spinlock_t *lock = NULL;
  65. struct sk_buff *skb;
  66. if (q->flags & TCQ_F_NOLOCK) {
  67. lock = qdisc_lock(q);
  68. spin_lock(lock);
  69. }
  70. skb = skb_peek(&q->skb_bad_txq);
  71. if (skb) {
  72. /* check the reason of requeuing without tx lock first */
  73. txq = skb_get_tx_queue(txq->dev, skb);
  74. if (!netif_xmit_frozen_or_stopped(txq)) {
  75. skb = __skb_dequeue(&q->skb_bad_txq);
  76. if (qdisc_is_percpu_stats(q)) {
  77. qdisc_qstats_cpu_backlog_dec(q, skb);
  78. qdisc_qstats_cpu_qlen_dec(q);
  79. } else {
  80. qdisc_qstats_backlog_dec(q, skb);
  81. q->q.qlen--;
  82. }
  83. } else {
  84. skb = SKB_XOFF_MAGIC;
  85. qdisc_maybe_clear_missed(q, txq);
  86. }
  87. }
  88. if (lock)
  89. spin_unlock(lock);
  90. return skb;
  91. }
  92. static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
  93. {
  94. struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
  95. if (unlikely(skb))
  96. skb = __skb_dequeue_bad_txq(q);
  97. return skb;
  98. }
  99. static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
  100. struct sk_buff *skb)
  101. {
  102. spinlock_t *lock = NULL;
  103. if (q->flags & TCQ_F_NOLOCK) {
  104. lock = qdisc_lock(q);
  105. spin_lock(lock);
  106. }
  107. __skb_queue_tail(&q->skb_bad_txq, skb);
  108. if (qdisc_is_percpu_stats(q)) {
  109. qdisc_qstats_cpu_backlog_inc(q, skb);
  110. qdisc_qstats_cpu_qlen_inc(q);
  111. } else {
  112. qdisc_qstats_backlog_inc(q, skb);
  113. q->q.qlen++;
  114. }
  115. if (lock)
  116. spin_unlock(lock);
  117. }
  118. static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
  119. {
  120. spinlock_t *lock = NULL;
  121. if (q->flags & TCQ_F_NOLOCK) {
  122. lock = qdisc_lock(q);
  123. spin_lock(lock);
  124. }
  125. while (skb) {
  126. struct sk_buff *next = skb->next;
  127. __skb_queue_tail(&q->gso_skb, skb);
  128. /* it's still part of the queue */
  129. if (qdisc_is_percpu_stats(q)) {
  130. qdisc_qstats_cpu_requeues_inc(q);
  131. qdisc_qstats_cpu_backlog_inc(q, skb);
  132. qdisc_qstats_cpu_qlen_inc(q);
  133. } else {
  134. q->qstats.requeues++;
  135. qdisc_qstats_backlog_inc(q, skb);
  136. q->q.qlen++;
  137. }
  138. skb = next;
  139. }
  140. if (lock)
  141. spin_unlock(lock);
  142. __netif_schedule(q);
  143. }
  144. static void try_bulk_dequeue_skb(struct Qdisc *q,
  145. struct sk_buff *skb,
  146. const struct netdev_queue *txq,
  147. int *packets)
  148. {
  149. int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
  150. while (bytelimit > 0) {
  151. struct sk_buff *nskb = q->dequeue(q);
  152. if (!nskb)
  153. break;
  154. bytelimit -= nskb->len; /* covers GSO len */
  155. skb->next = nskb;
  156. skb = nskb;
  157. (*packets)++; /* GSO counts as one pkt */
  158. }
  159. skb_mark_not_on_list(skb);
  160. }
  161. /* This variant of try_bulk_dequeue_skb() makes sure
  162. * all skbs in the chain are for the same txq
  163. */
  164. static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
  165. struct sk_buff *skb,
  166. int *packets)
  167. {
  168. int mapping = skb_get_queue_mapping(skb);
  169. struct sk_buff *nskb;
  170. int cnt = 0;
  171. do {
  172. nskb = q->dequeue(q);
  173. if (!nskb)
  174. break;
  175. if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
  176. qdisc_enqueue_skb_bad_txq(q, nskb);
  177. break;
  178. }
  179. skb->next = nskb;
  180. skb = nskb;
  181. } while (++cnt < 8);
  182. (*packets) += cnt;
  183. skb_mark_not_on_list(skb);
  184. }
  185. /* Note that dequeue_skb can possibly return a SKB list (via skb->next).
  186. * A requeued skb (via q->gso_skb) can also be a SKB list.
  187. */
  188. static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
  189. int *packets)
  190. {
  191. const struct netdev_queue *txq = q->dev_queue;
  192. struct sk_buff *skb = NULL;
  193. *packets = 1;
  194. if (unlikely(!skb_queue_empty(&q->gso_skb))) {
  195. spinlock_t *lock = NULL;
  196. if (q->flags & TCQ_F_NOLOCK) {
  197. lock = qdisc_lock(q);
  198. spin_lock(lock);
  199. }
  200. skb = skb_peek(&q->gso_skb);
  201. /* skb may be null if another cpu pulls gso_skb off in between
  202. * empty check and lock.
  203. */
  204. if (!skb) {
  205. if (lock)
  206. spin_unlock(lock);
  207. goto validate;
  208. }
  209. /* skb in gso_skb were already validated */
  210. *validate = false;
  211. if (xfrm_offload(skb))
  212. *validate = true;
  213. /* check the reason of requeuing without tx lock first */
  214. txq = skb_get_tx_queue(txq->dev, skb);
  215. if (!netif_xmit_frozen_or_stopped(txq)) {
  216. skb = __skb_dequeue(&q->gso_skb);
  217. if (qdisc_is_percpu_stats(q)) {
  218. qdisc_qstats_cpu_backlog_dec(q, skb);
  219. qdisc_qstats_cpu_qlen_dec(q);
  220. } else {
  221. qdisc_qstats_backlog_dec(q, skb);
  222. q->q.qlen--;
  223. }
  224. } else {
  225. skb = NULL;
  226. qdisc_maybe_clear_missed(q, txq);
  227. }
  228. if (lock)
  229. spin_unlock(lock);
  230. goto trace;
  231. }
  232. validate:
  233. *validate = true;
  234. if ((q->flags & TCQ_F_ONETXQUEUE) &&
  235. netif_xmit_frozen_or_stopped(txq)) {
  236. qdisc_maybe_clear_missed(q, txq);
  237. return skb;
  238. }
  239. skb = qdisc_dequeue_skb_bad_txq(q);
  240. if (unlikely(skb)) {
  241. if (skb == SKB_XOFF_MAGIC)
  242. return NULL;
  243. goto bulk;
  244. }
  245. skb = q->dequeue(q);
  246. if (skb) {
  247. bulk:
  248. if (qdisc_may_bulk(q))
  249. try_bulk_dequeue_skb(q, skb, txq, packets);
  250. else
  251. try_bulk_dequeue_skb_slow(q, skb, packets);
  252. }
  253. trace:
  254. trace_qdisc_dequeue(q, txq, *packets, skb);
  255. return skb;
  256. }
  257. /*
  258. * Transmit possibly several skbs, and handle the return status as
  259. * required. Owning running seqcount bit guarantees that
  260. * only one CPU can execute this function.
  261. *
  262. * Returns to the caller:
  263. * false - hardware queue frozen backoff
  264. * true - feel free to send more pkts
  265. */
  266. bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
  267. struct net_device *dev, struct netdev_queue *txq,
  268. spinlock_t *root_lock, bool validate)
  269. {
  270. int ret = NETDEV_TX_BUSY;
  271. bool again = false;
  272. /* And release qdisc */
  273. if (root_lock)
  274. spin_unlock(root_lock);
  275. /* Note that we validate skb (GSO, checksum, ...) outside of locks */
  276. if (validate)
  277. skb = validate_xmit_skb_list(skb, dev, &again);
  278. #ifdef CONFIG_XFRM_OFFLOAD
  279. if (unlikely(again)) {
  280. if (root_lock)
  281. spin_lock(root_lock);
  282. dev_requeue_skb(skb, q);
  283. return false;
  284. }
  285. #endif
  286. if (likely(skb)) {
  287. HARD_TX_LOCK(dev, txq, smp_processor_id());
  288. if (!netif_xmit_frozen_or_stopped(txq))
  289. skb = dev_hard_start_xmit(skb, dev, txq, &ret);
  290. else
  291. qdisc_maybe_clear_missed(q, txq);
  292. HARD_TX_UNLOCK(dev, txq);
  293. } else {
  294. if (root_lock)
  295. spin_lock(root_lock);
  296. return true;
  297. }
  298. if (root_lock)
  299. spin_lock(root_lock);
  300. if (!dev_xmit_complete(ret)) {
  301. /* Driver returned NETDEV_TX_BUSY - requeue skb */
  302. if (unlikely(ret != NETDEV_TX_BUSY))
  303. net_warn_ratelimited("BUG %s code %d qlen %d\n",
  304. dev->name, ret, q->q.qlen);
  305. dev_requeue_skb(skb, q);
  306. return false;
  307. }
  308. return true;
  309. }
  310. /*
  311. * NOTE: Called under qdisc_lock(q) with locally disabled BH.
  312. *
  313. * running seqcount guarantees only one CPU can process
  314. * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
  315. * this queue.
  316. *
  317. * netif_tx_lock serializes accesses to device driver.
  318. *
  319. * qdisc_lock(q) and netif_tx_lock are mutually exclusive,
  320. * if one is grabbed, another must be free.
  321. *
  322. * Note, that this procedure can be called by a watchdog timer
  323. *
  324. * Returns to the caller:
  325. * 0 - queue is empty or throttled.
  326. * >0 - queue is not empty.
  327. *
  328. */
  329. static inline bool qdisc_restart(struct Qdisc *q, int *packets)
  330. {
  331. spinlock_t *root_lock = NULL;
  332. struct netdev_queue *txq;
  333. struct net_device *dev;
  334. struct sk_buff *skb;
  335. bool validate;
  336. /* Dequeue packet */
  337. skb = dequeue_skb(q, &validate, packets);
  338. if (unlikely(!skb))
  339. return false;
  340. if (!(q->flags & TCQ_F_NOLOCK))
  341. root_lock = qdisc_lock(q);
  342. dev = qdisc_dev(q);
  343. txq = skb_get_tx_queue(dev, skb);
  344. return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
  345. }
  346. void __qdisc_run(struct Qdisc *q)
  347. {
  348. int quota = dev_tx_weight;
  349. int packets;
  350. while (qdisc_restart(q, &packets)) {
  351. quota -= packets;
  352. if (quota <= 0) {
  353. __netif_schedule(q);
  354. break;
  355. }
  356. }
  357. }
  358. unsigned long dev_trans_start(struct net_device *dev)
  359. {
  360. unsigned long val, res;
  361. unsigned int i;
  362. if (is_vlan_dev(dev))
  363. dev = vlan_dev_real_dev(dev);
  364. else if (netif_is_macvlan(dev))
  365. dev = macvlan_dev_real_dev(dev);
  366. res = netdev_get_tx_queue(dev, 0)->trans_start;
  367. for (i = 1; i < dev->num_tx_queues; i++) {
  368. val = netdev_get_tx_queue(dev, i)->trans_start;
  369. if (val && time_after(val, res))
  370. res = val;
  371. }
  372. return res;
  373. }
  374. EXPORT_SYMBOL(dev_trans_start);
  375. static void dev_watchdog(struct timer_list *t)
  376. {
  377. struct net_device *dev = from_timer(dev, t, watchdog_timer);
  378. netif_tx_lock(dev);
  379. if (!qdisc_tx_is_noop(dev)) {
  380. if (netif_device_present(dev) &&
  381. netif_running(dev) &&
  382. netif_carrier_ok(dev)) {
  383. int some_queue_timedout = 0;
  384. unsigned int i;
  385. unsigned long trans_start;
  386. for (i = 0; i < dev->num_tx_queues; i++) {
  387. struct netdev_queue *txq;
  388. txq = netdev_get_tx_queue(dev, i);
  389. trans_start = txq->trans_start;
  390. if (netif_xmit_stopped(txq) &&
  391. time_after(jiffies, (trans_start +
  392. dev->watchdog_timeo))) {
  393. some_queue_timedout = 1;
  394. txq->trans_timeout++;
  395. break;
  396. }
  397. }
  398. if (some_queue_timedout) {
  399. trace_net_dev_xmit_timeout(dev, i);
  400. WARN_ONCE(1, KERN_INFO "NETDEV WATCHDOG: %s (%s): transmit queue %u timed out\n",
  401. dev->name, netdev_drivername(dev), i);
  402. dev->netdev_ops->ndo_tx_timeout(dev, i);
  403. }
  404. if (!mod_timer(&dev->watchdog_timer,
  405. round_jiffies(jiffies +
  406. dev->watchdog_timeo)))
  407. dev_hold(dev);
  408. }
  409. }
  410. netif_tx_unlock(dev);
  411. dev_put(dev);
  412. }
  413. void __netdev_watchdog_up(struct net_device *dev)
  414. {
  415. if (dev->netdev_ops->ndo_tx_timeout) {
  416. if (dev->watchdog_timeo <= 0)
  417. dev->watchdog_timeo = 5*HZ;
  418. if (!mod_timer(&dev->watchdog_timer,
  419. round_jiffies(jiffies + dev->watchdog_timeo)))
  420. dev_hold(dev);
  421. }
  422. }
  423. EXPORT_SYMBOL_GPL(__netdev_watchdog_up);
  424. static void dev_watchdog_up(struct net_device *dev)
  425. {
  426. __netdev_watchdog_up(dev);
  427. }
  428. static void dev_watchdog_down(struct net_device *dev)
  429. {
  430. netif_tx_lock_bh(dev);
  431. if (del_timer(&dev->watchdog_timer))
  432. dev_put(dev);
  433. netif_tx_unlock_bh(dev);
  434. }
  435. /**
  436. * netif_carrier_on - set carrier
  437. * @dev: network device
  438. *
  439. * Device has detected acquisition of carrier.
  440. */
  441. void netif_carrier_on(struct net_device *dev)
  442. {
  443. if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
  444. if (dev->reg_state == NETREG_UNINITIALIZED)
  445. return;
  446. atomic_inc(&dev->carrier_up_count);
  447. linkwatch_fire_event(dev);
  448. if (netif_running(dev))
  449. __netdev_watchdog_up(dev);
  450. }
  451. }
  452. EXPORT_SYMBOL(netif_carrier_on);
  453. /**
  454. * netif_carrier_off - clear carrier
  455. * @dev: network device
  456. *
  457. * Device has detected loss of carrier.
  458. */
  459. void netif_carrier_off(struct net_device *dev)
  460. {
  461. if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
  462. if (dev->reg_state == NETREG_UNINITIALIZED)
  463. return;
  464. atomic_inc(&dev->carrier_down_count);
  465. linkwatch_fire_event(dev);
  466. }
  467. }
  468. EXPORT_SYMBOL(netif_carrier_off);
  469. /* "NOOP" scheduler: the best scheduler, recommended for all interfaces
  470. under all circumstances. It is difficult to invent anything faster or
  471. cheaper.
  472. */
  473. static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
  474. struct sk_buff **to_free)
  475. {
  476. __qdisc_drop(skb, to_free);
  477. return NET_XMIT_CN;
  478. }
  479. static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
  480. {
  481. return NULL;
  482. }
  483. struct Qdisc_ops noop_qdisc_ops __read_mostly = {
  484. .id = "noop",
  485. .priv_size = 0,
  486. .enqueue = noop_enqueue,
  487. .dequeue = noop_dequeue,
  488. .peek = noop_dequeue,
  489. .owner = THIS_MODULE,
  490. };
  491. static struct netdev_queue noop_netdev_queue = {
  492. RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
  493. .qdisc_sleeping = &noop_qdisc,
  494. };
  495. struct Qdisc noop_qdisc = {
  496. .enqueue = noop_enqueue,
  497. .dequeue = noop_dequeue,
  498. .flags = TCQ_F_BUILTIN,
  499. .ops = &noop_qdisc_ops,
  500. .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
  501. .dev_queue = &noop_netdev_queue,
  502. .running = SEQCNT_ZERO(noop_qdisc.running),
  503. .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
  504. .gso_skb = {
  505. .next = (struct sk_buff *)&noop_qdisc.gso_skb,
  506. .prev = (struct sk_buff *)&noop_qdisc.gso_skb,
  507. .qlen = 0,
  508. .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
  509. },
  510. .skb_bad_txq = {
  511. .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
  512. .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
  513. .qlen = 0,
  514. .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
  515. },
  516. };
  517. EXPORT_SYMBOL(noop_qdisc);
  518. static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
  519. struct netlink_ext_ack *extack)
  520. {
  521. /* register_qdisc() assigns a default of noop_enqueue if unset,
  522. * but __dev_queue_xmit() treats noqueue only as such
  523. * if this is NULL - so clear it here. */
  524. qdisc->enqueue = NULL;
  525. return 0;
  526. }
  527. struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
  528. .id = "noqueue",
  529. .priv_size = 0,
  530. .init = noqueue_init,
  531. .enqueue = noop_enqueue,
  532. .dequeue = noop_dequeue,
  533. .peek = noop_dequeue,
  534. .owner = THIS_MODULE,
  535. };
  536. static const u8 prio2band[TC_PRIO_MAX + 1] = {
  537. 1, 2, 2, 2, 1, 2, 0, 0 , 1, 1, 1, 1, 1, 1, 1, 1
  538. };
  539. /* 3-band FIFO queue: old style, but should be a bit faster than
  540. generic prio+fifo combination.
  541. */
  542. #define PFIFO_FAST_BANDS 3
  543. /*
  544. * Private data for a pfifo_fast scheduler containing:
  545. * - rings for priority bands
  546. */
  547. struct pfifo_fast_priv {
  548. struct skb_array q[PFIFO_FAST_BANDS];
  549. };
  550. static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
  551. int band)
  552. {
  553. return &priv->q[band];
  554. }
  555. static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
  556. struct sk_buff **to_free)
  557. {
  558. int band = prio2band[skb->priority & TC_PRIO_MAX];
  559. struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
  560. struct skb_array *q = band2list(priv, band);
  561. unsigned int pkt_len = qdisc_pkt_len(skb);
  562. int err;
  563. err = skb_array_produce(q, skb);
  564. if (unlikely(err)) {
  565. if (qdisc_is_percpu_stats(qdisc))
  566. return qdisc_drop_cpu(skb, qdisc, to_free);
  567. else
  568. return qdisc_drop(skb, qdisc, to_free);
  569. }
  570. qdisc_update_stats_at_enqueue(qdisc, pkt_len);
  571. return NET_XMIT_SUCCESS;
  572. }
  573. static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
  574. {
  575. struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
  576. struct sk_buff *skb = NULL;
  577. bool need_retry = true;
  578. int band;
  579. retry:
  580. for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
  581. struct skb_array *q = band2list(priv, band);
  582. if (__skb_array_empty(q))
  583. continue;
  584. skb = __skb_array_consume(q);
  585. }
  586. if (likely(skb)) {
  587. qdisc_update_stats_at_dequeue(qdisc, skb);
  588. } else if (need_retry &&
  589. test_bit(__QDISC_STATE_MISSED, &qdisc->state)) {
  590. /* Delay clearing the STATE_MISSED here to reduce
  591. * the overhead of the second spin_trylock() in
  592. * qdisc_run_begin() and __netif_schedule() calling
  593. * in qdisc_run_end().
  594. */
  595. clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
  596. /* Make sure dequeuing happens after clearing
  597. * STATE_MISSED.
  598. */
  599. smp_mb__after_atomic();
  600. need_retry = false;
  601. goto retry;
  602. } else {
  603. WRITE_ONCE(qdisc->empty, true);
  604. }
  605. return skb;
  606. }
  607. static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
  608. {
  609. struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
  610. struct sk_buff *skb = NULL;
  611. int band;
  612. for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
  613. struct skb_array *q = band2list(priv, band);
  614. skb = __skb_array_peek(q);
  615. }
  616. return skb;
  617. }
  618. static void pfifo_fast_reset(struct Qdisc *qdisc)
  619. {
  620. int i, band;
  621. struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
  622. for (band = 0; band < PFIFO_FAST_BANDS; band++) {
  623. struct skb_array *q = band2list(priv, band);
  624. struct sk_buff *skb;
  625. /* NULL ring is possible if destroy path is due to a failed
  626. * skb_array_init() in pfifo_fast_init() case.
  627. */
  628. if (!q->ring.queue)
  629. continue;
  630. while ((skb = __skb_array_consume(q)) != NULL)
  631. kfree_skb(skb);
  632. }
  633. if (qdisc_is_percpu_stats(qdisc)) {
  634. for_each_possible_cpu(i) {
  635. struct gnet_stats_queue *q;
  636. q = per_cpu_ptr(qdisc->cpu_qstats, i);
  637. q->backlog = 0;
  638. q->qlen = 0;
  639. }
  640. }
  641. }
  642. static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
  643. {
  644. struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
  645. memcpy(&opt.priomap, prio2band, TC_PRIO_MAX + 1);
  646. if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
  647. goto nla_put_failure;
  648. return skb->len;
  649. nla_put_failure:
  650. return -1;
  651. }
  652. static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
  653. struct netlink_ext_ack *extack)
  654. {
  655. unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
  656. struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
  657. int prio;
  658. /* guard against zero length rings */
  659. if (!qlen)
  660. return -EINVAL;
  661. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
  662. struct skb_array *q = band2list(priv, prio);
  663. int err;
  664. err = skb_array_init(q, qlen, GFP_KERNEL);
  665. if (err)
  666. return -ENOMEM;
  667. }
  668. /* Can by-pass the queue discipline */
  669. qdisc->flags |= TCQ_F_CAN_BYPASS;
  670. return 0;
  671. }
  672. static void pfifo_fast_destroy(struct Qdisc *sch)
  673. {
  674. struct pfifo_fast_priv *priv = qdisc_priv(sch);
  675. int prio;
  676. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
  677. struct skb_array *q = band2list(priv, prio);
  678. /* NULL ring is possible if destroy path is due to a failed
  679. * skb_array_init() in pfifo_fast_init() case.
  680. */
  681. if (!q->ring.queue)
  682. continue;
  683. /* Destroy ring but no need to kfree_skb because a call to
  684. * pfifo_fast_reset() has already done that work.
  685. */
  686. ptr_ring_cleanup(&q->ring, NULL);
  687. }
  688. }
  689. static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
  690. unsigned int new_len)
  691. {
  692. struct pfifo_fast_priv *priv = qdisc_priv(sch);
  693. struct skb_array *bands[PFIFO_FAST_BANDS];
  694. int prio;
  695. for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
  696. struct skb_array *q = band2list(priv, prio);
  697. bands[prio] = q;
  698. }
  699. return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len,
  700. GFP_KERNEL);
  701. }
  702. struct Qdisc_ops pfifo_fast_ops __read_mostly = {
  703. .id = "pfifo_fast",
  704. .priv_size = sizeof(struct pfifo_fast_priv),
  705. .enqueue = pfifo_fast_enqueue,
  706. .dequeue = pfifo_fast_dequeue,
  707. .peek = pfifo_fast_peek,
  708. .init = pfifo_fast_init,
  709. .destroy = pfifo_fast_destroy,
  710. .reset = pfifo_fast_reset,
  711. .dump = pfifo_fast_dump,
  712. .change_tx_queue_len = pfifo_fast_change_tx_queue_len,
  713. .owner = THIS_MODULE,
  714. .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
  715. };
  716. EXPORT_SYMBOL(pfifo_fast_ops);
  717. static struct lock_class_key qdisc_tx_busylock;
  718. static struct lock_class_key qdisc_running_key;
  719. struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
  720. const struct Qdisc_ops *ops,
  721. struct netlink_ext_ack *extack)
  722. {
  723. struct Qdisc *sch;
  724. unsigned int size = sizeof(*sch) + ops->priv_size;
  725. int err = -ENOBUFS;
  726. struct net_device *dev;
  727. if (!dev_queue) {
  728. NL_SET_ERR_MSG(extack, "No device queue given");
  729. err = -EINVAL;
  730. goto errout;
  731. }
  732. dev = dev_queue->dev;
  733. sch = kzalloc_node(size, GFP_KERNEL, netdev_queue_numa_node_read(dev_queue));
  734. if (!sch)
  735. goto errout;
  736. __skb_queue_head_init(&sch->gso_skb);
  737. __skb_queue_head_init(&sch->skb_bad_txq);
  738. qdisc_skb_head_init(&sch->q);
  739. spin_lock_init(&sch->q.lock);
  740. if (ops->static_flags & TCQ_F_CPUSTATS) {
  741. sch->cpu_bstats =
  742. netdev_alloc_pcpu_stats(struct gnet_stats_basic_cpu);
  743. if (!sch->cpu_bstats)
  744. goto errout1;
  745. sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
  746. if (!sch->cpu_qstats) {
  747. free_percpu(sch->cpu_bstats);
  748. goto errout1;
  749. }
  750. }
  751. spin_lock_init(&sch->busylock);
  752. lockdep_set_class(&sch->busylock,
  753. dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
  754. /* seqlock has the same scope of busylock, for NOLOCK qdisc */
  755. spin_lock_init(&sch->seqlock);
  756. lockdep_set_class(&sch->seqlock,
  757. dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
  758. seqcount_init(&sch->running);
  759. lockdep_set_class(&sch->running,
  760. dev->qdisc_running_key ?: &qdisc_running_key);
  761. sch->ops = ops;
  762. sch->flags = ops->static_flags;
  763. sch->enqueue = ops->enqueue;
  764. sch->dequeue = ops->dequeue;
  765. sch->dev_queue = dev_queue;
  766. sch->empty = true;
  767. dev_hold(dev);
  768. refcount_set(&sch->refcnt, 1);
  769. return sch;
  770. errout1:
  771. kfree(sch);
  772. errout:
  773. return ERR_PTR(err);
  774. }
  775. struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
  776. const struct Qdisc_ops *ops,
  777. unsigned int parentid,
  778. struct netlink_ext_ack *extack)
  779. {
  780. struct Qdisc *sch;
  781. if (!try_module_get(ops->owner)) {
  782. NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
  783. return NULL;
  784. }
  785. sch = qdisc_alloc(dev_queue, ops, extack);
  786. if (IS_ERR(sch)) {
  787. module_put(ops->owner);
  788. return NULL;
  789. }
  790. sch->parent = parentid;
  791. if (!ops->init || ops->init(sch, NULL, extack) == 0) {
  792. trace_qdisc_create(ops, dev_queue->dev, parentid);
  793. return sch;
  794. }
  795. qdisc_put(sch);
  796. return NULL;
  797. }
  798. EXPORT_SYMBOL(qdisc_create_dflt);
  799. /* Under qdisc_lock(qdisc) and BH! */
  800. void qdisc_reset(struct Qdisc *qdisc)
  801. {
  802. const struct Qdisc_ops *ops = qdisc->ops;
  803. struct sk_buff *skb, *tmp;
  804. trace_qdisc_reset(qdisc);
  805. if (ops->reset)
  806. ops->reset(qdisc);
  807. skb_queue_walk_safe(&qdisc->gso_skb, skb, tmp) {
  808. __skb_unlink(skb, &qdisc->gso_skb);
  809. kfree_skb_list(skb);
  810. }
  811. skb_queue_walk_safe(&qdisc->skb_bad_txq, skb, tmp) {
  812. __skb_unlink(skb, &qdisc->skb_bad_txq);
  813. kfree_skb_list(skb);
  814. }
  815. qdisc->q.qlen = 0;
  816. qdisc->qstats.backlog = 0;
  817. }
  818. EXPORT_SYMBOL(qdisc_reset);
  819. void qdisc_free(struct Qdisc *qdisc)
  820. {
  821. if (qdisc_is_percpu_stats(qdisc)) {
  822. free_percpu(qdisc->cpu_bstats);
  823. free_percpu(qdisc->cpu_qstats);
  824. }
  825. kfree(qdisc);
  826. }
  827. static void qdisc_free_cb(struct rcu_head *head)
  828. {
  829. struct Qdisc *q = container_of(head, struct Qdisc, rcu);
  830. qdisc_free(q);
  831. }
  832. static void qdisc_destroy(struct Qdisc *qdisc)
  833. {
  834. const struct Qdisc_ops *ops = qdisc->ops;
  835. #ifdef CONFIG_NET_SCHED
  836. qdisc_hash_del(qdisc);
  837. qdisc_put_stab(rtnl_dereference(qdisc->stab));
  838. #endif
  839. gen_kill_estimator(&qdisc->rate_est);
  840. qdisc_reset(qdisc);
  841. if (ops->destroy)
  842. ops->destroy(qdisc);
  843. module_put(ops->owner);
  844. dev_put(qdisc_dev(qdisc));
  845. trace_qdisc_destroy(qdisc);
  846. call_rcu(&qdisc->rcu, qdisc_free_cb);
  847. }
  848. void qdisc_put(struct Qdisc *qdisc)
  849. {
  850. if (!qdisc)
  851. return;
  852. if (qdisc->flags & TCQ_F_BUILTIN ||
  853. !refcount_dec_and_test(&qdisc->refcnt))
  854. return;
  855. qdisc_destroy(qdisc);
  856. }
  857. EXPORT_SYMBOL(qdisc_put);
  858. /* Version of qdisc_put() that is called with rtnl mutex unlocked.
  859. * Intended to be used as optimization, this function only takes rtnl lock if
  860. * qdisc reference counter reached zero.
  861. */
  862. void qdisc_put_unlocked(struct Qdisc *qdisc)
  863. {
  864. if (qdisc->flags & TCQ_F_BUILTIN ||
  865. !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
  866. return;
  867. qdisc_destroy(qdisc);
  868. rtnl_unlock();
  869. }
  870. EXPORT_SYMBOL(qdisc_put_unlocked);
  871. /* Attach toplevel qdisc to device queue. */
  872. struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
  873. struct Qdisc *qdisc)
  874. {
  875. struct Qdisc *oqdisc = dev_queue->qdisc_sleeping;
  876. spinlock_t *root_lock;
  877. root_lock = qdisc_lock(oqdisc);
  878. spin_lock_bh(root_lock);
  879. /* ... and graft new one */
  880. if (qdisc == NULL)
  881. qdisc = &noop_qdisc;
  882. dev_queue->qdisc_sleeping = qdisc;
  883. rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
  884. spin_unlock_bh(root_lock);
  885. return oqdisc;
  886. }
  887. EXPORT_SYMBOL(dev_graft_qdisc);
  888. static void attach_one_default_qdisc(struct net_device *dev,
  889. struct netdev_queue *dev_queue,
  890. void *_unused)
  891. {
  892. struct Qdisc *qdisc;
  893. const struct Qdisc_ops *ops = default_qdisc_ops;
  894. if (dev->priv_flags & IFF_NO_QUEUE)
  895. ops = &noqueue_qdisc_ops;
  896. else if(dev->type == ARPHRD_CAN)
  897. ops = &pfifo_fast_ops;
  898. qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
  899. if (!qdisc)
  900. return;
  901. if (!netif_is_multiqueue(dev))
  902. qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
  903. dev_queue->qdisc_sleeping = qdisc;
  904. }
  905. static void attach_default_qdiscs(struct net_device *dev)
  906. {
  907. struct netdev_queue *txq;
  908. struct Qdisc *qdisc;
  909. txq = netdev_get_tx_queue(dev, 0);
  910. if (!netif_is_multiqueue(dev) ||
  911. dev->priv_flags & IFF_NO_QUEUE) {
  912. netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
  913. qdisc = txq->qdisc_sleeping;
  914. rcu_assign_pointer(dev->qdisc, qdisc);
  915. qdisc_refcount_inc(qdisc);
  916. } else {
  917. qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
  918. if (qdisc) {
  919. rcu_assign_pointer(dev->qdisc, qdisc);
  920. qdisc->ops->attach(qdisc);
  921. }
  922. }
  923. qdisc = rtnl_dereference(dev->qdisc);
  924. /* Detect default qdisc setup/init failed and fallback to "noqueue" */
  925. if (qdisc == &noop_qdisc) {
  926. netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
  927. default_qdisc_ops->id, noqueue_qdisc_ops.id);
  928. dev->priv_flags |= IFF_NO_QUEUE;
  929. netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
  930. qdisc = txq->qdisc_sleeping;
  931. rcu_assign_pointer(dev->qdisc, qdisc);
  932. qdisc_refcount_inc(qdisc);
  933. dev->priv_flags ^= IFF_NO_QUEUE;
  934. }
  935. #ifdef CONFIG_NET_SCHED
  936. if (qdisc != &noop_qdisc)
  937. qdisc_hash_add(qdisc, false);
  938. #endif
  939. }
  940. static void transition_one_qdisc(struct net_device *dev,
  941. struct netdev_queue *dev_queue,
  942. void *_need_watchdog)
  943. {
  944. struct Qdisc *new_qdisc = dev_queue->qdisc_sleeping;
  945. int *need_watchdog_p = _need_watchdog;
  946. if (!(new_qdisc->flags & TCQ_F_BUILTIN))
  947. clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
  948. rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
  949. if (need_watchdog_p) {
  950. dev_queue->trans_start = 0;
  951. *need_watchdog_p = 1;
  952. }
  953. }
  954. void dev_activate(struct net_device *dev)
  955. {
  956. int need_watchdog;
  957. /* No queueing discipline is attached to device;
  958. * create default one for devices, which need queueing
  959. * and noqueue_qdisc for virtual interfaces
  960. */
  961. if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
  962. attach_default_qdiscs(dev);
  963. if (!netif_carrier_ok(dev))
  964. /* Delay activation until next carrier-on event */
  965. return;
  966. need_watchdog = 0;
  967. netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
  968. if (dev_ingress_queue(dev))
  969. transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
  970. if (need_watchdog) {
  971. netif_trans_update(dev);
  972. dev_watchdog_up(dev);
  973. }
  974. }
  975. EXPORT_SYMBOL(dev_activate);
  976. static void qdisc_deactivate(struct Qdisc *qdisc)
  977. {
  978. if (qdisc->flags & TCQ_F_BUILTIN)
  979. return;
  980. set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
  981. }
  982. static void dev_deactivate_queue(struct net_device *dev,
  983. struct netdev_queue *dev_queue,
  984. void *_qdisc_default)
  985. {
  986. struct Qdisc *qdisc_default = _qdisc_default;
  987. struct Qdisc *qdisc;
  988. qdisc = rtnl_dereference(dev_queue->qdisc);
  989. if (qdisc) {
  990. qdisc_deactivate(qdisc);
  991. rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
  992. }
  993. }
  994. static void dev_reset_queue(struct net_device *dev,
  995. struct netdev_queue *dev_queue,
  996. void *_unused)
  997. {
  998. struct Qdisc *qdisc;
  999. bool nolock;
  1000. qdisc = dev_queue->qdisc_sleeping;
  1001. if (!qdisc)
  1002. return;
  1003. nolock = qdisc->flags & TCQ_F_NOLOCK;
  1004. if (nolock)
  1005. spin_lock_bh(&qdisc->seqlock);
  1006. spin_lock_bh(qdisc_lock(qdisc));
  1007. qdisc_reset(qdisc);
  1008. spin_unlock_bh(qdisc_lock(qdisc));
  1009. if (nolock) {
  1010. clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
  1011. spin_unlock_bh(&qdisc->seqlock);
  1012. }
  1013. }
  1014. static bool some_qdisc_is_busy(struct net_device *dev)
  1015. {
  1016. unsigned int i;
  1017. for (i = 0; i < dev->num_tx_queues; i++) {
  1018. struct netdev_queue *dev_queue;
  1019. spinlock_t *root_lock;
  1020. struct Qdisc *q;
  1021. int val;
  1022. dev_queue = netdev_get_tx_queue(dev, i);
  1023. q = dev_queue->qdisc_sleeping;
  1024. root_lock = qdisc_lock(q);
  1025. spin_lock_bh(root_lock);
  1026. val = (qdisc_is_running(q) ||
  1027. test_bit(__QDISC_STATE_SCHED, &q->state));
  1028. spin_unlock_bh(root_lock);
  1029. if (val)
  1030. return true;
  1031. }
  1032. return false;
  1033. }
  1034. /**
  1035. * dev_deactivate_many - deactivate transmissions on several devices
  1036. * @head: list of devices to deactivate
  1037. *
  1038. * This function returns only when all outstanding transmissions
  1039. * have completed, unless all devices are in dismantle phase.
  1040. */
  1041. void dev_deactivate_many(struct list_head *head)
  1042. {
  1043. struct net_device *dev;
  1044. list_for_each_entry(dev, head, close_list) {
  1045. netdev_for_each_tx_queue(dev, dev_deactivate_queue,
  1046. &noop_qdisc);
  1047. if (dev_ingress_queue(dev))
  1048. dev_deactivate_queue(dev, dev_ingress_queue(dev),
  1049. &noop_qdisc);
  1050. dev_watchdog_down(dev);
  1051. }
  1052. /* Wait for outstanding qdisc-less dev_queue_xmit calls or
  1053. * outstanding qdisc enqueuing calls.
  1054. * This is avoided if all devices are in dismantle phase :
  1055. * Caller will call synchronize_net() for us
  1056. */
  1057. synchronize_net();
  1058. list_for_each_entry(dev, head, close_list) {
  1059. netdev_for_each_tx_queue(dev, dev_reset_queue, NULL);
  1060. if (dev_ingress_queue(dev))
  1061. dev_reset_queue(dev, dev_ingress_queue(dev), NULL);
  1062. }
  1063. /* Wait for outstanding qdisc_run calls. */
  1064. list_for_each_entry(dev, head, close_list) {
  1065. while (some_qdisc_is_busy(dev)) {
  1066. /* wait_event() would avoid this sleep-loop but would
  1067. * require expensive checks in the fast paths of packet
  1068. * processing which isn't worth it.
  1069. */
  1070. schedule_timeout_uninterruptible(1);
  1071. }
  1072. }
  1073. }
  1074. void dev_deactivate(struct net_device *dev)
  1075. {
  1076. LIST_HEAD(single);
  1077. list_add(&dev->close_list, &single);
  1078. dev_deactivate_many(&single);
  1079. list_del(&single);
  1080. }
  1081. EXPORT_SYMBOL(dev_deactivate);
  1082. static int qdisc_change_tx_queue_len(struct net_device *dev,
  1083. struct netdev_queue *dev_queue)
  1084. {
  1085. struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
  1086. const struct Qdisc_ops *ops = qdisc->ops;
  1087. if (ops->change_tx_queue_len)
  1088. return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
  1089. return 0;
  1090. }
  1091. int dev_qdisc_change_tx_queue_len(struct net_device *dev)
  1092. {
  1093. bool up = dev->flags & IFF_UP;
  1094. unsigned int i;
  1095. int ret = 0;
  1096. if (up)
  1097. dev_deactivate(dev);
  1098. for (i = 0; i < dev->num_tx_queues; i++) {
  1099. ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
  1100. /* TODO: revert changes on a partial failure */
  1101. if (ret)
  1102. break;
  1103. }
  1104. if (up)
  1105. dev_activate(dev);
  1106. return ret;
  1107. }
  1108. static void dev_init_scheduler_queue(struct net_device *dev,
  1109. struct netdev_queue *dev_queue,
  1110. void *_qdisc)
  1111. {
  1112. struct Qdisc *qdisc = _qdisc;
  1113. rcu_assign_pointer(dev_queue->qdisc, qdisc);
  1114. dev_queue->qdisc_sleeping = qdisc;
  1115. }
  1116. void dev_init_scheduler(struct net_device *dev)
  1117. {
  1118. rcu_assign_pointer(dev->qdisc, &noop_qdisc);
  1119. netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
  1120. if (dev_ingress_queue(dev))
  1121. dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
  1122. timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
  1123. }
  1124. static void shutdown_scheduler_queue(struct net_device *dev,
  1125. struct netdev_queue *dev_queue,
  1126. void *_qdisc_default)
  1127. {
  1128. struct Qdisc *qdisc = dev_queue->qdisc_sleeping;
  1129. struct Qdisc *qdisc_default = _qdisc_default;
  1130. if (qdisc) {
  1131. rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
  1132. dev_queue->qdisc_sleeping = qdisc_default;
  1133. qdisc_put(qdisc);
  1134. }
  1135. }
  1136. void dev_shutdown(struct net_device *dev)
  1137. {
  1138. netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
  1139. if (dev_ingress_queue(dev))
  1140. shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
  1141. qdisc_put(rtnl_dereference(dev->qdisc));
  1142. rcu_assign_pointer(dev->qdisc, &noop_qdisc);
  1143. WARN_ON(timer_pending(&dev->watchdog_timer));
  1144. }
  1145. void psched_ratecfg_precompute(struct psched_ratecfg *r,
  1146. const struct tc_ratespec *conf,
  1147. u64 rate64)
  1148. {
  1149. memset(r, 0, sizeof(*r));
  1150. r->overhead = conf->overhead;
  1151. r->mpu = conf->mpu;
  1152. r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
  1153. r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
  1154. r->mult = 1;
  1155. /*
  1156. * The deal here is to replace a divide by a reciprocal one
  1157. * in fast path (a reciprocal divide is a multiply and a shift)
  1158. *
  1159. * Normal formula would be :
  1160. * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
  1161. *
  1162. * We compute mult/shift to use instead :
  1163. * time_in_ns = (len * mult) >> shift;
  1164. *
  1165. * We try to get the highest possible mult value for accuracy,
  1166. * but have to make sure no overflows will ever happen.
  1167. */
  1168. if (r->rate_bytes_ps > 0) {
  1169. u64 factor = NSEC_PER_SEC;
  1170. for (;;) {
  1171. r->mult = div64_u64(factor, r->rate_bytes_ps);
  1172. if (r->mult & (1U << 31) || factor & (1ULL << 63))
  1173. break;
  1174. factor <<= 1;
  1175. r->shift++;
  1176. }
  1177. }
  1178. }
  1179. EXPORT_SYMBOL(psched_ratecfg_precompute);
  1180. static void mini_qdisc_rcu_func(struct rcu_head *head)
  1181. {
  1182. }
  1183. void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
  1184. struct tcf_proto *tp_head)
  1185. {
  1186. /* Protected with chain0->filter_chain_lock.
  1187. * Can't access chain directly because tp_head can be NULL.
  1188. */
  1189. struct mini_Qdisc *miniq_old =
  1190. rcu_dereference_protected(*miniqp->p_miniq, 1);
  1191. struct mini_Qdisc *miniq;
  1192. if (!tp_head) {
  1193. RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
  1194. /* Wait for flying RCU callback before it is freed. */
  1195. rcu_barrier();
  1196. return;
  1197. }
  1198. miniq = !miniq_old || miniq_old == &miniqp->miniq2 ?
  1199. &miniqp->miniq1 : &miniqp->miniq2;
  1200. /* We need to make sure that readers won't see the miniq
  1201. * we are about to modify. So wait until previous call_rcu callback
  1202. * is done.
  1203. */
  1204. rcu_barrier();
  1205. miniq->filter_list = tp_head;
  1206. rcu_assign_pointer(*miniqp->p_miniq, miniq);
  1207. if (miniq_old)
  1208. /* This is counterpart of the rcu barriers above. We need to
  1209. * block potential new user of miniq_old until all readers
  1210. * are not seeing it.
  1211. */
  1212. call_rcu(&miniq_old->rcu, mini_qdisc_rcu_func);
  1213. }
  1214. EXPORT_SYMBOL(mini_qdisc_pair_swap);
  1215. void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
  1216. struct tcf_block *block)
  1217. {
  1218. miniqp->miniq1.block = block;
  1219. miniqp->miniq2.block = block;
  1220. }
  1221. EXPORT_SYMBOL(mini_qdisc_pair_block_init);
  1222. void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
  1223. struct mini_Qdisc __rcu **p_miniq)
  1224. {
  1225. miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
  1226. miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
  1227. miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
  1228. miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
  1229. miniqp->p_miniq = p_miniq;
  1230. }
  1231. EXPORT_SYMBOL(mini_qdisc_pair_init);