projects / linux-block.git / blame_incremental
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (non-incremental) | history | HEAD
Merge tag 'probes-fixes-v6.16-rc6' of git://git.kernel.org/pub/scm/linux/kernel/git...
[linux-block.git] / net / sched / sch_generic.c
... / ...
CommitLineData
1// SPDX-License-Identifier: GPL-2.0-or-later
2/*
3 * net/sched/sch_generic.c Generic packet scheduler routines.
4 *
5 * Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
6 * Jamal Hadi Salim, <hadi@cyberus.ca> 990601
7 * - Ingress support
8 */
9
10#include <linux/bitops.h>
11#include <linux/module.h>
12#include <linux/types.h>
13#include <linux/kernel.h>
14#include <linux/sched.h>
15#include <linux/string.h>
16#include <linux/errno.h>
17#include <linux/netdevice.h>
18#include <linux/skbuff.h>
19#include <linux/rtnetlink.h>
20#include <linux/init.h>
21#include <linux/rcupdate.h>
22#include <linux/list.h>
23#include <linux/slab.h>
24#include <linux/if_vlan.h>
25#include <linux/skb_array.h>
26#include <linux/if_macvlan.h>
27#include <linux/bpf.h>
28#include <net/sch_generic.h>
29#include <net/pkt_sched.h>
30#include <net/dst.h>
31#include <net/hotdata.h>
32#include <trace/events/qdisc.h>
33#include <trace/events/net.h>
34#include <net/xfrm.h>
35
36/* Qdisc to use by default */
37const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
38EXPORT_SYMBOL(default_qdisc_ops);
39
40static void qdisc_maybe_clear_missed(struct Qdisc *q,
41 const struct netdev_queue *txq)
42{
43 clear_bit(__QDISC_STATE_MISSED, &q->state);
44
45 /* Make sure the below netif_xmit_frozen_or_stopped()
46 * checking happens after clearing STATE_MISSED.
47 */
48 smp_mb__after_atomic();
49
50 /* Checking netif_xmit_frozen_or_stopped() again to
51 * make sure STATE_MISSED is set if the STATE_MISSED
52 * set by netif_tx_wake_queue()'s rescheduling of
53 * net_tx_action() is cleared by the above clear_bit().
54 */
55 if (!netif_xmit_frozen_or_stopped(txq))
56 set_bit(__QDISC_STATE_MISSED, &q->state);
57 else
58 set_bit(__QDISC_STATE_DRAINING, &q->state);
59}
60
61/* Main transmission queue. */
62
63/* Modifications to data participating in scheduling must be protected with
64 * qdisc_lock(qdisc) spinlock.
65 *
66 * The idea is the following:
67 * - enqueue, dequeue are serialized via qdisc root lock
68 * - ingress filtering is also serialized via qdisc root lock
69 * - updates to tree and tree walking are only done under the rtnl mutex.
70 */
71
72#define SKB_XOFF_MAGIC ((struct sk_buff *)1UL)
73
74static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
75{
76 const struct netdev_queue *txq = q->dev_queue;
77 spinlock_t *lock = NULL;
78 struct sk_buff *skb;
79
80 if (q->flags & TCQ_F_NOLOCK) {
81 lock = qdisc_lock(q);
82 spin_lock(lock);
83 }
84
85 skb = skb_peek(&q->skb_bad_txq);
86 if (skb) {
87 /* check the reason of requeuing without tx lock first */
88 txq = skb_get_tx_queue(txq->dev, skb);
89 if (!netif_xmit_frozen_or_stopped(txq)) {
90 skb = __skb_dequeue(&q->skb_bad_txq);
91 if (qdisc_is_percpu_stats(q)) {
92 qdisc_qstats_cpu_backlog_dec(q, skb);
93 qdisc_qstats_cpu_qlen_dec(q);
94 } else {
95 qdisc_qstats_backlog_dec(q, skb);
96 q->q.qlen--;
97 }
98 } else {
99 skb = SKB_XOFF_MAGIC;
100 qdisc_maybe_clear_missed(q, txq);
101 }
102 }
103
104 if (lock)
105 spin_unlock(lock);
106
107 return skb;
108}
109
110static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
111{
112 struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
113
114 if (unlikely(skb))
115 skb = __skb_dequeue_bad_txq(q);
116
117 return skb;
118}
119
120static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
121 struct sk_buff *skb)
122{
123 spinlock_t *lock = NULL;
124
125 if (q->flags & TCQ_F_NOLOCK) {
126 lock = qdisc_lock(q);
127 spin_lock(lock);
128 }
129
130 __skb_queue_tail(&q->skb_bad_txq, skb);
131
132 if (qdisc_is_percpu_stats(q)) {
133 qdisc_qstats_cpu_backlog_inc(q, skb);
134 qdisc_qstats_cpu_qlen_inc(q);
135 } else {
136 qdisc_qstats_backlog_inc(q, skb);
137 q->q.qlen++;
138 }
139
140 if (lock)
141 spin_unlock(lock);
142}
143
144static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
145{
146 spinlock_t *lock = NULL;
147
148 if (q->flags & TCQ_F_NOLOCK) {
149 lock = qdisc_lock(q);
150 spin_lock(lock);
151 }
152
153 while (skb) {
154 struct sk_buff *next = skb->next;
155
156 __skb_queue_tail(&q->gso_skb, skb);
157
158 /* it's still part of the queue */
159 if (qdisc_is_percpu_stats(q)) {
160 qdisc_qstats_cpu_requeues_inc(q);
161 qdisc_qstats_cpu_backlog_inc(q, skb);
162 qdisc_qstats_cpu_qlen_inc(q);
163 } else {
164 q->qstats.requeues++;
165 qdisc_qstats_backlog_inc(q, skb);
166 q->q.qlen++;
167 }
168
169 skb = next;
170 }
171
172 if (lock) {
173 spin_unlock(lock);
174 set_bit(__QDISC_STATE_MISSED, &q->state);
175 } else {
176 __netif_schedule(q);
177 }
178}
179
180static void try_bulk_dequeue_skb(struct Qdisc *q,
181 struct sk_buff *skb,
182 const struct netdev_queue *txq,
183 int *packets)
184{
185 int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
186
187 while (bytelimit > 0) {
188 struct sk_buff *nskb = q->dequeue(q);
189
190 if (!nskb)
191 break;
192
193 bytelimit -= nskb->len; /* covers GSO len */
194 skb->next = nskb;
195 skb = nskb;
196 (*packets)++; /* GSO counts as one pkt */
197 }
198 skb_mark_not_on_list(skb);
199}
200
201/* This variant of try_bulk_dequeue_skb() makes sure
202 * all skbs in the chain are for the same txq
203 */
204static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
205 struct sk_buff *skb,
206 int *packets)
207{
208 int mapping = skb_get_queue_mapping(skb);
209 struct sk_buff *nskb;
210 int cnt = 0;
211
212 do {
213 nskb = q->dequeue(q);
214 if (!nskb)
215 break;
216 if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
217 qdisc_enqueue_skb_bad_txq(q, nskb);
218 break;
219 }
220 skb->next = nskb;
221 skb = nskb;
222 } while (++cnt < 8);
223 (*packets) += cnt;
224 skb_mark_not_on_list(skb);
225}
226
227/* Note that dequeue_skb can possibly return a SKB list (via skb->next).
228 * A requeued skb (via q->gso_skb) can also be a SKB list.
229 */
230static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
231 int *packets)
232{
233 const struct netdev_queue *txq = q->dev_queue;
234 struct sk_buff *skb = NULL;
235
236 *packets = 1;
237 if (unlikely(!skb_queue_empty(&q->gso_skb))) {
238 spinlock_t *lock = NULL;
239
240 if (q->flags & TCQ_F_NOLOCK) {
241 lock = qdisc_lock(q);
242 spin_lock(lock);
243 }
244
245 skb = skb_peek(&q->gso_skb);
246
247 /* skb may be null if another cpu pulls gso_skb off in between
248 * empty check and lock.
249 */
250 if (!skb) {
251 if (lock)
252 spin_unlock(lock);
253 goto validate;
254 }
255
256 /* skb in gso_skb were already validated */
257 *validate = false;
258 if (xfrm_offload(skb))
259 *validate = true;
260 /* check the reason of requeuing without tx lock first */
261 txq = skb_get_tx_queue(txq->dev, skb);
262 if (!netif_xmit_frozen_or_stopped(txq)) {
263 skb = __skb_dequeue(&q->gso_skb);
264 if (qdisc_is_percpu_stats(q)) {
265 qdisc_qstats_cpu_backlog_dec(q, skb);
266 qdisc_qstats_cpu_qlen_dec(q);
267 } else {
268 qdisc_qstats_backlog_dec(q, skb);
269 q->q.qlen--;
270 }
271 } else {
272 skb = NULL;
273 qdisc_maybe_clear_missed(q, txq);
274 }
275 if (lock)
276 spin_unlock(lock);
277 goto trace;
278 }
279validate:
280 *validate = true;
281
282 if ((q->flags & TCQ_F_ONETXQUEUE) &&
283 netif_xmit_frozen_or_stopped(txq)) {
284 qdisc_maybe_clear_missed(q, txq);
285 return skb;
286 }
287
288 skb = qdisc_dequeue_skb_bad_txq(q);
289 if (unlikely(skb)) {
290 if (skb == SKB_XOFF_MAGIC)
291 return NULL;
292 goto bulk;
293 }
294 skb = q->dequeue(q);
295 if (skb) {
296bulk:
297 if (qdisc_may_bulk(q))
298 try_bulk_dequeue_skb(q, skb, txq, packets);
299 else
300 try_bulk_dequeue_skb_slow(q, skb, packets);
301 }
302trace:
303 trace_qdisc_dequeue(q, txq, *packets, skb);
304 return skb;
305}
306
307/*
308 * Transmit possibly several skbs, and handle the return status as
309 * required. Owning qdisc running bit guarantees that only one CPU
310 * can execute this function.
311 *
312 * Returns to the caller:
313 * false - hardware queue frozen backoff
314 * true - feel free to send more pkts
315 */
316bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
317 struct net_device *dev, struct netdev_queue *txq,
318 spinlock_t *root_lock, bool validate)
319{
320 int ret = NETDEV_TX_BUSY;
321 bool again = false;
322
323 /* And release qdisc */
324 if (root_lock)
325 spin_unlock(root_lock);
326
327 /* Note that we validate skb (GSO, checksum, ...) outside of locks */
328 if (validate)
329 skb = validate_xmit_skb_list(skb, dev, &again);
330
331#ifdef CONFIG_XFRM_OFFLOAD
332 if (unlikely(again)) {
333 if (root_lock)
334 spin_lock(root_lock);
335
336 dev_requeue_skb(skb, q);
337 return false;
338 }
339#endif
340
341 if (likely(skb)) {
342 HARD_TX_LOCK(dev, txq, smp_processor_id());
343 if (!netif_xmit_frozen_or_stopped(txq))
344 skb = dev_hard_start_xmit(skb, dev, txq, &ret);
345 else
346 qdisc_maybe_clear_missed(q, txq);
347
348 HARD_TX_UNLOCK(dev, txq);
349 } else {
350 if (root_lock)
351 spin_lock(root_lock);
352 return true;
353 }
354
355 if (root_lock)
356 spin_lock(root_lock);
357
358 if (!dev_xmit_complete(ret)) {
359 /* Driver returned NETDEV_TX_BUSY - requeue skb */
360 if (unlikely(ret != NETDEV_TX_BUSY))
361 net_warn_ratelimited("BUG %s code %d qlen %d\n",
362 dev->name, ret, q->q.qlen);
363
364 dev_requeue_skb(skb, q);
365 return false;
366 }
367
368 return true;
369}
370
371/*
372 * NOTE: Called under qdisc_lock(q) with locally disabled BH.
373 *
374 * running seqcount guarantees only one CPU can process
375 * this qdisc at a time. qdisc_lock(q) serializes queue accesses for
376 * this queue.
377 *
378 * netif_tx_lock serializes accesses to device driver.
379 *
380 * qdisc_lock(q) and netif_tx_lock are mutually exclusive,
381 * if one is grabbed, another must be free.
382 *
383 * Note, that this procedure can be called by a watchdog timer
384 *
385 * Returns to the caller:
386 * 0 - queue is empty or throttled.
387 * >0 - queue is not empty.
388 *
389 */
390static inline bool qdisc_restart(struct Qdisc *q, int *packets)
391{
392 spinlock_t *root_lock = NULL;
393 struct netdev_queue *txq;
394 struct net_device *dev;
395 struct sk_buff *skb;
396 bool validate;
397
398 /* Dequeue packet */
399 skb = dequeue_skb(q, &validate, packets);
400 if (unlikely(!skb))
401 return false;
402
403 if (!(q->flags & TCQ_F_NOLOCK))
404 root_lock = qdisc_lock(q);
405
406 dev = qdisc_dev(q);
407 txq = skb_get_tx_queue(dev, skb);
408
409 return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
410}
411
412void __qdisc_run(struct Qdisc *q)
413{
414 int quota = READ_ONCE(net_hotdata.dev_tx_weight);
415 int packets;
416
417 while (qdisc_restart(q, &packets)) {
418 quota -= packets;
419 if (quota <= 0) {
420 if (q->flags & TCQ_F_NOLOCK)
421 set_bit(__QDISC_STATE_MISSED, &q->state);
422 else
423 __netif_schedule(q);
424
425 break;
426 }
427 }
428}
429
430unsigned long dev_trans_start(struct net_device *dev)
431{
432 unsigned long res = READ_ONCE(netdev_get_tx_queue(dev, 0)->trans_start);
433 unsigned long val;
434 unsigned int i;
435
436 for (i = 1; i < dev->num_tx_queues; i++) {
437 val = READ_ONCE(netdev_get_tx_queue(dev, i)->trans_start);
438 if (val && time_after(val, res))
439 res = val;
440 }
441
442 return res;
443}
444EXPORT_SYMBOL(dev_trans_start);
445
446static void netif_freeze_queues(struct net_device *dev)
447{
448 unsigned int i;
449 int cpu;
450
451 cpu = smp_processor_id();
452 for (i = 0; i < dev->num_tx_queues; i++) {
453 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
454
455 /* We are the only thread of execution doing a
456 * freeze, but we have to grab the _xmit_lock in
457 * order to synchronize with threads which are in
458 * the ->hard_start_xmit() handler and already
459 * checked the frozen bit.
460 */
461 __netif_tx_lock(txq, cpu);
462 set_bit(__QUEUE_STATE_FROZEN, &txq->state);
463 __netif_tx_unlock(txq);
464 }
465}
466
467void netif_tx_lock(struct net_device *dev)
468{
469 spin_lock(&dev->tx_global_lock);
470 netif_freeze_queues(dev);
471}
472EXPORT_SYMBOL(netif_tx_lock);
473
474static void netif_unfreeze_queues(struct net_device *dev)
475{
476 unsigned int i;
477
478 for (i = 0; i < dev->num_tx_queues; i++) {
479 struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
480
481 /* No need to grab the _xmit_lock here. If the
482 * queue is not stopped for another reason, we
483 * force a schedule.
484 */
485 clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
486 netif_schedule_queue(txq);
487 }
488}
489
490void netif_tx_unlock(struct net_device *dev)
491{
492 netif_unfreeze_queues(dev);
493 spin_unlock(&dev->tx_global_lock);
494}
495EXPORT_SYMBOL(netif_tx_unlock);
496
497static void dev_watchdog(struct timer_list *t)
498{
499 struct net_device *dev = timer_container_of(dev, t, watchdog_timer);
500 bool release = true;
501
502 spin_lock(&dev->tx_global_lock);
503 if (!qdisc_tx_is_noop(dev)) {
504 if (netif_device_present(dev) &&
505 netif_running(dev) &&
506 netif_carrier_ok(dev)) {
507 unsigned int timedout_ms = 0;
508 unsigned int i;
509 unsigned long trans_start;
510 unsigned long oldest_start = jiffies;
511
512 for (i = 0; i < dev->num_tx_queues; i++) {
513 struct netdev_queue *txq;
514
515 txq = netdev_get_tx_queue(dev, i);
516 if (!netif_xmit_stopped(txq))
517 continue;
518
519 /* Paired with WRITE_ONCE() + smp_mb...() in
520 * netdev_tx_sent_queue() and netif_tx_stop_queue().
521 */
522 smp_mb();
523 trans_start = READ_ONCE(txq->trans_start);
524
525 if (time_after(jiffies, trans_start + dev->watchdog_timeo)) {
526 timedout_ms = jiffies_to_msecs(jiffies - trans_start);
527 atomic_long_inc(&txq->trans_timeout);
528 break;
529 }
530 if (time_after(oldest_start, trans_start))
531 oldest_start = trans_start;
532 }
533
534 if (unlikely(timedout_ms)) {
535 trace_net_dev_xmit_timeout(dev, i);
536 netdev_crit(dev, "NETDEV WATCHDOG: CPU: %d: transmit queue %u timed out %u ms\n",
537 raw_smp_processor_id(),
538 i, timedout_ms);
539 netif_freeze_queues(dev);
540 dev->netdev_ops->ndo_tx_timeout(dev, i);
541 netif_unfreeze_queues(dev);
542 }
543 if (!mod_timer(&dev->watchdog_timer,
544 round_jiffies(oldest_start +
545 dev->watchdog_timeo)))
546 release = false;
547 }
548 }
549 spin_unlock(&dev->tx_global_lock);
550
551 if (release)
552 netdev_put(dev, &dev->watchdog_dev_tracker);
553}
554
555void netdev_watchdog_up(struct net_device *dev)
556{
557 if (!dev->netdev_ops->ndo_tx_timeout)
558 return;
559 if (dev->watchdog_timeo <= 0)
560 dev->watchdog_timeo = 5*HZ;
561 if (!mod_timer(&dev->watchdog_timer,
562 round_jiffies(jiffies + dev->watchdog_timeo)))
563 netdev_hold(dev, &dev->watchdog_dev_tracker,
564 GFP_ATOMIC);
565}
566EXPORT_SYMBOL_GPL(netdev_watchdog_up);
567
568static void netdev_watchdog_down(struct net_device *dev)
569{
570 netif_tx_lock_bh(dev);
571 if (timer_delete(&dev->watchdog_timer))
572 netdev_put(dev, &dev->watchdog_dev_tracker);
573 netif_tx_unlock_bh(dev);
574}
575
576/**
577 * netif_carrier_on - set carrier
578 * @dev: network device
579 *
580 * Device has detected acquisition of carrier.
581 */
582void netif_carrier_on(struct net_device *dev)
583{
584 if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
585 if (dev->reg_state == NETREG_UNINITIALIZED)
586 return;
587 atomic_inc(&dev->carrier_up_count);
588 linkwatch_fire_event(dev);
589 if (netif_running(dev))
590 netdev_watchdog_up(dev);
591 }
592}
593EXPORT_SYMBOL(netif_carrier_on);
594
595/**
596 * netif_carrier_off - clear carrier
597 * @dev: network device
598 *
599 * Device has detected loss of carrier.
600 */
601void netif_carrier_off(struct net_device *dev)
602{
603 if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
604 if (dev->reg_state == NETREG_UNINITIALIZED)
605 return;
606 atomic_inc(&dev->carrier_down_count);
607 linkwatch_fire_event(dev);
608 }
609}
610EXPORT_SYMBOL(netif_carrier_off);
611
612/**
613 * netif_carrier_event - report carrier state event
614 * @dev: network device
615 *
616 * Device has detected a carrier event but the carrier state wasn't changed.
617 * Use in drivers when querying carrier state asynchronously, to avoid missing
618 * events (link flaps) if link recovers before it's queried.
619 */
620void netif_carrier_event(struct net_device *dev)
621{
622 if (dev->reg_state == NETREG_UNINITIALIZED)
623 return;
624 atomic_inc(&dev->carrier_up_count);
625 atomic_inc(&dev->carrier_down_count);
626 linkwatch_fire_event(dev);
627}
628EXPORT_SYMBOL_GPL(netif_carrier_event);
629
630/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
631 under all circumstances. It is difficult to invent anything faster or
632 cheaper.
633 */
634
635static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
636 struct sk_buff **to_free)
637{
638 dev_core_stats_tx_dropped_inc(skb->dev);
639 __qdisc_drop(skb, to_free);
640 return NET_XMIT_CN;
641}
642
643static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
644{
645 return NULL;
646}
647
648struct Qdisc_ops noop_qdisc_ops __read_mostly = {
649 .id = "noop",
650 .priv_size = 0,
651 .enqueue = noop_enqueue,
652 .dequeue = noop_dequeue,
653 .peek = noop_dequeue,
654 .owner = THIS_MODULE,
655};
656
657static struct netdev_queue noop_netdev_queue = {
658 RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
659 RCU_POINTER_INITIALIZER(qdisc_sleeping, &noop_qdisc),
660};
661
662struct Qdisc noop_qdisc = {
663 .enqueue = noop_enqueue,
664 .dequeue = noop_dequeue,
665 .flags = TCQ_F_BUILTIN,
666 .ops = &noop_qdisc_ops,
667 .q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
668 .dev_queue = &noop_netdev_queue,
669 .busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
670 .gso_skb = {
671 .next = (struct sk_buff *)&noop_qdisc.gso_skb,
672 .prev = (struct sk_buff *)&noop_qdisc.gso_skb,
673 .qlen = 0,
674 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
675 },
676 .skb_bad_txq = {
677 .next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
678 .prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
679 .qlen = 0,
680 .lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
681 },
682 .owner = -1,
683};
684EXPORT_SYMBOL(noop_qdisc);
685
686static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
687 struct netlink_ext_ack *extack)
688{
689 /* register_qdisc() assigns a default of noop_enqueue if unset,
690 * but __dev_queue_xmit() treats noqueue only as such
691 * if this is NULL - so clear it here. */
692 qdisc->enqueue = NULL;
693 return 0;
694}
695
696struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
697 .id = "noqueue",
698 .priv_size = 0,
699 .init = noqueue_init,
700 .enqueue = noop_enqueue,
701 .dequeue = noop_dequeue,
702 .peek = noop_dequeue,
703 .owner = THIS_MODULE,
704};
705
706const u8 sch_default_prio2band[TC_PRIO_MAX + 1] = {
707 1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1
708};
709EXPORT_SYMBOL(sch_default_prio2band);
710
711/* 3-band FIFO queue: old style, but should be a bit faster than
712 generic prio+fifo combination.
713 */
714
715#define PFIFO_FAST_BANDS 3
716
717/*
718 * Private data for a pfifo_fast scheduler containing:
719 * - rings for priority bands
720 */
721struct pfifo_fast_priv {
722 struct skb_array q[PFIFO_FAST_BANDS];
723};
724
725static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
726 int band)
727{
728 return &priv->q[band];
729}
730
731static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
732 struct sk_buff **to_free)
733{
734 int band = sch_default_prio2band[skb->priority & TC_PRIO_MAX];
735 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
736 struct skb_array *q = band2list(priv, band);
737 unsigned int pkt_len = qdisc_pkt_len(skb);
738 int err;
739
740 err = skb_array_produce(q, skb);
741
742 if (unlikely(err)) {
743 if (qdisc_is_percpu_stats(qdisc))
744 return qdisc_drop_cpu(skb, qdisc, to_free);
745 else
746 return qdisc_drop(skb, qdisc, to_free);
747 }
748
749 qdisc_update_stats_at_enqueue(qdisc, pkt_len);
750 return NET_XMIT_SUCCESS;
751}
752
753static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
754{
755 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
756 struct sk_buff *skb = NULL;
757 bool need_retry = true;
758 int band;
759
760retry:
761 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
762 struct skb_array *q = band2list(priv, band);
763
764 if (__skb_array_empty(q))
765 continue;
766
767 skb = __skb_array_consume(q);
768 }
769 if (likely(skb)) {
770 qdisc_update_stats_at_dequeue(qdisc, skb);
771 } else if (need_retry &&
772 READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {
773 /* Delay clearing the STATE_MISSED here to reduce
774 * the overhead of the second spin_trylock() in
775 * qdisc_run_begin() and __netif_schedule() calling
776 * in qdisc_run_end().
777 */
778 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
779 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
780
781 /* Make sure dequeuing happens after clearing
782 * STATE_MISSED.
783 */
784 smp_mb__after_atomic();
785
786 need_retry = false;
787
788 goto retry;
789 }
790
791 return skb;
792}
793
794static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
795{
796 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
797 struct sk_buff *skb = NULL;
798 int band;
799
800 for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
801 struct skb_array *q = band2list(priv, band);
802
803 skb = __skb_array_peek(q);
804 }
805
806 return skb;
807}
808
809static void pfifo_fast_reset(struct Qdisc *qdisc)
810{
811 int i, band;
812 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
813
814 for (band = 0; band < PFIFO_FAST_BANDS; band++) {
815 struct skb_array *q = band2list(priv, band);
816 struct sk_buff *skb;
817
818 /* NULL ring is possible if destroy path is due to a failed
819 * skb_array_init() in pfifo_fast_init() case.
820 */
821 if (!q->ring.queue)
822 continue;
823
824 while ((skb = __skb_array_consume(q)) != NULL)
825 kfree_skb(skb);
826 }
827
828 if (qdisc_is_percpu_stats(qdisc)) {
829 for_each_possible_cpu(i) {
830 struct gnet_stats_queue *q;
831
832 q = per_cpu_ptr(qdisc->cpu_qstats, i);
833 q->backlog = 0;
834 q->qlen = 0;
835 }
836 }
837}
838
839static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
840{
841 struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
842
843 memcpy(&opt.priomap, sch_default_prio2band, TC_PRIO_MAX + 1);
844 if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
845 goto nla_put_failure;
846 return skb->len;
847
848nla_put_failure:
849 return -1;
850}
851
852static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
853 struct netlink_ext_ack *extack)
854{
855 unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
856 struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
857 int prio;
858
859 /* guard against zero length rings */
860 if (!qlen)
861 return -EINVAL;
862
863 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
864 struct skb_array *q = band2list(priv, prio);
865 int err;
866
867 err = skb_array_init(q, qlen, GFP_KERNEL);
868 if (err)
869 return -ENOMEM;
870 }
871
872 /* Can by-pass the queue discipline */
873 qdisc->flags |= TCQ_F_CAN_BYPASS;
874 return 0;
875}
876
877static void pfifo_fast_destroy(struct Qdisc *sch)
878{
879 struct pfifo_fast_priv *priv = qdisc_priv(sch);
880 int prio;
881
882 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
883 struct skb_array *q = band2list(priv, prio);
884
885 /* NULL ring is possible if destroy path is due to a failed
886 * skb_array_init() in pfifo_fast_init() case.
887 */
888 if (!q->ring.queue)
889 continue;
890 /* Destroy ring but no need to kfree_skb because a call to
891 * pfifo_fast_reset() has already done that work.
892 */
893 ptr_ring_cleanup(&q->ring, NULL);
894 }
895}
896
897static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
898 unsigned int new_len)
899{
900 struct pfifo_fast_priv *priv = qdisc_priv(sch);
901 struct skb_array *bands[PFIFO_FAST_BANDS];
902 int prio;
903
904 for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
905 struct skb_array *q = band2list(priv, prio);
906
907 bands[prio] = q;
908 }
909
910 return skb_array_resize_multiple_bh(bands, PFIFO_FAST_BANDS, new_len,
911 GFP_KERNEL);
912}
913
914struct Qdisc_ops pfifo_fast_ops __read_mostly = {
915 .id = "pfifo_fast",
916 .priv_size = sizeof(struct pfifo_fast_priv),
917 .enqueue = pfifo_fast_enqueue,
918 .dequeue = pfifo_fast_dequeue,
919 .peek = pfifo_fast_peek,
920 .init = pfifo_fast_init,
921 .destroy = pfifo_fast_destroy,
922 .reset = pfifo_fast_reset,
923 .dump = pfifo_fast_dump,
924 .change_tx_queue_len = pfifo_fast_change_tx_queue_len,
925 .owner = THIS_MODULE,
926 .static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
927};
928EXPORT_SYMBOL(pfifo_fast_ops);
929
930static struct lock_class_key qdisc_tx_busylock;
931
932struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
933 const struct Qdisc_ops *ops,
934 struct netlink_ext_ack *extack)
935{
936 struct Qdisc *sch;
937 unsigned int size = sizeof(*sch) + ops->priv_size;
938 int err = -ENOBUFS;
939 struct net_device *dev;
940
941 if (!dev_queue) {
942 NL_SET_ERR_MSG(extack, "No device queue given");
943 err = -EINVAL;
944 goto errout;
945 }
946
947 dev = dev_queue->dev;
948 sch = kzalloc_node(size, GFP_KERNEL, netdev_queue_numa_node_read(dev_queue));
949
950 if (!sch)
951 goto errout;
952 __skb_queue_head_init(&sch->gso_skb);
953 __skb_queue_head_init(&sch->skb_bad_txq);
954 gnet_stats_basic_sync_init(&sch->bstats);
955 lockdep_register_key(&sch->root_lock_key);
956 spin_lock_init(&sch->q.lock);
957 lockdep_set_class(&sch->q.lock, &sch->root_lock_key);
958
959 if (ops->static_flags & TCQ_F_CPUSTATS) {
960 sch->cpu_bstats =
961 netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync);
962 if (!sch->cpu_bstats)
963 goto errout1;
964
965 sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
966 if (!sch->cpu_qstats) {
967 free_percpu(sch->cpu_bstats);
968 goto errout1;
969 }
970 }
971
972 spin_lock_init(&sch->busylock);
973 lockdep_set_class(&sch->busylock,
974 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
975
976 /* seqlock has the same scope of busylock, for NOLOCK qdisc */
977 spin_lock_init(&sch->seqlock);
978 lockdep_set_class(&sch->seqlock,
979 dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
980
981 sch->ops = ops;
982 sch->flags = ops->static_flags;
983 sch->enqueue = ops->enqueue;
984 sch->dequeue = ops->dequeue;
985 sch->dev_queue = dev_queue;
986 sch->owner = -1;
987 netdev_hold(dev, &sch->dev_tracker, GFP_KERNEL);
988 refcount_set(&sch->refcnt, 1);
989
990 return sch;
991errout1:
992 lockdep_unregister_key(&sch->root_lock_key);
993 kfree(sch);
994errout:
995 return ERR_PTR(err);
996}
997
998struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
999 const struct Qdisc_ops *ops,
1000 unsigned int parentid,
1001 struct netlink_ext_ack *extack)
1002{
1003 struct Qdisc *sch;
1004
1005 if (!bpf_try_module_get(ops, ops->owner)) {
1006 NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
1007 return NULL;
1008 }
1009
1010 sch = qdisc_alloc(dev_queue, ops, extack);
1011 if (IS_ERR(sch)) {
1012 bpf_module_put(ops, ops->owner);
1013 return NULL;
1014 }
1015 sch->parent = parentid;
1016
1017 if (!ops->init || ops->init(sch, NULL, extack) == 0) {
1018 trace_qdisc_create(ops, dev_queue->dev, parentid);
1019 return sch;
1020 }
1021
1022 qdisc_put(sch);
1023 return NULL;
1024}
1025EXPORT_SYMBOL(qdisc_create_dflt);
1026
1027/* Under qdisc_lock(qdisc) and BH! */
1028
1029void qdisc_reset(struct Qdisc *qdisc)
1030{
1031 const struct Qdisc_ops *ops = qdisc->ops;
1032
1033 trace_qdisc_reset(qdisc);
1034
1035 if (ops->reset)
1036 ops->reset(qdisc);
1037
1038 __skb_queue_purge(&qdisc->gso_skb);
1039 __skb_queue_purge(&qdisc->skb_bad_txq);
1040
1041 qdisc->q.qlen = 0;
1042 qdisc->qstats.backlog = 0;
1043}
1044EXPORT_SYMBOL(qdisc_reset);
1045
1046void qdisc_free(struct Qdisc *qdisc)
1047{
1048 if (qdisc_is_percpu_stats(qdisc)) {
1049 free_percpu(qdisc->cpu_bstats);
1050 free_percpu(qdisc->cpu_qstats);
1051 }
1052
1053 kfree(qdisc);
1054}
1055
1056static void qdisc_free_cb(struct rcu_head *head)
1057{
1058 struct Qdisc *q = container_of(head, struct Qdisc, rcu);
1059
1060 qdisc_free(q);
1061}
1062
1063static void __qdisc_destroy(struct Qdisc *qdisc)
1064{
1065 const struct Qdisc_ops *ops = qdisc->ops;
1066 struct net_device *dev = qdisc_dev(qdisc);
1067
1068#ifdef CONFIG_NET_SCHED
1069 qdisc_hash_del(qdisc);
1070
1071 qdisc_put_stab(rtnl_dereference(qdisc->stab));
1072#endif
1073 gen_kill_estimator(&qdisc->rate_est);
1074
1075 qdisc_reset(qdisc);
1076
1077
1078 if (ops->destroy)
1079 ops->destroy(qdisc);
1080
1081 lockdep_unregister_key(&qdisc->root_lock_key);
1082 bpf_module_put(ops, ops->owner);
1083 netdev_put(dev, &qdisc->dev_tracker);
1084
1085 trace_qdisc_destroy(qdisc);
1086
1087 call_rcu(&qdisc->rcu, qdisc_free_cb);
1088}
1089
1090void qdisc_destroy(struct Qdisc *qdisc)
1091{
1092 if (qdisc->flags & TCQ_F_BUILTIN)
1093 return;
1094
1095 __qdisc_destroy(qdisc);
1096}
1097
1098void qdisc_put(struct Qdisc *qdisc)
1099{
1100 if (!qdisc)
1101 return;
1102
1103 if (qdisc->flags & TCQ_F_BUILTIN ||
1104 !refcount_dec_and_test(&qdisc->refcnt))
1105 return;
1106
1107 __qdisc_destroy(qdisc);
1108}
1109EXPORT_SYMBOL(qdisc_put);
1110
1111/* Version of qdisc_put() that is called with rtnl mutex unlocked.
1112 * Intended to be used as optimization, this function only takes rtnl lock if
1113 * qdisc reference counter reached zero.
1114 */
1115
1116void qdisc_put_unlocked(struct Qdisc *qdisc)
1117{
1118 if (qdisc->flags & TCQ_F_BUILTIN ||
1119 !refcount_dec_and_rtnl_lock(&qdisc->refcnt))
1120 return;
1121
1122 __qdisc_destroy(qdisc);
1123 rtnl_unlock();
1124}
1125EXPORT_SYMBOL(qdisc_put_unlocked);
1126
1127/* Attach toplevel qdisc to device queue. */
1128struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
1129 struct Qdisc *qdisc)
1130{
1131 struct Qdisc *oqdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1132 spinlock_t *root_lock;
1133
1134 root_lock = qdisc_lock(oqdisc);
1135 spin_lock_bh(root_lock);
1136
1137 /* ... and graft new one */
1138 if (qdisc == NULL)
1139 qdisc = &noop_qdisc;
1140 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1141 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1142
1143 spin_unlock_bh(root_lock);
1144
1145 return oqdisc;
1146}
1147EXPORT_SYMBOL(dev_graft_qdisc);
1148
1149static void shutdown_scheduler_queue(struct net_device *dev,
1150 struct netdev_queue *dev_queue,
1151 void *_qdisc_default)
1152{
1153 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1154 struct Qdisc *qdisc_default = _qdisc_default;
1155
1156 if (qdisc) {
1157 rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
1158 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc_default);
1159
1160 qdisc_put(qdisc);
1161 }
1162}
1163
1164static void attach_one_default_qdisc(struct net_device *dev,
1165 struct netdev_queue *dev_queue,
1166 void *_unused)
1167{
1168 struct Qdisc *qdisc;
1169 const struct Qdisc_ops *ops = default_qdisc_ops;
1170
1171 if (dev->priv_flags & IFF_NO_QUEUE)
1172 ops = &noqueue_qdisc_ops;
1173 else if(dev->type == ARPHRD_CAN)
1174 ops = &pfifo_fast_ops;
1175
1176 qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
1177 if (!qdisc)
1178 return;
1179
1180 if (!netif_is_multiqueue(dev))
1181 qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
1182 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1183}
1184
1185static void attach_default_qdiscs(struct net_device *dev)
1186{
1187 struct netdev_queue *txq;
1188 struct Qdisc *qdisc;
1189
1190 txq = netdev_get_tx_queue(dev, 0);
1191
1192 if (!netif_is_multiqueue(dev) ||
1193 dev->priv_flags & IFF_NO_QUEUE) {
1194 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
1195 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1196 rcu_assign_pointer(dev->qdisc, qdisc);
1197 qdisc_refcount_inc(qdisc);
1198 } else {
1199 qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
1200 if (qdisc) {
1201 rcu_assign_pointer(dev->qdisc, qdisc);
1202 qdisc->ops->attach(qdisc);
1203 }
1204 }
1205 qdisc = rtnl_dereference(dev->qdisc);
1206
1207 /* Detect default qdisc setup/init failed and fallback to "noqueue" */
1208 if (qdisc == &noop_qdisc) {
1209 netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
1210 default_qdisc_ops->id, noqueue_qdisc_ops.id);
1211 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
1212 dev->priv_flags |= IFF_NO_QUEUE;
1213 netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
1214 qdisc = rtnl_dereference(txq->qdisc_sleeping);
1215 rcu_assign_pointer(dev->qdisc, qdisc);
1216 qdisc_refcount_inc(qdisc);
1217 dev->priv_flags ^= IFF_NO_QUEUE;
1218 }
1219
1220#ifdef CONFIG_NET_SCHED
1221 if (qdisc != &noop_qdisc)
1222 qdisc_hash_add(qdisc, false);
1223#endif
1224}
1225
1226static void transition_one_qdisc(struct net_device *dev,
1227 struct netdev_queue *dev_queue,
1228 void *_need_watchdog)
1229{
1230 struct Qdisc *new_qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1231 int *need_watchdog_p = _need_watchdog;
1232
1233 if (!(new_qdisc->flags & TCQ_F_BUILTIN))
1234 clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
1235
1236 rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
1237 if (need_watchdog_p) {
1238 WRITE_ONCE(dev_queue->trans_start, 0);
1239 *need_watchdog_p = 1;
1240 }
1241}
1242
1243void dev_activate(struct net_device *dev)
1244{
1245 int need_watchdog;
1246
1247 /* No queueing discipline is attached to device;
1248 * create default one for devices, which need queueing
1249 * and noqueue_qdisc for virtual interfaces
1250 */
1251
1252 if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
1253 attach_default_qdiscs(dev);
1254
1255 if (!netif_carrier_ok(dev))
1256 /* Delay activation until next carrier-on event */
1257 return;
1258
1259 need_watchdog = 0;
1260 netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
1261 if (dev_ingress_queue(dev))
1262 transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
1263
1264 if (need_watchdog) {
1265 netif_trans_update(dev);
1266 netdev_watchdog_up(dev);
1267 }
1268}
1269EXPORT_SYMBOL(dev_activate);
1270
1271static void qdisc_deactivate(struct Qdisc *qdisc)
1272{
1273 if (qdisc->flags & TCQ_F_BUILTIN)
1274 return;
1275
1276 set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
1277}
1278
1279static void dev_deactivate_queue(struct net_device *dev,
1280 struct netdev_queue *dev_queue,
1281 void *_sync_needed)
1282{
1283 bool *sync_needed = _sync_needed;
1284 struct Qdisc *qdisc;
1285
1286 qdisc = rtnl_dereference(dev_queue->qdisc);
1287 if (qdisc) {
1288 if (qdisc->enqueue)
1289 *sync_needed = true;
1290 qdisc_deactivate(qdisc);
1291 rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
1292 }
1293}
1294
1295static void dev_reset_queue(struct net_device *dev,
1296 struct netdev_queue *dev_queue,
1297 void *_unused)
1298{
1299 struct Qdisc *qdisc;
1300 bool nolock;
1301
1302 qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1303 if (!qdisc)
1304 return;
1305
1306 nolock = qdisc->flags & TCQ_F_NOLOCK;
1307
1308 if (nolock)
1309 spin_lock_bh(&qdisc->seqlock);
1310 spin_lock_bh(qdisc_lock(qdisc));
1311
1312 qdisc_reset(qdisc);
1313
1314 spin_unlock_bh(qdisc_lock(qdisc));
1315 if (nolock) {
1316 clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
1317 clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
1318 spin_unlock_bh(&qdisc->seqlock);
1319 }
1320}
1321
1322static bool some_qdisc_is_busy(struct net_device *dev)
1323{
1324 unsigned int i;
1325
1326 for (i = 0; i < dev->num_tx_queues; i++) {
1327 struct netdev_queue *dev_queue;
1328 spinlock_t *root_lock;
1329 struct Qdisc *q;
1330 int val;
1331
1332 dev_queue = netdev_get_tx_queue(dev, i);
1333 q = rtnl_dereference(dev_queue->qdisc_sleeping);
1334
1335 root_lock = qdisc_lock(q);
1336 spin_lock_bh(root_lock);
1337
1338 val = (qdisc_is_running(q) ||
1339 test_bit(__QDISC_STATE_SCHED, &q->state));
1340
1341 spin_unlock_bh(root_lock);
1342
1343 if (val)
1344 return true;
1345 }
1346 return false;
1347}
1348
1349/**
1350 * dev_deactivate_many - deactivate transmissions on several devices
1351 * @head: list of devices to deactivate
1352 *
1353 * This function returns only when all outstanding transmissions
1354 * have completed, unless all devices are in dismantle phase.
1355 */
1356void dev_deactivate_many(struct list_head *head)
1357{
1358 bool sync_needed = false;
1359 struct net_device *dev;
1360
1361 list_for_each_entry(dev, head, close_list) {
1362 netdev_for_each_tx_queue(dev, dev_deactivate_queue,
1363 &sync_needed);
1364 if (dev_ingress_queue(dev))
1365 dev_deactivate_queue(dev, dev_ingress_queue(dev),
1366 &sync_needed);
1367
1368 netdev_watchdog_down(dev);
1369 }
1370
1371 /* Wait for outstanding qdisc enqueuing calls. */
1372 if (sync_needed)
1373 synchronize_net();
1374
1375 list_for_each_entry(dev, head, close_list) {
1376 netdev_for_each_tx_queue(dev, dev_reset_queue, NULL);
1377
1378 if (dev_ingress_queue(dev))
1379 dev_reset_queue(dev, dev_ingress_queue(dev), NULL);
1380 }
1381
1382 /* Wait for outstanding qdisc_run calls. */
1383 list_for_each_entry(dev, head, close_list) {
1384 while (some_qdisc_is_busy(dev)) {
1385 /* wait_event() would avoid this sleep-loop but would
1386 * require expensive checks in the fast paths of packet
1387 * processing which isn't worth it.
1388 */
1389 schedule_timeout_uninterruptible(1);
1390 }
1391 }
1392}
1393
1394void dev_deactivate(struct net_device *dev)
1395{
1396 LIST_HEAD(single);
1397
1398 list_add(&dev->close_list, &single);
1399 dev_deactivate_many(&single);
1400 list_del(&single);
1401}
1402EXPORT_SYMBOL(dev_deactivate);
1403
1404static int qdisc_change_tx_queue_len(struct net_device *dev,
1405 struct netdev_queue *dev_queue)
1406{
1407 struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
1408 const struct Qdisc_ops *ops = qdisc->ops;
1409
1410 if (ops->change_tx_queue_len)
1411 return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
1412 return 0;
1413}
1414
1415void dev_qdisc_change_real_num_tx(struct net_device *dev,
1416 unsigned int new_real_tx)
1417{
1418 struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
1419
1420 if (qdisc->ops->change_real_num_tx)
1421 qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
1422}
1423
1424void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx)
1425{
1426#ifdef CONFIG_NET_SCHED
1427 struct net_device *dev = qdisc_dev(sch);
1428 struct Qdisc *qdisc;
1429 unsigned int i;
1430
1431 for (i = new_real_tx; i < dev->real_num_tx_queues; i++) {
1432 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1433 /* Only update the default qdiscs we created,
1434 * qdiscs with handles are always hashed.
1435 */
1436 if (qdisc != &noop_qdisc && !qdisc->handle)
1437 qdisc_hash_del(qdisc);
1438 }
1439 for (i = dev->real_num_tx_queues; i < new_real_tx; i++) {
1440 qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
1441 if (qdisc != &noop_qdisc && !qdisc->handle)
1442 qdisc_hash_add(qdisc, false);
1443 }
1444#endif
1445}
1446EXPORT_SYMBOL(mq_change_real_num_tx);
1447
1448int dev_qdisc_change_tx_queue_len(struct net_device *dev)
1449{
1450 bool up = dev->flags & IFF_UP;
1451 unsigned int i;
1452 int ret = 0;
1453
1454 if (up)
1455 dev_deactivate(dev);
1456
1457 for (i = 0; i < dev->num_tx_queues; i++) {
1458 ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
1459
1460 /* TODO: revert changes on a partial failure */
1461 if (ret)
1462 break;
1463 }
1464
1465 if (up)
1466 dev_activate(dev);
1467 return ret;
1468}
1469
1470static void dev_init_scheduler_queue(struct net_device *dev,
1471 struct netdev_queue *dev_queue,
1472 void *_qdisc)
1473{
1474 struct Qdisc *qdisc = _qdisc;
1475
1476 rcu_assign_pointer(dev_queue->qdisc, qdisc);
1477 rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
1478}
1479
1480void dev_init_scheduler(struct net_device *dev)
1481{
1482 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1483 netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
1484 if (dev_ingress_queue(dev))
1485 dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
1486
1487 timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
1488}
1489
1490void dev_shutdown(struct net_device *dev)
1491{
1492 netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
1493 if (dev_ingress_queue(dev))
1494 shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
1495 qdisc_put(rtnl_dereference(dev->qdisc));
1496 rcu_assign_pointer(dev->qdisc, &noop_qdisc);
1497
1498 WARN_ON(timer_pending(&dev->watchdog_timer));
1499}
1500
1501/**
1502 * psched_ratecfg_precompute__() - Pre-compute values for reciprocal division
1503 * @rate: Rate to compute reciprocal division values of
1504 * @mult: Multiplier for reciprocal division
1505 * @shift: Shift for reciprocal division
1506 *
1507 * The multiplier and shift for reciprocal division by rate are stored
1508 * in mult and shift.
1509 *
1510 * The deal here is to replace a divide by a reciprocal one
1511 * in fast path (a reciprocal divide is a multiply and a shift)
1512 *
1513 * Normal formula would be :
1514 * time_in_ns = (NSEC_PER_SEC * len) / rate_bps
1515 *
1516 * We compute mult/shift to use instead :
1517 * time_in_ns = (len * mult) >> shift;
1518 *
1519 * We try to get the highest possible mult value for accuracy,
1520 * but have to make sure no overflows will ever happen.
1521 *
1522 * reciprocal_value() is not used here it doesn't handle 64-bit values.
1523 */
1524static void psched_ratecfg_precompute__(u64 rate, u32 *mult, u8 *shift)
1525{
1526 u64 factor = NSEC_PER_SEC;
1527
1528 *mult = 1;
1529 *shift = 0;
1530
1531 if (rate <= 0)
1532 return;
1533
1534 for (;;) {
1535 *mult = div64_u64(factor, rate);
1536 if (*mult & (1U << 31) || factor & (1ULL << 63))
1537 break;
1538 factor <<= 1;
1539 (*shift)++;
1540 }
1541}
1542
1543void psched_ratecfg_precompute(struct psched_ratecfg *r,
1544 const struct tc_ratespec *conf,
1545 u64 rate64)
1546{
1547 memset(r, 0, sizeof(*r));
1548 r->overhead = conf->overhead;
1549 r->mpu = conf->mpu;
1550 r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
1551 r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
1552 psched_ratecfg_precompute__(r->rate_bytes_ps, &r->mult, &r->shift);
1553}
1554EXPORT_SYMBOL(psched_ratecfg_precompute);
1555
1556void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64)
1557{
1558 r->rate_pkts_ps = pktrate64;
1559 psched_ratecfg_precompute__(r->rate_pkts_ps, &r->mult, &r->shift);
1560}
1561EXPORT_SYMBOL(psched_ppscfg_precompute);
1562
1563void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
1564 struct tcf_proto *tp_head)
1565{
1566 /* Protected with chain0->filter_chain_lock.
1567 * Can't access chain directly because tp_head can be NULL.
1568 */
1569 struct mini_Qdisc *miniq_old =
1570 rcu_dereference_protected(*miniqp->p_miniq, 1);
1571 struct mini_Qdisc *miniq;
1572
1573 if (!tp_head) {
1574 RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
1575 } else {
1576 miniq = miniq_old != &miniqp->miniq1 ?
1577 &miniqp->miniq1 : &miniqp->miniq2;
1578
1579 /* We need to make sure that readers won't see the miniq
1580 * we are about to modify. So ensure that at least one RCU
1581 * grace period has elapsed since the miniq was made
1582 * inactive.
1583 */
1584 if (IS_ENABLED(CONFIG_PREEMPT_RT))
1585 cond_synchronize_rcu(miniq->rcu_state);
1586 else if (!poll_state_synchronize_rcu(miniq->rcu_state))
1587 synchronize_rcu_expedited();
1588
1589 miniq->filter_list = tp_head;
1590 rcu_assign_pointer(*miniqp->p_miniq, miniq);
1591 }
1592
1593 if (miniq_old)
1594 /* This is counterpart of the rcu sync above. We need to
1595 * block potential new user of miniq_old until all readers
1596 * are not seeing it.
1597 */
1598 miniq_old->rcu_state = start_poll_synchronize_rcu();
1599}
1600EXPORT_SYMBOL(mini_qdisc_pair_swap);
1601
1602void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
1603 struct tcf_block *block)
1604{
1605 miniqp->miniq1.block = block;
1606 miniqp->miniq2.block = block;
1607}
1608EXPORT_SYMBOL(mini_qdisc_pair_block_init);
1609
1610void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
1611 struct mini_Qdisc __rcu **p_miniq)
1612{
1613 miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
1614 miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
1615 miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
1616 miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
1617 miniqp->miniq1.rcu_state = get_state_synchronize_rcu();
1618 miniqp->miniq2.rcu_state = miniqp->miniq1.rcu_state;
1619 miniqp->p_miniq = p_miniq;
1620}
1621EXPORT_SYMBOL(mini_qdisc_pair_init);
Linux 6.x block layer and io_uring tree(s)