2019-05-27 06:55:01 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
|
* net/sched/sch_generic.c Generic packet scheduler routines.
|
|
|
|
|
*
|
|
|
|
|
* Authors: Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
|
|
|
* Jamal Hadi Salim, <hadi@cyberus.ca> 990601
|
|
|
|
|
* - Ingress support
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#include <linux/bitops.h>
|
|
|
|
|
#include <linux/module.h>
|
|
|
|
|
#include <linux/types.h>
|
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
|
#include <linux/sched.h>
|
|
|
|
|
#include <linux/string.h>
|
|
|
|
|
#include <linux/errno.h>
|
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
|
#include <linux/rtnetlink.h>
|
|
|
|
|
#include <linux/init.h>
|
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
|
#include <linux/list.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
2013-08-03 20:07:47 +00:00
|
|
|
#include <linux/if_vlan.h>
|
2017-12-07 17:58:19 +00:00
|
|
|
#include <linux/skb_array.h>
|
2017-12-06 15:50:28 +00:00
|
|
|
#include <linux/if_macvlan.h>
|
2013-02-12 00:12:03 +00:00
|
|
|
#include <net/sch_generic.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/pkt_sched.h>
|
2010-05-11 23:19:48 +00:00
|
|
|
#include <net/dst.h>
|
2024-03-06 16:00:21 +00:00
|
|
|
#include <net/hotdata.h>
|
2017-08-15 19:11:03 +00:00
|
|
|
#include <trace/events/qdisc.h>
|
2019-05-02 02:56:59 +00:00
|
|
|
#include <trace/events/net.h>
|
2017-12-20 09:41:36 +00:00
|
|
|
#include <net/xfrm.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-08-31 17:15:33 +00:00
|
|
|
/* Qdisc to use by default */
|
|
|
|
|
const struct Qdisc_ops *default_qdisc_ops = &pfifo_fast_ops;
|
|
|
|
|
EXPORT_SYMBOL(default_qdisc_ops);
|
|
|
|
|
|
2021-05-14 03:17:01 +00:00
|
|
|
static void qdisc_maybe_clear_missed(struct Qdisc *q,
|
|
|
|
|
const struct netdev_queue *txq)
|
|
|
|
|
{
|
|
|
|
|
clear_bit(__QDISC_STATE_MISSED, &q->state);
|
|
|
|
|
|
|
|
|
|
/* Make sure the below netif_xmit_frozen_or_stopped()
|
|
|
|
|
* checking happens after clearing STATE_MISSED.
|
|
|
|
|
*/
|
|
|
|
|
smp_mb__after_atomic();
|
|
|
|
|
|
|
|
|
|
/* Checking netif_xmit_frozen_or_stopped() again to
|
|
|
|
|
* make sure STATE_MISSED is set if the STATE_MISSED
|
|
|
|
|
* set by netif_tx_wake_queue()'s rescheduling of
|
|
|
|
|
* net_tx_action() is cleared by the above clear_bit().
|
|
|
|
|
*/
|
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq))
|
|
|
|
|
set_bit(__QDISC_STATE_MISSED, &q->state);
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
else
|
|
|
|
|
set_bit(__QDISC_STATE_DRAINING, &q->state);
|
2021-05-14 03:17:01 +00:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Main transmission queue. */
|
|
|
|
|
|
2007-04-17 00:02:10 +00:00
|
|
|
/* Modifications to data participating in scheduling must be protected with
|
2008-08-03 03:02:43 +00:00
|
|
|
* qdisc_lock(qdisc) spinlock.
|
2007-04-17 00:02:10 +00:00
|
|
|
*
|
|
|
|
|
* The idea is the following:
|
2008-07-16 10:22:39 +00:00
|
|
|
* - enqueue, dequeue are serialized via qdisc root lock
|
|
|
|
|
* - ingress filtering is also serialized via qdisc root lock
|
2007-04-17 00:02:10 +00:00
|
|
|
* - updates to tree and tree walking are only done under the rtnl mutex.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2017-12-07 17:56:23 +00:00
|
|
|
|
2019-09-05 12:20:22 +00:00
|
|
|
#define SKB_XOFF_MAGIC ((struct sk_buff *)1UL)
|
|
|
|
|
|
2017-12-07 17:56:23 +00:00
|
|
|
static inline struct sk_buff *__skb_dequeue_bad_txq(struct Qdisc *q)
|
|
|
|
|
{
|
|
|
|
|
const struct netdev_queue *txq = q->dev_queue;
|
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
|
spin_lock(lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb = skb_peek(&q->skb_bad_txq);
|
|
|
|
|
if (skb) {
|
|
|
|
|
/* check the reason of requeuing without tx lock first */
|
|
|
|
|
txq = skb_get_tx_queue(txq->dev, skb);
|
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq)) {
|
|
|
|
|
skb = __skb_dequeue(&q->skb_bad_txq);
|
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
|
qdisc_qstats_cpu_backlog_dec(q, skb);
|
2019-04-10 12:32:41 +00:00
|
|
|
qdisc_qstats_cpu_qlen_dec(q);
|
2017-12-07 17:56:23 +00:00
|
|
|
} else {
|
|
|
|
|
qdisc_qstats_backlog_dec(q, skb);
|
|
|
|
|
q->q.qlen--;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
2019-09-05 12:20:22 +00:00
|
|
|
skb = SKB_XOFF_MAGIC;
|
2021-05-14 03:17:01 +00:00
|
|
|
qdisc_maybe_clear_missed(q, txq);
|
2017-12-07 17:56:23 +00:00
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (lock)
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline struct sk_buff *qdisc_dequeue_skb_bad_txq(struct Qdisc *q)
|
|
|
|
|
{
|
|
|
|
|
struct sk_buff *skb = skb_peek(&q->skb_bad_txq);
|
|
|
|
|
|
|
|
|
|
if (unlikely(skb))
|
|
|
|
|
skb = __skb_dequeue_bad_txq(q);
|
|
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static inline void qdisc_enqueue_skb_bad_txq(struct Qdisc *q,
|
|
|
|
|
struct sk_buff *skb)
|
|
|
|
|
{
|
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
|
spin_lock(lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
__skb_queue_tail(&q->skb_bad_txq, skb);
|
|
|
|
|
|
2018-03-15 01:53:00 +00:00
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
|
qdisc_qstats_cpu_backlog_inc(q, skb);
|
2019-04-10 12:32:41 +00:00
|
|
|
qdisc_qstats_cpu_qlen_inc(q);
|
2018-03-15 01:53:00 +00:00
|
|
|
} else {
|
|
|
|
|
qdisc_qstats_backlog_inc(q, skb);
|
|
|
|
|
q->q.qlen++;
|
|
|
|
|
}
|
|
|
|
|
|
2017-12-07 17:56:23 +00:00
|
|
|
if (lock)
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
|
}
|
|
|
|
|
|
2019-04-10 12:32:39 +00:00
|
|
|
static inline void dev_requeue_skb(struct sk_buff *skb, struct Qdisc *q)
|
2007-06-11 00:31:24 +00:00
|
|
|
{
|
2019-04-10 12:32:39 +00:00
|
|
|
spinlock_t *lock = NULL;
|
2017-12-27 09:05:52 +00:00
|
|
|
|
2019-04-10 12:32:39 +00:00
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
|
spin_lock(lock);
|
2017-12-27 09:05:52 +00:00
|
|
|
}
|
2007-06-11 00:31:24 +00:00
|
|
|
|
2017-12-27 09:05:52 +00:00
|
|
|
while (skb) {
|
|
|
|
|
struct sk_buff *next = skb->next;
|
|
|
|
|
|
|
|
|
|
__skb_queue_tail(&q->gso_skb, skb);
|
|
|
|
|
|
2019-04-10 12:32:39 +00:00
|
|
|
/* it's still part of the queue */
|
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
|
qdisc_qstats_cpu_requeues_inc(q);
|
|
|
|
|
qdisc_qstats_cpu_backlog_inc(q, skb);
|
2019-04-10 12:32:41 +00:00
|
|
|
qdisc_qstats_cpu_qlen_inc(q);
|
2019-04-10 12:32:39 +00:00
|
|
|
} else {
|
|
|
|
|
q->qstats.requeues++;
|
|
|
|
|
qdisc_qstats_backlog_inc(q, skb);
|
|
|
|
|
q->q.qlen++;
|
|
|
|
|
}
|
2017-12-27 09:05:52 +00:00
|
|
|
|
|
|
|
|
skb = next;
|
|
|
|
|
}
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
|
|
|
|
|
if (lock) {
|
2019-04-10 12:32:39 +00:00
|
|
|
spin_unlock(lock);
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
set_bit(__QDISC_STATE_MISSED, &q->state);
|
|
|
|
|
} else {
|
|
|
|
|
__netif_schedule(q);
|
|
|
|
|
}
|
2017-12-07 17:55:45 +00:00
|
|
|
}
|
|
|
|
|
|
2014-10-03 22:31:07 +00:00
|
|
|
static void try_bulk_dequeue_skb(struct Qdisc *q,
|
|
|
|
|
struct sk_buff *skb,
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
const struct netdev_queue *txq,
|
|
|
|
|
int *packets)
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
{
|
2014-10-03 22:31:07 +00:00
|
|
|
int bytelimit = qdisc_avail_bulklimit(txq) - skb->len;
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
|
|
|
|
|
while (bytelimit > 0) {
|
2014-10-03 22:31:07 +00:00
|
|
|
struct sk_buff *nskb = q->dequeue(q);
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
|
2014-10-03 22:31:07 +00:00
|
|
|
if (!nskb)
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
break;
|
|
|
|
|
|
2014-10-03 22:31:07 +00:00
|
|
|
bytelimit -= nskb->len; /* covers GSO len */
|
|
|
|
|
skb->next = nskb;
|
|
|
|
|
skb = nskb;
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
(*packets)++; /* GSO counts as one pkt */
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
}
|
2018-07-30 03:42:53 +00:00
|
|
|
skb_mark_not_on_list(skb);
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
}
|
|
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
/* This variant of try_bulk_dequeue_skb() makes sure
|
|
|
|
|
* all skbs in the chain are for the same txq
|
|
|
|
|
*/
|
|
|
|
|
static void try_bulk_dequeue_skb_slow(struct Qdisc *q,
|
|
|
|
|
struct sk_buff *skb,
|
|
|
|
|
int *packets)
|
|
|
|
|
{
|
|
|
|
|
int mapping = skb_get_queue_mapping(skb);
|
|
|
|
|
struct sk_buff *nskb;
|
|
|
|
|
int cnt = 0;
|
|
|
|
|
|
|
|
|
|
do {
|
|
|
|
|
nskb = q->dequeue(q);
|
|
|
|
|
if (!nskb)
|
|
|
|
|
break;
|
|
|
|
|
if (unlikely(skb_get_queue_mapping(nskb) != mapping)) {
|
2017-12-07 17:56:23 +00:00
|
|
|
qdisc_enqueue_skb_bad_txq(q, nskb);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
skb->next = nskb;
|
|
|
|
|
skb = nskb;
|
|
|
|
|
} while (++cnt < 8);
|
|
|
|
|
(*packets) += cnt;
|
2018-07-30 03:42:53 +00:00
|
|
|
skb_mark_not_on_list(skb);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
}
|
|
|
|
|
|
qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE
Based on DaveM's recent API work on dev_hard_start_xmit(), that allows
sending/processing an entire skb list.
This patch implements qdisc bulk dequeue, by allowing multiple packets
to be dequeued in dequeue_skb().
The optimization principle for this is two fold, (1) to amortize
locking cost and (2) avoid expensive tailptr update for notifying HW.
(1) Several packets are dequeued while holding the qdisc root_lock,
amortizing locking cost over several packet. The dequeued SKB list is
processed under the TXQ lock in dev_hard_start_xmit(), thus also
amortizing the cost of the TXQ lock.
(2) Further more, dev_hard_start_xmit() will utilize the skb->xmit_more
API to delay HW tailptr update, which also reduces the cost per
packet.
One restriction of the new API is that every SKB must belong to the
same TXQ. This patch takes the easy way out, by restricting bulk
dequeue to qdisc's with the TCQ_F_ONETXQUEUE flag, that specifies the
qdisc only have attached a single TXQ.
Some detail about the flow; dev_hard_start_xmit() will process the skb
list, and transmit packets individually towards the driver (see
xmit_one()). In case the driver stops midway in the list, the
remaining skb list is returned by dev_hard_start_xmit(). In
sch_direct_xmit() this returned list is requeued by dev_requeue_skb().
To avoid overshooting the HW limits, which results in requeuing, the
patch limits the amount of bytes dequeued, based on the drivers BQL
limits. In-effect bulking will only happen for BQL enabled drivers.
Small amounts for extra HoL blocking (2x MTU/0.24ms) were
measured at 100Mbit/s, with bulking 8 packets, but the
oscillating nature of the measurement indicate something, like
sched latency might be causing this effect. More comparisons
show, that this oscillation goes away occationally. Thus, we
disregard this artifact completely and remove any "magic" bulking
limit.
For now, as a conservative approach, stop bulking when seeing TSO and
segmented GSO packets. They already benefit from bulking on their own.
A followup patch add this, to allow easier bisect-ability for finding
regressions.
Jointed work with Hannes, Daniel and Florian.
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: Florian Westphal <fw@strlen.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-01 20:35:59 +00:00
|
|
|
/* Note that dequeue_skb can possibly return a SKB list (via skb->next).
|
|
|
|
|
* A requeued skb (via q->gso_skb) can also be a SKB list.
|
|
|
|
|
*/
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
static struct sk_buff *dequeue_skb(struct Qdisc *q, bool *validate,
|
|
|
|
|
int *packets)
|
2007-06-11 00:31:24 +00:00
|
|
|
{
|
2012-12-11 15:54:33 +00:00
|
|
|
const struct netdev_queue *txq = q->dev_queue;
|
2017-12-07 17:56:42 +00:00
|
|
|
struct sk_buff *skb = NULL;
|
2008-10-06 16:54:39 +00:00
|
|
|
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
*packets = 1;
|
2017-12-07 17:55:45 +00:00
|
|
|
if (unlikely(!skb_queue_empty(&q->gso_skb))) {
|
|
|
|
|
spinlock_t *lock = NULL;
|
|
|
|
|
|
|
|
|
|
if (q->flags & TCQ_F_NOLOCK) {
|
|
|
|
|
lock = qdisc_lock(q);
|
|
|
|
|
spin_lock(lock);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
skb = skb_peek(&q->gso_skb);
|
|
|
|
|
|
|
|
|
|
/* skb may be null if another cpu pulls gso_skb off in between
|
|
|
|
|
* empty check and lock.
|
|
|
|
|
*/
|
|
|
|
|
if (!skb) {
|
|
|
|
|
if (lock)
|
|
|
|
|
spin_unlock(lock);
|
|
|
|
|
goto validate;
|
|
|
|
|
}
|
|
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
/* skb in gso_skb were already validated */
|
|
|
|
|
*validate = false;
|
2017-12-20 09:41:36 +00:00
|
|
|
if (xfrm_offload(skb))
|
|
|
|
|
*validate = true;
|
2008-09-23 05:16:23 +00:00
|
|
|
/* check the reason of requeuing without tx lock first */
|
2014-08-27 09:11:27 +00:00
|
|
|
txq = skb_get_tx_queue(txq->dev, skb);
|
2011-11-28 16:32:44 +00:00
|
|
|
if (!netif_xmit_frozen_or_stopped(txq)) {
|
2017-12-07 17:55:45 +00:00
|
|
|
skb = __skb_dequeue(&q->gso_skb);
|
|
|
|
|
if (qdisc_is_percpu_stats(q)) {
|
|
|
|
|
qdisc_qstats_cpu_backlog_dec(q, skb);
|
2019-04-10 12:32:41 +00:00
|
|
|
qdisc_qstats_cpu_qlen_dec(q);
|
2017-12-07 17:55:45 +00:00
|
|
|
} else {
|
|
|
|
|
qdisc_qstats_backlog_dec(q, skb);
|
|
|
|
|
q->q.qlen--;
|
|
|
|
|
}
|
|
|
|
|
} else {
|
2008-09-23 05:16:23 +00:00
|
|
|
skb = NULL;
|
2021-05-14 03:17:01 +00:00
|
|
|
qdisc_maybe_clear_missed(q, txq);
|
2017-12-07 17:55:45 +00:00
|
|
|
}
|
|
|
|
|
if (lock)
|
|
|
|
|
spin_unlock(lock);
|
2017-08-15 19:11:03 +00:00
|
|
|
goto trace;
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
}
|
2017-12-07 17:55:45 +00:00
|
|
|
validate:
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
*validate = true;
|
2017-12-07 17:56:42 +00:00
|
|
|
|
|
|
|
|
if ((q->flags & TCQ_F_ONETXQUEUE) &&
|
2021-05-14 03:17:01 +00:00
|
|
|
netif_xmit_frozen_or_stopped(txq)) {
|
|
|
|
|
qdisc_maybe_clear_missed(q, txq);
|
2017-12-07 17:56:42 +00:00
|
|
|
return skb;
|
2021-05-14 03:17:01 +00:00
|
|
|
}
|
2017-12-07 17:56:42 +00:00
|
|
|
|
2017-12-07 17:56:23 +00:00
|
|
|
skb = qdisc_dequeue_skb_bad_txq(q);
|
2019-09-05 12:20:22 +00:00
|
|
|
if (unlikely(skb)) {
|
|
|
|
|
if (skb == SKB_XOFF_MAGIC)
|
|
|
|
|
return NULL;
|
2017-12-07 17:56:23 +00:00
|
|
|
goto bulk;
|
2019-09-05 12:20:22 +00:00
|
|
|
}
|
2017-12-07 17:56:42 +00:00
|
|
|
skb = q->dequeue(q);
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
if (skb) {
|
|
|
|
|
bulk:
|
|
|
|
|
if (qdisc_may_bulk(q))
|
|
|
|
|
try_bulk_dequeue_skb(q, skb, txq, packets);
|
|
|
|
|
else
|
|
|
|
|
try_bulk_dequeue_skb_slow(q, skb, packets);
|
2008-09-23 05:16:23 +00:00
|
|
|
}
|
2017-08-15 19:11:03 +00:00
|
|
|
trace:
|
|
|
|
|
trace_qdisc_dequeue(q, txq, *packets, skb);
|
2007-06-11 00:31:24 +00:00
|
|
|
return skb;
|
|
|
|
|
}
|
|
|
|
|
|
2007-02-09 14:25:16 +00:00
|
|
|
/*
|
2014-09-02 14:35:33 +00:00
|
|
|
* Transmit possibly several skbs, and handle the return status as
|
net: sched: Remove Qdisc::running sequence counter
The Qdisc::running sequence counter has two uses:
1. Reliably reading qdisc's tc statistics while the qdisc is running
(a seqcount read/retry loop at gnet_stats_add_basic()).
2. As a flag, indicating whether the qdisc in question is running
(without any retry loops).
For the first usage, the Qdisc::running sequence counter write section,
qdisc_run_begin() => qdisc_run_end(), covers a much wider area than what
is actually needed: the raw qdisc's bstats update. A u64_stats sync
point was thus introduced (in previous commits) inside the bstats
structure itself. A local u64_stats write section is then started and
stopped for the bstats updates.
Use that u64_stats sync point mechanism for the bstats read/retry loop
at gnet_stats_add_basic().
For the second qdisc->running usage, a __QDISC_STATE_RUNNING bit flag,
accessed with atomic bitops, is sufficient. Using a bit flag instead of
a sequence counter at qdisc_run_begin/end() and qdisc_is_running() leads
to the SMP barriers implicitly added through raw_read_seqcount() and
write_seqcount_begin/end() getting removed. All call sites have been
surveyed though, and no required ordering was identified.
Now that the qdisc->running sequence counter is no longer used, remove
it.
Note, using u64_stats implies no sequence counter protection for 64-bit
architectures. This can lead to the qdisc tc statistics "packets" vs.
"bytes" values getting out of sync on rare occasions. The individual
values will still be valid.
Signed-off-by: Ahmed S. Darwish <a.darwish@linutronix.de>
Signed-off-by: Sebastian Andrzej Siewior <bigeasy@linutronix.de>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-10-16 08:49:10 +00:00
|
|
|
* required. Owning qdisc running bit guarantees that only one CPU
|
|
|
|
|
* can execute this function.
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-25 02:56:09 +00:00
|
|
|
*
|
|
|
|
|
* Returns to the caller:
|
2017-12-07 17:54:47 +00:00
|
|
|
* false - hardware queue frozen backoff
|
|
|
|
|
* true - feel free to send more pkts
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-25 02:56:09 +00:00
|
|
|
*/
|
2017-12-07 17:54:47 +00:00
|
|
|
bool sch_direct_xmit(struct sk_buff *skb, struct Qdisc *q,
|
|
|
|
|
struct net_device *dev, struct netdev_queue *txq,
|
|
|
|
|
spinlock_t *root_lock, bool validate)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2007-11-14 04:40:55 +00:00
|
|
|
int ret = NETDEV_TX_BUSY;
|
2017-12-20 09:41:36 +00:00
|
|
|
bool again = false;
|
2008-07-16 08:42:40 +00:00
|
|
|
|
|
|
|
|
/* And release qdisc */
|
2017-12-07 17:54:25 +00:00
|
|
|
if (root_lock)
|
|
|
|
|
spin_unlock(root_lock);
|
2007-06-11 00:31:24 +00:00
|
|
|
|
2014-10-03 22:31:07 +00:00
|
|
|
/* Note that we validate skb (GSO, checksum, ...) outside of locks */
|
|
|
|
|
if (validate)
|
2017-12-20 09:41:36 +00:00
|
|
|
skb = validate_xmit_skb_list(skb, dev, &again);
|
|
|
|
|
|
|
|
|
|
#ifdef CONFIG_XFRM_OFFLOAD
|
|
|
|
|
if (unlikely(again)) {
|
|
|
|
|
if (root_lock)
|
|
|
|
|
spin_lock(root_lock);
|
|
|
|
|
|
|
|
|
|
dev_requeue_skb(skb, q);
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
#endif
|
2009-11-10 06:14:14 +00:00
|
|
|
|
2016-04-12 06:45:52 +00:00
|
|
|
if (likely(skb)) {
|
2014-10-03 22:31:07 +00:00
|
|
|
HARD_TX_LOCK(dev, txq, smp_processor_id());
|
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq))
|
|
|
|
|
skb = dev_hard_start_xmit(skb, dev, txq, &ret);
|
2021-05-14 03:17:01 +00:00
|
|
|
else
|
|
|
|
|
qdisc_maybe_clear_missed(q, txq);
|
2007-06-11 00:31:24 +00:00
|
|
|
|
2014-10-03 22:31:07 +00:00
|
|
|
HARD_TX_UNLOCK(dev, txq);
|
2016-04-12 06:45:52 +00:00
|
|
|
} else {
|
2017-12-07 17:54:25 +00:00
|
|
|
if (root_lock)
|
|
|
|
|
spin_lock(root_lock);
|
2017-12-07 17:54:47 +00:00
|
|
|
return true;
|
2014-10-03 22:31:07 +00:00
|
|
|
}
|
2017-12-07 17:54:25 +00:00
|
|
|
|
|
|
|
|
if (root_lock)
|
|
|
|
|
spin_lock(root_lock);
|
2007-06-11 00:31:24 +00:00
|
|
|
|
2017-12-07 17:54:47 +00:00
|
|
|
if (!dev_xmit_complete(ret)) {
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-25 02:56:09 +00:00
|
|
|
/* Driver returned NETDEV_TX_BUSY - requeue skb */
|
2012-05-13 21:56:26 +00:00
|
|
|
if (unlikely(ret != NETDEV_TX_BUSY))
|
|
|
|
|
net_warn_ratelimited("BUG %s code %d qlen %d\n",
|
|
|
|
|
dev->name, ret, q->q.qlen);
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-25 02:56:09 +00:00
|
|
|
|
2017-12-07 17:54:47 +00:00
|
|
|
dev_requeue_skb(skb, q);
|
|
|
|
|
return false;
|
[NET]: qdisc_restart - readability changes plus one bug fix.
New changes :
- Incorporated Peter Waskiewicz's comments.
- Re-added back one warning message (on driver returning wrong value).
Previous changes :
- Converted to use switch/case code which looks neater.
- "if (ret == NETDEV_TX_LOCKED && lockless)" is buggy, and the lockless
check should be removed, since driver will return NETDEV_TX_LOCKED only
if lockless is true and driver has to do the locking. In the original
code as well as the latest code, this code can result in a bug where
if LLTX is not set for a driver (lockless == 0) but the driver is written
wrongly to do a trylock (despite LLTX being set), the driver returns
LOCKED. But since lockless is zero, the packet is requeue'd instead of
calling collision code which will issue warning and free up the skb.
Instead this skb will be retried with this driver next time, and the same
result will ensue. Removing this check will catch these driver bugs instead
of hiding the problem. I am keeping this change to readability section
since :
a. it is confusing to check two things as it is; and
b. it is difficult to keep this check in the changed 'switch' code.
- Changed some names, like try_get_tx_pkt to dev_dequeue_skb (as that is
the work being done and easier to understand) and do_dev_requeue to
dev_requeue_skb, merged handle_dev_cpu_collision and tx_islocked to
dev_handle_collision (handle_dev_cpu_collision is a small routine with only
one caller, so there is no need to have two separate routines which also
results in getting rid of two macros, etc.
- Removed an XXX comment as it should never fail (I suspect this was related
to batch skb WIP, Jamal ?). Converted some functions to original coding
style of having the return values and the function name on same line, eg
prio2list.
Signed-off-by: Krishna Kumar <krkumar2@in.ibm.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-06-25 02:56:09 +00:00
|
|
|
}
|
2007-06-11 00:31:24 +00:00
|
|
|
|
2017-12-07 17:54:47 +00:00
|
|
|
return true;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2009-08-06 01:44:21 +00:00
|
|
|
/*
|
|
|
|
|
* NOTE: Called under qdisc_lock(q) with locally disabled BH.
|
|
|
|
|
*
|
2016-06-06 16:37:15 +00:00
|
|
|
* running seqcount guarantees only one CPU can process
|
2009-08-06 01:44:21 +00:00
|
|
|
* this qdisc at a time. qdisc_lock(q) serializes queue accesses for
|
|
|
|
|
* this queue.
|
|
|
|
|
*
|
|
|
|
|
* netif_tx_lock serializes accesses to device driver.
|
|
|
|
|
*
|
|
|
|
|
* qdisc_lock(q) and netif_tx_lock are mutually exclusive,
|
|
|
|
|
* if one is grabbed, another must be free.
|
|
|
|
|
*
|
|
|
|
|
* Note, that this procedure can be called by a watchdog timer
|
|
|
|
|
*
|
|
|
|
|
* Returns to the caller:
|
|
|
|
|
* 0 - queue is empty or throttled.
|
|
|
|
|
* >0 - queue is not empty.
|
|
|
|
|
*
|
|
|
|
|
*/
|
2017-12-07 17:54:47 +00:00
|
|
|
static inline bool qdisc_restart(struct Qdisc *q, int *packets)
|
2009-08-06 01:44:21 +00:00
|
|
|
{
|
2017-12-07 17:54:25 +00:00
|
|
|
spinlock_t *root_lock = NULL;
|
2009-08-06 01:44:21 +00:00
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
struct net_device *dev;
|
|
|
|
|
struct sk_buff *skb;
|
2018-05-15 08:50:31 +00:00
|
|
|
bool validate;
|
2009-08-06 01:44:21 +00:00
|
|
|
|
|
|
|
|
/* Dequeue packet */
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
skb = dequeue_skb(q, &validate, packets);
|
2018-05-15 08:50:31 +00:00
|
|
|
if (unlikely(!skb))
|
2017-12-07 17:54:47 +00:00
|
|
|
return false;
|
2014-08-27 09:11:27 +00:00
|
|
|
|
2018-05-15 08:50:31 +00:00
|
|
|
if (!(q->flags & TCQ_F_NOLOCK))
|
2017-12-07 17:54:25 +00:00
|
|
|
root_lock = qdisc_lock(q);
|
|
|
|
|
|
2009-08-06 01:44:21 +00:00
|
|
|
dev = qdisc_dev(q);
|
2014-08-27 09:11:27 +00:00
|
|
|
txq = skb_get_tx_queue(dev, skb);
|
2009-08-06 01:44:21 +00:00
|
|
|
|
2018-05-15 08:50:31 +00:00
|
|
|
return sch_direct_xmit(skb, q, dev, txq, root_lock, validate);
|
2009-08-06 01:44:21 +00:00
|
|
|
}
|
|
|
|
|
|
2008-07-16 09:15:04 +00:00
|
|
|
void __qdisc_run(struct Qdisc *q)
|
2006-06-20 06:57:59 +00:00
|
|
|
{
|
2024-03-06 16:00:21 +00:00
|
|
|
int quota = READ_ONCE(net_hotdata.dev_tx_weight);
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
int packets;
|
2008-03-28 23:25:26 +00:00
|
|
|
|
net_sched: restore qdisc quota fairness limits after bulk dequeue
Restore the quota fairness between qdisc's, that we broke with commit
5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE").
Before that commit, the quota in __qdisc_run() were in packets as
dequeue_skb() would only dequeue a single packet, that assumption
broke with bulk dequeue.
We choose not to account for the number of packets inside the TSO/GSO
packets (accessable via "skb_gso_segs"). As the previous fairness
also had this "defect". Thus, GSO/TSO packets counts as a single
packet.
Further more, we choose to slack on accuracy, by allowing a bulk
dequeue try_bulk_dequeue_skb() to exceed the "packets" limit, only
limited by the BQL bytelimit. This is done because BQL prefers to get
its full budget for appropriate feedback from TX completion.
In future, we might consider reworking this further and, if it allows,
switch to a time-based model, as suggested by Eric. Right now, we only
restore old semantics.
Joint work with Eric, Hannes, Daniel and Jesper. Hannes wrote the
first patch in cooperation with Daniel and Jesper. Eric rewrote the
patch.
Fixes: 5772e9a346 ("qdisc: bulk dequeue support for qdiscs with TCQ_F_ONETXQUEUE")
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: Hannes Frederic Sowa <hannes@stressinduktion.org>
Signed-off-by: Daniel Borkmann <dborkman@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-10-09 10:18:10 +00:00
|
|
|
while (qdisc_restart(q, &packets)) {
|
|
|
|
|
quota -= packets;
|
net_sched: remove need_resched() from qdisc_run()
The introduction of this schedule point was done in commit
2ba2506ca7ca ("[NET]: Add preemption point in qdisc_run")
at a time the loop was not bounded.
Then later in commit d5b8aa1d246f ("net_sched: fix dequeuer fairness")
we added a limit on the number of packets.
Now is the time to remove the schedule point, since the default
limit of 64 packets matches the number of packets a typical NAPI
poll can process in a row.
This solves a latency problem for most TCP receivers under moderate load :
1) host receives a packet.
NET_RX_SOFTIRQ is raised by NIC hard IRQ handler
2) __do_softirq() does its first loop, handling NET_RX_SOFTIRQ
and calling the driver napi->loop() function
3) TCP stores the skb in socket receive queue:
4) TCP calls sk->sk_data_ready() and wakeups a user thread
waiting for EPOLLIN (as a result, need_resched() might now be true)
5) TCP cooks an ACK and sends it.
6) qdisc_run() processes one packet from qdisc, and sees need_resched(),
this raises NET_TX_SOFTIRQ (even if there are no more packets in
the qdisc)
Then we go back to the __do_softirq() in 2), and we see that new
softirqs were raised. Since need_resched() is true, we end up waking
ksoftirqd in this path :
if (pending) {
if (time_before(jiffies, end) && !need_resched() &&
--max_restart)
goto restart;
wakeup_softirqd();
}
So we have many wakeups of ksoftirqd kernel threads,
and more calls to qdisc_run() with associated lock overhead.
Note that another way to solve the issue would be to change TCP
to first send the ACK packet, then signal the EPOLLIN,
but this changes P99 latencies, as sending the ACK packet
can add a long delay.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-01 21:02:36 +00:00
|
|
|
if (quota <= 0) {
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
if (q->flags & TCQ_F_NOLOCK)
|
|
|
|
|
set_bit(__QDISC_STATE_MISSED, &q->state);
|
|
|
|
|
else
|
|
|
|
|
__netif_schedule(q);
|
|
|
|
|
|
2007-05-10 11:55:14 +00:00
|
|
|
break;
|
2008-03-28 23:25:26 +00:00
|
|
|
}
|
|
|
|
|
}
|
2006-06-20 06:57:59 +00:00
|
|
|
}
|
|
|
|
|
|
2009-05-18 03:55:16 +00:00
|
|
|
unsigned long dev_trans_start(struct net_device *dev)
|
|
|
|
|
{
|
net/sched: remove hacks added to dev_trans_start() for bonding to work
Now that the bonding driver keeps track of the last TX time of ARP and
NS probes, we effectively revert the following commits:
32d3e51a82d4 ("net_sched: use macvlan real dev trans_start in dev_trans_start()")
07ce76aa9bcf ("net_sched: make dev_trans_start return vlan's real dev trans_start")
Note that the approach of continuing to hack at this function would not
get us very far, hence the desire to take a different approach. DSA is
also a virtual device that uses NETIF_F_LLTX, but there, many uppers
share the same lower (DSA master, i.e. the physical host port of a
switch). By making dev_trans_start() on a DSA interface return the
dev_trans_start() of the master, we effectively assume that all other
DSA interfaces are silent, otherwise this corrupts the validity of the
probe timestamp data from the bonding driver's perspective.
Furthermore, the hacks didn't take into consideration the fact that the
lower interface of @dev may not have been physical either. For example,
VLAN over VLAN, or DSA with 2 masters in a LAG.
And even furthermore, there are NETIF_F_LLTX devices which are not
stacked, like veth. The hack here would not work with those, because it
would not have to provide the bonding driver something to chew at all.
Signed-off-by: Vladimir Oltean <vladimir.oltean@nxp.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-07-31 12:41:06 +00:00
|
|
|
unsigned long res = READ_ONCE(netdev_get_tx_queue(dev, 0)->trans_start);
|
|
|
|
|
unsigned long val;
|
2009-05-18 03:55:16 +00:00
|
|
|
unsigned int i;
|
|
|
|
|
|
2016-05-03 14:33:14 +00:00
|
|
|
for (i = 1; i < dev->num_tx_queues; i++) {
|
2021-11-17 03:29:22 +00:00
|
|
|
val = READ_ONCE(netdev_get_tx_queue(dev, i)->trans_start);
|
2009-05-18 03:55:16 +00:00
|
|
|
if (val && time_after(val, res))
|
|
|
|
|
res = val;
|
|
|
|
|
}
|
2013-08-03 20:07:47 +00:00
|
|
|
|
2009-05-18 03:55:16 +00:00
|
|
|
return res;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(dev_trans_start);
|
|
|
|
|
|
2021-11-17 03:29:23 +00:00
|
|
|
static void netif_freeze_queues(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
unsigned int i;
|
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
|
|
cpu = smp_processor_id();
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
|
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
|
|
|
|
|
|
|
|
|
|
/* We are the only thread of execution doing a
|
|
|
|
|
* freeze, but we have to grab the _xmit_lock in
|
|
|
|
|
* order to synchronize with threads which are in
|
|
|
|
|
* the ->hard_start_xmit() handler and already
|
|
|
|
|
* checked the frozen bit.
|
|
|
|
|
*/
|
|
|
|
|
__netif_tx_lock(txq, cpu);
|
|
|
|
|
set_bit(__QUEUE_STATE_FROZEN, &txq->state);
|
|
|
|
|
__netif_tx_unlock(txq);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void netif_tx_lock(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
spin_lock(&dev->tx_global_lock);
|
|
|
|
|
netif_freeze_queues(dev);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(netif_tx_lock);
|
|
|
|
|
|
|
|
|
|
static void netif_unfreeze_queues(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
|
struct netdev_queue *txq = netdev_get_tx_queue(dev, i);
|
|
|
|
|
|
|
|
|
|
/* No need to grab the _xmit_lock here. If the
|
|
|
|
|
* queue is not stopped for another reason, we
|
|
|
|
|
* force a schedule.
|
|
|
|
|
*/
|
|
|
|
|
clear_bit(__QUEUE_STATE_FROZEN, &txq->state);
|
|
|
|
|
netif_schedule_queue(txq);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void netif_tx_unlock(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
netif_unfreeze_queues(dev);
|
|
|
|
|
spin_unlock(&dev->tx_global_lock);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(netif_tx_unlock);
|
|
|
|
|
|
2017-10-17 00:29:17 +00:00
|
|
|
static void dev_watchdog(struct timer_list *t)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2017-10-17 00:29:17 +00:00
|
|
|
struct net_device *dev = from_timer(dev, t, watchdog_timer);
|
2021-12-07 01:30:30 +00:00
|
|
|
bool release = true;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2021-11-17 03:29:24 +00:00
|
|
|
spin_lock(&dev->tx_global_lock);
|
2008-07-17 07:34:19 +00:00
|
|
|
if (!qdisc_tx_is_noop(dev)) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (netif_device_present(dev) &&
|
|
|
|
|
netif_running(dev) &&
|
|
|
|
|
netif_carrier_ok(dev)) {
|
2023-04-21 08:26:06 +00:00
|
|
|
unsigned int timedout_ms = 0;
|
2008-07-17 07:34:19 +00:00
|
|
|
unsigned int i;
|
2009-05-18 03:55:16 +00:00
|
|
|
unsigned long trans_start;
|
2024-05-08 13:36:17 +00:00
|
|
|
unsigned long oldest_start = jiffies;
|
2008-07-17 07:34:19 +00:00
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, i);
|
2021-11-17 03:29:22 +00:00
|
|
|
trans_start = READ_ONCE(txq->trans_start);
|
2024-05-08 13:36:17 +00:00
|
|
|
if (!netif_xmit_stopped(txq))
|
|
|
|
|
continue;
|
|
|
|
|
if (time_after(jiffies, trans_start + dev->watchdog_timeo)) {
|
2023-04-21 08:26:06 +00:00
|
|
|
timedout_ms = jiffies_to_msecs(jiffies - trans_start);
|
2021-11-17 03:29:21 +00:00
|
|
|
atomic_long_inc(&txq->trans_timeout);
|
2008-07-17 07:34:19 +00:00
|
|
|
break;
|
|
|
|
|
}
|
2024-05-08 13:36:17 +00:00
|
|
|
if (time_after(oldest_start, trans_start))
|
|
|
|
|
oldest_start = trans_start;
|
2008-07-17 07:34:19 +00:00
|
|
|
}
|
2006-05-16 22:02:12 +00:00
|
|
|
|
2023-04-21 08:26:06 +00:00
|
|
|
if (unlikely(timedout_ms)) {
|
2019-05-02 02:56:59 +00:00
|
|
|
trace_net_dev_xmit_timeout(dev, i);
|
2023-11-14 05:11:42 +00:00
|
|
|
netdev_crit(dev, "NETDEV WATCHDOG: CPU: %d: transmit queue %u timed out %u ms\n",
|
|
|
|
|
raw_smp_processor_id(),
|
|
|
|
|
i, timedout_ms);
|
2021-11-17 03:29:24 +00:00
|
|
|
netif_freeze_queues(dev);
|
netdev: pass the stuck queue to the timeout handler
This allows incrementing the correct timeout statistic without any mess.
Down the road, devices can learn to reset just the specific queue.
The patch was generated with the following script:
use strict;
use warnings;
our $^I = '.bak';
my @work = (
["arch/m68k/emu/nfeth.c", "nfeth_tx_timeout"],
["arch/um/drivers/net_kern.c", "uml_net_tx_timeout"],
["arch/um/drivers/vector_kern.c", "vector_net_tx_timeout"],
["arch/xtensa/platforms/iss/network.c", "iss_net_tx_timeout"],
["drivers/char/pcmcia/synclink_cs.c", "hdlcdev_tx_timeout"],
["drivers/infiniband/ulp/ipoib/ipoib_main.c", "ipoib_timeout"],
["drivers/infiniband/ulp/ipoib/ipoib_main.c", "ipoib_timeout"],
["drivers/message/fusion/mptlan.c", "mpt_lan_tx_timeout"],
["drivers/misc/sgi-xp/xpnet.c", "xpnet_dev_tx_timeout"],
["drivers/net/appletalk/cops.c", "cops_timeout"],
["drivers/net/arcnet/arcdevice.h", "arcnet_timeout"],
["drivers/net/arcnet/arcnet.c", "arcnet_timeout"],
["drivers/net/arcnet/com20020.c", "arcnet_timeout"],
["drivers/net/ethernet/3com/3c509.c", "el3_tx_timeout"],
["drivers/net/ethernet/3com/3c515.c", "corkscrew_timeout"],
["drivers/net/ethernet/3com/3c574_cs.c", "el3_tx_timeout"],
["drivers/net/ethernet/3com/3c589_cs.c", "el3_tx_timeout"],
["drivers/net/ethernet/3com/3c59x.c", "vortex_tx_timeout"],
["drivers/net/ethernet/3com/3c59x.c", "vortex_tx_timeout"],
["drivers/net/ethernet/3com/typhoon.c", "typhoon_tx_timeout"],
["drivers/net/ethernet/8390/8390.h", "ei_tx_timeout"],
["drivers/net/ethernet/8390/8390.h", "eip_tx_timeout"],
["drivers/net/ethernet/8390/8390.c", "ei_tx_timeout"],
["drivers/net/ethernet/8390/8390p.c", "eip_tx_timeout"],
["drivers/net/ethernet/8390/ax88796.c", "ax_ei_tx_timeout"],
["drivers/net/ethernet/8390/axnet_cs.c", "axnet_tx_timeout"],
["drivers/net/ethernet/8390/etherh.c", "__ei_tx_timeout"],
["drivers/net/ethernet/8390/hydra.c", "__ei_tx_timeout"],
["drivers/net/ethernet/8390/mac8390.c", "__ei_tx_timeout"],
["drivers/net/ethernet/8390/mcf8390.c", "__ei_tx_timeout"],
["drivers/net/ethernet/8390/lib8390.c", "__ei_tx_timeout"],
["drivers/net/ethernet/8390/ne2k-pci.c", "ei_tx_timeout"],
["drivers/net/ethernet/8390/pcnet_cs.c", "ei_tx_timeout"],
["drivers/net/ethernet/8390/smc-ultra.c", "ei_tx_timeout"],
["drivers/net/ethernet/8390/wd.c", "ei_tx_timeout"],
["drivers/net/ethernet/8390/zorro8390.c", "__ei_tx_timeout"],
["drivers/net/ethernet/adaptec/starfire.c", "tx_timeout"],
["drivers/net/ethernet/agere/et131x.c", "et131x_tx_timeout"],
["drivers/net/ethernet/allwinner/sun4i-emac.c", "emac_timeout"],
["drivers/net/ethernet/alteon/acenic.c", "ace_watchdog"],
["drivers/net/ethernet/amazon/ena/ena_netdev.c", "ena_tx_timeout"],
["drivers/net/ethernet/amd/7990.h", "lance_tx_timeout"],
["drivers/net/ethernet/amd/7990.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/a2065.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/am79c961a.c", "am79c961_timeout"],
["drivers/net/ethernet/amd/amd8111e.c", "amd8111e_tx_timeout"],
["drivers/net/ethernet/amd/ariadne.c", "ariadne_tx_timeout"],
["drivers/net/ethernet/amd/atarilance.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/au1000_eth.c", "au1000_tx_timeout"],
["drivers/net/ethernet/amd/declance.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/lance.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/mvme147.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/ni65.c", "ni65_timeout"],
["drivers/net/ethernet/amd/nmclan_cs.c", "mace_tx_timeout"],
["drivers/net/ethernet/amd/pcnet32.c", "pcnet32_tx_timeout"],
["drivers/net/ethernet/amd/sunlance.c", "lance_tx_timeout"],
["drivers/net/ethernet/amd/xgbe/xgbe-drv.c", "xgbe_tx_timeout"],
["drivers/net/ethernet/apm/xgene-v2/main.c", "xge_timeout"],
["drivers/net/ethernet/apm/xgene/xgene_enet_main.c", "xgene_enet_timeout"],
["drivers/net/ethernet/apple/macmace.c", "mace_tx_timeout"],
["drivers/net/ethernet/atheros/ag71xx.c", "ag71xx_tx_timeout"],
["drivers/net/ethernet/atheros/alx/main.c", "alx_tx_timeout"],
["drivers/net/ethernet/atheros/atl1c/atl1c_main.c", "atl1c_tx_timeout"],
["drivers/net/ethernet/atheros/atl1e/atl1e_main.c", "atl1e_tx_timeout"],
["drivers/net/ethernet/atheros/atlx/atl.c", "atlx_tx_timeout"],
["drivers/net/ethernet/atheros/atlx/atl1.c", "atlx_tx_timeout"],
["drivers/net/ethernet/atheros/atlx/atl2.c", "atl2_tx_timeout"],
["drivers/net/ethernet/broadcom/b44.c", "b44_tx_timeout"],
["drivers/net/ethernet/broadcom/bcmsysport.c", "bcm_sysport_tx_timeout"],
["drivers/net/ethernet/broadcom/bnx2.c", "bnx2_tx_timeout"],
["drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.h", "bnx2x_tx_timeout"],
["drivers/net/ethernet/broadcom/bnx2x/bnx2x_cmn.c", "bnx2x_tx_timeout"],
["drivers/net/ethernet/broadcom/bnx2x/bnx2x_main.c", "bnx2x_tx_timeout"],
["drivers/net/ethernet/broadcom/bnxt/bnxt.c", "bnxt_tx_timeout"],
["drivers/net/ethernet/broadcom/genet/bcmgenet.c", "bcmgenet_timeout"],
["drivers/net/ethernet/broadcom/sb1250-mac.c", "sbmac_tx_timeout"],
["drivers/net/ethernet/broadcom/tg3.c", "tg3_tx_timeout"],
["drivers/net/ethernet/calxeda/xgmac.c", "xgmac_tx_timeout"],
["drivers/net/ethernet/cavium/liquidio/lio_main.c", "liquidio_tx_timeout"],
["drivers/net/ethernet/cavium/liquidio/lio_vf_main.c", "liquidio_tx_timeout"],
["drivers/net/ethernet/cavium/liquidio/lio_vf_rep.c", "lio_vf_rep_tx_timeout"],
["drivers/net/ethernet/cavium/thunder/nicvf_main.c", "nicvf_tx_timeout"],
["drivers/net/ethernet/cirrus/cs89x0.c", "net_timeout"],
["drivers/net/ethernet/cisco/enic/enic_main.c", "enic_tx_timeout"],
["drivers/net/ethernet/cisco/enic/enic_main.c", "enic_tx_timeout"],
["drivers/net/ethernet/cortina/gemini.c", "gmac_tx_timeout"],
["drivers/net/ethernet/davicom/dm9000.c", "dm9000_timeout"],
["drivers/net/ethernet/dec/tulip/de2104x.c", "de_tx_timeout"],
["drivers/net/ethernet/dec/tulip/tulip_core.c", "tulip_tx_timeout"],
["drivers/net/ethernet/dec/tulip/winbond-840.c", "tx_timeout"],
["drivers/net/ethernet/dlink/dl2k.c", "rio_tx_timeout"],
["drivers/net/ethernet/dlink/sundance.c", "tx_timeout"],
["drivers/net/ethernet/emulex/benet/be_main.c", "be_tx_timeout"],
["drivers/net/ethernet/ethoc.c", "ethoc_tx_timeout"],
["drivers/net/ethernet/faraday/ftgmac100.c", "ftgmac100_tx_timeout"],
["drivers/net/ethernet/fealnx.c", "fealnx_tx_timeout"],
["drivers/net/ethernet/freescale/dpaa/dpaa_eth.c", "dpaa_tx_timeout"],
["drivers/net/ethernet/freescale/fec_main.c", "fec_timeout"],
["drivers/net/ethernet/freescale/fec_mpc52xx.c", "mpc52xx_fec_tx_timeout"],
["drivers/net/ethernet/freescale/fs_enet/fs_enet-main.c", "fs_timeout"],
["drivers/net/ethernet/freescale/gianfar.c", "gfar_timeout"],
["drivers/net/ethernet/freescale/ucc_geth.c", "ucc_geth_timeout"],
["drivers/net/ethernet/fujitsu/fmvj18x_cs.c", "fjn_tx_timeout"],
["drivers/net/ethernet/google/gve/gve_main.c", "gve_tx_timeout"],
["drivers/net/ethernet/hisilicon/hip04_eth.c", "hip04_timeout"],
["drivers/net/ethernet/hisilicon/hix5hd2_gmac.c", "hix5hd2_net_timeout"],
["drivers/net/ethernet/hisilicon/hns/hns_enet.c", "hns_nic_net_timeout"],
["drivers/net/ethernet/hisilicon/hns3/hns3_enet.c", "hns3_nic_net_timeout"],
["drivers/net/ethernet/huawei/hinic/hinic_main.c", "hinic_tx_timeout"],
["drivers/net/ethernet/i825xx/82596.c", "i596_tx_timeout"],
["drivers/net/ethernet/i825xx/ether1.c", "ether1_timeout"],
["drivers/net/ethernet/i825xx/lib82596.c", "i596_tx_timeout"],
["drivers/net/ethernet/i825xx/sun3_82586.c", "sun3_82586_timeout"],
["drivers/net/ethernet/ibm/ehea/ehea_main.c", "ehea_tx_watchdog"],
["drivers/net/ethernet/ibm/emac/core.c", "emac_tx_timeout"],
["drivers/net/ethernet/ibm/emac/core.c", "emac_tx_timeout"],
["drivers/net/ethernet/ibm/ibmvnic.c", "ibmvnic_tx_timeout"],
["drivers/net/ethernet/intel/e100.c", "e100_tx_timeout"],
["drivers/net/ethernet/intel/e1000/e1000_main.c", "e1000_tx_timeout"],
["drivers/net/ethernet/intel/e1000e/netdev.c", "e1000_tx_timeout"],
["drivers/net/ethernet/intel/fm10k/fm10k_netdev.c", "fm10k_tx_timeout"],
["drivers/net/ethernet/intel/i40e/i40e_main.c", "i40e_tx_timeout"],
["drivers/net/ethernet/intel/iavf/iavf_main.c", "iavf_tx_timeout"],
["drivers/net/ethernet/intel/ice/ice_main.c", "ice_tx_timeout"],
["drivers/net/ethernet/intel/ice/ice_main.c", "ice_tx_timeout"],
["drivers/net/ethernet/intel/igb/igb_main.c", "igb_tx_timeout"],
["drivers/net/ethernet/intel/igbvf/netdev.c", "igbvf_tx_timeout"],
["drivers/net/ethernet/intel/ixgb/ixgb_main.c", "ixgb_tx_timeout"],
["drivers/net/ethernet/intel/ixgbe/ixgbe_debugfs.c", "adapter->netdev->netdev_ops->ndo_tx_timeout(adapter->netdev);"],
["drivers/net/ethernet/intel/ixgbe/ixgbe_main.c", "ixgbe_tx_timeout"],
["drivers/net/ethernet/intel/ixgbevf/ixgbevf_main.c", "ixgbevf_tx_timeout"],
["drivers/net/ethernet/jme.c", "jme_tx_timeout"],
["drivers/net/ethernet/korina.c", "korina_tx_timeout"],
["drivers/net/ethernet/lantiq_etop.c", "ltq_etop_tx_timeout"],
["drivers/net/ethernet/marvell/mv643xx_eth.c", "mv643xx_eth_tx_timeout"],
["drivers/net/ethernet/marvell/pxa168_eth.c", "pxa168_eth_tx_timeout"],
["drivers/net/ethernet/marvell/skge.c", "skge_tx_timeout"],
["drivers/net/ethernet/marvell/sky2.c", "sky2_tx_timeout"],
["drivers/net/ethernet/marvell/sky2.c", "sky2_tx_timeout"],
["drivers/net/ethernet/mediatek/mtk_eth_soc.c", "mtk_tx_timeout"],
["drivers/net/ethernet/mellanox/mlx4/en_netdev.c", "mlx4_en_tx_timeout"],
["drivers/net/ethernet/mellanox/mlx4/en_netdev.c", "mlx4_en_tx_timeout"],
["drivers/net/ethernet/mellanox/mlx5/core/en_main.c", "mlx5e_tx_timeout"],
["drivers/net/ethernet/micrel/ks8842.c", "ks8842_tx_timeout"],
["drivers/net/ethernet/micrel/ksz884x.c", "netdev_tx_timeout"],
["drivers/net/ethernet/microchip/enc28j60.c", "enc28j60_tx_timeout"],
["drivers/net/ethernet/microchip/encx24j600.c", "encx24j600_tx_timeout"],
["drivers/net/ethernet/natsemi/sonic.h", "sonic_tx_timeout"],
["drivers/net/ethernet/natsemi/sonic.c", "sonic_tx_timeout"],
["drivers/net/ethernet/natsemi/jazzsonic.c", "sonic_tx_timeout"],
["drivers/net/ethernet/natsemi/macsonic.c", "sonic_tx_timeout"],
["drivers/net/ethernet/natsemi/natsemi.c", "ns_tx_timeout"],
["drivers/net/ethernet/natsemi/ns83820.c", "ns83820_tx_timeout"],
["drivers/net/ethernet/natsemi/xtsonic.c", "sonic_tx_timeout"],
["drivers/net/ethernet/neterion/s2io.h", "s2io_tx_watchdog"],
["drivers/net/ethernet/neterion/s2io.c", "s2io_tx_watchdog"],
["drivers/net/ethernet/neterion/vxge/vxge-main.c", "vxge_tx_watchdog"],
["drivers/net/ethernet/netronome/nfp/nfp_net_common.c", "nfp_net_tx_timeout"],
["drivers/net/ethernet/nvidia/forcedeth.c", "nv_tx_timeout"],
["drivers/net/ethernet/nvidia/forcedeth.c", "nv_tx_timeout"],
["drivers/net/ethernet/oki-semi/pch_gbe/pch_gbe_main.c", "pch_gbe_tx_timeout"],
["drivers/net/ethernet/packetengines/hamachi.c", "hamachi_tx_timeout"],
["drivers/net/ethernet/packetengines/yellowfin.c", "yellowfin_tx_timeout"],
["drivers/net/ethernet/pensando/ionic/ionic_lif.c", "ionic_tx_timeout"],
["drivers/net/ethernet/qlogic/netxen/netxen_nic_main.c", "netxen_tx_timeout"],
["drivers/net/ethernet/qlogic/qla3xxx.c", "ql3xxx_tx_timeout"],
["drivers/net/ethernet/qlogic/qlcnic/qlcnic_main.c", "qlcnic_tx_timeout"],
["drivers/net/ethernet/qualcomm/emac/emac.c", "emac_tx_timeout"],
["drivers/net/ethernet/qualcomm/qca_spi.c", "qcaspi_netdev_tx_timeout"],
["drivers/net/ethernet/qualcomm/qca_uart.c", "qcauart_netdev_tx_timeout"],
["drivers/net/ethernet/rdc/r6040.c", "r6040_tx_timeout"],
["drivers/net/ethernet/realtek/8139cp.c", "cp_tx_timeout"],
["drivers/net/ethernet/realtek/8139too.c", "rtl8139_tx_timeout"],
["drivers/net/ethernet/realtek/atp.c", "tx_timeout"],
["drivers/net/ethernet/realtek/r8169_main.c", "rtl8169_tx_timeout"],
["drivers/net/ethernet/renesas/ravb_main.c", "ravb_tx_timeout"],
["drivers/net/ethernet/renesas/sh_eth.c", "sh_eth_tx_timeout"],
["drivers/net/ethernet/renesas/sh_eth.c", "sh_eth_tx_timeout"],
["drivers/net/ethernet/samsung/sxgbe/sxgbe_main.c", "sxgbe_tx_timeout"],
["drivers/net/ethernet/seeq/ether3.c", "ether3_timeout"],
["drivers/net/ethernet/seeq/sgiseeq.c", "timeout"],
["drivers/net/ethernet/sfc/efx.c", "efx_watchdog"],
["drivers/net/ethernet/sfc/falcon/efx.c", "ef4_watchdog"],
["drivers/net/ethernet/sgi/ioc3-eth.c", "ioc3_timeout"],
["drivers/net/ethernet/sgi/meth.c", "meth_tx_timeout"],
["drivers/net/ethernet/silan/sc92031.c", "sc92031_tx_timeout"],
["drivers/net/ethernet/sis/sis190.c", "sis190_tx_timeout"],
["drivers/net/ethernet/sis/sis900.c", "sis900_tx_timeout"],
["drivers/net/ethernet/smsc/epic100.c", "epic_tx_timeout"],
["drivers/net/ethernet/smsc/smc911x.c", "smc911x_timeout"],
["drivers/net/ethernet/smsc/smc9194.c", "smc_timeout"],
["drivers/net/ethernet/smsc/smc91c92_cs.c", "smc_tx_timeout"],
["drivers/net/ethernet/smsc/smc91x.c", "smc_timeout"],
["drivers/net/ethernet/stmicro/stmmac/stmmac_main.c", "stmmac_tx_timeout"],
["drivers/net/ethernet/sun/cassini.c", "cas_tx_timeout"],
["drivers/net/ethernet/sun/ldmvsw.c", "sunvnet_tx_timeout_common"],
["drivers/net/ethernet/sun/niu.c", "niu_tx_timeout"],
["drivers/net/ethernet/sun/sunbmac.c", "bigmac_tx_timeout"],
["drivers/net/ethernet/sun/sungem.c", "gem_tx_timeout"],
["drivers/net/ethernet/sun/sunhme.c", "happy_meal_tx_timeout"],
["drivers/net/ethernet/sun/sunqe.c", "qe_tx_timeout"],
["drivers/net/ethernet/sun/sunvnet.c", "sunvnet_tx_timeout_common"],
["drivers/net/ethernet/sun/sunvnet_common.c", "sunvnet_tx_timeout_common"],
["drivers/net/ethernet/sun/sunvnet_common.h", "sunvnet_tx_timeout_common"],
["drivers/net/ethernet/synopsys/dwc-xlgmac-net.c", "xlgmac_tx_timeout"],
["drivers/net/ethernet/ti/cpmac.c", "cpmac_tx_timeout"],
["drivers/net/ethernet/ti/cpsw.c", "cpsw_ndo_tx_timeout"],
["drivers/net/ethernet/ti/cpsw_priv.c", "cpsw_ndo_tx_timeout"],
["drivers/net/ethernet/ti/cpsw_priv.h", "cpsw_ndo_tx_timeout"],
["drivers/net/ethernet/ti/davinci_emac.c", "emac_dev_tx_timeout"],
["drivers/net/ethernet/ti/netcp_core.c", "netcp_ndo_tx_timeout"],
["drivers/net/ethernet/ti/tlan.c", "tlan_tx_timeout"],
["drivers/net/ethernet/toshiba/ps3_gelic_net.h", "gelic_net_tx_timeout"],
["drivers/net/ethernet/toshiba/ps3_gelic_net.c", "gelic_net_tx_timeout"],
["drivers/net/ethernet/toshiba/ps3_gelic_wireless.c", "gelic_net_tx_timeout"],
["drivers/net/ethernet/toshiba/spider_net.c", "spider_net_tx_timeout"],
["drivers/net/ethernet/toshiba/tc35815.c", "tc35815_tx_timeout"],
["drivers/net/ethernet/via/via-rhine.c", "rhine_tx_timeout"],
["drivers/net/ethernet/wiznet/w5100.c", "w5100_tx_timeout"],
["drivers/net/ethernet/wiznet/w5300.c", "w5300_tx_timeout"],
["drivers/net/ethernet/xilinx/xilinx_emaclite.c", "xemaclite_tx_timeout"],
["drivers/net/ethernet/xircom/xirc2ps_cs.c", "xirc_tx_timeout"],
["drivers/net/fjes/fjes_main.c", "fjes_tx_retry"],
["drivers/net/slip/slip.c", "sl_tx_timeout"],
["include/linux/usb/usbnet.h", "usbnet_tx_timeout"],
["drivers/net/usb/aqc111.c", "usbnet_tx_timeout"],
["drivers/net/usb/asix_devices.c", "usbnet_tx_timeout"],
["drivers/net/usb/asix_devices.c", "usbnet_tx_timeout"],
["drivers/net/usb/asix_devices.c", "usbnet_tx_timeout"],
["drivers/net/usb/ax88172a.c", "usbnet_tx_timeout"],
["drivers/net/usb/ax88179_178a.c", "usbnet_tx_timeout"],
["drivers/net/usb/catc.c", "catc_tx_timeout"],
["drivers/net/usb/cdc_mbim.c", "usbnet_tx_timeout"],
["drivers/net/usb/cdc_ncm.c", "usbnet_tx_timeout"],
["drivers/net/usb/dm9601.c", "usbnet_tx_timeout"],
["drivers/net/usb/hso.c", "hso_net_tx_timeout"],
["drivers/net/usb/int51x1.c", "usbnet_tx_timeout"],
["drivers/net/usb/ipheth.c", "ipheth_tx_timeout"],
["drivers/net/usb/kaweth.c", "kaweth_tx_timeout"],
["drivers/net/usb/lan78xx.c", "lan78xx_tx_timeout"],
["drivers/net/usb/mcs7830.c", "usbnet_tx_timeout"],
["drivers/net/usb/pegasus.c", "pegasus_tx_timeout"],
["drivers/net/usb/qmi_wwan.c", "usbnet_tx_timeout"],
["drivers/net/usb/r8152.c", "rtl8152_tx_timeout"],
["drivers/net/usb/rndis_host.c", "usbnet_tx_timeout"],
["drivers/net/usb/rtl8150.c", "rtl8150_tx_timeout"],
["drivers/net/usb/sierra_net.c", "usbnet_tx_timeout"],
["drivers/net/usb/smsc75xx.c", "usbnet_tx_timeout"],
["drivers/net/usb/smsc95xx.c", "usbnet_tx_timeout"],
["drivers/net/usb/sr9700.c", "usbnet_tx_timeout"],
["drivers/net/usb/sr9800.c", "usbnet_tx_timeout"],
["drivers/net/usb/usbnet.c", "usbnet_tx_timeout"],
["drivers/net/vmxnet3/vmxnet3_drv.c", "vmxnet3_tx_timeout"],
["drivers/net/wan/cosa.c", "cosa_net_timeout"],
["drivers/net/wan/farsync.c", "fst_tx_timeout"],
["drivers/net/wan/fsl_ucc_hdlc.c", "uhdlc_tx_timeout"],
["drivers/net/wan/lmc/lmc_main.c", "lmc_driver_timeout"],
["drivers/net/wan/x25_asy.c", "x25_asy_timeout"],
["drivers/net/wimax/i2400m/netdev.c", "i2400m_tx_timeout"],
["drivers/net/wireless/intel/ipw2x00/ipw2100.c", "ipw2100_tx_timeout"],
["drivers/net/wireless/intersil/hostap/hostap_main.c", "prism2_tx_timeout"],
["drivers/net/wireless/intersil/hostap/hostap_main.c", "prism2_tx_timeout"],
["drivers/net/wireless/intersil/hostap/hostap_main.c", "prism2_tx_timeout"],
["drivers/net/wireless/intersil/orinoco/main.c", "orinoco_tx_timeout"],
["drivers/net/wireless/intersil/orinoco/orinoco_usb.c", "orinoco_tx_timeout"],
["drivers/net/wireless/intersil/orinoco/orinoco.h", "orinoco_tx_timeout"],
["drivers/net/wireless/intersil/prism54/islpci_dev.c", "islpci_eth_tx_timeout"],
["drivers/net/wireless/intersil/prism54/islpci_eth.c", "islpci_eth_tx_timeout"],
["drivers/net/wireless/intersil/prism54/islpci_eth.h", "islpci_eth_tx_timeout"],
["drivers/net/wireless/marvell/mwifiex/main.c", "mwifiex_tx_timeout"],
["drivers/net/wireless/quantenna/qtnfmac/core.c", "qtnf_netdev_tx_timeout"],
["drivers/net/wireless/quantenna/qtnfmac/core.h", "qtnf_netdev_tx_timeout"],
["drivers/net/wireless/rndis_wlan.c", "usbnet_tx_timeout"],
["drivers/net/wireless/wl3501_cs.c", "wl3501_tx_timeout"],
["drivers/net/wireless/zydas/zd1201.c", "zd1201_tx_timeout"],
["drivers/s390/net/qeth_core.h", "qeth_tx_timeout"],
["drivers/s390/net/qeth_core_main.c", "qeth_tx_timeout"],
["drivers/s390/net/qeth_l2_main.c", "qeth_tx_timeout"],
["drivers/s390/net/qeth_l2_main.c", "qeth_tx_timeout"],
["drivers/s390/net/qeth_l3_main.c", "qeth_tx_timeout"],
["drivers/s390/net/qeth_l3_main.c", "qeth_tx_timeout"],
["drivers/staging/ks7010/ks_wlan_net.c", "ks_wlan_tx_timeout"],
["drivers/staging/qlge/qlge_main.c", "qlge_tx_timeout"],
["drivers/staging/rtl8192e/rtl8192e/rtl_core.c", "_rtl92e_tx_timeout"],
["drivers/staging/rtl8192u/r8192U_core.c", "tx_timeout"],
["drivers/staging/unisys/visornic/visornic_main.c", "visornic_xmit_timeout"],
["drivers/staging/wlan-ng/p80211netdev.c", "p80211knetdev_tx_timeout"],
["drivers/tty/n_gsm.c", "gsm_mux_net_tx_timeout"],
["drivers/tty/synclink.c", "hdlcdev_tx_timeout"],
["drivers/tty/synclink_gt.c", "hdlcdev_tx_timeout"],
["drivers/tty/synclinkmp.c", "hdlcdev_tx_timeout"],
["net/atm/lec.c", "lec_tx_timeout"],
["net/bluetooth/bnep/netdev.c", "bnep_net_timeout"]
);
for my $p (@work) {
my @pair = @$p;
my $file = $pair[0];
my $func = $pair[1];
print STDERR $file , ": ", $func,"\n";
our @ARGV = ($file);
while (<ARGV>) {
if (m/($func\s*\(struct\s+net_device\s+\*[A-Za-z_]?[A-Za-z-0-9_]*)(\))/) {
print STDERR "found $1+$2 in $file\n";
}
if (s/($func\s*\(struct\s+net_device\s+\*[A-Za-z_]?[A-Za-z-0-9_]*)(\))/$1, unsigned int txqueue$2/) {
print STDERR "$func found in $file\n";
}
print;
}
}
where the list of files and functions is simply from:
git grep ndo_tx_timeout, with manual addition of headers
in the rare cases where the function is from a header,
then manually changing the few places which actually
call ndo_tx_timeout.
Signed-off-by: Michael S. Tsirkin <mst@redhat.com>
Acked-by: Heiner Kallweit <hkallweit1@gmail.com>
Acked-by: Jakub Kicinski <jakub.kicinski@netronome.com>
Acked-by: Shannon Nelson <snelson@pensando.io>
Reviewed-by: Martin Habets <mhabets@solarflare.com>
changes from v9:
fixup a forward declaration
changes from v9:
more leftovers from v3 change
changes from v8:
fix up a missing direct call to timeout
rebased on net-next
changes from v7:
fixup leftovers from v3 change
changes from v6:
fix typo in rtl driver
changes from v5:
add missing files (allow any net device argument name)
changes from v4:
add a missing driver header
changes from v3:
change queue # to unsigned
Changes from v2:
added headers
Changes from v1:
Fix errors found by kbuild:
generalize the pattern a bit, to pick up
a couple of instances missed by the previous
version.
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-12-10 14:23:51 +00:00
|
|
|
dev->netdev_ops->ndo_tx_timeout(dev, i);
|
2021-11-17 03:29:24 +00:00
|
|
|
netif_unfreeze_queues(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2008-07-17 07:34:19 +00:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
2024-05-08 13:36:17 +00:00
|
|
|
round_jiffies(oldest_start +
|
2008-07-17 07:34:19 +00:00
|
|
|
dev->watchdog_timeo)))
|
2021-12-07 01:30:30 +00:00
|
|
|
release = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
}
|
2021-11-17 03:29:24 +00:00
|
|
|
spin_unlock(&dev->tx_global_lock);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2021-12-07 01:30:30 +00:00
|
|
|
if (release)
|
2022-06-08 04:39:55 +00:00
|
|
|
netdev_put(dev, &dev->watchdog_dev_tracker);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void __netdev_watchdog_up(struct net_device *dev)
|
|
|
|
|
{
|
2008-11-20 05:32:24 +00:00
|
|
|
if (dev->netdev_ops->ndo_tx_timeout) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (dev->watchdog_timeo <= 0)
|
|
|
|
|
dev->watchdog_timeo = 5*HZ;
|
2007-06-01 04:28:44 +00:00
|
|
|
if (!mod_timer(&dev->watchdog_timer,
|
|
|
|
|
round_jiffies(jiffies + dev->watchdog_timeo)))
|
2022-06-08 04:39:55 +00:00
|
|
|
netdev_hold(dev, &dev->watchdog_dev_tracker,
|
|
|
|
|
GFP_ATOMIC);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
}
|
2020-06-09 20:11:54 +00:00
|
|
|
EXPORT_SYMBOL_GPL(__netdev_watchdog_up);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
static void dev_watchdog_up(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
static void dev_watchdog_down(struct net_device *dev)
|
|
|
|
|
{
|
2006-06-09 19:20:56 +00:00
|
|
|
netif_tx_lock_bh(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (del_timer(&dev->watchdog_timer))
|
2022-06-08 04:39:55 +00:00
|
|
|
netdev_put(dev, &dev->watchdog_dev_tracker);
|
2006-06-09 19:20:56 +00:00
|
|
|
netif_tx_unlock_bh(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 23:41:36 +00:00
|
|
|
/**
|
|
|
|
|
* netif_carrier_on - set carrier
|
|
|
|
|
* @dev: network device
|
|
|
|
|
*
|
2019-02-07 16:14:32 +00:00
|
|
|
* Device has detected acquisition of carrier.
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 23:41:36 +00:00
|
|
|
*/
|
2005-08-11 22:32:53 +00:00
|
|
|
void netif_carrier_on(struct net_device *dev)
|
|
|
|
|
{
|
2007-10-18 06:26:43 +00:00
|
|
|
if (test_and_clear_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
2008-11-19 23:33:54 +00:00
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
|
return;
|
2018-01-18 17:59:13 +00:00
|
|
|
atomic_inc(&dev->carrier_up_count);
|
2005-08-11 22:32:53 +00:00
|
|
|
linkwatch_fire_event(dev);
|
2007-10-18 06:26:43 +00:00
|
|
|
if (netif_running(dev))
|
|
|
|
|
__netdev_watchdog_up(dev);
|
|
|
|
|
}
|
2005-08-11 22:32:53 +00:00
|
|
|
}
|
2008-01-23 06:10:23 +00:00
|
|
|
EXPORT_SYMBOL(netif_carrier_on);
|
2005-08-11 22:32:53 +00:00
|
|
|
|
[NET]: Make NAPI polling independent of struct net_device objects.
Several devices have multiple independant RX queues per net
device, and some have a single interrupt doorbell for several
queues.
In either case, it's easier to support layouts like that if the
structure representing the poll is independant from the net
device itself.
The signature of the ->poll() call back goes from:
int foo_poll(struct net_device *dev, int *budget)
to
int foo_poll(struct napi_struct *napi, int budget)
The caller is returned the number of RX packets processed (or
the number of "NAPI credits" consumed if you want to get
abstract). The callee no longer messes around bumping
dev->quota, *budget, etc. because that is all handled in the
caller upon return.
The napi_struct is to be embedded in the device driver private data
structures.
Furthermore, it is the driver's responsibility to disable all NAPI
instances in it's ->stop() device close handler. Since the
napi_struct is privatized into the driver's private data structures,
only the driver knows how to get at all of the napi_struct instances
it may have per-device.
With lots of help and suggestions from Rusty Russell, Roland Dreier,
Michael Chan, Jeff Garzik, and Jamal Hadi Salim.
Bug fixes from Thomas Graf, Roland Dreier, Peter Zijlstra,
Joseph Fannin, Scott Wood, Hans J. Koch, and Michael Chan.
[ Ported to current tree and all drivers converted. Integrated
Stephen's follow-on kerneldoc additions, and restored poll_list
handling to the old style to fix mutual exclusion issues. -DaveM ]
Signed-off-by: Stephen Hemminger <shemminger@linux-foundation.org>
Signed-off-by: David S. Miller <davem@davemloft.net>
2007-10-03 23:41:36 +00:00
|
|
|
/**
|
|
|
|
|
* netif_carrier_off - clear carrier
|
|
|
|
|
* @dev: network device
|
|
|
|
|
*
|
|
|
|
|
* Device has detected loss of carrier.
|
|
|
|
|
*/
|
2005-08-11 22:32:53 +00:00
|
|
|
void netif_carrier_off(struct net_device *dev)
|
|
|
|
|
{
|
2008-11-19 23:33:54 +00:00
|
|
|
if (!test_and_set_bit(__LINK_STATE_NOCARRIER, &dev->state)) {
|
|
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
|
return;
|
2018-01-18 17:59:13 +00:00
|
|
|
atomic_inc(&dev->carrier_down_count);
|
2005-08-11 22:32:53 +00:00
|
|
|
linkwatch_fire_event(dev);
|
2008-11-19 23:33:54 +00:00
|
|
|
}
|
2005-08-11 22:32:53 +00:00
|
|
|
}
|
2008-01-23 06:10:23 +00:00
|
|
|
EXPORT_SYMBOL(netif_carrier_off);
|
2005-08-11 22:32:53 +00:00
|
|
|
|
2021-05-19 17:18:25 +00:00
|
|
|
/**
|
|
|
|
|
* netif_carrier_event - report carrier state event
|
|
|
|
|
* @dev: network device
|
|
|
|
|
*
|
|
|
|
|
* Device has detected a carrier event but the carrier state wasn't changed.
|
|
|
|
|
* Use in drivers when querying carrier state asynchronously, to avoid missing
|
|
|
|
|
* events (link flaps) if link recovers before it's queried.
|
|
|
|
|
*/
|
|
|
|
|
void netif_carrier_event(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
if (dev->reg_state == NETREG_UNINITIALIZED)
|
|
|
|
|
return;
|
|
|
|
|
atomic_inc(&dev->carrier_up_count);
|
|
|
|
|
atomic_inc(&dev->carrier_down_count);
|
|
|
|
|
linkwatch_fire_event(dev);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL_GPL(netif_carrier_event);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* "NOOP" scheduler: the best scheduler, recommended for all interfaces
|
|
|
|
|
under all circumstances. It is difficult to invent anything faster or
|
|
|
|
|
cheaper.
|
|
|
|
|
*/
|
|
|
|
|
|
2016-06-22 06:16:49 +00:00
|
|
|
static int noop_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
|
|
|
|
|
struct sk_buff **to_free)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2016-06-22 06:16:49 +00:00
|
|
|
__qdisc_drop(skb, to_free);
|
2005-04-16 22:20:36 +00:00
|
|
|
return NET_XMIT_CN;
|
|
|
|
|
}
|
|
|
|
|
|
2013-12-10 12:55:31 +00:00
|
|
|
static struct sk_buff *noop_dequeue(struct Qdisc *qdisc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
|
|
|
|
|
2007-11-14 09:44:41 +00:00
|
|
|
struct Qdisc_ops noop_qdisc_ops __read_mostly = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.id = "noop",
|
|
|
|
|
.priv_size = 0,
|
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 07:45:27 +00:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 22:20:36 +00:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
};
|
|
|
|
|
|
2008-07-16 08:42:40 +00:00
|
|
|
static struct netdev_queue noop_netdev_queue = {
|
2019-02-25 02:43:06 +00:00
|
|
|
RCU_POINTER_INITIALIZER(qdisc, &noop_qdisc),
|
2023-06-06 11:19:29 +00:00
|
|
|
RCU_POINTER_INITIALIZER(qdisc_sleeping, &noop_qdisc),
|
2008-07-16 08:42:40 +00:00
|
|
|
};
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
struct Qdisc noop_qdisc = {
|
|
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
|
.dequeue = noop_dequeue,
|
|
|
|
|
.flags = TCQ_F_BUILTIN,
|
2007-02-09 14:25:16 +00:00
|
|
|
.ops = &noop_qdisc_ops,
|
2008-07-17 07:53:03 +00:00
|
|
|
.q.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.q.lock),
|
2008-07-16 08:42:40 +00:00
|
|
|
.dev_queue = &noop_netdev_queue,
|
2010-10-15 19:22:34 +00:00
|
|
|
.busylock = __SPIN_LOCK_UNLOCKED(noop_qdisc.busylock),
|
2018-10-09 22:20:50 +00:00
|
|
|
.gso_skb = {
|
|
|
|
|
.next = (struct sk_buff *)&noop_qdisc.gso_skb,
|
|
|
|
|
.prev = (struct sk_buff *)&noop_qdisc.gso_skb,
|
|
|
|
|
.qlen = 0,
|
|
|
|
|
.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.gso_skb.lock),
|
|
|
|
|
},
|
|
|
|
|
.skb_bad_txq = {
|
|
|
|
|
.next = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
|
|
|
|
|
.prev = (struct sk_buff *)&noop_qdisc.skb_bad_txq,
|
|
|
|
|
.qlen = 0,
|
|
|
|
|
.lock = __SPIN_LOCK_UNLOCKED(noop_qdisc.skb_bad_txq.lock),
|
|
|
|
|
},
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
2008-01-23 06:10:23 +00:00
|
|
|
EXPORT_SYMBOL(noop_qdisc);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-12-20 17:35:13 +00:00
|
|
|
static int noqueue_init(struct Qdisc *qdisc, struct nlattr *opt,
|
|
|
|
|
struct netlink_ext_ack *extack)
|
2015-08-27 19:21:38 +00:00
|
|
|
{
|
|
|
|
|
/* register_qdisc() assigns a default of noop_enqueue if unset,
|
|
|
|
|
* but __dev_queue_xmit() treats noqueue only as such
|
|
|
|
|
* if this is NULL - so clear it here. */
|
|
|
|
|
qdisc->enqueue = NULL;
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
struct Qdisc_ops noqueue_qdisc_ops __read_mostly = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.id = "noqueue",
|
|
|
|
|
.priv_size = 0,
|
2015-08-27 19:21:38 +00:00
|
|
|
.init = noqueue_init,
|
2005-04-16 22:20:36 +00:00
|
|
|
.enqueue = noop_enqueue,
|
|
|
|
|
.dequeue = noop_dequeue,
|
2008-10-31 07:45:27 +00:00
|
|
|
.peek = noop_dequeue,
|
2005-04-16 22:20:36 +00:00
|
|
|
.owner = THIS_MODULE,
|
|
|
|
|
};
|
|
|
|
|
|
2023-10-02 13:17:36 +00:00
|
|
|
const u8 sch_default_prio2band[TC_PRIO_MAX + 1] = {
|
|
|
|
|
1, 2, 2, 2, 1, 2, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1
|
2011-01-19 19:26:56 +00:00
|
|
|
};
|
2023-10-02 13:17:36 +00:00
|
|
|
EXPORT_SYMBOL(sch_default_prio2band);
|
2008-07-21 16:56:13 +00:00
|
|
|
|
|
|
|
|
/* 3-band FIFO queue: old style, but should be a bit faster than
|
|
|
|
|
generic prio+fifo combination.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
#define PFIFO_FAST_BANDS 3
|
|
|
|
|
|
2009-08-18 21:55:59 +00:00
|
|
|
/*
|
|
|
|
|
* Private data for a pfifo_fast scheduler containing:
|
2017-12-07 17:58:19 +00:00
|
|
|
* - rings for priority bands
|
2009-08-18 21:55:59 +00:00
|
|
|
*/
|
|
|
|
|
struct pfifo_fast_priv {
|
2017-12-07 17:58:19 +00:00
|
|
|
struct skb_array q[PFIFO_FAST_BANDS];
|
2009-08-18 21:55:59 +00:00
|
|
|
};
|
|
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
static inline struct skb_array *band2list(struct pfifo_fast_priv *priv,
|
|
|
|
|
int band)
|
2008-07-21 16:56:13 +00:00
|
|
|
{
|
2017-12-07 17:58:19 +00:00
|
|
|
return &priv->q[band];
|
2008-07-21 16:56:13 +00:00
|
|
|
}
|
|
|
|
|
|
2016-06-22 06:16:49 +00:00
|
|
|
static int pfifo_fast_enqueue(struct sk_buff *skb, struct Qdisc *qdisc,
|
|
|
|
|
struct sk_buff **to_free)
|
2005-06-19 05:58:35 +00:00
|
|
|
{
|
2023-10-02 13:17:36 +00:00
|
|
|
int band = sch_default_prio2band[skb->priority & TC_PRIO_MAX];
|
2017-12-07 17:58:19 +00:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2018-03-15 01:53:00 +00:00
|
|
|
unsigned int pkt_len = qdisc_pkt_len(skb);
|
2017-12-07 17:58:19 +00:00
|
|
|
int err;
|
2005-06-19 05:58:15 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
err = skb_array_produce(q, skb);
|
|
|
|
|
|
net/sched: pfifo_fast: fix wrong dereference in pfifo_fast_enqueue
Now that 'TCQ_F_CPUSTATS' bit can be cleared, depending on the value of
'TCQ_F_NOLOCK' bit in the parent qdisc, we can't assume anymore that
per-cpu counters are there in the error path of skb_array_produce().
Otherwise, the following splat can be seen:
Unable to handle kernel paging request at virtual address 0000600dea430008
Mem abort info:
ESR = 0x96000005
Exception class = DABT (current EL), IL = 32 bits
SET = 0, FnV = 0
EA = 0, S1PTW = 0
Data abort info:
ISV = 0, ISS = 0x00000005
CM = 0, WnR = 0
user pgtable: 64k pages, 48-bit VAs, pgdp = 000000007b97530e
[0000600dea430008] pgd=0000000000000000, pud=0000000000000000
Internal error: Oops: 96000005 [#1] SMP
[...]
pstate: 10000005 (nzcV daif -PAN -UAO)
pc : pfifo_fast_enqueue+0x524/0x6e8
lr : pfifo_fast_enqueue+0x46c/0x6e8
sp : ffff800d39376fe0
x29: ffff800d39376fe0 x28: 1ffff001a07d1e40
x27: ffff800d03e8f188 x26: ffff800d03e8f200
x25: 0000000000000062 x24: ffff800d393772f0
x23: 0000000000000000 x22: 0000000000000403
x21: ffff800cca569a00 x20: ffff800d03e8ee00
x19: ffff800cca569a10 x18: 00000000000000bf
x17: 0000000000000000 x16: 0000000000000000
x15: 0000000000000000 x14: ffff1001a726edd0
x13: 1fffe4000276a9a4 x12: 0000000000000000
x11: dfff200000000000 x10: ffff800d03e8f1a0
x9 : 0000000000000003 x8 : 0000000000000000
x7 : 00000000f1f1f1f1 x6 : ffff1001a726edea
x5 : ffff800cca56a53c x4 : 1ffff001bf9a8003
x3 : 1ffff001bf9a8003 x2 : 1ffff001a07d1dcb
x1 : 0000600dea430000 x0 : 0000600dea430008
Process ping (pid: 6067, stack limit = 0x00000000dc0aa557)
Call trace:
pfifo_fast_enqueue+0x524/0x6e8
htb_enqueue+0x660/0x10e0 [sch_htb]
__dev_queue_xmit+0x123c/0x2de0
dev_queue_xmit+0x24/0x30
ip_finish_output2+0xc48/0x1720
ip_finish_output+0x548/0x9d8
ip_output+0x334/0x788
ip_local_out+0x90/0x138
ip_send_skb+0x44/0x1d0
ip_push_pending_frames+0x5c/0x78
raw_sendmsg+0xed8/0x28d0
inet_sendmsg+0xc4/0x5c0
sock_sendmsg+0xac/0x108
__sys_sendto+0x1ac/0x2a0
__arm64_sys_sendto+0xc4/0x138
el0_svc_handler+0x13c/0x298
el0_svc+0x8/0xc
Code: f9402e80 d538d081 91002000 8b010000 (885f7c03)
Fix this by testing the value of 'TCQ_F_CPUSTATS' bit in 'qdisc->flags',
before dereferencing 'qdisc->cpu_qstats'.
Fixes: 8a53e616de29 ("net: sched: when clearing NOLOCK, clear TCQ_F_CPUSTATS, too")
CC: Paolo Abeni <pabeni@redhat.com>
CC: Stefano Brivio <sbrivio@redhat.com>
Reported-by: Li Shuang <shuali@redhat.com>
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-08-27 21:18:53 +00:00
|
|
|
if (unlikely(err)) {
|
|
|
|
|
if (qdisc_is_percpu_stats(qdisc))
|
|
|
|
|
return qdisc_drop_cpu(skb, qdisc, to_free);
|
|
|
|
|
else
|
|
|
|
|
return qdisc_drop(skb, qdisc, to_free);
|
|
|
|
|
}
|
2017-12-07 17:58:19 +00:00
|
|
|
|
2019-04-10 12:32:40 +00:00
|
|
|
qdisc_update_stats_at_enqueue(qdisc, pkt_len);
|
2017-12-07 17:58:19 +00:00
|
|
|
return NET_XMIT_SUCCESS;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2011-01-19 19:26:56 +00:00
|
|
|
static struct sk_buff *pfifo_fast_dequeue(struct Qdisc *qdisc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2009-08-18 21:55:59 +00:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 17:58:19 +00:00
|
|
|
struct sk_buff *skb = NULL;
|
2021-05-14 03:16:59 +00:00
|
|
|
bool need_retry = true;
|
2017-12-07 17:58:19 +00:00
|
|
|
int band;
|
2016-09-17 22:57:32 +00:00
|
|
|
|
2021-05-14 03:16:59 +00:00
|
|
|
retry:
|
2017-12-07 17:58:19 +00:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2009-08-18 21:55:59 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
if (__skb_array_empty(q))
|
|
|
|
|
continue;
|
2009-08-18 21:55:59 +00:00
|
|
|
|
2018-05-15 14:24:37 +00:00
|
|
|
skb = __skb_array_consume(q);
|
2017-12-07 17:58:19 +00:00
|
|
|
}
|
|
|
|
|
if (likely(skb)) {
|
2019-04-10 12:32:40 +00:00
|
|
|
qdisc_update_stats_at_dequeue(qdisc, skb);
|
2021-05-14 03:16:59 +00:00
|
|
|
} else if (need_retry &&
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
READ_ONCE(qdisc->state) & QDISC_STATE_NON_EMPTY) {
|
2021-05-14 03:16:59 +00:00
|
|
|
/* Delay clearing the STATE_MISSED here to reduce
|
|
|
|
|
* the overhead of the second spin_trylock() in
|
|
|
|
|
* qdisc_run_begin() and __netif_schedule() calling
|
|
|
|
|
* in qdisc_run_end().
|
|
|
|
|
*/
|
|
|
|
|
clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
|
2021-05-14 03:16:59 +00:00
|
|
|
|
|
|
|
|
/* Make sure dequeuing happens after clearing
|
|
|
|
|
* STATE_MISSED.
|
|
|
|
|
*/
|
|
|
|
|
smp_mb__after_atomic();
|
|
|
|
|
|
|
|
|
|
need_retry = false;
|
|
|
|
|
|
|
|
|
|
goto retry;
|
2008-07-21 16:56:13 +00:00
|
|
|
}
|
2005-06-19 05:58:53 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
return skb;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2011-01-19 19:26:56 +00:00
|
|
|
static struct sk_buff *pfifo_fast_peek(struct Qdisc *qdisc)
|
2008-10-31 07:45:27 +00:00
|
|
|
{
|
2009-08-18 21:55:59 +00:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 17:58:19 +00:00
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
|
int band;
|
2009-08-18 21:55:59 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS && !skb; band++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
2008-10-31 07:45:27 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
skb = __skb_array_peek(q);
|
2008-10-31 07:45:27 +00:00
|
|
|
}
|
|
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
return skb;
|
2008-10-31 07:45:27 +00:00
|
|
|
}
|
|
|
|
|
|
2011-01-19 19:26:56 +00:00
|
|
|
static void pfifo_fast_reset(struct Qdisc *qdisc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2017-12-07 17:58:19 +00:00
|
|
|
int i, band;
|
2009-08-18 21:55:59 +00:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2008-07-21 16:56:13 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
for (band = 0; band < PFIFO_FAST_BANDS; band++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, band);
|
|
|
|
|
struct sk_buff *skb;
|
2008-07-21 16:56:13 +00:00
|
|
|
|
2017-12-18 22:34:26 +00:00
|
|
|
/* NULL ring is possible if destroy path is due to a failed
|
|
|
|
|
* skb_array_init() in pfifo_fast_init() case.
|
|
|
|
|
*/
|
|
|
|
|
if (!q->ring.queue)
|
|
|
|
|
continue;
|
|
|
|
|
|
2018-05-15 14:24:37 +00:00
|
|
|
while ((skb = __skb_array_consume(q)) != NULL)
|
2017-12-07 17:58:19 +00:00
|
|
|
kfree_skb(skb);
|
|
|
|
|
}
|
|
|
|
|
|
net/sched: pfifo_fast: fix wrong dereference when qdisc is reset
Now that 'TCQ_F_CPUSTATS' bit can be cleared, depending on the value of
'TCQ_F_NOLOCK' bit in the parent qdisc, we need to be sure that per-cpu
counters are present when 'reset()' is called for pfifo_fast qdiscs.
Otherwise, the following script:
# tc q a dev lo handle 1: root htb default 100
# tc c a dev lo parent 1: classid 1:100 htb \
> rate 95Mbit ceil 100Mbit burst 64k
[...]
# tc f a dev lo parent 1: protocol arp basic classid 1:100
[...]
# tc q a dev lo parent 1:100 handle 100: pfifo_fast
[...]
# tc q d dev lo root
can generate the following splat:
Unable to handle kernel paging request at virtual address dfff2c01bd148000
Mem abort info:
ESR = 0x96000004
Exception class = DABT (current EL), IL = 32 bits
SET = 0, FnV = 0
EA = 0, S1PTW = 0
Data abort info:
ISV = 0, ISS = 0x00000004
CM = 0, WnR = 0
[dfff2c01bd148000] address between user and kernel address ranges
Internal error: Oops: 96000004 [#1] SMP
[...]
pstate: 80000005 (Nzcv daif -PAN -UAO)
pc : pfifo_fast_reset+0x280/0x4d8
lr : pfifo_fast_reset+0x21c/0x4d8
sp : ffff800d09676fa0
x29: ffff800d09676fa0 x28: ffff200012ee22e4
x27: dfff200000000000 x26: 0000000000000000
x25: ffff800ca0799958 x24: ffff1001940f332b
x23: 0000000000000007 x22: ffff200012ee1ab8
x21: 0000600de8a40000 x20: 0000000000000000
x19: ffff800ca0799900 x18: 0000000000000000
x17: 0000000000000002 x16: 0000000000000000
x15: 0000000000000000 x14: 0000000000000000
x13: 0000000000000000 x12: ffff1001b922e6e2
x11: 1ffff001b922e6e1 x10: 0000000000000000
x9 : 1ffff001b922e6e1 x8 : dfff200000000000
x7 : 0000000000000000 x6 : 0000000000000000
x5 : 1fffe400025dc45c x4 : 1fffe400025dc357
x3 : 00000c01bd148000 x2 : 0000600de8a40000
x1 : 0000000000000007 x0 : 0000600de8a40004
Call trace:
pfifo_fast_reset+0x280/0x4d8
qdisc_reset+0x6c/0x370
htb_reset+0x150/0x3b8 [sch_htb]
qdisc_reset+0x6c/0x370
dev_deactivate_queue.constprop.5+0xe0/0x1a8
dev_deactivate_many+0xd8/0x908
dev_deactivate+0xe4/0x190
qdisc_graft+0x88c/0xbd0
tc_get_qdisc+0x418/0x8a8
rtnetlink_rcv_msg+0x3a8/0xa78
netlink_rcv_skb+0x18c/0x328
rtnetlink_rcv+0x28/0x38
netlink_unicast+0x3c4/0x538
netlink_sendmsg+0x538/0x9a0
sock_sendmsg+0xac/0xf8
___sys_sendmsg+0x53c/0x658
__sys_sendmsg+0xc8/0x140
__arm64_sys_sendmsg+0x74/0xa8
el0_svc_handler+0x164/0x468
el0_svc+0x10/0x14
Code: 910012a0 92400801 d343fc03 11000c21 (38fb6863)
Fix this by testing the value of 'TCQ_F_CPUSTATS' bit in 'qdisc->flags',
before dereferencing 'qdisc->cpu_qstats'.
Changes since v1:
- coding style improvements, thanks to Stefano Brivio
Fixes: 8a53e616de29 ("net: sched: when clearing NOLOCK, clear TCQ_F_CPUSTATS, too")
CC: Paolo Abeni <pabeni@redhat.com>
Reported-by: Li Shuang <shuali@redhat.com>
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-08-27 10:29:09 +00:00
|
|
|
if (qdisc_is_percpu_stats(qdisc)) {
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
|
struct gnet_stats_queue *q;
|
2017-12-07 17:58:19 +00:00
|
|
|
|
net/sched: pfifo_fast: fix wrong dereference when qdisc is reset
Now that 'TCQ_F_CPUSTATS' bit can be cleared, depending on the value of
'TCQ_F_NOLOCK' bit in the parent qdisc, we need to be sure that per-cpu
counters are present when 'reset()' is called for pfifo_fast qdiscs.
Otherwise, the following script:
# tc q a dev lo handle 1: root htb default 100
# tc c a dev lo parent 1: classid 1:100 htb \
> rate 95Mbit ceil 100Mbit burst 64k
[...]
# tc f a dev lo parent 1: protocol arp basic classid 1:100
[...]
# tc q a dev lo parent 1:100 handle 100: pfifo_fast
[...]
# tc q d dev lo root
can generate the following splat:
Unable to handle kernel paging request at virtual address dfff2c01bd148000
Mem abort info:
ESR = 0x96000004
Exception class = DABT (current EL), IL = 32 bits
SET = 0, FnV = 0
EA = 0, S1PTW = 0
Data abort info:
ISV = 0, ISS = 0x00000004
CM = 0, WnR = 0
[dfff2c01bd148000] address between user and kernel address ranges
Internal error: Oops: 96000004 [#1] SMP
[...]
pstate: 80000005 (Nzcv daif -PAN -UAO)
pc : pfifo_fast_reset+0x280/0x4d8
lr : pfifo_fast_reset+0x21c/0x4d8
sp : ffff800d09676fa0
x29: ffff800d09676fa0 x28: ffff200012ee22e4
x27: dfff200000000000 x26: 0000000000000000
x25: ffff800ca0799958 x24: ffff1001940f332b
x23: 0000000000000007 x22: ffff200012ee1ab8
x21: 0000600de8a40000 x20: 0000000000000000
x19: ffff800ca0799900 x18: 0000000000000000
x17: 0000000000000002 x16: 0000000000000000
x15: 0000000000000000 x14: 0000000000000000
x13: 0000000000000000 x12: ffff1001b922e6e2
x11: 1ffff001b922e6e1 x10: 0000000000000000
x9 : 1ffff001b922e6e1 x8 : dfff200000000000
x7 : 0000000000000000 x6 : 0000000000000000
x5 : 1fffe400025dc45c x4 : 1fffe400025dc357
x3 : 00000c01bd148000 x2 : 0000600de8a40000
x1 : 0000000000000007 x0 : 0000600de8a40004
Call trace:
pfifo_fast_reset+0x280/0x4d8
qdisc_reset+0x6c/0x370
htb_reset+0x150/0x3b8 [sch_htb]
qdisc_reset+0x6c/0x370
dev_deactivate_queue.constprop.5+0xe0/0x1a8
dev_deactivate_many+0xd8/0x908
dev_deactivate+0xe4/0x190
qdisc_graft+0x88c/0xbd0
tc_get_qdisc+0x418/0x8a8
rtnetlink_rcv_msg+0x3a8/0xa78
netlink_rcv_skb+0x18c/0x328
rtnetlink_rcv+0x28/0x38
netlink_unicast+0x3c4/0x538
netlink_sendmsg+0x538/0x9a0
sock_sendmsg+0xac/0xf8
___sys_sendmsg+0x53c/0x658
__sys_sendmsg+0xc8/0x140
__arm64_sys_sendmsg+0x74/0xa8
el0_svc_handler+0x164/0x468
el0_svc+0x10/0x14
Code: 910012a0 92400801 d343fc03 11000c21 (38fb6863)
Fix this by testing the value of 'TCQ_F_CPUSTATS' bit in 'qdisc->flags',
before dereferencing 'qdisc->cpu_qstats'.
Changes since v1:
- coding style improvements, thanks to Stefano Brivio
Fixes: 8a53e616de29 ("net: sched: when clearing NOLOCK, clear TCQ_F_CPUSTATS, too")
CC: Paolo Abeni <pabeni@redhat.com>
Reported-by: Li Shuang <shuali@redhat.com>
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Acked-by: Paolo Abeni <pabeni@redhat.com>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-08-27 10:29:09 +00:00
|
|
|
q = per_cpu_ptr(qdisc->cpu_qstats, i);
|
|
|
|
|
q->backlog = 0;
|
|
|
|
|
q->qlen = 0;
|
|
|
|
|
}
|
2017-12-07 17:58:19 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2008-07-21 16:56:13 +00:00
|
|
|
static int pfifo_fast_dump(struct Qdisc *qdisc, struct sk_buff *skb)
|
|
|
|
|
{
|
|
|
|
|
struct tc_prio_qopt opt = { .bands = PFIFO_FAST_BANDS };
|
|
|
|
|
|
2023-10-02 13:17:36 +00:00
|
|
|
memcpy(&opt.priomap, sch_default_prio2band, TC_PRIO_MAX + 1);
|
2012-03-29 09:11:39 +00:00
|
|
|
if (nla_put(skb, TCA_OPTIONS, sizeof(opt), &opt))
|
|
|
|
|
goto nla_put_failure;
|
2008-07-21 16:56:13 +00:00
|
|
|
return skb->len;
|
|
|
|
|
|
|
|
|
|
nla_put_failure:
|
|
|
|
|
return -1;
|
|
|
|
|
}
|
|
|
|
|
|
2017-12-20 17:35:13 +00:00
|
|
|
static int pfifo_fast_init(struct Qdisc *qdisc, struct nlattr *opt,
|
|
|
|
|
struct netlink_ext_ack *extack)
|
2008-07-21 16:56:13 +00:00
|
|
|
{
|
2017-12-07 17:58:19 +00:00
|
|
|
unsigned int qlen = qdisc_dev(qdisc)->tx_queue_len;
|
2009-08-18 21:55:59 +00:00
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(qdisc);
|
2017-12-07 17:58:19 +00:00
|
|
|
int prio;
|
|
|
|
|
|
|
|
|
|
/* guard against zero length rings */
|
|
|
|
|
if (!qlen)
|
|
|
|
|
return -EINVAL;
|
2008-07-21 16:56:13 +00:00
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
|
int err;
|
|
|
|
|
|
|
|
|
|
err = skb_array_init(q, qlen, GFP_KERNEL);
|
|
|
|
|
if (err)
|
|
|
|
|
return -ENOMEM;
|
|
|
|
|
}
|
2008-07-21 16:56:13 +00:00
|
|
|
|
2011-01-22 00:26:09 +00:00
|
|
|
/* Can by-pass the queue discipline */
|
|
|
|
|
qdisc->flags |= TCQ_F_CAN_BYPASS;
|
2008-07-21 16:56:13 +00:00
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2017-12-07 17:58:19 +00:00
|
|
|
static void pfifo_fast_destroy(struct Qdisc *sch)
|
|
|
|
|
{
|
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(sch);
|
|
|
|
|
int prio;
|
|
|
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
|
|
|
|
|
|
/* NULL ring is possible if destroy path is due to a failed
|
|
|
|
|
* skb_array_init() in pfifo_fast_init() case.
|
|
|
|
|
*/
|
2017-12-18 22:34:26 +00:00
|
|
|
if (!q->ring.queue)
|
2017-12-07 17:58:19 +00:00
|
|
|
continue;
|
|
|
|
|
/* Destroy ring but no need to kfree_skb because a call to
|
|
|
|
|
* pfifo_fast_reset() has already done that work.
|
|
|
|
|
*/
|
|
|
|
|
ptr_ring_cleanup(&q->ring, NULL);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2018-01-26 02:26:24 +00:00
|
|
|
static int pfifo_fast_change_tx_queue_len(struct Qdisc *sch,
|
|
|
|
|
unsigned int new_len)
|
|
|
|
|
{
|
|
|
|
|
struct pfifo_fast_priv *priv = qdisc_priv(sch);
|
|
|
|
|
struct skb_array *bands[PFIFO_FAST_BANDS];
|
|
|
|
|
int prio;
|
|
|
|
|
|
|
|
|
|
for (prio = 0; prio < PFIFO_FAST_BANDS; prio++) {
|
|
|
|
|
struct skb_array *q = band2list(priv, prio);
|
|
|
|
|
|
|
|
|
|
bands[prio] = q;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
return skb_array_resize_multiple(bands, PFIFO_FAST_BANDS, new_len,
|
|
|
|
|
GFP_KERNEL);
|
|
|
|
|
}
|
|
|
|
|
|
2009-09-06 08:58:51 +00:00
|
|
|
struct Qdisc_ops pfifo_fast_ops __read_mostly = {
|
2008-07-21 16:56:13 +00:00
|
|
|
.id = "pfifo_fast",
|
2009-08-18 21:55:59 +00:00
|
|
|
.priv_size = sizeof(struct pfifo_fast_priv),
|
2008-07-21 16:56:13 +00:00
|
|
|
.enqueue = pfifo_fast_enqueue,
|
|
|
|
|
.dequeue = pfifo_fast_dequeue,
|
2008-10-31 07:45:27 +00:00
|
|
|
.peek = pfifo_fast_peek,
|
2008-07-21 16:56:13 +00:00
|
|
|
.init = pfifo_fast_init,
|
2017-12-07 17:58:19 +00:00
|
|
|
.destroy = pfifo_fast_destroy,
|
2008-07-21 16:56:13 +00:00
|
|
|
.reset = pfifo_fast_reset,
|
|
|
|
|
.dump = pfifo_fast_dump,
|
2018-01-26 02:26:24 +00:00
|
|
|
.change_tx_queue_len = pfifo_fast_change_tx_queue_len,
|
2005-04-16 22:20:36 +00:00
|
|
|
.owner = THIS_MODULE,
|
2017-12-07 17:58:19 +00:00
|
|
|
.static_flags = TCQ_F_NOLOCK | TCQ_F_CPUSTATS,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
2016-03-02 16:21:43 +00:00
|
|
|
EXPORT_SYMBOL(pfifo_fast_ops);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-05-03 05:22:19 +00:00
|
|
|
static struct lock_class_key qdisc_tx_busylock;
|
|
|
|
|
|
2008-07-09 00:06:30 +00:00
|
|
|
struct Qdisc *qdisc_alloc(struct netdev_queue *dev_queue,
|
2017-12-20 17:35:20 +00:00
|
|
|
const struct Qdisc_ops *ops,
|
|
|
|
|
struct netlink_ext_ack *extack)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
|
struct Qdisc *sch;
|
2020-10-07 16:51:11 +00:00
|
|
|
unsigned int size = sizeof(*sch) + ops->priv_size;
|
2005-07-05 21:15:09 +00:00
|
|
|
int err = -ENOBUFS;
|
2017-10-17 01:01:23 +00:00
|
|
|
struct net_device *dev;
|
|
|
|
|
|
|
|
|
|
if (!dev_queue) {
|
2017-12-20 17:35:20 +00:00
|
|
|
NL_SET_ERR_MSG(extack, "No device queue given");
|
2017-10-17 01:01:23 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
|
goto errout;
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-10-17 01:01:23 +00:00
|
|
|
dev = dev_queue->dev;
|
2020-10-07 16:51:11 +00:00
|
|
|
sch = kzalloc_node(size, GFP_KERNEL, netdev_queue_numa_node_read(dev_queue));
|
2010-11-29 08:14:37 +00:00
|
|
|
|
2020-10-07 16:51:11 +00:00
|
|
|
if (!sch)
|
2005-07-05 21:15:09 +00:00
|
|
|
goto errout;
|
2017-12-07 17:55:45 +00:00
|
|
|
__skb_queue_head_init(&sch->gso_skb);
|
2017-12-07 17:56:23 +00:00
|
|
|
__skb_queue_head_init(&sch->skb_bad_txq);
|
2021-10-16 08:49:09 +00:00
|
|
|
gnet_stats_basic_sync_init(&sch->bstats);
|
net/sched: fix false lockdep warning on qdisc root lock
Xiumei and Christoph reported the following lockdep splat, complaining of
the qdisc root lock being taken twice:
============================================
WARNING: possible recursive locking detected
6.7.0-rc3+ #598 Not tainted
--------------------------------------------
swapper/2/0 is trying to acquire lock:
ffff888177190110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
but task is already holding lock:
ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&sch->q.lock);
lock(&sch->q.lock);
*** DEADLOCK ***
May be due to missing lock nesting notation
5 locks held by swapper/2/0:
#0: ffff888135a09d98 ((&in_dev->mr_ifc_timer)){+.-.}-{0:0}, at: call_timer_fn+0x11a/0x510
#1: ffffffffaaee5260 (rcu_read_lock){....}-{1:2}, at: ip_finish_output2+0x2c0/0x1ed0
#2: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
#3: ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
#4: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
stack backtrace:
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 6.7.0-rc3+ #598
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7353+9de0a3cc 04/01/2014
Call Trace:
<IRQ>
dump_stack_lvl+0x4a/0x80
__lock_acquire+0xfdd/0x3150
lock_acquire+0x1ca/0x540
_raw_spin_lock+0x34/0x80
__dev_queue_xmit+0x1560/0x2e70
tcf_mirred_act+0x82e/0x1260 [act_mirred]
tcf_action_exec+0x161/0x480
tcf_classify+0x689/0x1170
prio_enqueue+0x316/0x660 [sch_prio]
dev_qdisc_enqueue+0x46/0x220
__dev_queue_xmit+0x1615/0x2e70
ip_finish_output2+0x1218/0x1ed0
__ip_finish_output+0x8b3/0x1350
ip_output+0x163/0x4e0
igmp_ifc_timer_expire+0x44b/0x930
call_timer_fn+0x1a2/0x510
run_timer_softirq+0x54d/0x11a0
__do_softirq+0x1b3/0x88f
irq_exit_rcu+0x18f/0x1e0
sysvec_apic_timer_interrupt+0x6f/0x90
</IRQ>
This happens when TC does a mirred egress redirect from the root qdisc of
device A to the root qdisc of device B. As long as these two locks aren't
protecting the same qdisc, they can be acquired in chain: add a per-qdisc
lockdep key to silence false warnings.
This dynamic key should safely replace the static key we have in sch_htb:
it was added to allow enqueueing to the device "direct qdisc" while still
holding the qdisc root lock.
v2: don't use static keys anymore in HTB direct qdiscs (thanks Eric Dumazet)
CC: Maxim Mikityanskiy <maxim@isovalent.com>
CC: Xiumei Mu <xmu@redhat.com>
Reported-by: Christoph Paasch <cpaasch@apple.com>
Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/451
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Link: https://lore.kernel.org/r/7dc06d6158f72053cf877a82e2a7a5bd23692faa.1713448007.git.dcaratti@redhat.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2024-04-18 13:50:11 +00:00
|
|
|
lockdep_register_key(&sch->root_lock_key);
|
2016-09-17 22:57:34 +00:00
|
|
|
spin_lock_init(&sch->q.lock);
|
net/sched: fix false lockdep warning on qdisc root lock
Xiumei and Christoph reported the following lockdep splat, complaining of
the qdisc root lock being taken twice:
============================================
WARNING: possible recursive locking detected
6.7.0-rc3+ #598 Not tainted
--------------------------------------------
swapper/2/0 is trying to acquire lock:
ffff888177190110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
but task is already holding lock:
ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&sch->q.lock);
lock(&sch->q.lock);
*** DEADLOCK ***
May be due to missing lock nesting notation
5 locks held by swapper/2/0:
#0: ffff888135a09d98 ((&in_dev->mr_ifc_timer)){+.-.}-{0:0}, at: call_timer_fn+0x11a/0x510
#1: ffffffffaaee5260 (rcu_read_lock){....}-{1:2}, at: ip_finish_output2+0x2c0/0x1ed0
#2: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
#3: ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
#4: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
stack backtrace:
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 6.7.0-rc3+ #598
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7353+9de0a3cc 04/01/2014
Call Trace:
<IRQ>
dump_stack_lvl+0x4a/0x80
__lock_acquire+0xfdd/0x3150
lock_acquire+0x1ca/0x540
_raw_spin_lock+0x34/0x80
__dev_queue_xmit+0x1560/0x2e70
tcf_mirred_act+0x82e/0x1260 [act_mirred]
tcf_action_exec+0x161/0x480
tcf_classify+0x689/0x1170
prio_enqueue+0x316/0x660 [sch_prio]
dev_qdisc_enqueue+0x46/0x220
__dev_queue_xmit+0x1615/0x2e70
ip_finish_output2+0x1218/0x1ed0
__ip_finish_output+0x8b3/0x1350
ip_output+0x163/0x4e0
igmp_ifc_timer_expire+0x44b/0x930
call_timer_fn+0x1a2/0x510
run_timer_softirq+0x54d/0x11a0
__do_softirq+0x1b3/0x88f
irq_exit_rcu+0x18f/0x1e0
sysvec_apic_timer_interrupt+0x6f/0x90
</IRQ>
This happens when TC does a mirred egress redirect from the root qdisc of
device A to the root qdisc of device B. As long as these two locks aren't
protecting the same qdisc, they can be acquired in chain: add a per-qdisc
lockdep key to silence false warnings.
This dynamic key should safely replace the static key we have in sch_htb:
it was added to allow enqueueing to the device "direct qdisc" while still
holding the qdisc root lock.
v2: don't use static keys anymore in HTB direct qdiscs (thanks Eric Dumazet)
CC: Maxim Mikityanskiy <maxim@isovalent.com>
CC: Xiumei Mu <xmu@redhat.com>
Reported-by: Christoph Paasch <cpaasch@apple.com>
Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/451
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Link: https://lore.kernel.org/r/7dc06d6158f72053cf877a82e2a7a5bd23692faa.1713448007.git.dcaratti@redhat.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2024-04-18 13:50:11 +00:00
|
|
|
lockdep_set_class(&sch->q.lock, &sch->root_lock_key);
|
2012-09-05 01:02:56 +00:00
|
|
|
|
2017-12-07 17:55:26 +00:00
|
|
|
if (ops->static_flags & TCQ_F_CPUSTATS) {
|
|
|
|
|
sch->cpu_bstats =
|
2021-10-16 08:49:09 +00:00
|
|
|
netdev_alloc_pcpu_stats(struct gnet_stats_basic_sync);
|
2017-12-07 17:55:26 +00:00
|
|
|
if (!sch->cpu_bstats)
|
|
|
|
|
goto errout1;
|
|
|
|
|
|
|
|
|
|
sch->cpu_qstats = alloc_percpu(struct gnet_stats_queue);
|
|
|
|
|
if (!sch->cpu_qstats) {
|
|
|
|
|
free_percpu(sch->cpu_bstats);
|
|
|
|
|
goto errout1;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2010-06-02 12:09:29 +00:00
|
|
|
spin_lock_init(&sch->busylock);
|
2020-05-03 05:22:19 +00:00
|
|
|
lockdep_set_class(&sch->busylock,
|
|
|
|
|
dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
|
|
|
|
|
|
2018-05-15 14:24:36 +00:00
|
|
|
/* seqlock has the same scope of busylock, for NOLOCK qdisc */
|
|
|
|
|
spin_lock_init(&sch->seqlock);
|
2021-08-03 10:58:21 +00:00
|
|
|
lockdep_set_class(&sch->seqlock,
|
2020-05-03 05:22:19 +00:00
|
|
|
dev->qdisc_tx_busylock ?: &qdisc_tx_busylock);
|
2016-06-06 16:37:15 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
sch->ops = ops;
|
2017-12-07 17:55:26 +00:00
|
|
|
sch->flags = ops->static_flags;
|
2005-04-16 22:20:36 +00:00
|
|
|
sch->enqueue = ops->enqueue;
|
|
|
|
|
sch->dequeue = ops->dequeue;
|
2008-07-08 23:55:56 +00:00
|
|
|
sch->dev_queue = dev_queue;
|
2024-04-15 21:07:28 +00:00
|
|
|
sch->owner = -1;
|
2022-06-08 04:39:55 +00:00
|
|
|
netdev_hold(dev, &sch->dev_tracker, GFP_KERNEL);
|
2017-07-04 12:53:07 +00:00
|
|
|
refcount_set(&sch->refcnt, 1);
|
2005-07-05 21:15:09 +00:00
|
|
|
|
|
|
|
|
return sch;
|
2017-12-07 17:55:26 +00:00
|
|
|
errout1:
|
2024-04-30 17:11:13 +00:00
|
|
|
lockdep_unregister_key(&sch->root_lock_key);
|
2020-10-07 16:51:11 +00:00
|
|
|
kfree(sch);
|
2005-07-05 21:15:09 +00:00
|
|
|
errout:
|
2008-06-28 02:51:35 +00:00
|
|
|
return ERR_PTR(err);
|
2005-07-05 21:15:09 +00:00
|
|
|
}
|
|
|
|
|
|
2010-10-16 13:04:08 +00:00
|
|
|
struct Qdisc *qdisc_create_dflt(struct netdev_queue *dev_queue,
|
2013-08-31 17:15:50 +00:00
|
|
|
const struct Qdisc_ops *ops,
|
2017-12-20 17:35:21 +00:00
|
|
|
unsigned int parentid,
|
|
|
|
|
struct netlink_ext_ack *extack)
|
2005-07-05 21:15:09 +00:00
|
|
|
{
|
|
|
|
|
struct Qdisc *sch;
|
2007-02-09 14:25:16 +00:00
|
|
|
|
2017-12-20 17:35:21 +00:00
|
|
|
if (!try_module_get(ops->owner)) {
|
|
|
|
|
NL_SET_ERR_MSG(extack, "Failed to increase module reference counter");
|
2016-08-24 16:39:02 +00:00
|
|
|
return NULL;
|
2017-12-20 17:35:21 +00:00
|
|
|
}
|
2013-08-27 23:19:08 +00:00
|
|
|
|
2017-12-20 17:35:21 +00:00
|
|
|
sch = qdisc_alloc(dev_queue, ops, extack);
|
2016-08-24 16:39:02 +00:00
|
|
|
if (IS_ERR(sch)) {
|
|
|
|
|
module_put(ops->owner);
|
|
|
|
|
return NULL;
|
|
|
|
|
}
|
2006-11-30 01:35:18 +00:00
|
|
|
sch->parent = parentid;
|
2005-07-05 21:15:09 +00:00
|
|
|
|
2020-05-27 04:35:25 +00:00
|
|
|
if (!ops->init || ops->init(sch, NULL, extack) == 0) {
|
|
|
|
|
trace_qdisc_create(ops, dev_queue->dev, parentid);
|
2005-04-16 22:20:36 +00:00
|
|
|
return sch;
|
2020-05-27 04:35:25 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-09-24 16:22:50 +00:00
|
|
|
qdisc_put(sch);
|
2005-04-16 22:20:36 +00:00
|
|
|
return NULL;
|
|
|
|
|
}
|
2008-01-23 06:10:23 +00:00
|
|
|
EXPORT_SYMBOL(qdisc_create_dflt);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2008-08-03 03:02:43 +00:00
|
|
|
/* Under qdisc_lock(qdisc) and BH! */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
|
void qdisc_reset(struct Qdisc *qdisc)
|
|
|
|
|
{
|
2007-11-14 09:44:41 +00:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-05-27 04:35:24 +00:00
|
|
|
trace_qdisc_reset(qdisc);
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (ops->reset)
|
|
|
|
|
ops->reset(qdisc);
|
2008-11-03 10:52:50 +00:00
|
|
|
|
2022-04-14 01:10:04 +00:00
|
|
|
__skb_queue_purge(&qdisc->gso_skb);
|
|
|
|
|
__skb_queue_purge(&qdisc->skb_bad_txq);
|
2017-12-07 17:56:23 +00:00
|
|
|
|
net_sched: generalize bulk dequeue
When qdisc bulk dequeue was added in linux-3.18 (commit
5772e9a3463b "qdisc: bulk dequeue support for qdiscs
with TCQ_F_ONETXQUEUE"), it was constrained to some
specific qdiscs.
With some extra care, we can extend this to all qdiscs,
so that typical traffic shaping solutions can benefit from
small batches (8 packets in this patch).
For example, HTB is often used on some multi queue device.
And bonding/team are multi queue devices...
Idea is to bulk-dequeue packets mapping to the same transmit queue.
This brings between 35 and 80 % performance increase in HTB setup
under pressure on a bonding setup :
1) NUMA node contention : 610,000 pps -> 1,110,000 pps
2) No node contention : 1,380,000 pps -> 1,930,000 pps
Now we should work to add batches on the enqueue() side ;)
Signed-off-by: Eric Dumazet <edumazet@google.com>
Cc: John Fastabend <john.r.fastabend@intel.com>
Cc: Jesper Dangaard Brouer <brouer@redhat.com>
Cc: Hannes Frederic Sowa <hannes@stressinduktion.org>
Cc: Florian Westphal <fw@strlen.de>
Cc: Daniel Borkmann <daniel@iogearbox.net>
Acked-by: Jesper Dangaard Brouer <brouer@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2016-06-22 06:16:52 +00:00
|
|
|
qdisc->q.qlen = 0;
|
2017-09-20 12:45:36 +00:00
|
|
|
qdisc->qstats.backlog = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2008-01-23 06:10:23 +00:00
|
|
|
EXPORT_SYMBOL(qdisc_reset);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
net, sched: fix panic when updating miniq {b,q}stats
While working on fixing another bug, I ran into the following panic
on arm64 by simply attaching clsact qdisc, adding a filter and running
traffic on ingress to it:
[...]
[ 178.188591] Unable to handle kernel read from unreadable memory at virtual address 810fb501f000
[ 178.197314] Mem abort info:
[ 178.200121] ESR = 0x96000004
[ 178.203168] Exception class = DABT (current EL), IL = 32 bits
[ 178.209095] SET = 0, FnV = 0
[ 178.212157] EA = 0, S1PTW = 0
[ 178.215288] Data abort info:
[ 178.218175] ISV = 0, ISS = 0x00000004
[ 178.222019] CM = 0, WnR = 0
[ 178.224997] user pgtable: 4k pages, 48-bit VAs, pgd = 0000000023cb3f33
[ 178.231531] [0000810fb501f000] *pgd=0000000000000000
[ 178.236508] Internal error: Oops: 96000004 [#1] SMP
[...]
[ 178.311855] CPU: 73 PID: 2497 Comm: ping Tainted: G W 4.15.0-rc7+ #5
[ 178.319413] Hardware name: FOXCONN R2-1221R-A4/C2U4N_MB, BIOS G31FB18A 03/31/2017
[ 178.326887] pstate: 60400005 (nZCv daif +PAN -UAO)
[ 178.331685] pc : __netif_receive_skb_core+0x49c/0xac8
[ 178.336728] lr : __netif_receive_skb+0x28/0x78
[ 178.341161] sp : ffff00002344b750
[ 178.344465] x29: ffff00002344b750 x28: ffff810fbdfd0580
[ 178.349769] x27: 0000000000000000 x26: ffff000009378000
[...]
[ 178.418715] x1 : 0000000000000054 x0 : 0000000000000000
[ 178.424020] Process ping (pid: 2497, stack limit = 0x000000009f0a3ff4)
[ 178.430537] Call trace:
[ 178.432976] __netif_receive_skb_core+0x49c/0xac8
[ 178.437670] __netif_receive_skb+0x28/0x78
[ 178.441757] process_backlog+0x9c/0x160
[ 178.445584] net_rx_action+0x2f8/0x3f0
[...]
Reason is that sch_ingress and sch_clsact are doing mini_qdisc_pair_init()
which sets up miniq pointers to cpu_{b,q}stats from the underlying qdisc.
Problem is that this cannot work since they are actually set up right after
the qdisc ->init() callback in qdisc_create(), so first packet going into
sch_handle_ingress() tries to call mini_qdisc_bstats_cpu_update() and we
therefore panic.
In order to fix this, allocation of {b,q}stats needs to happen before we
call into ->init(). In net-next, there's already such option through commit
d59f5ffa59d8 ("net: sched: a dflt qdisc may be used with per cpu stats").
However, the bug needs to be fixed in net still for 4.15. Thus, include
these bits to reduce any merge churn and reuse the static_flags field to
set TCQ_F_CPUSTATS, and remove the allocation from qdisc_create() since
there is no other user left. Prashant Bhole ran into the same issue but
for net-next, thus adding him below as well as co-author. Same issue was
also reported by Sandipan Das when using bcc.
Fixes: 46209401f8f6 ("net: core: introduce mini_Qdisc and eliminate usage of tp->q for clsact fastpath")
Reference: https://lists.iovisor.org/pipermail/iovisor-dev/2018-January/001190.html
Reported-by: Sandipan Das <sandipan@linux.vnet.ibm.com>
Co-authored-by: Prashant Bhole <bhole_prashant_q7@lab.ntt.co.jp>
Co-authored-by: John Fastabend <john.fastabend@gmail.com>
Signed-off-by: Daniel Borkmann <daniel@iogearbox.net>
Cc: Jiri Pirko <jiri@resnulli.us>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-01-15 22:12:09 +00:00
|
|
|
void qdisc_free(struct Qdisc *qdisc)
|
2010-03-31 07:06:04 +00:00
|
|
|
{
|
2016-01-05 17:11:36 +00:00
|
|
|
if (qdisc_is_percpu_stats(qdisc)) {
|
2014-09-28 18:52:56 +00:00
|
|
|
free_percpu(qdisc->cpu_bstats);
|
2016-01-05 17:11:36 +00:00
|
|
|
free_percpu(qdisc->cpu_qstats);
|
|
|
|
|
}
|
2014-09-28 18:52:56 +00:00
|
|
|
|
2020-10-07 16:51:11 +00:00
|
|
|
kfree(qdisc);
|
2010-03-31 07:06:04 +00:00
|
|
|
}
|
|
|
|
|
|
2018-09-27 14:47:56 +00:00
|
|
|
static void qdisc_free_cb(struct rcu_head *head)
|
2018-09-24 16:22:51 +00:00
|
|
|
{
|
|
|
|
|
struct Qdisc *q = container_of(head, struct Qdisc, rcu);
|
|
|
|
|
|
|
|
|
|
qdisc_free(q);
|
|
|
|
|
}
|
|
|
|
|
|
net/sched: qdisc_destroy() old ingress and clsact Qdiscs before grafting
mini_Qdisc_pair::p_miniq is a double pointer to mini_Qdisc, initialized
in ingress_init() to point to net_device::miniq_ingress. ingress Qdiscs
access this per-net_device pointer in mini_qdisc_pair_swap(). Similar
for clsact Qdiscs and miniq_egress.
Unfortunately, after introducing RTNL-unlocked RTM_{NEW,DEL,GET}TFILTER
requests (thanks Hillf Danton for the hint), when replacing ingress or
clsact Qdiscs, for example, the old Qdisc ("@old") could access the same
miniq_{in,e}gress pointer(s) concurrently with the new Qdisc ("@new"),
causing race conditions [1] including a use-after-free bug in
mini_qdisc_pair_swap() reported by syzbot:
BUG: KASAN: slab-use-after-free in mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
Write of size 8 at addr ffff888045b31308 by task syz-executor690/14901
...
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106
print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:319
print_report mm/kasan/report.c:430 [inline]
kasan_report+0x11c/0x130 mm/kasan/report.c:536
mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
tcf_chain_head_change_item net/sched/cls_api.c:495 [inline]
tcf_chain0_head_change.isra.0+0xb9/0x120 net/sched/cls_api.c:509
tcf_chain_tp_insert net/sched/cls_api.c:1826 [inline]
tcf_chain_tp_insert_unique net/sched/cls_api.c:1875 [inline]
tc_new_tfilter+0x1de6/0x2290 net/sched/cls_api.c:2266
...
@old and @new should not affect each other. In other words, @old should
never modify miniq_{in,e}gress after @new, and @new should not update
@old's RCU state.
Fixing without changing sch_api.c turned out to be difficult (please
refer to Closes: for discussions). Instead, make sure @new's first call
always happen after @old's last call (in {ingress,clsact}_destroy()) has
finished:
In qdisc_graft(), return -EBUSY if @old has any ongoing filter requests,
and call qdisc_destroy() for @old before grafting @new.
Introduce qdisc_refcount_dec_if_one() as the counterpart of
qdisc_refcount_inc_nz() used for filter requests. Introduce a
non-static version of qdisc_destroy() that does a TCQ_F_BUILTIN check,
just like qdisc_put() etc.
Depends on patch "net/sched: Refactor qdisc_graft() for ingress and
clsact Qdiscs".
[1] To illustrate, the syzkaller reproducer adds ingress Qdiscs under
TC_H_ROOT (no longer possible after commit c7cfbd115001 ("net/sched:
sch_ingress: Only create under TC_H_INGRESS")) on eth0 that has 8
transmission queues:
Thread 1 creates ingress Qdisc A (containing mini Qdisc a1 and a2),
then adds a flower filter X to A.
Thread 2 creates another ingress Qdisc B (containing mini Qdisc b1 and
b2) to replace A, then adds a flower filter Y to B.
Thread 1 A's refcnt Thread 2
RTM_NEWQDISC (A, RTNL-locked)
qdisc_create(A) 1
qdisc_graft(A) 9
RTM_NEWTFILTER (X, RTNL-unlocked)
__tcf_qdisc_find(A) 10
tcf_chain0_head_change(A)
mini_qdisc_pair_swap(A) (1st)
|
| RTM_NEWQDISC (B, RTNL-locked)
RCU sync 2 qdisc_graft(B)
| 1 notify_and_destroy(A)
|
tcf_block_release(A) 0 RTM_NEWTFILTER (Y, RTNL-unlocked)
qdisc_destroy(A) tcf_chain0_head_change(B)
tcf_chain0_head_change_cb_del(A) mini_qdisc_pair_swap(B) (2nd)
mini_qdisc_pair_swap(A) (3rd) |
... ...
Here, B calls mini_qdisc_pair_swap(), pointing eth0->miniq_ingress to
its mini Qdisc, b1. Then, A calls mini_qdisc_pair_swap() again during
ingress_destroy(), setting eth0->miniq_ingress to NULL, so ingress
packets on eth0 will not find filter Y in sch_handle_ingress().
This is just one of the possible consequences of concurrently accessing
miniq_{in,e}gress pointers.
Fixes: 7a096d579e8e ("net: sched: ingress: set 'unlocked' flag for Qdisc ops")
Fixes: 87f373921c4e ("net: sched: ingress: set 'unlocked' flag for clsact Qdisc ops")
Reported-by: syzbot+b53a9c0d1ea4ad62da8b@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/r/0000000000006cf87705f79acf1a@google.com/
Cc: Hillf Danton <hdanton@sina.com>
Cc: Vlad Buslov <vladbu@mellanox.com>
Signed-off-by: Peilin Ye <peilin.ye@bytedance.com>
Acked-by: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2023-06-11 03:30:25 +00:00
|
|
|
static void __qdisc_destroy(struct Qdisc *qdisc)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2008-07-17 07:47:45 +00:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
2023-12-19 18:16:19 +00:00
|
|
|
struct net_device *dev = qdisc_dev(qdisc);
|
2008-07-17 07:47:45 +00:00
|
|
|
|
2008-07-21 01:13:01 +00:00
|
|
|
#ifdef CONFIG_NET_SCHED
|
2016-08-10 09:05:15 +00:00
|
|
|
qdisc_hash_del(qdisc);
|
2008-08-22 10:24:05 +00:00
|
|
|
|
2011-01-20 03:48:19 +00:00
|
|
|
qdisc_put_stab(rtnl_dereference(qdisc->stab));
|
2008-07-21 01:13:01 +00:00
|
|
|
#endif
|
2016-12-04 17:48:16 +00:00
|
|
|
gen_kill_estimator(&qdisc->rate_est);
|
2020-05-27 04:35:23 +00:00
|
|
|
|
|
|
|
|
qdisc_reset(qdisc);
|
|
|
|
|
|
2023-12-19 18:16:19 +00:00
|
|
|
|
2008-07-17 07:47:45 +00:00
|
|
|
if (ops->destroy)
|
|
|
|
|
ops->destroy(qdisc);
|
|
|
|
|
|
net/sched: fix false lockdep warning on qdisc root lock
Xiumei and Christoph reported the following lockdep splat, complaining of
the qdisc root lock being taken twice:
============================================
WARNING: possible recursive locking detected
6.7.0-rc3+ #598 Not tainted
--------------------------------------------
swapper/2/0 is trying to acquire lock:
ffff888177190110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
but task is already holding lock:
ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&sch->q.lock);
lock(&sch->q.lock);
*** DEADLOCK ***
May be due to missing lock nesting notation
5 locks held by swapper/2/0:
#0: ffff888135a09d98 ((&in_dev->mr_ifc_timer)){+.-.}-{0:0}, at: call_timer_fn+0x11a/0x510
#1: ffffffffaaee5260 (rcu_read_lock){....}-{1:2}, at: ip_finish_output2+0x2c0/0x1ed0
#2: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
#3: ffff88811995a110 (&sch->q.lock){+.-.}-{2:2}, at: __dev_queue_xmit+0x1560/0x2e70
#4: ffffffffaaee5200 (rcu_read_lock_bh){....}-{1:2}, at: __dev_queue_xmit+0x209/0x2e70
stack backtrace:
CPU: 2 PID: 0 Comm: swapper/2 Not tainted 6.7.0-rc3+ #598
Hardware name: Red Hat KVM, BIOS 1.13.0-2.module+el8.3.0+7353+9de0a3cc 04/01/2014
Call Trace:
<IRQ>
dump_stack_lvl+0x4a/0x80
__lock_acquire+0xfdd/0x3150
lock_acquire+0x1ca/0x540
_raw_spin_lock+0x34/0x80
__dev_queue_xmit+0x1560/0x2e70
tcf_mirred_act+0x82e/0x1260 [act_mirred]
tcf_action_exec+0x161/0x480
tcf_classify+0x689/0x1170
prio_enqueue+0x316/0x660 [sch_prio]
dev_qdisc_enqueue+0x46/0x220
__dev_queue_xmit+0x1615/0x2e70
ip_finish_output2+0x1218/0x1ed0
__ip_finish_output+0x8b3/0x1350
ip_output+0x163/0x4e0
igmp_ifc_timer_expire+0x44b/0x930
call_timer_fn+0x1a2/0x510
run_timer_softirq+0x54d/0x11a0
__do_softirq+0x1b3/0x88f
irq_exit_rcu+0x18f/0x1e0
sysvec_apic_timer_interrupt+0x6f/0x90
</IRQ>
This happens when TC does a mirred egress redirect from the root qdisc of
device A to the root qdisc of device B. As long as these two locks aren't
protecting the same qdisc, they can be acquired in chain: add a per-qdisc
lockdep key to silence false warnings.
This dynamic key should safely replace the static key we have in sch_htb:
it was added to allow enqueueing to the device "direct qdisc" while still
holding the qdisc root lock.
v2: don't use static keys anymore in HTB direct qdiscs (thanks Eric Dumazet)
CC: Maxim Mikityanskiy <maxim@isovalent.com>
CC: Xiumei Mu <xmu@redhat.com>
Reported-by: Christoph Paasch <cpaasch@apple.com>
Closes: https://github.com/multipath-tcp/mptcp_net-next/issues/451
Signed-off-by: Davide Caratti <dcaratti@redhat.com>
Link: https://lore.kernel.org/r/7dc06d6158f72053cf877a82e2a7a5bd23692faa.1713448007.git.dcaratti@redhat.com
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2024-04-18 13:50:11 +00:00
|
|
|
lockdep_unregister_key(&qdisc->root_lock_key);
|
2008-07-17 07:47:45 +00:00
|
|
|
module_put(ops->owner);
|
2023-12-19 18:16:19 +00:00
|
|
|
netdev_put(dev, &qdisc->dev_tracker);
|
2008-07-17 07:47:45 +00:00
|
|
|
|
2020-05-27 04:35:24 +00:00
|
|
|
trace_qdisc_destroy(qdisc);
|
|
|
|
|
|
2018-09-24 16:22:51 +00:00
|
|
|
call_rcu(&qdisc->rcu, qdisc_free_cb);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2018-09-24 16:22:50 +00:00
|
|
|
|
net/sched: qdisc_destroy() old ingress and clsact Qdiscs before grafting
mini_Qdisc_pair::p_miniq is a double pointer to mini_Qdisc, initialized
in ingress_init() to point to net_device::miniq_ingress. ingress Qdiscs
access this per-net_device pointer in mini_qdisc_pair_swap(). Similar
for clsact Qdiscs and miniq_egress.
Unfortunately, after introducing RTNL-unlocked RTM_{NEW,DEL,GET}TFILTER
requests (thanks Hillf Danton for the hint), when replacing ingress or
clsact Qdiscs, for example, the old Qdisc ("@old") could access the same
miniq_{in,e}gress pointer(s) concurrently with the new Qdisc ("@new"),
causing race conditions [1] including a use-after-free bug in
mini_qdisc_pair_swap() reported by syzbot:
BUG: KASAN: slab-use-after-free in mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
Write of size 8 at addr ffff888045b31308 by task syz-executor690/14901
...
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106
print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:319
print_report mm/kasan/report.c:430 [inline]
kasan_report+0x11c/0x130 mm/kasan/report.c:536
mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
tcf_chain_head_change_item net/sched/cls_api.c:495 [inline]
tcf_chain0_head_change.isra.0+0xb9/0x120 net/sched/cls_api.c:509
tcf_chain_tp_insert net/sched/cls_api.c:1826 [inline]
tcf_chain_tp_insert_unique net/sched/cls_api.c:1875 [inline]
tc_new_tfilter+0x1de6/0x2290 net/sched/cls_api.c:2266
...
@old and @new should not affect each other. In other words, @old should
never modify miniq_{in,e}gress after @new, and @new should not update
@old's RCU state.
Fixing without changing sch_api.c turned out to be difficult (please
refer to Closes: for discussions). Instead, make sure @new's first call
always happen after @old's last call (in {ingress,clsact}_destroy()) has
finished:
In qdisc_graft(), return -EBUSY if @old has any ongoing filter requests,
and call qdisc_destroy() for @old before grafting @new.
Introduce qdisc_refcount_dec_if_one() as the counterpart of
qdisc_refcount_inc_nz() used for filter requests. Introduce a
non-static version of qdisc_destroy() that does a TCQ_F_BUILTIN check,
just like qdisc_put() etc.
Depends on patch "net/sched: Refactor qdisc_graft() for ingress and
clsact Qdiscs".
[1] To illustrate, the syzkaller reproducer adds ingress Qdiscs under
TC_H_ROOT (no longer possible after commit c7cfbd115001 ("net/sched:
sch_ingress: Only create under TC_H_INGRESS")) on eth0 that has 8
transmission queues:
Thread 1 creates ingress Qdisc A (containing mini Qdisc a1 and a2),
then adds a flower filter X to A.
Thread 2 creates another ingress Qdisc B (containing mini Qdisc b1 and
b2) to replace A, then adds a flower filter Y to B.
Thread 1 A's refcnt Thread 2
RTM_NEWQDISC (A, RTNL-locked)
qdisc_create(A) 1
qdisc_graft(A) 9
RTM_NEWTFILTER (X, RTNL-unlocked)
__tcf_qdisc_find(A) 10
tcf_chain0_head_change(A)
mini_qdisc_pair_swap(A) (1st)
|
| RTM_NEWQDISC (B, RTNL-locked)
RCU sync 2 qdisc_graft(B)
| 1 notify_and_destroy(A)
|
tcf_block_release(A) 0 RTM_NEWTFILTER (Y, RTNL-unlocked)
qdisc_destroy(A) tcf_chain0_head_change(B)
tcf_chain0_head_change_cb_del(A) mini_qdisc_pair_swap(B) (2nd)
mini_qdisc_pair_swap(A) (3rd) |
... ...
Here, B calls mini_qdisc_pair_swap(), pointing eth0->miniq_ingress to
its mini Qdisc, b1. Then, A calls mini_qdisc_pair_swap() again during
ingress_destroy(), setting eth0->miniq_ingress to NULL, so ingress
packets on eth0 will not find filter Y in sch_handle_ingress().
This is just one of the possible consequences of concurrently accessing
miniq_{in,e}gress pointers.
Fixes: 7a096d579e8e ("net: sched: ingress: set 'unlocked' flag for Qdisc ops")
Fixes: 87f373921c4e ("net: sched: ingress: set 'unlocked' flag for clsact Qdisc ops")
Reported-by: syzbot+b53a9c0d1ea4ad62da8b@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/r/0000000000006cf87705f79acf1a@google.com/
Cc: Hillf Danton <hdanton@sina.com>
Cc: Vlad Buslov <vladbu@mellanox.com>
Signed-off-by: Peilin Ye <peilin.ye@bytedance.com>
Acked-by: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2023-06-11 03:30:25 +00:00
|
|
|
void qdisc_destroy(struct Qdisc *qdisc)
|
|
|
|
|
{
|
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
__qdisc_destroy(qdisc);
|
|
|
|
|
}
|
|
|
|
|
|
2018-09-24 16:22:50 +00:00
|
|
|
void qdisc_put(struct Qdisc *qdisc)
|
|
|
|
|
{
|
2019-09-12 17:22:30 +00:00
|
|
|
if (!qdisc)
|
|
|
|
|
return;
|
|
|
|
|
|
2018-09-24 16:22:50 +00:00
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
|
|
|
!refcount_dec_and_test(&qdisc->refcnt))
|
|
|
|
|
return;
|
|
|
|
|
|
net/sched: qdisc_destroy() old ingress and clsact Qdiscs before grafting
mini_Qdisc_pair::p_miniq is a double pointer to mini_Qdisc, initialized
in ingress_init() to point to net_device::miniq_ingress. ingress Qdiscs
access this per-net_device pointer in mini_qdisc_pair_swap(). Similar
for clsact Qdiscs and miniq_egress.
Unfortunately, after introducing RTNL-unlocked RTM_{NEW,DEL,GET}TFILTER
requests (thanks Hillf Danton for the hint), when replacing ingress or
clsact Qdiscs, for example, the old Qdisc ("@old") could access the same
miniq_{in,e}gress pointer(s) concurrently with the new Qdisc ("@new"),
causing race conditions [1] including a use-after-free bug in
mini_qdisc_pair_swap() reported by syzbot:
BUG: KASAN: slab-use-after-free in mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
Write of size 8 at addr ffff888045b31308 by task syz-executor690/14901
...
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106
print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:319
print_report mm/kasan/report.c:430 [inline]
kasan_report+0x11c/0x130 mm/kasan/report.c:536
mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
tcf_chain_head_change_item net/sched/cls_api.c:495 [inline]
tcf_chain0_head_change.isra.0+0xb9/0x120 net/sched/cls_api.c:509
tcf_chain_tp_insert net/sched/cls_api.c:1826 [inline]
tcf_chain_tp_insert_unique net/sched/cls_api.c:1875 [inline]
tc_new_tfilter+0x1de6/0x2290 net/sched/cls_api.c:2266
...
@old and @new should not affect each other. In other words, @old should
never modify miniq_{in,e}gress after @new, and @new should not update
@old's RCU state.
Fixing without changing sch_api.c turned out to be difficult (please
refer to Closes: for discussions). Instead, make sure @new's first call
always happen after @old's last call (in {ingress,clsact}_destroy()) has
finished:
In qdisc_graft(), return -EBUSY if @old has any ongoing filter requests,
and call qdisc_destroy() for @old before grafting @new.
Introduce qdisc_refcount_dec_if_one() as the counterpart of
qdisc_refcount_inc_nz() used for filter requests. Introduce a
non-static version of qdisc_destroy() that does a TCQ_F_BUILTIN check,
just like qdisc_put() etc.
Depends on patch "net/sched: Refactor qdisc_graft() for ingress and
clsact Qdiscs".
[1] To illustrate, the syzkaller reproducer adds ingress Qdiscs under
TC_H_ROOT (no longer possible after commit c7cfbd115001 ("net/sched:
sch_ingress: Only create under TC_H_INGRESS")) on eth0 that has 8
transmission queues:
Thread 1 creates ingress Qdisc A (containing mini Qdisc a1 and a2),
then adds a flower filter X to A.
Thread 2 creates another ingress Qdisc B (containing mini Qdisc b1 and
b2) to replace A, then adds a flower filter Y to B.
Thread 1 A's refcnt Thread 2
RTM_NEWQDISC (A, RTNL-locked)
qdisc_create(A) 1
qdisc_graft(A) 9
RTM_NEWTFILTER (X, RTNL-unlocked)
__tcf_qdisc_find(A) 10
tcf_chain0_head_change(A)
mini_qdisc_pair_swap(A) (1st)
|
| RTM_NEWQDISC (B, RTNL-locked)
RCU sync 2 qdisc_graft(B)
| 1 notify_and_destroy(A)
|
tcf_block_release(A) 0 RTM_NEWTFILTER (Y, RTNL-unlocked)
qdisc_destroy(A) tcf_chain0_head_change(B)
tcf_chain0_head_change_cb_del(A) mini_qdisc_pair_swap(B) (2nd)
mini_qdisc_pair_swap(A) (3rd) |
... ...
Here, B calls mini_qdisc_pair_swap(), pointing eth0->miniq_ingress to
its mini Qdisc, b1. Then, A calls mini_qdisc_pair_swap() again during
ingress_destroy(), setting eth0->miniq_ingress to NULL, so ingress
packets on eth0 will not find filter Y in sch_handle_ingress().
This is just one of the possible consequences of concurrently accessing
miniq_{in,e}gress pointers.
Fixes: 7a096d579e8e ("net: sched: ingress: set 'unlocked' flag for Qdisc ops")
Fixes: 87f373921c4e ("net: sched: ingress: set 'unlocked' flag for clsact Qdisc ops")
Reported-by: syzbot+b53a9c0d1ea4ad62da8b@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/r/0000000000006cf87705f79acf1a@google.com/
Cc: Hillf Danton <hdanton@sina.com>
Cc: Vlad Buslov <vladbu@mellanox.com>
Signed-off-by: Peilin Ye <peilin.ye@bytedance.com>
Acked-by: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2023-06-11 03:30:25 +00:00
|
|
|
__qdisc_destroy(qdisc);
|
2018-09-24 16:22:50 +00:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(qdisc_put);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-09-24 16:22:51 +00:00
|
|
|
/* Version of qdisc_put() that is called with rtnl mutex unlocked.
|
|
|
|
|
* Intended to be used as optimization, this function only takes rtnl lock if
|
|
|
|
|
* qdisc reference counter reached zero.
|
|
|
|
|
*/
|
|
|
|
|
|
|
|
|
|
void qdisc_put_unlocked(struct Qdisc *qdisc)
|
|
|
|
|
{
|
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN ||
|
|
|
|
|
!refcount_dec_and_rtnl_lock(&qdisc->refcnt))
|
|
|
|
|
return;
|
|
|
|
|
|
net/sched: qdisc_destroy() old ingress and clsact Qdiscs before grafting
mini_Qdisc_pair::p_miniq is a double pointer to mini_Qdisc, initialized
in ingress_init() to point to net_device::miniq_ingress. ingress Qdiscs
access this per-net_device pointer in mini_qdisc_pair_swap(). Similar
for clsact Qdiscs and miniq_egress.
Unfortunately, after introducing RTNL-unlocked RTM_{NEW,DEL,GET}TFILTER
requests (thanks Hillf Danton for the hint), when replacing ingress or
clsact Qdiscs, for example, the old Qdisc ("@old") could access the same
miniq_{in,e}gress pointer(s) concurrently with the new Qdisc ("@new"),
causing race conditions [1] including a use-after-free bug in
mini_qdisc_pair_swap() reported by syzbot:
BUG: KASAN: slab-use-after-free in mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
Write of size 8 at addr ffff888045b31308 by task syz-executor690/14901
...
Call Trace:
<TASK>
__dump_stack lib/dump_stack.c:88 [inline]
dump_stack_lvl+0xd9/0x150 lib/dump_stack.c:106
print_address_description.constprop.0+0x2c/0x3c0 mm/kasan/report.c:319
print_report mm/kasan/report.c:430 [inline]
kasan_report+0x11c/0x130 mm/kasan/report.c:536
mini_qdisc_pair_swap+0x1c2/0x1f0 net/sched/sch_generic.c:1573
tcf_chain_head_change_item net/sched/cls_api.c:495 [inline]
tcf_chain0_head_change.isra.0+0xb9/0x120 net/sched/cls_api.c:509
tcf_chain_tp_insert net/sched/cls_api.c:1826 [inline]
tcf_chain_tp_insert_unique net/sched/cls_api.c:1875 [inline]
tc_new_tfilter+0x1de6/0x2290 net/sched/cls_api.c:2266
...
@old and @new should not affect each other. In other words, @old should
never modify miniq_{in,e}gress after @new, and @new should not update
@old's RCU state.
Fixing without changing sch_api.c turned out to be difficult (please
refer to Closes: for discussions). Instead, make sure @new's first call
always happen after @old's last call (in {ingress,clsact}_destroy()) has
finished:
In qdisc_graft(), return -EBUSY if @old has any ongoing filter requests,
and call qdisc_destroy() for @old before grafting @new.
Introduce qdisc_refcount_dec_if_one() as the counterpart of
qdisc_refcount_inc_nz() used for filter requests. Introduce a
non-static version of qdisc_destroy() that does a TCQ_F_BUILTIN check,
just like qdisc_put() etc.
Depends on patch "net/sched: Refactor qdisc_graft() for ingress and
clsact Qdiscs".
[1] To illustrate, the syzkaller reproducer adds ingress Qdiscs under
TC_H_ROOT (no longer possible after commit c7cfbd115001 ("net/sched:
sch_ingress: Only create under TC_H_INGRESS")) on eth0 that has 8
transmission queues:
Thread 1 creates ingress Qdisc A (containing mini Qdisc a1 and a2),
then adds a flower filter X to A.
Thread 2 creates another ingress Qdisc B (containing mini Qdisc b1 and
b2) to replace A, then adds a flower filter Y to B.
Thread 1 A's refcnt Thread 2
RTM_NEWQDISC (A, RTNL-locked)
qdisc_create(A) 1
qdisc_graft(A) 9
RTM_NEWTFILTER (X, RTNL-unlocked)
__tcf_qdisc_find(A) 10
tcf_chain0_head_change(A)
mini_qdisc_pair_swap(A) (1st)
|
| RTM_NEWQDISC (B, RTNL-locked)
RCU sync 2 qdisc_graft(B)
| 1 notify_and_destroy(A)
|
tcf_block_release(A) 0 RTM_NEWTFILTER (Y, RTNL-unlocked)
qdisc_destroy(A) tcf_chain0_head_change(B)
tcf_chain0_head_change_cb_del(A) mini_qdisc_pair_swap(B) (2nd)
mini_qdisc_pair_swap(A) (3rd) |
... ...
Here, B calls mini_qdisc_pair_swap(), pointing eth0->miniq_ingress to
its mini Qdisc, b1. Then, A calls mini_qdisc_pair_swap() again during
ingress_destroy(), setting eth0->miniq_ingress to NULL, so ingress
packets on eth0 will not find filter Y in sch_handle_ingress().
This is just one of the possible consequences of concurrently accessing
miniq_{in,e}gress pointers.
Fixes: 7a096d579e8e ("net: sched: ingress: set 'unlocked' flag for Qdisc ops")
Fixes: 87f373921c4e ("net: sched: ingress: set 'unlocked' flag for clsact Qdisc ops")
Reported-by: syzbot+b53a9c0d1ea4ad62da8b@syzkaller.appspotmail.com
Closes: https://lore.kernel.org/r/0000000000006cf87705f79acf1a@google.com/
Cc: Hillf Danton <hdanton@sina.com>
Cc: Vlad Buslov <vladbu@mellanox.com>
Signed-off-by: Peilin Ye <peilin.ye@bytedance.com>
Acked-by: Jamal Hadi Salim <jhs@mojatatu.com>
Signed-off-by: Paolo Abeni <pabeni@redhat.com>
2023-06-11 03:30:25 +00:00
|
|
|
__qdisc_destroy(qdisc);
|
2018-09-24 16:22:51 +00:00
|
|
|
rtnl_unlock();
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(qdisc_put_unlocked);
|
|
|
|
|
|
2009-09-04 06:41:20 +00:00
|
|
|
/* Attach toplevel qdisc to device queue. */
|
|
|
|
|
struct Qdisc *dev_graft_qdisc(struct netdev_queue *dev_queue,
|
|
|
|
|
struct Qdisc *qdisc)
|
|
|
|
|
{
|
2023-06-06 11:19:29 +00:00
|
|
|
struct Qdisc *oqdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2009-09-04 06:41:20 +00:00
|
|
|
spinlock_t *root_lock;
|
|
|
|
|
|
|
|
|
|
root_lock = qdisc_lock(oqdisc);
|
|
|
|
|
spin_lock_bh(root_lock);
|
|
|
|
|
|
|
|
|
|
/* ... and graft new one */
|
|
|
|
|
if (qdisc == NULL)
|
|
|
|
|
qdisc = &noop_qdisc;
|
2023-06-06 11:19:29 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
|
2009-09-04 06:41:20 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, &noop_qdisc);
|
|
|
|
|
|
|
|
|
|
spin_unlock_bh(root_lock);
|
|
|
|
|
|
|
|
|
|
return oqdisc;
|
|
|
|
|
}
|
2011-01-17 08:06:09 +00:00
|
|
|
EXPORT_SYMBOL(dev_graft_qdisc);
|
2009-09-04 06:41:20 +00:00
|
|
|
|
2022-08-26 09:00:55 +00:00
|
|
|
static void shutdown_scheduler_queue(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
|
void *_qdisc_default)
|
|
|
|
|
{
|
2023-06-06 11:19:29 +00:00
|
|
|
struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2022-08-26 09:00:55 +00:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
|
|
|
|
|
|
|
|
|
if (qdisc) {
|
|
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2023-06-06 11:19:29 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc_default);
|
2022-08-26 09:00:55 +00:00
|
|
|
|
|
|
|
|
qdisc_put(qdisc);
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-17 07:34:19 +00:00
|
|
|
static void attach_one_default_qdisc(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
|
void *_unused)
|
|
|
|
|
{
|
2015-08-27 19:21:39 +00:00
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
const struct Qdisc_ops *ops = default_qdisc_ops;
|
2008-07-17 07:34:19 +00:00
|
|
|
|
2015-08-27 19:21:39 +00:00
|
|
|
if (dev->priv_flags & IFF_NO_QUEUE)
|
|
|
|
|
ops = &noqueue_qdisc_ops;
|
net: sch_generic: Use pfifo_fast as fallback scheduler for CAN hardware
There is networking hardware that isn't based on Ethernet for layers 1 and 2.
For example CAN.
CAN is a multi-master serial bus standard for connecting Electronic Control
Units [ECUs] also known as nodes. A frame on the CAN bus carries up to 8 bytes
of payload. Frame corruption is detected by a CRC. However frame loss due to
corruption is possible, but a quite unusual phenomenon.
While fq_codel works great for TCP/IP, it doesn't for CAN. There are a lot of
legacy protocols on top of CAN, which are not build with flow control or high
CAN frame drop rates in mind.
When using fq_codel, as soon as the queue reaches a certain delay based length,
skbs from the head of the queue are silently dropped. Silently meaning that the
user space using a send() or similar syscall doesn't get an error. However
TCP's flow control algorithm will detect dropped packages and adjust the
bandwidth accordingly.
When using fq_codel and sending raw frames over CAN, which is the common use
case, the user space thinks the package has been sent without problems, because
send() returned without an error. pfifo_fast will drop skbs, if the queue
length exceeds the maximum. But with this scheduler the skbs at the tail are
dropped, an error (-ENOBUFS) is propagated to user space. So that the user
space can slow down the package generation.
On distributions, where fq_codel is made default via CONFIG_DEFAULT_NET_SCH
during compile time, or set default during runtime with sysctl
net.core.default_qdisc (see [1]), we get a bad user experience. In my test case
with pfifo_fast, I can transfer thousands of million CAN frames without a frame
drop. On the other hand with fq_codel there is more then one lost CAN frame per
thousand frames.
As pointed out fq_codel is not suited for CAN hardware, so this patch changes
attach_one_default_qdisc() to use pfifo_fast for "ARPHRD_CAN" network devices.
During transition of a netdev from down to up state the default queuing
discipline is attached by attach_default_qdiscs() with the help of
attach_one_default_qdisc(). This patch modifies attach_one_default_qdisc() to
attach the pfifo_fast (pfifo_fast_ops) if the network device type is
"ARPHRD_CAN".
[1] https://github.com/systemd/systemd/issues/9194
Suggested-by: Marc Kleine-Budde <mkl@pengutronix.de>
Signed-off-by: Marc Kleine-Budde <mkl@pengutronix.de>
Signed-off-by: Vincent Prince <vincent.prince.fr@gmail.com>
Acked-by: Dave Taht <dave.taht@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-23 13:44:20 +00:00
|
|
|
else if(dev->type == ARPHRD_CAN)
|
|
|
|
|
ops = &pfifo_fast_ops;
|
2015-08-27 19:21:39 +00:00
|
|
|
|
2017-12-20 17:35:21 +00:00
|
|
|
qdisc = qdisc_create_dflt(dev_queue, ops, TC_H_ROOT, NULL);
|
2020-04-30 11:42:22 +00:00
|
|
|
if (!qdisc)
|
2015-08-27 19:21:39 +00:00
|
|
|
return;
|
2020-04-30 11:42:22 +00:00
|
|
|
|
2015-08-27 19:21:39 +00:00
|
|
|
if (!netif_is_multiqueue(dev))
|
2015-12-02 04:08:51 +00:00
|
|
|
qdisc->flags |= TCQ_F_ONETXQUEUE | TCQ_F_NOPARENT;
|
2023-06-06 11:19:29 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
|
2008-07-17 07:34:19 +00:00
|
|
|
}
|
|
|
|
|
|
2009-09-06 08:58:51 +00:00
|
|
|
static void attach_default_qdiscs(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
|
|
|
|
txq = netdev_get_tx_queue(dev, 0);
|
|
|
|
|
|
2015-08-13 17:01:07 +00:00
|
|
|
if (!netif_is_multiqueue(dev) ||
|
|
|
|
|
dev->priv_flags & IFF_NO_QUEUE) {
|
2009-09-06 08:58:51 +00:00
|
|
|
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
|
2023-06-06 11:19:29 +00:00
|
|
|
qdisc = rtnl_dereference(txq->qdisc_sleeping);
|
2022-02-11 20:06:23 +00:00
|
|
|
rcu_assign_pointer(dev->qdisc, qdisc);
|
|
|
|
|
qdisc_refcount_inc(qdisc);
|
2009-09-06 08:58:51 +00:00
|
|
|
} else {
|
2017-12-20 17:35:21 +00:00
|
|
|
qdisc = qdisc_create_dflt(txq, &mq_qdisc_ops, TC_H_ROOT, NULL);
|
2009-09-06 08:58:51 +00:00
|
|
|
if (qdisc) {
|
2022-02-11 20:06:23 +00:00
|
|
|
rcu_assign_pointer(dev->qdisc, qdisc);
|
2013-12-12 23:41:56 +00:00
|
|
|
qdisc->ops->attach(qdisc);
|
2009-09-06 08:58:51 +00:00
|
|
|
}
|
|
|
|
|
}
|
2022-02-11 20:06:23 +00:00
|
|
|
qdisc = rtnl_dereference(dev->qdisc);
|
2020-04-30 11:42:22 +00:00
|
|
|
|
|
|
|
|
/* Detect default qdisc setup/init failed and fallback to "noqueue" */
|
2022-02-11 20:06:23 +00:00
|
|
|
if (qdisc == &noop_qdisc) {
|
2020-04-30 11:42:22 +00:00
|
|
|
netdev_warn(dev, "default qdisc (%s) fail, fallback to %s\n",
|
|
|
|
|
default_qdisc_ops->id, noqueue_qdisc_ops.id);
|
2022-08-26 09:00:55 +00:00
|
|
|
netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
|
2020-04-30 11:42:22 +00:00
|
|
|
dev->priv_flags |= IFF_NO_QUEUE;
|
|
|
|
|
netdev_for_each_tx_queue(dev, attach_one_default_qdisc, NULL);
|
2023-06-06 11:19:29 +00:00
|
|
|
qdisc = rtnl_dereference(txq->qdisc_sleeping);
|
2022-02-11 20:06:23 +00:00
|
|
|
rcu_assign_pointer(dev->qdisc, qdisc);
|
|
|
|
|
qdisc_refcount_inc(qdisc);
|
2020-04-30 11:42:22 +00:00
|
|
|
dev->priv_flags ^= IFF_NO_QUEUE;
|
|
|
|
|
}
|
|
|
|
|
|
2016-08-10 09:05:15 +00:00
|
|
|
#ifdef CONFIG_NET_SCHED
|
2022-02-11 20:06:23 +00:00
|
|
|
if (qdisc != &noop_qdisc)
|
|
|
|
|
qdisc_hash_add(qdisc, false);
|
2016-08-10 09:05:15 +00:00
|
|
|
#endif
|
2009-09-06 08:58:51 +00:00
|
|
|
}
|
|
|
|
|
|
2008-07-17 07:34:19 +00:00
|
|
|
static void transition_one_qdisc(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
|
void *_need_watchdog)
|
|
|
|
|
{
|
2023-06-06 11:19:29 +00:00
|
|
|
struct Qdisc *new_qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2008-07-17 07:34:19 +00:00
|
|
|
int *need_watchdog_p = _need_watchdog;
|
|
|
|
|
|
2008-08-18 04:51:03 +00:00
|
|
|
if (!(new_qdisc->flags & TCQ_F_BUILTIN))
|
|
|
|
|
clear_bit(__QDISC_STATE_DEACTIVATED, &new_qdisc->state);
|
|
|
|
|
|
2008-07-17 07:53:03 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, new_qdisc);
|
2015-08-27 19:21:39 +00:00
|
|
|
if (need_watchdog_p) {
|
2021-11-17 03:29:22 +00:00
|
|
|
WRITE_ONCE(dev_queue->trans_start, 0);
|
2008-07-17 07:34:19 +00:00
|
|
|
*need_watchdog_p = 1;
|
2009-05-18 03:55:16 +00:00
|
|
|
}
|
2008-07-17 07:34:19 +00:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void dev_activate(struct net_device *dev)
|
|
|
|
|
{
|
2008-07-17 07:34:19 +00:00
|
|
|
int need_watchdog;
|
2008-07-09 00:42:10 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* No queueing discipline is attached to device;
|
2013-08-27 23:19:08 +00:00
|
|
|
* create default one for devices, which need queueing
|
|
|
|
|
* and noqueue_qdisc for virtual interfaces
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
|
2022-02-11 20:06:23 +00:00
|
|
|
if (rtnl_dereference(dev->qdisc) == &noop_qdisc)
|
2009-09-06 08:58:51 +00:00
|
|
|
attach_default_qdiscs(dev);
|
2009-09-04 06:41:18 +00:00
|
|
|
|
2005-05-03 23:18:52 +00:00
|
|
|
if (!netif_carrier_ok(dev))
|
|
|
|
|
/* Delay activation until next carrier-on event */
|
|
|
|
|
return;
|
|
|
|
|
|
2008-07-17 07:34:19 +00:00
|
|
|
need_watchdog = 0;
|
|
|
|
|
netdev_for_each_tx_queue(dev, transition_one_qdisc, &need_watchdog);
|
2010-10-02 06:11:55 +00:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
|
transition_one_qdisc(dev, dev_ingress_queue(dev), NULL);
|
2008-07-17 07:34:19 +00:00
|
|
|
|
|
|
|
|
if (need_watchdog) {
|
2016-05-03 14:33:13 +00:00
|
|
|
netif_trans_update(dev);
|
2005-04-16 22:20:36 +00:00
|
|
|
dev_watchdog_up(dev);
|
|
|
|
|
}
|
2008-07-09 00:42:10 +00:00
|
|
|
}
|
2011-01-17 08:06:09 +00:00
|
|
|
EXPORT_SYMBOL(dev_activate);
|
2008-07-09 00:42:10 +00:00
|
|
|
|
2020-05-27 04:35:26 +00:00
|
|
|
static void qdisc_deactivate(struct Qdisc *qdisc)
|
|
|
|
|
{
|
|
|
|
|
if (qdisc->flags & TCQ_F_BUILTIN)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
set_bit(__QDISC_STATE_DEACTIVATED, &qdisc->state);
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-17 07:34:19 +00:00
|
|
|
static void dev_deactivate_queue(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
|
void *_qdisc_default)
|
2008-07-09 00:42:10 +00:00
|
|
|
{
|
2008-07-17 07:34:19 +00:00
|
|
|
struct Qdisc *qdisc_default = _qdisc_default;
|
2008-07-09 06:10:33 +00:00
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
|
2014-09-13 03:04:52 +00:00
|
|
|
qdisc = rtnl_dereference(dev_queue->qdisc);
|
2008-07-09 00:42:10 +00:00
|
|
|
if (qdisc) {
|
2020-05-27 04:35:26 +00:00
|
|
|
qdisc_deactivate(qdisc);
|
2008-08-27 09:22:07 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc_default);
|
2008-07-09 00:42:10 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2020-09-08 11:02:34 +00:00
|
|
|
static void dev_reset_queue(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
|
|
|
|
void *_unused)
|
|
|
|
|
{
|
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
bool nolock;
|
|
|
|
|
|
2023-06-06 11:19:29 +00:00
|
|
|
qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2020-09-08 11:02:34 +00:00
|
|
|
if (!qdisc)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
nolock = qdisc->flags & TCQ_F_NOLOCK;
|
|
|
|
|
|
|
|
|
|
if (nolock)
|
|
|
|
|
spin_lock_bh(&qdisc->seqlock);
|
|
|
|
|
spin_lock_bh(qdisc_lock(qdisc));
|
|
|
|
|
|
|
|
|
|
qdisc_reset(qdisc);
|
|
|
|
|
|
|
|
|
|
spin_unlock_bh(qdisc_lock(qdisc));
|
net: sched: fix tx action rescheduling issue during deactivation
Currently qdisc_run() checks the STATE_DEACTIVATED of lockless
qdisc before calling __qdisc_run(), which ultimately clear the
STATE_MISSED when all the skb is dequeued. If STATE_DEACTIVATED
is set before clearing STATE_MISSED, there may be rescheduling
of net_tx_action() at the end of qdisc_run_end(), see below:
CPU0(net_tx_atcion) CPU1(__dev_xmit_skb) CPU2(dev_deactivate)
. . .
. set STATE_MISSED .
. __netif_schedule() .
. . set STATE_DEACTIVATED
. . qdisc_reset()
. . .
.<--------------- . synchronize_net()
clear __QDISC_STATE_SCHED | . .
. | . .
. | . some_qdisc_is_busy()
. | . return *false*
. | . .
test STATE_DEACTIVATED | . .
__qdisc_run() *not* called | . .
. | . .
test STATE_MISS | . .
__netif_schedule()--------| . .
. . .
. . .
__qdisc_run() is not called by net_tx_atcion() in CPU0 because
CPU2 has set STATE_DEACTIVATED flag during dev_deactivate(), and
STATE_MISSED is only cleared in __qdisc_run(), __netif_schedule
is called at the end of qdisc_run_end(), causing tx action
rescheduling problem.
qdisc_run() called by net_tx_action() runs in the softirq context,
which should has the same semantic as the qdisc_run() called by
__dev_xmit_skb() protected by rcu_read_lock_bh(). And there is a
synchronize_net() between STATE_DEACTIVATED flag being set and
qdisc_reset()/some_qdisc_is_busy in dev_deactivate(), we can safely
bail out for the deactived lockless qdisc in net_tx_action(), and
qdisc_reset() will reset all skb not dequeued yet.
So add the rcu_read_lock() explicitly to protect the qdisc_run()
and do the STATE_DEACTIVATED checking in net_tx_action() before
calling qdisc_run_begin(). Another option is to do the checking in
the qdisc_run_end(), but it will add unnecessary overhead for
non-tx_action case, because __dev_queue_xmit() will not see qdisc
with STATE_DEACTIVATED after synchronize_net(), the qdisc with
STATE_DEACTIVATED can only be seen by net_tx_action() because of
__netif_schedule().
The STATE_DEACTIVATED checking in qdisc_run() is to avoid race
between net_tx_action() and qdisc_reset(), see:
commit d518d2ed8640 ("net/sched: fix race between deactivation
and dequeue for NOLOCK qdisc"). As the bailout added above for
deactived lockless qdisc in net_tx_action() provides better
protection for the race without calling qdisc_run() at all, so
remove the STATE_DEACTIVATED checking in qdisc_run().
After qdisc_reset(), there is no skb in qdisc to be dequeued, so
clear the STATE_MISSED in dev_reset_queue() too.
Fixes: 6b3ba9146fe6 ("net: sched: allow qdiscs to handle locking")
Acked-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
V8: Clearing STATE_MISSED before calling __netif_schedule() has
avoid the endless rescheduling problem, but there may still
be a unnecessary rescheduling, so adjust the commit log.
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-05-14 03:17:00 +00:00
|
|
|
if (nolock) {
|
|
|
|
|
clear_bit(__QDISC_STATE_MISSED, &qdisc->state);
|
net: sched: implement TCQ_F_CAN_BYPASS for lockless qdisc
Currently pfifo_fast has both TCQ_F_CAN_BYPASS and TCQ_F_NOLOCK
flag set, but queue discipline by-pass does not work for lockless
qdisc because skb is always enqueued to qdisc even when the qdisc
is empty, see __dev_xmit_skb().
This patch calls sch_direct_xmit() to transmit the skb directly
to the driver for empty lockless qdisc, which aviod enqueuing
and dequeuing operation.
As qdisc->empty is not reliable to indicate a empty qdisc because
there is a time window between enqueuing and setting qdisc->empty.
So we use the MISSED state added in commit a90c57f2cedd ("net:
sched: fix packet stuck problem for lockless qdisc"), which
indicate there is lock contention, suggesting that it is better
not to do the qdisc bypass in order to avoid packet out of order
problem.
In order to make MISSED state reliable to indicate a empty qdisc,
we need to ensure that testing and clearing of MISSED state is
within the protection of qdisc->seqlock, only setting MISSED state
can be done without the protection of qdisc->seqlock. A MISSED
state testing is added without the protection of qdisc->seqlock to
aviod doing unnecessary spin_trylock() for contention case.
As the enqueuing is not within the protection of qdisc->seqlock,
there is still a potential data race as mentioned by Jakub [1]:
thread1 thread2 thread3
qdisc_run_begin() # true
qdisc_run_begin(q)
set(MISSED)
pfifo_fast_dequeue
clear(MISSED)
# recheck the queue
qdisc_run_end()
enqueue skb1
qdisc empty # true
qdisc_run_begin() # true
sch_direct_xmit() # skb2
qdisc_run_begin()
set(MISSED)
When above happens, skb1 enqueued by thread2 is transmited after
skb2 is transmited by thread3 because MISSED state setting and
enqueuing is not under the qdisc->seqlock. If qdisc bypass is
disabled, skb1 has better chance to be transmited quicker than
skb2.
This patch does not take care of the above data race, because we
view this as similar as below:
Even at the same time CPU1 and CPU2 write the skb to two socket
which both heading to the same qdisc, there is no guarantee that
which skb will hit the qdisc first, because there is a lot of
factor like interrupt/softirq/cache miss/scheduling afffecting
that.
There are below cases that need special handling:
1. When MISSED state is cleared before another round of dequeuing
in pfifo_fast_dequeue(), and __qdisc_run() might not be able to
dequeue all skb in one round and call __netif_schedule(), which
might result in a non-empty qdisc without MISSED set. In order
to avoid this, the MISSED state is set for lockless qdisc and
__netif_schedule() will be called at the end of qdisc_run_end.
2. The MISSED state also need to be set for lockless qdisc instead
of calling __netif_schedule() directly when requeuing a skb for
a similar reason.
3. For netdev queue stopped case, the MISSED case need clearing
while the netdev queue is stopped, otherwise there may be
unnecessary __netif_schedule() calling. So a new DRAINING state
is added to indicate this case, which also indicate a non-empty
qdisc.
4. As there is already netif_xmit_frozen_or_stopped() checking in
dequeue_skb() and sch_direct_xmit(), which are both within the
protection of qdisc->seqlock, but the same checking in
__dev_xmit_skb() is without the protection, which might cause
empty indication of a lockless qdisc to be not reliable. So
remove the checking in __dev_xmit_skb(), and the checking in
the protection of qdisc->seqlock seems enough to avoid the cpu
consumption problem for netdev queue stopped case.
1. https://lkml.org/lkml/2021/5/29/215
Acked-by: Jakub Kicinski <kuba@kernel.org>
Tested-by: Vladimir Oltean <vladimir.oltean@nxp.com> # flexcan
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-06-22 06:49:56 +00:00
|
|
|
clear_bit(__QDISC_STATE_DRAINING, &qdisc->state);
|
2020-09-08 11:02:34 +00:00
|
|
|
spin_unlock_bh(&qdisc->seqlock);
|
net: sched: fix tx action rescheduling issue during deactivation
Currently qdisc_run() checks the STATE_DEACTIVATED of lockless
qdisc before calling __qdisc_run(), which ultimately clear the
STATE_MISSED when all the skb is dequeued. If STATE_DEACTIVATED
is set before clearing STATE_MISSED, there may be rescheduling
of net_tx_action() at the end of qdisc_run_end(), see below:
CPU0(net_tx_atcion) CPU1(__dev_xmit_skb) CPU2(dev_deactivate)
. . .
. set STATE_MISSED .
. __netif_schedule() .
. . set STATE_DEACTIVATED
. . qdisc_reset()
. . .
.<--------------- . synchronize_net()
clear __QDISC_STATE_SCHED | . .
. | . .
. | . some_qdisc_is_busy()
. | . return *false*
. | . .
test STATE_DEACTIVATED | . .
__qdisc_run() *not* called | . .
. | . .
test STATE_MISS | . .
__netif_schedule()--------| . .
. . .
. . .
__qdisc_run() is not called by net_tx_atcion() in CPU0 because
CPU2 has set STATE_DEACTIVATED flag during dev_deactivate(), and
STATE_MISSED is only cleared in __qdisc_run(), __netif_schedule
is called at the end of qdisc_run_end(), causing tx action
rescheduling problem.
qdisc_run() called by net_tx_action() runs in the softirq context,
which should has the same semantic as the qdisc_run() called by
__dev_xmit_skb() protected by rcu_read_lock_bh(). And there is a
synchronize_net() between STATE_DEACTIVATED flag being set and
qdisc_reset()/some_qdisc_is_busy in dev_deactivate(), we can safely
bail out for the deactived lockless qdisc in net_tx_action(), and
qdisc_reset() will reset all skb not dequeued yet.
So add the rcu_read_lock() explicitly to protect the qdisc_run()
and do the STATE_DEACTIVATED checking in net_tx_action() before
calling qdisc_run_begin(). Another option is to do the checking in
the qdisc_run_end(), but it will add unnecessary overhead for
non-tx_action case, because __dev_queue_xmit() will not see qdisc
with STATE_DEACTIVATED after synchronize_net(), the qdisc with
STATE_DEACTIVATED can only be seen by net_tx_action() because of
__netif_schedule().
The STATE_DEACTIVATED checking in qdisc_run() is to avoid race
between net_tx_action() and qdisc_reset(), see:
commit d518d2ed8640 ("net/sched: fix race between deactivation
and dequeue for NOLOCK qdisc"). As the bailout added above for
deactived lockless qdisc in net_tx_action() provides better
protection for the race without calling qdisc_run() at all, so
remove the STATE_DEACTIVATED checking in qdisc_run().
After qdisc_reset(), there is no skb in qdisc to be dequeued, so
clear the STATE_MISSED in dev_reset_queue() too.
Fixes: 6b3ba9146fe6 ("net: sched: allow qdiscs to handle locking")
Acked-by: Jakub Kicinski <kuba@kernel.org>
Signed-off-by: Yunsheng Lin <linyunsheng@huawei.com>
V8: Clearing STATE_MISSED before calling __netif_schedule() has
avoid the endless rescheduling problem, but there may still
be a unnecessary rescheduling, so adjust the commit log.
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-05-14 03:17:00 +00:00
|
|
|
}
|
2020-09-08 11:02:34 +00:00
|
|
|
}
|
|
|
|
|
|
2008-08-18 04:58:07 +00:00
|
|
|
static bool some_qdisc_is_busy(struct net_device *dev)
|
2008-07-17 07:34:19 +00:00
|
|
|
{
|
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
|
struct netdev_queue *dev_queue;
|
2008-07-16 08:42:40 +00:00
|
|
|
spinlock_t *root_lock;
|
2008-07-16 07:56:32 +00:00
|
|
|
struct Qdisc *q;
|
2008-07-17 07:34:19 +00:00
|
|
|
int val;
|
|
|
|
|
|
|
|
|
|
dev_queue = netdev_get_tx_queue(dev, i);
|
2023-06-06 11:19:29 +00:00
|
|
|
q = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2008-07-17 07:34:19 +00:00
|
|
|
|
2018-05-15 08:50:31 +00:00
|
|
|
root_lock = qdisc_lock(q);
|
|
|
|
|
spin_lock_bh(root_lock);
|
2008-07-17 07:34:19 +00:00
|
|
|
|
2018-05-15 08:50:31 +00:00
|
|
|
val = (qdisc_is_running(q) ||
|
|
|
|
|
test_bit(__QDISC_STATE_SCHED, &q->state));
|
2008-07-17 07:34:19 +00:00
|
|
|
|
2018-05-15 08:50:31 +00:00
|
|
|
spin_unlock_bh(root_lock);
|
2008-07-17 07:34:19 +00:00
|
|
|
|
|
|
|
|
if (val)
|
|
|
|
|
return true;
|
|
|
|
|
}
|
|
|
|
|
return false;
|
|
|
|
|
}
|
|
|
|
|
|
2011-05-19 23:42:09 +00:00
|
|
|
/**
|
|
|
|
|
* dev_deactivate_many - deactivate transmissions on several devices
|
|
|
|
|
* @head: list of devices to deactivate
|
|
|
|
|
*
|
|
|
|
|
* This function returns only when all outstanding transmissions
|
|
|
|
|
* have completed, unless all devices are in dismantle phase.
|
|
|
|
|
*/
|
2010-12-13 12:44:07 +00:00
|
|
|
void dev_deactivate_many(struct list_head *head)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-12-13 12:44:07 +00:00
|
|
|
struct net_device *dev;
|
2007-05-10 21:12:47 +00:00
|
|
|
|
2013-10-06 02:26:05 +00:00
|
|
|
list_for_each_entry(dev, head, close_list) {
|
2010-12-13 12:44:07 +00:00
|
|
|
netdev_for_each_tx_queue(dev, dev_deactivate_queue,
|
|
|
|
|
&noop_qdisc);
|
|
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
|
dev_deactivate_queue(dev, dev_ingress_queue(dev),
|
|
|
|
|
&noop_qdisc);
|
|
|
|
|
|
|
|
|
|
dev_watchdog_down(dev);
|
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-09-08 11:02:34 +00:00
|
|
|
/* Wait for outstanding qdisc-less dev_queue_xmit calls or
|
|
|
|
|
* outstanding qdisc enqueuing calls.
|
2011-05-19 23:42:09 +00:00
|
|
|
* This is avoided if all devices are in dismantle phase :
|
|
|
|
|
* Caller will call synchronize_net() for us
|
|
|
|
|
*/
|
2017-12-07 17:56:04 +00:00
|
|
|
synchronize_net();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-09-08 11:02:34 +00:00
|
|
|
list_for_each_entry(dev, head, close_list) {
|
|
|
|
|
netdev_for_each_tx_queue(dev, dev_reset_queue, NULL);
|
|
|
|
|
|
|
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
|
dev_reset_queue(dev, dev_ingress_queue(dev), NULL);
|
|
|
|
|
}
|
|
|
|
|
|
2006-06-22 09:28:18 +00:00
|
|
|
/* Wait for outstanding qdisc_run calls. */
|
2017-12-07 17:56:04 +00:00
|
|
|
list_for_each_entry(dev, head, close_list) {
|
2019-10-16 08:28:33 +00:00
|
|
|
while (some_qdisc_is_busy(dev)) {
|
|
|
|
|
/* wait_event() would avoid this sleep-loop but would
|
|
|
|
|
* require expensive checks in the fast paths of packet
|
|
|
|
|
* processing which isn't worth it.
|
|
|
|
|
*/
|
|
|
|
|
schedule_timeout_uninterruptible(1);
|
|
|
|
|
}
|
2017-12-07 17:56:04 +00:00
|
|
|
}
|
2010-12-13 12:44:07 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
void dev_deactivate(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
LIST_HEAD(single);
|
|
|
|
|
|
2013-10-06 02:26:05 +00:00
|
|
|
list_add(&dev->close_list, &single);
|
2010-12-13 12:44:07 +00:00
|
|
|
dev_deactivate_many(&single);
|
2011-02-20 19:49:45 +00:00
|
|
|
list_del(&single);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2011-01-17 08:06:09 +00:00
|
|
|
EXPORT_SYMBOL(dev_deactivate);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-01-26 02:26:23 +00:00
|
|
|
static int qdisc_change_tx_queue_len(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue)
|
|
|
|
|
{
|
2023-06-06 11:19:29 +00:00
|
|
|
struct Qdisc *qdisc = rtnl_dereference(dev_queue->qdisc_sleeping);
|
2018-01-26 02:26:23 +00:00
|
|
|
const struct Qdisc_ops *ops = qdisc->ops;
|
|
|
|
|
|
|
|
|
|
if (ops->change_tx_queue_len)
|
|
|
|
|
return ops->change_tx_queue_len(qdisc, dev->tx_queue_len);
|
|
|
|
|
return 0;
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-13 22:53:30 +00:00
|
|
|
void dev_qdisc_change_real_num_tx(struct net_device *dev,
|
|
|
|
|
unsigned int new_real_tx)
|
|
|
|
|
{
|
2022-02-11 20:06:23 +00:00
|
|
|
struct Qdisc *qdisc = rtnl_dereference(dev->qdisc);
|
2021-09-13 22:53:30 +00:00
|
|
|
|
|
|
|
|
if (qdisc->ops->change_real_num_tx)
|
|
|
|
|
qdisc->ops->change_real_num_tx(qdisc, new_real_tx);
|
|
|
|
|
}
|
|
|
|
|
|
2021-09-17 13:55:06 +00:00
|
|
|
void mq_change_real_num_tx(struct Qdisc *sch, unsigned int new_real_tx)
|
|
|
|
|
{
|
|
|
|
|
#ifdef CONFIG_NET_SCHED
|
|
|
|
|
struct net_device *dev = qdisc_dev(sch);
|
|
|
|
|
struct Qdisc *qdisc;
|
|
|
|
|
unsigned int i;
|
|
|
|
|
|
|
|
|
|
for (i = new_real_tx; i < dev->real_num_tx_queues; i++) {
|
2023-06-06 11:19:29 +00:00
|
|
|
qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
|
2021-09-17 13:55:06 +00:00
|
|
|
/* Only update the default qdiscs we created,
|
|
|
|
|
* qdiscs with handles are always hashed.
|
|
|
|
|
*/
|
|
|
|
|
if (qdisc != &noop_qdisc && !qdisc->handle)
|
|
|
|
|
qdisc_hash_del(qdisc);
|
|
|
|
|
}
|
|
|
|
|
for (i = dev->real_num_tx_queues; i < new_real_tx; i++) {
|
2023-06-06 11:19:29 +00:00
|
|
|
qdisc = rtnl_dereference(netdev_get_tx_queue(dev, i)->qdisc_sleeping);
|
2021-09-17 13:55:06 +00:00
|
|
|
if (qdisc != &noop_qdisc && !qdisc->handle)
|
|
|
|
|
qdisc_hash_add(qdisc, false);
|
|
|
|
|
}
|
|
|
|
|
#endif
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(mq_change_real_num_tx);
|
|
|
|
|
|
2018-01-26 02:26:23 +00:00
|
|
|
int dev_qdisc_change_tx_queue_len(struct net_device *dev)
|
|
|
|
|
{
|
|
|
|
|
bool up = dev->flags & IFF_UP;
|
|
|
|
|
unsigned int i;
|
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
|
|
if (up)
|
|
|
|
|
dev_deactivate(dev);
|
|
|
|
|
|
|
|
|
|
for (i = 0; i < dev->num_tx_queues; i++) {
|
|
|
|
|
ret = qdisc_change_tx_queue_len(dev, &dev->_tx[i]);
|
|
|
|
|
|
|
|
|
|
/* TODO: revert changes on a partial failure */
|
|
|
|
|
if (ret)
|
|
|
|
|
break;
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
if (up)
|
|
|
|
|
dev_activate(dev);
|
|
|
|
|
return ret;
|
|
|
|
|
}
|
|
|
|
|
|
2008-07-09 00:42:10 +00:00
|
|
|
static void dev_init_scheduler_queue(struct net_device *dev,
|
|
|
|
|
struct netdev_queue *dev_queue,
|
2008-07-17 07:34:19 +00:00
|
|
|
void *_qdisc)
|
2008-07-09 00:42:10 +00:00
|
|
|
{
|
2008-07-17 07:34:19 +00:00
|
|
|
struct Qdisc *qdisc = _qdisc;
|
|
|
|
|
|
2014-09-13 03:04:52 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc, qdisc);
|
2023-06-06 11:19:29 +00:00
|
|
|
rcu_assign_pointer(dev_queue->qdisc_sleeping, qdisc);
|
2008-07-09 00:42:10 +00:00
|
|
|
}
|
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
void dev_init_scheduler(struct net_device *dev)
|
|
|
|
|
{
|
2022-02-11 20:06:23 +00:00
|
|
|
rcu_assign_pointer(dev->qdisc, &noop_qdisc);
|
2008-07-17 07:34:19 +00:00
|
|
|
netdev_for_each_tx_queue(dev, dev_init_scheduler_queue, &noop_qdisc);
|
2010-10-02 06:11:55 +00:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
|
dev_init_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-10-17 00:29:17 +00:00
|
|
|
timer_setup(&dev->watchdog_timer, dev_watchdog, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
2008-07-09 00:42:10 +00:00
|
|
|
void dev_shutdown(struct net_device *dev)
|
|
|
|
|
{
|
2008-07-17 07:34:19 +00:00
|
|
|
netdev_for_each_tx_queue(dev, shutdown_scheduler_queue, &noop_qdisc);
|
2010-10-02 06:11:55 +00:00
|
|
|
if (dev_ingress_queue(dev))
|
|
|
|
|
shutdown_scheduler_queue(dev, dev_ingress_queue(dev), &noop_qdisc);
|
2022-02-11 20:06:23 +00:00
|
|
|
qdisc_put(rtnl_dereference(dev->qdisc));
|
|
|
|
|
rcu_assign_pointer(dev->qdisc, &noop_qdisc);
|
2009-09-04 06:41:18 +00:00
|
|
|
|
2008-07-26 04:43:18 +00:00
|
|
|
WARN_ON(timer_pending(&dev->watchdog_timer));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2013-02-12 00:12:03 +00:00
|
|
|
|
2021-03-12 14:08:31 +00:00
|
|
|
/**
|
|
|
|
|
* psched_ratecfg_precompute__() - Pre-compute values for reciprocal division
|
|
|
|
|
* @rate: Rate to compute reciprocal division values of
|
|
|
|
|
* @mult: Multiplier for reciprocal division
|
|
|
|
|
* @shift: Shift for reciprocal division
|
|
|
|
|
*
|
|
|
|
|
* The multiplier and shift for reciprocal division by rate are stored
|
|
|
|
|
* in mult and shift.
|
|
|
|
|
*
|
|
|
|
|
* The deal here is to replace a divide by a reciprocal one
|
|
|
|
|
* in fast path (a reciprocal divide is a multiply and a shift)
|
|
|
|
|
*
|
|
|
|
|
* Normal formula would be :
|
|
|
|
|
* time_in_ns = (NSEC_PER_SEC * len) / rate_bps
|
|
|
|
|
*
|
|
|
|
|
* We compute mult/shift to use instead :
|
|
|
|
|
* time_in_ns = (len * mult) >> shift;
|
|
|
|
|
*
|
|
|
|
|
* We try to get the highest possible mult value for accuracy,
|
|
|
|
|
* but have to make sure no overflows will ever happen.
|
|
|
|
|
*
|
|
|
|
|
* reciprocal_value() is not used here it doesn't handle 64-bit values.
|
|
|
|
|
*/
|
|
|
|
|
static void psched_ratecfg_precompute__(u64 rate, u32 *mult, u8 *shift)
|
|
|
|
|
{
|
|
|
|
|
u64 factor = NSEC_PER_SEC;
|
|
|
|
|
|
|
|
|
|
*mult = 1;
|
|
|
|
|
*shift = 0;
|
|
|
|
|
|
|
|
|
|
if (rate <= 0)
|
|
|
|
|
return;
|
|
|
|
|
|
|
|
|
|
for (;;) {
|
|
|
|
|
*mult = div64_u64(factor, rate);
|
|
|
|
|
if (*mult & (1U << 31) || factor & (1ULL << 63))
|
|
|
|
|
break;
|
|
|
|
|
factor <<= 1;
|
|
|
|
|
(*shift)++;
|
|
|
|
|
}
|
|
|
|
|
}
|
|
|
|
|
|
2013-06-02 13:55:05 +00:00
|
|
|
void psched_ratecfg_precompute(struct psched_ratecfg *r,
|
2013-09-19 16:10:03 +00:00
|
|
|
const struct tc_ratespec *conf,
|
|
|
|
|
u64 rate64)
|
2013-02-12 00:12:03 +00:00
|
|
|
{
|
2013-06-02 13:55:05 +00:00
|
|
|
memset(r, 0, sizeof(*r));
|
|
|
|
|
r->overhead = conf->overhead;
|
net_sched: restore "mpu xxx" handling
commit 56b765b79e9a ("htb: improved accuracy at high rates") broke
"overhead X", "linklayer atm" and "mpu X" attributes.
"overhead X" and "linklayer atm" have already been fixed. This restores
the "mpu X" handling, as might be used by DOCSIS or Ethernet shaping:
tc class add ... htb rate X overhead 4 mpu 64
The code being fixed is used by htb, tbf and act_police. Cake has its
own mpu handling. qdisc_calculate_pkt_len still uses the size table
containing values adjusted for mpu by user space.
iproute2 tc has always passed mpu into the kernel via a tc_ratespec
structure, but the kernel never directly acted on it, merely stored it
so that it could be read back by `tc class show`.
Rather, tc would generate length-to-time tables that included the mpu
(and linklayer) in their construction, and the kernel used those tables.
Since v3.7, the tables were no longer used. Along with "mpu", this also
broke "overhead" and "linklayer" which were fixed in 01cb71d2d47b
("net_sched: restore "overhead xxx" handling", v3.10) and 8a8e3d84b171
("net_sched: restore "linklayer atm" handling", v3.11).
"overhead" was fixed by simply restoring use of tc_ratespec::overhead -
this had originally been used by the kernel but was initially omitted
from the new non-table-based calculations.
"linklayer" had been handled in the table like "mpu", but the mode was
not originally passed in tc_ratespec. The new implementation was made to
handle it by getting new versions of tc to pass the mode in an extended
tc_ratespec, and for older versions of tc the table contents were analysed
at load time to deduce linklayer.
As "mpu" has always been given to the kernel in tc_ratespec,
accompanying the mpu-based table, we can restore system functionality
with no userspace change by making the kernel act on the tc_ratespec
value.
Fixes: 56b765b79e9a ("htb: improved accuracy at high rates")
Signed-off-by: Kevin Bracey <kevin@bracey.fi>
Cc: Eric Dumazet <edumazet@google.com>
Cc: Jiri Pirko <jiri@resnulli.us>
Cc: Vimalkumar <j.vimal@gmail.com>
Link: https://lore.kernel.org/r/20220112170210.1014351-1-kevin@bracey.fi
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
2022-01-12 17:02:10 +00:00
|
|
|
r->mpu = conf->mpu;
|
2013-09-19 16:10:03 +00:00
|
|
|
r->rate_bytes_ps = max_t(u64, conf->rate, rate64);
|
2013-08-14 21:47:11 +00:00
|
|
|
r->linklayer = (conf->linklayer & TC_LINKLAYER_MASK);
|
2021-03-12 14:08:31 +00:00
|
|
|
psched_ratecfg_precompute__(r->rate_bytes_ps, &r->mult, &r->shift);
|
2013-02-12 00:12:03 +00:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(psched_ratecfg_precompute);
|
2017-11-03 10:46:25 +00:00
|
|
|
|
2021-03-12 14:08:31 +00:00
|
|
|
void psched_ppscfg_precompute(struct psched_pktrate *r, u64 pktrate64)
|
|
|
|
|
{
|
|
|
|
|
r->rate_pkts_ps = pktrate64;
|
|
|
|
|
psched_ratecfg_precompute__(r->rate_pkts_ps, &r->mult, &r->shift);
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(psched_ppscfg_precompute);
|
|
|
|
|
|
2017-11-03 10:46:25 +00:00
|
|
|
void mini_qdisc_pair_swap(struct mini_Qdisc_pair *miniqp,
|
|
|
|
|
struct tcf_proto *tp_head)
|
|
|
|
|
{
|
2019-02-11 08:55:38 +00:00
|
|
|
/* Protected with chain0->filter_chain_lock.
|
|
|
|
|
* Can't access chain directly because tp_head can be NULL.
|
|
|
|
|
*/
|
|
|
|
|
struct mini_Qdisc *miniq_old =
|
|
|
|
|
rcu_dereference_protected(*miniqp->p_miniq, 1);
|
2017-11-03 10:46:25 +00:00
|
|
|
struct mini_Qdisc *miniq;
|
|
|
|
|
|
|
|
|
|
if (!tp_head) {
|
|
|
|
|
RCU_INIT_POINTER(*miniqp->p_miniq, NULL);
|
2021-10-26 13:06:59 +00:00
|
|
|
} else {
|
2021-10-26 18:37:21 +00:00
|
|
|
miniq = miniq_old != &miniqp->miniq1 ?
|
2021-10-26 13:06:59 +00:00
|
|
|
&miniqp->miniq1 : &miniqp->miniq2;
|
2017-11-03 10:46:25 +00:00
|
|
|
|
2021-10-26 13:06:59 +00:00
|
|
|
/* We need to make sure that readers won't see the miniq
|
|
|
|
|
* we are about to modify. So ensure that at least one RCU
|
|
|
|
|
* grace period has elapsed since the miniq was made
|
|
|
|
|
* inactive.
|
|
|
|
|
*/
|
|
|
|
|
if (IS_ENABLED(CONFIG_PREEMPT_RT))
|
|
|
|
|
cond_synchronize_rcu(miniq->rcu_state);
|
|
|
|
|
else if (!poll_state_synchronize_rcu(miniq->rcu_state))
|
|
|
|
|
synchronize_rcu_expedited();
|
2017-11-03 10:46:25 +00:00
|
|
|
|
2021-10-26 13:06:59 +00:00
|
|
|
miniq->filter_list = tp_head;
|
|
|
|
|
rcu_assign_pointer(*miniqp->p_miniq, miniq);
|
|
|
|
|
}
|
2017-11-03 10:46:25 +00:00
|
|
|
|
|
|
|
|
if (miniq_old)
|
2021-10-26 13:06:59 +00:00
|
|
|
/* This is counterpart of the rcu sync above. We need to
|
2017-11-03 10:46:25 +00:00
|
|
|
* block potential new user of miniq_old until all readers
|
|
|
|
|
* are not seeing it.
|
|
|
|
|
*/
|
2021-10-26 13:06:59 +00:00
|
|
|
miniq_old->rcu_state = start_poll_synchronize_rcu();
|
2017-11-03 10:46:25 +00:00
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(mini_qdisc_pair_swap);
|
|
|
|
|
|
2020-02-16 10:01:22 +00:00
|
|
|
void mini_qdisc_pair_block_init(struct mini_Qdisc_pair *miniqp,
|
|
|
|
|
struct tcf_block *block)
|
|
|
|
|
{
|
|
|
|
|
miniqp->miniq1.block = block;
|
|
|
|
|
miniqp->miniq2.block = block;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(mini_qdisc_pair_block_init);
|
|
|
|
|
|
2017-11-03 10:46:25 +00:00
|
|
|
void mini_qdisc_pair_init(struct mini_Qdisc_pair *miniqp, struct Qdisc *qdisc,
|
|
|
|
|
struct mini_Qdisc __rcu **p_miniq)
|
|
|
|
|
{
|
|
|
|
|
miniqp->miniq1.cpu_bstats = qdisc->cpu_bstats;
|
|
|
|
|
miniqp->miniq1.cpu_qstats = qdisc->cpu_qstats;
|
|
|
|
|
miniqp->miniq2.cpu_bstats = qdisc->cpu_bstats;
|
|
|
|
|
miniqp->miniq2.cpu_qstats = qdisc->cpu_qstats;
|
2021-10-26 13:06:59 +00:00
|
|
|
miniqp->miniq1.rcu_state = get_state_synchronize_rcu();
|
|
|
|
|
miniqp->miniq2.rcu_state = miniqp->miniq1.rcu_state;
|
2017-11-03 10:46:25 +00:00
|
|
|
miniqp->p_miniq = p_miniq;
|
|
|
|
|
}
|
|
|
|
|
EXPORT_SYMBOL(mini_qdisc_pair_init);
|