selftests: forwarding: sch_ets: Add test coverage for ETS Qdisc

This tests the newly-added ETS Qdisc. It runs two to three streams of
traffic, each with a different priority. ETS Qdisc is supposed to allocate
bandwidth according to the DRR algorithm and given weights. After running
the traffic for a while, counters are compared for each stream to check
that the expected ratio is in fact observed.

In order for the DRR process to kick in, a traffic bottleneck must exist in
the first place. In slow path, such bottleneck can be implemented by
wrapping the ETS Qdisc inside a TBF or other shaper. This might however
make the configuration unoffloadable. Instead, on HW datapath, the
bottleneck would be set up by lowering port speed and configuring shared
buffer suitably.

Therefore the test is structured as a core component that implements the
testing, with two wrapper scripts that implement the details of slow path
resp. fast path configuration.

Signed-off-by: Petr Machata <petrm@mellanox.com>
Reviewed-by: Ido Schimmel <idosch@mellanox.com>
Acked-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Petr Machata 2019-12-18 14:55:22 +00:00 committed by David S. Miller
parent 4cf9b8f992
commit ddd3fd750f
5 changed files with 666 additions and 0 deletions

View File

@ -78,3 +78,31 @@ measure_rate()
echo $ir $er
return $ret
}
bail_on_lldpad()
{
if systemctl is-active --quiet lldpad; then
cat >/dev/stderr <<-EOF
WARNING: lldpad is running
lldpad will likely configure DCB, and this test will
configure Qdiscs. mlxsw does not support both at the
same time, one of them is arbitrarily going to overwrite
the other. That will cause spurious failures (or,
unlikely, passes) of this test.
EOF
if [[ -z $ALLOW_LLDPAD ]]; then
cat >/dev/stderr <<-EOF
If you want to run the test anyway, please set
an environment variable ALLOW_LLDPAD to a
non-empty string.
EOF
exit 1
else
return
fi
fi
}

View File

@ -0,0 +1,67 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
# A driver for the ETS selftest that implements testing in offloaded datapath.
lib_dir=$(dirname $0)/../../../net/forwarding
source $lib_dir/sch_ets_core.sh
source $lib_dir/devlink_lib.sh
source qos_lib.sh
ALL_TESTS="
ping_ipv4
priomap_mode
ets_test_strict
ets_test_mixed
ets_test_dwrr
"
switch_create()
{
ets_switch_create
# Create a bottleneck so that the DWRR process can kick in.
ethtool -s $h2 speed 1000 autoneg off
ethtool -s $swp2 speed 1000 autoneg off
# Set the ingress quota high and use the three egress TCs to limit the
# amount of traffic that is admitted to the shared buffers. This makes
# sure that there is always enough traffic of all types to select from
# for the DWRR process.
devlink_port_pool_th_set $swp1 0 12
devlink_tc_bind_pool_th_set $swp1 0 ingress 0 12
devlink_port_pool_th_set $swp2 4 12
devlink_tc_bind_pool_th_set $swp2 7 egress 4 5
devlink_tc_bind_pool_th_set $swp2 6 egress 4 5
devlink_tc_bind_pool_th_set $swp2 5 egress 4 5
# Note: sch_ets_core.sh uses VLAN ingress-qos-map to assign packet
# priorities at $swp1 based on their 802.1p headers. ingress-qos-map is
# not offloaded by mlxsw as of this writing, but the mapping used is
# 1:1, which is the mapping currently hard-coded by the driver.
}
switch_destroy()
{
devlink_tc_bind_pool_th_restore $swp2 5 egress
devlink_tc_bind_pool_th_restore $swp2 6 egress
devlink_tc_bind_pool_th_restore $swp2 7 egress
devlink_port_pool_th_restore $swp2 4
devlink_tc_bind_pool_th_restore $swp1 0 ingress
devlink_port_pool_th_restore $swp1 0
ethtool -s $swp2 autoneg on
ethtool -s $h2 autoneg on
ets_switch_destroy
}
# Callback from sch_ets_tests.sh
get_stats()
{
local band=$1; shift
ethtool_stats_get "$h2" rx_octets_prio_$band
}
bail_on_lldpad
ets_run

View File

@ -0,0 +1,44 @@
#!/bin/bash
# SPDX-License-Identifier: GPL-2.0
# A driver for the ETS selftest that implements testing in slowpath.
lib_dir=.
source sch_ets_core.sh
ALL_TESTS="
ping_ipv4
priomap_mode
ets_test_strict
ets_test_mixed
ets_test_dwrr
classifier_mode
ets_test_strict
ets_test_mixed
ets_test_dwrr
"
switch_create()
{
ets_switch_create
# Create a bottleneck so that the DWRR process can kick in.
tc qdisc add dev $swp2 root handle 1: tbf \
rate 1Gbit burst 1Mbit latency 100ms
PARENT="parent 1:"
}
switch_destroy()
{
ets_switch_destroy
tc qdisc del dev $swp2 root
}
# Callback from sch_ets_tests.sh
get_stats()
{
local stream=$1; shift
link_stats_get $h2.1$stream rx bytes
}
ets_run

View File

@ -0,0 +1,300 @@
# SPDX-License-Identifier: GPL-2.0
# This is a template for ETS Qdisc test.
#
# This test sends from H1 several traffic streams with 802.1p-tagged packets.
# The tags are used at $swp1 to prioritize the traffic. Each stream is then
# queued at a different ETS band according to the assigned priority. After
# runnig for a while, counters at H2 are consulted to determine whether the
# traffic scheduling was according to the ETS configuration.
#
# This template is supposed to be embedded by a test driver, which implements
# statistics collection, any HW-specific stuff, and prominently configures the
# system to assure that there is overcommitment at $swp2. That is necessary so
# that the ETS traffic selection algorithm kicks in and has to schedule some
# traffic at the expense of other.
#
# A driver for veth-based testing is in sch_ets.sh, an example of a driver for
# an offloaded data path is in selftests/drivers/net/mlxsw/sch_ets.sh.
#
# +---------------------------------------------------------------------+
# | H1 |
# | + $h1.10 + $h1.11 + $h1.12 |
# | | 192.0.2.1/28 | 192.0.2.17/28 | 192.0.2.33/28 |
# | | egress-qos-map | egress-qos-map | egress-qos-map |
# | | 0:0 | 0:1 | 0:2 |
# | \____________________ | ____________________/ |
# | \|/ |
# | + $h1 |
# +---------------------------|-----------------------------------------+
# |
# +---------------------------|-----------------------------------------+
# | SW + $swp1 |
# | | >1Gbps |
# | ____________________/|\____________________ |
# | / | \ |
# | +--|----------------+ +--|----------------+ +--|----------------+ |
# | | + $swp1.10 | | + $swp1.11 | | + $swp1.12 | |
# | | ingress-qos-map| | ingress-qos-map| | ingress-qos-map| |
# | | 0:0 1:1 2:2 | | 0:0 1:1 2:2 | | 0:0 1:1 2:2 | |
# | | | | | | | |
# | | BR10 | | BR11 | | BR12 | |
# | | | | | | | |
# | | + $swp2.10 | | + $swp2.11 | | + $swp2.12 | |
# | +--|----------------+ +--|----------------+ +--|----------------+ |
# | \____________________ | ____________________/ |
# | \|/ |
# | + $swp2 |
# | | 1Gbps (ethtool or HTB qdisc) |
# | | qdisc ets quanta $W0 $W1 $W2 |
# | | priomap 0 1 2 |
# +---------------------------|-----------------------------------------+
# |
# +---------------------------|-----------------------------------------+
# | H2 + $h2 |
# | ____________________/|\____________________ |
# | / | \ |
# | + $h2.10 + $h2.11 + $h2.12 |
# | 192.0.2.2/28 192.0.2.18/28 192.0.2.34/28 |
# +---------------------------------------------------------------------+
NUM_NETIFS=4
CHECK_TC=yes
source $lib_dir/lib.sh
source $lib_dir/sch_ets_tests.sh
PARENT=root
QDISC_DEV=
sip()
{
echo 192.0.2.$((16 * $1 + 1))
}
dip()
{
echo 192.0.2.$((16 * $1 + 2))
}
# Callback from sch_ets_tests.sh
ets_start_traffic()
{
local dst_mac=$(mac_get $h2)
local i=$1; shift
start_traffic $h1.1$i $(sip $i) $(dip $i) $dst_mac
}
ETS_CHANGE_QDISC=
priomap_mode()
{
echo "Running in priomap mode"
ets_delete_qdisc
ETS_CHANGE_QDISC=ets_change_qdisc_priomap
}
classifier_mode()
{
echo "Running in classifier mode"
ets_delete_qdisc
ETS_CHANGE_QDISC=ets_change_qdisc_classifier
}
ets_change_qdisc_priomap()
{
local dev=$1; shift
local nstrict=$1; shift
local priomap=$1; shift
local quanta=("${@}")
local op=$(if [[ -n $QDISC_DEV ]]; then echo change; else echo add; fi)
tc qdisc $op dev $dev $PARENT handle 10: ets \
$(if ((nstrict)); then echo strict $nstrict; fi) \
$(if ((${#quanta[@]})); then echo quanta ${quanta[@]}; fi) \
priomap $priomap
QDISC_DEV=$dev
}
ets_change_qdisc_classifier()
{
local dev=$1; shift
local nstrict=$1; shift
local priomap=$1; shift
local quanta=("${@}")
local op=$(if [[ -n $QDISC_DEV ]]; then echo change; else echo add; fi)
tc qdisc $op dev $dev $PARENT handle 10: ets \
$(if ((nstrict)); then echo strict $nstrict; fi) \
$(if ((${#quanta[@]})); then echo quanta ${quanta[@]}; fi)
if [[ $op == add ]]; then
local prio=0
local band
for band in $priomap; do
tc filter add dev $dev parent 10: basic \
match "meta(priority eq $prio)" \
flowid 10:$((band + 1))
((prio++))
done
fi
QDISC_DEV=$dev
}
# Callback from sch_ets_tests.sh
ets_change_qdisc()
{
if [[ -z "$ETS_CHANGE_QDISC" ]]; then
exit 1
fi
$ETS_CHANGE_QDISC "$@"
}
ets_delete_qdisc()
{
if [[ -n $QDISC_DEV ]]; then
tc qdisc del dev $QDISC_DEV $PARENT
QDISC_DEV=
fi
}
h1_create()
{
local i;
simple_if_init $h1
mtu_set $h1 9900
for i in {0..2}; do
vlan_create $h1 1$i v$h1 $(sip $i)/28
ip link set dev $h1.1$i type vlan egress 0:$i
done
}
h1_destroy()
{
local i
for i in {0..2}; do
vlan_destroy $h1 1$i
done
mtu_restore $h1
simple_if_fini $h1
}
h2_create()
{
local i
simple_if_init $h2
mtu_set $h2 9900
for i in {0..2}; do
vlan_create $h2 1$i v$h2 $(dip $i)/28
done
}
h2_destroy()
{
local i
for i in {0..2}; do
vlan_destroy $h2 1$i
done
mtu_restore $h2
simple_if_fini $h2
}
ets_switch_create()
{
local i
ip link set dev $swp1 up
mtu_set $swp1 9900
ip link set dev $swp2 up
mtu_set $swp2 9900
for i in {0..2}; do
vlan_create $swp1 1$i
ip link set dev $swp1.1$i type vlan ingress 0:0 1:1 2:2
vlan_create $swp2 1$i
ip link add dev br1$i type bridge
ip link set dev $swp1.1$i master br1$i
ip link set dev $swp2.1$i master br1$i
ip link set dev br1$i up
ip link set dev $swp1.1$i up
ip link set dev $swp2.1$i up
done
}
ets_switch_destroy()
{
local i
ets_delete_qdisc
for i in {0..2}; do
ip link del dev br1$i
vlan_destroy $swp2 1$i
vlan_destroy $swp1 1$i
done
mtu_restore $swp2
ip link set dev $swp2 down
mtu_restore $swp1
ip link set dev $swp1 down
}
setup_prepare()
{
h1=${NETIFS[p1]}
swp1=${NETIFS[p2]}
swp2=${NETIFS[p3]}
h2=${NETIFS[p4]}
put=$swp2
hut=$h2
vrf_prepare
h1_create
h2_create
switch_create
}
cleanup()
{
pre_cleanup
switch_destroy
h2_destroy
h1_destroy
vrf_cleanup
}
ping_ipv4()
{
ping_test $h1.10 $(dip 0) " vlan 10"
ping_test $h1.11 $(dip 1) " vlan 11"
ping_test $h1.12 $(dip 2) " vlan 12"
}
ets_run()
{
trap cleanup EXIT
setup_prepare
setup_wait
tests_run
exit $EXIT_STATUS
}

View File

@ -0,0 +1,227 @@
# SPDX-License-Identifier: GPL-2.0
# Global interface:
# $put -- port under test (e.g. $swp2)
# get_stats($band) -- A function to collect stats for band
# ets_start_traffic($band) -- Start traffic for this band
# ets_change_qdisc($op, $dev, $nstrict, $quanta...) -- Add or change qdisc
# WS describes the Qdisc configuration. It has one value per band (so the
# number of array elements indicates the number of bands). If the value is
# 0, it is a strict band, otherwise the it's a DRR band and the value is
# that band's quantum.
declare -a WS
qdisc_describe()
{
local nbands=${#WS[@]}
local nstrict=0
local i
for ((i = 0; i < nbands; i++)); do
if ((!${WS[$i]})); then
: $((nstrict++))
fi
done
echo -n "ets bands $nbands"
if ((nstrict)); then
echo -n " strict $nstrict"
fi
if ((nstrict < nbands)); then
echo -n " quanta"
for ((i = nstrict; i < nbands; i++)); do
echo -n " ${WS[$i]}"
done
fi
}
__strict_eval()
{
local desc=$1; shift
local d=$1; shift
local total=$1; shift
local above=$1; shift
RET=0
if ((! total)); then
check_err 1 "No traffic observed"
log_test "$desc"
return
fi
local ratio=$(echo "scale=2; 100 * $d / $total" | bc -l)
if ((above)); then
test $(echo "$ratio > 95.0" | bc -l) -eq 1
check_err $? "Not enough traffic"
log_test "$desc"
log_info "Expected ratio >95% Measured ratio $ratio"
else
test $(echo "$ratio < 5" | bc -l) -eq 1
check_err $? "Too much traffic"
log_test "$desc"
log_info "Expected ratio <5% Measured ratio $ratio"
fi
}
strict_eval()
{
__strict_eval "$@" 1
}
notraf_eval()
{
__strict_eval "$@" 0
}
__ets_dwrr_test()
{
local -a streams=("$@")
local low_stream=${streams[0]}
local seen_strict=0
local -a t0 t1 d
local stream
local total
local i
echo "Testing $(qdisc_describe), streams ${streams[@]}"
for stream in ${streams[@]}; do
ets_start_traffic $stream
done
sleep 10
t0=($(for stream in ${streams[@]}; do
get_stats $stream
done))
sleep 10
t1=($(for stream in ${streams[@]}; do
get_stats $stream
done))
d=($(for ((i = 0; i < ${#streams[@]}; i++)); do
echo $((${t1[$i]} - ${t0[$i]}))
done))
total=$(echo ${d[@]} | sed 's/ /+/g' | bc)
for ((i = 0; i < ${#streams[@]}; i++)); do
local stream=${streams[$i]}
if ((seen_strict)); then
notraf_eval "band $stream" ${d[$i]} $total
elif ((${WS[$stream]} == 0)); then
strict_eval "band $stream" ${d[$i]} $total
seen_strict=1
elif ((stream == low_stream)); then
# Low stream is used as DWRR evaluation reference.
continue
else
multipath_eval "bands $low_stream:$stream" \
${WS[$low_stream]} ${WS[$stream]} \
${d[0]} ${d[$i]}
fi
done
for stream in ${streams[@]}; do
stop_traffic
done
}
ets_dwrr_test_012()
{
__ets_dwrr_test 0 1 2
}
ets_dwrr_test_01()
{
__ets_dwrr_test 0 1
}
ets_dwrr_test_12()
{
__ets_dwrr_test 1 2
}
ets_qdisc_setup()
{
local dev=$1; shift
local nstrict=$1; shift
local -a quanta=("$@")
local ndwrr=${#quanta[@]}
local nbands=$((nstrict + ndwrr))
local nstreams=$(if ((nbands > 3)); then echo 3; else echo $nbands; fi)
local priomap=$(seq 0 $((nstreams - 1)))
local i
WS=($(
for ((i = 0; i < nstrict; i++)); do
echo 0
done
for ((i = 0; i < ndwrr; i++)); do
echo ${quanta[$i]}
done
))
ets_change_qdisc $dev $nstrict "$priomap" ${quanta[@]}
}
ets_set_dwrr_uniform()
{
ets_qdisc_setup $put 0 3300 3300 3300
}
ets_set_dwrr_varying()
{
ets_qdisc_setup $put 0 5000 3500 1500
}
ets_set_strict()
{
ets_qdisc_setup $put 3
}
ets_set_mixed()
{
ets_qdisc_setup $put 1 5000 2500 1500
}
ets_change_quantum()
{
tc class change dev $put classid 10:2 ets quantum 8000
WS[1]=8000
}
ets_set_dwrr_two_bands()
{
ets_qdisc_setup $put 0 5000 2500
}
ets_test_strict()
{
ets_set_strict
ets_dwrr_test_01
ets_dwrr_test_12
}
ets_test_mixed()
{
ets_set_mixed
ets_dwrr_test_01
ets_dwrr_test_12
}
ets_test_dwrr()
{
ets_set_dwrr_uniform
ets_dwrr_test_012
ets_set_dwrr_varying
ets_dwrr_test_012
ets_change_quantum
ets_dwrr_test_012
ets_set_dwrr_two_bands
ets_dwrr_test_01
}