2019-05-27 06:55:01 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* INET An implementation of the TCP/IP protocol suite for the LINUX
|
|
|
|
* operating system. INET is implemented using the BSD Socket
|
|
|
|
* interface as the means of communication with the user level.
|
|
|
|
*
|
|
|
|
* ROUTE - implementation of the IP router.
|
|
|
|
*
|
2005-05-05 23:16:16 +00:00
|
|
|
* Authors: Ross Biro
|
2005-04-16 22:20:36 +00:00
|
|
|
* Fred N. van Kempen, <waltje@uWalt.NL.Mugnet.ORG>
|
|
|
|
* Alan Cox, <gw4pts@gw4pts.ampr.org>
|
|
|
|
* Linus Torvalds, <Linus.Torvalds@helsinki.fi>
|
|
|
|
* Alexey Kuznetsov, <kuznet@ms2.inr.ac.ru>
|
|
|
|
*
|
|
|
|
* Fixes:
|
|
|
|
* Alan Cox : Verify area fixes.
|
|
|
|
* Alan Cox : cli() protects routing changes
|
|
|
|
* Rui Oliveira : ICMP routing table updates
|
|
|
|
* (rco@di.uminho.pt) Routing table insertion and update
|
|
|
|
* Linus Torvalds : Rewrote bits to be sensible
|
|
|
|
* Alan Cox : Added BSD route gw semantics
|
2007-02-09 14:24:47 +00:00
|
|
|
* Alan Cox : Super /proc >4K
|
2005-04-16 22:20:36 +00:00
|
|
|
* Alan Cox : MTU in route table
|
2021-03-10 21:13:43 +00:00
|
|
|
* Alan Cox : MSS actually. Also added the window
|
2005-04-16 22:20:36 +00:00
|
|
|
* clamper.
|
|
|
|
* Sam Lantinga : Fixed route matching in rt_del()
|
|
|
|
* Alan Cox : Routing cache support.
|
|
|
|
* Alan Cox : Removed compatibility cruft.
|
|
|
|
* Alan Cox : RTF_REJECT support.
|
|
|
|
* Alan Cox : TCP irtt support.
|
|
|
|
* Jonathan Naylor : Added Metric support.
|
|
|
|
* Miquel van Smoorenburg : BSD API fixes.
|
|
|
|
* Miquel van Smoorenburg : Metrics.
|
|
|
|
* Alan Cox : Use __u32 properly
|
|
|
|
* Alan Cox : Aligned routing errors more closely with BSD
|
|
|
|
* our system is still very different.
|
|
|
|
* Alan Cox : Faster /proc handling
|
|
|
|
* Alexey Kuznetsov : Massive rework to support tree based routing,
|
|
|
|
* routing caches and better behaviour.
|
2007-02-09 14:24:47 +00:00
|
|
|
*
|
2005-04-16 22:20:36 +00:00
|
|
|
* Olaf Erb : irtt wasn't being copied right.
|
|
|
|
* Bjorn Ekwall : Kerneld route support.
|
|
|
|
* Alan Cox : Multicast fixed (I hope)
|
2021-03-10 21:13:43 +00:00
|
|
|
* Pavel Krauz : Limited broadcast fixed
|
2005-04-16 22:20:36 +00:00
|
|
|
* Mike McLagan : Routing by source
|
|
|
|
* Alexey Kuznetsov : End of old history. Split to fib.c and
|
|
|
|
* route.c and rewritten from scratch.
|
|
|
|
* Andi Kleen : Load-limit warning messages.
|
|
|
|
* Vitaly E. Lavrov : Transparent proxy revived after year coma.
|
|
|
|
* Vitaly E. Lavrov : Race condition in ip_route_input_slow.
|
|
|
|
* Tobias Ringstrom : Uninitialized res.type in ip_route_output_slow.
|
|
|
|
* Vladimir V. Ivanov : IP rule info (flowid) is really useful.
|
|
|
|
* Marc Boucher : routing by fwmark
|
|
|
|
* Robert Olsson : Added rt_cache statistics
|
|
|
|
* Arnaldo C. Melo : Convert proc stuff to seq_file
|
2005-07-05 22:00:32 +00:00
|
|
|
* Eric Dumazet : hashed spinlocks and rt_check_expire() fixes.
|
2021-03-10 21:13:43 +00:00
|
|
|
* Ilia Sotnikov : Ignore TOS on PMTUD and Redirect
|
|
|
|
* Ilia Sotnikov : Removed TOS from hash calculations
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2012-03-12 07:03:32 +00:00
|
|
|
#define pr_fmt(fmt) "IPv4: " fmt
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/module.h>
|
2016-12-24 19:46:01 +00:00
|
|
|
#include <linux/uaccess.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/bitops.h>
|
|
|
|
#include <linux/types.h>
|
|
|
|
#include <linux/kernel.h>
|
|
|
|
#include <linux/mm.h>
|
2021-03-24 21:53:37 +00:00
|
|
|
#include <linux/memblock.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <linux/string.h>
|
|
|
|
#include <linux/socket.h>
|
|
|
|
#include <linux/sockios.h>
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/in.h>
|
|
|
|
#include <linux/inet.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/proc_fs.h>
|
|
|
|
#include <linux/init.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/inetdevice.h>
|
|
|
|
#include <linux/igmp.h>
|
|
|
|
#include <linux/pkt_sched.h>
|
|
|
|
#include <linux/mroute.h>
|
|
|
|
#include <linux/netfilter_ipv4.h>
|
|
|
|
#include <linux/random.h>
|
|
|
|
#include <linux/rcupdate.h>
|
|
|
|
#include <linux/times.h>
|
include cleanup: Update gfp.h and slab.h includes to prepare for breaking implicit slab.h inclusion from percpu.h
percpu.h is included by sched.h and module.h and thus ends up being
included when building most .c files. percpu.h includes slab.h which
in turn includes gfp.h making everything defined by the two files
universally available and complicating inclusion dependencies.
percpu.h -> slab.h dependency is about to be removed. Prepare for
this change by updating users of gfp and slab facilities include those
headers directly instead of assuming availability. As this conversion
needs to touch large number of source files, the following script is
used as the basis of conversion.
http://userweb.kernel.org/~tj/misc/slabh-sweep.py
The script does the followings.
* Scan files for gfp and slab usages and update includes such that
only the necessary includes are there. ie. if only gfp is used,
gfp.h, if slab is used, slab.h.
* When the script inserts a new include, it looks at the include
blocks and try to put the new include such that its order conforms
to its surrounding. It's put in the include block which contains
core kernel includes, in the same order that the rest are ordered -
alphabetical, Christmas tree, rev-Xmas-tree or at the end if there
doesn't seem to be any matching order.
* If the script can't find a place to put a new include (mostly
because the file doesn't have fitting include block), it prints out
an error message indicating which .h file needs to be added to the
file.
The conversion was done in the following steps.
1. The initial automatic conversion of all .c files updated slightly
over 4000 files, deleting around 700 includes and adding ~480 gfp.h
and ~3000 slab.h inclusions. The script emitted errors for ~400
files.
2. Each error was manually checked. Some didn't need the inclusion,
some needed manual addition while adding it to implementation .h or
embedding .c file was more appropriate for others. This step added
inclusions to around 150 files.
3. The script was run again and the output was compared to the edits
from #2 to make sure no file was left behind.
4. Several build tests were done and a couple of problems were fixed.
e.g. lib/decompress_*.c used malloc/free() wrappers around slab
APIs requiring slab.h to be added manually.
5. The script was run on all .h files but without automatically
editing them as sprinkling gfp.h and slab.h inclusions around .h
files could easily lead to inclusion dependency hell. Most gfp.h
inclusion directives were ignored as stuff from gfp.h was usually
wildly available and often used in preprocessor macros. Each
slab.h inclusion directive was examined and added manually as
necessary.
6. percpu.h was updated not to include slab.h.
7. Build test were done on the following configurations and failures
were fixed. CONFIG_GCOV_KERNEL was turned off for all tests (as my
distributed build env didn't work with gcov compiles) and a few
more options had to be turned off depending on archs to make things
build (like ipr on powerpc/64 which failed due to missing writeq).
* x86 and x86_64 UP and SMP allmodconfig and a custom test config.
* powerpc and powerpc64 SMP allmodconfig
* sparc and sparc64 SMP allmodconfig
* ia64 SMP allmodconfig
* s390 SMP allmodconfig
* alpha SMP allmodconfig
* um on x86_64 SMP allmodconfig
8. percpu.h modifications were reverted so that it could be applied as
a separate patch and serve as bisection point.
Given the fact that I had only a couple of failures from tests on step
6, I'm fairly confident about the coverage of this conversion patch.
If there is a breakage, it's likely to be something in one of the arch
headers which should be easily discoverable easily on most builds of
the specific arch.
Signed-off-by: Tejun Heo <tj@kernel.org>
Guess-its-ok-by: Christoph Lameter <cl@linux-foundation.org>
Cc: Ingo Molnar <mingo@redhat.com>
Cc: Lee Schermerhorn <Lee.Schermerhorn@hp.com>
2010-03-24 08:04:11 +00:00
|
|
|
#include <linux/slab.h>
|
inetpeer: get rid of ip_id_count
Ideally, we would need to generate IP ID using a per destination IP
generator.
linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.
1) each inet_peer struct consumes 192 bytes
2) inetpeer cache uses a binary tree of inet_peer structs,
with a nominal size of ~66000 elements under load.
3) lookups in this tree are hitting a lot of cache lines, as tree depth
is about 20.
4) If server deals with many tcp flows, we have a high probability of
not finding the inet_peer, allocating a fresh one, inserting it in
the tree with same initial ip_id_count, (cf secure_ip_id())
5) We garbage collect inet_peer aggressively.
IP ID generation do not have to be 'perfect'
Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.
We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.
ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)
secure_ip_id() and secure_ipv6_id() no longer are needed.
Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-02 12:26:03 +00:00
|
|
|
#include <linux/jhash.h>
|
2007-11-14 05:34:06 +00:00
|
|
|
#include <net/dst.h>
|
2015-07-21 08:43:59 +00:00
|
|
|
#include <net/dst_metadata.h>
|
2007-09-12 10:01:34 +00:00
|
|
|
#include <net/net_namespace.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/protocol.h>
|
|
|
|
#include <net/ip.h>
|
|
|
|
#include <net/route.h>
|
|
|
|
#include <net/inetpeer.h>
|
|
|
|
#include <net/sock.h>
|
|
|
|
#include <net/ip_fib.h>
|
2019-06-04 03:19:49 +00:00
|
|
|
#include <net/nexthop.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#include <net/arp.h>
|
|
|
|
#include <net/tcp.h>
|
|
|
|
#include <net/icmp.h>
|
|
|
|
#include <net/xfrm.h>
|
2015-07-21 08:43:47 +00:00
|
|
|
#include <net/lwtunnel.h>
|
2006-07-31 03:43:36 +00:00
|
|
|
#include <net/netevent.h>
|
2007-03-22 18:55:17 +00:00
|
|
|
#include <net/rtnetlink.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_SYSCTL
|
|
|
|
#include <linux/sysctl.h>
|
|
|
|
#endif
|
2011-08-04 03:50:44 +00:00
|
|
|
#include <net/secure_seq.h>
|
2015-07-21 08:43:59 +00:00
|
|
|
#include <net/ip_tunnels.h>
|
2015-09-30 03:07:13 +00:00
|
|
|
#include <net/l3mdev.h>
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-05-25 17:42:39 +00:00
|
|
|
#include "fib_lookup.h"
|
|
|
|
|
2011-03-12 01:07:33 +00:00
|
|
|
#define RT_FL_TOS(oldflp4) \
|
2011-12-02 11:39:42 +00:00
|
|
|
((oldflp4)->flowi4_tos & (IPTOS_RT_MASK | RTO_ONLINK))
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#define RT_GC_TIMEOUT (300*HZ)
|
|
|
|
|
|
|
|
static int ip_rt_max_size;
|
2008-03-23 00:43:59 +00:00
|
|
|
static int ip_rt_redirect_number __read_mostly = 9;
|
|
|
|
static int ip_rt_redirect_load __read_mostly = HZ / 50;
|
|
|
|
static int ip_rt_redirect_silence __read_mostly = ((HZ / 50) << (9 + 1));
|
|
|
|
static int ip_rt_error_cost __read_mostly = HZ;
|
|
|
|
static int ip_rt_error_burst __read_mostly = 5 * HZ;
|
|
|
|
static int ip_rt_mtu_expires __read_mostly = 10 * 60 * HZ;
|
2018-02-26 15:13:43 +00:00
|
|
|
static u32 ip_rt_min_pmtu __read_mostly = 512 + 20 + 20;
|
2008-03-23 00:43:59 +00:00
|
|
|
static int ip_rt_min_advmss __read_mostly = 256;
|
2011-12-21 20:47:16 +00:00
|
|
|
|
2016-02-18 13:21:19 +00:00
|
|
|
static int ip_rt_gc_timeout __read_mostly = RT_GC_TIMEOUT;
|
2018-02-26 15:13:43 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Interface to generic destination cache.
|
|
|
|
*/
|
|
|
|
|
2021-02-01 17:41:32 +00:00
|
|
|
INDIRECT_CALLABLE_SCOPE
|
|
|
|
struct dst_entry *ipv4_dst_check(struct dst_entry *dst, u32 cookie);
|
2010-12-13 20:52:14 +00:00
|
|
|
static unsigned int ipv4_default_advmss(const struct dst_entry *dst);
|
2021-02-01 17:41:31 +00:00
|
|
|
INDIRECT_CALLABLE_SCOPE
|
|
|
|
unsigned int ipv4_mtu(const struct dst_entry *dst);
|
2005-04-16 22:20:36 +00:00
|
|
|
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst);
|
|
|
|
static void ipv4_link_failure(struct sk_buff *skb);
|
2012-07-17 10:29:28 +00:00
|
|
|
static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
|
2019-12-22 02:51:09 +00:00
|
|
|
struct sk_buff *skb, u32 mtu,
|
|
|
|
bool confirm_neigh);
|
2012-07-17 10:29:28 +00:00
|
|
|
static void ip_do_redirect(struct dst_entry *dst, struct sock *sk,
|
|
|
|
struct sk_buff *skb);
|
2012-07-31 22:06:50 +00:00
|
|
|
static void ipv4_dst_destroy(struct dst_entry *dst);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
net: Implement read-only protection and COW'ing of metrics.
Routing metrics are now copy-on-write.
Initially a route entry points it's metrics at a read-only location.
If a routing table entry exists, it will point there. Else it will
point at the all zero metric place-holder called 'dst_default_metrics'.
The writeability state of the metrics is stored in the low bits of the
metrics pointer, we have two bits left to spare if we want to store
more states.
For the initial implementation, COW is implemented simply via kmalloc.
However future enhancements will change this to place the writable
metrics somewhere else, in order to increase sharing. Very likely
this "somewhere else" will be the inetpeer cache.
Note also that this means that metrics updates may transiently fail
if we cannot COW the metrics successfully.
But even by itself, this patch should decrease memory usage and
increase cache locality especially for routing workloads. In those
cases the read-only metric copies stay in place and never get written
to.
TCP workloads where metrics get updated, and those rare cases where
PMTU triggers occur, will take a very slight performance hit. But
that hit will be alleviated when the long-term writable metrics
move to a more sharable location.
Since the metrics storage went from a u32 array of RTAX_MAX entries to
what is essentially a pointer, some retooling of the dst_entry layout
was necessary.
Most importantly, we need to preserve the alignment of the reference
count so that it doesn't share cache lines with the read-mostly state,
as per Eric Dumazet's alignment assertion checks.
The only non-trivial bit here is the move of the 'flags' member into
the writeable cacheline. This is OK since we are always accessing the
flags around the same moment when we made a modification to the
reference count.
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-01-27 04:51:05 +00:00
|
|
|
static u32 *ipv4_cow_metrics(struct dst_entry *dst, unsigned long old)
|
|
|
|
{
|
2012-07-10 14:08:18 +00:00
|
|
|
WARN_ON(1);
|
|
|
|
return NULL;
|
net: Implement read-only protection and COW'ing of metrics.
Routing metrics are now copy-on-write.
Initially a route entry points it's metrics at a read-only location.
If a routing table entry exists, it will point there. Else it will
point at the all zero metric place-holder called 'dst_default_metrics'.
The writeability state of the metrics is stored in the low bits of the
metrics pointer, we have two bits left to spare if we want to store
more states.
For the initial implementation, COW is implemented simply via kmalloc.
However future enhancements will change this to place the writable
metrics somewhere else, in order to increase sharing. Very likely
this "somewhere else" will be the inetpeer cache.
Note also that this means that metrics updates may transiently fail
if we cannot COW the metrics successfully.
But even by itself, this patch should decrease memory usage and
increase cache locality especially for routing workloads. In those
cases the read-only metric copies stay in place and never get written
to.
TCP workloads where metrics get updated, and those rare cases where
PMTU triggers occur, will take a very slight performance hit. But
that hit will be alleviated when the long-term writable metrics
move to a more sharable location.
Since the metrics storage went from a u32 array of RTAX_MAX entries to
what is essentially a pointer, some retooling of the dst_entry layout
was necessary.
Most importantly, we need to preserve the alignment of the reference
count so that it doesn't share cache lines with the read-mostly state,
as per Eric Dumazet's alignment assertion checks.
The only non-trivial bit here is the move of the 'flags' member into
the writeable cacheline. This is OK since we are always accessing the
flags around the same moment when we made a modification to the
reference count.
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-01-27 04:51:05 +00:00
|
|
|
}
|
|
|
|
|
2012-07-03 04:52:24 +00:00
|
|
|
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
const void *daddr);
|
2017-02-06 21:14:15 +00:00
|
|
|
static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr);
|
2011-07-18 07:40:17 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static struct dst_ops ipv4_dst_ops = {
|
|
|
|
.family = AF_INET,
|
|
|
|
.check = ipv4_dst_check,
|
2010-12-13 20:52:14 +00:00
|
|
|
.default_advmss = ipv4_default_advmss,
|
2011-11-23 02:12:51 +00:00
|
|
|
.mtu = ipv4_mtu,
|
net: Implement read-only protection and COW'ing of metrics.
Routing metrics are now copy-on-write.
Initially a route entry points it's metrics at a read-only location.
If a routing table entry exists, it will point there. Else it will
point at the all zero metric place-holder called 'dst_default_metrics'.
The writeability state of the metrics is stored in the low bits of the
metrics pointer, we have two bits left to spare if we want to store
more states.
For the initial implementation, COW is implemented simply via kmalloc.
However future enhancements will change this to place the writable
metrics somewhere else, in order to increase sharing. Very likely
this "somewhere else" will be the inetpeer cache.
Note also that this means that metrics updates may transiently fail
if we cannot COW the metrics successfully.
But even by itself, this patch should decrease memory usage and
increase cache locality especially for routing workloads. In those
cases the read-only metric copies stay in place and never get written
to.
TCP workloads where metrics get updated, and those rare cases where
PMTU triggers occur, will take a very slight performance hit. But
that hit will be alleviated when the long-term writable metrics
move to a more sharable location.
Since the metrics storage went from a u32 array of RTAX_MAX entries to
what is essentially a pointer, some retooling of the dst_entry layout
was necessary.
Most importantly, we need to preserve the alignment of the reference
count so that it doesn't share cache lines with the read-mostly state,
as per Eric Dumazet's alignment assertion checks.
The only non-trivial bit here is the move of the 'flags' member into
the writeable cacheline. This is OK since we are always accessing the
flags around the same moment when we made a modification to the
reference count.
Signed-off-by: David S. Miller <davem@davemloft.net>
2011-01-27 04:51:05 +00:00
|
|
|
.cow_metrics = ipv4_cow_metrics,
|
2012-07-31 22:06:50 +00:00
|
|
|
.destroy = ipv4_dst_destroy,
|
2005-04-16 22:20:36 +00:00
|
|
|
.negative_advice = ipv4_negative_advice,
|
|
|
|
.link_failure = ipv4_link_failure,
|
|
|
|
.update_pmtu = ip_rt_update_pmtu,
|
2012-07-12 03:55:47 +00:00
|
|
|
.redirect = ip_do_redirect,
|
2015-10-07 21:48:37 +00:00
|
|
|
.local_out = __ip_local_out,
|
2011-07-18 07:40:17 +00:00
|
|
|
.neigh_lookup = ipv4_neigh_lookup,
|
2017-02-06 21:14:15 +00:00
|
|
|
.confirm_neigh = ipv4_confirm_neigh,
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define ECN_OR_COST(class) TC_PRIO_##class
|
|
|
|
|
2007-07-09 22:32:57 +00:00
|
|
|
const __u8 ip_tos2prio[16] = {
|
2005-04-16 22:20:36 +00:00
|
|
|
TC_PRIO_BESTEFFORT,
|
2011-03-15 13:56:07 +00:00
|
|
|
ECN_OR_COST(BESTEFFORT),
|
2005-04-16 22:20:36 +00:00
|
|
|
TC_PRIO_BESTEFFORT,
|
|
|
|
ECN_OR_COST(BESTEFFORT),
|
|
|
|
TC_PRIO_BULK,
|
|
|
|
ECN_OR_COST(BULK),
|
|
|
|
TC_PRIO_BULK,
|
|
|
|
ECN_OR_COST(BULK),
|
|
|
|
TC_PRIO_INTERACTIVE,
|
|
|
|
ECN_OR_COST(INTERACTIVE),
|
|
|
|
TC_PRIO_INTERACTIVE,
|
|
|
|
ECN_OR_COST(INTERACTIVE),
|
|
|
|
TC_PRIO_INTERACTIVE_BULK,
|
|
|
|
ECN_OR_COST(INTERACTIVE_BULK),
|
|
|
|
TC_PRIO_INTERACTIVE_BULK,
|
|
|
|
ECN_OR_COST(INTERACTIVE_BULK)
|
|
|
|
};
|
2012-04-04 21:33:28 +00:00
|
|
|
EXPORT_SYMBOL(ip_tos2prio);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
[IPV4]: rt_cache_stat can be statically defined
Using __get_cpu_var(obj) is slightly faster than per_cpu_ptr(obj,
raw_smp_processor_id()).
1) Smaller code and memory use
For static and small objects, DEFINE_PER_CPU(type, object) is preferred over a
alloc_percpu() : Better and smaller code to access them, and no extra memory
(storing the pointer, and the percpu array of pointers)
x86_64 code before patch
mov 1237577(%rip),%rax # ffffffff803e5990 <rt_cache_stat>
not %rax # part of per_cpu machinery
mov %gs:0x3c,%edx # get cpu number
movslq %edx,%rdx # extend 32 bits cpu number to 64 bits
mov (%rax,%rdx,8),%rax # get the pointer for this cpu
incl 0x38(%rax)
x86_64 code after patch
mov $per_cpu__rt_cache_stat,%rdx
mov %gs:0x48,%rax # get percpu data offset
incl 0x38(%rax,%rdx,1)
2) False sharing avoidance for SMP :
For a small NR_CPUS, the array of per cpu pointers allocated in alloc_percpu()
can be <= 32 bytes. This let slab code gives a part of a cache line. If the
other part of this 64 bytes (or 128 bytes) cache line is used by a mostly
written object, we can have false sharing and expensive per_cpu_ptr() operations.
Size of rt_cache_stat is 64 bytes, so this patch is not a danger of a too big
increase of bss (in UP mode) or static per_cpu data for SMP
(PERCPU_ENOUGH_ROOM is currently 32768 bytes)
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-01-17 10:54:36 +00:00
|
|
|
static DEFINE_PER_CPU(struct rt_cache_stat, rt_cache_stat);
|
2014-04-07 22:39:40 +00:00
|
|
|
#define RT_CACHE_STAT_INC(field) raw_cpu_inc(rt_cache_stat.field)
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_PROC_FS
|
|
|
|
static void *rt_cache_seq_start(struct seq_file *seq, loff_t *pos)
|
|
|
|
{
|
[IPV4] route cache: Introduce rt_genid for smooth cache invalidation
Current ip route cache implementation is not suited to large caches.
We can consume a lot of CPU when cache must be invalidated, since we
currently need to evict all cache entries, and this eviction is
sometimes asynchronous. min_delay & max_delay can somewhat control this
asynchronism behavior, but whole thing is a kludge, regularly triggering
infamous soft lockup messages. When entries are still in use, this also
consumes a lot of ram, filling dst_garbage.list.
A better scheme is to use a generation identifier on each entry,
so that cache invalidation can be performed by changing the table
identifier, without having to scan all entries.
No more delayed flushing, no more stalling when secret_interval expires.
Invalidated entries will then be freed at GC time (controled by
ip_rt_gc_timeout or stress), or when an invalidated entry is found
in a chain when an insert is done.
Thus we keep a normal equilibrium.
This patch :
- renames rt_hash_rnd to rt_genid (and makes it an atomic_t)
- Adds a new rt_genid field to 'struct rtable' (filling a hole on 64bit)
- Checks entry->rt_genid at appropriate places :
2008-02-01 01:05:09 +00:00
|
|
|
if (*pos)
|
2012-07-17 18:00:09 +00:00
|
|
|
return NULL;
|
[IPV4] route cache: Introduce rt_genid for smooth cache invalidation
Current ip route cache implementation is not suited to large caches.
We can consume a lot of CPU when cache must be invalidated, since we
currently need to evict all cache entries, and this eviction is
sometimes asynchronous. min_delay & max_delay can somewhat control this
asynchronism behavior, but whole thing is a kludge, regularly triggering
infamous soft lockup messages. When entries are still in use, this also
consumes a lot of ram, filling dst_garbage.list.
A better scheme is to use a generation identifier on each entry,
so that cache invalidation can be performed by changing the table
identifier, without having to scan all entries.
No more delayed flushing, no more stalling when secret_interval expires.
Invalidated entries will then be freed at GC time (controled by
ip_rt_gc_timeout or stress), or when an invalidated entry is found
in a chain when an insert is done.
Thus we keep a normal equilibrium.
This patch :
- renames rt_hash_rnd to rt_genid (and makes it an atomic_t)
- Adds a new rt_genid field to 'struct rtable' (filling a hole on 64bit)
- Checks entry->rt_genid at appropriate places :
2008-02-01 01:05:09 +00:00
|
|
|
return SEQ_START_TOKEN;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void *rt_cache_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
|
|
{
|
|
|
|
++*pos;
|
2012-07-17 18:00:09 +00:00
|
|
|
return NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void rt_cache_seq_stop(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rt_cache_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
if (v == SEQ_START_TOKEN)
|
|
|
|
seq_printf(seq, "%-127s\n",
|
|
|
|
"Iface\tDestination\tGateway \tFlags\t\tRefCnt\tUse\t"
|
|
|
|
"Metric\tSource\t\tMTU\tWindow\tIRTT\tTOS\tHHRef\t"
|
|
|
|
"HHUptod\tSpecDst");
|
2007-02-09 14:24:47 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2007-03-12 21:34:29 +00:00
|
|
|
static const struct seq_operations rt_cache_seq_ops = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.start = rt_cache_seq_start,
|
|
|
|
.next = rt_cache_seq_next,
|
|
|
|
.stop = rt_cache_seq_stop,
|
|
|
|
.show = rt_cache_seq_show,
|
|
|
|
};
|
|
|
|
|
|
|
|
static void *rt_cpu_seq_start(struct seq_file *seq, loff_t *pos)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
if (*pos == 0)
|
|
|
|
return SEQ_START_TOKEN;
|
|
|
|
|
2008-12-29 12:23:42 +00:00
|
|
|
for (cpu = *pos-1; cpu < nr_cpu_ids; ++cpu) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!cpu_possible(cpu))
|
|
|
|
continue;
|
|
|
|
*pos = cpu+1;
|
[IPV4]: rt_cache_stat can be statically defined
Using __get_cpu_var(obj) is slightly faster than per_cpu_ptr(obj,
raw_smp_processor_id()).
1) Smaller code and memory use
For static and small objects, DEFINE_PER_CPU(type, object) is preferred over a
alloc_percpu() : Better and smaller code to access them, and no extra memory
(storing the pointer, and the percpu array of pointers)
x86_64 code before patch
mov 1237577(%rip),%rax # ffffffff803e5990 <rt_cache_stat>
not %rax # part of per_cpu machinery
mov %gs:0x3c,%edx # get cpu number
movslq %edx,%rdx # extend 32 bits cpu number to 64 bits
mov (%rax,%rdx,8),%rax # get the pointer for this cpu
incl 0x38(%rax)
x86_64 code after patch
mov $per_cpu__rt_cache_stat,%rdx
mov %gs:0x48,%rax # get percpu data offset
incl 0x38(%rax,%rdx,1)
2) False sharing avoidance for SMP :
For a small NR_CPUS, the array of per cpu pointers allocated in alloc_percpu()
can be <= 32 bytes. This let slab code gives a part of a cache line. If the
other part of this 64 bytes (or 128 bytes) cache line is used by a mostly
written object, we can have false sharing and expensive per_cpu_ptr() operations.
Size of rt_cache_stat is 64 bytes, so this patch is not a danger of a too big
increase of bss (in UP mode) or static per_cpu data for SMP
(PERCPU_ENOUGH_ROOM is currently 32768 bytes)
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-01-17 10:54:36 +00:00
|
|
|
return &per_cpu(rt_cache_stat, cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void *rt_cpu_seq_next(struct seq_file *seq, void *v, loff_t *pos)
|
|
|
|
{
|
|
|
|
int cpu;
|
|
|
|
|
2008-12-29 12:23:42 +00:00
|
|
|
for (cpu = *pos; cpu < nr_cpu_ids; ++cpu) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!cpu_possible(cpu))
|
|
|
|
continue;
|
|
|
|
*pos = cpu+1;
|
[IPV4]: rt_cache_stat can be statically defined
Using __get_cpu_var(obj) is slightly faster than per_cpu_ptr(obj,
raw_smp_processor_id()).
1) Smaller code and memory use
For static and small objects, DEFINE_PER_CPU(type, object) is preferred over a
alloc_percpu() : Better and smaller code to access them, and no extra memory
(storing the pointer, and the percpu array of pointers)
x86_64 code before patch
mov 1237577(%rip),%rax # ffffffff803e5990 <rt_cache_stat>
not %rax # part of per_cpu machinery
mov %gs:0x3c,%edx # get cpu number
movslq %edx,%rdx # extend 32 bits cpu number to 64 bits
mov (%rax,%rdx,8),%rax # get the pointer for this cpu
incl 0x38(%rax)
x86_64 code after patch
mov $per_cpu__rt_cache_stat,%rdx
mov %gs:0x48,%rax # get percpu data offset
incl 0x38(%rax,%rdx,1)
2) False sharing avoidance for SMP :
For a small NR_CPUS, the array of per cpu pointers allocated in alloc_percpu()
can be <= 32 bytes. This let slab code gives a part of a cache line. If the
other part of this 64 bytes (or 128 bytes) cache line is used by a mostly
written object, we can have false sharing and expensive per_cpu_ptr() operations.
Size of rt_cache_stat is 64 bytes, so this patch is not a danger of a too big
increase of bss (in UP mode) or static per_cpu data for SMP
(PERCPU_ENOUGH_ROOM is currently 32768 bytes)
Signed-off-by: Eric Dumazet <dada1@cosmosbay.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2006-01-17 10:54:36 +00:00
|
|
|
return &per_cpu(rt_cache_stat, cpu);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2020-01-23 07:11:35 +00:00
|
|
|
(*pos)++;
|
2005-04-16 22:20:36 +00:00
|
|
|
return NULL;
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void rt_cpu_seq_stop(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
|
|
|
|
}
|
|
|
|
|
|
|
|
static int rt_cpu_seq_show(struct seq_file *seq, void *v)
|
|
|
|
{
|
|
|
|
struct rt_cache_stat *st = v;
|
|
|
|
|
|
|
|
if (v == SEQ_START_TOKEN) {
|
2021-08-02 08:05:08 +00:00
|
|
|
seq_puts(seq, "entries in_hit in_slow_tot in_slow_mc in_no_route in_brd in_martian_dst in_martian_src out_hit out_slow_tot out_slow_mc gc_total gc_ignored gc_goal_miss gc_dst_overflow in_hlist_search out_hlist_search\n");
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2021-08-02 08:05:08 +00:00
|
|
|
seq_printf(seq, "%08x %08x %08x %08x %08x %08x %08x "
|
|
|
|
"%08x %08x %08x %08x %08x %08x "
|
|
|
|
"%08x %08x %08x %08x\n",
|
2010-10-08 06:37:34 +00:00
|
|
|
dst_entries_get_slow(&ipv4_dst_ops),
|
2013-10-16 09:49:04 +00:00
|
|
|
0, /* st->in_hit */
|
2005-04-16 22:20:36 +00:00
|
|
|
st->in_slow_tot,
|
|
|
|
st->in_slow_mc,
|
|
|
|
st->in_no_route,
|
|
|
|
st->in_brd,
|
|
|
|
st->in_martian_dst,
|
|
|
|
st->in_martian_src,
|
|
|
|
|
2013-10-16 09:49:04 +00:00
|
|
|
0, /* st->out_hit */
|
2005-04-16 22:20:36 +00:00
|
|
|
st->out_slow_tot,
|
2007-02-09 14:24:47 +00:00
|
|
|
st->out_slow_mc,
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2013-10-16 09:49:04 +00:00
|
|
|
0, /* st->gc_total */
|
|
|
|
0, /* st->gc_ignored */
|
|
|
|
0, /* st->gc_goal_miss */
|
|
|
|
0, /* st->gc_dst_overflow */
|
|
|
|
0, /* st->in_hlist_search */
|
|
|
|
0 /* st->out_hlist_search */
|
2005-04-16 22:20:36 +00:00
|
|
|
);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2007-03-12 21:34:29 +00:00
|
|
|
static const struct seq_operations rt_cpu_seq_ops = {
|
2005-04-16 22:20:36 +00:00
|
|
|
.start = rt_cpu_seq_start,
|
|
|
|
.next = rt_cpu_seq_next,
|
|
|
|
.stop = rt_cpu_seq_stop,
|
|
|
|
.show = rt_cpu_seq_show,
|
|
|
|
};
|
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2009-11-25 23:40:35 +00:00
|
|
|
static int rt_acct_proc_show(struct seq_file *m, void *v)
|
2007-12-06 05:13:48 +00:00
|
|
|
{
|
2009-11-25 23:40:35 +00:00
|
|
|
struct ip_rt_acct *dst, *src;
|
|
|
|
unsigned int i, j;
|
|
|
|
|
|
|
|
dst = kcalloc(256, sizeof(struct ip_rt_acct), GFP_KERNEL);
|
|
|
|
if (!dst)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
src = (struct ip_rt_acct *)per_cpu_ptr(ip_rt_acct, i);
|
|
|
|
for (j = 0; j < 256; j++) {
|
|
|
|
dst[j].o_bytes += src[j].o_bytes;
|
|
|
|
dst[j].o_packets += src[j].o_packets;
|
|
|
|
dst[j].i_bytes += src[j].i_bytes;
|
|
|
|
dst[j].i_packets += src[j].i_packets;
|
|
|
|
}
|
2007-12-06 05:13:48 +00:00
|
|
|
}
|
|
|
|
|
2009-11-25 23:40:35 +00:00
|
|
|
seq_write(m, dst, 256 * sizeof(struct ip_rt_acct));
|
|
|
|
kfree(dst);
|
|
|
|
return 0;
|
|
|
|
}
|
2007-12-06 05:13:48 +00:00
|
|
|
#endif
|
2007-12-06 05:14:28 +00:00
|
|
|
|
2008-02-29 04:51:18 +00:00
|
|
|
static int __net_init ip_rt_do_proc_init(struct net *net)
|
2007-12-06 05:14:28 +00:00
|
|
|
{
|
|
|
|
struct proc_dir_entry *pde;
|
|
|
|
|
2021-03-16 02:57:36 +00:00
|
|
|
pde = proc_create_seq("rt_cache", 0444, net->proc_net,
|
|
|
|
&rt_cache_seq_ops);
|
2007-12-06 05:14:28 +00:00
|
|
|
if (!pde)
|
|
|
|
goto err1;
|
|
|
|
|
2021-03-16 02:57:36 +00:00
|
|
|
pde = proc_create_seq("rt_cache", 0444, net->proc_net_stat,
|
|
|
|
&rt_cpu_seq_ops);
|
2007-12-06 05:14:28 +00:00
|
|
|
if (!pde)
|
|
|
|
goto err2;
|
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2018-05-15 13:57:23 +00:00
|
|
|
pde = proc_create_single("rt_acct", 0, net->proc_net,
|
|
|
|
rt_acct_proc_show);
|
2007-12-06 05:14:28 +00:00
|
|
|
if (!pde)
|
|
|
|
goto err3;
|
|
|
|
#endif
|
|
|
|
return 0;
|
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2007-12-06 05:14:28 +00:00
|
|
|
err3:
|
|
|
|
remove_proc_entry("rt_cache", net->proc_net_stat);
|
|
|
|
#endif
|
|
|
|
err2:
|
|
|
|
remove_proc_entry("rt_cache", net->proc_net);
|
|
|
|
err1:
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
2008-02-29 04:51:18 +00:00
|
|
|
|
|
|
|
static void __net_exit ip_rt_do_proc_exit(struct net *net)
|
|
|
|
{
|
|
|
|
remove_proc_entry("rt_cache", net->proc_net_stat);
|
|
|
|
remove_proc_entry("rt_cache", net->proc_net);
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2008-02-29 04:51:18 +00:00
|
|
|
remove_proc_entry("rt_acct", net->proc_net);
|
2010-01-17 03:32:50 +00:00
|
|
|
#endif
|
2008-02-29 04:51:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct pernet_operations ip_rt_proc_ops __net_initdata = {
|
|
|
|
.init = ip_rt_do_proc_init,
|
|
|
|
.exit = ip_rt_do_proc_exit,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int __init ip_rt_proc_init(void)
|
|
|
|
{
|
|
|
|
return register_pernet_subsys(&ip_rt_proc_ops);
|
|
|
|
}
|
|
|
|
|
2007-12-06 05:14:28 +00:00
|
|
|
#else
|
2008-02-29 04:51:18 +00:00
|
|
|
static inline int ip_rt_proc_init(void)
|
2007-12-06 05:14:28 +00:00
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif /* CONFIG_PROC_FS */
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2012-07-25 05:11:23 +00:00
|
|
|
static inline bool rt_is_expired(const struct rtable *rth)
|
2008-07-06 02:04:32 +00:00
|
|
|
{
|
2013-07-30 00:33:53 +00:00
|
|
|
return rth->rt_genid != rt_genid_ipv4(dev_net(rth->dst.dev));
|
2008-07-06 02:04:32 +00:00
|
|
|
}
|
|
|
|
|
2012-09-07 00:45:29 +00:00
|
|
|
void rt_cache_flush(struct net *net)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-07-30 00:33:53 +00:00
|
|
|
rt_genid_bump_ipv4(net);
|
2010-03-08 03:20:00 +00:00
|
|
|
}
|
|
|
|
|
2012-07-03 04:52:24 +00:00
|
|
|
static struct neighbour *ipv4_neigh_lookup(const struct dst_entry *dst,
|
|
|
|
struct sk_buff *skb,
|
|
|
|
const void *daddr)
|
2011-07-11 22:44:24 +00:00
|
|
|
{
|
2019-04-05 23:30:27 +00:00
|
|
|
const struct rtable *rt = container_of(dst, struct rtable, dst);
|
2011-07-18 07:40:17 +00:00
|
|
|
struct net_device *dev = dst->dev;
|
2011-07-11 22:44:24 +00:00
|
|
|
struct neighbour *n;
|
|
|
|
|
ipv4: Add helpers for neigh lookup for nexthop
A common theme in the output path is looking up a neigh entry for a
nexthop, either the gateway in an rtable or a fallback to the daddr
in the skb:
nexthop = (__force u32)rt_nexthop(rt, ip_hdr(skb)->daddr);
neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
if (unlikely(!neigh))
neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
To allow the nexthop to be an IPv6 address we need to consider the
family of the nexthop and then call __ipv{4,6}_neigh_lookup_noref based
on it.
To make this simpler, add a ip_neigh_gw4 helper similar to ip_neigh_gw6
added in an earlier patch which handles:
neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
if (unlikely(!neigh))
neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
And then add a second one, ip_neigh_for_gw, that calls either
ip_neigh_gw4 or ip_neigh_gw6 based on the address family of the gateway.
Update the output paths in the VRF driver and core v4 code to use
ip_neigh_for_gw simplifying the family based lookup and making both
ready for a v6 nexthop.
ipv4_neigh_lookup has a different need - the potential to resolve a
passed in address in addition to any gateway in the rtable or skb. Since
this is a one-off, add ip_neigh_gw4 and ip_neigh_gw6 diectly. The
difference between __neigh_create used by the helpers and neigh_create
called by ipv4_neigh_lookup is taking a refcount, so add rcu_read_lock_bh
and bump the refcnt on the neigh entry.
Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-04-05 23:30:34 +00:00
|
|
|
rcu_read_lock_bh();
|
|
|
|
|
|
|
|
if (likely(rt->rt_gw_family == AF_INET)) {
|
|
|
|
n = ip_neigh_gw4(dev, rt->rt_gw4);
|
|
|
|
} else if (rt->rt_gw_family == AF_INET6) {
|
|
|
|
n = ip_neigh_gw6(dev, &rt->rt_gw6);
|
|
|
|
} else {
|
|
|
|
__be32 pkey;
|
|
|
|
|
|
|
|
pkey = skb ? ip_hdr(skb)->daddr : *((__be32 *) daddr);
|
|
|
|
n = ip_neigh_gw4(dev, pkey);
|
|
|
|
}
|
|
|
|
|
2019-07-04 16:26:38 +00:00
|
|
|
if (!IS_ERR(n) && !refcount_inc_not_zero(&n->refcnt))
|
ipv4: Add helpers for neigh lookup for nexthop
A common theme in the output path is looking up a neigh entry for a
nexthop, either the gateway in an rtable or a fallback to the daddr
in the skb:
nexthop = (__force u32)rt_nexthop(rt, ip_hdr(skb)->daddr);
neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
if (unlikely(!neigh))
neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
To allow the nexthop to be an IPv6 address we need to consider the
family of the nexthop and then call __ipv{4,6}_neigh_lookup_noref based
on it.
To make this simpler, add a ip_neigh_gw4 helper similar to ip_neigh_gw6
added in an earlier patch which handles:
neigh = __ipv4_neigh_lookup_noref(dev, nexthop);
if (unlikely(!neigh))
neigh = __neigh_create(&arp_tbl, &nexthop, dev, false);
And then add a second one, ip_neigh_for_gw, that calls either
ip_neigh_gw4 or ip_neigh_gw6 based on the address family of the gateway.
Update the output paths in the VRF driver and core v4 code to use
ip_neigh_for_gw simplifying the family based lookup and making both
ready for a v6 nexthop.
ipv4_neigh_lookup has a different need - the potential to resolve a
passed in address in addition to any gateway in the rtable or skb. Since
this is a one-off, add ip_neigh_gw4 and ip_neigh_gw6 diectly. The
difference between __neigh_create used by the helpers and neigh_create
called by ipv4_neigh_lookup is taking a refcount, so add rcu_read_lock_bh
and bump the refcnt on the neigh entry.
Signed-off-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-04-05 23:30:34 +00:00
|
|
|
n = NULL;
|
|
|
|
|
|
|
|
rcu_read_unlock_bh();
|
|
|
|
|
|
|
|
return n;
|
2011-07-18 07:40:17 +00:00
|
|
|
}
|
|
|
|
|
2017-02-06 21:14:15 +00:00
|
|
|
static void ipv4_confirm_neigh(const struct dst_entry *dst, const void *daddr)
|
|
|
|
{
|
2019-04-05 23:30:27 +00:00
|
|
|
const struct rtable *rt = container_of(dst, struct rtable, dst);
|
2017-02-06 21:14:15 +00:00
|
|
|
struct net_device *dev = dst->dev;
|
|
|
|
const __be32 *pkey = daddr;
|
|
|
|
|
2019-04-05 23:30:36 +00:00
|
|
|
if (rt->rt_gw_family == AF_INET) {
|
2019-04-05 23:30:27 +00:00
|
|
|
pkey = (const __be32 *)&rt->rt_gw4;
|
2019-04-05 23:30:36 +00:00
|
|
|
} else if (rt->rt_gw_family == AF_INET6) {
|
|
|
|
return __ipv6_confirm_neigh_stub(dev, &rt->rt_gw6);
|
|
|
|
} else if (!daddr ||
|
2017-02-06 21:14:15 +00:00
|
|
|
(rt->rt_flags &
|
2019-04-05 23:30:36 +00:00
|
|
|
(RTCF_MULTICAST | RTCF_BROADCAST | RTCF_LOCAL))) {
|
2017-02-06 21:14:15 +00:00
|
|
|
return;
|
2019-04-05 23:30:36 +00:00
|
|
|
}
|
2017-02-06 21:14:15 +00:00
|
|
|
__ipv4_confirm_neigh(dev, *(__force u32 *)pkey);
|
|
|
|
}
|
|
|
|
|
2021-03-24 21:53:37 +00:00
|
|
|
/* Hash tables of size 2048..262144 depending on RAM size.
|
|
|
|
* Each bucket uses 8 bytes.
|
|
|
|
*/
|
|
|
|
static u32 ip_idents_mask __read_mostly;
|
2015-05-01 17:37:49 +00:00
|
|
|
static atomic_t *ip_idents __read_mostly;
|
|
|
|
static u32 *ip_tstamps __read_mostly;
|
ip: make IP identifiers less predictable
In "Counting Packets Sent Between Arbitrary Internet Hosts", Jeffrey and
Jedidiah describe ways exploiting linux IP identifier generation to
infer whether two machines are exchanging packets.
With commit 73f156a6e8c1 ("inetpeer: get rid of ip_id_count"), we
changed IP id generation, but this does not really prevent this
side-channel technique.
This patch adds a random amount of perturbation so that IP identifiers
for a given destination [1] are no longer monotonically increasing after
an idle period.
Note that prandom_u32_max(1) returns 0, so if generator is used at most
once per jiffy, this patch inserts no hole in the ID suite and do not
increase collision probability.
This is jiffies based, so in the worst case (HZ=1000), the id can
rollover after ~65 seconds of idle time, which should be fine.
We also change the hash used in __ip_select_ident() to not only hash
on daddr, but also saddr and protocol, so that ICMP probes can not be
used to infer information for other protocols.
For IPv6, adds saddr into the hash as well, but not nexthdr.
If I ping the patched target, we can see ID are now hard to predict.
21:57:11.008086 IP (...)
A > target: ICMP echo request, seq 1, length 64
21:57:11.010752 IP (... id 2081 ...)
target > A: ICMP echo reply, seq 1, length 64
21:57:12.013133 IP (...)
A > target: ICMP echo request, seq 2, length 64
21:57:12.015737 IP (... id 3039 ...)
target > A: ICMP echo reply, seq 2, length 64
21:57:13.016580 IP (...)
A > target: ICMP echo request, seq 3, length 64
21:57:13.019251 IP (... id 3437 ...)
target > A: ICMP echo reply, seq 3, length 64
[1] TCP sessions uses a per flow ID generator not changed by this patch.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Jeffrey Knockel <jeffk@cs.unm.edu>
Reported-by: Jedidiah R. Crandall <crandall@cs.unm.edu>
Cc: Willy Tarreau <w@1wt.eu>
Cc: Hannes Frederic Sowa <hannes@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-07-26 06:58:10 +00:00
|
|
|
|
|
|
|
/* In order to protect privacy, we add a perturbation to identifiers
|
|
|
|
* if one generator is seldom used. This makes hard for an attacker
|
|
|
|
* to infer how many packets were sent between two points in time.
|
|
|
|
*/
|
|
|
|
u32 ip_idents_reserve(u32 hash, int segs)
|
|
|
|
{
|
2021-03-24 21:53:37 +00:00
|
|
|
u32 bucket, old, now = (u32)jiffies;
|
|
|
|
atomic_t *p_id;
|
|
|
|
u32 *p_tstamp;
|
2020-05-16 03:46:49 +00:00
|
|
|
u32 delta = 0;
|
ip: make IP identifiers less predictable
In "Counting Packets Sent Between Arbitrary Internet Hosts", Jeffrey and
Jedidiah describe ways exploiting linux IP identifier generation to
infer whether two machines are exchanging packets.
With commit 73f156a6e8c1 ("inetpeer: get rid of ip_id_count"), we
changed IP id generation, but this does not really prevent this
side-channel technique.
This patch adds a random amount of perturbation so that IP identifiers
for a given destination [1] are no longer monotonically increasing after
an idle period.
Note that prandom_u32_max(1) returns 0, so if generator is used at most
once per jiffy, this patch inserts no hole in the ID suite and do not
increase collision probability.
This is jiffies based, so in the worst case (HZ=1000), the id can
rollover after ~65 seconds of idle time, which should be fine.
We also change the hash used in __ip_select_ident() to not only hash
on daddr, but also saddr and protocol, so that ICMP probes can not be
used to infer information for other protocols.
For IPv6, adds saddr into the hash as well, but not nexthdr.
If I ping the patched target, we can see ID are now hard to predict.
21:57:11.008086 IP (...)
A > target: ICMP echo request, seq 1, length 64
21:57:11.010752 IP (... id 2081 ...)
target > A: ICMP echo reply, seq 1, length 64
21:57:12.013133 IP (...)
A > target: ICMP echo request, seq 2, length 64
21:57:12.015737 IP (... id 3039 ...)
target > A: ICMP echo reply, seq 2, length 64
21:57:13.016580 IP (...)
A > target: ICMP echo request, seq 3, length 64
21:57:13.019251 IP (... id 3437 ...)
target > A: ICMP echo reply, seq 3, length 64
[1] TCP sessions uses a per flow ID generator not changed by this patch.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Jeffrey Knockel <jeffk@cs.unm.edu>
Reported-by: Jedidiah R. Crandall <crandall@cs.unm.edu>
Cc: Willy Tarreau <w@1wt.eu>
Cc: Hannes Frederic Sowa <hannes@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-07-26 06:58:10 +00:00
|
|
|
|
2021-03-24 21:53:37 +00:00
|
|
|
bucket = hash & ip_idents_mask;
|
|
|
|
p_tstamp = ip_tstamps + bucket;
|
|
|
|
p_id = ip_idents + bucket;
|
|
|
|
old = READ_ONCE(*p_tstamp);
|
|
|
|
|
2015-05-01 17:37:49 +00:00
|
|
|
if (old != now && cmpxchg(p_tstamp, old, now) == old)
|
ip: make IP identifiers less predictable
In "Counting Packets Sent Between Arbitrary Internet Hosts", Jeffrey and
Jedidiah describe ways exploiting linux IP identifier generation to
infer whether two machines are exchanging packets.
With commit 73f156a6e8c1 ("inetpeer: get rid of ip_id_count"), we
changed IP id generation, but this does not really prevent this
side-channel technique.
This patch adds a random amount of perturbation so that IP identifiers
for a given destination [1] are no longer monotonically increasing after
an idle period.
Note that prandom_u32_max(1) returns 0, so if generator is used at most
once per jiffy, this patch inserts no hole in the ID suite and do not
increase collision probability.
This is jiffies based, so in the worst case (HZ=1000), the id can
rollover after ~65 seconds of idle time, which should be fine.
We also change the hash used in __ip_select_ident() to not only hash
on daddr, but also saddr and protocol, so that ICMP probes can not be
used to infer information for other protocols.
For IPv6, adds saddr into the hash as well, but not nexthdr.
If I ping the patched target, we can see ID are now hard to predict.
21:57:11.008086 IP (...)
A > target: ICMP echo request, seq 1, length 64
21:57:11.010752 IP (... id 2081 ...)
target > A: ICMP echo reply, seq 1, length 64
21:57:12.013133 IP (...)
A > target: ICMP echo request, seq 2, length 64
21:57:12.015737 IP (... id 3039 ...)
target > A: ICMP echo reply, seq 2, length 64
21:57:13.016580 IP (...)
A > target: ICMP echo request, seq 3, length 64
21:57:13.019251 IP (... id 3437 ...)
target > A: ICMP echo reply, seq 3, length 64
[1] TCP sessions uses a per flow ID generator not changed by this patch.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Jeffrey Knockel <jeffk@cs.unm.edu>
Reported-by: Jedidiah R. Crandall <crandall@cs.unm.edu>
Cc: Willy Tarreau <w@1wt.eu>
Cc: Hannes Frederic Sowa <hannes@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-07-26 06:58:10 +00:00
|
|
|
delta = prandom_u32_max(now - old);
|
|
|
|
|
2020-05-16 03:46:49 +00:00
|
|
|
/* If UBSAN reports an error there, please make sure your compiler
|
|
|
|
* supports -fno-strict-overflow before reporting it that was a bug
|
|
|
|
* in UBSAN, and it has been fixed in GCC-8.
|
|
|
|
*/
|
|
|
|
return atomic_add_return(segs + delta, p_id) - segs;
|
ip: make IP identifiers less predictable
In "Counting Packets Sent Between Arbitrary Internet Hosts", Jeffrey and
Jedidiah describe ways exploiting linux IP identifier generation to
infer whether two machines are exchanging packets.
With commit 73f156a6e8c1 ("inetpeer: get rid of ip_id_count"), we
changed IP id generation, but this does not really prevent this
side-channel technique.
This patch adds a random amount of perturbation so that IP identifiers
for a given destination [1] are no longer monotonically increasing after
an idle period.
Note that prandom_u32_max(1) returns 0, so if generator is used at most
once per jiffy, this patch inserts no hole in the ID suite and do not
increase collision probability.
This is jiffies based, so in the worst case (HZ=1000), the id can
rollover after ~65 seconds of idle time, which should be fine.
We also change the hash used in __ip_select_ident() to not only hash
on daddr, but also saddr and protocol, so that ICMP probes can not be
used to infer information for other protocols.
For IPv6, adds saddr into the hash as well, but not nexthdr.
If I ping the patched target, we can see ID are now hard to predict.
21:57:11.008086 IP (...)
A > target: ICMP echo request, seq 1, length 64
21:57:11.010752 IP (... id 2081 ...)
target > A: ICMP echo reply, seq 1, length 64
21:57:12.013133 IP (...)
A > target: ICMP echo request, seq 2, length 64
21:57:12.015737 IP (... id 3039 ...)
target > A: ICMP echo reply, seq 2, length 64
21:57:13.016580 IP (...)
A > target: ICMP echo request, seq 3, length 64
21:57:13.019251 IP (... id 3437 ...)
target > A: ICMP echo reply, seq 3, length 64
[1] TCP sessions uses a per flow ID generator not changed by this patch.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Jeffrey Knockel <jeffk@cs.unm.edu>
Reported-by: Jedidiah R. Crandall <crandall@cs.unm.edu>
Cc: Willy Tarreau <w@1wt.eu>
Cc: Hannes Frederic Sowa <hannes@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-07-26 06:58:10 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(ip_idents_reserve);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-03-25 16:07:44 +00:00
|
|
|
void __ip_select_ident(struct net *net, struct iphdr *iph, int segs)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
inetpeer: get rid of ip_id_count
Ideally, we would need to generate IP ID using a per destination IP
generator.
linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.
1) each inet_peer struct consumes 192 bytes
2) inetpeer cache uses a binary tree of inet_peer structs,
with a nominal size of ~66000 elements under load.
3) lookups in this tree are hitting a lot of cache lines, as tree depth
is about 20.
4) If server deals with many tcp flows, we have a high probability of
not finding the inet_peer, allocating a fresh one, inserting it in
the tree with same initial ip_id_count, (cf secure_ip_id())
5) We garbage collect inet_peer aggressively.
IP ID generation do not have to be 'perfect'
Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.
We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.
ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)
secure_ip_id() and secure_ipv6_id() no longer are needed.
Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-02 12:26:03 +00:00
|
|
|
u32 hash, id;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-03-27 19:40:33 +00:00
|
|
|
/* Note the following code is not safe, but this is okay. */
|
|
|
|
if (unlikely(siphash_key_is_zero(&net->ipv4.ip_id_key)))
|
|
|
|
get_random_bytes(&net->ipv4.ip_id_key,
|
|
|
|
sizeof(net->ipv4.ip_id_key));
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-03-27 19:40:33 +00:00
|
|
|
hash = siphash_3u32((__force u32)iph->daddr,
|
ip: make IP identifiers less predictable
In "Counting Packets Sent Between Arbitrary Internet Hosts", Jeffrey and
Jedidiah describe ways exploiting linux IP identifier generation to
infer whether two machines are exchanging packets.
With commit 73f156a6e8c1 ("inetpeer: get rid of ip_id_count"), we
changed IP id generation, but this does not really prevent this
side-channel technique.
This patch adds a random amount of perturbation so that IP identifiers
for a given destination [1] are no longer monotonically increasing after
an idle period.
Note that prandom_u32_max(1) returns 0, so if generator is used at most
once per jiffy, this patch inserts no hole in the ID suite and do not
increase collision probability.
This is jiffies based, so in the worst case (HZ=1000), the id can
rollover after ~65 seconds of idle time, which should be fine.
We also change the hash used in __ip_select_ident() to not only hash
on daddr, but also saddr and protocol, so that ICMP probes can not be
used to infer information for other protocols.
For IPv6, adds saddr into the hash as well, but not nexthdr.
If I ping the patched target, we can see ID are now hard to predict.
21:57:11.008086 IP (...)
A > target: ICMP echo request, seq 1, length 64
21:57:11.010752 IP (... id 2081 ...)
target > A: ICMP echo reply, seq 1, length 64
21:57:12.013133 IP (...)
A > target: ICMP echo request, seq 2, length 64
21:57:12.015737 IP (... id 3039 ...)
target > A: ICMP echo reply, seq 2, length 64
21:57:13.016580 IP (...)
A > target: ICMP echo request, seq 3, length 64
21:57:13.019251 IP (... id 3437 ...)
target > A: ICMP echo reply, seq 3, length 64
[1] TCP sessions uses a per flow ID generator not changed by this patch.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Reported-by: Jeffrey Knockel <jeffk@cs.unm.edu>
Reported-by: Jedidiah R. Crandall <crandall@cs.unm.edu>
Cc: Willy Tarreau <w@1wt.eu>
Cc: Hannes Frederic Sowa <hannes@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-07-26 06:58:10 +00:00
|
|
|
(__force u32)iph->saddr,
|
2019-03-27 19:40:33 +00:00
|
|
|
iph->protocol,
|
|
|
|
&net->ipv4.ip_id_key);
|
inetpeer: get rid of ip_id_count
Ideally, we would need to generate IP ID using a per destination IP
generator.
linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.
1) each inet_peer struct consumes 192 bytes
2) inetpeer cache uses a binary tree of inet_peer structs,
with a nominal size of ~66000 elements under load.
3) lookups in this tree are hitting a lot of cache lines, as tree depth
is about 20.
4) If server deals with many tcp flows, we have a high probability of
not finding the inet_peer, allocating a fresh one, inserting it in
the tree with same initial ip_id_count, (cf secure_ip_id())
5) We garbage collect inet_peer aggressively.
IP ID generation do not have to be 'perfect'
Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.
We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.
ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)
secure_ip_id() and secure_ipv6_id() no longer are needed.
Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-02 12:26:03 +00:00
|
|
|
id = ip_idents_reserve(hash, segs);
|
|
|
|
iph->id = htons(id);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2010-07-09 21:22:10 +00:00
|
|
|
EXPORT_SYMBOL(__ip_select_ident);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
static void __build_flow_key(const struct net *net, struct flowi4 *fl4,
|
|
|
|
const struct sock *sk,
|
2012-07-17 11:19:00 +00:00
|
|
|
const struct iphdr *iph,
|
|
|
|
int oif, u8 tos,
|
|
|
|
u8 prot, u32 mark, int flow_flags)
|
|
|
|
{
|
|
|
|
if (sk) {
|
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
|
|
|
|
|
|
|
oif = sk->sk_bound_dev_if;
|
|
|
|
mark = sk->sk_mark;
|
|
|
|
tos = RT_CONN_FLAGS(sk);
|
|
|
|
prot = inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol;
|
|
|
|
}
|
|
|
|
flowi4_init_output(fl4, oif, mark, tos,
|
|
|
|
RT_SCOPE_UNIVERSE, prot,
|
|
|
|
flow_flags,
|
2016-11-03 17:23:43 +00:00
|
|
|
iph->daddr, iph->saddr, 0, 0,
|
|
|
|
sock_net_uid(net, sk));
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 20:42:13 +00:00
|
|
|
static void build_skb_flow_key(struct flowi4 *fl4, const struct sk_buff *skb,
|
|
|
|
const struct sock *sk)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
2016-11-29 17:56:47 +00:00
|
|
|
const struct net *net = dev_net(skb->dev);
|
2012-07-17 11:19:00 +00:00
|
|
|
const struct iphdr *iph = ip_hdr(skb);
|
|
|
|
int oif = skb->dev->ifindex;
|
|
|
|
u8 tos = RT_TOS(iph->tos);
|
|
|
|
u8 prot = iph->protocol;
|
|
|
|
u32 mark = skb->mark;
|
|
|
|
|
2016-11-29 17:56:47 +00:00
|
|
|
__build_flow_key(net, fl4, sk, iph, oif, tos, prot, mark, 0);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 20:42:13 +00:00
|
|
|
static void build_sk_flow_key(struct flowi4 *fl4, const struct sock *sk)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
|
|
|
const struct inet_sock *inet = inet_sk(sk);
|
2012-07-17 20:42:13 +00:00
|
|
|
const struct ip_options_rcu *inet_opt;
|
2012-07-17 11:19:00 +00:00
|
|
|
__be32 daddr = inet->inet_daddr;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
|
|
|
inet_opt = rcu_dereference(inet->inet_opt);
|
|
|
|
if (inet_opt && inet_opt->opt.srr)
|
|
|
|
daddr = inet_opt->opt.faddr;
|
|
|
|
flowi4_init_output(fl4, sk->sk_bound_dev_if, sk->sk_mark,
|
|
|
|
RT_CONN_FLAGS(sk), RT_SCOPE_UNIVERSE,
|
|
|
|
inet->hdrincl ? IPPROTO_RAW : sk->sk_protocol,
|
|
|
|
inet_sk_flowi_flags(sk),
|
2016-11-03 17:23:43 +00:00
|
|
|
daddr, inet->inet_saddr, 0, 0, sk->sk_uid);
|
2012-07-17 11:19:00 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
|
2012-07-17 20:42:13 +00:00
|
|
|
static void ip_rt_build_flow_key(struct flowi4 *fl4, const struct sock *sk,
|
|
|
|
const struct sk_buff *skb)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
|
|
|
if (skb)
|
|
|
|
build_skb_flow_key(fl4, skb, sk);
|
|
|
|
else
|
|
|
|
build_sk_flow_key(fl4, sk);
|
|
|
|
}
|
|
|
|
|
2012-07-31 22:02:02 +00:00
|
|
|
static DEFINE_SPINLOCK(fnhe_lock);
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2013-06-27 07:27:05 +00:00
|
|
|
static void fnhe_flush_routes(struct fib_nh_exception *fnhe)
|
|
|
|
{
|
|
|
|
struct rtable *rt;
|
|
|
|
|
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_input);
|
|
|
|
if (rt) {
|
|
|
|
RCU_INIT_POINTER(fnhe->fnhe_rth_input, NULL);
|
2017-06-17 17:42:30 +00:00
|
|
|
dst_dev_put(&rt->dst);
|
2017-06-17 17:42:29 +00:00
|
|
|
dst_release(&rt->dst);
|
2013-06-27 07:27:05 +00:00
|
|
|
}
|
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_output);
|
|
|
|
if (rt) {
|
|
|
|
RCU_INIT_POINTER(fnhe->fnhe_rth_output, NULL);
|
2017-06-17 17:42:30 +00:00
|
|
|
dst_dev_put(&rt->dst);
|
2017-06-17 17:42:29 +00:00
|
|
|
dst_release(&rt->dst);
|
2013-06-27 07:27:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-08-29 22:16:15 +00:00
|
|
|
static void fnhe_remove_oldest(struct fnhe_hash_bucket *hash)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
2021-08-29 22:16:15 +00:00
|
|
|
struct fib_nh_exception __rcu **fnhe_p, **oldest_p;
|
|
|
|
struct fib_nh_exception *fnhe, *oldest = NULL;
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2021-08-29 22:16:15 +00:00
|
|
|
for (fnhe_p = &hash->chain; ; fnhe_p = &fnhe->fnhe_next) {
|
|
|
|
fnhe = rcu_dereference_protected(*fnhe_p,
|
|
|
|
lockdep_is_held(&fnhe_lock));
|
|
|
|
if (!fnhe)
|
|
|
|
break;
|
|
|
|
if (!oldest ||
|
|
|
|
time_before(fnhe->fnhe_stamp, oldest->fnhe_stamp)) {
|
2012-07-17 11:19:00 +00:00
|
|
|
oldest = fnhe;
|
2021-08-29 22:16:15 +00:00
|
|
|
oldest_p = fnhe_p;
|
|
|
|
}
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
2013-06-27 07:27:05 +00:00
|
|
|
fnhe_flush_routes(oldest);
|
2021-08-29 22:16:15 +00:00
|
|
|
*oldest_p = oldest->fnhe_next;
|
|
|
|
kfree_rcu(oldest, rcu);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2021-08-25 23:17:29 +00:00
|
|
|
static u32 fnhe_hashfun(__be32 daddr)
|
2012-07-17 20:23:08 +00:00
|
|
|
{
|
2021-08-25 23:17:29 +00:00
|
|
|
static siphash_key_t fnhe_hash_key __read_mostly;
|
|
|
|
u64 hval;
|
2012-07-17 20:23:08 +00:00
|
|
|
|
2021-08-25 23:17:29 +00:00
|
|
|
net_get_random_once(&fnhe_hash_key, sizeof(fnhe_hash_key));
|
|
|
|
hval = siphash_1u32((__force u32)daddr, &fnhe_hash_key);
|
|
|
|
return hash_64(hval, FNHE_HASH_SHIFT);
|
2012-07-17 20:23:08 +00:00
|
|
|
}
|
|
|
|
|
2013-05-27 20:46:31 +00:00
|
|
|
static void fill_route_from_fnhe(struct rtable *rt, struct fib_nh_exception *fnhe)
|
|
|
|
{
|
|
|
|
rt->rt_pmtu = fnhe->fnhe_pmtu;
|
2018-03-14 09:21:14 +00:00
|
|
|
rt->rt_mtu_locked = fnhe->fnhe_mtu_locked;
|
2013-05-27 20:46:31 +00:00
|
|
|
rt->dst.expires = fnhe->fnhe_expires;
|
|
|
|
|
|
|
|
if (fnhe->fnhe_gw) {
|
|
|
|
rt->rt_flags |= RTCF_REDIRECTED;
|
2019-09-17 17:39:49 +00:00
|
|
|
rt->rt_uses_gateway = 1;
|
2019-04-05 23:30:27 +00:00
|
|
|
rt->rt_gw_family = AF_INET;
|
|
|
|
rt->rt_gw4 = fnhe->fnhe_gw;
|
2013-05-27 20:46:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
static void update_or_create_fnhe(struct fib_nh_common *nhc, __be32 daddr,
|
|
|
|
__be32 gw, u32 pmtu, bool lock,
|
|
|
|
unsigned long expires)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
2012-07-18 10:15:35 +00:00
|
|
|
struct fnhe_hash_bucket *hash;
|
2012-07-17 11:19:00 +00:00
|
|
|
struct fib_nh_exception *fnhe;
|
2013-05-27 20:46:31 +00:00
|
|
|
struct rtable *rt;
|
2017-11-17 06:27:18 +00:00
|
|
|
u32 genid, hval;
|
2013-05-27 20:46:31 +00:00
|
|
|
unsigned int i;
|
2012-07-17 11:19:00 +00:00
|
|
|
int depth;
|
2017-11-17 06:27:18 +00:00
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
genid = fnhe_genid(dev_net(nhc->nhc_dev));
|
2017-11-17 06:27:18 +00:00
|
|
|
hval = fnhe_hashfun(daddr);
|
2012-07-18 10:15:35 +00:00
|
|
|
|
2012-07-31 22:02:02 +00:00
|
|
|
spin_lock_bh(&fnhe_lock);
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
hash = rcu_dereference(nhc->nhc_exceptions);
|
2012-07-17 11:19:00 +00:00
|
|
|
if (!hash) {
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
hash = kcalloc(FNHE_HASH_SIZE, sizeof(*hash), GFP_ATOMIC);
|
2012-07-17 11:19:00 +00:00
|
|
|
if (!hash)
|
2012-07-18 10:15:35 +00:00
|
|
|
goto out_unlock;
|
2019-04-30 14:45:50 +00:00
|
|
|
rcu_assign_pointer(nhc->nhc_exceptions, hash);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
hash += hval;
|
|
|
|
|
|
|
|
depth = 0;
|
|
|
|
for (fnhe = rcu_dereference(hash->chain); fnhe;
|
|
|
|
fnhe = rcu_dereference(fnhe->fnhe_next)) {
|
|
|
|
if (fnhe->fnhe_daddr == daddr)
|
2012-07-18 10:15:35 +00:00
|
|
|
break;
|
2012-07-17 11:19:00 +00:00
|
|
|
depth++;
|
|
|
|
}
|
|
|
|
|
2012-07-18 10:15:35 +00:00
|
|
|
if (fnhe) {
|
2017-11-17 06:27:18 +00:00
|
|
|
if (fnhe->fnhe_genid != genid)
|
|
|
|
fnhe->fnhe_genid = genid;
|
2012-07-18 10:15:35 +00:00
|
|
|
if (gw)
|
|
|
|
fnhe->fnhe_gw = gw;
|
2018-03-14 09:21:14 +00:00
|
|
|
if (pmtu) {
|
2012-07-18 10:15:35 +00:00
|
|
|
fnhe->fnhe_pmtu = pmtu;
|
2018-03-14 09:21:14 +00:00
|
|
|
fnhe->fnhe_mtu_locked = lock;
|
|
|
|
}
|
2017-11-17 06:27:06 +00:00
|
|
|
fnhe->fnhe_expires = max(1UL, expires);
|
2013-05-27 20:46:31 +00:00
|
|
|
/* Update all cached dsts too */
|
2013-06-27 07:27:05 +00:00
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_input);
|
|
|
|
if (rt)
|
|
|
|
fill_route_from_fnhe(rt, fnhe);
|
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_output);
|
2013-05-27 20:46:31 +00:00
|
|
|
if (rt)
|
|
|
|
fill_route_from_fnhe(rt, fnhe);
|
2012-07-18 10:15:35 +00:00
|
|
|
} else {
|
2021-08-29 22:16:15 +00:00
|
|
|
/* Randomize max depth to avoid some side channels attacks. */
|
|
|
|
int max_depth = FNHE_RECLAIM_DEPTH +
|
|
|
|
prandom_u32_max(FNHE_RECLAIM_DEPTH);
|
|
|
|
|
|
|
|
while (depth > max_depth) {
|
|
|
|
fnhe_remove_oldest(hash);
|
|
|
|
depth--;
|
2012-07-18 10:15:35 +00:00
|
|
|
}
|
2021-08-29 22:16:15 +00:00
|
|
|
|
|
|
|
fnhe = kzalloc(sizeof(*fnhe), GFP_ATOMIC);
|
|
|
|
if (!fnhe)
|
|
|
|
goto out_unlock;
|
|
|
|
|
|
|
|
fnhe->fnhe_next = hash->chain;
|
|
|
|
|
2017-11-17 06:27:18 +00:00
|
|
|
fnhe->fnhe_genid = genid;
|
2012-07-18 10:15:35 +00:00
|
|
|
fnhe->fnhe_daddr = daddr;
|
|
|
|
fnhe->fnhe_gw = gw;
|
|
|
|
fnhe->fnhe_pmtu = pmtu;
|
2018-03-14 09:21:14 +00:00
|
|
|
fnhe->fnhe_mtu_locked = lock;
|
2018-05-02 06:41:19 +00:00
|
|
|
fnhe->fnhe_expires = max(1UL, expires);
|
2013-05-27 20:46:31 +00:00
|
|
|
|
2021-08-29 22:16:15 +00:00
|
|
|
rcu_assign_pointer(hash->chain, fnhe);
|
|
|
|
|
2013-05-27 20:46:31 +00:00
|
|
|
/* Exception created; mark the cached routes for the nexthop
|
|
|
|
* stale, so anyone caching it rechecks if this exception
|
|
|
|
* applies to them.
|
|
|
|
*/
|
2019-04-30 14:45:48 +00:00
|
|
|
rt = rcu_dereference(nhc->nhc_rth_input);
|
2013-06-27 07:27:05 +00:00
|
|
|
if (rt)
|
|
|
|
rt->dst.obsolete = DST_OBSOLETE_KILL;
|
|
|
|
|
2013-05-27 20:46:31 +00:00
|
|
|
for_each_possible_cpu(i) {
|
|
|
|
struct rtable __rcu **prt;
|
2021-03-12 07:30:05 +00:00
|
|
|
|
2019-04-30 14:45:48 +00:00
|
|
|
prt = per_cpu_ptr(nhc->nhc_pcpu_rth_output, i);
|
2013-05-27 20:46:31 +00:00
|
|
|
rt = rcu_dereference(*prt);
|
|
|
|
if (rt)
|
|
|
|
rt->dst.obsolete = DST_OBSOLETE_KILL;
|
|
|
|
}
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
fnhe->fnhe_stamp = jiffies;
|
2012-07-18 10:15:35 +00:00
|
|
|
|
|
|
|
out_unlock:
|
2012-07-31 22:02:02 +00:00
|
|
|
spin_unlock_bh(&fnhe_lock);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 18:31:28 +00:00
|
|
|
static void __ip_do_redirect(struct rtable *rt, struct sk_buff *skb, struct flowi4 *fl4,
|
|
|
|
bool kill_route)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-07-12 03:55:47 +00:00
|
|
|
__be32 new_gw = icmp_hdr(skb)->un.gateway;
|
2012-07-12 03:38:08 +00:00
|
|
|
__be32 old_gw = ip_hdr(skb)->saddr;
|
2012-07-12 03:55:47 +00:00
|
|
|
struct net_device *dev = skb->dev;
|
|
|
|
struct in_device *in_dev;
|
2012-07-17 11:19:00 +00:00
|
|
|
struct fib_result res;
|
2012-07-12 03:55:47 +00:00
|
|
|
struct neighbour *n;
|
2008-02-29 04:50:06 +00:00
|
|
|
struct net *net;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-07-12 03:38:08 +00:00
|
|
|
switch (icmp_hdr(skb)->code & 7) {
|
|
|
|
case ICMP_REDIR_NET:
|
|
|
|
case ICMP_REDIR_NETTOS:
|
|
|
|
case ICMP_REDIR_HOST:
|
|
|
|
case ICMP_REDIR_HOSTTOS:
|
|
|
|
break;
|
|
|
|
|
|
|
|
default:
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2019-04-05 23:30:27 +00:00
|
|
|
if (rt->rt_gw_family != AF_INET || rt->rt_gw4 != old_gw)
|
2012-07-12 03:55:47 +00:00
|
|
|
return;
|
|
|
|
|
|
|
|
in_dev = __in_dev_get_rcu(dev);
|
|
|
|
if (!in_dev)
|
|
|
|
return;
|
|
|
|
|
2008-03-25 12:47:49 +00:00
|
|
|
net = dev_net(dev);
|
2009-11-23 18:41:23 +00:00
|
|
|
if (new_gw == old_gw || !IN_DEV_RX_REDIRECTS(in_dev) ||
|
|
|
|
ipv4_is_multicast(new_gw) || ipv4_is_lbcast(new_gw) ||
|
|
|
|
ipv4_is_zeronet(new_gw))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto reject_redirect;
|
|
|
|
|
|
|
|
if (!IN_DEV_SHARED_MEDIA(in_dev)) {
|
|
|
|
if (!inet_addr_onlink(in_dev, new_gw, old_gw))
|
|
|
|
goto reject_redirect;
|
|
|
|
if (IN_DEV_SEC_REDIRECTS(in_dev) && ip_fib_check_default(new_gw, dev))
|
|
|
|
goto reject_redirect;
|
|
|
|
} else {
|
2008-02-29 04:50:06 +00:00
|
|
|
if (inet_addr_type(net, new_gw) != RTN_UNICAST)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto reject_redirect;
|
|
|
|
}
|
|
|
|
|
2016-11-10 16:16:15 +00:00
|
|
|
n = __ipv4_neigh_lookup(rt->dst.dev, new_gw);
|
|
|
|
if (!n)
|
|
|
|
n = neigh_create(&arp_tbl, &new_gw, rt->dst.dev);
|
2014-09-25 00:07:53 +00:00
|
|
|
if (!IS_ERR(n)) {
|
2012-07-12 03:55:47 +00:00
|
|
|
if (!(n->nud_state & NUD_VALID)) {
|
|
|
|
neigh_event_send(n, NULL);
|
|
|
|
} else {
|
2015-06-23 17:45:37 +00:00
|
|
|
if (fib_lookup(net, fl4, &res, 0) == 0) {
|
2020-09-15 03:03:54 +00:00
|
|
|
struct fib_nh_common *nhc;
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2020-09-15 03:03:54 +00:00
|
|
|
fib_select_path(net, &res, fl4, skb);
|
|
|
|
nhc = FIB_RES_NHC(res);
|
2019-04-30 14:45:50 +00:00
|
|
|
update_or_create_fnhe(nhc, fl4->daddr, new_gw,
|
2018-03-14 09:21:14 +00:00
|
|
|
0, false,
|
|
|
|
jiffies + ip_rt_gc_timeout);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
2012-07-17 18:31:28 +00:00
|
|
|
if (kill_route)
|
|
|
|
rt->dst.obsolete = DST_OBSOLETE_KILL;
|
2012-07-12 03:55:47 +00:00
|
|
|
call_netevent_notifiers(NETEVENT_NEIGH_UPDATE, n);
|
|
|
|
}
|
|
|
|
neigh_release(n);
|
|
|
|
}
|
|
|
|
return;
|
|
|
|
|
|
|
|
reject_redirect:
|
|
|
|
#ifdef CONFIG_IP_ROUTE_VERBOSE
|
2012-07-12 14:40:05 +00:00
|
|
|
if (IN_DEV_LOG_MARTIANS(in_dev)) {
|
|
|
|
const struct iphdr *iph = (const struct iphdr *) skb->data;
|
|
|
|
__be32 daddr = iph->daddr;
|
|
|
|
__be32 saddr = iph->saddr;
|
|
|
|
|
2012-07-12 03:55:47 +00:00
|
|
|
net_info_ratelimited("Redirect from %pI4 on %s about %pI4 ignored\n"
|
|
|
|
" Advised path = %pI4 -> %pI4\n",
|
|
|
|
&old_gw, dev->name, &new_gw,
|
|
|
|
&saddr, &daddr);
|
2012-07-12 14:40:05 +00:00
|
|
|
}
|
2012-07-12 03:55:47 +00:00
|
|
|
#endif
|
|
|
|
;
|
|
|
|
}
|
|
|
|
|
2012-07-17 11:19:00 +00:00
|
|
|
static void ip_do_redirect(struct dst_entry *dst, struct sock *sk, struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct rtable *rt;
|
|
|
|
struct flowi4 fl4;
|
2013-05-28 06:26:49 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *) skb->data;
|
2016-12-22 15:33:57 +00:00
|
|
|
struct net *net = dev_net(skb->dev);
|
2013-05-28 06:26:49 +00:00
|
|
|
int oif = skb->dev->ifindex;
|
|
|
|
u8 tos = RT_TOS(iph->tos);
|
|
|
|
u8 prot = iph->protocol;
|
|
|
|
u32 mark = skb->mark;
|
2012-07-17 11:19:00 +00:00
|
|
|
|
|
|
|
rt = (struct rtable *) dst;
|
|
|
|
|
2016-12-22 15:33:57 +00:00
|
|
|
__build_flow_key(net, &fl4, sk, iph, oif, tos, prot, mark, 0);
|
2012-07-17 18:31:28 +00:00
|
|
|
__ip_do_redirect(rt, skb, &fl4, true);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
static struct dst_entry *ipv4_negative_advice(struct dst_entry *dst)
|
|
|
|
{
|
2008-03-06 02:30:47 +00:00
|
|
|
struct rtable *rt = (struct rtable *)dst;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct dst_entry *ret = dst;
|
|
|
|
|
|
|
|
if (rt) {
|
2010-03-18 23:20:20 +00:00
|
|
|
if (dst->obsolete > 0) {
|
2005-04-16 22:20:36 +00:00
|
|
|
ip_rt_put(rt);
|
|
|
|
ret = NULL;
|
2012-07-10 13:58:42 +00:00
|
|
|
} else if ((rt->rt_flags & RTCF_REDIRECTED) ||
|
|
|
|
rt->dst.expires) {
|
2012-07-17 18:00:09 +00:00
|
|
|
ip_rt_put(rt);
|
2005-04-16 22:20:36 +00:00
|
|
|
ret = NULL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Algorithm:
|
|
|
|
* 1. The first ip_rt_redirect_number redirects are sent
|
|
|
|
* with exponential backoff, then we stop sending them at all,
|
|
|
|
* assuming that the host ignores our redirects.
|
|
|
|
* 2. If we did not see packets requiring redirects
|
|
|
|
* during ip_rt_redirect_silence, we assume that the host
|
|
|
|
* forgot redirected route and start to send redirects again.
|
|
|
|
*
|
|
|
|
* This algorithm is much cheaper and more intelligent than dumb load limiting
|
|
|
|
* in icmp.c.
|
|
|
|
*
|
|
|
|
* NOTE. Do not forget to inhibit load limiting for redirects (redundant)
|
|
|
|
* and "frag. need" (breaks PMTU discovery) in icmp.c.
|
|
|
|
*/
|
|
|
|
|
|
|
|
void ip_rt_send_redirect(struct sk_buff *skb)
|
|
|
|
{
|
2009-06-02 05:14:27 +00:00
|
|
|
struct rtable *rt = skb_rtable(skb);
|
2009-08-29 06:52:01 +00:00
|
|
|
struct in_device *in_dev;
|
2011-02-04 23:55:25 +00:00
|
|
|
struct inet_peer *peer;
|
2012-07-10 10:58:16 +00:00
|
|
|
struct net *net;
|
2009-08-29 06:52:01 +00:00
|
|
|
int log_martians;
|
2015-08-27 23:07:03 +00:00
|
|
|
int vif;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-08-29 06:52:01 +00:00
|
|
|
rcu_read_lock();
|
2010-06-11 06:31:35 +00:00
|
|
|
in_dev = __in_dev_get_rcu(rt->dst.dev);
|
2009-08-29 06:52:01 +00:00
|
|
|
if (!in_dev || !IN_DEV_TX_REDIRECTS(in_dev)) {
|
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
return;
|
2009-08-29 06:52:01 +00:00
|
|
|
}
|
|
|
|
log_martians = IN_DEV_LOG_MARTIANS(in_dev);
|
2015-09-30 03:07:13 +00:00
|
|
|
vif = l3mdev_master_ifindex_rcu(rt->dst.dev);
|
2009-08-29 06:52:01 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-07-10 10:58:16 +00:00
|
|
|
net = dev_net(rt->dst.dev);
|
2015-08-27 23:07:03 +00:00
|
|
|
peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr, vif, 1);
|
2011-02-04 23:55:25 +00:00
|
|
|
if (!peer) {
|
2012-10-08 11:41:15 +00:00
|
|
|
icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST,
|
|
|
|
rt_nexthop(rt, ip_hdr(skb)->daddr));
|
2011-02-04 23:55:25 +00:00
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* No redirected packets during ip_rt_redirect_silence;
|
|
|
|
* reset the algorithm.
|
|
|
|
*/
|
2019-02-06 18:18:04 +00:00
|
|
|
if (time_after(jiffies, peer->rate_last + ip_rt_redirect_silence)) {
|
2011-02-04 23:55:25 +00:00
|
|
|
peer->rate_tokens = 0;
|
2019-02-06 18:18:04 +00:00
|
|
|
peer->n_redirects = 0;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Too many ignored redirects; do not send anything
|
2010-06-11 06:31:35 +00:00
|
|
|
* set dst.rate_last to the last seen redirected packet.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2019-02-06 18:18:04 +00:00
|
|
|
if (peer->n_redirects >= ip_rt_redirect_number) {
|
2011-02-04 23:55:25 +00:00
|
|
|
peer->rate_last = jiffies;
|
2012-07-10 10:58:16 +00:00
|
|
|
goto out_put_peer;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* Check for load limit; set rate_last to the latest sent
|
|
|
|
* redirect.
|
|
|
|
*/
|
2020-05-08 17:28:34 +00:00
|
|
|
if (peer->n_redirects == 0 ||
|
2006-12-18 08:26:35 +00:00
|
|
|
time_after(jiffies,
|
2011-02-04 23:55:25 +00:00
|
|
|
(peer->rate_last +
|
2019-10-04 13:11:17 +00:00
|
|
|
(ip_rt_redirect_load << peer->n_redirects)))) {
|
2012-10-08 11:41:15 +00:00
|
|
|
__be32 gw = rt_nexthop(rt, ip_hdr(skb)->daddr);
|
|
|
|
|
|
|
|
icmp_send(skb, ICMP_REDIRECT, ICMP_REDIR_HOST, gw);
|
2011-02-04 23:55:25 +00:00
|
|
|
peer->rate_last = jiffies;
|
2019-02-06 18:18:04 +00:00
|
|
|
++peer->n_redirects;
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_VERBOSE
|
2009-08-29 06:52:01 +00:00
|
|
|
if (log_martians &&
|
2019-10-04 13:11:17 +00:00
|
|
|
peer->n_redirects == ip_rt_redirect_number)
|
2012-05-13 21:56:26 +00:00
|
|
|
net_warn_ratelimited("host %pI4/if%d ignores redirects for %pI4 to %pI4\n",
|
2012-07-23 23:29:00 +00:00
|
|
|
&ip_hdr(skb)->saddr, inet_iif(skb),
|
2012-10-08 11:41:15 +00:00
|
|
|
&ip_hdr(skb)->daddr, &gw);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
}
|
2012-07-10 10:58:16 +00:00
|
|
|
out_put_peer:
|
|
|
|
inet_putpeer(peer);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int ip_error(struct sk_buff *skb)
|
|
|
|
{
|
2009-06-02 05:14:27 +00:00
|
|
|
struct rtable *rt = skb_rtable(skb);
|
2018-02-28 17:20:44 +00:00
|
|
|
struct net_device *dev = skb->dev;
|
|
|
|
struct in_device *in_dev;
|
2011-02-04 23:55:25 +00:00
|
|
|
struct inet_peer *peer;
|
2005-04-16 22:20:36 +00:00
|
|
|
unsigned long now;
|
2012-06-26 23:27:09 +00:00
|
|
|
struct net *net;
|
2011-02-04 23:55:25 +00:00
|
|
|
bool send;
|
2005-04-16 22:20:36 +00:00
|
|
|
int code;
|
|
|
|
|
2018-02-28 17:20:44 +00:00
|
|
|
if (netif_is_l3_master(skb->dev)) {
|
|
|
|
dev = __dev_get_by_index(dev_net(skb->dev), IPCB(skb)->iif);
|
|
|
|
if (!dev)
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
in_dev = __in_dev_get_rcu(dev);
|
|
|
|
|
2015-05-22 09:58:12 +00:00
|
|
|
/* IP on this device is disabled. */
|
|
|
|
if (!in_dev)
|
|
|
|
goto out;
|
|
|
|
|
2012-06-26 23:27:09 +00:00
|
|
|
net = dev_net(rt->dst.dev);
|
|
|
|
if (!IN_DEV_FORWARD(in_dev)) {
|
|
|
|
switch (rt->dst.error) {
|
|
|
|
case EHOSTUNREACH:
|
2016-04-27 23:44:35 +00:00
|
|
|
__IP_INC_STATS(net, IPSTATS_MIB_INADDRERRORS);
|
2012-06-26 23:27:09 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
case ENETUNREACH:
|
2016-04-27 23:44:35 +00:00
|
|
|
__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
|
2012-06-26 23:27:09 +00:00
|
|
|
break;
|
|
|
|
}
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2010-06-11 06:31:35 +00:00
|
|
|
switch (rt->dst.error) {
|
2011-07-01 09:43:07 +00:00
|
|
|
case EINVAL:
|
|
|
|
default:
|
|
|
|
goto out;
|
|
|
|
case EHOSTUNREACH:
|
|
|
|
code = ICMP_HOST_UNREACH;
|
|
|
|
break;
|
|
|
|
case ENETUNREACH:
|
|
|
|
code = ICMP_NET_UNREACH;
|
2016-04-27 23:44:35 +00:00
|
|
|
__IP_INC_STATS(net, IPSTATS_MIB_INNOROUTES);
|
2011-07-01 09:43:07 +00:00
|
|
|
break;
|
|
|
|
case EACCES:
|
|
|
|
code = ICMP_PKT_FILTERED;
|
|
|
|
break;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2015-08-27 23:07:03 +00:00
|
|
|
peer = inet_getpeer_v4(net->ipv4.peers, ip_hdr(skb)->saddr,
|
2015-09-30 03:07:13 +00:00
|
|
|
l3mdev_master_ifindex(skb->dev), 1);
|
2011-02-04 23:55:25 +00:00
|
|
|
|
|
|
|
send = true;
|
|
|
|
if (peer) {
|
|
|
|
now = jiffies;
|
|
|
|
peer->rate_tokens += now - peer->rate_last;
|
|
|
|
if (peer->rate_tokens > ip_rt_error_burst)
|
|
|
|
peer->rate_tokens = ip_rt_error_burst;
|
|
|
|
peer->rate_last = now;
|
|
|
|
if (peer->rate_tokens >= ip_rt_error_cost)
|
|
|
|
peer->rate_tokens -= ip_rt_error_cost;
|
|
|
|
else
|
|
|
|
send = false;
|
2012-07-10 10:58:16 +00:00
|
|
|
inet_putpeer(peer);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2011-02-04 23:55:25 +00:00
|
|
|
if (send)
|
|
|
|
icmp_send(skb, ICMP_DEST_UNREACH, code, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
out: kfree_skb(skb);
|
|
|
|
return 0;
|
2007-02-09 14:24:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-10-07 22:47:25 +00:00
|
|
|
static void __ip_rt_update_pmtu(struct rtable *rt, struct flowi4 *fl4, u32 mtu)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-10-07 22:47:25 +00:00
|
|
|
struct dst_entry *dst = &rt->dst;
|
2020-09-15 03:03:54 +00:00
|
|
|
struct net *net = dev_net(dst->dev);
|
2012-07-17 11:19:00 +00:00
|
|
|
struct fib_result res;
|
2018-03-14 09:21:14 +00:00
|
|
|
bool lock = false;
|
2020-08-25 03:20:28 +00:00
|
|
|
u32 old_mtu;
|
2011-02-10 04:42:07 +00:00
|
|
|
|
2018-03-14 09:21:14 +00:00
|
|
|
if (ip_mtu_locked(dst))
|
2013-01-16 20:58:10 +00:00
|
|
|
return;
|
|
|
|
|
2020-08-25 03:20:28 +00:00
|
|
|
old_mtu = ipv4_mtu(dst);
|
net: ipv4: don't let PMTU updates increase route MTU
When an MTU update with PMTU smaller than net.ipv4.route.min_pmtu is
received, we must clamp its value. However, we can receive a PMTU
exception with PMTU < old_mtu < ip_rt_min_pmtu, which would lead to an
increase in PMTU.
To fix this, take the smallest of the old MTU and ip_rt_min_pmtu.
Before this patch, in case of an update, the exception's MTU would
always change. Now, an exception can have only its lock flag updated,
but not the MTU, so we need to add a check on locking to the following
"is this exception getting updated, or close to expiring?" test.
Fixes: d52e5a7e7ca4 ("ipv4: lock mtu in fnhe when received PMTU < net.ipv4.route.min_pmtu")
Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-10-09 15:48:15 +00:00
|
|
|
if (old_mtu < mtu)
|
2015-01-29 08:09:03 +00:00
|
|
|
return;
|
|
|
|
|
2018-03-14 09:21:14 +00:00
|
|
|
if (mtu < ip_rt_min_pmtu) {
|
|
|
|
lock = true;
|
net: ipv4: don't let PMTU updates increase route MTU
When an MTU update with PMTU smaller than net.ipv4.route.min_pmtu is
received, we must clamp its value. However, we can receive a PMTU
exception with PMTU < old_mtu < ip_rt_min_pmtu, which would lead to an
increase in PMTU.
To fix this, take the smallest of the old MTU and ip_rt_min_pmtu.
Before this patch, in case of an update, the exception's MTU would
always change. Now, an exception can have only its lock flag updated,
but not the MTU, so we need to add a check on locking to the following
"is this exception getting updated, or close to expiring?" test.
Fixes: d52e5a7e7ca4 ("ipv4: lock mtu in fnhe when received PMTU < net.ipv4.route.min_pmtu")
Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-10-09 15:48:15 +00:00
|
|
|
mtu = min(old_mtu, ip_rt_min_pmtu);
|
2018-03-14 09:21:14 +00:00
|
|
|
}
|
2011-02-10 04:42:07 +00:00
|
|
|
|
net: ipv4: don't let PMTU updates increase route MTU
When an MTU update with PMTU smaller than net.ipv4.route.min_pmtu is
received, we must clamp its value. However, we can receive a PMTU
exception with PMTU < old_mtu < ip_rt_min_pmtu, which would lead to an
increase in PMTU.
To fix this, take the smallest of the old MTU and ip_rt_min_pmtu.
Before this patch, in case of an update, the exception's MTU would
always change. Now, an exception can have only its lock flag updated,
but not the MTU, so we need to add a check on locking to the following
"is this exception getting updated, or close to expiring?" test.
Fixes: d52e5a7e7ca4 ("ipv4: lock mtu in fnhe when received PMTU < net.ipv4.route.min_pmtu")
Signed-off-by: Sabrina Dubroca <sd@queasysnail.net>
Reviewed-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2018-10-09 15:48:15 +00:00
|
|
|
if (rt->rt_pmtu == mtu && !lock &&
|
2013-05-27 20:46:32 +00:00
|
|
|
time_before(jiffies, dst->expires - ip_rt_mtu_expires / 2))
|
|
|
|
return;
|
|
|
|
|
2012-08-28 12:33:07 +00:00
|
|
|
rcu_read_lock();
|
2020-09-15 03:03:54 +00:00
|
|
|
if (fib_lookup(net, fl4, &res, 0) == 0) {
|
|
|
|
struct fib_nh_common *nhc;
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2020-09-15 03:03:54 +00:00
|
|
|
fib_select_path(net, &res, fl4, NULL);
|
|
|
|
nhc = FIB_RES_NHC(res);
|
2019-04-30 14:45:50 +00:00
|
|
|
update_or_create_fnhe(nhc, fl4->daddr, 0, mtu, lock,
|
2012-07-18 10:15:35 +00:00
|
|
|
jiffies + ip_rt_mtu_expires);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
2012-08-28 12:33:07 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 11:19:00 +00:00
|
|
|
static void ip_rt_update_pmtu(struct dst_entry *dst, struct sock *sk,
|
2019-12-22 02:51:09 +00:00
|
|
|
struct sk_buff *skb, u32 mtu,
|
|
|
|
bool confirm_neigh)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
|
|
|
struct rtable *rt = (struct rtable *) dst;
|
|
|
|
struct flowi4 fl4;
|
|
|
|
|
|
|
|
ip_rt_build_flow_key(&fl4, sk, skb);
|
ipv4: route: Ignore output interface in FIB lookup for PMTU route
Currently, processes sending traffic to a local bridge with an
encapsulation device as a port don't get ICMP errors if they exceed
the PMTU of the encapsulated link.
David Ahern suggested this as a hack, but it actually looks like
the correct solution: when we update the PMTU for a given destination
by means of updating or creating a route exception, the encapsulation
might trigger this because of PMTU discovery happening either on the
encapsulation device itself, or its lower layer. This happens on
bridged encapsulations only.
The output interface shouldn't matter, because we already have a
valid destination. Drop the output interface restriction from the
associated route lookup.
For UDP tunnels, we will now have a route exception created for the
encapsulation itself, with a MTU value reflecting its headroom, which
allows a bridge forwarding IP packets originated locally to deliver
errors back to the sending socket.
The behaviour is now consistent with IPv6 and verified with selftests
pmtu_ipv{4,6}_br_{geneve,vxlan}{4,6}_exception introduced later in
this series.
v2:
- reset output interface only for bridge ports (David Ahern)
- add and use netif_is_any_bridge_port() helper (David Ahern)
Suggested-by: David Ahern <dsahern@gmail.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-08-04 05:53:42 +00:00
|
|
|
|
|
|
|
/* Don't make lookup fail for bridged encapsulations */
|
|
|
|
if (skb && netif_is_any_bridge_port(skb->dev))
|
|
|
|
fl4.flowi4_oif = 0;
|
|
|
|
|
2012-10-07 22:47:25 +00:00
|
|
|
__ip_rt_update_pmtu(rt, &fl4, mtu);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
|
2012-06-15 05:21:46 +00:00
|
|
|
void ipv4_update_pmtu(struct sk_buff *skb, struct net *net, u32 mtu,
|
2018-09-26 03:56:26 +00:00
|
|
|
int oif, u8 protocol)
|
2012-06-15 05:21:46 +00:00
|
|
|
{
|
2020-08-29 09:21:30 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2012-06-15 05:21:46 +00:00
|
|
|
struct flowi4 fl4;
|
|
|
|
struct rtable *rt;
|
2018-09-26 03:56:26 +00:00
|
|
|
u32 mark = IP4_REPLY_MARK(net, skb->mark);
|
2014-05-13 17:17:34 +00:00
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
__build_flow_key(net, &fl4, NULL, iph, oif,
|
2018-09-26 03:56:26 +00:00
|
|
|
RT_TOS(iph->tos), protocol, mark, 0);
|
2012-06-15 05:21:46 +00:00
|
|
|
rt = __ip_route_output_key(net, &fl4);
|
|
|
|
if (!IS_ERR(rt)) {
|
2012-07-17 11:19:00 +00:00
|
|
|
__ip_rt_update_pmtu(rt, &fl4, mtu);
|
2012-06-15 05:21:46 +00:00
|
|
|
ip_rt_put(rt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(ipv4_update_pmtu);
|
|
|
|
|
2013-01-21 01:59:11 +00:00
|
|
|
static void __ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
|
2012-06-15 05:21:46 +00:00
|
|
|
{
|
2020-08-25 12:32:11 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2012-07-17 11:19:00 +00:00
|
|
|
struct flowi4 fl4;
|
|
|
|
struct rtable *rt;
|
2012-06-15 05:21:46 +00:00
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
__build_flow_key(sock_net(sk), &fl4, sk, iph, 0, 0, 0, 0, 0);
|
2014-05-13 17:17:34 +00:00
|
|
|
|
|
|
|
if (!fl4.flowi4_mark)
|
|
|
|
fl4.flowi4_mark = IP4_REPLY_MARK(sock_net(sk), skb->mark);
|
|
|
|
|
2012-07-17 11:19:00 +00:00
|
|
|
rt = __ip_route_output_key(sock_net(sk), &fl4);
|
|
|
|
if (!IS_ERR(rt)) {
|
|
|
|
__ip_rt_update_pmtu(rt, &fl4, mtu);
|
|
|
|
ip_rt_put(rt);
|
|
|
|
}
|
2012-06-15 05:21:46 +00:00
|
|
|
}
|
2013-01-21 01:59:11 +00:00
|
|
|
|
|
|
|
void ipv4_sk_update_pmtu(struct sk_buff *skb, struct sock *sk, u32 mtu)
|
|
|
|
{
|
2020-08-29 09:21:30 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2013-01-21 01:59:11 +00:00
|
|
|
struct flowi4 fl4;
|
|
|
|
struct rtable *rt;
|
2014-06-30 08:26:23 +00:00
|
|
|
struct dst_entry *odst = NULL;
|
2013-01-22 00:01:28 +00:00
|
|
|
bool new = false;
|
2016-11-03 17:23:43 +00:00
|
|
|
struct net *net = sock_net(sk);
|
2013-01-21 01:59:11 +00:00
|
|
|
|
|
|
|
bh_lock_sock(sk);
|
2013-11-05 01:24:17 +00:00
|
|
|
|
|
|
|
if (!ip_sk_accept_pmtu(sk))
|
|
|
|
goto out;
|
|
|
|
|
2014-06-30 08:26:23 +00:00
|
|
|
odst = sk_dst_get(sk);
|
2013-01-21 01:59:11 +00:00
|
|
|
|
2014-06-30 08:26:23 +00:00
|
|
|
if (sock_owned_by_user(sk) || !odst) {
|
2013-01-21 01:59:11 +00:00
|
|
|
__ipv4_sk_update_pmtu(skb, sk, mtu);
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
|
2013-01-21 01:59:11 +00:00
|
|
|
|
2014-06-30 08:26:23 +00:00
|
|
|
rt = (struct rtable *)odst;
|
2015-04-03 08:17:26 +00:00
|
|
|
if (odst->obsolete && !odst->ops->check(odst, 0)) {
|
2013-01-21 01:59:11 +00:00
|
|
|
rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
|
|
|
|
if (IS_ERR(rt))
|
|
|
|
goto out;
|
2013-01-22 00:01:28 +00:00
|
|
|
|
|
|
|
new = true;
|
2013-01-21 01:59:11 +00:00
|
|
|
}
|
|
|
|
|
2020-08-25 12:32:11 +00:00
|
|
|
__ip_rt_update_pmtu((struct rtable *)xfrm_dst_path(&rt->dst), &fl4, mtu);
|
2013-01-21 01:59:11 +00:00
|
|
|
|
2014-06-30 08:26:23 +00:00
|
|
|
if (!dst_check(&rt->dst, 0)) {
|
2013-01-22 00:01:28 +00:00
|
|
|
if (new)
|
|
|
|
dst_release(&rt->dst);
|
|
|
|
|
2013-01-21 01:59:11 +00:00
|
|
|
rt = ip_route_output_flow(sock_net(sk), &fl4, sk);
|
|
|
|
if (IS_ERR(rt))
|
|
|
|
goto out;
|
|
|
|
|
2013-01-22 00:01:28 +00:00
|
|
|
new = true;
|
2013-01-21 01:59:11 +00:00
|
|
|
}
|
|
|
|
|
2013-01-22 00:01:28 +00:00
|
|
|
if (new)
|
2014-06-30 08:26:23 +00:00
|
|
|
sk_dst_set(sk, &rt->dst);
|
2013-01-21 01:59:11 +00:00
|
|
|
|
|
|
|
out:
|
|
|
|
bh_unlock_sock(sk);
|
2014-06-30 08:26:23 +00:00
|
|
|
dst_release(odst);
|
2013-01-21 01:59:11 +00:00
|
|
|
}
|
2012-06-15 05:21:46 +00:00
|
|
|
EXPORT_SYMBOL_GPL(ipv4_sk_update_pmtu);
|
2011-02-10 06:00:16 +00:00
|
|
|
|
2012-07-12 04:25:45 +00:00
|
|
|
void ipv4_redirect(struct sk_buff *skb, struct net *net,
|
2018-09-26 03:56:27 +00:00
|
|
|
int oif, u8 protocol)
|
2012-07-12 04:25:45 +00:00
|
|
|
{
|
2020-08-29 09:21:30 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2012-07-12 04:25:45 +00:00
|
|
|
struct flowi4 fl4;
|
|
|
|
struct rtable *rt;
|
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
__build_flow_key(net, &fl4, NULL, iph, oif,
|
2018-09-26 03:56:27 +00:00
|
|
|
RT_TOS(iph->tos), protocol, 0, 0);
|
2012-07-12 04:25:45 +00:00
|
|
|
rt = __ip_route_output_key(net, &fl4);
|
|
|
|
if (!IS_ERR(rt)) {
|
2012-07-17 18:31:28 +00:00
|
|
|
__ip_do_redirect(rt, skb, &fl4, false);
|
2012-07-12 04:25:45 +00:00
|
|
|
ip_rt_put(rt);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(ipv4_redirect);
|
|
|
|
|
|
|
|
void ipv4_sk_redirect(struct sk_buff *skb, struct sock *sk)
|
|
|
|
{
|
2020-08-25 12:32:11 +00:00
|
|
|
const struct iphdr *iph = (const struct iphdr *)skb->data;
|
2012-07-17 11:19:00 +00:00
|
|
|
struct flowi4 fl4;
|
|
|
|
struct rtable *rt;
|
2016-11-03 17:23:43 +00:00
|
|
|
struct net *net = sock_net(sk);
|
2012-07-12 04:25:45 +00:00
|
|
|
|
2016-11-03 17:23:43 +00:00
|
|
|
__build_flow_key(net, &fl4, sk, iph, 0, 0, 0, 0, 0);
|
|
|
|
rt = __ip_route_output_key(net, &fl4);
|
2012-07-17 11:19:00 +00:00
|
|
|
if (!IS_ERR(rt)) {
|
2012-07-17 18:31:28 +00:00
|
|
|
__ip_do_redirect(rt, skb, &fl4, false);
|
2012-07-17 11:19:00 +00:00
|
|
|
ip_rt_put(rt);
|
|
|
|
}
|
2012-07-12 04:25:45 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(ipv4_sk_redirect);
|
|
|
|
|
2021-02-01 17:41:32 +00:00
|
|
|
INDIRECT_CALLABLE_SCOPE struct dst_entry *ipv4_dst_check(struct dst_entry *dst,
|
|
|
|
u32 cookie)
|
2011-12-01 18:38:59 +00:00
|
|
|
{
|
|
|
|
struct rtable *rt = (struct rtable *) dst;
|
|
|
|
|
2012-07-17 18:31:28 +00:00
|
|
|
/* All IPV4 dsts are created with ->obsolete set to the value
|
|
|
|
* DST_OBSOLETE_FORCE_CHK which forces validation calls down
|
|
|
|
* into this function always.
|
|
|
|
*
|
2013-05-27 20:46:31 +00:00
|
|
|
* When a PMTU/redirect information update invalidates a route,
|
|
|
|
* this is indicated by setting obsolete to DST_OBSOLETE_KILL or
|
2019-03-20 19:02:56 +00:00
|
|
|
* DST_OBSOLETE_DEAD.
|
2012-07-17 18:31:28 +00:00
|
|
|
*/
|
2013-05-27 20:46:31 +00:00
|
|
|
if (dst->obsolete != DST_OBSOLETE_FORCE_CHK || rt_is_expired(rt))
|
2011-12-01 18:38:59 +00:00
|
|
|
return NULL;
|
2010-03-18 23:20:20 +00:00
|
|
|
return dst;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2021-02-04 18:18:39 +00:00
|
|
|
EXPORT_INDIRECT_CALLABLE(ipv4_dst_check);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-04-24 15:04:05 +00:00
|
|
|
static void ipv4_send_dest_unreach(struct sk_buff *skb)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2019-04-12 20:19:27 +00:00
|
|
|
struct ip_options opt;
|
2019-04-14 00:32:21 +00:00
|
|
|
int res;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-04-12 20:19:27 +00:00
|
|
|
/* Recompile ip options since IPCB may not be valid anymore.
|
2019-04-24 15:04:05 +00:00
|
|
|
* Also check we have a reasonable ipv4 header.
|
2019-04-12 20:19:27 +00:00
|
|
|
*/
|
2019-04-24 15:04:05 +00:00
|
|
|
if (!pskb_network_may_pull(skb, sizeof(struct iphdr)) ||
|
|
|
|
ip_hdr(skb)->version != 4 || ip_hdr(skb)->ihl < 5)
|
|
|
|
return;
|
2019-04-14 00:32:21 +00:00
|
|
|
|
2019-04-24 15:04:05 +00:00
|
|
|
memset(&opt, 0, sizeof(opt));
|
|
|
|
if (ip_hdr(skb)->ihl > 5) {
|
|
|
|
if (!pskb_network_may_pull(skb, ip_hdr(skb)->ihl * 4))
|
|
|
|
return;
|
|
|
|
opt.optlen = ip_hdr(skb)->ihl * 4 - sizeof(struct iphdr);
|
2019-04-14 00:32:21 +00:00
|
|
|
|
2019-04-24 15:04:05 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
res = __ip_options_compile(dev_net(skb->dev), &opt, skb, NULL);
|
|
|
|
rcu_read_unlock();
|
2019-04-12 20:19:27 +00:00
|
|
|
|
2019-04-24 15:04:05 +00:00
|
|
|
if (res)
|
|
|
|
return;
|
|
|
|
}
|
2019-04-12 20:19:27 +00:00
|
|
|
__icmp_send(skb, ICMP_DEST_UNREACH, ICMP_HOST_UNREACH, 0, &opt);
|
2019-04-24 15:04:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void ipv4_link_failure(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct rtable *rt;
|
|
|
|
|
|
|
|
ipv4_send_dest_unreach(skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2009-06-02 05:14:27 +00:00
|
|
|
rt = skb_rtable(skb);
|
2012-07-10 13:58:42 +00:00
|
|
|
if (rt)
|
|
|
|
dst_set_expires(&rt->dst, 0);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2015-10-07 21:48:47 +00:00
|
|
|
static int ip_rt_bug(struct net *net, struct sock *sk, struct sk_buff *skb)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-05-15 14:11:54 +00:00
|
|
|
pr_debug("%s: %pI4 -> %pI4, %s\n",
|
|
|
|
__func__, &ip_hdr(skb)->saddr, &ip_hdr(skb)->daddr,
|
|
|
|
skb->dev ? skb->dev->name : "?");
|
2005-04-16 22:20:36 +00:00
|
|
|
kfree_skb(skb);
|
2011-05-21 07:16:42 +00:00
|
|
|
WARN_ON(1);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2021-03-12 07:30:05 +00:00
|
|
|
* We do not cache source address of outgoing interface,
|
|
|
|
* because it is used only by IP RR, TS and SRR options,
|
|
|
|
* so that it out of fast path.
|
|
|
|
*
|
|
|
|
* BTW remember: "addr" is allowed to be not aligned
|
|
|
|
* in IP options!
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2011-05-13 21:29:41 +00:00
|
|
|
void ip_rt_get_source(u8 *addr, struct sk_buff *skb, struct rtable *rt)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2006-09-27 04:27:54 +00:00
|
|
|
__be32 src;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-11-12 01:07:48 +00:00
|
|
|
if (rt_is_output_route(rt))
|
2011-05-13 22:01:21 +00:00
|
|
|
src = ip_hdr(skb)->saddr;
|
2010-10-05 10:41:36 +00:00
|
|
|
else {
|
2011-05-13 21:29:41 +00:00
|
|
|
struct fib_result res;
|
2018-09-30 06:44:46 +00:00
|
|
|
struct iphdr *iph = ip_hdr(skb);
|
|
|
|
struct flowi4 fl4 = {
|
|
|
|
.daddr = iph->daddr,
|
|
|
|
.saddr = iph->saddr,
|
|
|
|
.flowi4_tos = RT_TOS(iph->tos),
|
|
|
|
.flowi4_oif = rt->dst.dev->ifindex,
|
|
|
|
.flowi4_iif = skb->dev->ifindex,
|
|
|
|
.flowi4_mark = skb->mark,
|
|
|
|
};
|
2011-03-05 05:47:09 +00:00
|
|
|
|
2010-10-05 10:41:36 +00:00
|
|
|
rcu_read_lock();
|
2015-06-23 17:45:37 +00:00
|
|
|
if (fib_lookup(dev_net(rt->dst.dev), &fl4, &res, 0) == 0)
|
2019-04-02 21:11:55 +00:00
|
|
|
src = fib_result_prefsrc(dev_net(rt->dst.dev), &res);
|
2010-10-05 10:41:36 +00:00
|
|
|
else
|
2012-07-13 12:03:45 +00:00
|
|
|
src = inet_select_addr(rt->dst.dev,
|
|
|
|
rt_nexthop(rt, iph->daddr),
|
|
|
|
RT_SCOPE_UNIVERSE);
|
2010-10-05 10:41:36 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
memcpy(addr, &src, 4);
|
|
|
|
}
|
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2005-04-16 22:20:36 +00:00
|
|
|
static void set_class_tag(struct rtable *rt, u32 tag)
|
|
|
|
{
|
2010-06-11 06:31:35 +00:00
|
|
|
if (!(rt->dst.tclassid & 0xFFFF))
|
|
|
|
rt->dst.tclassid |= tag & 0xFFFF;
|
|
|
|
if (!(rt->dst.tclassid & 0xFFFF0000))
|
|
|
|
rt->dst.tclassid |= tag & 0xFFFF0000;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
#endif
|
|
|
|
|
2010-12-13 20:52:14 +00:00
|
|
|
static unsigned int ipv4_default_advmss(const struct dst_entry *dst)
|
|
|
|
{
|
2017-04-12 04:34:03 +00:00
|
|
|
unsigned int header_size = sizeof(struct tcphdr) + sizeof(struct iphdr);
|
2017-10-19 00:02:03 +00:00
|
|
|
unsigned int advmss = max_t(unsigned int, ipv4_mtu(dst) - header_size,
|
2017-04-12 04:34:03 +00:00
|
|
|
ip_rt_min_advmss);
|
2010-12-13 20:52:14 +00:00
|
|
|
|
2017-04-12 04:34:03 +00:00
|
|
|
return min(advmss, IPV4_MAX_PMTU - header_size);
|
2010-12-13 20:52:14 +00:00
|
|
|
}
|
|
|
|
|
2021-02-01 17:41:31 +00:00
|
|
|
INDIRECT_CALLABLE_SCOPE unsigned int ipv4_mtu(const struct dst_entry *dst)
|
2010-12-14 21:01:14 +00:00
|
|
|
{
|
2021-07-20 20:06:28 +00:00
|
|
|
return ip_dst_mtu_maybe_forward(dst, false);
|
2010-12-14 21:01:14 +00:00
|
|
|
}
|
2021-02-04 18:18:39 +00:00
|
|
|
EXPORT_INDIRECT_CALLABLE(ipv4_mtu);
|
2010-12-14 21:01:14 +00:00
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
static void ip_del_fnhe(struct fib_nh_common *nhc, __be32 daddr)
|
2018-05-02 06:41:19 +00:00
|
|
|
{
|
|
|
|
struct fnhe_hash_bucket *hash;
|
|
|
|
struct fib_nh_exception *fnhe, __rcu **fnhe_p;
|
|
|
|
u32 hval = fnhe_hashfun(daddr);
|
|
|
|
|
|
|
|
spin_lock_bh(&fnhe_lock);
|
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
hash = rcu_dereference_protected(nhc->nhc_exceptions,
|
2018-05-02 06:41:19 +00:00
|
|
|
lockdep_is_held(&fnhe_lock));
|
|
|
|
hash += hval;
|
|
|
|
|
|
|
|
fnhe_p = &hash->chain;
|
|
|
|
fnhe = rcu_dereference_protected(*fnhe_p, lockdep_is_held(&fnhe_lock));
|
|
|
|
while (fnhe) {
|
|
|
|
if (fnhe->fnhe_daddr == daddr) {
|
|
|
|
rcu_assign_pointer(*fnhe_p, rcu_dereference_protected(
|
|
|
|
fnhe->fnhe_next, lockdep_is_held(&fnhe_lock)));
|
2019-03-08 06:50:54 +00:00
|
|
|
/* set fnhe_daddr to 0 to ensure it won't bind with
|
|
|
|
* new dsts in rt_bind_exception().
|
|
|
|
*/
|
|
|
|
fnhe->fnhe_daddr = 0;
|
2018-05-02 06:41:19 +00:00
|
|
|
fnhe_flush_routes(fnhe);
|
|
|
|
kfree_rcu(fnhe, rcu);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
fnhe_p = &fnhe->fnhe_next;
|
|
|
|
fnhe = rcu_dereference_protected(fnhe->fnhe_next,
|
|
|
|
lockdep_is_held(&fnhe_lock));
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_bh(&fnhe_lock);
|
|
|
|
}
|
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
static struct fib_nh_exception *find_exception(struct fib_nh_common *nhc,
|
|
|
|
__be32 daddr)
|
2012-07-17 11:19:00 +00:00
|
|
|
{
|
2019-04-30 14:45:50 +00:00
|
|
|
struct fnhe_hash_bucket *hash = rcu_dereference(nhc->nhc_exceptions);
|
2012-07-17 11:19:00 +00:00
|
|
|
struct fib_nh_exception *fnhe;
|
|
|
|
u32 hval;
|
|
|
|
|
2012-07-17 19:20:47 +00:00
|
|
|
if (!hash)
|
|
|
|
return NULL;
|
|
|
|
|
2012-07-17 20:23:08 +00:00
|
|
|
hval = fnhe_hashfun(daddr);
|
2012-07-17 11:19:00 +00:00
|
|
|
|
|
|
|
for (fnhe = rcu_dereference(hash[hval].chain); fnhe;
|
|
|
|
fnhe = rcu_dereference(fnhe->fnhe_next)) {
|
2018-05-02 06:41:19 +00:00
|
|
|
if (fnhe->fnhe_daddr == daddr) {
|
|
|
|
if (fnhe->fnhe_expires &&
|
|
|
|
time_after(jiffies, fnhe->fnhe_expires)) {
|
2019-04-30 14:45:50 +00:00
|
|
|
ip_del_fnhe(nhc, daddr);
|
2018-05-02 06:41:19 +00:00
|
|
|
break;
|
|
|
|
}
|
2012-07-17 19:20:47 +00:00
|
|
|
return fnhe;
|
2018-05-02 06:41:19 +00:00
|
|
|
}
|
2012-07-17 19:20:47 +00:00
|
|
|
}
|
|
|
|
return NULL;
|
|
|
|
}
|
2012-07-18 10:15:35 +00:00
|
|
|
|
2018-05-21 16:08:13 +00:00
|
|
|
/* MTU selection:
|
|
|
|
* 1. mtu on route is locked - use it
|
|
|
|
* 2. mtu from nexthop exception
|
|
|
|
* 3. mtu from egress device
|
|
|
|
*/
|
|
|
|
|
|
|
|
u32 ip_mtu_from_fib_result(struct fib_result *res, __be32 daddr)
|
|
|
|
{
|
2019-04-02 21:11:55 +00:00
|
|
|
struct fib_nh_common *nhc = res->nhc;
|
|
|
|
struct net_device *dev = nhc->nhc_dev;
|
2018-05-21 16:08:13 +00:00
|
|
|
struct fib_info *fi = res->fi;
|
|
|
|
u32 mtu = 0;
|
|
|
|
|
|
|
|
if (dev_net(dev)->ipv4.sysctl_ip_fwd_use_pmtu ||
|
|
|
|
fi->fib_metrics->metrics[RTAX_LOCK - 1] & (1 << RTAX_MTU))
|
|
|
|
mtu = fi->fib_mtu;
|
|
|
|
|
|
|
|
if (likely(!mtu)) {
|
|
|
|
struct fib_nh_exception *fnhe;
|
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
fnhe = find_exception(nhc, daddr);
|
2018-05-21 16:08:13 +00:00
|
|
|
if (fnhe && !time_after_eq(jiffies, fnhe->fnhe_expires))
|
|
|
|
mtu = fnhe->fnhe_pmtu;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (likely(!mtu))
|
|
|
|
mtu = min(READ_ONCE(dev->mtu), IP_MAX_MTU);
|
|
|
|
|
2019-04-02 21:11:55 +00:00
|
|
|
return mtu - lwtunnel_headroom(nhc->nhc_lwtstate, mtu);
|
2018-05-21 16:08:13 +00:00
|
|
|
}
|
|
|
|
|
2012-07-31 22:06:50 +00:00
|
|
|
static bool rt_bind_exception(struct rtable *rt, struct fib_nh_exception *fnhe,
|
2017-06-17 17:42:42 +00:00
|
|
|
__be32 daddr, const bool do_cache)
|
2012-07-17 19:20:47 +00:00
|
|
|
{
|
2012-07-31 22:06:50 +00:00
|
|
|
bool ret = false;
|
|
|
|
|
2012-07-31 22:02:02 +00:00
|
|
|
spin_lock_bh(&fnhe_lock);
|
2012-07-17 19:20:47 +00:00
|
|
|
|
2012-07-31 22:02:02 +00:00
|
|
|
if (daddr == fnhe->fnhe_daddr) {
|
2013-06-27 07:27:05 +00:00
|
|
|
struct rtable __rcu **porig;
|
|
|
|
struct rtable *orig;
|
2013-05-27 20:46:33 +00:00
|
|
|
int genid = fnhe_genid(dev_net(rt->dst.dev));
|
2013-06-27 07:27:05 +00:00
|
|
|
|
|
|
|
if (rt_is_input_route(rt))
|
|
|
|
porig = &fnhe->fnhe_rth_input;
|
|
|
|
else
|
|
|
|
porig = &fnhe->fnhe_rth_output;
|
|
|
|
orig = rcu_dereference(*porig);
|
2013-05-27 20:46:33 +00:00
|
|
|
|
|
|
|
if (fnhe->fnhe_genid != genid) {
|
|
|
|
fnhe->fnhe_genid = genid;
|
2012-10-17 21:17:44 +00:00
|
|
|
fnhe->fnhe_gw = 0;
|
|
|
|
fnhe->fnhe_pmtu = 0;
|
|
|
|
fnhe->fnhe_expires = 0;
|
2018-05-09 10:06:44 +00:00
|
|
|
fnhe->fnhe_mtu_locked = false;
|
2013-06-27 07:27:05 +00:00
|
|
|
fnhe_flush_routes(fnhe);
|
|
|
|
orig = NULL;
|
2012-10-17 21:17:44 +00:00
|
|
|
}
|
2013-05-27 20:46:31 +00:00
|
|
|
fill_route_from_fnhe(rt, fnhe);
|
2019-04-05 23:30:27 +00:00
|
|
|
if (!rt->rt_gw4) {
|
|
|
|
rt->rt_gw4 = daddr;
|
|
|
|
rt->rt_gw_family = AF_INET;
|
|
|
|
}
|
2012-07-17 19:20:47 +00:00
|
|
|
|
2017-06-17 17:42:42 +00:00
|
|
|
if (do_cache) {
|
2017-06-17 17:42:29 +00:00
|
|
|
dst_hold(&rt->dst);
|
2013-06-27 07:27:05 +00:00
|
|
|
rcu_assign_pointer(*porig, rt);
|
2017-06-17 17:42:29 +00:00
|
|
|
if (orig) {
|
2017-06-17 17:42:30 +00:00
|
|
|
dst_dev_put(&orig->dst);
|
2017-06-17 17:42:29 +00:00
|
|
|
dst_release(&orig->dst);
|
|
|
|
}
|
2013-06-27 07:27:05 +00:00
|
|
|
ret = true;
|
|
|
|
}
|
2012-07-31 22:02:02 +00:00
|
|
|
|
|
|
|
fnhe->fnhe_stamp = jiffies;
|
|
|
|
}
|
|
|
|
spin_unlock_bh(&fnhe_lock);
|
2012-07-31 22:06:50 +00:00
|
|
|
|
|
|
|
return ret;
|
2012-07-31 01:08:23 +00:00
|
|
|
}
|
|
|
|
|
2019-04-30 14:45:49 +00:00
|
|
|
static bool rt_cache_route(struct fib_nh_common *nhc, struct rtable *rt)
|
2012-07-17 19:20:47 +00:00
|
|
|
{
|
2012-07-31 05:45:30 +00:00
|
|
|
struct rtable *orig, *prev, **p;
|
2012-07-31 22:06:50 +00:00
|
|
|
bool ret = true;
|
2012-07-17 19:20:47 +00:00
|
|
|
|
2012-07-31 05:45:30 +00:00
|
|
|
if (rt_is_input_route(rt)) {
|
2019-04-30 14:45:48 +00:00
|
|
|
p = (struct rtable **)&nhc->nhc_rth_input;
|
2012-07-31 05:45:30 +00:00
|
|
|
} else {
|
2019-04-30 14:45:48 +00:00
|
|
|
p = (struct rtable **)raw_cpu_ptr(nhc->nhc_pcpu_rth_output);
|
2012-07-31 05:45:30 +00:00
|
|
|
}
|
2012-07-17 19:20:47 +00:00
|
|
|
orig = *p;
|
|
|
|
|
2017-06-17 17:42:29 +00:00
|
|
|
/* hold dst before doing cmpxchg() to avoid race condition
|
|
|
|
* on this dst
|
|
|
|
*/
|
|
|
|
dst_hold(&rt->dst);
|
2012-07-17 19:20:47 +00:00
|
|
|
prev = cmpxchg(p, orig, rt);
|
|
|
|
if (prev == orig) {
|
2017-06-17 17:42:29 +00:00
|
|
|
if (orig) {
|
ipv4: fix race condition between route lookup and invalidation
Jesse and Ido reported the following race condition:
<CPU A, t0> - Received packet A is forwarded and cached dst entry is
taken from the nexthop ('nhc->nhc_rth_input'). Calls skb_dst_set()
<t1> - Given Jesse has busy routers ("ingesting full BGP routing tables
from multiple ISPs"), route is added / deleted and rt_cache_flush() is
called
<CPU B, t2> - Received packet B tries to use the same cached dst entry
from t0, but rt_cache_valid() is no longer true and it is replaced in
rt_cache_route() by the newer one. This calls dst_dev_put() on the
original dst entry which assigns the blackhole netdev to 'dst->dev'
<CPU A, t3> - dst_input(skb) is called on packet A and it is dropped due
to 'dst->dev' being the blackhole netdev
There are 2 issues in the v4 routing code:
1. A per-netns counter is used to do the validation of the route. That
means whenever a route is changed in the netns, users of all routes in
the netns needs to redo lookup. v6 has an implementation of only
updating fn_sernum for routes that are affected.
2. When rt_cache_valid() returns false, rt_cache_route() is called to
throw away the current cache, and create a new one. This seems
unnecessary because as long as this route does not change, the route
cache does not need to be recreated.
To fully solve the above 2 issues, it probably needs quite some code
changes and requires careful testing, and does not suite for net branch.
So this patch only tries to add the deleted cached rt into the uncached
list, so user could still be able to use it to receive packets until
it's done.
Fixes: 95c47f9cf5e0 ("ipv4: call dst_dev_put() properly")
Signed-off-by: Wei Wang <weiwan@google.com>
Reported-by: Ido Schimmel <idosch@idosch.org>
Reported-by: Jesse Hathaway <jesse@mbuki-mvuki.org>
Tested-by: Jesse Hathaway <jesse@mbuki-mvuki.org>
Acked-by: Martin KaFai Lau <kafai@fb.com>
Cc: David Ahern <dsahern@gmail.com>
Reviewed-by: Ido Schimmel <idosch@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-16 19:03:15 +00:00
|
|
|
rt_add_uncached_list(orig);
|
2017-06-17 17:42:29 +00:00
|
|
|
dst_release(&orig->dst);
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
dst_release(&rt->dst);
|
2012-07-31 22:06:50 +00:00
|
|
|
ret = false;
|
2017-06-17 17:42:29 +00:00
|
|
|
}
|
2012-07-31 22:06:50 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2015-01-14 23:17:06 +00:00
|
|
|
struct uncached_list {
|
|
|
|
spinlock_t lock;
|
|
|
|
struct list_head head;
|
|
|
|
};
|
|
|
|
|
|
|
|
static DEFINE_PER_CPU_ALIGNED(struct uncached_list, rt_uncached_list);
|
2012-07-31 22:06:50 +00:00
|
|
|
|
xfrm: reuse uncached_list to track xdsts
In early time, when freeing a xdst, it would be inserted into
dst_garbage.list first. Then if it's refcnt was still held
somewhere, later it would be put into dst_busy_list in
dst_gc_task().
When one dev was being unregistered, the dev of these dsts in
dst_busy_list would be set with loopback_dev and put this dev.
So that this dev's removal wouldn't get blocked, and avoid the
kmsg warning:
kernel:unregister_netdevice: waiting for veth0 to become \
free. Usage count = 2
However after Commit 52df157f17e5 ("xfrm: take refcnt of dst
when creating struct xfrm_dst bundle"), the xdst will not be
freed with dst gc, and this warning happens.
To fix it, we need to find these xdsts that are still held by
others when removing the dev, and free xdst's dev and set it
with loopback_dev.
But unfortunately after flow_cache for xfrm was deleted, no
list tracks them anymore. So we need to save these xdsts
somewhere to release the xdst's dev later.
To make this easier, this patch is to reuse uncached_list to
track xdsts, so that the dev refcnt can be released in the
event NETDEV_UNREGISTER process of fib_netdev_notifier.
Thanks to Florian, we could move forward this fix quickly.
Fixes: 52df157f17e5 ("xfrm: take refcnt of dst when creating struct xfrm_dst bundle")
Reported-by: Jianlin Shi <jishi@redhat.com>
Reported-by: Hangbin Liu <liuhangbin@gmail.com>
Tested-by: Eyal Birger <eyal.birger@gmail.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2018-02-14 11:06:02 +00:00
|
|
|
void rt_add_uncached_list(struct rtable *rt)
|
2012-07-31 22:06:50 +00:00
|
|
|
{
|
2015-01-14 23:17:06 +00:00
|
|
|
struct uncached_list *ul = raw_cpu_ptr(&rt_uncached_list);
|
|
|
|
|
|
|
|
rt->rt_uncached_list = ul;
|
|
|
|
|
|
|
|
spin_lock_bh(&ul->lock);
|
|
|
|
list_add_tail(&rt->rt_uncached, &ul->head);
|
|
|
|
spin_unlock_bh(&ul->lock);
|
2012-07-31 22:06:50 +00:00
|
|
|
}
|
|
|
|
|
xfrm: reuse uncached_list to track xdsts
In early time, when freeing a xdst, it would be inserted into
dst_garbage.list first. Then if it's refcnt was still held
somewhere, later it would be put into dst_busy_list in
dst_gc_task().
When one dev was being unregistered, the dev of these dsts in
dst_busy_list would be set with loopback_dev and put this dev.
So that this dev's removal wouldn't get blocked, and avoid the
kmsg warning:
kernel:unregister_netdevice: waiting for veth0 to become \
free. Usage count = 2
However after Commit 52df157f17e5 ("xfrm: take refcnt of dst
when creating struct xfrm_dst bundle"), the xdst will not be
freed with dst gc, and this warning happens.
To fix it, we need to find these xdsts that are still held by
others when removing the dev, and free xdst's dev and set it
with loopback_dev.
But unfortunately after flow_cache for xfrm was deleted, no
list tracks them anymore. So we need to save these xdsts
somewhere to release the xdst's dev later.
To make this easier, this patch is to reuse uncached_list to
track xdsts, so that the dev refcnt can be released in the
event NETDEV_UNREGISTER process of fib_netdev_notifier.
Thanks to Florian, we could move forward this fix quickly.
Fixes: 52df157f17e5 ("xfrm: take refcnt of dst when creating struct xfrm_dst bundle")
Reported-by: Jianlin Shi <jishi@redhat.com>
Reported-by: Hangbin Liu <liuhangbin@gmail.com>
Tested-by: Eyal Birger <eyal.birger@gmail.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2018-02-14 11:06:02 +00:00
|
|
|
void rt_del_uncached_list(struct rtable *rt)
|
2012-07-31 22:06:50 +00:00
|
|
|
{
|
2012-08-24 05:40:47 +00:00
|
|
|
if (!list_empty(&rt->rt_uncached)) {
|
2015-01-14 23:17:06 +00:00
|
|
|
struct uncached_list *ul = rt->rt_uncached_list;
|
|
|
|
|
|
|
|
spin_lock_bh(&ul->lock);
|
2012-07-31 22:06:50 +00:00
|
|
|
list_del(&rt->rt_uncached);
|
2015-01-14 23:17:06 +00:00
|
|
|
spin_unlock_bh(&ul->lock);
|
2012-07-31 22:06:50 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
xfrm: reuse uncached_list to track xdsts
In early time, when freeing a xdst, it would be inserted into
dst_garbage.list first. Then if it's refcnt was still held
somewhere, later it would be put into dst_busy_list in
dst_gc_task().
When one dev was being unregistered, the dev of these dsts in
dst_busy_list would be set with loopback_dev and put this dev.
So that this dev's removal wouldn't get blocked, and avoid the
kmsg warning:
kernel:unregister_netdevice: waiting for veth0 to become \
free. Usage count = 2
However after Commit 52df157f17e5 ("xfrm: take refcnt of dst
when creating struct xfrm_dst bundle"), the xdst will not be
freed with dst gc, and this warning happens.
To fix it, we need to find these xdsts that are still held by
others when removing the dev, and free xdst's dev and set it
with loopback_dev.
But unfortunately after flow_cache for xfrm was deleted, no
list tracks them anymore. So we need to save these xdsts
somewhere to release the xdst's dev later.
To make this easier, this patch is to reuse uncached_list to
track xdsts, so that the dev refcnt can be released in the
event NETDEV_UNREGISTER process of fib_netdev_notifier.
Thanks to Florian, we could move forward this fix quickly.
Fixes: 52df157f17e5 ("xfrm: take refcnt of dst when creating struct xfrm_dst bundle")
Reported-by: Jianlin Shi <jishi@redhat.com>
Reported-by: Hangbin Liu <liuhangbin@gmail.com>
Tested-by: Eyal Birger <eyal.birger@gmail.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2018-02-14 11:06:02 +00:00
|
|
|
static void ipv4_dst_destroy(struct dst_entry *dst)
|
|
|
|
{
|
|
|
|
struct rtable *rt = (struct rtable *)dst;
|
|
|
|
|
2018-10-05 03:07:54 +00:00
|
|
|
ip_dst_metrics_put(dst);
|
xfrm: reuse uncached_list to track xdsts
In early time, when freeing a xdst, it would be inserted into
dst_garbage.list first. Then if it's refcnt was still held
somewhere, later it would be put into dst_busy_list in
dst_gc_task().
When one dev was being unregistered, the dev of these dsts in
dst_busy_list would be set with loopback_dev and put this dev.
So that this dev's removal wouldn't get blocked, and avoid the
kmsg warning:
kernel:unregister_netdevice: waiting for veth0 to become \
free. Usage count = 2
However after Commit 52df157f17e5 ("xfrm: take refcnt of dst
when creating struct xfrm_dst bundle"), the xdst will not be
freed with dst gc, and this warning happens.
To fix it, we need to find these xdsts that are still held by
others when removing the dev, and free xdst's dev and set it
with loopback_dev.
But unfortunately after flow_cache for xfrm was deleted, no
list tracks them anymore. So we need to save these xdsts
somewhere to release the xdst's dev later.
To make this easier, this patch is to reuse uncached_list to
track xdsts, so that the dev refcnt can be released in the
event NETDEV_UNREGISTER process of fib_netdev_notifier.
Thanks to Florian, we could move forward this fix quickly.
Fixes: 52df157f17e5 ("xfrm: take refcnt of dst when creating struct xfrm_dst bundle")
Reported-by: Jianlin Shi <jishi@redhat.com>
Reported-by: Hangbin Liu <liuhangbin@gmail.com>
Tested-by: Eyal Birger <eyal.birger@gmail.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2018-02-14 11:06:02 +00:00
|
|
|
rt_del_uncached_list(rt);
|
|
|
|
}
|
|
|
|
|
2012-07-31 22:06:50 +00:00
|
|
|
void rt_flush_dev(struct net_device *dev)
|
|
|
|
{
|
2015-01-14 23:17:06 +00:00
|
|
|
struct rtable *rt;
|
|
|
|
int cpu;
|
|
|
|
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
|
2012-07-31 22:06:50 +00:00
|
|
|
|
2015-01-14 23:17:06 +00:00
|
|
|
spin_lock_bh(&ul->lock);
|
|
|
|
list_for_each_entry(rt, &ul->head, rt_uncached) {
|
2012-07-31 22:06:50 +00:00
|
|
|
if (rt->dst.dev != dev)
|
|
|
|
continue;
|
2019-07-01 21:38:57 +00:00
|
|
|
rt->dst.dev = blackhole_netdev;
|
2012-07-31 22:06:50 +00:00
|
|
|
dev_hold(rt->dst.dev);
|
|
|
|
dev_put(dev);
|
|
|
|
}
|
2015-01-14 23:17:06 +00:00
|
|
|
spin_unlock_bh(&ul->lock);
|
2012-07-17 11:19:00 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2012-07-25 05:11:23 +00:00
|
|
|
static bool rt_cache_valid(const struct rtable *rt)
|
2012-07-17 19:58:50 +00:00
|
|
|
{
|
2012-07-25 05:11:23 +00:00
|
|
|
return rt &&
|
|
|
|
rt->dst.obsolete == DST_OBSOLETE_FORCE_CHK &&
|
|
|
|
!rt_is_expired(rt);
|
2012-07-17 19:58:50 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 19:20:47 +00:00
|
|
|
static void rt_set_nexthop(struct rtable *rt, __be32 daddr,
|
2011-03-05 05:47:09 +00:00
|
|
|
const struct fib_result *res,
|
2012-07-17 19:20:47 +00:00
|
|
|
struct fib_nh_exception *fnhe,
|
2017-06-17 17:42:42 +00:00
|
|
|
struct fib_info *fi, u16 type, u32 itag,
|
|
|
|
const bool do_cache)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2012-07-31 22:06:50 +00:00
|
|
|
bool cached = false;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (fi) {
|
2019-04-02 21:11:55 +00:00
|
|
|
struct fib_nh_common *nhc = FIB_RES_NHC(*res);
|
2012-07-17 11:19:00 +00:00
|
|
|
|
2019-04-05 23:30:29 +00:00
|
|
|
if (nhc->nhc_gw_family && nhc->nhc_scope == RT_SCOPE_LINK) {
|
2019-09-17 17:39:49 +00:00
|
|
|
rt->rt_uses_gateway = 1;
|
2019-04-05 23:30:29 +00:00
|
|
|
rt->rt_gw_family = nhc->nhc_gw_family;
|
|
|
|
/* only INET and INET6 are supported */
|
|
|
|
if (likely(nhc->nhc_gw_family == AF_INET))
|
|
|
|
rt->rt_gw4 = nhc->nhc_gw.ipv4;
|
|
|
|
else
|
|
|
|
rt->rt_gw6 = nhc->nhc_gw.ipv6;
|
2012-10-08 11:41:18 +00:00
|
|
|
}
|
2019-04-05 23:30:29 +00:00
|
|
|
|
2018-10-05 03:07:53 +00:00
|
|
|
ip_dst_init_metrics(&rt->dst, fi->fib_metrics);
|
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2019-06-04 03:19:50 +00:00
|
|
|
if (nhc->nhc_family == AF_INET) {
|
2019-04-30 14:45:49 +00:00
|
|
|
struct fib_nh *nh;
|
|
|
|
|
|
|
|
nh = container_of(nhc, struct fib_nh, nh_common);
|
|
|
|
rt->dst.tclassid = nh->nh_tclassid;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2019-04-30 14:45:49 +00:00
|
|
|
rt->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);
|
2012-07-31 22:02:02 +00:00
|
|
|
if (unlikely(fnhe))
|
2017-06-17 17:42:42 +00:00
|
|
|
cached = rt_bind_exception(rt, fnhe, daddr, do_cache);
|
|
|
|
else if (do_cache)
|
2019-04-30 14:45:49 +00:00
|
|
|
cached = rt_cache_route(nhc, rt);
|
2012-10-08 11:41:18 +00:00
|
|
|
if (unlikely(!cached)) {
|
|
|
|
/* Routes we intend to cache in nexthop exception or
|
|
|
|
* FIB nexthop have the DST_NOCACHE bit clear.
|
|
|
|
* However, if we are unsuccessful at storing this
|
|
|
|
* route into the cache we really need to set it.
|
|
|
|
*/
|
2019-04-05 23:30:27 +00:00
|
|
|
if (!rt->rt_gw4) {
|
|
|
|
rt->rt_gw_family = AF_INET;
|
|
|
|
rt->rt_gw4 = daddr;
|
|
|
|
}
|
2012-10-08 11:41:18 +00:00
|
|
|
rt_add_uncached_list(rt);
|
|
|
|
}
|
|
|
|
} else
|
2012-07-31 22:06:50 +00:00
|
|
|
rt_add_uncached_list(rt);
|
2010-12-09 05:16:57 +00:00
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_IP_MULTIPLE_TABLES
|
2012-07-13 15:21:29 +00:00
|
|
|
set_class_tag(rt, res->tclassid);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
set_class_tag(rt, itag);
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2016-04-07 18:10:06 +00:00
|
|
|
struct rtable *rt_dst_alloc(struct net_device *dev,
|
|
|
|
unsigned int flags, u16 type,
|
2020-03-23 14:31:19 +00:00
|
|
|
bool nopolicy, bool noxfrm)
|
2011-02-17 23:42:37 +00:00
|
|
|
{
|
2015-09-02 20:58:34 +00:00
|
|
|
struct rtable *rt;
|
|
|
|
|
|
|
|
rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
|
|
|
|
(nopolicy ? DST_NOPOLICY : 0) |
|
2017-06-17 17:42:41 +00:00
|
|
|
(noxfrm ? DST_NOXFRM : 0));
|
2015-09-02 20:58:34 +00:00
|
|
|
|
|
|
|
if (rt) {
|
|
|
|
rt->rt_genid = rt_genid_ipv4(dev_net(dev));
|
|
|
|
rt->rt_flags = flags;
|
|
|
|
rt->rt_type = type;
|
|
|
|
rt->rt_is_input = 0;
|
|
|
|
rt->rt_iif = 0;
|
|
|
|
rt->rt_pmtu = 0;
|
2018-03-14 09:21:14 +00:00
|
|
|
rt->rt_mtu_locked = 0;
|
2019-09-17 17:39:49 +00:00
|
|
|
rt->rt_uses_gateway = 0;
|
2019-04-05 23:30:27 +00:00
|
|
|
rt->rt_gw_family = 0;
|
|
|
|
rt->rt_gw4 = 0;
|
2015-09-02 20:58:34 +00:00
|
|
|
INIT_LIST_HEAD(&rt->rt_uncached);
|
|
|
|
|
|
|
|
rt->dst.output = ip_output;
|
|
|
|
if (flags & RTCF_LOCAL)
|
|
|
|
rt->dst.input = ip_local_deliver;
|
|
|
|
}
|
|
|
|
|
|
|
|
return rt;
|
2011-02-17 23:42:37 +00:00
|
|
|
}
|
2016-04-07 18:10:06 +00:00
|
|
|
EXPORT_SYMBOL(rt_dst_alloc);
|
2011-02-17 23:42:37 +00:00
|
|
|
|
2019-06-26 06:21:16 +00:00
|
|
|
struct rtable *rt_dst_clone(struct net_device *dev, struct rtable *rt)
|
|
|
|
{
|
|
|
|
struct rtable *new_rt;
|
|
|
|
|
|
|
|
new_rt = dst_alloc(&ipv4_dst_ops, dev, 1, DST_OBSOLETE_FORCE_CHK,
|
|
|
|
rt->dst.flags);
|
|
|
|
|
|
|
|
if (new_rt) {
|
|
|
|
new_rt->rt_genid = rt_genid_ipv4(dev_net(dev));
|
|
|
|
new_rt->rt_flags = rt->rt_flags;
|
|
|
|
new_rt->rt_type = rt->rt_type;
|
|
|
|
new_rt->rt_is_input = rt->rt_is_input;
|
|
|
|
new_rt->rt_iif = rt->rt_iif;
|
|
|
|
new_rt->rt_pmtu = rt->rt_pmtu;
|
|
|
|
new_rt->rt_mtu_locked = rt->rt_mtu_locked;
|
|
|
|
new_rt->rt_gw_family = rt->rt_gw_family;
|
|
|
|
if (rt->rt_gw_family == AF_INET)
|
|
|
|
new_rt->rt_gw4 = rt->rt_gw4;
|
|
|
|
else if (rt->rt_gw_family == AF_INET6)
|
|
|
|
new_rt->rt_gw6 = rt->rt_gw6;
|
|
|
|
INIT_LIST_HEAD(&new_rt->rt_uncached);
|
|
|
|
|
|
|
|
new_rt->dst.input = rt->dst.input;
|
|
|
|
new_rt->dst.output = rt->dst.output;
|
|
|
|
new_rt->dst.error = rt->dst.error;
|
|
|
|
new_rt->dst.lastuse = jiffies;
|
|
|
|
new_rt->dst.lwtstate = lwtstate_get(rt->dst.lwtstate);
|
|
|
|
}
|
|
|
|
return new_rt;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(rt_dst_clone);
|
|
|
|
|
2010-06-02 19:21:31 +00:00
|
|
|
/* called in rcu_read_lock() section */
|
2017-09-28 13:51:37 +00:00
|
|
|
int ip_mc_validate_source(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
|
|
|
u8 tos, struct net_device *dev,
|
|
|
|
struct in_device *in_dev, u32 *itag)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-06-02 12:05:27 +00:00
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* Primary sanity checks. */
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!in_dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2008-01-21 11:18:08 +00:00
|
|
|
if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr) ||
|
2012-06-12 00:44:01 +00:00
|
|
|
skb->protocol != htons(ETH_P_IP))
|
2017-09-28 13:51:37 +00:00
|
|
|
return -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-09-28 18:10:38 +00:00
|
|
|
if (ipv4_is_loopback(saddr) && !IN_DEV_ROUTE_LOCALNET(in_dev))
|
2017-09-28 13:51:37 +00:00
|
|
|
return -EINVAL;
|
2012-06-12 00:44:01 +00:00
|
|
|
|
2007-12-16 21:45:43 +00:00
|
|
|
if (ipv4_is_zeronet(saddr)) {
|
2019-01-31 23:00:40 +00:00
|
|
|
if (!ipv4_is_local_multicast(daddr) &&
|
|
|
|
ip_hdr(skb)->protocol != IPPROTO_IGMP)
|
2017-09-28 13:51:37 +00:00
|
|
|
return -EINVAL;
|
2010-06-02 12:05:27 +00:00
|
|
|
} else {
|
2012-06-29 01:54:02 +00:00
|
|
|
err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
|
2017-09-28 13:51:37 +00:00
|
|
|
in_dev, itag);
|
2010-06-02 12:05:27 +00:00
|
|
|
if (err < 0)
|
2017-09-28 13:51:37 +00:00
|
|
|
return err;
|
2010-06-02 12:05:27 +00:00
|
|
|
}
|
2017-09-28 13:51:37 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* called in rcu_read_lock() section */
|
|
|
|
static int ip_route_input_mc(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
|
|
|
u8 tos, struct net_device *dev, int our)
|
|
|
|
{
|
|
|
|
struct in_device *in_dev = __in_dev_get_rcu(dev);
|
|
|
|
unsigned int flags = RTCF_MULTICAST;
|
|
|
|
struct rtable *rth;
|
|
|
|
u32 itag = 0;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
err = ip_mc_validate_source(skb, daddr, saddr, tos, dev, in_dev, &itag);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
2015-09-02 20:58:34 +00:00
|
|
|
if (our)
|
|
|
|
flags |= RTCF_LOCAL;
|
|
|
|
|
|
|
|
rth = rt_dst_alloc(dev_net(dev)->loopback_dev, flags, RTN_MULTICAST,
|
2020-11-07 19:35:15 +00:00
|
|
|
IN_DEV_ORCONF(in_dev, NOPOLICY), false);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!rth)
|
2017-09-28 13:51:37 +00:00
|
|
|
return -ENOBUFS;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-04-28 21:31:47 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
|
|
|
rth->dst.tclassid = itag;
|
|
|
|
#endif
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.output = ip_rt_bug;
|
2012-07-17 21:44:26 +00:00
|
|
|
rth->rt_is_input= 1;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
#ifdef CONFIG_IP_MROUTE
|
2007-12-16 21:45:43 +00:00
|
|
|
if (!ipv4_is_local_multicast(daddr) && IN_DEV_MFORWARD(in_dev))
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.input = ip_mr_input;
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
RT_CACHE_STAT_INC(in_slow_mc);
|
|
|
|
|
2012-07-17 18:00:09 +00:00
|
|
|
skb_dst_set(skb, &rth->dst);
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
|
|
static void ip_handle_martian_source(struct net_device *dev,
|
|
|
|
struct in_device *in_dev,
|
|
|
|
struct sk_buff *skb,
|
2006-09-27 04:25:20 +00:00
|
|
|
__be32 daddr,
|
|
|
|
__be32 saddr)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
RT_CACHE_STAT_INC(in_martian_src);
|
|
|
|
#ifdef CONFIG_IP_ROUTE_VERBOSE
|
|
|
|
if (IN_DEV_LOG_MARTIANS(in_dev) && net_ratelimit()) {
|
|
|
|
/*
|
|
|
|
* RFC1812 recommendation, if source is martian,
|
|
|
|
* the only hint is MAC header.
|
|
|
|
*/
|
2012-03-11 18:36:11 +00:00
|
|
|
pr_warn("martian source %pI4 from %pI4, on dev %s\n",
|
2008-10-31 07:53:57 +00:00
|
|
|
&daddr, &saddr, dev->name);
|
2007-03-19 22:33:04 +00:00
|
|
|
if (dev->hard_header_len && skb_mac_header_was_set(skb)) {
|
2012-03-11 18:36:11 +00:00
|
|
|
print_hex_dump(KERN_WARNING, "ll header: ",
|
|
|
|
DUMP_PREFIX_OFFSET, 16, 1,
|
|
|
|
skb_mac_header(skb),
|
2018-11-20 18:15:36 +00:00
|
|
|
dev->hard_header_len, false);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2010-06-03 04:13:21 +00:00
|
|
|
/* called in rcu_read_lock() section */
|
2008-04-10 08:52:09 +00:00
|
|
|
static int __mkroute_input(struct sk_buff *skb,
|
2011-02-17 05:44:24 +00:00
|
|
|
const struct fib_result *res,
|
2008-04-10 08:52:09 +00:00
|
|
|
struct in_device *in_dev,
|
2012-07-26 11:14:38 +00:00
|
|
|
__be32 daddr, __be32 saddr, u32 tos)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2019-04-02 21:11:55 +00:00
|
|
|
struct fib_nh_common *nhc = FIB_RES_NHC(*res);
|
|
|
|
struct net_device *dev = nhc->nhc_dev;
|
2013-06-27 07:27:05 +00:00
|
|
|
struct fib_nh_exception *fnhe;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct rtable *rth;
|
|
|
|
int err;
|
|
|
|
struct in_device *out_dev;
|
2012-07-17 19:58:50 +00:00
|
|
|
bool do_cache;
|
2014-05-22 08:36:55 +00:00
|
|
|
u32 itag = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* get a working reference to the output device */
|
2019-04-02 21:11:55 +00:00
|
|
|
out_dev = __in_dev_get_rcu(dev);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!out_dev) {
|
2012-05-13 21:56:26 +00:00
|
|
|
net_crit_ratelimited("Bug in ip_route_input_slow(). Please report.\n");
|
2005-04-16 22:20:36 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2011-04-07 04:51:50 +00:00
|
|
|
err = fib_validate_source(skb, saddr, daddr, tos, FIB_RES_OIF(*res),
|
2012-06-29 01:54:02 +00:00
|
|
|
in_dev->dev, in_dev, &itag);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err < 0) {
|
2007-02-09 14:24:47 +00:00
|
|
|
ip_handle_martian_source(in_dev->dev, in_dev, skb, daddr,
|
2005-04-16 22:20:36 +00:00
|
|
|
saddr);
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
goto cleanup;
|
|
|
|
}
|
|
|
|
|
2012-10-08 11:41:15 +00:00
|
|
|
do_cache = res->fi && !itag;
|
|
|
|
if (out_dev == in_dev && err && IN_DEV_TX_REDIRECTS(out_dev) &&
|
2019-04-02 21:11:55 +00:00
|
|
|
skb->protocol == htons(ETH_P_IP)) {
|
2019-04-05 23:30:26 +00:00
|
|
|
__be32 gw;
|
2019-04-02 21:11:55 +00:00
|
|
|
|
2019-04-05 23:30:26 +00:00
|
|
|
gw = nhc->nhc_gw_family == AF_INET ? nhc->nhc_gw.ipv4 : 0;
|
2019-04-02 21:11:55 +00:00
|
|
|
if (IN_DEV_SHARED_MEDIA(out_dev) ||
|
|
|
|
inet_addr_onlink(out_dev, saddr, gw))
|
|
|
|
IPCB(skb)->flags |= IPSKB_DOREDIRECT;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (skb->protocol != htons(ETH_P_IP)) {
|
|
|
|
/* Not IP (i.e. ARP). Do not create route, if it is
|
|
|
|
* invalid for proxy arp. DNAT routes are always valid.
|
2010-01-05 05:50:47 +00:00
|
|
|
*
|
|
|
|
* Proxy arp feature have been extended to allow, ARP
|
|
|
|
* replies back to the same interface, to support
|
|
|
|
* Private VLAN switch technologies. See arp.c.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2010-01-05 05:50:47 +00:00
|
|
|
if (out_dev == in_dev &&
|
|
|
|
IN_DEV_PROXY_ARP_PVLAN(in_dev) == 0) {
|
2005-04-16 22:20:36 +00:00
|
|
|
err = -EINVAL;
|
|
|
|
goto cleanup;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
fnhe = find_exception(nhc, daddr);
|
2012-10-08 11:41:15 +00:00
|
|
|
if (do_cache) {
|
2018-05-02 06:41:19 +00:00
|
|
|
if (fnhe)
|
2013-06-27 07:27:05 +00:00
|
|
|
rth = rcu_dereference(fnhe->fnhe_rth_input);
|
2018-05-02 06:41:19 +00:00
|
|
|
else
|
2019-04-30 14:45:48 +00:00
|
|
|
rth = rcu_dereference(nhc->nhc_rth_input);
|
2012-10-08 11:41:15 +00:00
|
|
|
if (rt_cache_valid(rth)) {
|
|
|
|
skb_dst_set_noref(skb, &rth->dst);
|
|
|
|
goto out;
|
2012-07-17 19:58:50 +00:00
|
|
|
}
|
|
|
|
}
|
2012-07-17 19:20:47 +00:00
|
|
|
|
2015-09-02 20:58:34 +00:00
|
|
|
rth = rt_dst_alloc(out_dev->dev, 0, res->type,
|
2020-11-07 19:35:15 +00:00
|
|
|
IN_DEV_ORCONF(in_dev, NOPOLICY),
|
|
|
|
IN_DEV_ORCONF(out_dev, NOXFRM));
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!rth) {
|
|
|
|
err = -ENOBUFS;
|
|
|
|
goto cleanup;
|
|
|
|
}
|
|
|
|
|
2012-07-17 21:44:26 +00:00
|
|
|
rth->rt_is_input = 1;
|
2014-02-17 07:23:43 +00:00
|
|
|
RT_CACHE_STAT_INC(in_slow_tot);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.input = ip_forward;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-06-17 17:42:42 +00:00
|
|
|
rt_set_nexthop(rth, daddr, res, fnhe, res->fi, res->type, itag,
|
|
|
|
do_cache);
|
2018-02-14 04:32:04 +00:00
|
|
|
lwtunnel_set_redirect(&rth->dst);
|
2012-07-26 11:14:38 +00:00
|
|
|
skb_dst_set(skb, &rth->dst);
|
2012-07-17 19:58:50 +00:00
|
|
|
out:
|
2005-04-16 22:20:36 +00:00
|
|
|
err = 0;
|
|
|
|
cleanup:
|
|
|
|
return err;
|
2007-02-09 14:24:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2015-09-30 08:12:22 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_MULTIPATH
|
|
|
|
/* To make ICMP packets follow the right flow, the multipath hash is
|
2017-03-16 13:28:00 +00:00
|
|
|
* calculated from the inner IP addresses.
|
2015-09-30 08:12:22 +00:00
|
|
|
*/
|
2017-03-16 13:28:00 +00:00
|
|
|
static void ip_multipath_l3_keys(const struct sk_buff *skb,
|
|
|
|
struct flow_keys *hash_keys)
|
2015-09-30 08:12:22 +00:00
|
|
|
{
|
|
|
|
const struct iphdr *outer_iph = ip_hdr(skb);
|
2018-03-02 16:32:13 +00:00
|
|
|
const struct iphdr *key_iph = outer_iph;
|
2017-03-16 13:28:00 +00:00
|
|
|
const struct iphdr *inner_iph;
|
2015-09-30 08:12:22 +00:00
|
|
|
const struct icmphdr *icmph;
|
|
|
|
struct iphdr _inner_iph;
|
2017-03-16 13:28:00 +00:00
|
|
|
struct icmphdr _icmph;
|
|
|
|
|
|
|
|
if (likely(outer_iph->protocol != IPPROTO_ICMP))
|
2018-03-02 16:32:13 +00:00
|
|
|
goto out;
|
2015-09-30 08:12:22 +00:00
|
|
|
|
|
|
|
if (unlikely((outer_iph->frag_off & htons(IP_OFFSET)) != 0))
|
2018-03-02 16:32:13 +00:00
|
|
|
goto out;
|
2015-09-30 08:12:22 +00:00
|
|
|
|
|
|
|
icmph = skb_header_pointer(skb, outer_iph->ihl * 4, sizeof(_icmph),
|
|
|
|
&_icmph);
|
|
|
|
if (!icmph)
|
2018-03-02 16:32:13 +00:00
|
|
|
goto out;
|
2015-09-30 08:12:22 +00:00
|
|
|
|
2019-11-02 00:12:04 +00:00
|
|
|
if (!icmp_is_err(icmph->type))
|
2018-03-02 16:32:13 +00:00
|
|
|
goto out;
|
2015-09-30 08:12:22 +00:00
|
|
|
|
|
|
|
inner_iph = skb_header_pointer(skb,
|
|
|
|
outer_iph->ihl * 4 + sizeof(_icmph),
|
|
|
|
sizeof(_inner_iph), &_inner_iph);
|
|
|
|
if (!inner_iph)
|
2018-03-02 16:32:13 +00:00
|
|
|
goto out;
|
|
|
|
|
|
|
|
key_iph = inner_iph;
|
|
|
|
out:
|
|
|
|
hash_keys->addrs.v4addrs.src = key_iph->saddr;
|
|
|
|
hash_keys->addrs.v4addrs.dst = key_iph->daddr;
|
2017-03-16 13:28:00 +00:00
|
|
|
}
|
2015-09-30 08:12:22 +00:00
|
|
|
|
2021-05-17 18:15:19 +00:00
|
|
|
static u32 fib_multipath_custom_hash_outer(const struct net *net,
|
|
|
|
const struct sk_buff *skb,
|
|
|
|
bool *p_has_inner)
|
|
|
|
{
|
|
|
|
u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields;
|
|
|
|
struct flow_keys keys, hash_keys;
|
|
|
|
|
|
|
|
if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
skb_flow_dissect_flow_keys(skb, &keys, FLOW_DISSECTOR_F_STOP_AT_ENCAP);
|
|
|
|
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
|
|
|
|
hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
|
|
|
|
hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
|
|
|
|
hash_keys.basic.ip_proto = keys.basic.ip_proto;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
|
|
|
|
hash_keys.ports.src = keys.ports.src;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
|
|
|
|
hash_keys.ports.dst = keys.ports.dst;
|
|
|
|
|
|
|
|
*p_has_inner = !!(keys.control.flags & FLOW_DIS_ENCAPSULATION);
|
|
|
|
return flow_hash_from_keys(&hash_keys);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 fib_multipath_custom_hash_inner(const struct net *net,
|
|
|
|
const struct sk_buff *skb,
|
|
|
|
bool has_inner)
|
|
|
|
{
|
|
|
|
u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields;
|
|
|
|
struct flow_keys keys, hash_keys;
|
|
|
|
|
|
|
|
/* We assume the packet carries an encapsulation, but if none was
|
|
|
|
* encountered during dissection of the outer flow, then there is no
|
|
|
|
* point in calling the flow dissector again.
|
|
|
|
*/
|
|
|
|
if (!has_inner)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_MASK))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
skb_flow_dissect_flow_keys(skb, &keys, 0);
|
|
|
|
|
|
|
|
if (!(keys.control.flags & FLOW_DIS_ENCAPSULATION))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
|
|
|
|
hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
|
|
|
|
hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
|
|
|
|
} else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_IP)
|
|
|
|
hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_IP)
|
|
|
|
hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_FLOWLABEL)
|
|
|
|
hash_keys.tags.flow_label = keys.tags.flow_label;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_IP_PROTO)
|
|
|
|
hash_keys.basic.ip_proto = keys.basic.ip_proto;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_SRC_PORT)
|
|
|
|
hash_keys.ports.src = keys.ports.src;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_INNER_DST_PORT)
|
|
|
|
hash_keys.ports.dst = keys.ports.dst;
|
|
|
|
|
|
|
|
return flow_hash_from_keys(&hash_keys);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 fib_multipath_custom_hash_skb(const struct net *net,
|
|
|
|
const struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
u32 mhash, mhash_inner;
|
|
|
|
bool has_inner = true;
|
|
|
|
|
|
|
|
mhash = fib_multipath_custom_hash_outer(net, skb, &has_inner);
|
|
|
|
mhash_inner = fib_multipath_custom_hash_inner(net, skb, has_inner);
|
|
|
|
|
|
|
|
return jhash_2words(mhash, mhash_inner, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static u32 fib_multipath_custom_hash_fl4(const struct net *net,
|
|
|
|
const struct flowi4 *fl4)
|
|
|
|
{
|
|
|
|
u32 hash_fields = net->ipv4.sysctl_fib_multipath_hash_fields;
|
|
|
|
struct flow_keys hash_keys;
|
|
|
|
|
|
|
|
if (!(hash_fields & FIB_MULTIPATH_HASH_FIELD_OUTER_MASK))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_IP)
|
|
|
|
hash_keys.addrs.v4addrs.src = fl4->saddr;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_IP)
|
|
|
|
hash_keys.addrs.v4addrs.dst = fl4->daddr;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_IP_PROTO)
|
|
|
|
hash_keys.basic.ip_proto = fl4->flowi4_proto;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_SRC_PORT)
|
|
|
|
hash_keys.ports.src = fl4->fl4_sport;
|
|
|
|
if (hash_fields & FIB_MULTIPATH_HASH_FIELD_DST_PORT)
|
|
|
|
hash_keys.ports.dst = fl4->fl4_dport;
|
|
|
|
|
|
|
|
return flow_hash_from_keys(&hash_keys);
|
|
|
|
}
|
|
|
|
|
2017-03-16 13:28:00 +00:00
|
|
|
/* if skb is set it will be used and fl4 can be NULL */
|
2018-03-02 16:32:12 +00:00
|
|
|
int fib_multipath_hash(const struct net *net, const struct flowi4 *fl4,
|
2018-03-01 03:42:41 +00:00
|
|
|
const struct sk_buff *skb, struct flow_keys *flkeys)
|
2017-03-16 13:28:00 +00:00
|
|
|
{
|
2019-03-01 13:38:43 +00:00
|
|
|
u32 multipath_hash = fl4 ? fl4->flowi4_multipath_hash : 0;
|
2017-03-16 13:28:00 +00:00
|
|
|
struct flow_keys hash_keys;
|
2021-05-17 18:15:17 +00:00
|
|
|
u32 mhash = 0;
|
2015-09-30 08:12:22 +00:00
|
|
|
|
2017-03-16 13:28:00 +00:00
|
|
|
switch (net->ipv4.sysctl_fib_multipath_hash_policy) {
|
|
|
|
case 0:
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
if (skb) {
|
|
|
|
ip_multipath_l3_keys(skb, &hash_keys);
|
|
|
|
} else {
|
|
|
|
hash_keys.addrs.v4addrs.src = fl4->saddr;
|
|
|
|
hash_keys.addrs.v4addrs.dst = fl4->daddr;
|
|
|
|
}
|
2021-05-17 18:15:17 +00:00
|
|
|
mhash = flow_hash_from_keys(&hash_keys);
|
2017-03-16 13:28:00 +00:00
|
|
|
break;
|
|
|
|
case 1:
|
|
|
|
/* skb is currently provided only when forwarding */
|
|
|
|
if (skb) {
|
|
|
|
unsigned int flag = FLOW_DISSECTOR_F_STOP_AT_ENCAP;
|
|
|
|
struct flow_keys keys;
|
|
|
|
|
|
|
|
/* short-circuit if we already have L4 hash present */
|
|
|
|
if (skb->l4_hash)
|
|
|
|
return skb_get_hash_raw(skb) >> 1;
|
2018-03-02 16:32:14 +00:00
|
|
|
|
2017-03-16 13:28:00 +00:00
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
2018-02-21 19:00:54 +00:00
|
|
|
|
2018-03-02 16:32:14 +00:00
|
|
|
if (!flkeys) {
|
2018-03-01 03:42:41 +00:00
|
|
|
skb_flow_dissect_flow_keys(skb, &keys, flag);
|
2018-03-02 16:32:14 +00:00
|
|
|
flkeys = &keys;
|
2018-03-01 03:42:41 +00:00
|
|
|
}
|
2018-03-02 16:32:14 +00:00
|
|
|
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
hash_keys.addrs.v4addrs.src = flkeys->addrs.v4addrs.src;
|
|
|
|
hash_keys.addrs.v4addrs.dst = flkeys->addrs.v4addrs.dst;
|
|
|
|
hash_keys.ports.src = flkeys->ports.src;
|
|
|
|
hash_keys.ports.dst = flkeys->ports.dst;
|
|
|
|
hash_keys.basic.ip_proto = flkeys->basic.ip_proto;
|
2017-03-16 13:28:00 +00:00
|
|
|
} else {
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
hash_keys.addrs.v4addrs.src = fl4->saddr;
|
|
|
|
hash_keys.addrs.v4addrs.dst = fl4->daddr;
|
|
|
|
hash_keys.ports.src = fl4->fl4_sport;
|
|
|
|
hash_keys.ports.dst = fl4->fl4_dport;
|
|
|
|
hash_keys.basic.ip_proto = fl4->flowi4_proto;
|
|
|
|
}
|
2021-05-17 18:15:17 +00:00
|
|
|
mhash = flow_hash_from_keys(&hash_keys);
|
2017-03-16 13:28:00 +00:00
|
|
|
break;
|
2019-06-13 18:38:58 +00:00
|
|
|
case 2:
|
|
|
|
memset(&hash_keys, 0, sizeof(hash_keys));
|
|
|
|
/* skb is currently provided only when forwarding */
|
|
|
|
if (skb) {
|
|
|
|
struct flow_keys keys;
|
|
|
|
|
|
|
|
skb_flow_dissect_flow_keys(skb, &keys, 0);
|
2019-07-06 14:55:17 +00:00
|
|
|
/* Inner can be v4 or v6 */
|
|
|
|
if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV4_ADDRS) {
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
hash_keys.addrs.v4addrs.src = keys.addrs.v4addrs.src;
|
|
|
|
hash_keys.addrs.v4addrs.dst = keys.addrs.v4addrs.dst;
|
|
|
|
} else if (keys.control.addr_type == FLOW_DISSECTOR_KEY_IPV6_ADDRS) {
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV6_ADDRS;
|
|
|
|
hash_keys.addrs.v6addrs.src = keys.addrs.v6addrs.src;
|
|
|
|
hash_keys.addrs.v6addrs.dst = keys.addrs.v6addrs.dst;
|
|
|
|
hash_keys.tags.flow_label = keys.tags.flow_label;
|
|
|
|
hash_keys.basic.ip_proto = keys.basic.ip_proto;
|
|
|
|
} else {
|
|
|
|
/* Same as case 0 */
|
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
|
|
|
ip_multipath_l3_keys(skb, &hash_keys);
|
|
|
|
}
|
2019-06-13 18:38:58 +00:00
|
|
|
} else {
|
|
|
|
/* Same as case 0 */
|
2019-07-06 14:55:17 +00:00
|
|
|
hash_keys.control.addr_type = FLOW_DISSECTOR_KEY_IPV4_ADDRS;
|
2019-06-13 18:38:58 +00:00
|
|
|
hash_keys.addrs.v4addrs.src = fl4->saddr;
|
|
|
|
hash_keys.addrs.v4addrs.dst = fl4->daddr;
|
|
|
|
}
|
2021-05-17 18:15:17 +00:00
|
|
|
mhash = flow_hash_from_keys(&hash_keys);
|
2019-06-13 18:38:58 +00:00
|
|
|
break;
|
2021-05-17 18:15:19 +00:00
|
|
|
case 3:
|
|
|
|
if (skb)
|
|
|
|
mhash = fib_multipath_custom_hash_skb(net, skb);
|
|
|
|
else
|
|
|
|
mhash = fib_multipath_custom_hash_fl4(net, fl4);
|
|
|
|
break;
|
2017-03-16 13:28:00 +00:00
|
|
|
}
|
2015-09-30 08:12:22 +00:00
|
|
|
|
2019-02-24 03:36:20 +00:00
|
|
|
if (multipath_hash)
|
|
|
|
mhash = jhash_2words(mhash, multipath_hash, 0);
|
|
|
|
|
2017-03-16 13:28:00 +00:00
|
|
|
return mhash >> 1;
|
|
|
|
}
|
2015-09-30 08:12:22 +00:00
|
|
|
#endif /* CONFIG_IP_ROUTE_MULTIPATH */
|
|
|
|
|
2008-04-10 08:52:09 +00:00
|
|
|
static int ip_mkroute_input(struct sk_buff *skb,
|
|
|
|
struct fib_result *res,
|
|
|
|
struct in_device *in_dev,
|
2018-03-01 03:42:41 +00:00
|
|
|
__be32 daddr, __be32 saddr, u32 tos,
|
|
|
|
struct flow_keys *hkeys)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
#ifdef CONFIG_IP_ROUTE_MULTIPATH
|
2019-06-04 03:19:49 +00:00
|
|
|
if (res->fi && fib_info_num_path(res->fi) > 1) {
|
2018-03-02 16:32:12 +00:00
|
|
|
int h = fib_multipath_hash(res->fi->fib_net, NULL, skb, hkeys);
|
2015-09-30 08:12:21 +00:00
|
|
|
|
|
|
|
fib_select_multipath(res, h);
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
|
|
|
/* create a routing cache entry */
|
2012-07-26 11:14:38 +00:00
|
|
|
return __mkroute_input(skb, res, in_dev, daddr, saddr, tos);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2019-11-20 12:47:37 +00:00
|
|
|
/* Implements all the saddr-related checks as ip_route_input_slow(),
|
|
|
|
* assuming daddr is valid and the destination is not a local broadcast one.
|
|
|
|
* Uses the provided hint instead of performing a route lookup.
|
|
|
|
*/
|
|
|
|
int ip_route_use_hint(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
|
|
|
u8 tos, struct net_device *dev,
|
|
|
|
const struct sk_buff *hint)
|
|
|
|
{
|
|
|
|
struct in_device *in_dev = __in_dev_get_rcu(dev);
|
2020-06-27 07:47:51 +00:00
|
|
|
struct rtable *rt = skb_rtable(hint);
|
2019-11-20 12:47:37 +00:00
|
|
|
struct net *net = dev_net(dev);
|
|
|
|
int err = -EINVAL;
|
|
|
|
u32 tag = 0;
|
|
|
|
|
|
|
|
if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
|
|
|
|
goto martian_source;
|
|
|
|
|
|
|
|
if (ipv4_is_zeronet(saddr))
|
|
|
|
goto martian_source;
|
|
|
|
|
|
|
|
if (ipv4_is_loopback(saddr) && !IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
|
|
|
|
goto martian_source;
|
|
|
|
|
|
|
|
if (rt->rt_type != RTN_LOCAL)
|
|
|
|
goto skip_validate_source;
|
|
|
|
|
|
|
|
tos &= IPTOS_RT_MASK;
|
|
|
|
err = fib_validate_source(skb, saddr, daddr, tos, 0, dev, in_dev, &tag);
|
|
|
|
if (err < 0)
|
|
|
|
goto martian_source;
|
|
|
|
|
|
|
|
skip_validate_source:
|
|
|
|
skb_dst_copy(skb, hint);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
martian_source:
|
|
|
|
ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2021-06-13 00:24:59 +00:00
|
|
|
/* get device for dst_alloc with local routes */
|
|
|
|
static struct net_device *ip_rt_get_dev(struct net *net,
|
|
|
|
const struct fib_result *res)
|
|
|
|
{
|
|
|
|
struct fib_nh_common *nhc = res->fi ? res->nhc : NULL;
|
|
|
|
struct net_device *dev = NULL;
|
|
|
|
|
|
|
|
if (nhc)
|
|
|
|
dev = l3mdev_master_dev_rcu(nhc->nhc_dev);
|
|
|
|
|
|
|
|
return dev ? : net->loopback_dev;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* NOTE. We drop all the packets that has local source
|
|
|
|
* addresses, because every properly looped back packet
|
|
|
|
* must have correct destination already attached by output routine.
|
2019-11-20 12:47:37 +00:00
|
|
|
* Changes in the enforced policies must be applied also to
|
|
|
|
* ip_route_use_hint().
|
2005-04-16 22:20:36 +00:00
|
|
|
*
|
|
|
|
* Such approach solves two big problems:
|
|
|
|
* 1. Not simplex devices are handled properly.
|
|
|
|
* 2. IP spoofing attempts are filtered with 100% of guarantee.
|
2010-10-05 10:41:36 +00:00
|
|
|
* called with rcu_read_lock()
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2006-09-27 04:25:20 +00:00
|
|
|
static int ip_route_input_slow(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
2017-05-25 17:42:34 +00:00
|
|
|
u8 tos, struct net_device *dev,
|
|
|
|
struct fib_result *res)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2010-06-02 19:21:31 +00:00
|
|
|
struct in_device *in_dev = __in_dev_get_rcu(dev);
|
2018-03-01 03:42:41 +00:00
|
|
|
struct flow_keys *flkeys = NULL, _flkeys;
|
|
|
|
struct net *net = dev_net(dev);
|
2015-07-21 08:43:59 +00:00
|
|
|
struct ip_tunnel_info *tun_info;
|
2018-03-01 03:42:41 +00:00
|
|
|
int err = -EINVAL;
|
2012-04-15 05:58:06 +00:00
|
|
|
unsigned int flags = 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
u32 itag = 0;
|
2012-04-15 05:58:06 +00:00
|
|
|
struct rtable *rth;
|
2018-03-01 03:42:41 +00:00
|
|
|
struct flowi4 fl4;
|
2019-06-02 11:10:24 +00:00
|
|
|
bool do_cache = true;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* IP on this device is disabled. */
|
|
|
|
|
|
|
|
if (!in_dev)
|
|
|
|
goto out;
|
|
|
|
|
|
|
|
/* Check for the most weird martians, which can be not detected
|
2021-03-12 07:30:05 +00:00
|
|
|
* by fib_lookup.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2015-08-20 11:56:25 +00:00
|
|
|
tun_info = skb_tunnel_info(skb);
|
2015-08-28 18:48:19 +00:00
|
|
|
if (tun_info && !(tun_info->mode & IP_TUNNEL_INFO_TX))
|
2015-07-21 08:43:59 +00:00
|
|
|
fl4.flowi4_tun_key.tun_id = tun_info->key.tun_id;
|
|
|
|
else
|
|
|
|
fl4.flowi4_tun_key.tun_id = 0;
|
2015-07-21 08:43:56 +00:00
|
|
|
skb_dst_drop(skb);
|
|
|
|
|
2012-06-12 00:44:01 +00:00
|
|
|
if (ipv4_is_multicast(saddr) || ipv4_is_lbcast(saddr))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto martian_source;
|
|
|
|
|
2017-05-25 17:42:34 +00:00
|
|
|
res->fi = NULL;
|
|
|
|
res->table = NULL;
|
2010-10-17 15:11:22 +00:00
|
|
|
if (ipv4_is_lbcast(daddr) || (saddr == 0 && daddr == 0))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto brd_input;
|
|
|
|
|
|
|
|
/* Accept zero addresses only to limited broadcast;
|
|
|
|
* I even do not know to fix it or not. Waiting for complains :-)
|
|
|
|
*/
|
2007-12-16 21:45:43 +00:00
|
|
|
if (ipv4_is_zeronet(saddr))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto martian_source;
|
|
|
|
|
2012-06-12 00:44:01 +00:00
|
|
|
if (ipv4_is_zeronet(daddr))
|
2005-04-16 22:20:36 +00:00
|
|
|
goto martian_destination;
|
|
|
|
|
2012-08-03 21:27:25 +00:00
|
|
|
/* Following code try to avoid calling IN_DEV_NET_ROUTE_LOCALNET(),
|
|
|
|
* and call it once if daddr or/and saddr are loopback addresses
|
|
|
|
*/
|
|
|
|
if (ipv4_is_loopback(daddr)) {
|
|
|
|
if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
|
2012-06-12 00:44:01 +00:00
|
|
|
goto martian_destination;
|
2012-08-03 21:27:25 +00:00
|
|
|
} else if (ipv4_is_loopback(saddr)) {
|
|
|
|
if (!IN_DEV_NET_ROUTE_LOCALNET(in_dev, net))
|
2012-06-12 00:44:01 +00:00
|
|
|
goto martian_source;
|
|
|
|
}
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/*
|
|
|
|
* Now we are ready to route packet.
|
|
|
|
*/
|
2011-03-12 01:07:33 +00:00
|
|
|
fl4.flowi4_oif = 0;
|
2016-09-10 19:09:57 +00:00
|
|
|
fl4.flowi4_iif = dev->ifindex;
|
2011-03-12 01:07:33 +00:00
|
|
|
fl4.flowi4_mark = skb->mark;
|
|
|
|
fl4.flowi4_tos = tos;
|
|
|
|
fl4.flowi4_scope = RT_SCOPE_UNIVERSE;
|
2015-09-30 02:07:07 +00:00
|
|
|
fl4.flowi4_flags = 0;
|
2011-03-12 01:07:33 +00:00
|
|
|
fl4.daddr = daddr;
|
|
|
|
fl4.saddr = saddr;
|
2017-02-26 13:50:52 +00:00
|
|
|
fl4.flowi4_uid = sock_net_uid(net, NULL);
|
2020-09-13 18:43:39 +00:00
|
|
|
fl4.flowi4_multipath_hash = 0;
|
2018-03-01 03:42:41 +00:00
|
|
|
|
2018-05-16 20:36:40 +00:00
|
|
|
if (fib4_rules_early_flow_dissect(net, skb, &fl4, &_flkeys)) {
|
2018-03-01 03:42:41 +00:00
|
|
|
flkeys = &_flkeys;
|
2018-05-16 20:36:40 +00:00
|
|
|
} else {
|
|
|
|
fl4.flowi4_proto = 0;
|
|
|
|
fl4.fl4_sport = 0;
|
|
|
|
fl4.fl4_dport = 0;
|
|
|
|
}
|
2018-03-01 03:42:41 +00:00
|
|
|
|
2017-05-25 17:42:34 +00:00
|
|
|
err = fib_lookup(net, &fl4, res, 0);
|
2014-02-14 10:26:22 +00:00
|
|
|
if (err != 0) {
|
|
|
|
if (!IN_DEV_FORWARD(in_dev))
|
|
|
|
err = -EHOSTUNREACH;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto no_route;
|
2014-02-14 10:26:22 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-07-27 08:37:28 +00:00
|
|
|
if (res->type == RTN_BROADCAST) {
|
|
|
|
if (IN_DEV_BFORWARD(in_dev))
|
|
|
|
goto make_route;
|
2019-06-02 11:10:24 +00:00
|
|
|
/* not do cache if bc_forwarding is enabled */
|
|
|
|
if (IPV4_DEVCONF_ALL(net, BC_FORWARDING))
|
|
|
|
do_cache = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto brd_input;
|
2018-07-27 08:37:28 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-05-25 17:42:34 +00:00
|
|
|
if (res->type == RTN_LOCAL) {
|
2011-04-07 04:51:50 +00:00
|
|
|
err = fib_validate_source(skb, saddr, daddr, tos,
|
2014-04-15 23:25:35 +00:00
|
|
|
0, dev, in_dev, &itag);
|
2010-06-02 12:05:27 +00:00
|
|
|
if (err < 0)
|
2015-09-28 18:10:44 +00:00
|
|
|
goto martian_source;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto local_input;
|
|
|
|
}
|
|
|
|
|
2014-02-14 10:26:22 +00:00
|
|
|
if (!IN_DEV_FORWARD(in_dev)) {
|
|
|
|
err = -EHOSTUNREACH;
|
2012-06-26 23:27:09 +00:00
|
|
|
goto no_route;
|
2014-02-14 10:26:22 +00:00
|
|
|
}
|
2017-05-25 17:42:34 +00:00
|
|
|
if (res->type != RTN_UNICAST)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto martian_destination;
|
|
|
|
|
2018-07-27 08:37:28 +00:00
|
|
|
make_route:
|
2018-03-01 03:42:41 +00:00
|
|
|
err = ip_mkroute_input(skb, res, in_dev, daddr, saddr, tos, flkeys);
|
2005-04-16 22:20:36 +00:00
|
|
|
out: return err;
|
|
|
|
|
|
|
|
brd_input:
|
|
|
|
if (skb->protocol != htons(ETH_P_IP))
|
|
|
|
goto e_inval;
|
|
|
|
|
2012-06-28 11:05:27 +00:00
|
|
|
if (!ipv4_is_zeronet(saddr)) {
|
2012-06-29 01:54:02 +00:00
|
|
|
err = fib_validate_source(skb, saddr, 0, tos, 0, dev,
|
|
|
|
in_dev, &itag);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err < 0)
|
2015-09-28 18:10:44 +00:00
|
|
|
goto martian_source;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
flags |= RTCF_BROADCAST;
|
2017-05-25 17:42:34 +00:00
|
|
|
res->type = RTN_BROADCAST;
|
2005-04-16 22:20:36 +00:00
|
|
|
RT_CACHE_STAT_INC(in_brd);
|
|
|
|
|
|
|
|
local_input:
|
2019-06-02 11:10:24 +00:00
|
|
|
do_cache &= res->fi && !itag;
|
|
|
|
if (do_cache) {
|
|
|
|
struct fib_nh_common *nhc = FIB_RES_NHC(*res);
|
2019-04-02 21:11:55 +00:00
|
|
|
|
2019-06-02 11:10:24 +00:00
|
|
|
rth = rcu_dereference(nhc->nhc_rth_input);
|
|
|
|
if (rt_cache_valid(rth)) {
|
|
|
|
skb_dst_set_noref(skb, &rth->dst);
|
|
|
|
err = 0;
|
|
|
|
goto out;
|
2012-07-17 19:58:50 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2021-06-13 00:24:59 +00:00
|
|
|
rth = rt_dst_alloc(ip_rt_get_dev(net, res),
|
2017-05-25 17:42:34 +00:00
|
|
|
flags | RTCF_LOCAL, res->type,
|
2020-11-07 19:35:15 +00:00
|
|
|
IN_DEV_ORCONF(in_dev, NOPOLICY), false);
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!rth)
|
|
|
|
goto e_nobufs;
|
|
|
|
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.output= ip_rt_bug;
|
2011-04-28 21:31:47 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
|
|
|
rth->dst.tclassid = itag;
|
|
|
|
#endif
|
2012-07-17 21:44:26 +00:00
|
|
|
rth->rt_is_input = 1;
|
2015-07-21 08:43:47 +00:00
|
|
|
|
2014-02-17 07:23:43 +00:00
|
|
|
RT_CACHE_STAT_INC(in_slow_tot);
|
2017-05-25 17:42:34 +00:00
|
|
|
if (res->type == RTN_UNREACHABLE) {
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.input= ip_error;
|
|
|
|
rth->dst.error= -err;
|
2021-03-10 21:13:43 +00:00
|
|
|
rth->rt_flags &= ~RTCF_LOCAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2016-11-30 16:10:09 +00:00
|
|
|
|
2013-11-20 03:12:34 +00:00
|
|
|
if (do_cache) {
|
2019-04-02 21:11:55 +00:00
|
|
|
struct fib_nh_common *nhc = FIB_RES_NHC(*res);
|
2016-11-30 16:10:09 +00:00
|
|
|
|
2019-04-02 21:11:55 +00:00
|
|
|
rth->dst.lwtstate = lwtstate_get(nhc->nhc_lwtstate);
|
2016-11-30 16:10:09 +00:00
|
|
|
if (lwtunnel_input_redirect(rth->dst.lwtstate)) {
|
|
|
|
WARN_ON(rth->dst.input == lwtunnel_input);
|
|
|
|
rth->dst.lwtstate->orig_input = rth->dst.input;
|
|
|
|
rth->dst.input = lwtunnel_input;
|
|
|
|
}
|
|
|
|
|
2019-04-30 14:45:49 +00:00
|
|
|
if (unlikely(!rt_cache_route(nhc, rth)))
|
2013-11-20 03:12:34 +00:00
|
|
|
rt_add_uncached_list(rth);
|
|
|
|
}
|
2012-07-17 18:00:09 +00:00
|
|
|
skb_dst_set(skb, &rth->dst);
|
2011-03-02 22:31:35 +00:00
|
|
|
err = 0;
|
2010-10-05 10:41:36 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
no_route:
|
|
|
|
RT_CACHE_STAT_INC(in_no_route);
|
2017-05-25 17:42:34 +00:00
|
|
|
res->type = RTN_UNREACHABLE;
|
|
|
|
res->fi = NULL;
|
|
|
|
res->table = NULL;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto local_input;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Do not cache martian addresses: they should be logged (RFC1812)
|
|
|
|
*/
|
|
|
|
martian_destination:
|
|
|
|
RT_CACHE_STAT_INC(in_martian_dst);
|
|
|
|
#ifdef CONFIG_IP_ROUTE_VERBOSE
|
2012-05-13 21:56:26 +00:00
|
|
|
if (IN_DEV_LOG_MARTIANS(in_dev))
|
|
|
|
net_warn_ratelimited("martian destination %pI4 from %pI4, dev %s\n",
|
|
|
|
&daddr, &saddr, dev->name);
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2005-06-28 20:06:23 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
e_inval:
|
|
|
|
err = -EINVAL;
|
2010-10-05 10:41:36 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
e_nobufs:
|
|
|
|
err = -ENOBUFS;
|
2010-10-05 10:41:36 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
martian_source:
|
|
|
|
ip_handle_martian_source(dev, in_dev, skb, daddr, saddr);
|
2010-10-05 10:41:36 +00:00
|
|
|
goto out;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-07-26 11:14:38 +00:00
|
|
|
int ip_route_input_noref(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
|
|
|
u8 tos, struct net_device *dev)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2017-05-25 17:42:34 +00:00
|
|
|
struct fib_result res;
|
|
|
|
int err;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-02-26 15:14:35 +00:00
|
|
|
tos &= IPTOS_RT_MASK;
|
2010-06-02 19:21:31 +00:00
|
|
|
rcu_read_lock();
|
2017-05-25 17:42:34 +00:00
|
|
|
err = ip_route_input_rcu(skb, daddr, saddr, tos, dev, &res);
|
|
|
|
rcu_read_unlock();
|
2010-06-02 19:21:31 +00:00
|
|
|
|
2017-05-25 17:42:34 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(ip_route_input_noref);
|
|
|
|
|
|
|
|
/* called with rcu_read_lock held */
|
|
|
|
int ip_route_input_rcu(struct sk_buff *skb, __be32 daddr, __be32 saddr,
|
|
|
|
u8 tos, struct net_device *dev, struct fib_result *res)
|
|
|
|
{
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Multicast recognition logic is moved from route cache to here.
|
2021-03-12 07:30:05 +00:00
|
|
|
* The problem was that too many Ethernet cards have broken/missing
|
|
|
|
* hardware multicast filters :-( As result the host on multicasting
|
|
|
|
* network acquires a lot of useless route cache entries, sort of
|
|
|
|
* SDR messages from all the world. Now we try to get rid of them.
|
|
|
|
* Really, provided software IP multicast filter is organized
|
|
|
|
* reasonably (at least, hashed), it does not result in a slowdown
|
|
|
|
* comparing with route cache reject entries.
|
|
|
|
* Note, that multicast routers are not affected, because
|
|
|
|
* route cache entry is created eventually.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2007-12-16 21:45:43 +00:00
|
|
|
if (ipv4_is_multicast(daddr)) {
|
2010-06-02 19:21:31 +00:00
|
|
|
struct in_device *in_dev = __in_dev_get_rcu(dev);
|
2016-10-31 22:54:00 +00:00
|
|
|
int our = 0;
|
2017-05-25 17:42:34 +00:00
|
|
|
int err = -EINVAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-03-06 09:42:53 +00:00
|
|
|
if (!in_dev)
|
|
|
|
return err;
|
|
|
|
our = ip_check_mc_rcu(in_dev, daddr, saddr,
|
|
|
|
ip_hdr(skb)->protocol);
|
2016-10-31 22:54:00 +00:00
|
|
|
|
|
|
|
/* check l3 master if no match yet */
|
2019-03-06 09:42:53 +00:00
|
|
|
if (!our && netif_is_l3_slave(dev)) {
|
2016-10-31 22:54:00 +00:00
|
|
|
struct in_device *l3_in_dev;
|
|
|
|
|
|
|
|
l3_in_dev = __in_dev_get_rcu(skb->dev);
|
|
|
|
if (l3_in_dev)
|
|
|
|
our = ip_check_mc_rcu(l3_in_dev, daddr, saddr,
|
|
|
|
ip_hdr(skb)->protocol);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (our
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_IP_MROUTE
|
2016-10-31 22:54:00 +00:00
|
|
|
||
|
|
|
|
(!ipv4_is_local_multicast(daddr) &&
|
|
|
|
IN_DEV_MFORWARD(in_dev))
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2016-10-31 22:54:00 +00:00
|
|
|
) {
|
2017-05-25 17:42:34 +00:00
|
|
|
err = ip_route_input_mc(skb, daddr, saddr,
|
2016-10-31 22:54:00 +00:00
|
|
|
tos, dev, our);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2017-05-25 17:42:34 +00:00
|
|
|
return err;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2017-05-25 17:42:34 +00:00
|
|
|
|
|
|
|
return ip_route_input_slow(skb, daddr, saddr, tos, dev, res);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2010-10-05 10:41:36 +00:00
|
|
|
/* called with rcu_read_lock() */
|
2011-02-17 05:44:24 +00:00
|
|
|
static struct rtable *__mkroute_output(const struct fib_result *res,
|
2012-07-01 02:02:56 +00:00
|
|
|
const struct flowi4 *fl4, int orig_oif,
|
2011-12-02 11:39:42 +00:00
|
|
|
struct net_device *dev_out,
|
2011-02-17 23:29:00 +00:00
|
|
|
unsigned int flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-02-17 05:44:24 +00:00
|
|
|
struct fib_info *fi = res->fi;
|
2012-07-17 19:20:47 +00:00
|
|
|
struct fib_nh_exception *fnhe;
|
2011-02-17 23:29:00 +00:00
|
|
|
struct in_device *in_dev;
|
2011-02-17 05:44:24 +00:00
|
|
|
u16 type = res->type;
|
2011-02-17 23:29:00 +00:00
|
|
|
struct rtable *rth;
|
2012-10-08 11:41:19 +00:00
|
|
|
bool do_cache;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-06-12 00:44:01 +00:00
|
|
|
in_dev = __in_dev_get_rcu(dev_out);
|
|
|
|
if (!in_dev)
|
2011-02-17 23:29:00 +00:00
|
|
|
return ERR_PTR(-EINVAL);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-06-12 00:44:01 +00:00
|
|
|
if (likely(!IN_DEV_ROUTE_LOCALNET(in_dev)))
|
2016-09-10 19:09:54 +00:00
|
|
|
if (ipv4_is_loopback(fl4->saddr) &&
|
|
|
|
!(dev_out->flags & IFF_LOOPBACK) &&
|
|
|
|
!netif_is_l3_master(dev_out))
|
2012-06-12 00:44:01 +00:00
|
|
|
return ERR_PTR(-EINVAL);
|
|
|
|
|
2011-03-12 01:07:33 +00:00
|
|
|
if (ipv4_is_lbcast(fl4->daddr))
|
2011-02-17 05:44:24 +00:00
|
|
|
type = RTN_BROADCAST;
|
2011-03-12 01:07:33 +00:00
|
|
|
else if (ipv4_is_multicast(fl4->daddr))
|
2011-02-17 05:44:24 +00:00
|
|
|
type = RTN_MULTICAST;
|
2011-03-12 01:07:33 +00:00
|
|
|
else if (ipv4_is_zeronet(fl4->daddr))
|
2011-02-17 23:29:00 +00:00
|
|
|
return ERR_PTR(-EINVAL);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (dev_out->flags & IFF_LOOPBACK)
|
|
|
|
flags |= RTCF_LOCAL;
|
|
|
|
|
2012-11-22 21:04:14 +00:00
|
|
|
do_cache = true;
|
2011-02-17 05:44:24 +00:00
|
|
|
if (type == RTN_BROADCAST) {
|
2005-04-16 22:20:36 +00:00
|
|
|
flags |= RTCF_BROADCAST | RTCF_LOCAL;
|
2011-02-17 05:44:24 +00:00
|
|
|
fi = NULL;
|
|
|
|
} else if (type == RTN_MULTICAST) {
|
2010-09-29 11:53:50 +00:00
|
|
|
flags |= RTCF_MULTICAST | RTCF_LOCAL;
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!ip_check_mc_rcu(in_dev, fl4->daddr, fl4->saddr,
|
|
|
|
fl4->flowi4_proto))
|
2005-04-16 22:20:36 +00:00
|
|
|
flags &= ~RTCF_LOCAL;
|
2012-11-22 21:04:14 +00:00
|
|
|
else
|
|
|
|
do_cache = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
/* If multicast route do not exist use
|
2010-09-29 11:53:50 +00:00
|
|
|
* default one, but do not gateway in this case.
|
|
|
|
* Yes, it is hack.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
2011-02-17 05:44:24 +00:00
|
|
|
if (fi && res->prefixlen < 4)
|
|
|
|
fi = NULL;
|
2016-04-08 21:21:30 +00:00
|
|
|
} else if ((type == RTN_LOCAL) && (orig_oif != 0) &&
|
|
|
|
(orig_oif != dev_out->ifindex)) {
|
|
|
|
/* For local routes that require a particular output interface
|
|
|
|
* we do not want to cache the result. Caching the result
|
|
|
|
* causes incorrect behaviour when there are multiple source
|
|
|
|
* addresses on the interface, the end result being that if the
|
|
|
|
* intended recipient is waiting on that interface for the
|
|
|
|
* packet he won't receive it because it will be delivered on
|
|
|
|
* the loopback interface and the IP_PKTINFO ipi_ifindex will
|
|
|
|
* be set to the loopback interface as well.
|
|
|
|
*/
|
2018-05-02 06:41:19 +00:00
|
|
|
do_cache = false;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2012-07-17 19:20:47 +00:00
|
|
|
fnhe = NULL;
|
2012-11-22 21:04:14 +00:00
|
|
|
do_cache &= fi != NULL;
|
2018-05-02 06:41:19 +00:00
|
|
|
if (fi) {
|
2019-04-02 21:11:55 +00:00
|
|
|
struct fib_nh_common *nhc = FIB_RES_NHC(*res);
|
2012-07-31 22:02:02 +00:00
|
|
|
struct rtable __rcu **prth;
|
2012-07-31 05:45:30 +00:00
|
|
|
|
2019-04-30 14:45:50 +00:00
|
|
|
fnhe = find_exception(nhc, fl4->daddr);
|
2018-05-02 06:41:19 +00:00
|
|
|
if (!do_cache)
|
|
|
|
goto add;
|
2016-02-18 13:21:19 +00:00
|
|
|
if (fnhe) {
|
2013-06-27 07:27:05 +00:00
|
|
|
prth = &fnhe->fnhe_rth_output;
|
2018-05-02 06:41:19 +00:00
|
|
|
} else {
|
|
|
|
if (unlikely(fl4->flowi4_flags &
|
|
|
|
FLOWI_FLAG_KNOWN_NH &&
|
2019-04-05 23:30:26 +00:00
|
|
|
!(nhc->nhc_gw_family &&
|
2019-04-02 21:11:55 +00:00
|
|
|
nhc->nhc_scope == RT_SCOPE_LINK))) {
|
2018-05-02 06:41:19 +00:00
|
|
|
do_cache = false;
|
|
|
|
goto add;
|
2012-10-08 11:41:19 +00:00
|
|
|
}
|
2019-04-30 14:45:48 +00:00
|
|
|
prth = raw_cpu_ptr(nhc->nhc_pcpu_rth_output);
|
2012-10-08 11:41:19 +00:00
|
|
|
}
|
2012-07-31 22:02:02 +00:00
|
|
|
rth = rcu_dereference(*prth);
|
2017-06-17 17:42:31 +00:00
|
|
|
if (rt_cache_valid(rth) && dst_hold_safe(&rth->dst))
|
2012-07-31 22:02:02 +00:00
|
|
|
return rth;
|
2012-07-17 19:20:47 +00:00
|
|
|
}
|
2012-10-08 11:41:19 +00:00
|
|
|
|
|
|
|
add:
|
2015-09-02 20:58:34 +00:00
|
|
|
rth = rt_dst_alloc(dev_out, flags, type,
|
2020-11-07 19:35:15 +00:00
|
|
|
IN_DEV_ORCONF(in_dev, NOPOLICY),
|
|
|
|
IN_DEV_ORCONF(in_dev, NOXFRM));
|
2010-10-07 14:48:38 +00:00
|
|
|
if (!rth)
|
2011-02-17 23:29:00 +00:00
|
|
|
return ERR_PTR(-ENOBUFS);
|
2010-10-07 14:48:38 +00:00
|
|
|
|
2017-08-12 00:02:02 +00:00
|
|
|
rth->rt_iif = orig_oif;
|
2015-09-02 20:58:35 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
RT_CACHE_STAT_INC(out_slow_tot);
|
|
|
|
|
|
|
|
if (flags & (RTCF_BROADCAST | RTCF_MULTICAST)) {
|
2007-02-09 14:24:47 +00:00
|
|
|
if (flags & RTCF_LOCAL &&
|
2005-04-16 22:20:36 +00:00
|
|
|
!(dev_out->flags & IFF_LOOPBACK)) {
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.output = ip_mc_output;
|
2005-04-16 22:20:36 +00:00
|
|
|
RT_CACHE_STAT_INC(out_slow_mc);
|
|
|
|
}
|
|
|
|
#ifdef CONFIG_IP_MROUTE
|
2011-02-17 05:44:24 +00:00
|
|
|
if (type == RTN_MULTICAST) {
|
2005-04-16 22:20:36 +00:00
|
|
|
if (IN_DEV_MFORWARD(in_dev) &&
|
2011-04-28 21:48:42 +00:00
|
|
|
!ipv4_is_local_multicast(fl4->daddr)) {
|
2010-06-11 06:31:35 +00:00
|
|
|
rth->dst.input = ip_mr_input;
|
|
|
|
rth->dst.output = ip_mc_output;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
}
|
|
|
|
|
2017-06-17 17:42:42 +00:00
|
|
|
rt_set_nexthop(rth, fl4->daddr, res, fnhe, fi, type, 0, do_cache);
|
2018-02-14 04:32:04 +00:00
|
|
|
lwtunnel_set_redirect(&rth->dst);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-02-17 23:29:00 +00:00
|
|
|
return rth;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Major route resolver routine.
|
|
|
|
*/
|
|
|
|
|
2017-05-25 17:42:33 +00:00
|
|
|
struct rtable *ip_route_output_key_hash(struct net *net, struct flowi4 *fl4,
|
|
|
|
const struct sk_buff *skb)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-12-02 11:39:42 +00:00
|
|
|
__u8 tos = RT_FL_TOS(fl4);
|
2018-04-07 20:42:42 +00:00
|
|
|
struct fib_result res = {
|
|
|
|
.type = RTN_UNSPEC,
|
|
|
|
.fi = NULL,
|
|
|
|
.table = NULL,
|
|
|
|
.tclassid = 0,
|
|
|
|
};
|
2011-02-17 23:29:00 +00:00
|
|
|
struct rtable *rth;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2012-08-08 21:53:36 +00:00
|
|
|
fl4->flowi4_iif = LOOPBACK_IFINDEX;
|
2011-04-28 21:48:42 +00:00
|
|
|
fl4->flowi4_tos = tos & IPTOS_RT_MASK;
|
|
|
|
fl4->flowi4_scope = ((tos & RTO_ONLINK) ?
|
|
|
|
RT_SCOPE_LINK : RT_SCOPE_UNIVERSE);
|
ipv4: Optimize flow initialization in output route lookup.
We burn a lot of useless cycles, cpu store buffer traffic, and
memory operations memset()'ing the on-stack flow used to perform
output route lookups in __ip_route_output_key().
Only the first half of the flow object members even matter for
output route lookups in this context, specifically:
FIB rules matching cares about:
dst, src, tos, iif, oif, mark
FIB trie lookup cares about:
dst
FIB semantic match cares about:
tos, scope, oif
Therefore only initialize these specific members and elide the
memset entirely.
On Niagara2 this kills about ~300 cycles from the output route
lookup path.
Likely, we can take things further, since all callers of output
route lookups essentially throw away the on-stack flow they use.
So they don't care if we use it as a scratch-pad to compute the
final flow key.
Signed-off-by: David S. Miller <davem@davemloft.net>
Acked-by: Eric Dumazet <eric.dumazet@gmail.com>
2011-03-05 05:24:47 +00:00
|
|
|
|
2011-02-17 23:37:09 +00:00
|
|
|
rcu_read_lock();
|
2017-05-25 17:42:33 +00:00
|
|
|
rth = ip_route_output_key_hash_rcu(net, fl4, &res, skb);
|
|
|
|
rcu_read_unlock();
|
|
|
|
|
|
|
|
return rth;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(ip_route_output_key_hash);
|
|
|
|
|
|
|
|
struct rtable *ip_route_output_key_hash_rcu(struct net *net, struct flowi4 *fl4,
|
|
|
|
struct fib_result *res,
|
|
|
|
const struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct net_device *dev_out = NULL;
|
|
|
|
int orig_oif = fl4->flowi4_oif;
|
|
|
|
unsigned int flags = 0;
|
|
|
|
struct rtable *rth;
|
ipv4: Return -ENETUNREACH if we can't create route but saddr is valid
...instead of -EINVAL. An issue was found with older kernel versions
while unplugging a NFS client with pending RPCs, and the wrong error
code here prevented it from recovering once link is back up with a
configured address.
Incidentally, this is not an issue anymore since commit 4f8943f80883
("SUNRPC: Replace direct task wakeups from softirq context"), included
in 5.2-rc7, had the effect of decoupling the forwarding of this error
by using SO_ERROR in xs_wake_error(), as pointed out by Benjamin
Coddington.
To the best of my knowledge, this isn't currently causing any further
issue, but the error code doesn't look appropriate anyway, and we
might hit this in other paths as well.
In detail, as analysed by Gonzalo Siero, once the route is deleted
because the interface is down, and can't be resolved and we return
-EINVAL here, this ends up, courtesy of inet_sk_rebuild_header(),
as the socket error seen by tcp_write_err(), called by
tcp_retransmit_timer().
In turn, tcp_write_err() indirectly calls xs_error_report(), which
wakes up the RPC pending tasks with a status of -EINVAL. This is then
seen by call_status() in the SUN RPC implementation, which aborts the
RPC call calling rpc_exit(), instead of handling this as a
potentially temporary condition, i.e. as a timeout.
Return -EINVAL only if the input parameters passed to
ip_route_output_key_hash_rcu() are actually invalid (this is the case
if the specified source address is multicast, limited broadcast or
all zeroes), but return -ENETUNREACH in all cases where, at the given
moment, the given source address doesn't allow resolving the route.
While at it, drop the initialisation of err to -ENETUNREACH, which
was added to __ip_route_output_key() back then by commit
0315e3827048 ("net: Fix behaviour of unreachable, blackhole and
prohibit routes"), but actually had no effect, as it was, and is,
overwritten by the fib_lookup() return code assignment, and anyway
ignored in all other branches, including the if (fl4->saddr) one:
I find this rather confusing, as it would look like -ENETUNREACH is
the "default" error, while that statement has no effect.
Also note that after commit fc75fc8339e7 ("ipv4: dont create routes
on down devices"), we would get -ENETUNREACH if the device is down,
but -EINVAL if the source address is specified and we can't resolve
the route, and this appears to be rather inconsistent.
Reported-by: Stefan Walter <walteste@inf.ethz.ch>
Analysed-by: Benjamin Coddington <bcodding@redhat.com>
Analysed-by: Gonzalo Siero <gsierohu@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-16 18:52:09 +00:00
|
|
|
int err;
|
2017-05-25 17:42:33 +00:00
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (fl4->saddr) {
|
|
|
|
if (ipv4_is_multicast(fl4->saddr) ||
|
|
|
|
ipv4_is_lbcast(fl4->saddr) ||
|
ipv4: Return -ENETUNREACH if we can't create route but saddr is valid
...instead of -EINVAL. An issue was found with older kernel versions
while unplugging a NFS client with pending RPCs, and the wrong error
code here prevented it from recovering once link is back up with a
configured address.
Incidentally, this is not an issue anymore since commit 4f8943f80883
("SUNRPC: Replace direct task wakeups from softirq context"), included
in 5.2-rc7, had the effect of decoupling the forwarding of this error
by using SO_ERROR in xs_wake_error(), as pointed out by Benjamin
Coddington.
To the best of my knowledge, this isn't currently causing any further
issue, but the error code doesn't look appropriate anyway, and we
might hit this in other paths as well.
In detail, as analysed by Gonzalo Siero, once the route is deleted
because the interface is down, and can't be resolved and we return
-EINVAL here, this ends up, courtesy of inet_sk_rebuild_header(),
as the socket error seen by tcp_write_err(), called by
tcp_retransmit_timer().
In turn, tcp_write_err() indirectly calls xs_error_report(), which
wakes up the RPC pending tasks with a status of -EINVAL. This is then
seen by call_status() in the SUN RPC implementation, which aborts the
RPC call calling rpc_exit(), instead of handling this as a
potentially temporary condition, i.e. as a timeout.
Return -EINVAL only if the input parameters passed to
ip_route_output_key_hash_rcu() are actually invalid (this is the case
if the specified source address is multicast, limited broadcast or
all zeroes), but return -ENETUNREACH in all cases where, at the given
moment, the given source address doesn't allow resolving the route.
While at it, drop the initialisation of err to -ENETUNREACH, which
was added to __ip_route_output_key() back then by commit
0315e3827048 ("net: Fix behaviour of unreachable, blackhole and
prohibit routes"), but actually had no effect, as it was, and is,
overwritten by the fib_lookup() return code assignment, and anyway
ignored in all other branches, including the if (fl4->saddr) one:
I find this rather confusing, as it would look like -ENETUNREACH is
the "default" error, while that statement has no effect.
Also note that after commit fc75fc8339e7 ("ipv4: dont create routes
on down devices"), we would get -ENETUNREACH if the device is down,
but -EINVAL if the source address is specified and we can't resolve
the route, and this appears to be rather inconsistent.
Reported-by: Stefan Walter <walteste@inf.ethz.ch>
Analysed-by: Benjamin Coddington <bcodding@redhat.com>
Analysed-by: Gonzalo Siero <gsierohu@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-16 18:52:09 +00:00
|
|
|
ipv4_is_zeronet(fl4->saddr)) {
|
|
|
|
rth = ERR_PTR(-EINVAL);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
ipv4: Return -ENETUNREACH if we can't create route but saddr is valid
...instead of -EINVAL. An issue was found with older kernel versions
while unplugging a NFS client with pending RPCs, and the wrong error
code here prevented it from recovering once link is back up with a
configured address.
Incidentally, this is not an issue anymore since commit 4f8943f80883
("SUNRPC: Replace direct task wakeups from softirq context"), included
in 5.2-rc7, had the effect of decoupling the forwarding of this error
by using SO_ERROR in xs_wake_error(), as pointed out by Benjamin
Coddington.
To the best of my knowledge, this isn't currently causing any further
issue, but the error code doesn't look appropriate anyway, and we
might hit this in other paths as well.
In detail, as analysed by Gonzalo Siero, once the route is deleted
because the interface is down, and can't be resolved and we return
-EINVAL here, this ends up, courtesy of inet_sk_rebuild_header(),
as the socket error seen by tcp_write_err(), called by
tcp_retransmit_timer().
In turn, tcp_write_err() indirectly calls xs_error_report(), which
wakes up the RPC pending tasks with a status of -EINVAL. This is then
seen by call_status() in the SUN RPC implementation, which aborts the
RPC call calling rpc_exit(), instead of handling this as a
potentially temporary condition, i.e. as a timeout.
Return -EINVAL only if the input parameters passed to
ip_route_output_key_hash_rcu() are actually invalid (this is the case
if the specified source address is multicast, limited broadcast or
all zeroes), but return -ENETUNREACH in all cases where, at the given
moment, the given source address doesn't allow resolving the route.
While at it, drop the initialisation of err to -ENETUNREACH, which
was added to __ip_route_output_key() back then by commit
0315e3827048 ("net: Fix behaviour of unreachable, blackhole and
prohibit routes"), but actually had no effect, as it was, and is,
overwritten by the fib_lookup() return code assignment, and anyway
ignored in all other branches, including the if (fl4->saddr) one:
I find this rather confusing, as it would look like -ENETUNREACH is
the "default" error, while that statement has no effect.
Also note that after commit fc75fc8339e7 ("ipv4: dont create routes
on down devices"), we would get -ENETUNREACH if the device is down,
but -EINVAL if the source address is specified and we can't resolve
the route, and this appears to be rather inconsistent.
Reported-by: Stefan Walter <walteste@inf.ethz.ch>
Analysed-by: Benjamin Coddington <bcodding@redhat.com>
Analysed-by: Gonzalo Siero <gsierohu@redhat.com>
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-10-16 18:52:09 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
rth = ERR_PTR(-ENETUNREACH);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
/* I removed check for oif == dev_out->oif here.
|
2021-03-12 07:30:05 +00:00
|
|
|
* It was wrong for two reasons:
|
|
|
|
* 1. ip_dev_find(net, saddr) can return wrong iface, if saddr
|
|
|
|
* is assigned to multiple interfaces.
|
|
|
|
* 2. Moreover, we are allowed to send packets with saddr
|
|
|
|
* of another iface. --ANK
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (fl4->flowi4_oif == 0 &&
|
|
|
|
(ipv4_is_multicast(fl4->daddr) ||
|
|
|
|
ipv4_is_lbcast(fl4->daddr))) {
|
2008-10-01 14:28:28 +00:00
|
|
|
/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
|
2011-04-28 21:48:42 +00:00
|
|
|
dev_out = __ip_dev_find(net, fl4->saddr, false);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!dev_out)
|
2008-10-01 14:28:28 +00:00
|
|
|
goto out;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Special hack: user can direct multicasts
|
2021-03-12 07:30:05 +00:00
|
|
|
* and limited broadcast via necessary interface
|
|
|
|
* without fiddling with IP_MULTICAST_IF or IP_PKTINFO.
|
|
|
|
* This hack is not just for fun, it allows
|
|
|
|
* vic,vat and friends to work.
|
|
|
|
* They bind socket to loopback, set ttl to zero
|
|
|
|
* and expect that it will work.
|
|
|
|
* From the viewpoint of routing cache they are broken,
|
|
|
|
* because we are not allowed to build multicast path
|
|
|
|
* with loopback source addr (look, routing cache
|
|
|
|
* cannot know, that ttl is zero, so that packet
|
|
|
|
* will not leave this host and route is valid).
|
|
|
|
* Luckily, this hack is good workaround.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
fl4->flowi4_oif = dev_out->ifindex;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto make_route;
|
|
|
|
}
|
2008-10-01 14:28:28 +00:00
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!(fl4->flowi4_flags & FLOWI_FLAG_ANYSRC)) {
|
2008-10-01 14:28:28 +00:00
|
|
|
/* It is equivalent to inet_addr_type(saddr) == RTN_LOCAL */
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!__ip_dev_find(net, fl4->saddr, false))
|
2008-10-01 14:28:28 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (fl4->flowi4_oif) {
|
|
|
|
dev_out = dev_get_by_index_rcu(net, fl4->flowi4_oif);
|
2011-03-02 22:31:35 +00:00
|
|
|
rth = ERR_PTR(-ENODEV);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!dev_out)
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
2005-10-03 21:35:55 +00:00
|
|
|
|
|
|
|
/* RACE: Check return value of inet_select_addr instead. */
|
2010-12-22 04:39:39 +00:00
|
|
|
if (!(dev_out->flags & IFF_UP) || !__in_dev_get_rcu(dev_out)) {
|
2011-03-02 22:31:35 +00:00
|
|
|
rth = ERR_PTR(-ENETUNREACH);
|
2010-12-22 04:39:39 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2011-04-28 21:48:42 +00:00
|
|
|
if (ipv4_is_local_multicast(fl4->daddr) ||
|
2015-05-01 14:39:54 +00:00
|
|
|
ipv4_is_lbcast(fl4->daddr) ||
|
|
|
|
fl4->flowi4_proto == IPPROTO_IGMP) {
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!fl4->saddr)
|
|
|
|
fl4->saddr = inet_select_addr(dev_out, 0,
|
|
|
|
RT_SCOPE_LINK);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto make_route;
|
|
|
|
}
|
2013-10-04 15:04:48 +00:00
|
|
|
if (!fl4->saddr) {
|
2011-04-28 21:48:42 +00:00
|
|
|
if (ipv4_is_multicast(fl4->daddr))
|
|
|
|
fl4->saddr = inet_select_addr(dev_out, 0,
|
|
|
|
fl4->flowi4_scope);
|
|
|
|
else if (!fl4->daddr)
|
|
|
|
fl4->saddr = inet_select_addr(dev_out, 0,
|
|
|
|
RT_SCOPE_HOST);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!fl4->daddr) {
|
|
|
|
fl4->daddr = fl4->saddr;
|
|
|
|
if (!fl4->daddr)
|
|
|
|
fl4->daddr = fl4->saddr = htonl(INADDR_LOOPBACK);
|
2008-01-23 06:06:19 +00:00
|
|
|
dev_out = net->loopback_dev;
|
2012-08-08 21:53:36 +00:00
|
|
|
fl4->flowi4_oif = LOOPBACK_IFINDEX;
|
2017-05-25 17:42:33 +00:00
|
|
|
res->type = RTN_LOCAL;
|
2005-04-16 22:20:36 +00:00
|
|
|
flags |= RTCF_LOCAL;
|
|
|
|
goto make_route;
|
|
|
|
}
|
|
|
|
|
2017-05-25 17:42:33 +00:00
|
|
|
err = fib_lookup(net, fl4, res, 0);
|
2015-09-17 14:01:32 +00:00
|
|
|
if (err) {
|
2017-05-25 17:42:33 +00:00
|
|
|
res->fi = NULL;
|
|
|
|
res->table = NULL;
|
2016-10-12 20:20:11 +00:00
|
|
|
if (fl4->flowi4_oif &&
|
2016-10-31 22:54:00 +00:00
|
|
|
(ipv4_is_multicast(fl4->daddr) ||
|
|
|
|
!netif_index_is_l3_master(net, fl4->flowi4_oif))) {
|
2005-04-16 22:20:36 +00:00
|
|
|
/* Apparently, routing tables are wrong. Assume,
|
2021-03-12 07:30:05 +00:00
|
|
|
* that the destination is on link.
|
|
|
|
*
|
|
|
|
* WHY? DW.
|
|
|
|
* Because we are allowed to send to iface
|
|
|
|
* even if it has NO routes and NO assigned
|
|
|
|
* addresses. When oif is specified, routing
|
|
|
|
* tables are looked up with only one purpose:
|
|
|
|
* to catch if destination is gatewayed, rather than
|
|
|
|
* direct. Moreover, if MSG_DONTROUTE is set,
|
|
|
|
* we send packet, ignoring both routing tables
|
|
|
|
* and ifaddr state. --ANK
|
|
|
|
*
|
|
|
|
*
|
|
|
|
* We could make it even if oif is unknown,
|
|
|
|
* likely IPv6, but we do not.
|
2005-04-16 22:20:36 +00:00
|
|
|
*/
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
if (fl4->saddr == 0)
|
|
|
|
fl4->saddr = inet_select_addr(dev_out, 0,
|
|
|
|
RT_SCOPE_LINK);
|
2017-05-25 17:42:33 +00:00
|
|
|
res->type = RTN_UNICAST;
|
2005-04-16 22:20:36 +00:00
|
|
|
goto make_route;
|
|
|
|
}
|
2015-09-17 14:01:32 +00:00
|
|
|
rth = ERR_PTR(err);
|
2005-04-16 22:20:36 +00:00
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2017-05-25 17:42:33 +00:00
|
|
|
if (res->type == RTN_LOCAL) {
|
2011-04-28 21:48:42 +00:00
|
|
|
if (!fl4->saddr) {
|
2017-05-25 17:42:33 +00:00
|
|
|
if (res->fi->fib_prefsrc)
|
|
|
|
fl4->saddr = res->fi->fib_prefsrc;
|
2011-01-03 20:24:20 +00:00
|
|
|
else
|
2011-04-28 21:48:42 +00:00
|
|
|
fl4->saddr = fl4->daddr;
|
2011-01-03 20:24:20 +00:00
|
|
|
}
|
2016-09-10 19:09:54 +00:00
|
|
|
|
|
|
|
/* L3 master device is the loopback for that domain */
|
2017-05-25 17:42:33 +00:00
|
|
|
dev_out = l3mdev_master_dev_rcu(FIB_RES_DEV(*res)) ? :
|
2017-04-21 20:34:59 +00:00
|
|
|
net->loopback_dev;
|
2017-08-10 20:49:10 +00:00
|
|
|
|
|
|
|
/* make sure orig_oif points to fib result device even
|
|
|
|
* though packet rx/tx happens over loopback or l3mdev
|
|
|
|
*/
|
|
|
|
orig_oif = FIB_RES_OIF(*res);
|
|
|
|
|
2011-04-28 21:48:42 +00:00
|
|
|
fl4->flowi4_oif = dev_out->ifindex;
|
2005-04-16 22:20:36 +00:00
|
|
|
flags |= RTCF_LOCAL;
|
|
|
|
goto make_route;
|
|
|
|
}
|
|
|
|
|
2017-05-25 17:42:33 +00:00
|
|
|
fib_select_path(net, res, fl4, skb);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-05-25 17:42:33 +00:00
|
|
|
dev_out = FIB_RES_DEV(*res);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
make_route:
|
2017-05-25 17:42:33 +00:00
|
|
|
rth = __mkroute_output(res, fl4, orig_oif, dev_out, flags);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-02-17 23:37:09 +00:00
|
|
|
out:
|
2011-03-02 22:31:35 +00:00
|
|
|
return rth;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-08-10 03:12:12 +00:00
|
|
|
|
2007-05-25 01:17:54 +00:00
|
|
|
static struct dst_ops ipv4_dst_blackhole_ops = {
|
2021-03-10 00:38:09 +00:00
|
|
|
.family = AF_INET,
|
|
|
|
.default_advmss = ipv4_default_advmss,
|
|
|
|
.neigh_lookup = ipv4_neigh_lookup,
|
|
|
|
.check = dst_blackhole_check,
|
|
|
|
.cow_metrics = dst_blackhole_cow_metrics,
|
|
|
|
.update_pmtu = dst_blackhole_update_pmtu,
|
|
|
|
.redirect = dst_blackhole_redirect,
|
|
|
|
.mtu = dst_blackhole_mtu,
|
2007-05-25 01:17:54 +00:00
|
|
|
};
|
|
|
|
|
2011-03-01 22:59:04 +00:00
|
|
|
struct dst_entry *ipv4_blackhole_route(struct net *net, struct dst_entry *dst_orig)
|
2007-05-25 01:17:54 +00:00
|
|
|
{
|
2011-03-01 22:59:04 +00:00
|
|
|
struct rtable *ort = (struct rtable *) dst_orig;
|
2012-07-19 19:31:33 +00:00
|
|
|
struct rtable *rt;
|
2007-05-25 01:17:54 +00:00
|
|
|
|
2017-10-09 06:43:55 +00:00
|
|
|
rt = dst_alloc(&ipv4_dst_blackhole_ops, NULL, 1, DST_OBSOLETE_DEAD, 0);
|
2007-05-25 01:17:54 +00:00
|
|
|
if (rt) {
|
2010-06-11 06:31:35 +00:00
|
|
|
struct dst_entry *new = &rt->dst;
|
2007-05-25 01:17:54 +00:00
|
|
|
|
|
|
|
new->__use = 1;
|
2007-11-14 05:34:06 +00:00
|
|
|
new->input = dst_discard;
|
2015-10-07 21:48:47 +00:00
|
|
|
new->output = dst_discard_out;
|
2007-05-25 01:17:54 +00:00
|
|
|
|
2017-06-17 17:42:26 +00:00
|
|
|
new->dev = net->loopback_dev;
|
2021-08-05 11:55:27 +00:00
|
|
|
dev_hold(new->dev);
|
2007-05-25 01:17:54 +00:00
|
|
|
|
2012-07-17 21:44:26 +00:00
|
|
|
rt->rt_is_input = ort->rt_is_input;
|
2011-03-05 05:47:09 +00:00
|
|
|
rt->rt_iif = ort->rt_iif;
|
2012-07-10 13:58:42 +00:00
|
|
|
rt->rt_pmtu = ort->rt_pmtu;
|
2018-03-14 09:21:14 +00:00
|
|
|
rt->rt_mtu_locked = ort->rt_mtu_locked;
|
2007-05-25 01:17:54 +00:00
|
|
|
|
2013-07-30 00:33:53 +00:00
|
|
|
rt->rt_genid = rt_genid_ipv4(net);
|
2007-05-25 01:17:54 +00:00
|
|
|
rt->rt_flags = ort->rt_flags;
|
|
|
|
rt->rt_type = ort->rt_type;
|
2019-09-17 17:39:49 +00:00
|
|
|
rt->rt_uses_gateway = ort->rt_uses_gateway;
|
2019-04-05 23:30:27 +00:00
|
|
|
rt->rt_gw_family = ort->rt_gw_family;
|
|
|
|
if (rt->rt_gw_family == AF_INET)
|
|
|
|
rt->rt_gw4 = ort->rt_gw4;
|
2019-04-05 23:30:29 +00:00
|
|
|
else if (rt->rt_gw_family == AF_INET6)
|
|
|
|
rt->rt_gw6 = ort->rt_gw6;
|
2007-05-25 01:17:54 +00:00
|
|
|
|
2012-07-31 22:06:50 +00:00
|
|
|
INIT_LIST_HEAD(&rt->rt_uncached);
|
2007-05-25 01:17:54 +00:00
|
|
|
}
|
|
|
|
|
2011-03-01 22:59:04 +00:00
|
|
|
dst_release(dst_orig);
|
|
|
|
|
|
|
|
return rt ? &rt->dst : ERR_PTR(-ENOMEM);
|
2007-05-25 01:17:54 +00:00
|
|
|
}
|
|
|
|
|
2011-03-12 06:12:47 +00:00
|
|
|
struct rtable *ip_route_output_flow(struct net *net, struct flowi4 *flp4,
|
2015-09-25 14:39:10 +00:00
|
|
|
const struct sock *sk)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2011-03-12 06:12:47 +00:00
|
|
|
struct rtable *rt = __ip_route_output_key(net, flp4);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-03-02 22:31:35 +00:00
|
|
|
if (IS_ERR(rt))
|
|
|
|
return rt;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2020-10-09 18:01:01 +00:00
|
|
|
if (flp4->flowi4_proto) {
|
|
|
|
flp4->flowi4_oif = rt->dst.dev->ifindex;
|
2014-09-16 08:08:40 +00:00
|
|
|
rt = (struct rtable *)xfrm_lookup_route(net, &rt->dst,
|
|
|
|
flowi4_to_flowi(flp4),
|
|
|
|
sk, 0);
|
2020-10-09 18:01:01 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2011-03-02 22:31:35 +00:00
|
|
|
return rt;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2005-08-10 03:12:12 +00:00
|
|
|
EXPORT_SYMBOL_GPL(ip_route_output_flow);
|
|
|
|
|
2020-02-24 05:27:50 +00:00
|
|
|
struct rtable *ip_route_output_tunnel(struct sk_buff *skb,
|
|
|
|
struct net_device *dev,
|
|
|
|
struct net *net, __be32 *saddr,
|
|
|
|
const struct ip_tunnel_info *info,
|
|
|
|
u8 protocol, bool use_cache)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_DST_CACHE
|
|
|
|
struct dst_cache *dst_cache;
|
|
|
|
#endif
|
|
|
|
struct rtable *rt = NULL;
|
|
|
|
struct flowi4 fl4;
|
|
|
|
__u8 tos;
|
|
|
|
|
|
|
|
#ifdef CONFIG_DST_CACHE
|
|
|
|
dst_cache = (struct dst_cache *)&info->dst_cache;
|
|
|
|
if (use_cache) {
|
|
|
|
rt = dst_cache_get_ip4(dst_cache, saddr);
|
|
|
|
if (rt)
|
|
|
|
return rt;
|
|
|
|
}
|
|
|
|
#endif
|
|
|
|
memset(&fl4, 0, sizeof(fl4));
|
|
|
|
fl4.flowi4_mark = skb->mark;
|
|
|
|
fl4.flowi4_proto = protocol;
|
|
|
|
fl4.daddr = info->key.u.ipv4.dst;
|
|
|
|
fl4.saddr = info->key.u.ipv4.src;
|
|
|
|
tos = info->key.tos;
|
|
|
|
fl4.flowi4_tos = RT_TOS(tos);
|
|
|
|
|
|
|
|
rt = ip_route_output_key(net, &fl4);
|
|
|
|
if (IS_ERR(rt)) {
|
|
|
|
netdev_dbg(dev, "no route to %pI4\n", &fl4.daddr);
|
|
|
|
return ERR_PTR(-ENETUNREACH);
|
|
|
|
}
|
|
|
|
if (rt->dst.dev == dev) { /* is this necessary? */
|
|
|
|
netdev_dbg(dev, "circular route to %pI4\n", &fl4.daddr);
|
|
|
|
ip_rt_put(rt);
|
|
|
|
return ERR_PTR(-ELOOP);
|
|
|
|
}
|
|
|
|
#ifdef CONFIG_DST_CACHE
|
|
|
|
if (use_cache)
|
|
|
|
dst_cache_set_ip4(dst_cache, &rt->dst, fl4.saddr);
|
|
|
|
#endif
|
|
|
|
*saddr = fl4.saddr;
|
|
|
|
return rt;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(ip_route_output_tunnel);
|
|
|
|
|
2017-05-25 17:42:36 +00:00
|
|
|
/* called with rcu_read_lock held */
|
2018-05-22 21:03:27 +00:00
|
|
|
static int rt_fill_info(struct net *net, __be32 dst, __be32 src,
|
|
|
|
struct rtable *rt, u32 table_id, struct flowi4 *fl4,
|
2019-08-24 00:11:38 +00:00
|
|
|
struct sk_buff *skb, u32 portid, u32 seq,
|
|
|
|
unsigned int flags)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
struct rtmsg *r;
|
2006-08-18 01:15:17 +00:00
|
|
|
struct nlmsghdr *nlh;
|
2011-10-11 01:12:02 +00:00
|
|
|
unsigned long expires = 0;
|
2012-07-10 14:26:01 +00:00
|
|
|
u32 error;
|
2012-07-20 09:02:08 +00:00
|
|
|
u32 metrics[RTAX_MAX];
|
2006-08-18 01:15:17 +00:00
|
|
|
|
2019-08-24 00:11:38 +00:00
|
|
|
nlh = nlmsg_put(skb, portid, seq, RTM_NEWROUTE, sizeof(*r), flags);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!nlh)
|
2007-02-01 07:16:40 +00:00
|
|
|
return -EMSGSIZE;
|
2006-08-18 01:15:17 +00:00
|
|
|
|
|
|
|
r = nlmsg_data(nlh);
|
2005-04-16 22:20:36 +00:00
|
|
|
r->rtm_family = AF_INET;
|
|
|
|
r->rtm_dst_len = 32;
|
|
|
|
r->rtm_src_len = 0;
|
2019-06-21 15:45:22 +00:00
|
|
|
r->rtm_tos = fl4 ? fl4->flowi4_tos : 0;
|
2017-01-11 23:42:17 +00:00
|
|
|
r->rtm_table = table_id < 256 ? table_id : RT_TABLE_COMPAT;
|
2015-09-02 20:58:36 +00:00
|
|
|
if (nla_put_u32(skb, RTA_TABLE, table_id))
|
2012-04-02 00:39:02 +00:00
|
|
|
goto nla_put_failure;
|
2005-04-16 22:20:36 +00:00
|
|
|
r->rtm_type = rt->rt_type;
|
|
|
|
r->rtm_scope = RT_SCOPE_UNIVERSE;
|
|
|
|
r->rtm_protocol = RTPROT_UNSPEC;
|
|
|
|
r->rtm_flags = (rt->rt_flags & ~0xFFFF) | RTM_F_CLONED;
|
|
|
|
if (rt->rt_flags & RTCF_NOTIFY)
|
|
|
|
r->rtm_flags |= RTM_F_NOTIFY;
|
2015-01-23 11:01:26 +00:00
|
|
|
if (IPCB(skb)->flags & IPSKB_DOREDIRECT)
|
|
|
|
r->rtm_flags |= RTCF_DOREDIRECT;
|
2006-08-18 01:15:17 +00:00
|
|
|
|
2015-03-29 14:59:25 +00:00
|
|
|
if (nla_put_in_addr(skb, RTA_DST, dst))
|
2012-04-02 00:39:02 +00:00
|
|
|
goto nla_put_failure;
|
2012-07-01 02:02:56 +00:00
|
|
|
if (src) {
|
2005-04-16 22:20:36 +00:00
|
|
|
r->rtm_src_len = 32;
|
2015-03-29 14:59:25 +00:00
|
|
|
if (nla_put_in_addr(skb, RTA_SRC, src))
|
2012-04-02 00:39:02 +00:00
|
|
|
goto nla_put_failure;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2012-04-02 00:39:02 +00:00
|
|
|
if (rt->dst.dev &&
|
|
|
|
nla_put_u32(skb, RTA_OIF, rt->dst.dev->ifindex))
|
|
|
|
goto nla_put_failure;
|
2020-11-13 08:55:17 +00:00
|
|
|
if (rt->dst.lwtstate &&
|
|
|
|
lwtunnel_fill_encap(skb, rt->dst.lwtstate, RTA_ENCAP, RTA_ENCAP_TYPE) < 0)
|
|
|
|
goto nla_put_failure;
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2012-04-02 00:39:02 +00:00
|
|
|
if (rt->dst.tclassid &&
|
|
|
|
nla_put_u32(skb, RTA_FLOW, rt->dst.tclassid))
|
|
|
|
goto nla_put_failure;
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2019-06-21 15:45:22 +00:00
|
|
|
if (fl4 && !rt_is_input_route(rt) &&
|
2012-07-01 02:02:59 +00:00
|
|
|
fl4->saddr != src) {
|
2015-03-29 14:59:25 +00:00
|
|
|
if (nla_put_in_addr(skb, RTA_PREFSRC, fl4->saddr))
|
2012-04-02 00:39:02 +00:00
|
|
|
goto nla_put_failure;
|
|
|
|
}
|
2019-09-17 17:39:49 +00:00
|
|
|
if (rt->rt_uses_gateway) {
|
|
|
|
if (rt->rt_gw_family == AF_INET &&
|
|
|
|
nla_put_in_addr(skb, RTA_GATEWAY, rt->rt_gw4)) {
|
2019-04-05 23:30:29 +00:00
|
|
|
goto nla_put_failure;
|
2019-09-17 17:39:49 +00:00
|
|
|
} else if (rt->rt_gw_family == AF_INET6) {
|
|
|
|
int alen = sizeof(struct in6_addr);
|
|
|
|
struct nlattr *nla;
|
|
|
|
struct rtvia *via;
|
|
|
|
|
|
|
|
nla = nla_reserve(skb, RTA_VIA, alen + 2);
|
|
|
|
if (!nla)
|
|
|
|
goto nla_put_failure;
|
|
|
|
|
|
|
|
via = nla_data(nla);
|
|
|
|
via->rtvia_family = AF_INET6;
|
|
|
|
memcpy(via->rtvia_addr, &rt->rt_gw6, alen);
|
|
|
|
}
|
2019-04-05 23:30:29 +00:00
|
|
|
}
|
2006-08-18 01:15:17 +00:00
|
|
|
|
2012-10-08 00:56:54 +00:00
|
|
|
expires = rt->dst.expires;
|
|
|
|
if (expires) {
|
|
|
|
unsigned long now = jiffies;
|
|
|
|
|
|
|
|
if (time_before(now, expires))
|
|
|
|
expires -= now;
|
|
|
|
else
|
|
|
|
expires = 0;
|
|
|
|
}
|
|
|
|
|
2012-07-20 09:02:08 +00:00
|
|
|
memcpy(metrics, dst_metrics_ptr(&rt->dst), sizeof(metrics));
|
2012-10-08 00:56:54 +00:00
|
|
|
if (rt->rt_pmtu && expires)
|
2012-07-20 09:02:08 +00:00
|
|
|
metrics[RTAX_MTU - 1] = rt->rt_pmtu;
|
2018-03-14 09:21:14 +00:00
|
|
|
if (rt->rt_mtu_locked && expires)
|
|
|
|
metrics[RTAX_LOCK - 1] |= BIT(RTAX_MTU);
|
2012-07-20 09:02:08 +00:00
|
|
|
if (rtnetlink_put_metrics(skb, metrics) < 0)
|
2006-08-18 01:15:17 +00:00
|
|
|
goto nla_put_failure;
|
|
|
|
|
2019-06-21 15:45:22 +00:00
|
|
|
if (fl4) {
|
|
|
|
if (fl4->flowi4_mark &&
|
|
|
|
nla_put_u32(skb, RTA_MARK, fl4->flowi4_mark))
|
|
|
|
goto nla_put_failure;
|
2016-11-03 17:23:42 +00:00
|
|
|
|
2019-06-21 15:45:22 +00:00
|
|
|
if (!uid_eq(fl4->flowi4_uid, INVALID_UID) &&
|
|
|
|
nla_put_u32(skb, RTA_UID,
|
|
|
|
from_kuid_munged(current_user_ns(),
|
|
|
|
fl4->flowi4_uid)))
|
|
|
|
goto nla_put_failure;
|
2006-08-18 01:15:17 +00:00
|
|
|
|
2019-06-21 15:45:22 +00:00
|
|
|
if (rt_is_input_route(rt)) {
|
2012-12-04 01:03:07 +00:00
|
|
|
#ifdef CONFIG_IP_MROUTE
|
2019-06-21 15:45:22 +00:00
|
|
|
if (ipv4_is_multicast(dst) &&
|
|
|
|
!ipv4_is_local_multicast(dst) &&
|
|
|
|
IPV4_DEVCONF_ALL(net, MC_FORWARDING)) {
|
|
|
|
int err = ipmr_get_route(net, skb,
|
|
|
|
fl4->saddr, fl4->daddr,
|
|
|
|
r, portid);
|
|
|
|
|
|
|
|
if (err <= 0) {
|
|
|
|
if (err == 0)
|
|
|
|
return 0;
|
|
|
|
goto nla_put_failure;
|
|
|
|
}
|
|
|
|
} else
|
2012-12-04 01:03:07 +00:00
|
|
|
#endif
|
2019-06-21 15:45:22 +00:00
|
|
|
if (nla_put_u32(skb, RTA_IIF, fl4->flowi4_iif))
|
|
|
|
goto nla_put_failure;
|
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2019-06-21 15:45:22 +00:00
|
|
|
error = rt->dst.error;
|
|
|
|
|
2012-07-10 14:26:01 +00:00
|
|
|
if (rtnl_put_cacheinfo(skb, &rt->dst, 0, expires, error) < 0)
|
2006-11-27 17:27:07 +00:00
|
|
|
goto nla_put_failure;
|
2006-08-18 01:15:17 +00:00
|
|
|
|
2015-01-16 21:09:00 +00:00
|
|
|
nlmsg_end(skb, nlh);
|
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-08-18 01:15:17 +00:00
|
|
|
nla_put_failure:
|
2007-02-01 07:16:40 +00:00
|
|
|
nlmsg_cancel(skb, nlh);
|
|
|
|
return -EMSGSIZE;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
static int fnhe_dump_bucket(struct net *net, struct sk_buff *skb,
|
|
|
|
struct netlink_callback *cb, u32 table_id,
|
|
|
|
struct fnhe_hash_bucket *bucket, int genid,
|
2019-08-24 00:11:38 +00:00
|
|
|
int *fa_index, int fa_start, unsigned int flags)
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < FNHE_HASH_SIZE; i++) {
|
|
|
|
struct fib_nh_exception *fnhe;
|
|
|
|
|
|
|
|
for (fnhe = rcu_dereference(bucket[i].chain); fnhe;
|
|
|
|
fnhe = rcu_dereference(fnhe->fnhe_next)) {
|
|
|
|
struct rtable *rt;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (*fa_index < fa_start)
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
if (fnhe->fnhe_genid != genid)
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
if (fnhe->fnhe_expires &&
|
|
|
|
time_after(jiffies, fnhe->fnhe_expires))
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_input);
|
|
|
|
if (!rt)
|
|
|
|
rt = rcu_dereference(fnhe->fnhe_rth_output);
|
|
|
|
if (!rt)
|
|
|
|
goto next;
|
|
|
|
|
|
|
|
err = rt_fill_info(net, fnhe->fnhe_daddr, 0, rt,
|
|
|
|
table_id, NULL, skb,
|
|
|
|
NETLINK_CB(cb->skb).portid,
|
2019-08-24 00:11:38 +00:00
|
|
|
cb->nlh->nlmsg_seq, flags);
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
next:
|
|
|
|
(*fa_index)++;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
int fib_dump_info_fnhe(struct sk_buff *skb, struct netlink_callback *cb,
|
|
|
|
u32 table_id, struct fib_info *fi,
|
2019-08-24 00:11:38 +00:00
|
|
|
int *fa_index, int fa_start, unsigned int flags)
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
{
|
|
|
|
struct net *net = sock_net(cb->skb->sk);
|
|
|
|
int nhsel, genid = fnhe_genid(net);
|
|
|
|
|
|
|
|
for (nhsel = 0; nhsel < fib_info_num_path(fi); nhsel++) {
|
|
|
|
struct fib_nh_common *nhc = fib_info_nhc(fi, nhsel);
|
|
|
|
struct fnhe_hash_bucket *bucket;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (nhc->nhc_flags & RTNH_F_DEAD)
|
|
|
|
continue;
|
|
|
|
|
2019-06-26 10:04:50 +00:00
|
|
|
rcu_read_lock();
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
bucket = rcu_dereference(nhc->nhc_exceptions);
|
2019-06-26 10:04:50 +00:00
|
|
|
err = 0;
|
|
|
|
if (bucket)
|
|
|
|
err = fnhe_dump_bucket(net, skb, cb, table_id, bucket,
|
2019-08-24 00:11:38 +00:00
|
|
|
genid, fa_index, fa_start,
|
|
|
|
flags);
|
2019-06-26 10:04:50 +00:00
|
|
|
rcu_read_unlock();
|
ipv4: Dump route exceptions if requested
Since commit 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions."), cached
exception routes are stored as a separate entity, so they are not dumped
on a FIB dump, even if the RTM_F_CLONED flag is passed.
This implies that the command 'ip route list cache' doesn't return any
result anymore.
If the RTM_F_CLONED is passed, and strict checking requested, retrieve
nexthop exception routes and dump them. If no strict checking is
requested, filtering can't be performed consistently: dump everything in
that case.
With this, we need to add an argument to the netlink callback in order to
track how many entries were already dumped for the last leaf included in
a partial netlink dump.
A single additional argument is sufficient, even if we traverse logically
nested structures (nexthop objects, hash table buckets, bucket chains): it
doesn't matter if we stop in the middle of any of those, because they are
always traversed the same way. As an example, s_i values in [], s_fa
values in ():
node (fa) #1 [1]
nexthop #1
bucket #1 -> #0 in chain (1)
bucket #2 -> #0 in chain (2) -> #1 in chain (3) -> #2 in chain (4)
bucket #3 -> #0 in chain (5) -> #1 in chain (6)
nexthop #2
bucket #1 -> #0 in chain (7) -> #1 in chain (8)
bucket #2 -> #0 in chain (9)
--
node (fa) #2 [2]
nexthop #1
bucket #1 -> #0 in chain (1) -> #1 in chain (2)
bucket #2 -> #0 in chain (3)
it doesn't matter if we stop at (3), (4), (7) for "node #1", or at (2)
for "node #2": walking flattens all that.
It would even be possible to drop the distinction between the in-tree
(s_i) and in-node (s_fa) counter, but a further improvement might
advise against this. This is only as accurate as the existing tracking
mechanism for leaves: if a partial dump is restarted after exceptions
are removed or expired, we might skip some non-dumped entries.
To improve this, we could attach a 'sernum' attribute (similar to the
one used for IPv6) to nexthop entities, and bump this counter whenever
exceptions change: having a distinction between the two counters would
make this more convenient.
Listing of exception routes (modified routes pre-3.5) was tested against
these versions of kernel and iproute2:
iproute2
kernel 4.14.0 4.15.0 4.19.0 5.0.0 5.1.0
3.5-rc4 + + + + +
4.4
4.9
4.14
4.15
4.19
5.0
5.1
fixed + + + + +
v7:
- Move loop over nexthop objects to route.c, and pass struct fib_info
and table ID to it, not a struct fib_alias (suggested by David Ahern)
- While at it, note that the NULL check on fa->fa_info is redundant,
and the check on RTNH_F_DEAD is also not consistent with what's done
with regular route listing: just keep it for nhc_flags
- Rename entry point function for dumping exceptions to
fib_dump_info_fnhe(), and rearrange arguments for consistency with
fib_dump_info()
- Rename fnhe_dump_buckets() to fnhe_dump_bucket() and make it handle
one bucket at a time
- Expand commit message to describe why we can have a single "skip"
counter for all exceptions stored in bucket chains in nexthop objects
(suggested by David Ahern)
v6:
- Rebased onto net-next
- Loop over nexthop paths too. Move loop over fnhe buckets to route.c,
avoids need to export rt_fill_info() and to touch exceptions from
fib_trie.c. Pass NULL as flow to rt_fill_info(), it now allows that
(suggested by David Ahern)
Fixes: 4895c771c7f0 ("ipv4: Add FIB nexthop exceptions.")
Signed-off-by: Stefano Brivio <sbrivio@redhat.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-06-21 15:45:23 +00:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
static struct sk_buff *inet_rtm_getroute_build_skb(__be32 src, __be32 dst,
|
|
|
|
u8 ip_proto, __be16 sport,
|
|
|
|
__be16 dport)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
struct iphdr *iph;
|
|
|
|
|
|
|
|
skb = alloc_skb(NLMSG_GOODSIZE, GFP_KERNEL);
|
|
|
|
if (!skb)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Reserve room for dummy headers, this skb can pass
|
|
|
|
* through good chunk of routing engine.
|
|
|
|
*/
|
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
skb_reset_network_header(skb);
|
|
|
|
skb->protocol = htons(ETH_P_IP);
|
|
|
|
iph = skb_put(skb, sizeof(struct iphdr));
|
|
|
|
iph->protocol = ip_proto;
|
|
|
|
iph->saddr = src;
|
|
|
|
iph->daddr = dst;
|
|
|
|
iph->version = 0x4;
|
|
|
|
iph->frag_off = 0;
|
|
|
|
iph->ihl = 0x5;
|
|
|
|
skb_set_transport_header(skb, skb->len);
|
|
|
|
|
|
|
|
switch (iph->protocol) {
|
|
|
|
case IPPROTO_UDP: {
|
|
|
|
struct udphdr *udph;
|
|
|
|
|
|
|
|
udph = skb_put_zero(skb, sizeof(struct udphdr));
|
|
|
|
udph->source = sport;
|
|
|
|
udph->dest = dport;
|
2021-08-31 02:02:10 +00:00
|
|
|
udph->len = htons(sizeof(struct udphdr));
|
2018-05-22 21:03:27 +00:00
|
|
|
udph->check = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case IPPROTO_TCP: {
|
|
|
|
struct tcphdr *tcph;
|
|
|
|
|
|
|
|
tcph = skb_put_zero(skb, sizeof(struct tcphdr));
|
|
|
|
tcph->source = sport;
|
|
|
|
tcph->dest = dport;
|
|
|
|
tcph->doff = sizeof(struct tcphdr) / 4;
|
|
|
|
tcph->rst = 1;
|
|
|
|
tcph->check = ~tcp_v4_check(sizeof(struct tcphdr),
|
|
|
|
src, dst, 0);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
case IPPROTO_ICMP: {
|
|
|
|
struct icmphdr *icmph;
|
|
|
|
|
|
|
|
icmph = skb_put_zero(skb, sizeof(struct icmphdr));
|
|
|
|
icmph->type = ICMP_ECHO;
|
|
|
|
icmph->code = 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2019-01-18 18:46:19 +00:00
|
|
|
static int inet_rtm_valid_getroute_req(struct sk_buff *skb,
|
|
|
|
const struct nlmsghdr *nlh,
|
|
|
|
struct nlattr **tb,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
struct rtmsg *rtm;
|
|
|
|
int i, err;
|
|
|
|
|
|
|
|
if (nlh->nlmsg_len < nlmsg_msg_size(sizeof(*rtm))) {
|
|
|
|
NL_SET_ERR_MSG(extack,
|
|
|
|
"ipv4: Invalid header for route get request");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!netlink_strict_get_check(skb))
|
netlink: make validation more configurable for future strictness
We currently have two levels of strict validation:
1) liberal (default)
- undefined (type >= max) & NLA_UNSPEC attributes accepted
- attribute length >= expected accepted
- garbage at end of message accepted
2) strict (opt-in)
- NLA_UNSPEC attributes accepted
- attribute length >= expected accepted
Split out parsing strictness into four different options:
* TRAILING - check that there's no trailing data after parsing
attributes (in message or nested)
* MAXTYPE - reject attrs > max known type
* UNSPEC - reject attributes with NLA_UNSPEC policy entries
* STRICT_ATTRS - strictly validate attribute size
The default for future things should be *everything*.
The current *_strict() is a combination of TRAILING and MAXTYPE,
and is renamed to _deprecated_strict().
The current regular parsing has none of this, and is renamed to
*_parse_deprecated().
Additionally it allows us to selectively set one of the new flags
even on old policies. Notably, the UNSPEC flag could be useful in
this case, since it can be arranged (by filling in the policy) to
not be an incompatible userspace ABI change, but would then going
forward prevent forgetting attribute entries. Similar can apply
to the POLICY flag.
We end up with the following renames:
* nla_parse -> nla_parse_deprecated
* nla_parse_strict -> nla_parse_deprecated_strict
* nlmsg_parse -> nlmsg_parse_deprecated
* nlmsg_parse_strict -> nlmsg_parse_deprecated_strict
* nla_parse_nested -> nla_parse_nested_deprecated
* nla_validate_nested -> nla_validate_nested_deprecated
Using spatch, of course:
@@
expression TB, MAX, HEAD, LEN, POL, EXT;
@@
-nla_parse(TB, MAX, HEAD, LEN, POL, EXT)
+nla_parse_deprecated(TB, MAX, HEAD, LEN, POL, EXT)
@@
expression NLH, HDRLEN, TB, MAX, POL, EXT;
@@
-nlmsg_parse(NLH, HDRLEN, TB, MAX, POL, EXT)
+nlmsg_parse_deprecated(NLH, HDRLEN, TB, MAX, POL, EXT)
@@
expression NLH, HDRLEN, TB, MAX, POL, EXT;
@@
-nlmsg_parse_strict(NLH, HDRLEN, TB, MAX, POL, EXT)
+nlmsg_parse_deprecated_strict(NLH, HDRLEN, TB, MAX, POL, EXT)
@@
expression TB, MAX, NLA, POL, EXT;
@@
-nla_parse_nested(TB, MAX, NLA, POL, EXT)
+nla_parse_nested_deprecated(TB, MAX, NLA, POL, EXT)
@@
expression START, MAX, POL, EXT;
@@
-nla_validate_nested(START, MAX, POL, EXT)
+nla_validate_nested_deprecated(START, MAX, POL, EXT)
@@
expression NLH, HDRLEN, MAX, POL, EXT;
@@
-nlmsg_validate(NLH, HDRLEN, MAX, POL, EXT)
+nlmsg_validate_deprecated(NLH, HDRLEN, MAX, POL, EXT)
For this patch, don't actually add the strict, non-renamed versions
yet so that it breaks compile if I get it wrong.
Also, while at it, make nla_validate and nla_parse go down to a
common __nla_validate_parse() function to avoid code duplication.
Ultimately, this allows us to have very strict validation for every
new caller of nla_parse()/nlmsg_parse() etc as re-introduced in the
next patch, while existing things will continue to work as is.
In effect then, this adds fully strict validation for any new command.
Signed-off-by: Johannes Berg <johannes.berg@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-04-26 12:07:28 +00:00
|
|
|
return nlmsg_parse_deprecated(nlh, sizeof(*rtm), tb, RTA_MAX,
|
|
|
|
rtm_ipv4_policy, extack);
|
2019-01-18 18:46:19 +00:00
|
|
|
|
|
|
|
rtm = nlmsg_data(nlh);
|
|
|
|
if ((rtm->rtm_src_len && rtm->rtm_src_len != 32) ||
|
|
|
|
(rtm->rtm_dst_len && rtm->rtm_dst_len != 32) ||
|
|
|
|
rtm->rtm_table || rtm->rtm_protocol ||
|
|
|
|
rtm->rtm_scope || rtm->rtm_type) {
|
|
|
|
NL_SET_ERR_MSG(extack, "ipv4: Invalid values in header for route get request");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (rtm->rtm_flags & ~(RTM_F_NOTIFY |
|
|
|
|
RTM_F_LOOKUP_TABLE |
|
|
|
|
RTM_F_FIB_MATCH)) {
|
|
|
|
NL_SET_ERR_MSG(extack, "ipv4: Unsupported rtm_flags for route get request");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
netlink: make validation more configurable for future strictness
We currently have two levels of strict validation:
1) liberal (default)
- undefined (type >= max) & NLA_UNSPEC attributes accepted
- attribute length >= expected accepted
- garbage at end of message accepted
2) strict (opt-in)
- NLA_UNSPEC attributes accepted
- attribute length >= expected accepted
Split out parsing strictness into four different options:
* TRAILING - check that there's no trailing data after parsing
attributes (in message or nested)
* MAXTYPE - reject attrs > max known type
* UNSPEC - reject attributes with NLA_UNSPEC policy entries
* STRICT_ATTRS - strictly validate attribute size
The default for future things should be *everything*.
The current *_strict() is a combination of TRAILING and MAXTYPE,
and is renamed to _deprecated_strict().
The current regular parsing has none of this, and is renamed to
*_parse_deprecated().
Additionally it allows us to selectively set one of the new flags
even on old policies. Notably, the UNSPEC flag could be useful in
this case, since it can be arranged (by filling in the policy) to
not be an incompatible userspace ABI change, but would then going
forward prevent forgetting attribute entries. Similar can apply
to the POLICY flag.
We end up with the following renames:
* nla_parse -> nla_parse_deprecated
* nla_parse_strict -> nla_parse_deprecated_strict
* nlmsg_parse -> nlmsg_parse_deprecated
* nlmsg_parse_strict -> nlmsg_parse_deprecated_strict
* nla_parse_nested -> nla_parse_nested_deprecated
* nla_validate_nested -> nla_validate_nested_deprecated
Using spatch, of course:
@@
expression TB, MAX, HEAD, LEN, POL, EXT;
@@
-nla_parse(TB, MAX, HEAD, LEN, POL, EXT)
+nla_parse_deprecated(TB, MAX, HEAD, LEN, POL, EXT)
@@
expression NLH, HDRLEN, TB, MAX, POL, EXT;
@@
-nlmsg_parse(NLH, HDRLEN, TB, MAX, POL, EXT)
+nlmsg_parse_deprecated(NLH, HDRLEN, TB, MAX, POL, EXT)
@@
expression NLH, HDRLEN, TB, MAX, POL, EXT;
@@
-nlmsg_parse_strict(NLH, HDRLEN, TB, MAX, POL, EXT)
+nlmsg_parse_deprecated_strict(NLH, HDRLEN, TB, MAX, POL, EXT)
@@
expression TB, MAX, NLA, POL, EXT;
@@
-nla_parse_nested(TB, MAX, NLA, POL, EXT)
+nla_parse_nested_deprecated(TB, MAX, NLA, POL, EXT)
@@
expression START, MAX, POL, EXT;
@@
-nla_validate_nested(START, MAX, POL, EXT)
+nla_validate_nested_deprecated(START, MAX, POL, EXT)
@@
expression NLH, HDRLEN, MAX, POL, EXT;
@@
-nlmsg_validate(NLH, HDRLEN, MAX, POL, EXT)
+nlmsg_validate_deprecated(NLH, HDRLEN, MAX, POL, EXT)
For this patch, don't actually add the strict, non-renamed versions
yet so that it breaks compile if I get it wrong.
Also, while at it, make nla_validate and nla_parse go down to a
common __nla_validate_parse() function to avoid code duplication.
Ultimately, this allows us to have very strict validation for every
new caller of nla_parse()/nlmsg_parse() etc as re-introduced in the
next patch, while existing things will continue to work as is.
In effect then, this adds fully strict validation for any new command.
Signed-off-by: Johannes Berg <johannes.berg@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-04-26 12:07:28 +00:00
|
|
|
err = nlmsg_parse_deprecated_strict(nlh, sizeof(*rtm), tb, RTA_MAX,
|
|
|
|
rtm_ipv4_policy, extack);
|
2019-01-18 18:46:19 +00:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
if ((tb[RTA_SRC] && !rtm->rtm_src_len) ||
|
|
|
|
(tb[RTA_DST] && !rtm->rtm_dst_len)) {
|
|
|
|
NL_SET_ERR_MSG(extack, "ipv4: rtm_src_len and rtm_dst_len must be 32 for IPv4");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
for (i = 0; i <= RTA_MAX; i++) {
|
|
|
|
if (!tb[i])
|
|
|
|
continue;
|
|
|
|
|
|
|
|
switch (i) {
|
|
|
|
case RTA_IIF:
|
|
|
|
case RTA_OIF:
|
|
|
|
case RTA_SRC:
|
|
|
|
case RTA_DST:
|
|
|
|
case RTA_IP_PROTO:
|
|
|
|
case RTA_SPORT:
|
|
|
|
case RTA_DPORT:
|
|
|
|
case RTA_MARK:
|
|
|
|
case RTA_UID:
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
NL_SET_ERR_MSG(extack, "ipv4: Unsupported attribute in route get request");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-04-16 16:48:24 +00:00
|
|
|
static int inet_rtm_getroute(struct sk_buff *in_skb, struct nlmsghdr *nlh,
|
|
|
|
struct netlink_ext_ack *extack)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2008-03-25 17:26:21 +00:00
|
|
|
struct net *net = sock_net(in_skb->sk);
|
2006-08-18 01:15:44 +00:00
|
|
|
struct nlattr *tb[RTA_MAX+1];
|
2018-05-22 21:03:27 +00:00
|
|
|
u32 table_id = RT_TABLE_MAIN;
|
|
|
|
__be16 sport = 0, dport = 0;
|
2017-05-25 17:42:36 +00:00
|
|
|
struct fib_result res = {};
|
2018-05-22 21:03:27 +00:00
|
|
|
u8 ip_proto = IPPROTO_UDP;
|
2005-04-16 22:20:36 +00:00
|
|
|
struct rtable *rt = NULL;
|
2018-05-22 21:03:27 +00:00
|
|
|
struct sk_buff *skb;
|
|
|
|
struct rtmsg *rtm;
|
2018-09-30 06:44:47 +00:00
|
|
|
struct flowi4 fl4 = {};
|
2006-09-27 04:25:20 +00:00
|
|
|
__be32 dst = 0;
|
|
|
|
__be32 src = 0;
|
2018-05-22 21:03:27 +00:00
|
|
|
kuid_t uid;
|
2006-09-27 04:25:20 +00:00
|
|
|
u32 iif;
|
2006-08-18 01:15:44 +00:00
|
|
|
int err;
|
2010-07-20 22:03:14 +00:00
|
|
|
int mark;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2019-01-18 18:46:19 +00:00
|
|
|
err = inet_rtm_valid_getroute_req(in_skb, nlh, tb, extack);
|
2006-08-18 01:15:44 +00:00
|
|
|
if (err < 0)
|
2018-05-22 21:03:27 +00:00
|
|
|
return err;
|
2006-08-18 01:15:44 +00:00
|
|
|
|
|
|
|
rtm = nlmsg_data(nlh);
|
2015-03-29 14:59:26 +00:00
|
|
|
src = tb[RTA_SRC] ? nla_get_in_addr(tb[RTA_SRC]) : 0;
|
|
|
|
dst = tb[RTA_DST] ? nla_get_in_addr(tb[RTA_DST]) : 0;
|
2006-08-18 01:15:44 +00:00
|
|
|
iif = tb[RTA_IIF] ? nla_get_u32(tb[RTA_IIF]) : 0;
|
2010-07-20 22:03:14 +00:00
|
|
|
mark = tb[RTA_MARK] ? nla_get_u32(tb[RTA_MARK]) : 0;
|
2016-11-03 17:23:42 +00:00
|
|
|
if (tb[RTA_UID])
|
|
|
|
uid = make_kuid(current_user_ns(), nla_get_u32(tb[RTA_UID]));
|
|
|
|
else
|
|
|
|
uid = (iif ? INVALID_UID : current_uid());
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
if (tb[RTA_IP_PROTO]) {
|
|
|
|
err = rtm_getroute_parse_ip_proto(tb[RTA_IP_PROTO],
|
2019-02-27 08:15:29 +00:00
|
|
|
&ip_proto, AF_INET, extack);
|
2018-05-22 21:03:27 +00:00
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
}
|
2017-04-07 12:42:20 +00:00
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
if (tb[RTA_SPORT])
|
|
|
|
sport = nla_get_be16(tb[RTA_SPORT]);
|
2017-04-07 12:42:20 +00:00
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
if (tb[RTA_DPORT])
|
|
|
|
dport = nla_get_be16(tb[RTA_DPORT]);
|
|
|
|
|
|
|
|
skb = inet_rtm_getroute_build_skb(src, dst, ip_proto, sport, dport);
|
|
|
|
if (!skb)
|
|
|
|
return -ENOBUFS;
|
2017-04-07 12:42:20 +00:00
|
|
|
|
2012-07-01 02:02:59 +00:00
|
|
|
fl4.daddr = dst;
|
|
|
|
fl4.saddr = src;
|
2020-11-26 18:09:22 +00:00
|
|
|
fl4.flowi4_tos = rtm->rtm_tos & IPTOS_RT_MASK;
|
2012-07-01 02:02:59 +00:00
|
|
|
fl4.flowi4_oif = tb[RTA_OIF] ? nla_get_u32(tb[RTA_OIF]) : 0;
|
|
|
|
fl4.flowi4_mark = mark;
|
2016-11-03 17:23:42 +00:00
|
|
|
fl4.flowi4_uid = uid;
|
2018-05-22 21:03:27 +00:00
|
|
|
if (sport)
|
|
|
|
fl4.fl4_sport = sport;
|
|
|
|
if (dport)
|
|
|
|
fl4.fl4_dport = dport;
|
|
|
|
fl4.flowi4_proto = ip_proto;
|
2012-07-01 02:02:59 +00:00
|
|
|
|
2017-05-25 17:42:36 +00:00
|
|
|
rcu_read_lock();
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (iif) {
|
2006-08-18 01:15:44 +00:00
|
|
|
struct net_device *dev;
|
|
|
|
|
2017-05-25 17:42:36 +00:00
|
|
|
dev = dev_get_by_index_rcu(net, iif);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!dev) {
|
2006-08-18 01:15:44 +00:00
|
|
|
err = -ENODEV;
|
2018-05-22 21:03:27 +00:00
|
|
|
goto errout_rcu;
|
2006-08-18 01:15:44 +00:00
|
|
|
}
|
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
fl4.flowi4_iif = iif; /* for rt_fill_info */
|
2005-04-16 22:20:36 +00:00
|
|
|
skb->dev = dev;
|
2010-07-20 22:03:14 +00:00
|
|
|
skb->mark = mark;
|
2020-11-26 18:09:22 +00:00
|
|
|
err = ip_route_input_rcu(skb, dst, src,
|
|
|
|
rtm->rtm_tos & IPTOS_RT_MASK, dev,
|
|
|
|
&res);
|
2006-08-18 01:15:44 +00:00
|
|
|
|
2009-06-02 05:14:27 +00:00
|
|
|
rt = skb_rtable(skb);
|
2010-06-11 06:31:35 +00:00
|
|
|
if (err == 0 && rt->dst.error)
|
|
|
|
err = -rt->dst.error;
|
2005-04-16 22:20:36 +00:00
|
|
|
} else {
|
2018-01-11 09:36:26 +00:00
|
|
|
fl4.flowi4_iif = LOOPBACK_IFINDEX;
|
2018-12-20 17:03:27 +00:00
|
|
|
skb->dev = net->loopback_dev;
|
2017-05-25 17:42:36 +00:00
|
|
|
rt = ip_route_output_key_hash_rcu(net, &fl4, &res, skb);
|
2011-03-02 22:31:35 +00:00
|
|
|
err = 0;
|
|
|
|
if (IS_ERR(rt))
|
|
|
|
err = PTR_ERR(rt);
|
2017-08-13 22:52:58 +00:00
|
|
|
else
|
|
|
|
skb_dst_set(skb, &rt->dst);
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
2006-08-18 01:15:44 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (err)
|
2018-05-22 21:03:27 +00:00
|
|
|
goto errout_rcu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
if (rtm->rtm_flags & RTM_F_NOTIFY)
|
|
|
|
rt->rt_flags |= RTCF_NOTIFY;
|
|
|
|
|
2015-09-02 20:58:36 +00:00
|
|
|
if (rtm->rtm_flags & RTM_F_LOOKUP_TABLE)
|
2018-02-14 22:24:28 +00:00
|
|
|
table_id = res.table ? res.table->tb_id : 0;
|
2015-09-02 20:58:36 +00:00
|
|
|
|
2018-05-22 21:03:27 +00:00
|
|
|
/* reset skb for netlink reply msg */
|
|
|
|
skb_trim(skb, 0);
|
|
|
|
skb_reset_network_header(skb);
|
|
|
|
skb_reset_transport_header(skb);
|
|
|
|
skb_reset_mac_header(skb);
|
|
|
|
|
2017-08-16 19:38:52 +00:00
|
|
|
if (rtm->rtm_flags & RTM_F_FIB_MATCH) {
|
2020-01-14 11:23:10 +00:00
|
|
|
struct fib_rt_info fri;
|
|
|
|
|
2017-08-16 19:38:52 +00:00
|
|
|
if (!res.fi) {
|
|
|
|
err = fib_props[res.type].error;
|
|
|
|
if (!err)
|
|
|
|
err = -EHOSTUNREACH;
|
2018-05-22 21:03:27 +00:00
|
|
|
goto errout_rcu;
|
2017-08-16 19:38:52 +00:00
|
|
|
}
|
2020-01-14 11:23:10 +00:00
|
|
|
fri.fi = res.fi;
|
|
|
|
fri.tb_id = table_id;
|
|
|
|
fri.dst = res.prefix;
|
|
|
|
fri.dst_len = res.prefixlen;
|
|
|
|
fri.tos = fl4.flowi4_tos;
|
|
|
|
fri.type = rt->rt_type;
|
ipv4: Add "offload" and "trap" indications to routes
When performing L3 offload, routes and nexthops are usually programmed
into two different tables in the underlying device. Therefore, the fact
that a nexthop resides in hardware does not necessarily mean that all
the associated routes also reside in hardware and vice-versa.
While the kernel can signal to user space the presence of a nexthop in
hardware (via 'RTNH_F_OFFLOAD'), it does not have a corresponding flag
for routes. In addition, the fact that a route resides in hardware does
not necessarily mean that the traffic is offloaded. For example,
unreachable routes (i.e., 'RTN_UNREACHABLE') are programmed to trap
packets to the CPU so that the kernel will be able to generate the
appropriate ICMP error packet.
This patch adds an "offload" and "trap" indications to IPv4 routes, so
that users will have better visibility into the offload process.
'struct fib_alias' is extended with two new fields that indicate if the
route resides in hardware or not and if it is offloading traffic from
the kernel or trapping packets to it. Note that the new fields are added
in the 6 bytes hole and therefore the struct still fits in a single
cache line [1].
Capable drivers are expected to invoke fib_alias_hw_flags_set() with the
route's key in order to set the flags.
The indications are dumped to user space via a new flags (i.e.,
'RTM_F_OFFLOAD' and 'RTM_F_TRAP') in the 'rtm_flags' field in the
ancillary header.
v2:
* Make use of 'struct fib_rt_info' in fib_alias_hw_flags_set()
[1]
struct fib_alias {
struct hlist_node fa_list; /* 0 16 */
struct fib_info * fa_info; /* 16 8 */
u8 fa_tos; /* 24 1 */
u8 fa_type; /* 25 1 */
u8 fa_state; /* 26 1 */
u8 fa_slen; /* 27 1 */
u32 tb_id; /* 28 4 */
s16 fa_default; /* 32 2 */
u8 offload:1; /* 34: 0 1 */
u8 trap:1; /* 34: 1 1 */
u8 unused:6; /* 34: 2 1 */
/* XXX 5 bytes hole, try to pack */
struct callback_head rcu __attribute__((__aligned__(8))); /* 40 16 */
/* size: 56, cachelines: 1, members: 12 */
/* sum members: 50, holes: 1, sum holes: 5 */
/* sum bitfield members: 8 bits (1 bytes) */
/* forced alignments: 1, forced holes: 1, sum forced holes: 5 */
/* last cacheline: 56 bytes */
} __attribute__((__aligned__(8)));
Signed-off-by: Ido Schimmel <idosch@mellanox.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Reviewed-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-14 11:23:11 +00:00
|
|
|
fri.offload = 0;
|
|
|
|
fri.trap = 0;
|
IPv4: Add "offload failed" indication to routes
After installing a route to the kernel, user space receives an
acknowledgment, which means the route was installed in the kernel, but not
necessarily in hardware.
The asynchronous nature of route installation in hardware can lead to a
routing daemon advertising a route before it was actually installed in
hardware. This can result in packet loss or mis-routed packets until the
route is installed in hardware.
To avoid such cases, previous patch set added the ability to emit
RTM_NEWROUTE notifications whenever RTM_F_OFFLOAD/RTM_F_TRAP flags
are changed, this behavior is controlled by sysctl.
With the above mentioned behavior, it is possible to know from user-space
if the route was offloaded, but if the offload fails there is no indication
to user-space. Following a failure, a routing daemon will wait indefinitely
for a notification that will never come.
This patch adds an "offload_failed" indication to IPv4 routes, so that
users will have better visibility into the offload process.
'struct fib_alias', and 'struct fib_rt_info' are extended with new field
that indicates if route offload failed. Note that the new field is added
using unused bit and therefore there is no need to increase structs size.
Signed-off-by: Amit Cohen <amcohen@nvidia.com>
Signed-off-by: Ido Schimmel <idosch@nvidia.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2021-02-07 08:22:50 +00:00
|
|
|
fri.offload_failed = 0;
|
ipv4: Add "offload" and "trap" indications to routes
When performing L3 offload, routes and nexthops are usually programmed
into two different tables in the underlying device. Therefore, the fact
that a nexthop resides in hardware does not necessarily mean that all
the associated routes also reside in hardware and vice-versa.
While the kernel can signal to user space the presence of a nexthop in
hardware (via 'RTNH_F_OFFLOAD'), it does not have a corresponding flag
for routes. In addition, the fact that a route resides in hardware does
not necessarily mean that the traffic is offloaded. For example,
unreachable routes (i.e., 'RTN_UNREACHABLE') are programmed to trap
packets to the CPU so that the kernel will be able to generate the
appropriate ICMP error packet.
This patch adds an "offload" and "trap" indications to IPv4 routes, so
that users will have better visibility into the offload process.
'struct fib_alias' is extended with two new fields that indicate if the
route resides in hardware or not and if it is offloading traffic from
the kernel or trapping packets to it. Note that the new fields are added
in the 6 bytes hole and therefore the struct still fits in a single
cache line [1].
Capable drivers are expected to invoke fib_alias_hw_flags_set() with the
route's key in order to set the flags.
The indications are dumped to user space via a new flags (i.e.,
'RTM_F_OFFLOAD' and 'RTM_F_TRAP') in the 'rtm_flags' field in the
ancillary header.
v2:
* Make use of 'struct fib_rt_info' in fib_alias_hw_flags_set()
[1]
struct fib_alias {
struct hlist_node fa_list; /* 0 16 */
struct fib_info * fa_info; /* 16 8 */
u8 fa_tos; /* 24 1 */
u8 fa_type; /* 25 1 */
u8 fa_state; /* 26 1 */
u8 fa_slen; /* 27 1 */
u32 tb_id; /* 28 4 */
s16 fa_default; /* 32 2 */
u8 offload:1; /* 34: 0 1 */
u8 trap:1; /* 34: 1 1 */
u8 unused:6; /* 34: 2 1 */
/* XXX 5 bytes hole, try to pack */
struct callback_head rcu __attribute__((__aligned__(8))); /* 40 16 */
/* size: 56, cachelines: 1, members: 12 */
/* sum members: 50, holes: 1, sum holes: 5 */
/* sum bitfield members: 8 bits (1 bytes) */
/* forced alignments: 1, forced holes: 1, sum forced holes: 5 */
/* last cacheline: 56 bytes */
} __attribute__((__aligned__(8)));
Signed-off-by: Ido Schimmel <idosch@mellanox.com>
Reviewed-by: David Ahern <dsahern@gmail.com>
Reviewed-by: Jiri Pirko <jiri@mellanox.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2020-01-14 11:23:11 +00:00
|
|
|
if (res.fa_head) {
|
|
|
|
struct fib_alias *fa;
|
|
|
|
|
|
|
|
hlist_for_each_entry_rcu(fa, res.fa_head, fa_list) {
|
|
|
|
u8 slen = 32 - fri.dst_len;
|
|
|
|
|
|
|
|
if (fa->fa_slen == slen &&
|
|
|
|
fa->tb_id == fri.tb_id &&
|
|
|
|
fa->fa_tos == fri.tos &&
|
|
|
|
fa->fa_info == res.fi &&
|
|
|
|
fa->fa_type == fri.type) {
|
|
|
|
fri.offload = fa->offload;
|
|
|
|
fri.trap = fa->trap;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
2017-05-25 17:42:39 +00:00
|
|
|
err = fib_dump_info(skb, NETLINK_CB(in_skb).portid,
|
2020-01-14 11:23:10 +00:00
|
|
|
nlh->nlmsg_seq, RTM_NEWROUTE, &fri, 0);
|
2017-08-16 19:38:52 +00:00
|
|
|
} else {
|
2018-05-22 21:03:27 +00:00
|
|
|
err = rt_fill_info(net, dst, src, rt, table_id, &fl4, skb,
|
2019-08-24 00:11:38 +00:00
|
|
|
NETLINK_CB(in_skb).portid,
|
|
|
|
nlh->nlmsg_seq, 0);
|
2017-08-16 19:38:52 +00:00
|
|
|
}
|
2015-01-19 04:36:08 +00:00
|
|
|
if (err < 0)
|
2018-05-22 21:03:27 +00:00
|
|
|
goto errout_rcu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2017-05-25 17:42:36 +00:00
|
|
|
rcu_read_unlock();
|
|
|
|
|
2012-09-07 20:12:54 +00:00
|
|
|
err = rtnl_unicast(skb, net, NETLINK_CB(in_skb).portid);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2006-08-18 01:15:44 +00:00
|
|
|
errout_free:
|
2018-05-22 21:03:27 +00:00
|
|
|
return err;
|
|
|
|
errout_rcu:
|
2017-05-25 17:42:36 +00:00
|
|
|
rcu_read_unlock();
|
2005-04-16 22:20:36 +00:00
|
|
|
kfree_skb(skb);
|
2018-05-22 21:03:27 +00:00
|
|
|
goto errout_free;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
void ip_rt_multicast_event(struct in_device *in_dev)
|
|
|
|
{
|
2012-09-07 00:45:29 +00:00
|
|
|
rt_cache_flush(dev_net(in_dev->dev));
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
#ifdef CONFIG_SYSCTL
|
2013-02-19 00:43:12 +00:00
|
|
|
static int ip_rt_gc_interval __read_mostly = 60 * HZ;
|
|
|
|
static int ip_rt_gc_min_interval __read_mostly = HZ / 2;
|
|
|
|
static int ip_rt_gc_elasticity __read_mostly = 8;
|
2018-02-28 13:32:48 +00:00
|
|
|
static int ip_min_valid_pmtu __read_mostly = IPV4_MIN_MTU;
|
2013-02-19 00:43:12 +00:00
|
|
|
|
2013-06-12 06:04:25 +00:00
|
|
|
static int ipv4_sysctl_rtcache_flush(struct ctl_table *__ctl, int write,
|
2020-04-24 06:43:38 +00:00
|
|
|
void *buffer, size_t *lenp, loff_t *ppos)
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
2013-05-27 20:46:33 +00:00
|
|
|
struct net *net = (struct net *)__ctl->extra1;
|
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
if (write) {
|
2013-05-27 20:46:33 +00:00
|
|
|
rt_cache_flush(net);
|
|
|
|
fnhe_genid_bump(net);
|
2005-04-16 22:20:36 +00:00
|
|
|
return 0;
|
2007-02-09 14:24:47 +00:00
|
|
|
}
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2013-06-12 06:04:25 +00:00
|
|
|
static struct ctl_table ipv4_route_table[] = {
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
.procname = "gc_thresh",
|
|
|
|
.data = &ipv4_dst_ops.gc_thresh,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "max_size",
|
|
|
|
.data = &ip_rt_max_size,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
/* Deprecated. Use gc_min_interval_ms */
|
2007-02-09 14:24:47 +00:00
|
|
|
|
2005-04-16 22:20:36 +00:00
|
|
|
.procname = "gc_min_interval",
|
|
|
|
.data = &ip_rt_gc_min_interval,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec_jiffies,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "gc_min_interval_ms",
|
|
|
|
.data = &ip_rt_gc_min_interval,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec_ms_jiffies,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "gc_timeout",
|
|
|
|
.data = &ip_rt_gc_timeout,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec_jiffies,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
2011-12-21 20:47:16 +00:00
|
|
|
{
|
|
|
|
.procname = "gc_interval",
|
|
|
|
.data = &ip_rt_gc_interval,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
|
|
|
.proc_handler = proc_dointvec_jiffies,
|
|
|
|
},
|
2005-04-16 22:20:36 +00:00
|
|
|
{
|
|
|
|
.procname = "redirect_load",
|
|
|
|
.data = &ip_rt_redirect_load,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "redirect_number",
|
|
|
|
.data = &ip_rt_redirect_number,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "redirect_silence",
|
|
|
|
.data = &ip_rt_redirect_silence,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "error_cost",
|
|
|
|
.data = &ip_rt_error_cost,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "error_burst",
|
|
|
|
.data = &ip_rt_error_burst,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "gc_elasticity",
|
|
|
|
.data = &ip_rt_gc_elasticity,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "mtu_expires",
|
|
|
|
.data = &ip_rt_mtu_expires,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec_jiffies,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "min_pmtu",
|
|
|
|
.data = &ip_rt_min_pmtu,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2018-02-26 15:13:43 +00:00
|
|
|
.proc_handler = proc_dointvec_minmax,
|
|
|
|
.extra1 = &ip_min_valid_pmtu,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
|
|
|
{
|
|
|
|
.procname = "min_adv_mss",
|
|
|
|
.data = &ip_rt_min_advmss,
|
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0644,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = proc_dointvec,
|
2005-04-16 22:20:36 +00:00
|
|
|
},
|
2009-11-05 21:32:03 +00:00
|
|
|
{ }
|
2005-04-16 22:20:36 +00:00
|
|
|
};
|
2008-07-06 02:02:33 +00:00
|
|
|
|
2019-06-24 13:29:23 +00:00
|
|
|
static const char ipv4_route_flush_procname[] = "flush";
|
|
|
|
|
2008-07-06 02:02:33 +00:00
|
|
|
static struct ctl_table ipv4_route_flush_table[] = {
|
|
|
|
{
|
2019-06-24 13:29:23 +00:00
|
|
|
.procname = ipv4_route_flush_procname,
|
2008-07-06 02:02:33 +00:00
|
|
|
.maxlen = sizeof(int),
|
|
|
|
.mode = 0200,
|
2008-11-04 02:21:05 +00:00
|
|
|
.proc_handler = ipv4_sysctl_rtcache_flush,
|
2008-07-06 02:02:33 +00:00
|
|
|
},
|
2009-11-05 21:32:03 +00:00
|
|
|
{ },
|
2008-07-06 02:02:33 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static __net_init int sysctl_route_net_init(struct net *net)
|
|
|
|
{
|
|
|
|
struct ctl_table *tbl;
|
|
|
|
|
|
|
|
tbl = ipv4_route_flush_table;
|
2009-11-25 23:14:13 +00:00
|
|
|
if (!net_eq(net, &init_net)) {
|
2008-07-06 02:02:33 +00:00
|
|
|
tbl = kmemdup(tbl, sizeof(ipv4_route_flush_table), GFP_KERNEL);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!tbl)
|
2008-07-06 02:02:33 +00:00
|
|
|
goto err_dup;
|
2012-11-16 03:02:59 +00:00
|
|
|
|
2019-06-24 13:29:23 +00:00
|
|
|
/* Don't export non-whitelisted sysctls to unprivileged users */
|
|
|
|
if (net->user_ns != &init_user_ns) {
|
|
|
|
if (tbl[0].procname != ipv4_route_flush_procname)
|
|
|
|
tbl[0].procname = NULL;
|
|
|
|
}
|
2008-07-06 02:02:33 +00:00
|
|
|
}
|
|
|
|
tbl[0].extra1 = net;
|
|
|
|
|
2012-04-19 13:44:49 +00:00
|
|
|
net->ipv4.route_hdr = register_net_sysctl(net, "net/ipv4/route", tbl);
|
2015-04-03 08:17:26 +00:00
|
|
|
if (!net->ipv4.route_hdr)
|
2008-07-06 02:02:33 +00:00
|
|
|
goto err_reg;
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_reg:
|
|
|
|
if (tbl != ipv4_route_flush_table)
|
|
|
|
kfree(tbl);
|
|
|
|
err_dup:
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
static __net_exit void sysctl_route_net_exit(struct net *net)
|
|
|
|
{
|
|
|
|
struct ctl_table *tbl;
|
|
|
|
|
|
|
|
tbl = net->ipv4.route_hdr->ctl_table_arg;
|
|
|
|
unregister_net_sysctl_table(net->ipv4.route_hdr);
|
|
|
|
BUG_ON(tbl == ipv4_route_flush_table);
|
|
|
|
kfree(tbl);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __net_initdata struct pernet_operations sysctl_route_ops = {
|
|
|
|
.init = sysctl_route_net_init,
|
|
|
|
.exit = sysctl_route_net_exit,
|
|
|
|
};
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
|
|
|
|
2010-05-08 08:57:52 +00:00
|
|
|
static __net_init int rt_genid_init(struct net *net)
|
2008-07-06 02:02:59 +00:00
|
|
|
{
|
2013-07-30 00:33:53 +00:00
|
|
|
atomic_set(&net->ipv4.rt_genid, 0);
|
2013-05-27 20:46:33 +00:00
|
|
|
atomic_set(&net->fnhe_genid, 0);
|
2017-06-08 03:01:20 +00:00
|
|
|
atomic_set(&net->ipv4.dev_addr_genid, get_random_int());
|
2008-07-06 02:02:59 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2010-05-08 08:57:52 +00:00
|
|
|
static __net_initdata struct pernet_operations rt_genid_ops = {
|
|
|
|
.init = rt_genid_init,
|
2008-07-06 02:02:59 +00:00
|
|
|
};
|
|
|
|
|
2012-06-09 23:27:05 +00:00
|
|
|
static int __net_init ipv4_inetpeer_init(struct net *net)
|
|
|
|
{
|
|
|
|
struct inet_peer_base *bp = kmalloc(sizeof(*bp), GFP_KERNEL);
|
|
|
|
|
|
|
|
if (!bp)
|
|
|
|
return -ENOMEM;
|
|
|
|
inet_peer_base_init(bp);
|
|
|
|
net->ipv4.peers = bp;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __net_exit ipv4_inetpeer_exit(struct net *net)
|
|
|
|
{
|
|
|
|
struct inet_peer_base *bp = net->ipv4.peers;
|
|
|
|
|
|
|
|
net->ipv4.peers = NULL;
|
2012-06-09 23:32:41 +00:00
|
|
|
inetpeer_invalidate_tree(bp);
|
2012-06-09 23:27:05 +00:00
|
|
|
kfree(bp);
|
|
|
|
}
|
|
|
|
|
|
|
|
static __net_initdata struct pernet_operations ipv4_inetpeer_ops = {
|
|
|
|
.init = ipv4_inetpeer_init,
|
|
|
|
.exit = ipv4_inetpeer_exit,
|
|
|
|
};
|
2008-07-06 02:02:59 +00:00
|
|
|
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2010-02-16 15:20:26 +00:00
|
|
|
struct ip_rt_acct __percpu *ip_rt_acct __read_mostly;
|
2011-01-14 12:36:42 +00:00
|
|
|
#endif /* CONFIG_IP_ROUTE_CLASSID */
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
int __init ip_rt_init(void)
|
|
|
|
{
|
2021-03-24 21:53:37 +00:00
|
|
|
void *idents_hash;
|
2015-01-14 23:17:06 +00:00
|
|
|
int cpu;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2021-03-24 21:53:37 +00:00
|
|
|
/* For modern hosts, this will use 2 MB of memory */
|
|
|
|
idents_hash = alloc_large_system_hash("IP idents",
|
|
|
|
sizeof(*ip_idents) + sizeof(*ip_tstamps),
|
|
|
|
0,
|
|
|
|
16, /* one bucket per 64 KB */
|
|
|
|
HASH_ZERO,
|
|
|
|
NULL,
|
|
|
|
&ip_idents_mask,
|
|
|
|
2048,
|
|
|
|
256*1024);
|
|
|
|
|
|
|
|
ip_idents = idents_hash;
|
inetpeer: get rid of ip_id_count
Ideally, we would need to generate IP ID using a per destination IP
generator.
linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.
1) each inet_peer struct consumes 192 bytes
2) inetpeer cache uses a binary tree of inet_peer structs,
with a nominal size of ~66000 elements under load.
3) lookups in this tree are hitting a lot of cache lines, as tree depth
is about 20.
4) If server deals with many tcp flows, we have a high probability of
not finding the inet_peer, allocating a fresh one, inserting it in
the tree with same initial ip_id_count, (cf secure_ip_id())
5) We garbage collect inet_peer aggressively.
IP ID generation do not have to be 'perfect'
Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.
We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.
ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)
secure_ip_id() and secure_ipv6_id() no longer are needed.
Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-02 12:26:03 +00:00
|
|
|
|
2021-03-24 21:53:37 +00:00
|
|
|
prandom_bytes(ip_idents, (ip_idents_mask + 1) * sizeof(*ip_idents));
|
inetpeer: get rid of ip_id_count
Ideally, we would need to generate IP ID using a per destination IP
generator.
linux kernels used inet_peer cache for this purpose, but this had a huge
cost on servers disabling MTU discovery.
1) each inet_peer struct consumes 192 bytes
2) inetpeer cache uses a binary tree of inet_peer structs,
with a nominal size of ~66000 elements under load.
3) lookups in this tree are hitting a lot of cache lines, as tree depth
is about 20.
4) If server deals with many tcp flows, we have a high probability of
not finding the inet_peer, allocating a fresh one, inserting it in
the tree with same initial ip_id_count, (cf secure_ip_id())
5) We garbage collect inet_peer aggressively.
IP ID generation do not have to be 'perfect'
Goal is trying to avoid duplicates in a short period of time,
so that reassembly units have a chance to complete reassembly of
fragments belonging to one message before receiving other fragments
with a recycled ID.
We simply use an array of generators, and a Jenkin hash using the dst IP
as a key.
ipv6_select_ident() is put back into net/ipv6/ip6_output.c where it
belongs (it is only used from this file)
secure_ip_id() and secure_ipv6_id() no longer are needed.
Rename ip_select_ident_more() to ip_select_ident_segs() to avoid
unnecessary decrement/increment of the number of segments.
Signed-off-by: Eric Dumazet <edumazet@google.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2014-06-02 12:26:03 +00:00
|
|
|
|
2021-03-24 21:53:37 +00:00
|
|
|
ip_tstamps = idents_hash + (ip_idents_mask + 1) * sizeof(*ip_idents);
|
2015-05-01 17:37:49 +00:00
|
|
|
|
2015-01-14 23:17:06 +00:00
|
|
|
for_each_possible_cpu(cpu) {
|
|
|
|
struct uncached_list *ul = &per_cpu(rt_uncached_list, cpu);
|
|
|
|
|
|
|
|
INIT_LIST_HEAD(&ul->head);
|
|
|
|
spin_lock_init(&ul->lock);
|
|
|
|
}
|
2011-01-14 12:36:42 +00:00
|
|
|
#ifdef CONFIG_IP_ROUTE_CLASSID
|
2009-02-25 13:07:33 +00:00
|
|
|
ip_rt_acct = __alloc_percpu(256 * sizeof(struct ip_rt_acct), __alignof__(struct ip_rt_acct));
|
2005-04-16 22:20:36 +00:00
|
|
|
if (!ip_rt_acct)
|
|
|
|
panic("IP: failed to allocate ip_rt_acct\n");
|
|
|
|
#endif
|
|
|
|
|
2006-08-27 02:25:52 +00:00
|
|
|
ipv4_dst_ops.kmem_cachep =
|
|
|
|
kmem_cache_create("ip_dst_cache", sizeof(struct rtable), 0,
|
2007-07-20 01:11:58 +00:00
|
|
|
SLAB_HWCACHE_ALIGN|SLAB_PANIC, NULL);
|
2005-04-16 22:20:36 +00:00
|
|
|
|
2007-05-25 01:17:54 +00:00
|
|
|
ipv4_dst_blackhole_ops.kmem_cachep = ipv4_dst_ops.kmem_cachep;
|
|
|
|
|
2010-10-08 06:37:34 +00:00
|
|
|
if (dst_entries_init(&ipv4_dst_ops) < 0)
|
|
|
|
panic("IP: failed to allocate ipv4_dst_ops counter\n");
|
|
|
|
|
|
|
|
if (dst_entries_init(&ipv4_dst_blackhole_ops) < 0)
|
|
|
|
panic("IP: failed to allocate ipv4_dst_blackhole_ops counter\n");
|
|
|
|
|
2012-07-17 18:00:09 +00:00
|
|
|
ipv4_dst_ops.gc_thresh = ~0;
|
|
|
|
ip_rt_max_size = INT_MAX;
|
2005-04-16 22:20:36 +00:00
|
|
|
|
|
|
|
devinet_init();
|
|
|
|
ip_fib_init();
|
|
|
|
|
2008-02-29 04:51:18 +00:00
|
|
|
if (ip_rt_proc_init())
|
2012-03-11 18:36:11 +00:00
|
|
|
pr_err("Unable to create route proc files\n");
|
2005-04-16 22:20:36 +00:00
|
|
|
#ifdef CONFIG_XFRM
|
|
|
|
xfrm_init();
|
2012-11-13 07:52:24 +00:00
|
|
|
xfrm4_init();
|
2005-04-16 22:20:36 +00:00
|
|
|
#endif
|
2017-08-15 14:34:44 +00:00
|
|
|
rtnl_register(PF_INET, RTM_GETROUTE, inet_rtm_getroute, NULL,
|
|
|
|
RTNL_FLAG_DOIT_UNLOCKED);
|
2007-03-22 18:55:17 +00:00
|
|
|
|
2008-07-06 02:02:33 +00:00
|
|
|
#ifdef CONFIG_SYSCTL
|
|
|
|
register_pernet_subsys(&sysctl_route_ops);
|
|
|
|
#endif
|
2010-05-08 08:57:52 +00:00
|
|
|
register_pernet_subsys(&rt_genid_ops);
|
2012-06-09 23:27:05 +00:00
|
|
|
register_pernet_subsys(&ipv4_inetpeer_ops);
|
2017-10-04 19:59:49 +00:00
|
|
|
return 0;
|
2005-04-16 22:20:36 +00:00
|
|
|
}
|
|
|
|
|
2008-07-30 10:32:52 +00:00
|
|
|
#ifdef CONFIG_SYSCTL
|
2008-07-27 07:59:33 +00:00
|
|
|
/*
|
|
|
|
* We really need to sanitize the damn ipv4 init order, then all
|
|
|
|
* this nonsense will go away.
|
|
|
|
*/
|
|
|
|
void __init ip_static_sysctl_init(void)
|
|
|
|
{
|
2012-04-19 13:32:39 +00:00
|
|
|
register_net_sysctl(&init_net, "net/ipv4/route", ipv4_route_table);
|
2008-07-27 07:59:33 +00:00
|
|
|
}
|
2008-07-30 10:32:52 +00:00
|
|
|
#endif
|