2019-05-27 06:55:01 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-or-later
|
2017-04-14 08:05:53 +00:00
|
|
|
/*
|
|
|
|
* xfrm_device.c - IPsec device offloading code.
|
|
|
|
*
|
|
|
|
* Copyright (c) 2015 secunet Security Networks AG
|
|
|
|
*
|
|
|
|
* Author:
|
|
|
|
* Steffen Klassert <steffen.klassert@secunet.com>
|
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/errno.h>
|
|
|
|
#include <linux/module.h>
|
|
|
|
#include <linux/netdevice.h>
|
|
|
|
#include <linux/skbuff.h>
|
|
|
|
#include <linux/slab.h>
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <net/dst.h>
|
|
|
|
#include <net/xfrm.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
|
2017-06-01 06:57:56 +00:00
|
|
|
#ifdef CONFIG_XFRM_OFFLOAD
|
2019-03-29 20:16:26 +00:00
|
|
|
static void __xfrm_transport_prep(struct xfrm_state *x, struct sk_buff *skb,
|
|
|
|
unsigned int hsize)
|
|
|
|
{
|
|
|
|
struct xfrm_offload *xo = xfrm_offload(skb);
|
|
|
|
|
|
|
|
skb_reset_mac_len(skb);
|
2020-04-10 09:06:31 +00:00
|
|
|
if (xo->flags & XFRM_GSO_SEGMENT)
|
2019-03-29 20:16:26 +00:00
|
|
|
skb->transport_header -= x->props.header_len;
|
2020-04-10 09:06:31 +00:00
|
|
|
|
|
|
|
pskb_pull(skb, skb_transport_offset(skb) + x->props.header_len);
|
2019-03-29 20:16:26 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void __xfrm_mode_tunnel_prep(struct xfrm_state *x, struct sk_buff *skb,
|
|
|
|
unsigned int hsize)
|
|
|
|
|
|
|
|
{
|
|
|
|
struct xfrm_offload *xo = xfrm_offload(skb);
|
|
|
|
|
|
|
|
if (xo->flags & XFRM_GSO_SEGMENT)
|
|
|
|
skb->transport_header = skb->network_header + hsize;
|
|
|
|
|
|
|
|
skb_reset_mac_len(skb);
|
|
|
|
pskb_pull(skb, skb->mac_len + x->props.header_len);
|
|
|
|
}
|
|
|
|
|
2020-03-26 09:02:31 +00:00
|
|
|
static void __xfrm_mode_beet_prep(struct xfrm_state *x, struct sk_buff *skb,
|
|
|
|
unsigned int hsize)
|
|
|
|
{
|
|
|
|
struct xfrm_offload *xo = xfrm_offload(skb);
|
|
|
|
int phlen = 0;
|
|
|
|
|
|
|
|
if (xo->flags & XFRM_GSO_SEGMENT)
|
|
|
|
skb->transport_header = skb->network_header + hsize;
|
|
|
|
|
|
|
|
skb_reset_mac_len(skb);
|
|
|
|
if (x->sel.family != AF_INET6) {
|
|
|
|
phlen = IPV4_BEET_PHMAXLEN;
|
|
|
|
if (x->outer_mode.family == AF_INET6)
|
|
|
|
phlen += sizeof(struct ipv6hdr) - sizeof(struct iphdr);
|
|
|
|
}
|
|
|
|
|
|
|
|
pskb_pull(skb, skb->mac_len + hsize + (x->props.header_len - phlen));
|
|
|
|
}
|
|
|
|
|
2019-03-29 20:16:26 +00:00
|
|
|
/* Adjust pointers into the packet when IPsec is done at layer2 */
|
|
|
|
static void xfrm_outer_mode_prep(struct xfrm_state *x, struct sk_buff *skb)
|
|
|
|
{
|
2019-03-29 20:16:32 +00:00
|
|
|
switch (x->outer_mode.encap) {
|
2019-03-29 20:16:26 +00:00
|
|
|
case XFRM_MODE_TUNNEL:
|
2019-03-29 20:16:32 +00:00
|
|
|
if (x->outer_mode.family == AF_INET)
|
2019-03-29 20:16:26 +00:00
|
|
|
return __xfrm_mode_tunnel_prep(x, skb,
|
|
|
|
sizeof(struct iphdr));
|
2019-03-29 20:16:32 +00:00
|
|
|
if (x->outer_mode.family == AF_INET6)
|
2019-03-29 20:16:26 +00:00
|
|
|
return __xfrm_mode_tunnel_prep(x, skb,
|
|
|
|
sizeof(struct ipv6hdr));
|
|
|
|
break;
|
|
|
|
case XFRM_MODE_TRANSPORT:
|
2019-03-29 20:16:32 +00:00
|
|
|
if (x->outer_mode.family == AF_INET)
|
2019-03-29 20:16:26 +00:00
|
|
|
return __xfrm_transport_prep(x, skb,
|
|
|
|
sizeof(struct iphdr));
|
2019-03-29 20:16:32 +00:00
|
|
|
if (x->outer_mode.family == AF_INET6)
|
2019-03-29 20:16:26 +00:00
|
|
|
return __xfrm_transport_prep(x, skb,
|
|
|
|
sizeof(struct ipv6hdr));
|
|
|
|
break;
|
2020-03-26 09:02:31 +00:00
|
|
|
case XFRM_MODE_BEET:
|
|
|
|
if (x->outer_mode.family == AF_INET)
|
|
|
|
return __xfrm_mode_beet_prep(x, skb,
|
|
|
|
sizeof(struct iphdr));
|
|
|
|
if (x->outer_mode.family == AF_INET6)
|
|
|
|
return __xfrm_mode_beet_prep(x, skb,
|
|
|
|
sizeof(struct ipv6hdr));
|
|
|
|
break;
|
2019-03-29 20:16:26 +00:00
|
|
|
case XFRM_MODE_ROUTEOPTIMIZATION:
|
|
|
|
case XFRM_MODE_IN_TRIGGER:
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2022-10-17 06:34:47 +00:00
|
|
|
static inline bool xmit_xfrm_check_overflow(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct xfrm_offload *xo = xfrm_offload(skb);
|
|
|
|
__u32 seq = xo->seq.low;
|
|
|
|
|
|
|
|
seq += skb_shinfo(skb)->gso_segs;
|
|
|
|
if (unlikely(seq < xo->seq.low))
|
|
|
|
return true;
|
|
|
|
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-12-20 09:41:36 +00:00
|
|
|
struct sk_buff *validate_xmit_xfrm(struct sk_buff *skb, netdev_features_t features, bool *again)
|
2017-04-14 08:07:28 +00:00
|
|
|
{
|
|
|
|
int err;
|
2017-12-20 09:41:36 +00:00
|
|
|
unsigned long flags;
|
2017-04-14 08:07:28 +00:00
|
|
|
struct xfrm_state *x;
|
2017-12-20 09:41:36 +00:00
|
|
|
struct softnet_data *sd;
|
esp: remove the skb from the chain when it's enqueued in cryptd_wq
Xiumei found a panic in esp offload:
BUG: unable to handle kernel NULL pointer dereference at 0000000000000020
RIP: 0010:esp_output_done+0x101/0x160 [esp4]
Call Trace:
? esp_output+0x180/0x180 [esp4]
cryptd_aead_crypt+0x4c/0x90
cryptd_queue_worker+0x6e/0xa0
process_one_work+0x1a7/0x3b0
worker_thread+0x30/0x390
? create_worker+0x1a0/0x1a0
kthread+0x112/0x130
? kthread_flush_work_fn+0x10/0x10
ret_from_fork+0x35/0x40
It was caused by that skb secpath is used in esp_output_done() after it's
been released elsewhere.
The tx path for esp offload is:
__dev_queue_xmit()->
validate_xmit_skb_list()->
validate_xmit_xfrm()->
esp_xmit()->
esp_output_tail()->
aead_request_set_callback(esp_output_done) <--[1]
crypto_aead_encrypt() <--[2]
In [1], .callback is set, and in [2] it will trigger the worker schedule,
later on a kernel thread will call .callback(esp_output_done), as the call
trace shows.
But in validate_xmit_xfrm():
skb_list_walk_safe(skb, skb2, nskb) {
...
err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit]
...
}
When the err is -EINPROGRESS, which means this skb2 will be enqueued and
later gets encrypted and sent out by .callback later in a kernel thread,
skb2 should be removed fromt skb chain. Otherwise, it will get processed
again outside validate_xmit_xfrm(), which could release skb secpath, and
cause the panic above.
This patch is to remove the skb from the chain when it's enqueued in
cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check.
Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.")
Reported-by: Xiumei Mu <xmu@redhat.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 08:51:42 +00:00
|
|
|
struct sk_buff *skb2, *nskb, *pskb = NULL;
|
2017-12-20 09:41:31 +00:00
|
|
|
netdev_features_t esp_features = features;
|
2017-04-14 08:07:28 +00:00
|
|
|
struct xfrm_offload *xo = xfrm_offload(skb);
|
2020-06-19 14:31:52 +00:00
|
|
|
struct net_device *dev = skb->dev;
|
2018-12-18 16:15:20 +00:00
|
|
|
struct sec_path *sp;
|
2017-04-14 08:07:28 +00:00
|
|
|
|
2020-06-01 21:39:37 +00:00
|
|
|
if (!xo || (xo->flags & XFRM_XMIT))
|
2017-12-20 09:41:31 +00:00
|
|
|
return skb;
|
2017-04-14 08:07:28 +00:00
|
|
|
|
2017-12-20 09:41:31 +00:00
|
|
|
if (!(features & NETIF_F_HW_ESP))
|
|
|
|
esp_features = features & ~(NETIF_F_SG | NETIF_F_CSUM_MASK);
|
2017-04-14 08:07:28 +00:00
|
|
|
|
2018-12-18 16:15:20 +00:00
|
|
|
sp = skb_sec_path(skb);
|
|
|
|
x = sp->xvec[sp->len - 1];
|
2022-05-05 10:06:41 +00:00
|
|
|
if (xo->flags & XFRM_GRO || x->xso.dir == XFRM_DEV_OFFLOAD_IN)
|
2017-12-20 09:41:31 +00:00
|
|
|
return skb;
|
|
|
|
|
2022-12-02 18:41:30 +00:00
|
|
|
/* The packet was sent to HW IPsec packet offload engine,
|
|
|
|
* but to wrong device. Drop the packet, so it won't skip
|
|
|
|
* XFRM stack.
|
|
|
|
*/
|
|
|
|
if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET && x->xso.dev != dev) {
|
|
|
|
kfree_skb(skb);
|
|
|
|
dev_core_stats_tx_dropped_inc(dev);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2020-06-23 20:40:01 +00:00
|
|
|
/* This skb was already validated on the upper/virtual dev */
|
|
|
|
if ((x->xso.dev != dev) && (x->xso.real_dev == dev))
|
2020-06-19 14:31:52 +00:00
|
|
|
return skb;
|
|
|
|
|
2017-12-20 09:41:36 +00:00
|
|
|
local_irq_save(flags);
|
|
|
|
sd = this_cpu_ptr(&softnet_data);
|
|
|
|
err = !skb_queue_empty(&sd->xfrm_backlog);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
*again = true;
|
|
|
|
return skb;
|
|
|
|
}
|
|
|
|
|
2022-10-17 06:34:47 +00:00
|
|
|
if (skb_is_gso(skb) && (unlikely(x->xso.dev != dev) ||
|
|
|
|
unlikely(xmit_xfrm_check_overflow(skb)))) {
|
2020-06-19 14:31:52 +00:00
|
|
|
struct sk_buff *segs;
|
2017-12-20 09:41:31 +00:00
|
|
|
|
2020-06-19 14:31:52 +00:00
|
|
|
/* Packet got rerouted, fixup features and segment it. */
|
|
|
|
esp_features = esp_features & ~(NETIF_F_HW_ESP | NETIF_F_GSO_ESP);
|
2017-12-20 09:41:31 +00:00
|
|
|
|
2020-06-19 14:31:52 +00:00
|
|
|
segs = skb_gso_segment(skb, esp_features);
|
|
|
|
if (IS_ERR(segs)) {
|
|
|
|
kfree_skb(skb);
|
2022-03-11 05:14:20 +00:00
|
|
|
dev_core_stats_tx_dropped_inc(dev);
|
2020-06-19 14:31:52 +00:00
|
|
|
return NULL;
|
|
|
|
} else {
|
|
|
|
consume_skb(skb);
|
|
|
|
skb = segs;
|
2017-12-20 09:41:31 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!skb->next) {
|
2019-03-21 14:41:37 +00:00
|
|
|
esp_features |= skb->dev->gso_partial_features;
|
2019-03-29 20:16:26 +00:00
|
|
|
xfrm_outer_mode_prep(x, skb);
|
2017-04-14 08:07:28 +00:00
|
|
|
|
2017-12-20 09:41:36 +00:00
|
|
|
xo->flags |= XFRM_DEV_RESUME;
|
|
|
|
|
2017-12-20 09:41:31 +00:00
|
|
|
err = x->type_offload->xmit(x, skb, esp_features);
|
2017-04-14 08:07:28 +00:00
|
|
|
if (err) {
|
2017-12-20 09:41:36 +00:00
|
|
|
if (err == -EINPROGRESS)
|
|
|
|
return NULL;
|
|
|
|
|
2017-04-14 08:07:28 +00:00
|
|
|
XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
|
2017-12-20 09:41:31 +00:00
|
|
|
kfree_skb(skb);
|
|
|
|
return NULL;
|
2017-04-14 08:07:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
skb_push(skb, skb->data - skb_mac_header(skb));
|
2017-12-20 09:41:31 +00:00
|
|
|
|
|
|
|
return skb;
|
2017-04-14 08:07:28 +00:00
|
|
|
}
|
|
|
|
|
2020-01-13 23:42:28 +00:00
|
|
|
skb_list_walk_safe(skb, skb2, nskb) {
|
2019-03-21 14:41:37 +00:00
|
|
|
esp_features |= skb->dev->gso_partial_features;
|
2018-07-30 03:42:53 +00:00
|
|
|
skb_mark_not_on_list(skb2);
|
2017-12-20 09:41:31 +00:00
|
|
|
|
|
|
|
xo = xfrm_offload(skb2);
|
2017-12-20 09:41:36 +00:00
|
|
|
xo->flags |= XFRM_DEV_RESUME;
|
2017-12-20 09:41:31 +00:00
|
|
|
|
2019-03-29 20:16:26 +00:00
|
|
|
xfrm_outer_mode_prep(x, skb2);
|
2017-12-20 09:41:31 +00:00
|
|
|
|
|
|
|
err = x->type_offload->xmit(x, skb2, esp_features);
|
2017-12-20 09:41:36 +00:00
|
|
|
if (!err) {
|
|
|
|
skb2->next = nskb;
|
|
|
|
} else if (err != -EINPROGRESS) {
|
2017-12-20 09:41:31 +00:00
|
|
|
XFRM_INC_STATS(xs_net(x), LINUX_MIB_XFRMOUTSTATEPROTOERROR);
|
|
|
|
skb2->next = nskb;
|
|
|
|
kfree_skb_list(skb2);
|
|
|
|
return NULL;
|
2017-12-20 09:41:36 +00:00
|
|
|
} else {
|
|
|
|
if (skb == skb2)
|
|
|
|
skb = nskb;
|
esp: remove the skb from the chain when it's enqueued in cryptd_wq
Xiumei found a panic in esp offload:
BUG: unable to handle kernel NULL pointer dereference at 0000000000000020
RIP: 0010:esp_output_done+0x101/0x160 [esp4]
Call Trace:
? esp_output+0x180/0x180 [esp4]
cryptd_aead_crypt+0x4c/0x90
cryptd_queue_worker+0x6e/0xa0
process_one_work+0x1a7/0x3b0
worker_thread+0x30/0x390
? create_worker+0x1a0/0x1a0
kthread+0x112/0x130
? kthread_flush_work_fn+0x10/0x10
ret_from_fork+0x35/0x40
It was caused by that skb secpath is used in esp_output_done() after it's
been released elsewhere.
The tx path for esp offload is:
__dev_queue_xmit()->
validate_xmit_skb_list()->
validate_xmit_xfrm()->
esp_xmit()->
esp_output_tail()->
aead_request_set_callback(esp_output_done) <--[1]
crypto_aead_encrypt() <--[2]
In [1], .callback is set, and in [2] it will trigger the worker schedule,
later on a kernel thread will call .callback(esp_output_done), as the call
trace shows.
But in validate_xmit_xfrm():
skb_list_walk_safe(skb, skb2, nskb) {
...
err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit]
...
}
When the err is -EINPROGRESS, which means this skb2 will be enqueued and
later gets encrypted and sent out by .callback later in a kernel thread,
skb2 should be removed fromt skb chain. Otherwise, it will get processed
again outside validate_xmit_xfrm(), which could release skb secpath, and
cause the panic above.
This patch is to remove the skb from the chain when it's enqueued in
cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check.
Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.")
Reported-by: Xiumei Mu <xmu@redhat.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 08:51:42 +00:00
|
|
|
else
|
|
|
|
pskb->next = nskb;
|
2017-12-20 09:41:31 +00:00
|
|
|
|
2020-01-13 23:42:28 +00:00
|
|
|
continue;
|
2017-12-20 09:41:36 +00:00
|
|
|
}
|
2017-12-20 09:41:31 +00:00
|
|
|
|
|
|
|
skb_push(skb2, skb2->data - skb_mac_header(skb2));
|
esp: remove the skb from the chain when it's enqueued in cryptd_wq
Xiumei found a panic in esp offload:
BUG: unable to handle kernel NULL pointer dereference at 0000000000000020
RIP: 0010:esp_output_done+0x101/0x160 [esp4]
Call Trace:
? esp_output+0x180/0x180 [esp4]
cryptd_aead_crypt+0x4c/0x90
cryptd_queue_worker+0x6e/0xa0
process_one_work+0x1a7/0x3b0
worker_thread+0x30/0x390
? create_worker+0x1a0/0x1a0
kthread+0x112/0x130
? kthread_flush_work_fn+0x10/0x10
ret_from_fork+0x35/0x40
It was caused by that skb secpath is used in esp_output_done() after it's
been released elsewhere.
The tx path for esp offload is:
__dev_queue_xmit()->
validate_xmit_skb_list()->
validate_xmit_xfrm()->
esp_xmit()->
esp_output_tail()->
aead_request_set_callback(esp_output_done) <--[1]
crypto_aead_encrypt() <--[2]
In [1], .callback is set, and in [2] it will trigger the worker schedule,
later on a kernel thread will call .callback(esp_output_done), as the call
trace shows.
But in validate_xmit_xfrm():
skb_list_walk_safe(skb, skb2, nskb) {
...
err = x->type_offload->xmit(x, skb2, esp_features); [esp_xmit]
...
}
When the err is -EINPROGRESS, which means this skb2 will be enqueued and
later gets encrypted and sent out by .callback later in a kernel thread,
skb2 should be removed fromt skb chain. Otherwise, it will get processed
again outside validate_xmit_xfrm(), which could release skb secpath, and
cause the panic above.
This patch is to remove the skb from the chain when it's enqueued in
cryptd_wq. While at it, remove the unnecessary 'if (!skb)' check.
Fixes: 3dca3f38cfb8 ("xfrm: Separate ESP handling from segmentation for GRO packets.")
Reported-by: Xiumei Mu <xmu@redhat.com>
Signed-off-by: Xin Long <lucien.xin@gmail.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2020-03-04 08:51:42 +00:00
|
|
|
pskb = skb2;
|
2020-01-13 23:42:28 +00:00
|
|
|
}
|
2017-12-20 09:41:31 +00:00
|
|
|
|
|
|
|
return skb;
|
2017-04-14 08:07:28 +00:00
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(validate_xmit_xfrm);
|
|
|
|
|
2017-04-14 08:06:10 +00:00
|
|
|
int xfrm_dev_state_add(struct net *net, struct xfrm_state *x,
|
2022-09-14 17:04:03 +00:00
|
|
|
struct xfrm_user_offload *xuo,
|
|
|
|
struct netlink_ext_ack *extack)
|
2017-04-14 08:06:10 +00:00
|
|
|
{
|
|
|
|
int err;
|
|
|
|
struct dst_entry *dst;
|
|
|
|
struct net_device *dev;
|
2022-05-05 10:06:40 +00:00
|
|
|
struct xfrm_dev_offload *xso = &x->xso;
|
2017-04-14 08:06:10 +00:00
|
|
|
xfrm_address_t *saddr;
|
|
|
|
xfrm_address_t *daddr;
|
2022-12-02 18:41:28 +00:00
|
|
|
bool is_packet_offload;
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2022-09-14 17:04:03 +00:00
|
|
|
if (!x->type_offload) {
|
|
|
|
NL_SET_ERR_MSG(extack, "Type doesn't support offload");
|
2017-08-01 09:49:08 +00:00
|
|
|
return -EINVAL;
|
2022-09-14 17:04:03 +00:00
|
|
|
}
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2018-01-14 09:39:10 +00:00
|
|
|
/* We don't yet support UDP encapsulation and TFC padding. */
|
2022-09-14 17:04:03 +00:00
|
|
|
if (x->encap || x->tfcpad) {
|
|
|
|
NL_SET_ERR_MSG(extack, "Encapsulation and TFC padding can't be offloaded");
|
2017-11-28 09:49:29 +00:00
|
|
|
return -EINVAL;
|
2022-09-14 17:04:03 +00:00
|
|
|
}
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2022-12-02 18:41:28 +00:00
|
|
|
if (xuo->flags &
|
|
|
|
~(XFRM_OFFLOAD_IPV6 | XFRM_OFFLOAD_INBOUND | XFRM_OFFLOAD_PACKET)) {
|
2022-09-14 17:04:03 +00:00
|
|
|
NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
|
2022-02-08 14:14:32 +00:00
|
|
|
return -EINVAL;
|
2022-09-14 17:04:03 +00:00
|
|
|
}
|
2022-02-08 14:14:32 +00:00
|
|
|
|
2022-12-02 18:41:28 +00:00
|
|
|
is_packet_offload = xuo->flags & XFRM_OFFLOAD_PACKET;
|
2017-04-14 08:06:10 +00:00
|
|
|
dev = dev_get_by_index(net, xuo->ifindex);
|
|
|
|
if (!dev) {
|
|
|
|
if (!(xuo->flags & XFRM_OFFLOAD_INBOUND)) {
|
|
|
|
saddr = &x->props.saddr;
|
|
|
|
daddr = &x->id.daddr;
|
|
|
|
} else {
|
|
|
|
saddr = &x->id.daddr;
|
|
|
|
daddr = &x->props.saddr;
|
|
|
|
}
|
|
|
|
|
net: xfrm: support setting an output mark.
On systems that use mark-based routing it may be necessary for
routing lookups to use marks in order for packets to be routed
correctly. An example of such a system is Android, which uses
socket marks to route packets via different networks.
Currently, routing lookups in tunnel mode always use a mark of
zero, making routing incorrect on such systems.
This patch adds a new output_mark element to the xfrm state and
a corresponding XFRMA_OUTPUT_MARK netlink attribute. The output
mark differs from the existing xfrm mark in two ways:
1. The xfrm mark is used to match xfrm policies and states, while
the xfrm output mark is used to set the mark (and influence
the routing) of the packets emitted by those states.
2. The existing mark is constrained to be a subset of the bits of
the originating socket or transformed packet, but the output
mark is arbitrary and depends only on the state.
The use of a separate mark provides additional flexibility. For
example:
- A packet subject to two transforms (e.g., transport mode inside
tunnel mode) can have two different output marks applied to it,
one for the transport mode SA and one for the tunnel mode SA.
- On a system where socket marks determine routing, the packets
emitted by an IPsec tunnel can be routed based on a mark that
is determined by the tunnel, not by the marks of the
unencrypted packets.
- Support for setting the output marks can be introduced without
breaking any existing setups that employ both mark-based
routing and xfrm tunnel mode. Simply changing the code to use
the xfrm mark for routing output packets could xfrm mark could
change behaviour in a way that breaks these setups.
If the output mark is unspecified or set to zero, the mark is not
set or changed.
Tested: make allyesconfig; make -j64
Tested: https://android-review.googlesource.com/452776
Signed-off-by: Lorenzo Colitti <lorenzo@google.com>
Signed-off-by: Steffen Klassert <steffen.klassert@secunet.com>
2017-08-10 17:11:33 +00:00
|
|
|
dst = __xfrm_dst_lookup(net, 0, 0, saddr, daddr,
|
2018-06-12 10:44:26 +00:00
|
|
|
x->props.family,
|
|
|
|
xfrm_smark_get(0, x));
|
2017-04-14 08:06:10 +00:00
|
|
|
if (IS_ERR(dst))
|
2022-12-02 18:41:28 +00:00
|
|
|
return (is_packet_offload) ? -EINVAL : 0;
|
2017-04-14 08:06:10 +00:00
|
|
|
|
|
|
|
dev = dst->dev;
|
|
|
|
|
|
|
|
dev_hold(dev);
|
|
|
|
dst_release(dst);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_state_add) {
|
2017-09-04 08:59:55 +00:00
|
|
|
xso->dev = NULL;
|
2017-04-14 08:06:10 +00:00
|
|
|
dev_put(dev);
|
2022-12-02 18:41:28 +00:00
|
|
|
return (is_packet_offload) ? -EINVAL : 0;
|
2017-04-14 08:06:10 +00:00
|
|
|
}
|
|
|
|
|
2023-03-30 08:02:26 +00:00
|
|
|
if (!is_packet_offload && x->props.flags & XFRM_STATE_ESN &&
|
2018-01-14 09:39:10 +00:00
|
|
|
!dev->xfrmdev_ops->xdo_dev_state_advance_esn) {
|
2022-09-14 17:04:03 +00:00
|
|
|
NL_SET_ERR_MSG(extack, "Device doesn't support offload with ESN");
|
2018-01-14 09:39:10 +00:00
|
|
|
xso->dev = NULL;
|
|
|
|
dev_put(dev);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2017-04-14 08:06:10 +00:00
|
|
|
xso->dev = dev;
|
2021-12-09 15:44:51 +00:00
|
|
|
netdev_tracker_alloc(dev, &xso->dev_tracker, GFP_ATOMIC);
|
2020-06-23 20:40:01 +00:00
|
|
|
xso->real_dev = dev;
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2022-05-05 10:06:41 +00:00
|
|
|
if (xuo->flags & XFRM_OFFLOAD_INBOUND)
|
|
|
|
xso->dir = XFRM_DEV_OFFLOAD_IN;
|
|
|
|
else
|
|
|
|
xso->dir = XFRM_DEV_OFFLOAD_OUT;
|
|
|
|
|
2022-12-02 18:41:28 +00:00
|
|
|
if (is_packet_offload)
|
|
|
|
xso->type = XFRM_DEV_OFFLOAD_PACKET;
|
|
|
|
else
|
|
|
|
xso->type = XFRM_DEV_OFFLOAD_CRYPTO;
|
2022-12-02 18:41:27 +00:00
|
|
|
|
2023-01-24 11:54:59 +00:00
|
|
|
err = dev->xfrmdev_ops->xdo_dev_state_add(x, extack);
|
2017-04-14 08:06:10 +00:00
|
|
|
if (err) {
|
2018-01-18 13:41:51 +00:00
|
|
|
xso->dev = NULL;
|
2022-05-05 10:06:41 +00:00
|
|
|
xso->dir = 0;
|
2021-06-22 03:55:31 +00:00
|
|
|
xso->real_dev = NULL;
|
2022-06-08 04:39:55 +00:00
|
|
|
netdev_put(dev, &xso->dev_tracker);
|
2022-12-02 18:41:27 +00:00
|
|
|
xso->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
|
2018-08-22 21:38:10 +00:00
|
|
|
|
2022-12-02 18:41:28 +00:00
|
|
|
/* User explicitly requested packet offload mode and configured
|
|
|
|
* policy in addition to the XFRM state. So be civil to users,
|
|
|
|
* and return an error instead of taking fallback path.
|
|
|
|
*
|
|
|
|
* This WARN_ON() can be seen as a documentation for driver
|
|
|
|
* authors to do not return -EOPNOTSUPP in packet offload mode.
|
|
|
|
*/
|
|
|
|
WARN_ON(err == -EOPNOTSUPP && is_packet_offload);
|
2023-01-31 13:31:57 +00:00
|
|
|
if (err != -EOPNOTSUPP || is_packet_offload) {
|
|
|
|
NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this state");
|
2018-08-22 21:38:10 +00:00
|
|
|
return err;
|
2023-01-31 13:31:57 +00:00
|
|
|
}
|
2017-04-14 08:06:10 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(xfrm_dev_state_add);
|
|
|
|
|
2022-12-02 18:41:29 +00:00
|
|
|
int xfrm_dev_policy_add(struct net *net, struct xfrm_policy *xp,
|
|
|
|
struct xfrm_user_offload *xuo, u8 dir,
|
|
|
|
struct netlink_ext_ack *extack)
|
|
|
|
{
|
|
|
|
struct xfrm_dev_offload *xdo = &xp->xdo;
|
|
|
|
struct net_device *dev;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (!xuo->flags || xuo->flags & ~XFRM_OFFLOAD_PACKET) {
|
|
|
|
/* We support only packet offload mode and it means
|
|
|
|
* that user must set XFRM_OFFLOAD_PACKET bit.
|
|
|
|
*/
|
|
|
|
NL_SET_ERR_MSG(extack, "Unrecognized flags in offload request");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
dev = dev_get_by_index(net, xuo->ifindex);
|
|
|
|
if (!dev)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (!dev->xfrmdev_ops || !dev->xfrmdev_ops->xdo_dev_policy_add) {
|
|
|
|
xdo->dev = NULL;
|
|
|
|
dev_put(dev);
|
|
|
|
NL_SET_ERR_MSG(extack, "Policy offload is not supported");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
xdo->dev = dev;
|
|
|
|
netdev_tracker_alloc(dev, &xdo->dev_tracker, GFP_ATOMIC);
|
|
|
|
xdo->real_dev = dev;
|
|
|
|
xdo->type = XFRM_DEV_OFFLOAD_PACKET;
|
|
|
|
switch (dir) {
|
|
|
|
case XFRM_POLICY_IN:
|
|
|
|
xdo->dir = XFRM_DEV_OFFLOAD_IN;
|
|
|
|
break;
|
|
|
|
case XFRM_POLICY_OUT:
|
|
|
|
xdo->dir = XFRM_DEV_OFFLOAD_OUT;
|
|
|
|
break;
|
|
|
|
case XFRM_POLICY_FWD:
|
|
|
|
xdo->dir = XFRM_DEV_OFFLOAD_FWD;
|
|
|
|
break;
|
|
|
|
default:
|
|
|
|
xdo->dev = NULL;
|
2023-04-19 12:19:08 +00:00
|
|
|
netdev_put(dev, &xdo->dev_tracker);
|
2022-12-07 09:23:14 +00:00
|
|
|
NL_SET_ERR_MSG(extack, "Unrecognized offload direction");
|
2022-12-02 18:41:29 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2023-01-24 11:54:57 +00:00
|
|
|
err = dev->xfrmdev_ops->xdo_dev_policy_add(xp, extack);
|
2022-12-02 18:41:29 +00:00
|
|
|
if (err) {
|
|
|
|
xdo->dev = NULL;
|
|
|
|
xdo->real_dev = NULL;
|
|
|
|
xdo->type = XFRM_DEV_OFFLOAD_UNSPECIFIED;
|
|
|
|
xdo->dir = 0;
|
|
|
|
netdev_put(dev, &xdo->dev_tracker);
|
2023-01-31 13:31:57 +00:00
|
|
|
NL_SET_ERR_MSG_WEAK(extack, "Device failed to offload this policy");
|
2022-12-02 18:41:29 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(xfrm_dev_policy_add);
|
|
|
|
|
2017-04-14 08:06:10 +00:00
|
|
|
bool xfrm_dev_offload_ok(struct sk_buff *skb, struct xfrm_state *x)
|
|
|
|
{
|
|
|
|
int mtu;
|
|
|
|
struct dst_entry *dst = skb_dst(skb);
|
|
|
|
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
|
|
|
|
struct net_device *dev = x->xso.dev;
|
|
|
|
|
|
|
|
if (!x->type_offload || x->encap)
|
|
|
|
return false;
|
|
|
|
|
2022-12-02 18:41:30 +00:00
|
|
|
if (x->xso.type == XFRM_DEV_OFFLOAD_PACKET ||
|
|
|
|
((!dev || (dev == xfrm_dst_path(dst)->dev)) &&
|
|
|
|
!xdst->child->xfrm)) {
|
2019-06-24 20:04:48 +00:00
|
|
|
mtu = xfrm_state_mtu(x, xdst->child_mtu_cached);
|
2017-04-14 08:06:10 +00:00
|
|
|
if (skb->len <= mtu)
|
|
|
|
goto ok;
|
|
|
|
|
2018-03-01 06:13:37 +00:00
|
|
|
if (skb_is_gso(skb) && skb_gso_validate_network_len(skb, mtu))
|
2017-04-14 08:06:10 +00:00
|
|
|
goto ok;
|
|
|
|
}
|
|
|
|
|
|
|
|
return false;
|
|
|
|
|
|
|
|
ok:
|
|
|
|
if (dev && dev->xfrmdev_ops && dev->xfrmdev_ops->xdo_dev_offload_ok)
|
|
|
|
return x->xso.dev->xfrmdev_ops->xdo_dev_offload_ok(skb, x);
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(xfrm_dev_offload_ok);
|
2017-12-20 09:41:36 +00:00
|
|
|
|
|
|
|
void xfrm_dev_resume(struct sk_buff *skb)
|
|
|
|
{
|
|
|
|
struct net_device *dev = skb->dev;
|
|
|
|
int ret = NETDEV_TX_BUSY;
|
|
|
|
struct netdev_queue *txq;
|
|
|
|
struct softnet_data *sd;
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
rcu_read_lock();
|
2019-03-20 10:02:04 +00:00
|
|
|
txq = netdev_core_pick_tx(dev, skb, NULL);
|
2017-12-20 09:41:36 +00:00
|
|
|
|
|
|
|
HARD_TX_LOCK(dev, txq, smp_processor_id());
|
|
|
|
if (!netif_xmit_frozen_or_stopped(txq))
|
|
|
|
skb = dev_hard_start_xmit(skb, dev, txq, &ret);
|
|
|
|
HARD_TX_UNLOCK(dev, txq);
|
|
|
|
|
|
|
|
if (!dev_xmit_complete(ret)) {
|
|
|
|
local_irq_save(flags);
|
|
|
|
sd = this_cpu_ptr(&softnet_data);
|
|
|
|
skb_queue_tail(&sd->xfrm_backlog, skb);
|
|
|
|
raise_softirq_irqoff(NET_TX_SOFTIRQ);
|
|
|
|
local_irq_restore(flags);
|
|
|
|
}
|
|
|
|
rcu_read_unlock();
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL_GPL(xfrm_dev_resume);
|
|
|
|
|
|
|
|
void xfrm_dev_backlog(struct softnet_data *sd)
|
|
|
|
{
|
|
|
|
struct sk_buff_head *xfrm_backlog = &sd->xfrm_backlog;
|
|
|
|
struct sk_buff_head list;
|
|
|
|
struct sk_buff *skb;
|
|
|
|
|
|
|
|
if (skb_queue_empty(xfrm_backlog))
|
|
|
|
return;
|
|
|
|
|
|
|
|
__skb_queue_head_init(&list);
|
|
|
|
|
|
|
|
spin_lock(&xfrm_backlog->lock);
|
|
|
|
skb_queue_splice_init(xfrm_backlog, &list);
|
|
|
|
spin_unlock(&xfrm_backlog->lock);
|
|
|
|
|
|
|
|
while (!skb_queue_empty(&list)) {
|
|
|
|
skb = __skb_dequeue(&list);
|
|
|
|
xfrm_dev_resume(skb);
|
|
|
|
}
|
|
|
|
|
|
|
|
}
|
2017-06-01 06:57:56 +00:00
|
|
|
#endif
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2017-12-19 23:35:48 +00:00
|
|
|
static int xfrm_api_check(struct net_device *dev)
|
2017-04-14 08:06:10 +00:00
|
|
|
{
|
2017-12-19 23:35:48 +00:00
|
|
|
#ifdef CONFIG_XFRM_OFFLOAD
|
2017-04-14 08:06:10 +00:00
|
|
|
if ((dev->features & NETIF_F_HW_ESP_TX_CSUM) &&
|
|
|
|
!(dev->features & NETIF_F_HW_ESP))
|
|
|
|
return NOTIFY_BAD;
|
|
|
|
|
2017-12-19 23:35:48 +00:00
|
|
|
if ((dev->features & NETIF_F_HW_ESP) &&
|
|
|
|
(!(dev->xfrmdev_ops &&
|
|
|
|
dev->xfrmdev_ops->xdo_dev_state_add &&
|
|
|
|
dev->xfrmdev_ops->xdo_dev_state_delete)))
|
|
|
|
return NOTIFY_BAD;
|
|
|
|
#else
|
|
|
|
if (dev->features & (NETIF_F_HW_ESP | NETIF_F_HW_ESP_TX_CSUM))
|
|
|
|
return NOTIFY_BAD;
|
|
|
|
#endif
|
|
|
|
|
2017-04-14 08:06:10 +00:00
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int xfrm_dev_down(struct net_device *dev)
|
|
|
|
{
|
2022-12-02 18:41:29 +00:00
|
|
|
if (dev->features & NETIF_F_HW_ESP) {
|
2017-04-14 08:06:10 +00:00
|
|
|
xfrm_dev_state_flush(dev_net(dev), dev, true);
|
2022-12-02 18:41:29 +00:00
|
|
|
xfrm_dev_policy_flush(dev_net(dev), dev, true);
|
|
|
|
}
|
2017-04-14 08:06:10 +00:00
|
|
|
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
2017-04-14 08:05:53 +00:00
|
|
|
static int xfrm_dev_event(struct notifier_block *this, unsigned long event, void *ptr)
|
|
|
|
{
|
|
|
|
struct net_device *dev = netdev_notifier_info_to_dev(ptr);
|
|
|
|
|
|
|
|
switch (event) {
|
2017-04-14 08:06:10 +00:00
|
|
|
case NETDEV_REGISTER:
|
2022-02-01 07:35:28 +00:00
|
|
|
return xfrm_api_check(dev);
|
2017-04-14 08:06:10 +00:00
|
|
|
|
|
|
|
case NETDEV_FEAT_CHANGE:
|
2022-02-01 07:35:28 +00:00
|
|
|
return xfrm_api_check(dev);
|
2017-04-14 08:06:10 +00:00
|
|
|
|
2017-04-14 08:05:53 +00:00
|
|
|
case NETDEV_DOWN:
|
2020-02-02 11:19:34 +00:00
|
|
|
case NETDEV_UNREGISTER:
|
2017-04-14 08:06:10 +00:00
|
|
|
return xfrm_dev_down(dev);
|
2017-04-14 08:05:53 +00:00
|
|
|
}
|
|
|
|
return NOTIFY_DONE;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct notifier_block xfrm_dev_notifier = {
|
|
|
|
.notifier_call = xfrm_dev_event,
|
|
|
|
};
|
|
|
|
|
2018-03-29 14:03:25 +00:00
|
|
|
void __init xfrm_dev_init(void)
|
2017-04-14 08:05:53 +00:00
|
|
|
{
|
|
|
|
register_netdevice_notifier(&xfrm_dev_notifier);
|
|
|
|
}
|