linux-stable/net/ipv4/xfrm4_policy.c
Eric Dumazet fc66f95c68 net dst: use a percpu_counter to track entries
struct dst_ops tracks number of allocated dst in an atomic_t field,
subject to high cache line contention in stress workload.

Switch to a percpu_counter, to reduce number of time we need to dirty a
central location. Place it on a separate cache line to avoid dirtying
read only fields.

Stress test :

(Sending 160.000.000 UDP frames,
IP route cache disabled, dual E5540 @2.53GHz,
32bit kernel, FIB_TRIE, SLUB/NUMA)

Before:

real    0m51.179s
user    0m15.329s
sys     10m15.942s

After:

real	0m45.570s
user	0m15.525s
sys	9m56.669s

With a small reordering of struct neighbour fields, subject of a
following patch, (to separate refcnt from other read mostly fields)

real	0m41.841s
user	0m15.261s
sys	8m45.949s

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2010-10-11 13:06:53 -07:00

299 lines
6.7 KiB
C

/*
* xfrm4_policy.c
*
* Changes:
* Kazunori MIYAZAWA @USAGI
* YOSHIFUJI Hideaki @USAGI
* Split up af-specific portion
*
*/
#include <linux/err.h>
#include <linux/kernel.h>
#include <linux/inetdevice.h>
#include <net/dst.h>
#include <net/xfrm.h>
#include <net/ip.h>
static struct xfrm_policy_afinfo xfrm4_policy_afinfo;
static struct dst_entry *xfrm4_dst_lookup(struct net *net, int tos,
xfrm_address_t *saddr,
xfrm_address_t *daddr)
{
struct flowi fl = {
.nl_u = {
.ip4_u = {
.tos = tos,
.daddr = daddr->a4,
},
},
};
struct dst_entry *dst;
struct rtable *rt;
int err;
if (saddr)
fl.fl4_src = saddr->a4;
err = __ip_route_output_key(net, &rt, &fl);
dst = &rt->dst;
if (err)
dst = ERR_PTR(err);
return dst;
}
static int xfrm4_get_saddr(struct net *net,
xfrm_address_t *saddr, xfrm_address_t *daddr)
{
struct dst_entry *dst;
struct rtable *rt;
dst = xfrm4_dst_lookup(net, 0, NULL, daddr);
if (IS_ERR(dst))
return -EHOSTUNREACH;
rt = (struct rtable *)dst;
saddr->a4 = rt->rt_src;
dst_release(dst);
return 0;
}
static int xfrm4_get_tos(struct flowi *fl)
{
return IPTOS_RT_MASK & fl->fl4_tos; /* Strip ECN bits */
}
static int xfrm4_init_path(struct xfrm_dst *path, struct dst_entry *dst,
int nfheader_len)
{
return 0;
}
static int xfrm4_fill_dst(struct xfrm_dst *xdst, struct net_device *dev,
struct flowi *fl)
{
struct rtable *rt = (struct rtable *)xdst->route;
xdst->u.rt.fl = *fl;
xdst->u.dst.dev = dev;
dev_hold(dev);
xdst->u.rt.idev = in_dev_get(dev);
if (!xdst->u.rt.idev)
return -ENODEV;
xdst->u.rt.peer = rt->peer;
if (rt->peer)
atomic_inc(&rt->peer->refcnt);
/* Sheit... I remember I did this right. Apparently,
* it was magically lost, so this code needs audit */
xdst->u.rt.rt_flags = rt->rt_flags & (RTCF_BROADCAST | RTCF_MULTICAST |
RTCF_LOCAL);
xdst->u.rt.rt_type = rt->rt_type;
xdst->u.rt.rt_src = rt->rt_src;
xdst->u.rt.rt_dst = rt->rt_dst;
xdst->u.rt.rt_gateway = rt->rt_gateway;
xdst->u.rt.rt_spec_dst = rt->rt_spec_dst;
return 0;
}
static void
_decode_session4(struct sk_buff *skb, struct flowi *fl, int reverse)
{
struct iphdr *iph = ip_hdr(skb);
u8 *xprth = skb_network_header(skb) + iph->ihl * 4;
memset(fl, 0, sizeof(struct flowi));
fl->mark = skb->mark;
if (!(iph->frag_off & htons(IP_MF | IP_OFFSET))) {
switch (iph->protocol) {
case IPPROTO_UDP:
case IPPROTO_UDPLITE:
case IPPROTO_TCP:
case IPPROTO_SCTP:
case IPPROTO_DCCP:
if (xprth + 4 < skb->data ||
pskb_may_pull(skb, xprth + 4 - skb->data)) {
__be16 *ports = (__be16 *)xprth;
fl->fl_ip_sport = ports[!!reverse];
fl->fl_ip_dport = ports[!reverse];
}
break;
case IPPROTO_ICMP:
if (pskb_may_pull(skb, xprth + 2 - skb->data)) {
u8 *icmp = xprth;
fl->fl_icmp_type = icmp[0];
fl->fl_icmp_code = icmp[1];
}
break;
case IPPROTO_ESP:
if (pskb_may_pull(skb, xprth + 4 - skb->data)) {
__be32 *ehdr = (__be32 *)xprth;
fl->fl_ipsec_spi = ehdr[0];
}
break;
case IPPROTO_AH:
if (pskb_may_pull(skb, xprth + 8 - skb->data)) {
__be32 *ah_hdr = (__be32*)xprth;
fl->fl_ipsec_spi = ah_hdr[1];
}
break;
case IPPROTO_COMP:
if (pskb_may_pull(skb, xprth + 4 - skb->data)) {
__be16 *ipcomp_hdr = (__be16 *)xprth;
fl->fl_ipsec_spi = htonl(ntohs(ipcomp_hdr[1]));
}
break;
default:
fl->fl_ipsec_spi = 0;
break;
}
}
fl->proto = iph->protocol;
fl->fl4_dst = reverse ? iph->saddr : iph->daddr;
fl->fl4_src = reverse ? iph->daddr : iph->saddr;
fl->fl4_tos = iph->tos;
}
static inline int xfrm4_garbage_collect(struct dst_ops *ops)
{
struct net *net = container_of(ops, struct net, xfrm.xfrm4_dst_ops);
xfrm4_policy_afinfo.garbage_collect(net);
return (dst_entries_get_slow(ops) > ops->gc_thresh * 2);
}
static void xfrm4_update_pmtu(struct dst_entry *dst, u32 mtu)
{
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
struct dst_entry *path = xdst->route;
path->ops->update_pmtu(path, mtu);
}
static void xfrm4_dst_destroy(struct dst_entry *dst)
{
struct xfrm_dst *xdst = (struct xfrm_dst *)dst;
if (likely(xdst->u.rt.idev))
in_dev_put(xdst->u.rt.idev);
if (likely(xdst->u.rt.peer))
inet_putpeer(xdst->u.rt.peer);
xfrm_dst_destroy(xdst);
}
static void xfrm4_dst_ifdown(struct dst_entry *dst, struct net_device *dev,
int unregister)
{
struct xfrm_dst *xdst;
if (!unregister)
return;
xdst = (struct xfrm_dst *)dst;
if (xdst->u.rt.idev->dev == dev) {
struct in_device *loopback_idev =
in_dev_get(dev_net(dev)->loopback_dev);
BUG_ON(!loopback_idev);
do {
in_dev_put(xdst->u.rt.idev);
xdst->u.rt.idev = loopback_idev;
in_dev_hold(loopback_idev);
xdst = (struct xfrm_dst *)xdst->u.dst.child;
} while (xdst->u.dst.xfrm);
__in_dev_put(loopback_idev);
}
xfrm_dst_ifdown(dst, dev);
}
static struct dst_ops xfrm4_dst_ops = {
.family = AF_INET,
.protocol = cpu_to_be16(ETH_P_IP),
.gc = xfrm4_garbage_collect,
.update_pmtu = xfrm4_update_pmtu,
.destroy = xfrm4_dst_destroy,
.ifdown = xfrm4_dst_ifdown,
.local_out = __ip_local_out,
.gc_thresh = 1024,
};
static struct xfrm_policy_afinfo xfrm4_policy_afinfo = {
.family = AF_INET,
.dst_ops = &xfrm4_dst_ops,
.dst_lookup = xfrm4_dst_lookup,
.get_saddr = xfrm4_get_saddr,
.decode_session = _decode_session4,
.get_tos = xfrm4_get_tos,
.init_path = xfrm4_init_path,
.fill_dst = xfrm4_fill_dst,
};
#ifdef CONFIG_SYSCTL
static struct ctl_table xfrm4_policy_table[] = {
{
.procname = "xfrm4_gc_thresh",
.data = &init_net.xfrm.xfrm4_dst_ops.gc_thresh,
.maxlen = sizeof(int),
.mode = 0644,
.proc_handler = proc_dointvec,
},
{ }
};
static struct ctl_table_header *sysctl_hdr;
#endif
static void __init xfrm4_policy_init(void)
{
xfrm_policy_register_afinfo(&xfrm4_policy_afinfo);
}
static void __exit xfrm4_policy_fini(void)
{
#ifdef CONFIG_SYSCTL
if (sysctl_hdr)
unregister_net_sysctl_table(sysctl_hdr);
#endif
xfrm_policy_unregister_afinfo(&xfrm4_policy_afinfo);
}
void __init xfrm4_init(int rt_max_size)
{
/*
* Select a default value for the gc_thresh based on the main route
* table hash size. It seems to me the worst case scenario is when
* we have ipsec operating in transport mode, in which we create a
* dst_entry per socket. The xfrm gc algorithm starts trying to remove
* entries at gc_thresh, and prevents new allocations as 2*gc_thresh
* so lets set an initial xfrm gc_thresh value at the rt_max_size/2.
* That will let us store an ipsec connection per route table entry,
* and start cleaning when were 1/2 full
*/
xfrm4_dst_ops.gc_thresh = rt_max_size/2;
dst_entries_init(&xfrm4_dst_ops);
xfrm4_state_init();
xfrm4_policy_init();
#ifdef CONFIG_SYSCTL
sysctl_hdr = register_net_sysctl_table(&init_net, net_ipv4_ctl_path,
xfrm4_policy_table);
#endif
}