mirrors
/
linux-stable
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
1
0
Fork 0
Code Issues Releases Wiki Activity

rps: shortcut net_rps_action() 

net_rps_action() is a bit expensive on NR_CPUS=64..4096 kernels, even if
RPS is not active.

Tom Herbert used two bitmasks to hold information needed to send IPI,
but a single LIFO list seems more appropriate.

Move all RPS logic into net_rps_action() to cleanup net_rx_action() code
(remove two ifdefs)

Move rps_remote_softirq_cpus into softnet_data to share its first cache
line, filling an existing hole.

In a future patch, we could call net_rps_action() from process_backlog()
to make sure we send IPI before handling this cpu backlog.

Signed-off-by: Eric Dumazet <eric.dumazet@gmail.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
This commit is contained in:
Eric Dumazet 2010-04-19 05:07:33 +00:00 committed by David S. Miller
parent a03b1a5c95
commit 88751275b8
2 changed files with 38 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
include/linux/netdevice.h
View File

@ -1381,17 +1381,20 @@ static inline int unregister_gifconf(unsigned int family)
}
/*
* Incoming packets are placed on per-cpu queues so that
* no locking is needed.
* Incoming packets are placed on per-cpu queues
*/
struct softnet_data {
struct Qdisc *output_queue;
struct list_head poll_list;
struct sk_buff *completion_queue;
/* Elements below can be accessed between CPUs for RPS */
#ifdef CONFIG_RPS
struct softnet_data *rps_ipi_list;
/* Elements below can be accessed between CPUs for RPS */
struct call_single_data csd ____cacheline_aligned_in_smp;
struct softnet_data *rps_ipi_next;
unsigned int cpu;
unsigned int input_queue_head;
#endif
struct sk_buff_head input_pkt_queue;

79
net/core/dev.c
View File

@ -2345,21 +2345,6 @@ done:
return cpu;
}
/*
* This structure holds the per-CPU mask of CPUs for which IPIs are scheduled
* to be sent to kick remote softirq processing. There are two masks since
* the sending of IPIs must be done with interrupts enabled. The select field
* indicates the current mask that enqueue_backlog uses to schedule IPIs.
* select is flipped before net_rps_action is called while still under lock,
* net_rps_action then uses the non-selected mask to send the IPIs and clears
* it without conflicting with enqueue_backlog operation.
*/
struct rps_remote_softirq_cpus {
cpumask_t mask[2];
int select;
};
static DEFINE_PER_CPU(struct rps_remote_softirq_cpus, rps_remote_softirq_cpus);
/* Called from hardirq (IPI) context */
static void trigger_softirq(void *data)
{
@ -2402,10 +2387,12 @@ enqueue:
if (napi_schedule_prep(&queue->backlog)) {
#ifdef CONFIG_RPS
if (cpu != smp_processor_id()) {
struct rps_remote_softirq_cpus *rcpus =
&__get_cpu_var(rps_remote_softirq_cpus);
struct softnet_data *myqueue;
myqueue = &__get_cpu_var(softnet_data);
queue->rps_ipi_next = myqueue->rps_ipi_list;
myqueue->rps_ipi_list = queue;
cpu_set(cpu, rcpus->mask[rcpus->select]);
__raise_softirq_irqoff(NET_RX_SOFTIRQ);
goto enqueue;
}
@ -2910,7 +2897,9 @@ int netif_receive_skb(struct sk_buff *skb)
}
EXPORT_SYMBOL(netif_receive_skb);
/* Network device is going away, flush any packets still pending */
/* Network device is going away, flush any packets still pending
* Called with irqs disabled.
*/
static void flush_backlog(void *arg)
{
struct net_device *dev = arg;
@ -3338,24 +3327,33 @@ void netif_napi_del(struct napi_struct *napi)
}
EXPORT_SYMBOL(netif_napi_del);
#ifdef CONFIG_RPS
/*
* net_rps_action sends any pending IPI's for rps. This is only called from
* softirq and interrupts must be enabled.
* net_rps_action sends any pending IPI's for rps.
* Note: called with local irq disabled, but exits with local irq enabled.
*/
static void net_rps_action(cpumask_t *mask)
static void net_rps_action(void)
{
int cpu;
#ifdef CONFIG_RPS
struct softnet_data *locqueue = &__get_cpu_var(softnet_data);
struct softnet_data *remqueue = locqueue->rps_ipi_list;
/* Send pending IPI's to kick RPS processing on remote cpus. */
for_each_cpu_mask_nr(cpu, *mask) {
struct softnet_data *queue = &per_cpu(softnet_data, cpu);
if (cpu_online(cpu))
__smp_call_function_single(cpu, &queue->csd, 0);
}
cpus_clear(*mask);
}
if (remqueue) {
locqueue->rps_ipi_list = NULL;
local_irq_enable();
/* Send pending IPI's to kick RPS processing on remote cpus. */
while (remqueue) {
struct softnet_data *next = remqueue->rps_ipi_next;
if (cpu_online(remqueue->cpu))
__smp_call_function_single(remqueue->cpu,
&remqueue->csd, 0);
remqueue = next;
}
} else
#endif
local_irq_enable();
}
static void net_rx_action(struct softirq_action *h)
{
@ -3363,10 +3361,6 @@ static void net_rx_action(struct softirq_action *h)
unsigned long time_limit = jiffies + 2;
int budget = netdev_budget;
void *have;
#ifdef CONFIG_RPS
int select;
struct rps_remote_softirq_cpus *rcpus;
#endif
local_irq_disable();
@ -3429,17 +3423,7 @@ static void net_rx_action(struct softirq_action *h)
netpoll_poll_unlock(have);
}
out:
#ifdef CONFIG_RPS
rcpus = &__get_cpu_var(rps_remote_softirq_cpus);
select = rcpus->select;
rcpus->select ^= 1;
local_irq_enable();
net_rps_action(&rcpus->mask[select]);
#else
local_irq_enable();
#endif
net_rps_action();
#ifdef CONFIG_NET_DMA
/*
@ -5839,6 +5823,7 @@ static int __init net_dev_init(void)
queue->csd.func = trigger_softirq;
queue->csd.info = queue;
queue->csd.flags = 0;
queue->cpu = i;
#endif
queue->backlog.poll = process_backlog;