mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-10-03 23:58:05 +00:00
886fcee939
Apparently the spinlock on incoming_handshake's skb_queue is highly
contended, and a torrent of handshake or cookie packets can bring the
data plane to its knees, simply by virtue of enqueueing the handshake
packets to be processed asynchronously. So, we try switching this to a
ring buffer to hopefully have less lock contention. This alleviates the
problem somewhat, though it still isn't perfect, so future patches will
have to improve this further. However, it at least doesn't completely
diminish the data plane.
Reported-by: Streun Fabio <fstreun@student.ethz.ch>
Reported-by: Joel Wanner <joel.wanner@inf.ethz.ch>
Fixes: e7096c131e ("net: WireGuard secure network tunnel")
Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com>
Signed-off-by: Jakub Kicinski <kuba@kernel.org>
107 lines
2.6 KiB
C
107 lines
2.6 KiB
C
// SPDX-License-Identifier: GPL-2.0
|
|
/*
|
|
* Copyright (C) 2015-2019 Jason A. Donenfeld <Jason@zx2c4.com>. All Rights Reserved.
|
|
*/
|
|
|
|
#include "queueing.h"
|
|
|
|
struct multicore_worker __percpu *
|
|
wg_packet_percpu_multicore_worker_alloc(work_func_t function, void *ptr)
|
|
{
|
|
int cpu;
|
|
struct multicore_worker __percpu *worker = alloc_percpu(struct multicore_worker);
|
|
|
|
if (!worker)
|
|
return NULL;
|
|
|
|
for_each_possible_cpu(cpu) {
|
|
per_cpu_ptr(worker, cpu)->ptr = ptr;
|
|
INIT_WORK(&per_cpu_ptr(worker, cpu)->work, function);
|
|
}
|
|
return worker;
|
|
}
|
|
|
|
int wg_packet_queue_init(struct crypt_queue *queue, work_func_t function,
|
|
unsigned int len)
|
|
{
|
|
int ret;
|
|
|
|
memset(queue, 0, sizeof(*queue));
|
|
ret = ptr_ring_init(&queue->ring, len, GFP_KERNEL);
|
|
if (ret)
|
|
return ret;
|
|
queue->worker = wg_packet_percpu_multicore_worker_alloc(function, queue);
|
|
if (!queue->worker) {
|
|
ptr_ring_cleanup(&queue->ring, NULL);
|
|
return -ENOMEM;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
void wg_packet_queue_free(struct crypt_queue *queue, bool purge)
|
|
{
|
|
free_percpu(queue->worker);
|
|
WARN_ON(!purge && !__ptr_ring_empty(&queue->ring));
|
|
ptr_ring_cleanup(&queue->ring, purge ? (void(*)(void*))kfree_skb : NULL);
|
|
}
|
|
|
|
#define NEXT(skb) ((skb)->prev)
|
|
#define STUB(queue) ((struct sk_buff *)&queue->empty)
|
|
|
|
void wg_prev_queue_init(struct prev_queue *queue)
|
|
{
|
|
NEXT(STUB(queue)) = NULL;
|
|
queue->head = queue->tail = STUB(queue);
|
|
queue->peeked = NULL;
|
|
atomic_set(&queue->count, 0);
|
|
BUILD_BUG_ON(
|
|
offsetof(struct sk_buff, next) != offsetof(struct prev_queue, empty.next) -
|
|
offsetof(struct prev_queue, empty) ||
|
|
offsetof(struct sk_buff, prev) != offsetof(struct prev_queue, empty.prev) -
|
|
offsetof(struct prev_queue, empty));
|
|
}
|
|
|
|
static void __wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)
|
|
{
|
|
WRITE_ONCE(NEXT(skb), NULL);
|
|
WRITE_ONCE(NEXT(xchg_release(&queue->head, skb)), skb);
|
|
}
|
|
|
|
bool wg_prev_queue_enqueue(struct prev_queue *queue, struct sk_buff *skb)
|
|
{
|
|
if (!atomic_add_unless(&queue->count, 1, MAX_QUEUED_PACKETS))
|
|
return false;
|
|
__wg_prev_queue_enqueue(queue, skb);
|
|
return true;
|
|
}
|
|
|
|
struct sk_buff *wg_prev_queue_dequeue(struct prev_queue *queue)
|
|
{
|
|
struct sk_buff *tail = queue->tail, *next = smp_load_acquire(&NEXT(tail));
|
|
|
|
if (tail == STUB(queue)) {
|
|
if (!next)
|
|
return NULL;
|
|
queue->tail = next;
|
|
tail = next;
|
|
next = smp_load_acquire(&NEXT(next));
|
|
}
|
|
if (next) {
|
|
queue->tail = next;
|
|
atomic_dec(&queue->count);
|
|
return tail;
|
|
}
|
|
if (tail != READ_ONCE(queue->head))
|
|
return NULL;
|
|
__wg_prev_queue_enqueue(queue, STUB(queue));
|
|
next = smp_load_acquire(&NEXT(tail));
|
|
if (next) {
|
|
queue->tail = next;
|
|
atomic_dec(&queue->count);
|
|
return tail;
|
|
}
|
|
return NULL;
|
|
}
|
|
|
|
#undef NEXT
|
|
#undef STUB
|