mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-29 22:02:02 +00:00
Code Issues Releases Wiki Activity

io_uring: IRQ rw completion batching

Browse source 
Employ inline completion logic for read/write completions done via
io_req_task_complete(). If ->uring_lock is contended, just do normal
request completion, but if not, make tctx_task_work() to grab the lock
and do batched inline completions in io_req_task_complete().

Signed-off-by: Pavel Begunkov <asml.silence@gmail.com>
Link: https://lore.kernel.org/r/94589c3ce69eaed86a21bb1ec696407a54fab1aa.1629286357.git.asml.silence@gmail.com
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Pavel Begunkov 2021-08-18 12:42:47 +01:00 committed by Jens Axboe
parent f237c30a56
commit 126180b95f
1 changed files with 16 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
fs/io_uring.c
View file

@ -2067,6 +2067,8 @@ static void tctx_task_work(struct callback_head *cb)
if (req->ctx != ctx) {
ctx_flush_and_put(ctx, &locked);
ctx = req->ctx;
/* if not contended, grab and improve batching */
locked = mutex_trylock(&ctx->uring_lock);
percpu_ref_get(&ctx->refs);
}
req->io_task_work.func(req, &locked);
@ -2582,7 +2584,20 @@ static bool __io_complete_rw_common(struct io_kiocb *req, long res)
static void io_req_task_complete(struct io_kiocb *req, bool *locked)
{
__io_req_complete(req, 0, req->result, io_put_rw_kbuf(req));
unsigned int cflags = io_put_rw_kbuf(req);
long res = req->result;
if (*locked) {
struct io_ring_ctx *ctx = req->ctx;
struct io_submit_state *state = &ctx->submit_state;
io_req_complete_state(req, res, cflags);
state->compl_reqs[state->compl_nr++] = req;
if (state->compl_nr == ARRAY_SIZE(state->compl_reqs))
io_submit_flush_completions(ctx);
} else {
io_req_complete_post(req, res, cflags);
}
}
static void __io_complete_rw(struct io_kiocb *req, long res, long res2,