mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-06 02:50:43 +00:00
io_uring: deduplicate core cancellations sequence
Files and task cancellations go over same steps trying to cancel requests in io-wq, poll, etc. Deduplicate it with a helper. note: new io_uring_try_cancel_requests() is former __io_uring_cancel_task_requests() with files passed as an agrument and flushing overflowed requests. Signed-off-by: Pavel Begunkov <asml.silence@gmail.com> Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
57cd657b82
commit
9936c7c2bc
1 changed files with 40 additions and 45 deletions
|
@ -1003,9 +1003,9 @@ enum io_mem_account {
|
|||
ACCT_PINNED,
|
||||
};
|
||||
|
||||
static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
|
||||
struct task_struct *task);
|
||||
|
||||
static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
|
||||
struct task_struct *task,
|
||||
struct files_struct *files);
|
||||
static void destroy_fixed_rsrc_ref_node(struct fixed_rsrc_ref_node *ref_node);
|
||||
static struct fixed_rsrc_ref_node *alloc_fixed_rsrc_ref_node(
|
||||
struct io_ring_ctx *ctx);
|
||||
|
@ -8817,7 +8817,7 @@ static void io_ring_exit_work(struct work_struct *work)
|
|||
* as nobody else will be looking for them.
|
||||
*/
|
||||
do {
|
||||
__io_uring_cancel_task_requests(ctx, NULL);
|
||||
io_uring_try_cancel_requests(ctx, NULL, NULL);
|
||||
} while (!wait_for_completion_timeout(&ctx->ref_comp, HZ/20));
|
||||
io_ring_ctx_free(ctx);
|
||||
}
|
||||
|
@ -8931,6 +8931,40 @@ static void io_cancel_defer_files(struct io_ring_ctx *ctx,
|
|||
}
|
||||
}
|
||||
|
||||
static void io_uring_try_cancel_requests(struct io_ring_ctx *ctx,
|
||||
struct task_struct *task,
|
||||
struct files_struct *files)
|
||||
{
|
||||
struct io_task_cancel cancel = { .task = task, .files = files, };
|
||||
|
||||
while (1) {
|
||||
enum io_wq_cancel cret;
|
||||
bool ret = false;
|
||||
|
||||
if (ctx->io_wq) {
|
||||
cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb,
|
||||
&cancel, true);
|
||||
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
|
||||
}
|
||||
|
||||
/* SQPOLL thread does its own polling */
|
||||
if (!(ctx->flags & IORING_SETUP_SQPOLL) && !files) {
|
||||
while (!list_empty_careful(&ctx->iopoll_list)) {
|
||||
io_iopoll_try_reap_events(ctx);
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
ret |= io_poll_remove_all(ctx, task, files);
|
||||
ret |= io_kill_timeouts(ctx, task, files);
|
||||
ret |= io_run_task_work();
|
||||
io_cqring_overflow_flush(ctx, true, task, files);
|
||||
if (!ret)
|
||||
break;
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
|
||||
static int io_uring_count_inflight(struct io_ring_ctx *ctx,
|
||||
struct task_struct *task,
|
||||
struct files_struct *files)
|
||||
|
@ -8950,7 +8984,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
|
|||
struct files_struct *files)
|
||||
{
|
||||
while (!list_empty_careful(&ctx->inflight_list)) {
|
||||
struct io_task_cancel cancel = { .task = task, .files = files };
|
||||
DEFINE_WAIT(wait);
|
||||
int inflight;
|
||||
|
||||
|
@ -8958,13 +8991,7 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
|
|||
if (!inflight)
|
||||
break;
|
||||
|
||||
io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb, &cancel, true);
|
||||
io_poll_remove_all(ctx, task, files);
|
||||
io_kill_timeouts(ctx, task, files);
|
||||
io_cqring_overflow_flush(ctx, true, task, files);
|
||||
/* cancellations _may_ trigger task work */
|
||||
io_run_task_work();
|
||||
|
||||
io_uring_try_cancel_requests(ctx, task, files);
|
||||
prepare_to_wait(&task->io_uring->wait, &wait,
|
||||
TASK_UNINTERRUPTIBLE);
|
||||
if (inflight == io_uring_count_inflight(ctx, task, files))
|
||||
|
@ -8973,37 +9000,6 @@ static void io_uring_cancel_files(struct io_ring_ctx *ctx,
|
|||
}
|
||||
}
|
||||
|
||||
static void __io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
|
||||
struct task_struct *task)
|
||||
{
|
||||
while (1) {
|
||||
struct io_task_cancel cancel = { .task = task, .files = NULL, };
|
||||
enum io_wq_cancel cret;
|
||||
bool ret = false;
|
||||
|
||||
if (ctx->io_wq) {
|
||||
cret = io_wq_cancel_cb(ctx->io_wq, io_cancel_task_cb,
|
||||
&cancel, true);
|
||||
ret |= (cret != IO_WQ_CANCEL_NOTFOUND);
|
||||
}
|
||||
|
||||
/* SQPOLL thread does its own polling */
|
||||
if (!(ctx->flags & IORING_SETUP_SQPOLL)) {
|
||||
while (!list_empty_careful(&ctx->iopoll_list)) {
|
||||
io_iopoll_try_reap_events(ctx);
|
||||
ret = true;
|
||||
}
|
||||
}
|
||||
|
||||
ret |= io_poll_remove_all(ctx, task, NULL);
|
||||
ret |= io_kill_timeouts(ctx, task, NULL);
|
||||
ret |= io_run_task_work();
|
||||
if (!ret)
|
||||
break;
|
||||
cond_resched();
|
||||
}
|
||||
}
|
||||
|
||||
static void io_disable_sqo_submit(struct io_ring_ctx *ctx)
|
||||
{
|
||||
mutex_lock(&ctx->uring_lock);
|
||||
|
@ -9033,11 +9029,10 @@ static void io_uring_cancel_task_requests(struct io_ring_ctx *ctx,
|
|||
}
|
||||
|
||||
io_cancel_defer_files(ctx, task, files);
|
||||
io_cqring_overflow_flush(ctx, true, task, files);
|
||||
|
||||
io_uring_cancel_files(ctx, task, files);
|
||||
if (!files)
|
||||
__io_uring_cancel_task_requests(ctx, task);
|
||||
io_uring_try_cancel_requests(ctx, task, NULL);
|
||||
|
||||
if ((ctx->flags & IORING_SETUP_SQPOLL) && ctx->sq_data) {
|
||||
atomic_dec(&task->io_uring->in_idle);
|
||||
|
|
Loading…
Reference in a new issue