mirrors/linux-stable
1
0
Fork 0
mirror of https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git synced 2024-09-21 01:51:18 +00:00
Code Issues Releases Wiki Activity

io_uring: implement our own schedule timeout handling

Browse source 
In preparation for having two distinct timeouts and avoid waking the
task if we don't need to.

Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe 2024-01-04 08:46:23 -07:00
parent 45a41e74b8
commit cebf123c63
2 changed files with 33 additions and 4 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

35
io_uring/io_uring.c
View file

@ -2350,6 +2350,34 @@ static bool current_pending_io(void)
return percpu_counter_read_positive(&tctx->inflight);
}
static enum hrtimer_restart io_cqring_timer_wakeup(struct hrtimer *timer)
{
struct io_wait_queue *iowq = container_of(timer, struct io_wait_queue, t);
WRITE_ONCE(iowq->hit_timeout, 1);
wake_up_process(iowq->wq.private);
return HRTIMER_NORESTART;
}
static int io_cqring_schedule_timeout(struct io_wait_queue *iowq,
clockid_t clock_id)
{
iowq->hit_timeout = 0;
hrtimer_init_on_stack(&iowq->t, clock_id, HRTIMER_MODE_ABS);
iowq->t.function = io_cqring_timer_wakeup;
hrtimer_set_expires_range_ns(&iowq->t, iowq->timeout, 0);
hrtimer_start_expires(&iowq->t, HRTIMER_MODE_ABS);
if (!READ_ONCE(iowq->hit_timeout))
schedule();
hrtimer_cancel(&iowq->t);
destroy_hrtimer_on_stack(&iowq->t);
__set_current_state(TASK_RUNNING);
return READ_ONCE(iowq->hit_timeout) ? -ETIME : 0;
}
static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
struct io_wait_queue *iowq)
{
@ -2362,11 +2390,10 @@ static int __io_cqring_wait_schedule(struct io_ring_ctx *ctx,
*/
if (current_pending_io())
current->in_iowait = 1;
if (iowq->timeout == KTIME_MAX)
if (iowq->timeout != KTIME_MAX)
ret = io_cqring_schedule_timeout(iowq, ctx->clockid);
else
schedule();
else if (!schedule_hrtimeout_range_clock(&iowq->timeout, 0,
HRTIMER_MODE_ABS, ctx->clockid))
ret = -ETIME;
current->in_iowait = 0;
return ret;
}

2
io_uring/io_uring.h
View file

@ -40,7 +40,9 @@ struct io_wait_queue {
struct io_ring_ctx *ctx;
unsigned cq_tail;
unsigned nr_timeouts;
int hit_timeout;
ktime_t timeout;
struct hrtimer t;
#ifdef CONFIG_NET_RX_BUSY_POLL
ktime_t napi_busy_poll_dt;