mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-03 17:40:04 +00:00
io_uring: allow POLL_ADD with double poll_wait() users
Some file descriptors use separate waitqueues for their f_ops->poll() handler, most commonly one for read and one for write. The io_uring poll implementation doesn't work with that, as the 2nd poll_wait() call will cause the io_uring poll request to -EINVAL. This affects (at least) tty devices and /dev/random as well. This is a big problem for event loops where some file descriptors work, and others don't. With this fix, io_uring handles multiple waitqueues. Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
4a38aed2a0
commit
18bceab101
1 changed files with 146 additions and 72 deletions
218
fs/io_uring.c
218
fs/io_uring.c
|
@ -4099,27 +4099,6 @@ struct io_poll_table {
|
|||
int error;
|
||||
};
|
||||
|
||||
static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
|
||||
struct wait_queue_head *head)
|
||||
{
|
||||
if (unlikely(poll->head)) {
|
||||
pt->error = -EINVAL;
|
||||
return;
|
||||
}
|
||||
|
||||
pt->error = 0;
|
||||
poll->head = head;
|
||||
add_wait_queue(head, &poll->wait);
|
||||
}
|
||||
|
||||
static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
|
||||
struct poll_table_struct *p)
|
||||
{
|
||||
struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
|
||||
|
||||
__io_queue_proc(&pt->req->apoll->poll, pt, head);
|
||||
}
|
||||
|
||||
static int __io_async_wake(struct io_kiocb *req, struct io_poll_iocb *poll,
|
||||
__poll_t mask, task_work_func_t func)
|
||||
{
|
||||
|
@ -4171,6 +4150,144 @@ static bool io_poll_rewait(struct io_kiocb *req, struct io_poll_iocb *poll)
|
|||
return false;
|
||||
}
|
||||
|
||||
static void io_poll_remove_double(struct io_kiocb *req)
|
||||
{
|
||||
struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;
|
||||
|
||||
lockdep_assert_held(&req->ctx->completion_lock);
|
||||
|
||||
if (poll && poll->head) {
|
||||
struct wait_queue_head *head = poll->head;
|
||||
|
||||
spin_lock(&head->lock);
|
||||
list_del_init(&poll->wait.entry);
|
||||
if (poll->wait.private)
|
||||
refcount_dec(&req->refs);
|
||||
poll->head = NULL;
|
||||
spin_unlock(&head->lock);
|
||||
}
|
||||
}
|
||||
|
||||
static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
|
||||
{
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
io_poll_remove_double(req);
|
||||
req->poll.done = true;
|
||||
io_cqring_fill_event(req, error ? error : mangle_poll(mask));
|
||||
io_commit_cqring(ctx);
|
||||
}
|
||||
|
||||
static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
|
||||
{
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
if (io_poll_rewait(req, &req->poll)) {
|
||||
spin_unlock_irq(&ctx->completion_lock);
|
||||
return;
|
||||
}
|
||||
|
||||
hash_del(&req->hash_node);
|
||||
io_poll_complete(req, req->result, 0);
|
||||
req->flags |= REQ_F_COMP_LOCKED;
|
||||
io_put_req_find_next(req, nxt);
|
||||
spin_unlock_irq(&ctx->completion_lock);
|
||||
|
||||
io_cqring_ev_posted(ctx);
|
||||
}
|
||||
|
||||
static void io_poll_task_func(struct callback_head *cb)
|
||||
{
|
||||
struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
|
||||
struct io_kiocb *nxt = NULL;
|
||||
|
||||
io_poll_task_handler(req, &nxt);
|
||||
if (nxt) {
|
||||
struct io_ring_ctx *ctx = nxt->ctx;
|
||||
|
||||
mutex_lock(&ctx->uring_lock);
|
||||
__io_queue_sqe(nxt, NULL);
|
||||
mutex_unlock(&ctx->uring_lock);
|
||||
}
|
||||
}
|
||||
|
||||
static int io_poll_double_wake(struct wait_queue_entry *wait, unsigned mode,
|
||||
int sync, void *key)
|
||||
{
|
||||
struct io_kiocb *req = wait->private;
|
||||
struct io_poll_iocb *poll = (struct io_poll_iocb *) req->io;
|
||||
__poll_t mask = key_to_poll(key);
|
||||
|
||||
/* for instances that support it check for an event match first: */
|
||||
if (mask && !(mask & poll->events))
|
||||
return 0;
|
||||
|
||||
if (req->poll.head) {
|
||||
bool done;
|
||||
|
||||
spin_lock(&req->poll.head->lock);
|
||||
done = list_empty(&req->poll.wait.entry);
|
||||
if (!done)
|
||||
list_del_init(&req->poll.wait.entry);
|
||||
spin_unlock(&req->poll.head->lock);
|
||||
if (!done)
|
||||
__io_async_wake(req, poll, mask, io_poll_task_func);
|
||||
}
|
||||
refcount_dec(&req->refs);
|
||||
return 1;
|
||||
}
|
||||
|
||||
static void io_init_poll_iocb(struct io_poll_iocb *poll, __poll_t events,
|
||||
wait_queue_func_t wake_func)
|
||||
{
|
||||
poll->head = NULL;
|
||||
poll->done = false;
|
||||
poll->canceled = false;
|
||||
poll->events = events;
|
||||
INIT_LIST_HEAD(&poll->wait.entry);
|
||||
init_waitqueue_func_entry(&poll->wait, wake_func);
|
||||
}
|
||||
|
||||
static void __io_queue_proc(struct io_poll_iocb *poll, struct io_poll_table *pt,
|
||||
struct wait_queue_head *head)
|
||||
{
|
||||
struct io_kiocb *req = pt->req;
|
||||
|
||||
/*
|
||||
* If poll->head is already set, it's because the file being polled
|
||||
* uses multiple waitqueues for poll handling (eg one for read, one
|
||||
* for write). Setup a separate io_poll_iocb if this happens.
|
||||
*/
|
||||
if (unlikely(poll->head)) {
|
||||
/* already have a 2nd entry, fail a third attempt */
|
||||
if (req->io) {
|
||||
pt->error = -EINVAL;
|
||||
return;
|
||||
}
|
||||
poll = kmalloc(sizeof(*poll), GFP_ATOMIC);
|
||||
if (!poll) {
|
||||
pt->error = -ENOMEM;
|
||||
return;
|
||||
}
|
||||
io_init_poll_iocb(poll, req->poll.events, io_poll_double_wake);
|
||||
refcount_inc(&req->refs);
|
||||
poll->wait.private = req;
|
||||
req->io = (void *) poll;
|
||||
}
|
||||
|
||||
pt->error = 0;
|
||||
poll->head = head;
|
||||
add_wait_queue(head, &poll->wait);
|
||||
}
|
||||
|
||||
static void io_async_queue_proc(struct file *file, struct wait_queue_head *head,
|
||||
struct poll_table_struct *p)
|
||||
{
|
||||
struct io_poll_table *pt = container_of(p, struct io_poll_table, pt);
|
||||
|
||||
__io_queue_proc(&pt->req->apoll->poll, pt, head);
|
||||
}
|
||||
|
||||
static void io_async_task_func(struct callback_head *cb)
|
||||
{
|
||||
struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
|
||||
|
@ -4246,18 +4363,13 @@ static __poll_t __io_arm_poll_handler(struct io_kiocb *req,
|
|||
bool cancel = false;
|
||||
|
||||
poll->file = req->file;
|
||||
poll->head = NULL;
|
||||
poll->done = poll->canceled = false;
|
||||
poll->events = mask;
|
||||
io_init_poll_iocb(poll, mask, wake_func);
|
||||
poll->wait.private = req;
|
||||
|
||||
ipt->pt._key = mask;
|
||||
ipt->req = req;
|
||||
ipt->error = -EINVAL;
|
||||
|
||||
INIT_LIST_HEAD(&poll->wait.entry);
|
||||
init_waitqueue_func_entry(&poll->wait, wake_func);
|
||||
poll->wait.private = req;
|
||||
|
||||
mask = vfs_poll(req->file, &ipt->pt) & poll->events;
|
||||
|
||||
spin_lock_irq(&ctx->completion_lock);
|
||||
|
@ -4288,6 +4400,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
|
|||
struct async_poll *apoll;
|
||||
struct io_poll_table ipt;
|
||||
__poll_t mask, ret;
|
||||
bool had_io;
|
||||
|
||||
if (!req->file || !file_can_poll(req->file))
|
||||
return false;
|
||||
|
@ -4302,6 +4415,7 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
|
|||
|
||||
req->flags |= REQ_F_POLLED;
|
||||
memcpy(&apoll->work, &req->work, sizeof(req->work));
|
||||
had_io = req->io != NULL;
|
||||
|
||||
get_task_struct(current);
|
||||
req->task = current;
|
||||
|
@ -4321,7 +4435,9 @@ static bool io_arm_poll_handler(struct io_kiocb *req)
|
|||
io_async_wake);
|
||||
if (ret) {
|
||||
ipt.error = 0;
|
||||
apoll->poll.done = true;
|
||||
/* only remove double add if we did it here */
|
||||
if (!had_io)
|
||||
io_poll_remove_double(req);
|
||||
spin_unlock_irq(&ctx->completion_lock);
|
||||
memcpy(&req->work, &apoll->work, sizeof(req->work));
|
||||
kfree(apoll);
|
||||
|
@ -4354,6 +4470,7 @@ static bool io_poll_remove_one(struct io_kiocb *req)
|
|||
bool do_complete;
|
||||
|
||||
if (req->opcode == IORING_OP_POLL_ADD) {
|
||||
io_poll_remove_double(req);
|
||||
do_complete = __io_poll_remove_one(req, &req->poll);
|
||||
} else {
|
||||
apoll = req->apoll;
|
||||
|
@ -4455,49 +4572,6 @@ static int io_poll_remove(struct io_kiocb *req)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static void io_poll_complete(struct io_kiocb *req, __poll_t mask, int error)
|
||||
{
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
|
||||
req->poll.done = true;
|
||||
io_cqring_fill_event(req, error ? error : mangle_poll(mask));
|
||||
io_commit_cqring(ctx);
|
||||
}
|
||||
|
||||
static void io_poll_task_handler(struct io_kiocb *req, struct io_kiocb **nxt)
|
||||
{
|
||||
struct io_ring_ctx *ctx = req->ctx;
|
||||
struct io_poll_iocb *poll = &req->poll;
|
||||
|
||||
if (io_poll_rewait(req, poll)) {
|
||||
spin_unlock_irq(&ctx->completion_lock);
|
||||
return;
|
||||
}
|
||||
|
||||
hash_del(&req->hash_node);
|
||||
io_poll_complete(req, req->result, 0);
|
||||
req->flags |= REQ_F_COMP_LOCKED;
|
||||
io_put_req_find_next(req, nxt);
|
||||
spin_unlock_irq(&ctx->completion_lock);
|
||||
|
||||
io_cqring_ev_posted(ctx);
|
||||
}
|
||||
|
||||
static void io_poll_task_func(struct callback_head *cb)
|
||||
{
|
||||
struct io_kiocb *req = container_of(cb, struct io_kiocb, task_work);
|
||||
struct io_kiocb *nxt = NULL;
|
||||
|
||||
io_poll_task_handler(req, &nxt);
|
||||
if (nxt) {
|
||||
struct io_ring_ctx *ctx = nxt->ctx;
|
||||
|
||||
mutex_lock(&ctx->uring_lock);
|
||||
__io_queue_sqe(nxt, NULL);
|
||||
mutex_unlock(&ctx->uring_lock);
|
||||
}
|
||||
}
|
||||
|
||||
static int io_poll_wake(struct wait_queue_entry *wait, unsigned mode, int sync,
|
||||
void *key)
|
||||
{
|
||||
|
|
Loading…
Reference in a new issue