diff --git a/include/linux/audit.h b/include/linux/audit.h index 00f7a80f1a3e..3608992848d3 100644 --- a/include/linux/audit.h +++ b/include/linux/audit.h @@ -285,7 +285,6 @@ static inline int audit_signal_info(int sig, struct task_struct *t) /* These are defined in auditsc.c */ /* Public API */ extern int audit_alloc(struct task_struct *task); -extern int audit_alloc_kernel(struct task_struct *task); extern void __audit_free(struct task_struct *task); extern void __audit_uring_entry(u8 op); extern void __audit_uring_exit(int success, long code); @@ -578,10 +577,6 @@ static inline int audit_alloc(struct task_struct *task) { return 0; } -static inline int audit_alloc_kernel(struct task_struct *task) -{ - return 0; -} static inline void audit_free(struct task_struct *task) { } static inline void audit_uring_entry(u8 op) diff --git a/include/linux/fs.h b/include/linux/fs.h index 5113f65c786f..9eced4cc286e 100644 --- a/include/linux/fs.h +++ b/include/linux/fs.h @@ -340,17 +340,12 @@ enum rw_hint { struct kiocb { struct file *ki_filp; - - /* The 'ki_filp' pointer is shared in a union for aio */ - randomized_struct_fields_start - loff_t ki_pos; void (*ki_complete)(struct kiocb *iocb, long ret); void *private; int ki_flags; u16 ki_ioprio; /* See linux/ioprio.h */ struct wait_page_queue *ki_waitq; /* for async buffered IO */ - randomized_struct_fields_end }; static inline bool is_sync_kiocb(struct kiocb *kiocb) diff --git a/include/linux/io_uring_types.h b/include/linux/io_uring_types.h index f7fab3758cb9..677a25d44d7f 100644 --- a/include/linux/io_uring_types.h +++ b/include/linux/io_uring_types.h @@ -491,7 +491,14 @@ struct io_cmd_data { __u8 data[56]; }; -#define io_kiocb_to_cmd(req) ((void *) &(req)->cmd) +static inline void io_kiocb_cmd_sz_check(size_t cmd_sz) +{ + BUILD_BUG_ON(cmd_sz > sizeof(struct io_cmd_data)); +} +#define io_kiocb_to_cmd(req, cmd_type) ( \ + io_kiocb_cmd_sz_check(sizeof(cmd_type)) , \ + ((cmd_type *)&(req)->cmd) \ +) #define cmd_to_io_kiocb(ptr) ((struct io_kiocb *) ptr) struct io_kiocb { diff --git a/io_uring/advise.c b/io_uring/advise.c index 581956934c0b..449c6f14649f 100644 --- a/io_uring/advise.c +++ b/io_uring/advise.c @@ -31,7 +31,7 @@ struct io_madvise { int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) - struct io_madvise *ma = io_kiocb_to_cmd(req); + struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise); if (sqe->buf_index || sqe->off || sqe->splice_fd_in) return -EINVAL; @@ -48,7 +48,7 @@ int io_madvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_madvise(struct io_kiocb *req, unsigned int issue_flags) { #if defined(CONFIG_ADVISE_SYSCALLS) && defined(CONFIG_MMU) - struct io_madvise *ma = io_kiocb_to_cmd(req); + struct io_madvise *ma = io_kiocb_to_cmd(req, struct io_madvise); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -64,7 +64,7 @@ int io_madvise(struct io_kiocb *req, unsigned int issue_flags) int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_fadvise *fa = io_kiocb_to_cmd(req); + struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); if (sqe->buf_index || sqe->addr || sqe->splice_fd_in) return -EINVAL; @@ -77,7 +77,7 @@ int io_fadvise_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fadvise(struct io_kiocb *req, unsigned int issue_flags) { - struct io_fadvise *fa = io_kiocb_to_cmd(req); + struct io_fadvise *fa = io_kiocb_to_cmd(req, struct io_fadvise); int ret; if (issue_flags & IO_URING_F_NONBLOCK) { diff --git a/io_uring/cancel.c b/io_uring/cancel.c index 8435a1eba59a..e4e1dc0325f0 100644 --- a/io_uring/cancel.c +++ b/io_uring/cancel.c @@ -107,7 +107,7 @@ int io_try_cancel(struct io_uring_task *tctx, struct io_cancel_data *cd, int io_async_cancel_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_cancel *cancel = io_kiocb_to_cmd(req); + struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); if (unlikely(req->flags & REQ_F_BUFFER_SELECT)) return -EINVAL; @@ -164,7 +164,7 @@ static int __io_async_cancel(struct io_cancel_data *cd, int io_async_cancel(struct io_kiocb *req, unsigned int issue_flags) { - struct io_cancel *cancel = io_kiocb_to_cmd(req); + struct io_cancel *cancel = io_kiocb_to_cmd(req, struct io_cancel); struct io_cancel_data cd = { .ctx = req->ctx, .data = cancel->addr, diff --git a/io_uring/epoll.c b/io_uring/epoll.c index a8b794471d6b..9aa74d2c80bc 100644 --- a/io_uring/epoll.c +++ b/io_uring/epoll.c @@ -23,7 +23,7 @@ struct io_epoll { int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_epoll *epoll = io_kiocb_to_cmd(req); + struct io_epoll *epoll = io_kiocb_to_cmd(req, struct io_epoll); pr_warn_once("%s: epoll_ctl support in io_uring is deprecated and will " "be removed in a future Linux kernel version.\n", @@ -49,7 +49,7 @@ int io_epoll_ctl_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_epoll_ctl(struct io_kiocb *req, unsigned int issue_flags) { - struct io_epoll *ie = io_kiocb_to_cmd(req); + struct io_epoll *ie = io_kiocb_to_cmd(req, struct io_epoll); int ret; bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; diff --git a/io_uring/fs.c b/io_uring/fs.c index 0de4f549bb7d..7100c293c13a 100644 --- a/io_uring/fs.c +++ b/io_uring/fs.c @@ -49,7 +49,7 @@ struct io_link { int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); const char __user *oldf, *newf; if (sqe->buf_index || sqe->splice_fd_in) @@ -79,7 +79,7 @@ int io_renameat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_renameat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -95,7 +95,7 @@ int io_renameat(struct io_kiocb *req, unsigned int issue_flags) void io_renameat_cleanup(struct io_kiocb *req) { - struct io_rename *ren = io_kiocb_to_cmd(req); + struct io_rename *ren = io_kiocb_to_cmd(req, struct io_rename); putname(ren->oldpath); putname(ren->newpath); @@ -103,7 +103,7 @@ void io_renameat_cleanup(struct io_kiocb *req) int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_unlink *un = io_kiocb_to_cmd(req); + struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink); const char __user *fname; if (sqe->off || sqe->len || sqe->buf_index || sqe->splice_fd_in) @@ -128,7 +128,7 @@ int io_unlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_unlink *un = io_kiocb_to_cmd(req); + struct io_unlink *un = io_kiocb_to_cmd(req, struct io_unlink); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -146,14 +146,14 @@ int io_unlinkat(struct io_kiocb *req, unsigned int issue_flags) void io_unlinkat_cleanup(struct io_kiocb *req) { - struct io_unlink *ul = io_kiocb_to_cmd(req); + struct io_unlink *ul = io_kiocb_to_cmd(req, struct io_unlink); putname(ul->filename); } int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_mkdir *mkd = io_kiocb_to_cmd(req); + struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir); const char __user *fname; if (sqe->off || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -175,7 +175,7 @@ int io_mkdirat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_mkdir *mkd = io_kiocb_to_cmd(req); + struct io_mkdir *mkd = io_kiocb_to_cmd(req, struct io_mkdir); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -190,14 +190,14 @@ int io_mkdirat(struct io_kiocb *req, unsigned int issue_flags) void io_mkdirat_cleanup(struct io_kiocb *req) { - struct io_mkdir *md = io_kiocb_to_cmd(req); + struct io_mkdir *md = io_kiocb_to_cmd(req, struct io_mkdir); putname(md->filename); } int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); const char __user *oldpath, *newpath; if (sqe->len || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -225,7 +225,7 @@ int io_symlinkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -240,7 +240,7 @@ int io_symlinkat(struct io_kiocb *req, unsigned int issue_flags) int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_link *lnk = io_kiocb_to_cmd(req); + struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link); const char __user *oldf, *newf; if (sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in) @@ -270,7 +270,7 @@ int io_linkat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_linkat(struct io_kiocb *req, unsigned int issue_flags) { - struct io_link *lnk = io_kiocb_to_cmd(req); + struct io_link *lnk = io_kiocb_to_cmd(req, struct io_link); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -286,7 +286,7 @@ int io_linkat(struct io_kiocb *req, unsigned int issue_flags) void io_link_cleanup(struct io_kiocb *req) { - struct io_link *sl = io_kiocb_to_cmd(req); + struct io_link *sl = io_kiocb_to_cmd(req, struct io_link); putname(sl->oldpath); putname(sl->newpath); diff --git a/io_uring/io-wq.c b/io_uring/io-wq.c index 77df5b43bf52..c6536d4b2da0 100644 --- a/io_uring/io-wq.c +++ b/io_uring/io-wq.c @@ -624,8 +624,6 @@ static int io_wqe_worker(void *data) snprintf(buf, sizeof(buf), "iou-wrk-%d", wq->task->pid); set_task_comm(current, buf); - audit_alloc_kernel(current); - while (!test_bit(IO_WQ_BIT_EXIT, &wq->state)) { long ret; @@ -660,7 +658,6 @@ static int io_wqe_worker(void *data) if (test_bit(IO_WQ_BIT_EXIT, &wq->state)) io_worker_handle_work(worker); - audit_free(current); io_worker_exit(worker); return 0; } diff --git a/io_uring/io_uring.c b/io_uring/io_uring.c index b54218da075c..ebfdb2212ec2 100644 --- a/io_uring/io_uring.c +++ b/io_uring/io_uring.c @@ -3885,13 +3885,15 @@ SYSCALL_DEFINE4(io_uring_register, unsigned int, fd, unsigned int, opcode, static int __init io_uring_init(void) { -#define __BUILD_BUG_VERIFY_ELEMENT(stype, eoffset, etype, ename) do { \ +#define __BUILD_BUG_VERIFY_OFFSET_SIZE(stype, eoffset, esize, ename) do { \ BUILD_BUG_ON(offsetof(stype, ename) != eoffset); \ - BUILD_BUG_ON(sizeof(etype) != sizeof_field(stype, ename)); \ + BUILD_BUG_ON(sizeof_field(stype, ename) != esize); \ } while (0) #define BUILD_BUG_SQE_ELEM(eoffset, etype, ename) \ - __BUILD_BUG_VERIFY_ELEMENT(struct io_uring_sqe, eoffset, etype, ename) + __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, sizeof(etype), ename) +#define BUILD_BUG_SQE_ELEM_SIZE(eoffset, esize, ename) \ + __BUILD_BUG_VERIFY_OFFSET_SIZE(struct io_uring_sqe, eoffset, esize, ename) BUILD_BUG_ON(sizeof(struct io_uring_sqe) != 64); BUILD_BUG_SQE_ELEM(0, __u8, opcode); BUILD_BUG_SQE_ELEM(1, __u8, flags); @@ -3899,6 +3901,8 @@ static int __init io_uring_init(void) BUILD_BUG_SQE_ELEM(4, __s32, fd); BUILD_BUG_SQE_ELEM(8, __u64, off); BUILD_BUG_SQE_ELEM(8, __u64, addr2); + BUILD_BUG_SQE_ELEM(8, __u32, cmd_op); + BUILD_BUG_SQE_ELEM(12, __u32, __pad1); BUILD_BUG_SQE_ELEM(16, __u64, addr); BUILD_BUG_SQE_ELEM(16, __u64, splice_off_in); BUILD_BUG_SQE_ELEM(24, __u32, len); @@ -3917,13 +3921,22 @@ static int __init io_uring_init(void) BUILD_BUG_SQE_ELEM(28, __u32, statx_flags); BUILD_BUG_SQE_ELEM(28, __u32, fadvise_advice); BUILD_BUG_SQE_ELEM(28, __u32, splice_flags); + BUILD_BUG_SQE_ELEM(28, __u32, rename_flags); + BUILD_BUG_SQE_ELEM(28, __u32, unlink_flags); + BUILD_BUG_SQE_ELEM(28, __u32, hardlink_flags); + BUILD_BUG_SQE_ELEM(28, __u32, xattr_flags); + BUILD_BUG_SQE_ELEM(28, __u32, msg_ring_flags); BUILD_BUG_SQE_ELEM(32, __u64, user_data); BUILD_BUG_SQE_ELEM(40, __u16, buf_index); BUILD_BUG_SQE_ELEM(40, __u16, buf_group); BUILD_BUG_SQE_ELEM(42, __u16, personality); BUILD_BUG_SQE_ELEM(44, __s32, splice_fd_in); BUILD_BUG_SQE_ELEM(44, __u32, file_index); + BUILD_BUG_SQE_ELEM(44, __u16, notification_idx); + BUILD_BUG_SQE_ELEM(46, __u16, addr_len); BUILD_BUG_SQE_ELEM(48, __u64, addr3); + BUILD_BUG_SQE_ELEM_SIZE(48, 0, cmd); + BUILD_BUG_SQE_ELEM(56, __u64, __pad2); BUILD_BUG_ON(sizeof(struct io_uring_files_update) != sizeof(struct io_uring_rsrc_update)); diff --git a/io_uring/kbuf.c b/io_uring/kbuf.c index e538fa7cb727..25cd724ade18 100644 --- a/io_uring/kbuf.c +++ b/io_uring/kbuf.c @@ -272,7 +272,7 @@ void io_destroy_buffers(struct io_ring_ctx *ctx) int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); u64 tmp; if (sqe->rw_flags || sqe->addr || sqe->len || sqe->off || @@ -291,7 +291,7 @@ int io_remove_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; int ret = 0; @@ -319,7 +319,7 @@ int io_remove_buffers(struct io_kiocb *req, unsigned int issue_flags) int io_provide_buffers_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { unsigned long size, tmp_check; - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); u64 tmp; if (sqe->rw_flags || sqe->splice_fd_in) @@ -421,7 +421,7 @@ static int io_add_buffers(struct io_ring_ctx *ctx, struct io_provide_buf *pbuf, int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) { - struct io_provide_buf *p = io_kiocb_to_cmd(req); + struct io_provide_buf *p = io_kiocb_to_cmd(req, struct io_provide_buf); struct io_ring_ctx *ctx = req->ctx; struct io_buffer_list *bl; int ret = 0; @@ -436,7 +436,7 @@ int io_provide_buffers(struct io_kiocb *req, unsigned int issue_flags) bl = io_buffer_get_list(ctx, p->bgid); if (unlikely(!bl)) { - bl = kzalloc(sizeof(*bl), GFP_KERNEL); + bl = kzalloc(sizeof(*bl), GFP_KERNEL_ACCOUNT); if (!bl) { ret = -ENOMEM; goto err; diff --git a/io_uring/msg_ring.c b/io_uring/msg_ring.c index 753d16734319..976c4ba68ee7 100644 --- a/io_uring/msg_ring.c +++ b/io_uring/msg_ring.c @@ -26,7 +26,7 @@ struct io_msg { static int io_msg_ring_data(struct io_kiocb *req) { struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); if (msg->src_fd || msg->dst_fd || msg->flags) return -EINVAL; @@ -76,7 +76,7 @@ static int io_double_lock_ctx(struct io_ring_ctx *ctx, static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *target_ctx = req->file->private_data; - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); struct io_ring_ctx *ctx = req->ctx; unsigned long file_ptr; struct file *src_file; @@ -122,7 +122,7 @@ static int io_msg_send_fd(struct io_kiocb *req, unsigned int issue_flags) int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); if (unlikely(sqe->buf_index || sqe->personality)) return -EINVAL; @@ -141,7 +141,7 @@ int io_msg_ring_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_msg_ring(struct io_kiocb *req, unsigned int issue_flags) { - struct io_msg *msg = io_kiocb_to_cmd(req); + struct io_msg *msg = io_kiocb_to_cmd(req, struct io_msg); int ret; ret = -EBADFD; diff --git a/io_uring/net.c b/io_uring/net.c index 32fc3da04e41..6d71748e2c5a 100644 --- a/io_uring/net.c +++ b/io_uring/net.c @@ -70,13 +70,14 @@ struct io_sendzc { unsigned flags; unsigned addr_len; void __user *addr; + size_t done_io; }; #define IO_APOLL_MULTI_POLLED (REQ_F_APOLL_MULTISHOT | REQ_F_POLLED) int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); + struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); if (unlikely(sqe->off || sqe->addr || sqe->rw_flags || sqe->buf_index || sqe->splice_fd_in)) @@ -88,7 +89,7 @@ int io_shutdown_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_shutdown(struct io_kiocb *req, unsigned int issue_flags) { - struct io_shutdown *shutdown = io_kiocb_to_cmd(req); + struct io_shutdown *shutdown = io_kiocb_to_cmd(req, struct io_shutdown); struct socket *sock; int ret; @@ -173,7 +174,7 @@ static int io_setup_async_msg(struct io_kiocb *req, static int io_sendmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); iomsg->msg.msg_name = &iomsg->addr; iomsg->free_iov = iomsg->fast_iov; @@ -200,7 +201,7 @@ void io_sendmsg_recvmsg_cleanup(struct io_kiocb *req) int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); if (unlikely(sqe->file_index || sqe->addr2)) return -EINVAL; @@ -224,7 +225,7 @@ int io_sendmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr iomsg, *kmsg; struct socket *sock; unsigned flags; @@ -283,7 +284,7 @@ int io_sendmsg(struct io_kiocb *req, unsigned int issue_flags) int io_send(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct iovec iov; struct socket *sock; @@ -357,7 +358,7 @@ static bool io_recvmsg_multishot_overflow(struct io_async_msghdr *iomsg) static int __io_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct user_msghdr msg; int ret; @@ -404,7 +405,7 @@ static int __io_recvmsg_copy_hdr(struct io_kiocb *req, static int __io_compat_recvmsg_copy_hdr(struct io_kiocb *req, struct io_async_msghdr *iomsg) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct compat_msghdr msg; struct compat_iovec __user *uiov; int ret; @@ -482,7 +483,7 @@ int io_recvmsg_prep_async(struct io_kiocb *req) int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); if (unlikely(sqe->file_index || sqe->addr2)) return -EINVAL; @@ -517,7 +518,7 @@ int io_recvmsg_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static inline void io_recv_prep_retry(struct io_kiocb *req) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); sr->done_io = 0; sr->len = 0; /* get from the provided buffer */ @@ -575,12 +576,12 @@ static int io_recvmsg_prep_multishot(struct io_async_msghdr *kmsg, if (kmsg->controllen) { unsigned long control = ubuf + hdr - kmsg->controllen; - kmsg->msg.msg_control_user = (void *) control; + kmsg->msg.msg_control_user = (void __user *) control; kmsg->msg.msg_controllen = kmsg->controllen; } sr->buf = *buf; /* stash for later copy */ - *buf = (void *) (ubuf + hdr); + *buf = (void __user *) (ubuf + hdr); kmsg->payloadlen = *len = *len - hdr; return 0; } @@ -646,7 +647,7 @@ static int io_recvmsg_multishot(struct socket *sock, struct io_sr_msg *io, int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct io_async_msghdr iomsg, *kmsg; struct socket *sock; unsigned int cflags; @@ -758,7 +759,7 @@ int io_recvmsg(struct io_kiocb *req, unsigned int issue_flags) int io_recv(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sr_msg *sr = io_kiocb_to_cmd(req); + struct io_sr_msg *sr = io_kiocb_to_cmd(req, struct io_sr_msg); struct msghdr msg; struct socket *sock; struct iovec iov; @@ -849,7 +850,7 @@ int io_recv(struct io_kiocb *req, unsigned int issue_flags) int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sendzc *zc = io_kiocb_to_cmd(req); + struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); struct io_ring_ctx *ctx = req->ctx; if (READ_ONCE(sqe->__pad2[0]) || READ_ONCE(sqe->addr3)) @@ -878,6 +879,7 @@ int io_sendzc_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) zc->addr = u64_to_user_ptr(READ_ONCE(sqe->addr2)); zc->addr_len = READ_ONCE(sqe->addr_len); + zc->done_io = 0; #ifdef CONFIG_COMPAT if (req->ctx->compat) @@ -944,7 +946,7 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) { struct sockaddr_storage address; struct io_ring_ctx *ctx = req->ctx; - struct io_sendzc *zc = io_kiocb_to_cmd(req); + struct io_sendzc *zc = io_kiocb_to_cmd(req, struct io_sendzc); struct io_notif_slot *notif_slot; struct io_kiocb *notif; struct msghdr msg; @@ -1012,18 +1014,30 @@ int io_sendzc(struct io_kiocb *req, unsigned int issue_flags) if (unlikely(ret < min_ret)) { if (ret == -EAGAIN && (issue_flags & IO_URING_F_NONBLOCK)) return -EAGAIN; - return ret == -ERESTARTSYS ? -EINTR : ret; + if (ret > 0 && io_net_retry(sock, msg.msg_flags)) { + zc->len -= ret; + zc->buf += ret; + zc->done_io += ret; + req->flags |= REQ_F_PARTIAL_IO; + return -EAGAIN; + } + if (ret == -ERESTARTSYS) + ret = -EINTR; + } else if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) { + io_notif_slot_flush_submit(notif_slot, 0); } - if (zc->flags & IORING_RECVSEND_NOTIF_FLUSH) - io_notif_slot_flush_submit(notif_slot, 0); + if (ret >= 0) + ret += zc->done_io; + else if (zc->done_io) + ret = zc->done_io; io_req_set_res(req, ret, 0); return IOU_OK; } int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_accept *accept = io_kiocb_to_cmd(req); + struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); unsigned flags; if (sqe->len || sqe->buf_index) @@ -1057,7 +1071,7 @@ int io_accept_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_accept(struct io_kiocb *req, unsigned int issue_flags) { struct io_ring_ctx *ctx = req->ctx; - struct io_accept *accept = io_kiocb_to_cmd(req); + struct io_accept *accept = io_kiocb_to_cmd(req, struct io_accept); bool force_nonblock = issue_flags & IO_URING_F_NONBLOCK; unsigned int file_flags = force_nonblock ? O_NONBLOCK : 0; bool fixed = !!accept->file_slot; @@ -1115,7 +1129,7 @@ int io_accept(struct io_kiocb *req, unsigned int issue_flags) int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_socket *sock = io_kiocb_to_cmd(req); + struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); if (sqe->addr || sqe->rw_flags || sqe->buf_index) return -EINVAL; @@ -1136,7 +1150,7 @@ int io_socket_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_socket(struct io_kiocb *req, unsigned int issue_flags) { - struct io_socket *sock = io_kiocb_to_cmd(req); + struct io_socket *sock = io_kiocb_to_cmd(req, struct io_socket); bool fixed = !!sock->file_slot; struct file *file; int ret, fd; @@ -1170,14 +1184,14 @@ int io_socket(struct io_kiocb *req, unsigned int issue_flags) int io_connect_prep_async(struct io_kiocb *req) { struct io_async_connect *io = req->async_data; - struct io_connect *conn = io_kiocb_to_cmd(req); + struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); return move_addr_to_kernel(conn->addr, conn->addr_len, &io->address); } int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_connect *conn = io_kiocb_to_cmd(req); + struct io_connect *conn = io_kiocb_to_cmd(req, struct io_connect); if (sqe->len || sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) return -EINVAL; @@ -1189,7 +1203,7 @@ int io_connect_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_connect(struct io_kiocb *req, unsigned int issue_flags) { - struct io_connect *connect = io_kiocb_to_cmd(req); + struct io_connect *connect = io_kiocb_to_cmd(req, struct io_connect); struct io_async_connect __io, *io; unsigned file_flags; int ret; diff --git a/io_uring/notif.c b/io_uring/notif.c index b5f989dff9de..977736e82c1a 100644 --- a/io_uring/notif.c +++ b/io_uring/notif.c @@ -100,7 +100,7 @@ __cold int io_notif_unregister(struct io_ring_ctx *ctx) if (!notif) continue; - nd = io_kiocb_to_cmd(notif); + nd = io_notif_to_data(notif); slot->notif = NULL; if (!refcount_dec_and_test(&nd->uarg.refcnt)) continue; @@ -123,8 +123,6 @@ __cold int io_notif_register(struct io_ring_ctx *ctx, struct io_uring_notification_register reg; unsigned i; - BUILD_BUG_ON(sizeof(struct io_notif_data) > 64); - if (ctx->nr_notif_slots) return -EBUSY; if (size != sizeof(reg)) diff --git a/io_uring/notif.h b/io_uring/notif.h index 0819304d7e00..65f0b42f2555 100644 --- a/io_uring/notif.h +++ b/io_uring/notif.h @@ -46,7 +46,7 @@ struct io_kiocb *io_alloc_notif(struct io_ring_ctx *ctx, static inline struct io_notif_data *io_notif_to_data(struct io_kiocb *notif) { - return io_kiocb_to_cmd(notif); + return io_kiocb_to_cmd(notif, struct io_notif_data); } static inline struct io_kiocb *io_get_notif(struct io_ring_ctx *ctx, diff --git a/io_uring/openclose.c b/io_uring/openclose.c index d1818ec9169b..67178e4bb282 100644 --- a/io_uring/openclose.c +++ b/io_uring/openclose.c @@ -33,7 +33,7 @@ struct io_close { static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); const char __user *fname; int ret; @@ -66,7 +66,7 @@ static int __io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); u64 mode = READ_ONCE(sqe->len); u64 flags = READ_ONCE(sqe->open_flags); @@ -76,7 +76,7 @@ int io_openat_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_how __user *how; size_t len; int ret; @@ -95,7 +95,7 @@ int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_openat2(struct io_kiocb *req, unsigned int issue_flags) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); struct open_flags op; struct file *file; bool resolve_nonblock, nonblock_set; @@ -167,7 +167,7 @@ int io_openat(struct io_kiocb *req, unsigned int issue_flags) void io_open_cleanup(struct io_kiocb *req) { - struct io_open *open = io_kiocb_to_cmd(req); + struct io_open *open = io_kiocb_to_cmd(req, struct io_open); if (open->filename) putname(open->filename); @@ -187,14 +187,14 @@ int __io_close_fixed(struct io_ring_ctx *ctx, unsigned int issue_flags, static inline int io_close_fixed(struct io_kiocb *req, unsigned int issue_flags) { - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); return __io_close_fixed(req->ctx, issue_flags, close->file_slot - 1); } int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); if (sqe->off || sqe->addr || sqe->len || sqe->rw_flags || sqe->buf_index) return -EINVAL; @@ -212,7 +212,7 @@ int io_close_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_close(struct io_kiocb *req, unsigned int issue_flags) { struct files_struct *files = current->files; - struct io_close *close = io_kiocb_to_cmd(req); + struct io_close *close = io_kiocb_to_cmd(req, struct io_close); struct fdtable *fdt; struct file *file; int ret = -EBADF; diff --git a/io_uring/poll.c b/io_uring/poll.c index dadd293749b0..d5bad0bea6e4 100644 --- a/io_uring/poll.c +++ b/io_uring/poll.c @@ -85,7 +85,7 @@ static struct io_poll *io_poll_get_double(struct io_kiocb *req) static struct io_poll *io_poll_get_single(struct io_kiocb *req) { if (req->opcode == IORING_OP_POLL_ADD) - return io_kiocb_to_cmd(req); + return io_kiocb_to_cmd(req, struct io_poll); return &req->apoll->poll; } @@ -274,7 +274,7 @@ static void io_poll_task_func(struct io_kiocb *req, bool *locked) return; if (ret == IOU_POLL_DONE) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); req->cqe.res = mangle_poll(req->cqe.res & poll->events); } else if (ret != IOU_POLL_REMOVE_POLL_USE_RES) { req->cqe.res = ret; @@ -475,7 +475,7 @@ static void io_poll_queue_proc(struct file *file, struct wait_queue_head *head, struct poll_table_struct *p) { struct io_poll_table *pt = container_of(p, struct io_poll_table, pt); - struct io_poll *poll = io_kiocb_to_cmd(pt->req); + struct io_poll *poll = io_kiocb_to_cmd(pt->req, struct io_poll); __io_queue_proc(poll, pt, head, (struct io_poll **) &pt->req->async_data); @@ -821,7 +821,7 @@ static __poll_t io_poll_parse_events(const struct io_uring_sqe *sqe, int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_poll_update *upd = io_kiocb_to_cmd(req); + struct io_poll_update *upd = io_kiocb_to_cmd(req, struct io_poll_update); u32 flags; if (sqe->buf_index || sqe->splice_fd_in) @@ -851,7 +851,7 @@ int io_poll_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); u32 flags; if (sqe->buf_index || sqe->off || sqe->addr) @@ -868,7 +868,7 @@ int io_poll_add_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) { - struct io_poll *poll = io_kiocb_to_cmd(req); + struct io_poll *poll = io_kiocb_to_cmd(req, struct io_poll); struct io_poll_table ipt; int ret; @@ -891,7 +891,7 @@ int io_poll_add(struct io_kiocb *req, unsigned int issue_flags) int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) { - struct io_poll_update *poll_update = io_kiocb_to_cmd(req); + struct io_poll_update *poll_update = io_kiocb_to_cmd(req, struct io_poll_update); struct io_cancel_data cd = { .data = poll_update->old_user_data, }; struct io_ring_ctx *ctx = req->ctx; struct io_hash_bucket *bucket; @@ -930,7 +930,7 @@ int io_poll_remove(struct io_kiocb *req, unsigned int issue_flags) if (poll_update->update_events || poll_update->update_user_data) { /* only mask one event flags, keep behavior flags */ if (poll_update->update_events) { - struct io_poll *poll = io_kiocb_to_cmd(preq); + struct io_poll *poll = io_kiocb_to_cmd(preq, struct io_poll); poll->events &= ~0xffff; poll->events |= poll_update->events & 0xffff; diff --git a/io_uring/rsrc.c b/io_uring/rsrc.c index 59704b9ac537..71359a4d0bd4 100644 --- a/io_uring/rsrc.c +++ b/io_uring/rsrc.c @@ -657,7 +657,7 @@ __cold int io_register_rsrc(struct io_ring_ctx *ctx, void __user *arg, int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) return -EINVAL; @@ -676,7 +676,7 @@ int io_rsrc_update_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int io_files_update_with_index_alloc(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); __s32 __user *fds = u64_to_user_ptr(up->arg); unsigned int done; struct file *file; @@ -714,7 +714,7 @@ static int io_files_update_with_index_alloc(struct io_kiocb *req, static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); struct io_ring_ctx *ctx = req->ctx; struct io_uring_rsrc_update2 up2; int ret; @@ -743,7 +743,7 @@ static int io_files_update(struct io_kiocb *req, unsigned int issue_flags) static int io_notif_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); struct io_ring_ctx *ctx = req->ctx; unsigned len = up->nr_args; unsigned idx_end, idx = up->offset; @@ -778,7 +778,7 @@ static int io_notif_update(struct io_kiocb *req, unsigned int issue_flags) int io_rsrc_update(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rsrc_update *up = io_kiocb_to_cmd(req); + struct io_rsrc_update *up = io_kiocb_to_cmd(req, struct io_rsrc_update); switch (up->type) { case IORING_RSRC_UPDATE_FILES: diff --git a/io_uring/rw.c b/io_uring/rw.c index b20ba87e4926..1babd77da79c 100644 --- a/io_uring/rw.c +++ b/io_uring/rw.c @@ -35,7 +35,7 @@ static inline bool io_file_supports_nowait(struct io_kiocb *req) int io_prep_rw(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); unsigned ioprio; int ret; @@ -102,7 +102,7 @@ static inline void io_rw_done(struct kiocb *kiocb, ssize_t ret) static inline loff_t *io_kiocb_update_pos(struct io_kiocb *req) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_pos != -1) return &rw->kiocb.ki_pos; @@ -186,7 +186,7 @@ static void kiocb_end_write(struct io_kiocb *req) static bool __io_complete_rw_common(struct io_kiocb *req, long res) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (rw->kiocb.ki_flags & IOCB_WRITE) { kiocb_end_write(req); @@ -241,7 +241,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, unsigned int issue_flags) { struct io_async_rw *io = req->async_data; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); /* add previously done IO, if any */ if (req_has_async_data(req) && io->bytes_done > 0) { @@ -277,7 +277,7 @@ static int kiocb_done(struct io_kiocb *req, ssize_t ret, static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct compat_iovec __user *uiov; compat_ssize_t clen; void __user *buf; @@ -305,7 +305,7 @@ static ssize_t io_compat_import(struct io_kiocb *req, struct iovec *iov, static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct iovec __user *uiov = u64_to_user_ptr(rw->addr); void __user *buf; ssize_t len; @@ -328,7 +328,7 @@ static ssize_t __io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, static ssize_t io_iov_buffer_select(struct io_kiocb *req, struct iovec *iov, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); if (req->flags & (REQ_F_BUFFER_SELECTED|REQ_F_BUFFER_RING)) { iov[0].iov_base = u64_to_user_ptr(rw->addr); @@ -350,7 +350,7 @@ static struct iovec *__io_import_iovec(int ddir, struct io_kiocb *req, struct io_rw_state *s, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct iov_iter *iter = &s->iter; u8 opcode = req->opcode; struct iovec *iovec; @@ -571,7 +571,7 @@ static int io_async_buf_func(struct wait_queue_entry *wait, unsigned mode, { struct wait_page_queue *wpq; struct io_kiocb *req = wait->private; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct wait_page_key *key = arg; wpq = container_of(wait, struct wait_page_queue, wait); @@ -601,7 +601,7 @@ static bool io_rw_should_retry(struct io_kiocb *req) { struct io_async_rw *io = req->async_data; struct wait_page_queue *wait = &io->wpq; - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct kiocb *kiocb = &rw->kiocb; /* never retry for NOWAIT, we just complete with -EAGAIN */ @@ -649,7 +649,7 @@ static bool need_complete_io(struct io_kiocb *req) static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct kiocb *kiocb = &rw->kiocb; struct io_ring_ctx *ctx = req->ctx; struct file *file = req->file; @@ -694,7 +694,7 @@ static int io_rw_init_file(struct io_kiocb *req, fmode_t mode) int io_read(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &rw->kiocb; @@ -839,7 +839,7 @@ int io_read(struct io_kiocb *req, unsigned int issue_flags) int io_write(struct io_kiocb *req, unsigned int issue_flags) { - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); struct io_rw_state __s, *s = &__s; struct iovec *iovec; struct kiocb *kiocb = &rw->kiocb; @@ -994,7 +994,7 @@ int io_do_iopoll(struct io_ring_ctx *ctx, bool force_nonspin) wq_list_for_each(pos, start, &ctx->iopoll_list) { struct io_kiocb *req = container_of(pos, struct io_kiocb, comp_list); - struct io_rw *rw = io_kiocb_to_cmd(req); + struct io_rw *rw = io_kiocb_to_cmd(req, struct io_rw); int ret; /* diff --git a/io_uring/splice.c b/io_uring/splice.c index b013ba34bffa..53e4232d0866 100644 --- a/io_uring/splice.c +++ b/io_uring/splice.c @@ -26,7 +26,7 @@ struct io_splice { static int __io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); unsigned int valid_flags = SPLICE_F_FD_IN_FIXED | SPLICE_F_ALL; sp->len = READ_ONCE(sqe->len); @@ -46,7 +46,7 @@ int io_tee_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_tee(struct io_kiocb *req, unsigned int issue_flags) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); struct file *out = sp->file_out; unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; struct file *in; @@ -78,7 +78,7 @@ int io_tee(struct io_kiocb *req, unsigned int issue_flags) int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); sp->off_in = READ_ONCE(sqe->splice_off_in); sp->off_out = READ_ONCE(sqe->off); @@ -87,7 +87,7 @@ int io_splice_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_splice(struct io_kiocb *req, unsigned int issue_flags) { - struct io_splice *sp = io_kiocb_to_cmd(req); + struct io_splice *sp = io_kiocb_to_cmd(req, struct io_splice); struct file *out = sp->file_out; unsigned int flags = sp->flags & ~SPLICE_F_FD_IN_FIXED; loff_t *poff_in, *poff_out; diff --git a/io_uring/sqpoll.c b/io_uring/sqpoll.c index 76d4d70c733a..559652380672 100644 --- a/io_uring/sqpoll.c +++ b/io_uring/sqpoll.c @@ -235,8 +235,6 @@ static int io_sq_thread(void *data) set_cpus_allowed_ptr(current, cpu_online_mask); current->flags |= PF_NO_SETAFFINITY; - audit_alloc_kernel(current); - mutex_lock(&sqd->lock); while (1) { bool cap_entries, sqt_spin = false; @@ -310,8 +308,6 @@ static int io_sq_thread(void *data) io_run_task_work(); mutex_unlock(&sqd->lock); - audit_free(current); - complete(&sqd->exited); do_exit(0); } diff --git a/io_uring/statx.c b/io_uring/statx.c index 6056cd7f4876..d8fc933d3f59 100644 --- a/io_uring/statx.c +++ b/io_uring/statx.c @@ -22,7 +22,7 @@ struct io_statx { int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); const char __user *path; if (sqe->buf_index || sqe->splice_fd_in) @@ -53,7 +53,7 @@ int io_statx_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_statx(struct io_kiocb *req, unsigned int issue_flags) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -66,7 +66,7 @@ int io_statx(struct io_kiocb *req, unsigned int issue_flags) void io_statx_cleanup(struct io_kiocb *req) { - struct io_statx *sx = io_kiocb_to_cmd(req); + struct io_statx *sx = io_kiocb_to_cmd(req, struct io_statx); if (sx->filename) putname(sx->filename); diff --git a/io_uring/sync.c b/io_uring/sync.c index f2102afa79ca..64e87ea2b8fb 100644 --- a/io_uring/sync.c +++ b/io_uring/sync.c @@ -24,7 +24,7 @@ struct io_sync { int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in)) return -EINVAL; @@ -37,7 +37,7 @@ int io_sfr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); int ret; /* sync_file_range always requires a blocking context */ @@ -51,7 +51,7 @@ int io_sync_file_range(struct io_kiocb *req, unsigned int issue_flags) int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (unlikely(sqe->addr || sqe->buf_index || sqe->splice_fd_in)) return -EINVAL; @@ -67,7 +67,7 @@ int io_fsync_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fsync(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); loff_t end = sync->off + sync->len; int ret; @@ -83,7 +83,7 @@ int io_fsync(struct io_kiocb *req, unsigned int issue_flags) int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); if (sqe->buf_index || sqe->rw_flags || sqe->splice_fd_in) return -EINVAL; @@ -96,7 +96,7 @@ int io_fallocate_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fallocate(struct io_kiocb *req, unsigned int issue_flags) { - struct io_sync *sync = io_kiocb_to_cmd(req); + struct io_sync *sync = io_kiocb_to_cmd(req, struct io_sync); int ret; /* fallocate always requiring blocking context */ diff --git a/io_uring/timeout.c b/io_uring/timeout.c index 2f9e56935479..78ea2c64b70e 100644 --- a/io_uring/timeout.c +++ b/io_uring/timeout.c @@ -36,7 +36,7 @@ struct io_timeout_rem { static inline bool io_is_timeout_noseq(struct io_kiocb *req) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); return !timeout->off; } @@ -56,7 +56,7 @@ static bool io_kill_timeout(struct io_kiocb *req, int status) struct io_timeout_data *io = req->async_data; if (hrtimer_try_to_cancel(&io->timer) != -1) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); if (status) req_set_fail(req); @@ -188,7 +188,7 @@ struct io_kiocb *__io_disarm_linked_timeout(struct io_kiocb *req, __must_hold(&req->ctx->timeout_lock) { struct io_timeout_data *io = link->async_data; - struct io_timeout *timeout = io_kiocb_to_cmd(link); + struct io_timeout *timeout = io_kiocb_to_cmd(link, struct io_timeout); io_remove_next_linked(req); timeout->head = NULL; @@ -205,7 +205,7 @@ static enum hrtimer_restart io_timeout_fn(struct hrtimer *timer) struct io_timeout_data *data = container_of(timer, struct io_timeout_data, timer); struct io_kiocb *req = data->req; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -252,7 +252,7 @@ static struct io_kiocb *io_timeout_extract(struct io_ring_ctx *ctx, io = req->async_data; if (hrtimer_try_to_cancel(&io->timer) == -1) return ERR_PTR(-EALREADY); - timeout = io_kiocb_to_cmd(req); + timeout = io_kiocb_to_cmd(req, struct io_timeout); list_del_init(&timeout->list); return req; } @@ -275,7 +275,7 @@ int io_timeout_cancel(struct io_ring_ctx *ctx, struct io_cancel_data *cd) static void io_req_task_link_timeout(struct io_kiocb *req, bool *locked) { unsigned issue_flags = *locked ? 0 : IO_URING_F_UNLOCKED; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_kiocb *prev = timeout->prev; int ret = -ENOENT; @@ -302,7 +302,7 @@ static enum hrtimer_restart io_link_timeout_fn(struct hrtimer *timer) struct io_timeout_data *data = container_of(timer, struct io_timeout_data, timer); struct io_kiocb *prev, *req = data->req; - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; unsigned long flags; @@ -378,7 +378,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, { struct io_cancel_data cd = { .data = user_data, }; struct io_kiocb *req = io_timeout_extract(ctx, &cd); - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data; if (IS_ERR(req)) @@ -395,7 +395,7 @@ static int io_timeout_update(struct io_ring_ctx *ctx, __u64 user_data, int io_timeout_remove_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); + struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); if (unlikely(req->flags & (REQ_F_FIXED_FILE | REQ_F_BUFFER_SELECT))) return -EINVAL; @@ -435,7 +435,7 @@ static inline enum hrtimer_mode io_translate_timeout_mode(unsigned int flags) */ int io_timeout_remove(struct io_kiocb *req, unsigned int issue_flags) { - struct io_timeout_rem *tr = io_kiocb_to_cmd(req); + struct io_timeout_rem *tr = io_kiocb_to_cmd(req, struct io_timeout_rem); struct io_ring_ctx *ctx = req->ctx; int ret; @@ -466,7 +466,7 @@ static int __io_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe, bool is_timeout_link) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_timeout_data *data; unsigned flags; u32 off = READ_ONCE(sqe->off); @@ -532,7 +532,7 @@ int io_link_timeout_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_timeout(struct io_kiocb *req, unsigned int issue_flags) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; struct io_timeout_data *data = req->async_data; struct list_head *entry; @@ -583,7 +583,7 @@ int io_timeout(struct io_kiocb *req, unsigned int issue_flags) void io_queue_linked_timeout(struct io_kiocb *req) { - struct io_timeout *timeout = io_kiocb_to_cmd(req); + struct io_timeout *timeout = io_kiocb_to_cmd(req, struct io_timeout); struct io_ring_ctx *ctx = req->ctx; spin_lock_irq(&ctx->timeout_lock); diff --git a/io_uring/uring_cmd.c b/io_uring/uring_cmd.c index 0a421ed51e7e..8e0cc2d9205e 100644 --- a/io_uring/uring_cmd.c +++ b/io_uring/uring_cmd.c @@ -11,7 +11,7 @@ static void io_uring_cmd_work(struct io_kiocb *req, bool *locked) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); ioucmd->task_work_cb(ioucmd); } @@ -46,7 +46,7 @@ void io_uring_cmd_done(struct io_uring_cmd *ioucmd, ssize_t ret, ssize_t res2) if (ret < 0) req_set_fail(req); - io_req_set_res(req, 0, ret); + io_req_set_res(req, ret, 0); if (req->ctx->flags & IORING_SETUP_CQE32) io_req_set_cqe32_extra(req, res2, 0); __io_req_complete(req, 0); @@ -55,9 +55,12 @@ EXPORT_SYMBOL_GPL(io_uring_cmd_done); int io_uring_cmd_prep_async(struct io_kiocb *req) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); size_t cmd_size; + BUILD_BUG_ON(uring_cmd_pdu_size(0) != 16); + BUILD_BUG_ON(uring_cmd_pdu_size(1) != 80); + cmd_size = uring_cmd_pdu_size(req->ctx->flags & IORING_SETUP_SQE128); memcpy(req->async_data, ioucmd->cmd, cmd_size); @@ -66,7 +69,7 @@ int io_uring_cmd_prep_async(struct io_kiocb *req) int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); if (sqe->rw_flags || sqe->__pad1) return -EINVAL; @@ -77,7 +80,7 @@ int io_uring_cmd_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) { - struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req); + struct io_uring_cmd *ioucmd = io_kiocb_to_cmd(req, struct io_uring_cmd); struct io_ring_ctx *ctx = req->ctx; struct file *file = req->file; int ret; @@ -106,7 +109,9 @@ int io_uring_cmd(struct io_kiocb *req, unsigned int issue_flags) } if (ret != -EIOCBQUEUED) { - io_uring_cmd_done(ioucmd, ret, 0); + if (ret < 0) + req_set_fail(req); + io_req_set_res(req, ret, 0); return IOU_OK; } diff --git a/io_uring/xattr.c b/io_uring/xattr.c index b179f9acd5ac..84180afd090b 100644 --- a/io_uring/xattr.c +++ b/io_uring/xattr.c @@ -24,7 +24,7 @@ struct io_xattr { void io_xattr_cleanup(struct io_kiocb *req) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); if (ix->filename) putname(ix->filename); @@ -44,7 +44,7 @@ static void io_xattr_finish(struct io_kiocb *req, int ret) static int __io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *name; int ret; @@ -85,7 +85,7 @@ int io_fgetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *path; int ret; @@ -106,7 +106,7 @@ int io_getxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; if (issue_flags & IO_URING_F_NONBLOCK) @@ -122,7 +122,7 @@ int io_fgetxattr(struct io_kiocb *req, unsigned int issue_flags) int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); unsigned int lookup_flags = LOOKUP_FOLLOW; struct path path; int ret; @@ -151,7 +151,7 @@ int io_getxattr(struct io_kiocb *req, unsigned int issue_flags) static int __io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *name; int ret; @@ -181,7 +181,7 @@ static int __io_setxattr_prep(struct io_kiocb *req, int io_setxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); const char __user *path; int ret; @@ -208,7 +208,7 @@ int io_fsetxattr_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe) static int __io_setxattr(struct io_kiocb *req, unsigned int issue_flags, struct path *path) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); int ret; ret = mnt_want_write(path->mnt); @@ -234,7 +234,7 @@ int io_fsetxattr(struct io_kiocb *req, unsigned int issue_flags) int io_setxattr(struct io_kiocb *req, unsigned int issue_flags) { - struct io_xattr *ix = io_kiocb_to_cmd(req); + struct io_xattr *ix = io_kiocb_to_cmd(req, struct io_xattr); unsigned int lookup_flags = LOOKUP_FOLLOW; struct path path; int ret; diff --git a/kernel/auditsc.c b/kernel/auditsc.c index 3a8c9d744800..dd8d9ab747c3 100644 --- a/kernel/auditsc.c +++ b/kernel/auditsc.c @@ -1073,31 +1073,6 @@ int audit_alloc(struct task_struct *tsk) return 0; } -/** - * audit_alloc_kernel - allocate an audit_context for a kernel task - * @tsk: the kernel task - * - * Similar to the audit_alloc() function, but intended for kernel private - * threads. Returns zero on success, negative values on failure. - */ -int audit_alloc_kernel(struct task_struct *tsk) -{ - /* - * At the moment we are just going to call into audit_alloc() to - * simplify the code, but there two things to keep in mind with this - * approach: - * - * 1. Filtering internal kernel tasks is a bit laughable in almost all - * cases, but there is at least one case where there is a benefit: - * the '-a task,never' case allows the admin to effectively disable - * task auditing at runtime. - * - * 2. The {set,clear}_task_syscall_work() ops likely have zero effect - * on these internal kernel tasks, but they probably don't hurt either. - */ - return audit_alloc(tsk); -} - static inline void audit_free_context(struct audit_context *context) { /* resetting is extra work, but it is likely just noise */