mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-30 06:10:56 +00:00
io_uring: separate out file table handling code
Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
parent
f4c163dd7d
commit
453b329be5
5 changed files with 117 additions and 93 deletions
|
@ -3,5 +3,5 @@
|
||||||
# Makefile for io_uring
|
# Makefile for io_uring
|
||||||
|
|
||||||
obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \
|
obj-$(CONFIG_IO_URING) += io_uring.o xattr.o nop.o fs.o splice.o \
|
||||||
sync.o advise.o
|
sync.o advise.o filetable.o
|
||||||
obj-$(CONFIG_IO_WQ) += io-wq.o
|
obj-$(CONFIG_IO_WQ) += io-wq.o
|
||||||
|
|
57
io_uring/filetable.c
Normal file
57
io_uring/filetable.c
Normal file
|
@ -0,0 +1,57 @@
|
||||||
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
#include <linux/kernel.h>
|
||||||
|
#include <linux/errno.h>
|
||||||
|
#include <linux/file.h>
|
||||||
|
#include <linux/mm.h>
|
||||||
|
#include <linux/slab.h>
|
||||||
|
#include <linux/io_uring.h>
|
||||||
|
|
||||||
|
#include <uapi/linux/io_uring.h>
|
||||||
|
|
||||||
|
#include "io_uring_types.h"
|
||||||
|
#include "io_uring.h"
|
||||||
|
|
||||||
|
int io_file_bitmap_get(struct io_ring_ctx *ctx)
|
||||||
|
{
|
||||||
|
struct io_file_table *table = &ctx->file_table;
|
||||||
|
unsigned long nr = ctx->nr_user_files;
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
do {
|
||||||
|
ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
|
||||||
|
if (ret != nr)
|
||||||
|
return ret;
|
||||||
|
|
||||||
|
if (!table->alloc_hint)
|
||||||
|
break;
|
||||||
|
|
||||||
|
nr = table->alloc_hint;
|
||||||
|
table->alloc_hint = 0;
|
||||||
|
} while (1);
|
||||||
|
|
||||||
|
return -ENFILE;
|
||||||
|
}
|
||||||
|
|
||||||
|
bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
|
||||||
|
{
|
||||||
|
table->files = kvcalloc(nr_files, sizeof(table->files[0]),
|
||||||
|
GFP_KERNEL_ACCOUNT);
|
||||||
|
if (unlikely(!table->files))
|
||||||
|
return false;
|
||||||
|
|
||||||
|
table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
|
||||||
|
if (unlikely(!table->bitmap)) {
|
||||||
|
kvfree(table->files);
|
||||||
|
return false;
|
||||||
|
}
|
||||||
|
|
||||||
|
return true;
|
||||||
|
}
|
||||||
|
|
||||||
|
void io_free_file_tables(struct io_file_table *table)
|
||||||
|
{
|
||||||
|
kvfree(table->files);
|
||||||
|
bitmap_free(table->bitmap);
|
||||||
|
table->files = NULL;
|
||||||
|
table->bitmap = NULL;
|
||||||
|
}
|
58
io_uring/filetable.h
Normal file
58
io_uring/filetable.h
Normal file
|
@ -0,0 +1,58 @@
|
||||||
|
// SPDX-License-Identifier: GPL-2.0
|
||||||
|
#ifndef IOU_FILE_TABLE_H
|
||||||
|
#define IOU_FILE_TABLE_H
|
||||||
|
|
||||||
|
struct io_ring_ctx;
|
||||||
|
|
||||||
|
/*
|
||||||
|
* FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0
|
||||||
|
* and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we
|
||||||
|
* can't safely always dereference the file when the task has exited and ring
|
||||||
|
* cleanup is done. If a file is tracked and part of SCM, then unix gc on
|
||||||
|
* process exit may reap it before __io_sqe_files_unregister() is run.
|
||||||
|
*/
|
||||||
|
#define FFS_NOWAIT 0x1UL
|
||||||
|
#define FFS_ISREG 0x2UL
|
||||||
|
#if defined(CONFIG_64BIT)
|
||||||
|
#define FFS_SCM 0x4UL
|
||||||
|
#else
|
||||||
|
#define IO_URING_SCM_ALL
|
||||||
|
#define FFS_SCM 0x0UL
|
||||||
|
#endif
|
||||||
|
#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM)
|
||||||
|
|
||||||
|
struct io_fixed_file {
|
||||||
|
/* file * with additional FFS_* flags */
|
||||||
|
unsigned long file_ptr;
|
||||||
|
};
|
||||||
|
|
||||||
|
struct io_file_table {
|
||||||
|
struct io_fixed_file *files;
|
||||||
|
unsigned long *bitmap;
|
||||||
|
unsigned int alloc_hint;
|
||||||
|
};
|
||||||
|
|
||||||
|
bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files);
|
||||||
|
void io_free_file_tables(struct io_file_table *table);
|
||||||
|
int io_file_bitmap_get(struct io_ring_ctx *ctx);
|
||||||
|
|
||||||
|
static inline void io_file_bitmap_clear(struct io_file_table *table, int bit)
|
||||||
|
{
|
||||||
|
__clear_bit(bit, table->bitmap);
|
||||||
|
table->alloc_hint = bit;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline void io_file_bitmap_set(struct io_file_table *table, int bit)
|
||||||
|
{
|
||||||
|
WARN_ON_ONCE(test_bit(bit, table->bitmap));
|
||||||
|
__set_bit(bit, table->bitmap);
|
||||||
|
table->alloc_hint = bit + 1;
|
||||||
|
}
|
||||||
|
|
||||||
|
static inline struct io_fixed_file *
|
||||||
|
io_fixed_file_slot(struct io_file_table *table, unsigned i)
|
||||||
|
{
|
||||||
|
return &table->files[i];
|
||||||
|
}
|
||||||
|
|
||||||
|
#endif
|
|
@ -146,28 +146,6 @@ struct io_overflow_cqe {
|
||||||
struct io_uring_cqe cqe;
|
struct io_uring_cqe cqe;
|
||||||
};
|
};
|
||||||
|
|
||||||
/*
|
|
||||||
* FFS_SCM is only available on 64-bit archs, for 32-bit we just define it as 0
|
|
||||||
* and define IO_URING_SCM_ALL. For this case, we use SCM for all files as we
|
|
||||||
* can't safely always dereference the file when the task has exited and ring
|
|
||||||
* cleanup is done. If a file is tracked and part of SCM, then unix gc on
|
|
||||||
* process exit may reap it before __io_sqe_files_unregister() is run.
|
|
||||||
*/
|
|
||||||
#define FFS_NOWAIT 0x1UL
|
|
||||||
#define FFS_ISREG 0x2UL
|
|
||||||
#if defined(CONFIG_64BIT)
|
|
||||||
#define FFS_SCM 0x4UL
|
|
||||||
#else
|
|
||||||
#define IO_URING_SCM_ALL
|
|
||||||
#define FFS_SCM 0x0UL
|
|
||||||
#endif
|
|
||||||
#define FFS_MASK ~(FFS_NOWAIT|FFS_ISREG|FFS_SCM)
|
|
||||||
|
|
||||||
struct io_fixed_file {
|
|
||||||
/* file * with additional FFS_* flags */
|
|
||||||
unsigned long file_ptr;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct io_rsrc_put {
|
struct io_rsrc_put {
|
||||||
struct list_head list;
|
struct list_head list;
|
||||||
u64 tag;
|
u64 tag;
|
||||||
|
@ -3983,27 +3961,6 @@ static int io_openat2_prep(struct io_kiocb *req, const struct io_uring_sqe *sqe)
|
||||||
return __io_openat_prep(req, sqe);
|
return __io_openat_prep(req, sqe);
|
||||||
}
|
}
|
||||||
|
|
||||||
static int io_file_bitmap_get(struct io_ring_ctx *ctx)
|
|
||||||
{
|
|
||||||
struct io_file_table *table = &ctx->file_table;
|
|
||||||
unsigned long nr = ctx->nr_user_files;
|
|
||||||
int ret;
|
|
||||||
|
|
||||||
do {
|
|
||||||
ret = find_next_zero_bit(table->bitmap, nr, table->alloc_hint);
|
|
||||||
if (ret != nr)
|
|
||||||
return ret;
|
|
||||||
|
|
||||||
if (!table->alloc_hint)
|
|
||||||
break;
|
|
||||||
|
|
||||||
nr = table->alloc_hint;
|
|
||||||
table->alloc_hint = 0;
|
|
||||||
} while (1);
|
|
||||||
|
|
||||||
return -ENFILE;
|
|
||||||
}
|
|
||||||
|
|
||||||
/*
|
/*
|
||||||
* Note when io_fixed_fd_install() returns error value, it will ensure
|
* Note when io_fixed_fd_install() returns error value, it will ensure
|
||||||
* fput() is called correspondingly.
|
* fput() is called correspondingly.
|
||||||
|
@ -6832,12 +6789,6 @@ static void io_wq_submit_work(struct io_wq_work *work)
|
||||||
io_req_task_queue_fail(req, ret);
|
io_req_task_queue_fail(req, ret);
|
||||||
}
|
}
|
||||||
|
|
||||||
static inline struct io_fixed_file *io_fixed_file_slot(struct io_file_table *table,
|
|
||||||
unsigned i)
|
|
||||||
{
|
|
||||||
return &table->files[i];
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
|
static inline struct file *io_file_from_index(struct io_ring_ctx *ctx,
|
||||||
int index)
|
int index)
|
||||||
{
|
{
|
||||||
|
@ -7934,43 +7885,6 @@ static __cold int io_rsrc_data_alloc(struct io_ring_ctx *ctx, rsrc_put_fn *do_pu
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
static bool io_alloc_file_tables(struct io_file_table *table, unsigned nr_files)
|
|
||||||
{
|
|
||||||
table->files = kvcalloc(nr_files, sizeof(table->files[0]),
|
|
||||||
GFP_KERNEL_ACCOUNT);
|
|
||||||
if (unlikely(!table->files))
|
|
||||||
return false;
|
|
||||||
|
|
||||||
table->bitmap = bitmap_zalloc(nr_files, GFP_KERNEL_ACCOUNT);
|
|
||||||
if (unlikely(!table->bitmap)) {
|
|
||||||
kvfree(table->files);
|
|
||||||
return false;
|
|
||||||
}
|
|
||||||
|
|
||||||
return true;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void io_free_file_tables(struct io_file_table *table)
|
|
||||||
{
|
|
||||||
kvfree(table->files);
|
|
||||||
bitmap_free(table->bitmap);
|
|
||||||
table->files = NULL;
|
|
||||||
table->bitmap = NULL;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void io_file_bitmap_set(struct io_file_table *table, int bit)
|
|
||||||
{
|
|
||||||
WARN_ON_ONCE(test_bit(bit, table->bitmap));
|
|
||||||
__set_bit(bit, table->bitmap);
|
|
||||||
table->alloc_hint = bit + 1;
|
|
||||||
}
|
|
||||||
|
|
||||||
static inline void io_file_bitmap_clear(struct io_file_table *table, int bit)
|
|
||||||
{
|
|
||||||
__clear_bit(bit, table->bitmap);
|
|
||||||
table->alloc_hint = bit;
|
|
||||||
}
|
|
||||||
|
|
||||||
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
|
static void __io_sqe_files_unregister(struct io_ring_ctx *ctx)
|
||||||
{
|
{
|
||||||
#if !defined(IO_URING_SCM_ALL)
|
#if !defined(IO_URING_SCM_ALL)
|
||||||
|
|
|
@ -5,6 +5,7 @@
|
||||||
#include <linux/task_work.h>
|
#include <linux/task_work.h>
|
||||||
|
|
||||||
#include "io-wq.h"
|
#include "io-wq.h"
|
||||||
|
#include "filetable.h"
|
||||||
|
|
||||||
struct io_uring {
|
struct io_uring {
|
||||||
u32 head ____cacheline_aligned_in_smp;
|
u32 head ____cacheline_aligned_in_smp;
|
||||||
|
@ -122,12 +123,6 @@ struct io_ev_fd {
|
||||||
struct rcu_head rcu;
|
struct rcu_head rcu;
|
||||||
};
|
};
|
||||||
|
|
||||||
struct io_file_table {
|
|
||||||
struct io_fixed_file *files;
|
|
||||||
unsigned long *bitmap;
|
|
||||||
unsigned int alloc_hint;
|
|
||||||
};
|
|
||||||
|
|
||||||
struct io_ring_ctx {
|
struct io_ring_ctx {
|
||||||
/* const or read-mostly hot data */
|
/* const or read-mostly hot data */
|
||||||
struct {
|
struct {
|
||||||
|
|
Loading…
Reference in a new issue