2019-05-29 23:57:49 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0-only
|
2014-10-01 23:07:05 +00:00
|
|
|
/*
|
|
|
|
* Copyright (C) 2013 Shaohua Li <shli@kernel.org>
|
|
|
|
* Copyright (C) 2014 Red Hat, Inc.
|
2015-04-23 18:47:00 +00:00
|
|
|
* Copyright (C) 2015 Arrikto, Inc.
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
* Copyright (C) 2017 Chinamobile, Inc.
|
2014-10-01 23:07:05 +00:00
|
|
|
*/
|
|
|
|
|
|
|
|
#include <linux/spinlock.h>
|
|
|
|
#include <linux/module.h>
|
2015-05-08 08:11:12 +00:00
|
|
|
#include <linux/kernel.h>
|
2014-10-01 23:07:05 +00:00
|
|
|
#include <linux/timer.h>
|
|
|
|
#include <linux/parser.h>
|
2015-05-28 18:35:41 +00:00
|
|
|
#include <linux/vmalloc.h>
|
2014-10-01 23:07:05 +00:00
|
|
|
#include <linux/uio_driver.h>
|
2021-02-24 18:53:34 +00:00
|
|
|
#include <linux/xarray.h>
|
2015-09-03 23:39:56 +00:00
|
|
|
#include <linux/stringify.h>
|
2016-02-26 22:59:57 +00:00
|
|
|
#include <linux/bitops.h>
|
2016-11-18 23:31:45 +00:00
|
|
|
#include <linux/highmem.h>
|
2017-03-18 22:04:13 +00:00
|
|
|
#include <linux/configfs.h>
|
2017-05-02 03:38:06 +00:00
|
|
|
#include <linux/mutex.h>
|
2017-11-28 18:40:30 +00:00
|
|
|
#include <linux/workqueue.h>
|
2014-10-01 23:07:05 +00:00
|
|
|
#include <net/genetlink.h>
|
2015-05-08 08:11:12 +00:00
|
|
|
#include <scsi/scsi_common.h>
|
|
|
|
#include <scsi/scsi_proto.h>
|
2014-10-01 23:07:05 +00:00
|
|
|
#include <target/target_core_base.h>
|
|
|
|
#include <target/target_core_fabric.h>
|
|
|
|
#include <target/target_core_backend.h>
|
2014-11-28 05:11:24 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
#include <linux/target_core_user.h>
|
|
|
|
|
2018-04-14 17:51:05 +00:00
|
|
|
/**
|
|
|
|
* DOC: Userspace I/O
|
|
|
|
* Userspace I/O
|
|
|
|
* -------------
|
|
|
|
*
|
2014-10-01 23:07:05 +00:00
|
|
|
* Define a shared-memory interface for LIO to pass SCSI commands and
|
|
|
|
* data to userspace for processing. This is to allow backends that
|
|
|
|
* are too complex for in-kernel support to be possible.
|
|
|
|
*
|
|
|
|
* It uses the UIO framework to do a lot of the device-creation and
|
|
|
|
* introspection work for us.
|
|
|
|
*
|
|
|
|
* See the .h file for how the ring is laid out. Note that while the
|
|
|
|
* command ring is defined, the particulars of the data area are
|
|
|
|
* not. Offset values in the command entry point to other locations
|
2018-04-14 17:51:05 +00:00
|
|
|
* internal to the mmap-ed area. There is separate space outside the
|
2014-10-01 23:07:05 +00:00
|
|
|
* command ring for data buffers. This leaves maximum flexibility for
|
|
|
|
* moving buffer allocations, or even page flipping or other
|
|
|
|
* allocation techniques, without altering the command ring layout.
|
|
|
|
*
|
|
|
|
* SECURITY:
|
|
|
|
* The user process must be assumed to be malicious. There's no way to
|
|
|
|
* prevent it breaking the command ring protocol if it wants, but in
|
|
|
|
* order to prevent other issues we must only ever read *data* from
|
|
|
|
* the shared memory area, not offsets or sizes. This applies to
|
|
|
|
* command ring entries as well as the mailbox. Extra code needed for
|
|
|
|
* this may have a 'UAM' comment.
|
|
|
|
*/
|
|
|
|
|
|
|
|
#define TCMU_TIME_OUT (30 * MSEC_PER_SEC)
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
/* For cmd area, the size is fixed 8MB */
|
|
|
|
#define CMDR_SIZE (8 * 1024 * 1024)
|
2016-02-26 22:59:57 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
/*
|
|
|
|
* For data area, the block size is PAGE_SIZE and
|
|
|
|
* the total size is 256K * PAGE_SIZE.
|
|
|
|
*/
|
|
|
|
#define DATA_BLOCK_SIZE PAGE_SIZE
|
2017-11-28 18:40:41 +00:00
|
|
|
#define DATA_BLOCK_SHIFT PAGE_SHIFT
|
|
|
|
#define DATA_BLOCK_BITS_DEF (256 * 1024)
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
#define TCMU_MBS_TO_BLOCKS(_mbs) (_mbs << (20 - DATA_BLOCK_SHIFT))
|
|
|
|
#define TCMU_BLOCKS_TO_MBS(_blocks) (_blocks >> (20 - DATA_BLOCK_SHIFT))
|
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
/*
|
|
|
|
* Default number of global data blocks(512K * PAGE_SIZE)
|
|
|
|
* when the unmap thread will be started.
|
|
|
|
*/
|
2017-11-28 18:40:41 +00:00
|
|
|
#define TCMU_GLOBAL_MAX_BLOCKS_DEF (512 * 1024)
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
static u8 tcmu_kern_cmd_reply_supported;
|
2018-06-22 21:40:22 +00:00
|
|
|
static u8 tcmu_netlink_blocked;
|
2017-06-23 06:18:15 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static struct device *tcmu_root_device;
|
|
|
|
|
|
|
|
struct tcmu_hba {
|
|
|
|
u32 host_id;
|
|
|
|
};
|
|
|
|
|
|
|
|
#define TCMU_CONFIG_LEN 256
|
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
static DEFINE_MUTEX(tcmu_nl_cmd_mutex);
|
|
|
|
static LIST_HEAD(tcmu_nl_cmd_list);
|
|
|
|
|
|
|
|
struct tcmu_dev;
|
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
struct tcmu_nl_cmd {
|
|
|
|
/* wake up thread waiting for reply */
|
|
|
|
struct completion complete;
|
2018-06-22 21:40:19 +00:00
|
|
|
struct list_head nl_list;
|
|
|
|
struct tcmu_dev *udev;
|
2017-06-23 06:18:15 +00:00
|
|
|
int cmd;
|
|
|
|
int status;
|
|
|
|
};
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
struct tcmu_dev {
|
2017-05-02 03:38:06 +00:00
|
|
|
struct list_head node;
|
2017-05-17 09:34:37 +00:00
|
|
|
struct kref kref;
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
struct se_device se_dev;
|
2021-02-27 17:00:04 +00:00
|
|
|
struct se_dev_plug se_plug;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
char *name;
|
|
|
|
struct se_hba *hba;
|
|
|
|
|
|
|
|
#define TCMU_DEV_BIT_OPEN 0
|
|
|
|
#define TCMU_DEV_BIT_BROKEN 1
|
2017-12-19 10:03:58 +00:00
|
|
|
#define TCMU_DEV_BIT_BLOCKED 2
|
2020-07-26 15:35:10 +00:00
|
|
|
#define TCMU_DEV_BIT_TMR_NOTIFY 3
|
2021-02-27 17:00:04 +00:00
|
|
|
#define TCM_DEV_BIT_PLUGGED 4
|
2014-10-01 23:07:05 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
struct uio_info uio_info;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
struct inode *inode;
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
struct tcmu_mailbox *mb_addr;
|
2018-07-23 19:07:51 +00:00
|
|
|
uint64_t dev_size;
|
2014-10-01 23:07:05 +00:00
|
|
|
u32 cmdr_size;
|
|
|
|
u32 cmdr_last_cleaned;
|
2016-08-25 15:55:54 +00:00
|
|
|
/* Offset of data area from start of mb */
|
2016-02-26 22:59:57 +00:00
|
|
|
/* Must add data_off and mb_addr to get the address */
|
2014-10-01 23:07:05 +00:00
|
|
|
size_t data_off;
|
|
|
|
size_t data_size;
|
2017-11-28 18:40:41 +00:00
|
|
|
uint32_t max_blocks;
|
|
|
|
size_t ring_size;
|
2016-02-26 22:59:57 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
struct mutex cmdr_lock;
|
2018-11-23 01:15:30 +00:00
|
|
|
struct list_head qfull_queue;
|
2020-07-26 15:35:09 +00:00
|
|
|
struct list_head tmr_queue;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
uint32_t dbi_max;
|
2017-05-02 03:38:06 +00:00
|
|
|
uint32_t dbi_thresh;
|
2017-11-28 18:40:41 +00:00
|
|
|
unsigned long *data_bitmap;
|
2021-02-24 18:53:35 +00:00
|
|
|
struct xarray data_blocks;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
|
2021-02-24 18:53:34 +00:00
|
|
|
struct xarray commands;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
struct timer_list cmd_timer;
|
2017-03-09 08:42:09 +00:00
|
|
|
unsigned int cmd_time_out;
|
2018-11-23 01:15:30 +00:00
|
|
|
struct list_head inflight_queue;
|
2017-11-28 18:40:40 +00:00
|
|
|
|
|
|
|
struct timer_list qfull_timer;
|
|
|
|
int qfull_time_out;
|
|
|
|
|
2017-11-28 18:40:31 +00:00
|
|
|
struct list_head timedout_entry;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
struct tcmu_nl_cmd curr_nl_cmd;
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
char dev_config[TCMU_CONFIG_LEN];
|
2017-09-13 05:01:22 +00:00
|
|
|
|
|
|
|
int nl_reply_supported;
|
2014-10-01 23:07:05 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define TCMU_DEV(_se_dev) container_of(_se_dev, struct tcmu_dev, se_dev)
|
|
|
|
|
|
|
|
#define CMDR_OFF sizeof(struct tcmu_mailbox)
|
|
|
|
|
|
|
|
struct tcmu_cmd {
|
|
|
|
struct se_cmd *se_cmd;
|
|
|
|
struct tcmu_dev *tcmu_dev;
|
2018-11-23 01:15:30 +00:00
|
|
|
struct list_head queue_entry;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
uint16_t cmd_id;
|
|
|
|
|
2016-02-26 22:59:57 +00:00
|
|
|
/* Can't use se_cmd when cleaning up expired cmds, because if
|
2014-10-01 23:07:05 +00:00
|
|
|
cmd has been completed then accessing se_cmd is off limits */
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
uint32_t dbi_cnt;
|
2020-09-10 15:50:39 +00:00
|
|
|
uint32_t dbi_bidi_cnt;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
uint32_t dbi_cur;
|
|
|
|
uint32_t *dbi;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:41 +00:00
|
|
|
uint32_t data_len_bidi;
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
unsigned long deadline;
|
|
|
|
|
|
|
|
#define TCMU_CMD_BIT_EXPIRED 0
|
|
|
|
unsigned long flags;
|
|
|
|
};
|
2020-07-26 15:35:09 +00:00
|
|
|
|
|
|
|
struct tcmu_tmr {
|
|
|
|
struct list_head queue_entry;
|
|
|
|
|
|
|
|
uint8_t tmr_type;
|
|
|
|
uint32_t tmr_cmd_cnt;
|
2020-08-31 13:25:42 +00:00
|
|
|
int16_t tmr_cmd_ids[];
|
2020-07-26 15:35:09 +00:00
|
|
|
};
|
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
/*
|
|
|
|
* To avoid dead lock the mutex lock order should always be:
|
|
|
|
*
|
|
|
|
* mutex_lock(&root_udev_mutex);
|
|
|
|
* ...
|
|
|
|
* mutex_lock(&tcmu_dev->cmdr_lock);
|
|
|
|
* mutex_unlock(&tcmu_dev->cmdr_lock);
|
|
|
|
* ...
|
|
|
|
* mutex_unlock(&root_udev_mutex);
|
|
|
|
*/
|
2017-05-02 03:38:06 +00:00
|
|
|
static DEFINE_MUTEX(root_udev_mutex);
|
|
|
|
static LIST_HEAD(root_udev);
|
|
|
|
|
2017-11-28 18:40:31 +00:00
|
|
|
static DEFINE_SPINLOCK(timed_out_udevs_lock);
|
|
|
|
static LIST_HEAD(timed_out_udevs);
|
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
static struct kmem_cache *tcmu_cmd_cache;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
static atomic_t global_db_count = ATOMIC_INIT(0);
|
2017-11-28 18:40:39 +00:00
|
|
|
static struct delayed_work tcmu_unmap_work;
|
2017-11-28 18:40:41 +00:00
|
|
|
static int tcmu_global_max_blocks = TCMU_GLOBAL_MAX_BLOCKS_DEF;
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
static int tcmu_set_global_max_data_area(const char *str,
|
|
|
|
const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
int ret, max_area_mb;
|
|
|
|
|
|
|
|
ret = kstrtoint(str, 10, &max_area_mb);
|
|
|
|
if (ret)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (max_area_mb <= 0) {
|
|
|
|
pr_err("global_max_data_area must be larger than 0.\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
tcmu_global_max_blocks = TCMU_MBS_TO_BLOCKS(max_area_mb);
|
|
|
|
if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
|
|
|
|
schedule_delayed_work(&tcmu_unmap_work, 0);
|
|
|
|
else
|
|
|
|
cancel_delayed_work_sync(&tcmu_unmap_work);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_get_global_max_data_area(char *buffer,
|
|
|
|
const struct kernel_param *kp)
|
|
|
|
{
|
2020-09-03 11:29:33 +00:00
|
|
|
return sprintf(buffer, "%d\n", TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
|
2017-11-28 18:40:41 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static const struct kernel_param_ops tcmu_global_max_data_area_op = {
|
|
|
|
.set = tcmu_set_global_max_data_area,
|
|
|
|
.get = tcmu_get_global_max_data_area,
|
|
|
|
};
|
|
|
|
|
|
|
|
module_param_cb(global_max_data_area_mb, &tcmu_global_max_data_area_op, NULL,
|
|
|
|
S_IWUSR | S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(global_max_data_area_mb,
|
|
|
|
"Max MBs allowed to be allocated to all the tcmu device's "
|
|
|
|
"data areas.");
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2018-06-22 21:40:22 +00:00
|
|
|
static int tcmu_get_block_netlink(char *buffer,
|
|
|
|
const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
return sprintf(buffer, "%s\n", tcmu_netlink_blocked ?
|
|
|
|
"blocked" : "unblocked");
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_set_block_netlink(const char *str,
|
|
|
|
const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
int ret;
|
|
|
|
u8 val;
|
|
|
|
|
|
|
|
ret = kstrtou8(str, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (val > 1) {
|
|
|
|
pr_err("Invalid block netlink value %u\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
tcmu_netlink_blocked = val;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct kernel_param_ops tcmu_block_netlink_op = {
|
|
|
|
.set = tcmu_set_block_netlink,
|
|
|
|
.get = tcmu_get_block_netlink,
|
|
|
|
};
|
|
|
|
|
|
|
|
module_param_cb(block_netlink, &tcmu_block_netlink_op, NULL, S_IWUSR | S_IRUGO);
|
|
|
|
MODULE_PARM_DESC(block_netlink, "Block new netlink commands.");
|
|
|
|
|
|
|
|
static int tcmu_fail_netlink_cmd(struct tcmu_nl_cmd *nl_cmd)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = nl_cmd->udev;
|
|
|
|
|
|
|
|
if (!tcmu_netlink_blocked) {
|
|
|
|
pr_err("Could not reset device's netlink interface. Netlink is not blocked.\n");
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
|
|
|
|
pr_debug("Aborting nl cmd %d on %s\n", nl_cmd->cmd, udev->name);
|
|
|
|
nl_cmd->status = -EINTR;
|
|
|
|
list_del(&nl_cmd->nl_list);
|
|
|
|
complete(&nl_cmd->complete);
|
|
|
|
}
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_set_reset_netlink(const char *str,
|
|
|
|
const struct kernel_param *kp)
|
|
|
|
{
|
|
|
|
struct tcmu_nl_cmd *nl_cmd, *tmp_cmd;
|
|
|
|
int ret;
|
|
|
|
u8 val;
|
|
|
|
|
|
|
|
ret = kstrtou8(str, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (val != 1) {
|
|
|
|
pr_err("Invalid reset netlink value %u\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
mutex_lock(&tcmu_nl_cmd_mutex);
|
|
|
|
list_for_each_entry_safe(nl_cmd, tmp_cmd, &tcmu_nl_cmd_list, nl_list) {
|
|
|
|
ret = tcmu_fail_netlink_cmd(nl_cmd);
|
|
|
|
if (ret)
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct kernel_param_ops tcmu_reset_netlink_op = {
|
|
|
|
.set = tcmu_set_reset_netlink,
|
|
|
|
};
|
|
|
|
|
|
|
|
module_param_cb(reset_netlink, &tcmu_reset_netlink_op, NULL, S_IWUSR);
|
|
|
|
MODULE_PARM_DESC(reset_netlink, "Reset netlink commands.");
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
/* multicast group */
|
|
|
|
enum tcmu_multicast_groups {
|
|
|
|
TCMU_MCGRP_CONFIG,
|
|
|
|
};
|
|
|
|
|
|
|
|
static const struct genl_multicast_group tcmu_mcgrps[] = {
|
|
|
|
[TCMU_MCGRP_CONFIG] = { .name = "config", },
|
|
|
|
};
|
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
static struct nla_policy tcmu_attr_policy[TCMU_ATTR_MAX+1] = {
|
|
|
|
[TCMU_ATTR_DEVICE] = { .type = NLA_STRING },
|
|
|
|
[TCMU_ATTR_MINOR] = { .type = NLA_U32 },
|
|
|
|
[TCMU_ATTR_CMD_STATUS] = { .type = NLA_S32 },
|
|
|
|
[TCMU_ATTR_DEVICE_ID] = { .type = NLA_U32 },
|
|
|
|
[TCMU_ATTR_SUPP_KERN_CMD_REPLY] = { .type = NLA_U8 },
|
|
|
|
};
|
|
|
|
|
|
|
|
static int tcmu_genl_cmd_done(struct genl_info *info, int completed_cmd)
|
|
|
|
{
|
2018-06-22 21:40:19 +00:00
|
|
|
struct tcmu_dev *udev = NULL;
|
2017-06-23 06:18:15 +00:00
|
|
|
struct tcmu_nl_cmd *nl_cmd;
|
|
|
|
int dev_id, rc, ret = 0;
|
|
|
|
|
|
|
|
if (!info->attrs[TCMU_ATTR_CMD_STATUS] ||
|
|
|
|
!info->attrs[TCMU_ATTR_DEVICE_ID]) {
|
|
|
|
printk(KERN_ERR "TCMU_ATTR_CMD_STATUS or TCMU_ATTR_DEVICE_ID not set, doing nothing\n");
|
2018-06-07 06:31:36 +00:00
|
|
|
return -EINVAL;
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
dev_id = nla_get_u32(info->attrs[TCMU_ATTR_DEVICE_ID]);
|
|
|
|
rc = nla_get_s32(info->attrs[TCMU_ATTR_CMD_STATUS]);
|
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_lock(&tcmu_nl_cmd_mutex);
|
|
|
|
list_for_each_entry(nl_cmd, &tcmu_nl_cmd_list, nl_list) {
|
|
|
|
if (nl_cmd->udev->se_dev.dev_index == dev_id) {
|
|
|
|
udev = nl_cmd->udev;
|
|
|
|
break;
|
|
|
|
}
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
if (!udev) {
|
2018-06-26 16:47:20 +00:00
|
|
|
pr_err("tcmu nl cmd %u/%d completion could not find device with dev id %u.\n",
|
2018-06-22 21:40:19 +00:00
|
|
|
completed_cmd, rc, dev_id);
|
|
|
|
ret = -ENODEV;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
list_del(&nl_cmd->nl_list);
|
2017-06-23 06:18:15 +00:00
|
|
|
|
2018-06-22 21:40:22 +00:00
|
|
|
pr_debug("%s genl cmd done got id %d curr %d done %d rc %d stat %d\n",
|
|
|
|
udev->name, dev_id, nl_cmd->cmd, completed_cmd, rc,
|
|
|
|
nl_cmd->status);
|
2017-06-23 06:18:15 +00:00
|
|
|
|
|
|
|
if (nl_cmd->cmd != completed_cmd) {
|
2018-06-22 21:40:19 +00:00
|
|
|
pr_err("Mismatched commands on %s (Expecting reply for %d. Current %d).\n",
|
|
|
|
udev->name, completed_cmd, nl_cmd->cmd);
|
2017-06-23 06:18:15 +00:00
|
|
|
ret = -EINVAL;
|
2018-06-22 21:40:19 +00:00
|
|
|
goto unlock;
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
nl_cmd->status = rc;
|
|
|
|
complete(&nl_cmd->complete);
|
|
|
|
unlock:
|
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
2017-06-23 06:18:15 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_genl_rm_dev_done(struct sk_buff *skb, struct genl_info *info)
|
|
|
|
{
|
|
|
|
return tcmu_genl_cmd_done(info, TCMU_CMD_REMOVED_DEVICE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_genl_add_dev_done(struct sk_buff *skb, struct genl_info *info)
|
|
|
|
{
|
|
|
|
return tcmu_genl_cmd_done(info, TCMU_CMD_ADDED_DEVICE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_genl_reconfig_dev_done(struct sk_buff *skb,
|
|
|
|
struct genl_info *info)
|
|
|
|
{
|
|
|
|
return tcmu_genl_cmd_done(info, TCMU_CMD_RECONFIG_DEVICE);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_genl_set_features(struct sk_buff *skb, struct genl_info *info)
|
|
|
|
{
|
|
|
|
if (info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]) {
|
|
|
|
tcmu_kern_cmd_reply_supported =
|
|
|
|
nla_get_u8(info->attrs[TCMU_ATTR_SUPP_KERN_CMD_REPLY]);
|
|
|
|
printk(KERN_INFO "tcmu daemon: command reply support %u.\n",
|
|
|
|
tcmu_kern_cmd_reply_supported);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-10-02 21:49:54 +00:00
|
|
|
static const struct genl_small_ops tcmu_genl_ops[] = {
|
2017-06-23 06:18:15 +00:00
|
|
|
{
|
|
|
|
.cmd = TCMU_CMD_SET_FEATURES,
|
2019-04-26 12:07:31 +00:00
|
|
|
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
|
2017-06-23 06:18:15 +00:00
|
|
|
.flags = GENL_ADMIN_PERM,
|
|
|
|
.doit = tcmu_genl_set_features,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.cmd = TCMU_CMD_ADDED_DEVICE_DONE,
|
2019-04-26 12:07:31 +00:00
|
|
|
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
|
2017-06-23 06:18:15 +00:00
|
|
|
.flags = GENL_ADMIN_PERM,
|
|
|
|
.doit = tcmu_genl_add_dev_done,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.cmd = TCMU_CMD_REMOVED_DEVICE_DONE,
|
2019-04-26 12:07:31 +00:00
|
|
|
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
|
2017-06-23 06:18:15 +00:00
|
|
|
.flags = GENL_ADMIN_PERM,
|
|
|
|
.doit = tcmu_genl_rm_dev_done,
|
|
|
|
},
|
|
|
|
{
|
|
|
|
.cmd = TCMU_CMD_RECONFIG_DEVICE_DONE,
|
2019-04-26 12:07:31 +00:00
|
|
|
.validate = GENL_DONT_VALIDATE_STRICT | GENL_DONT_VALIDATE_DUMP,
|
2017-06-23 06:18:15 +00:00
|
|
|
.flags = GENL_ADMIN_PERM,
|
|
|
|
.doit = tcmu_genl_reconfig_dev_done,
|
|
|
|
},
|
|
|
|
};
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
/* Our generic netlink family */
|
2016-10-24 12:40:05 +00:00
|
|
|
static struct genl_family tcmu_genl_family __ro_after_init = {
|
2016-10-24 12:40:03 +00:00
|
|
|
.module = THIS_MODULE,
|
2014-10-01 23:07:05 +00:00
|
|
|
.hdrsize = 0,
|
|
|
|
.name = "TCM-USER",
|
2017-06-23 06:18:15 +00:00
|
|
|
.version = 2,
|
2014-10-01 23:07:05 +00:00
|
|
|
.maxattr = TCMU_ATTR_MAX,
|
genetlink: make policy common to family
Since maxattr is common, the policy can't really differ sanely,
so make it common as well.
The only user that did in fact manage to make a non-common policy
is taskstats, which has to be really careful about it (since it's
still using a common maxattr!). This is no longer supported, but
we can fake it using pre_doit.
This reduces the size of e.g. nl80211.o (which has lots of commands):
text data bss dec hex filename
398745 14323 2240 415308 6564c net/wireless/nl80211.o (before)
397913 14331 2240 414484 65314 net/wireless/nl80211.o (after)
--------------------------------
-832 +8 0 -824
Which is obviously just 8 bytes for each command, and an added 8
bytes for the new policy pointer. I'm not sure why the ops list is
counted as .text though.
Most of the code transformations were done using the following spatch:
@ops@
identifier OPS;
expression POLICY;
@@
struct genl_ops OPS[] = {
...,
{
- .policy = POLICY,
},
...
};
@@
identifier ops.OPS;
expression ops.POLICY;
identifier fam;
expression M;
@@
struct genl_family fam = {
.ops = OPS,
.maxattr = M,
+ .policy = POLICY,
...
};
This also gets rid of devlink_nl_cmd_region_read_dumpit() accessing
the cb->data as ops, which we want to change in a later genl patch.
Signed-off-by: Johannes Berg <johannes.berg@intel.com>
Signed-off-by: David S. Miller <davem@davemloft.net>
2019-03-21 21:51:02 +00:00
|
|
|
.policy = tcmu_attr_policy,
|
2014-10-01 23:07:05 +00:00
|
|
|
.mcgrps = tcmu_mcgrps,
|
|
|
|
.n_mcgrps = ARRAY_SIZE(tcmu_mcgrps),
|
2016-01-14 01:26:13 +00:00
|
|
|
.netnsok = true,
|
2020-10-02 21:49:54 +00:00
|
|
|
.small_ops = tcmu_genl_ops,
|
|
|
|
.n_small_ops = ARRAY_SIZE(tcmu_genl_ops),
|
2014-10-01 23:07:05 +00:00
|
|
|
};
|
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
#define tcmu_cmd_set_dbi_cur(cmd, index) ((cmd)->dbi_cur = (index))
|
|
|
|
#define tcmu_cmd_reset_dbi_cur(cmd) tcmu_cmd_set_dbi_cur(cmd, 0)
|
|
|
|
#define tcmu_cmd_set_dbi(cmd, index) ((cmd)->dbi[(cmd)->dbi_cur++] = (index))
|
|
|
|
#define tcmu_cmd_get_dbi(cmd) ((cmd)->dbi[(cmd)->dbi_cur++])
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
static void tcmu_cmd_free_data(struct tcmu_cmd *tcmu_cmd, uint32_t len)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
|
|
|
|
uint32_t i;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
for (i = 0; i < len; i++)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
clear_bit(tcmu_cmd->dbi[i], udev->data_bitmap);
|
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
static inline int tcmu_get_empty_block(struct tcmu_dev *udev,
|
|
|
|
struct tcmu_cmd *tcmu_cmd,
|
|
|
|
int prev_dbi, int *iov_cnt)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
{
|
2017-05-02 03:38:06 +00:00
|
|
|
struct page *page;
|
2021-02-24 18:53:35 +00:00
|
|
|
int dbi;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
dbi = find_first_zero_bit(udev->data_bitmap, udev->dbi_thresh);
|
|
|
|
if (dbi == udev->dbi_thresh)
|
2020-09-10 15:50:40 +00:00
|
|
|
return -1;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
|
2021-02-24 18:53:35 +00:00
|
|
|
page = xa_load(&udev->data_blocks, dbi);
|
2017-05-02 03:38:06 +00:00
|
|
|
if (!page) {
|
|
|
|
if (atomic_add_return(1, &global_db_count) >
|
2017-11-28 18:40:41 +00:00
|
|
|
tcmu_global_max_blocks)
|
2017-11-28 18:40:39 +00:00
|
|
|
schedule_delayed_work(&tcmu_unmap_work, 0);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
/* try to get new page from the mm */
|
2019-11-08 08:29:01 +00:00
|
|
|
page = alloc_page(GFP_NOIO);
|
2017-05-02 03:38:06 +00:00
|
|
|
if (!page)
|
2017-07-11 10:06:41 +00:00
|
|
|
goto err_alloc;
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2021-03-05 19:00:09 +00:00
|
|
|
if (xa_store(&udev->data_blocks, dbi, page, GFP_NOIO))
|
2017-07-11 10:06:41 +00:00
|
|
|
goto err_insert;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
}
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
if (dbi > udev->dbi_max)
|
|
|
|
udev->dbi_max = dbi;
|
|
|
|
|
|
|
|
set_bit(dbi, udev->data_bitmap);
|
|
|
|
tcmu_cmd_set_dbi(tcmu_cmd, dbi);
|
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
if (dbi != prev_dbi + 1)
|
|
|
|
*iov_cnt += 1;
|
|
|
|
|
|
|
|
return dbi;
|
2017-07-11 10:06:41 +00:00
|
|
|
err_insert:
|
|
|
|
__free_page(page);
|
|
|
|
err_alloc:
|
|
|
|
atomic_dec(&global_db_count);
|
2020-09-10 15:50:40 +00:00
|
|
|
return -1;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
static int tcmu_get_empty_blocks(struct tcmu_dev *udev,
|
|
|
|
struct tcmu_cmd *tcmu_cmd, int dbi_cnt)
|
2017-05-02 03:38:06 +00:00
|
|
|
{
|
2020-09-10 15:50:40 +00:00
|
|
|
/* start value of dbi + 1 must not be a valid dbi */
|
|
|
|
int dbi = -2;
|
|
|
|
int i, iov_cnt = 0;
|
|
|
|
|
|
|
|
for (i = 0; i < dbi_cnt; i++) {
|
|
|
|
dbi = tcmu_get_empty_block(udev, tcmu_cmd, dbi, &iov_cnt);
|
|
|
|
if (dbi < 0)
|
|
|
|
return -1;
|
2017-05-02 03:38:06 +00:00
|
|
|
}
|
2020-09-10 15:50:40 +00:00
|
|
|
return iov_cnt;
|
2017-05-02 03:38:06 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline struct page *
|
|
|
|
tcmu_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
{
|
2021-02-24 18:53:35 +00:00
|
|
|
return xa_load(&udev->data_blocks, dbi);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tcmu_free_cmd(struct tcmu_cmd *tcmu_cmd)
|
|
|
|
{
|
|
|
|
kfree(tcmu_cmd->dbi);
|
|
|
|
kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
|
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:39 +00:00
|
|
|
static inline void tcmu_cmd_set_block_cnts(struct tcmu_cmd *cmd)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
{
|
2020-09-10 15:50:39 +00:00
|
|
|
int i, len;
|
|
|
|
struct se_cmd *se_cmd = cmd->se_cmd;
|
|
|
|
|
|
|
|
cmd->dbi_cnt = DIV_ROUND_UP(se_cmd->data_length, DATA_BLOCK_SIZE);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
|
|
|
|
if (se_cmd->se_cmd_flags & SCF_BIDI) {
|
|
|
|
BUG_ON(!(se_cmd->t_bidi_data_sg && se_cmd->t_bidi_data_nents));
|
2020-09-10 15:50:39 +00:00
|
|
|
for (i = 0, len = 0; i < se_cmd->t_bidi_data_nents; i++)
|
|
|
|
len += se_cmd->t_bidi_data_sg[i].length;
|
|
|
|
cmd->dbi_bidi_cnt = DIV_ROUND_UP(len, DATA_BLOCK_SIZE);
|
|
|
|
cmd->dbi_cnt += cmd->dbi_bidi_cnt;
|
2020-09-10 15:50:41 +00:00
|
|
|
cmd->data_len_bidi = len;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:41 +00:00
|
|
|
static int new_block_to_iov(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
|
2020-10-19 11:51:18 +00:00
|
|
|
struct iovec **iov, int prev_dbi, int len)
|
2020-09-10 15:50:41 +00:00
|
|
|
{
|
|
|
|
/* Get the next dbi */
|
|
|
|
int dbi = tcmu_cmd_get_dbi(cmd);
|
2020-10-19 11:51:18 +00:00
|
|
|
|
2020-09-10 15:50:41 +00:00
|
|
|
/* Do not add more than DATA_BLOCK_SIZE to iov */
|
2020-10-19 11:51:18 +00:00
|
|
|
if (len > DATA_BLOCK_SIZE)
|
|
|
|
len = DATA_BLOCK_SIZE;
|
2020-09-10 15:50:41 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* The following code will gather and map the blocks to the same iovec
|
|
|
|
* when the blocks are all next to each other.
|
|
|
|
*/
|
|
|
|
if (dbi != prev_dbi + 1) {
|
|
|
|
/* dbi is not next to previous dbi, so start new iov */
|
|
|
|
if (prev_dbi >= 0)
|
|
|
|
(*iov)++;
|
|
|
|
/* write offset relative to mb_addr */
|
|
|
|
(*iov)->iov_base = (void __user *)
|
|
|
|
(udev->data_off + dbi * DATA_BLOCK_SIZE);
|
|
|
|
}
|
|
|
|
(*iov)->iov_len += len;
|
|
|
|
|
|
|
|
return dbi;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_setup_iovs(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
|
|
|
|
struct iovec **iov, int data_length)
|
|
|
|
{
|
|
|
|
/* start value of dbi + 1 must not be a valid dbi */
|
|
|
|
int dbi = -2;
|
|
|
|
|
|
|
|
/* We prepare the IOVs for DMA_FROM_DEVICE transfer direction */
|
2020-10-19 11:51:18 +00:00
|
|
|
for (; data_length > 0; data_length -= DATA_BLOCK_SIZE)
|
|
|
|
dbi = new_block_to_iov(udev, cmd, iov, dbi, data_length);
|
2020-09-10 15:50:41 +00:00
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static struct tcmu_cmd *tcmu_alloc_cmd(struct se_cmd *se_cmd)
|
|
|
|
{
|
|
|
|
struct se_device *se_dev = se_cmd->se_dev;
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
struct tcmu_cmd *tcmu_cmd;
|
|
|
|
|
2019-11-08 08:29:01 +00:00
|
|
|
tcmu_cmd = kmem_cache_zalloc(tcmu_cmd_cache, GFP_NOIO);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (!tcmu_cmd)
|
|
|
|
return NULL;
|
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
INIT_LIST_HEAD(&tcmu_cmd->queue_entry);
|
2014-10-01 23:07:05 +00:00
|
|
|
tcmu_cmd->se_cmd = se_cmd;
|
|
|
|
tcmu_cmd->tcmu_dev = udev;
|
|
|
|
|
2020-09-10 15:50:39 +00:00
|
|
|
tcmu_cmd_set_block_cnts(tcmu_cmd);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
tcmu_cmd->dbi = kcalloc(tcmu_cmd->dbi_cnt, sizeof(uint32_t),
|
2019-11-08 08:29:01 +00:00
|
|
|
GFP_NOIO);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
if (!tcmu_cmd->dbi) {
|
|
|
|
kmem_cache_free(tcmu_cmd_cache, tcmu_cmd);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
return tcmu_cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline void tcmu_flush_dcache_range(void *vaddr, size_t size)
|
|
|
|
{
|
2015-11-25 13:49:27 +00:00
|
|
|
unsigned long offset = offset_in_page(vaddr);
|
2017-11-28 18:40:27 +00:00
|
|
|
void *start = vaddr - offset;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
size = round_up(size+offset, PAGE_SIZE);
|
|
|
|
|
|
|
|
while (size) {
|
2020-06-18 13:16:32 +00:00
|
|
|
flush_dcache_page(vmalloc_to_page(start));
|
2017-11-28 18:40:27 +00:00
|
|
|
start += PAGE_SIZE;
|
2014-10-01 23:07:05 +00:00
|
|
|
size -= PAGE_SIZE;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Some ring helper functions. We don't assume size is a power of 2 so
|
|
|
|
* we can't use circ_buf.h.
|
|
|
|
*/
|
|
|
|
static inline size_t spc_used(size_t head, size_t tail, size_t size)
|
|
|
|
{
|
|
|
|
int diff = head - tail;
|
|
|
|
|
|
|
|
if (diff >= 0)
|
|
|
|
return diff;
|
|
|
|
else
|
|
|
|
return size + diff;
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline size_t spc_free(size_t head, size_t tail, size_t size)
|
|
|
|
{
|
|
|
|
/* Keep 1 byte unused or we can't tell full from empty */
|
|
|
|
return (size - spc_used(head, tail, size) - 1);
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline size_t head_to_end(size_t head, size_t size)
|
|
|
|
{
|
|
|
|
return size - head;
|
|
|
|
}
|
|
|
|
|
|
|
|
#define UPDATE_HEAD(head, used, size) smp_store_release(&head, ((head % size) + used) % size)
|
|
|
|
|
2020-10-19 11:51:18 +00:00
|
|
|
#define TCMU_SG_TO_DATA_AREA 1
|
|
|
|
#define TCMU_DATA_AREA_TO_SG 2
|
|
|
|
|
|
|
|
static inline void tcmu_copy_data(struct tcmu_dev *udev,
|
|
|
|
struct tcmu_cmd *tcmu_cmd, uint32_t direction,
|
|
|
|
struct scatterlist *sg, unsigned int sg_nents,
|
|
|
|
struct iovec **iov, size_t data_len)
|
2016-02-26 22:59:57 +00:00
|
|
|
{
|
2020-09-10 15:50:41 +00:00
|
|
|
/* start value of dbi + 1 must not be a valid dbi */
|
2020-10-19 11:51:18 +00:00
|
|
|
int dbi = -2;
|
|
|
|
size_t block_remaining, cp_len;
|
|
|
|
struct sg_mapping_iter sg_iter;
|
|
|
|
unsigned int sg_flags;
|
|
|
|
struct page *page;
|
|
|
|
void *data_page_start, *data_addr;
|
2017-11-28 18:40:36 +00:00
|
|
|
|
2020-10-19 11:51:18 +00:00
|
|
|
if (direction == TCMU_SG_TO_DATA_AREA)
|
|
|
|
sg_flags = SG_MITER_ATOMIC | SG_MITER_FROM_SG;
|
|
|
|
else
|
|
|
|
sg_flags = SG_MITER_ATOMIC | SG_MITER_TO_SG;
|
|
|
|
sg_miter_start(&sg_iter, sg, sg_nents, sg_flags);
|
|
|
|
|
|
|
|
while (data_len) {
|
|
|
|
if (direction == TCMU_SG_TO_DATA_AREA)
|
|
|
|
dbi = new_block_to_iov(udev, tcmu_cmd, iov, dbi,
|
|
|
|
data_len);
|
|
|
|
else
|
|
|
|
dbi = tcmu_cmd_get_dbi(tcmu_cmd);
|
|
|
|
page = tcmu_get_block_page(udev, dbi);
|
|
|
|
if (direction == TCMU_DATA_AREA_TO_SG)
|
|
|
|
flush_dcache_page(page);
|
|
|
|
data_page_start = kmap_atomic(page);
|
|
|
|
block_remaining = DATA_BLOCK_SIZE;
|
|
|
|
|
|
|
|
while (block_remaining && data_len) {
|
|
|
|
if (!sg_miter_next(&sg_iter)) {
|
|
|
|
/* set length to 0 to abort outer loop */
|
|
|
|
data_len = 0;
|
|
|
|
pr_debug("tcmu_move_data: aborting data copy due to exhausted sg_list\n");
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
cp_len = min3(sg_iter.length, block_remaining, data_len);
|
|
|
|
|
|
|
|
data_addr = data_page_start +
|
|
|
|
DATA_BLOCK_SIZE - block_remaining;
|
|
|
|
if (direction == TCMU_SG_TO_DATA_AREA)
|
|
|
|
memcpy(data_addr, sg_iter.addr, cp_len);
|
|
|
|
else
|
|
|
|
memcpy(sg_iter.addr, data_addr, cp_len);
|
|
|
|
|
|
|
|
data_len -= cp_len;
|
|
|
|
block_remaining -= cp_len;
|
|
|
|
sg_iter.consumed = cp_len;
|
2015-04-23 18:47:00 +00:00
|
|
|
}
|
2020-10-19 11:51:18 +00:00
|
|
|
sg_miter_stop(&sg_iter);
|
2017-11-28 18:40:36 +00:00
|
|
|
|
2020-10-19 11:51:18 +00:00
|
|
|
kunmap_atomic(data_page_start);
|
|
|
|
if (direction == TCMU_SG_TO_DATA_AREA)
|
|
|
|
flush_dcache_page(page);
|
2020-06-18 13:16:31 +00:00
|
|
|
}
|
2016-02-26 22:59:56 +00:00
|
|
|
}
|
|
|
|
|
2020-10-19 11:51:18 +00:00
|
|
|
static void scatter_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,
|
|
|
|
struct iovec **iov)
|
|
|
|
{
|
|
|
|
struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
|
|
|
|
|
|
|
|
tcmu_copy_data(udev, tcmu_cmd, TCMU_SG_TO_DATA_AREA, se_cmd->t_data_sg,
|
|
|
|
se_cmd->t_data_nents, iov, se_cmd->data_length);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void gather_data_area(struct tcmu_dev *udev, struct tcmu_cmd *tcmu_cmd,
|
2018-05-24 16:49:41 +00:00
|
|
|
bool bidi, uint32_t read_len)
|
2015-04-23 18:47:00 +00:00
|
|
|
{
|
2020-10-19 11:51:18 +00:00
|
|
|
struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
|
|
|
|
struct scatterlist *data_sg;
|
2017-03-31 02:35:25 +00:00
|
|
|
unsigned int data_nents;
|
|
|
|
|
|
|
|
if (!bidi) {
|
|
|
|
data_sg = se_cmd->t_data_sg;
|
|
|
|
data_nents = se_cmd->t_data_nents;
|
|
|
|
} else {
|
|
|
|
/*
|
|
|
|
* For bidi case, the first count blocks are for Data-Out
|
|
|
|
* buffer blocks, and before gathering the Data-In buffer
|
2020-09-10 15:50:39 +00:00
|
|
|
* the Data-Out buffer blocks should be skipped.
|
2017-03-31 02:35:25 +00:00
|
|
|
*/
|
2020-10-19 11:51:18 +00:00
|
|
|
tcmu_cmd_set_dbi_cur(tcmu_cmd,
|
|
|
|
tcmu_cmd->dbi_cnt - tcmu_cmd->dbi_bidi_cnt);
|
2017-03-31 02:35:25 +00:00
|
|
|
|
|
|
|
data_sg = se_cmd->t_bidi_data_sg;
|
|
|
|
data_nents = se_cmd->t_bidi_data_nents;
|
|
|
|
}
|
2015-04-23 18:47:00 +00:00
|
|
|
|
2020-10-19 11:51:18 +00:00
|
|
|
tcmu_copy_data(udev, tcmu_cmd, TCMU_DATA_AREA_TO_SG, data_sg,
|
|
|
|
data_nents, NULL, read_len);
|
2015-04-23 18:47:00 +00:00
|
|
|
}
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
static inline size_t spc_bitmap_free(unsigned long *bitmap, uint32_t thresh)
|
2016-02-26 22:59:57 +00:00
|
|
|
{
|
2017-11-28 18:40:35 +00:00
|
|
|
return thresh - bitmap_weight(bitmap, thresh);
|
2016-02-26 22:59:57 +00:00
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
/*
|
2020-09-10 15:50:40 +00:00
|
|
|
* We can't queue a command until we have space available on the cmd ring.
|
2014-10-01 23:07:05 +00:00
|
|
|
*
|
|
|
|
* Called with ring lock held.
|
|
|
|
*/
|
2020-09-10 15:50:40 +00:00
|
|
|
static bool is_ring_space_avail(struct tcmu_dev *udev, size_t cmd_size)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
|
|
|
struct tcmu_mailbox *mb = udev->mb_addr;
|
2016-02-28 02:25:22 +00:00
|
|
|
size_t space, cmd_needed;
|
2014-10-01 23:07:05 +00:00
|
|
|
u32 cmd_head;
|
|
|
|
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
|
|
|
|
cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
|
|
|
|
|
2014-10-02 17:23:15 +00:00
|
|
|
/*
|
|
|
|
* If cmd end-of-ring space is too small then we need space for a NOP plus
|
|
|
|
* original cmd - cmds are internally contiguous.
|
|
|
|
*/
|
|
|
|
if (head_to_end(cmd_head, udev->cmdr_size) >= cmd_size)
|
|
|
|
cmd_needed = cmd_size;
|
|
|
|
else
|
|
|
|
cmd_needed = cmd_size + head_to_end(cmd_head, udev->cmdr_size);
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
space = spc_free(cmd_head, udev->cmdr_last_cleaned, udev->cmdr_size);
|
|
|
|
if (space < cmd_needed) {
|
|
|
|
pr_debug("no cmd space: %u %u %u\n", cmd_head,
|
|
|
|
udev->cmdr_last_cleaned, udev->cmdr_size);
|
|
|
|
return false;
|
|
|
|
}
|
2020-09-10 15:50:40 +00:00
|
|
|
return true;
|
|
|
|
}
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
/*
|
|
|
|
* We have to allocate data buffers before we can queue a command.
|
|
|
|
* Returns -1 on error (not enough space) or number of needed iovs on success
|
|
|
|
*
|
|
|
|
* Called with ring lock held.
|
|
|
|
*/
|
|
|
|
static int tcmu_alloc_data_space(struct tcmu_dev *udev, struct tcmu_cmd *cmd,
|
|
|
|
int *iov_bidi_cnt)
|
|
|
|
{
|
|
|
|
int space, iov_cnt = 0, ret = 0;
|
|
|
|
|
|
|
|
if (!cmd->dbi_cnt)
|
|
|
|
goto wr_iov_cnts;
|
2020-07-26 15:35:09 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
/* try to check and get the data blocks as needed */
|
|
|
|
space = spc_bitmap_free(udev->data_bitmap, udev->dbi_thresh);
|
2020-09-10 15:50:39 +00:00
|
|
|
if (space < cmd->dbi_cnt) {
|
2017-11-28 18:40:41 +00:00
|
|
|
unsigned long blocks_left =
|
|
|
|
(udev->max_blocks - udev->dbi_thresh) + space;
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2020-09-10 15:50:39 +00:00
|
|
|
if (blocks_left < cmd->dbi_cnt) {
|
2020-09-10 15:50:40 +00:00
|
|
|
pr_debug("no data space: only %lu available, but ask for %lu\n",
|
2017-05-02 03:38:06 +00:00
|
|
|
blocks_left * DATA_BLOCK_SIZE,
|
2020-09-10 15:50:39 +00:00
|
|
|
cmd->dbi_cnt * DATA_BLOCK_SIZE);
|
2020-09-10 15:50:40 +00:00
|
|
|
return -1;
|
2017-05-02 03:38:06 +00:00
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:39 +00:00
|
|
|
udev->dbi_thresh += cmd->dbi_cnt;
|
2017-11-28 18:40:41 +00:00
|
|
|
if (udev->dbi_thresh > udev->max_blocks)
|
|
|
|
udev->dbi_thresh = udev->max_blocks;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
iov_cnt = tcmu_get_empty_blocks(udev, cmd,
|
|
|
|
cmd->dbi_cnt - cmd->dbi_bidi_cnt);
|
|
|
|
if (iov_cnt < 0)
|
|
|
|
return -1;
|
|
|
|
|
|
|
|
if (cmd->dbi_bidi_cnt) {
|
|
|
|
ret = tcmu_get_empty_blocks(udev, cmd, cmd->dbi_bidi_cnt);
|
|
|
|
if (ret < 0)
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
wr_iov_cnts:
|
|
|
|
*iov_bidi_cnt = ret;
|
|
|
|
return iov_cnt + ret;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2017-05-02 07:54:29 +00:00
|
|
|
static inline size_t tcmu_cmd_get_base_cmd_size(size_t iov_cnt)
|
|
|
|
{
|
|
|
|
return max(offsetof(struct tcmu_cmd_entry, req.iov[iov_cnt]),
|
|
|
|
sizeof(struct tcmu_cmd_entry));
|
|
|
|
}
|
|
|
|
|
|
|
|
static inline size_t tcmu_cmd_get_cmd_size(struct tcmu_cmd *tcmu_cmd,
|
|
|
|
size_t base_command_size)
|
|
|
|
{
|
|
|
|
struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
|
|
|
|
size_t command_size;
|
|
|
|
|
|
|
|
command_size = base_command_size +
|
|
|
|
round_up(scsi_command_size(se_cmd->t_task_cdb),
|
|
|
|
TCMU_OP_ALIGN_SIZE);
|
|
|
|
|
|
|
|
WARN_ON(command_size & (TCMU_OP_ALIGN_SIZE-1));
|
|
|
|
|
|
|
|
return command_size;
|
|
|
|
}
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
static void tcmu_setup_cmd_timer(struct tcmu_cmd *tcmu_cmd, unsigned int tmo,
|
|
|
|
struct timer_list *timer)
|
2017-10-25 16:47:15 +00:00
|
|
|
{
|
2017-11-28 18:40:40 +00:00
|
|
|
if (!tmo)
|
2020-05-18 16:48:33 +00:00
|
|
|
return;
|
2017-11-28 18:40:40 +00:00
|
|
|
|
2017-10-25 16:47:15 +00:00
|
|
|
tcmu_cmd->deadline = round_jiffies_up(jiffies + msecs_to_jiffies(tmo));
|
2018-11-23 01:15:30 +00:00
|
|
|
if (!timer_pending(timer))
|
|
|
|
mod_timer(timer, tcmu_cmd->deadline);
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
pr_debug("Timeout set up for cmd %p, dev = %s, tmo = %lu\n", tcmu_cmd,
|
|
|
|
tcmu_cmd->tcmu_dev->name, tmo / MSEC_PER_SEC);
|
2017-10-25 16:47:15 +00:00
|
|
|
}
|
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
static int add_to_qfull_queue(struct tcmu_cmd *tcmu_cmd)
|
2017-11-28 18:40:39 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
|
2017-11-28 18:40:40 +00:00
|
|
|
unsigned int tmo;
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
/*
|
|
|
|
* For backwards compat if qfull_time_out is not set use
|
|
|
|
* cmd_time_out and if that's not set use the default time out.
|
|
|
|
*/
|
|
|
|
if (!udev->qfull_time_out)
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
else if (udev->qfull_time_out > 0)
|
|
|
|
tmo = udev->qfull_time_out;
|
|
|
|
else if (udev->cmd_time_out)
|
|
|
|
tmo = udev->cmd_time_out;
|
|
|
|
else
|
|
|
|
tmo = TCMU_TIME_OUT;
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
tcmu_setup_cmd_timer(tcmu_cmd, tmo, &udev->qfull_timer);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
list_add_tail(&tcmu_cmd->queue_entry, &udev->qfull_queue);
|
2020-05-18 16:48:33 +00:00
|
|
|
pr_debug("adding cmd %p on dev %s to ring space wait queue\n",
|
|
|
|
tcmu_cmd, udev->name);
|
2017-11-28 18:40:39 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2020-07-26 15:35:07 +00:00
|
|
|
static uint32_t ring_insert_padding(struct tcmu_dev *udev, size_t cmd_size)
|
|
|
|
{
|
|
|
|
struct tcmu_cmd_entry_hdr *hdr;
|
|
|
|
struct tcmu_mailbox *mb = udev->mb_addr;
|
|
|
|
uint32_t cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
|
|
|
|
|
|
|
|
/* Insert a PAD if end-of-ring space is too small */
|
|
|
|
if (head_to_end(cmd_head, udev->cmdr_size) < cmd_size) {
|
|
|
|
size_t pad_size = head_to_end(cmd_head, udev->cmdr_size);
|
|
|
|
|
|
|
|
hdr = (void *) mb + CMDR_OFF + cmd_head;
|
|
|
|
tcmu_hdr_set_op(&hdr->len_op, TCMU_OP_PAD);
|
|
|
|
tcmu_hdr_set_len(&hdr->len_op, pad_size);
|
|
|
|
hdr->cmd_id = 0; /* not used for PAD */
|
|
|
|
hdr->kflags = 0;
|
|
|
|
hdr->uflags = 0;
|
|
|
|
tcmu_flush_dcache_range(hdr, sizeof(*hdr));
|
|
|
|
|
|
|
|
UPDATE_HEAD(mb->cmd_head, pad_size, udev->cmdr_size);
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
|
|
|
|
cmd_head = mb->cmd_head % udev->cmdr_size; /* UAM */
|
|
|
|
WARN_ON(cmd_head != 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
return cmd_head;
|
|
|
|
}
|
|
|
|
|
2021-02-27 17:00:04 +00:00
|
|
|
static void tcmu_unplug_device(struct se_dev_plug *se_plug)
|
|
|
|
{
|
|
|
|
struct se_device *se_dev = se_plug->se_dev;
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
|
|
|
|
clear_bit(TCM_DEV_BIT_PLUGGED, &udev->flags);
|
|
|
|
uio_event_notify(&udev->uio_info);
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct se_dev_plug *tcmu_plug_device(struct se_device *se_dev)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
|
|
|
|
if (!test_and_set_bit(TCM_DEV_BIT_PLUGGED, &udev->flags))
|
|
|
|
return &udev->se_plug;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:37 +00:00
|
|
|
/**
|
|
|
|
* queue_cmd_ring - queue cmd to ring or internally
|
|
|
|
* @tcmu_cmd: cmd to queue
|
|
|
|
* @scsi_err: TCM error code if failure (-1) returned.
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* -1 we cannot queue internally or to the ring.
|
|
|
|
* 0 success
|
2017-11-28 18:40:39 +00:00
|
|
|
* 1 internally queued to wait for ring memory to free.
|
2017-11-28 18:40:37 +00:00
|
|
|
*/
|
2018-11-27 23:51:55 +00:00
|
|
|
static int queue_cmd_ring(struct tcmu_cmd *tcmu_cmd, sense_reason_t *scsi_err)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = tcmu_cmd->tcmu_dev;
|
|
|
|
struct se_cmd *se_cmd = tcmu_cmd->se_cmd;
|
|
|
|
size_t base_command_size, command_size;
|
2020-07-26 15:35:07 +00:00
|
|
|
struct tcmu_mailbox *mb = udev->mb_addr;
|
2014-10-01 23:07:05 +00:00
|
|
|
struct tcmu_cmd_entry *entry;
|
|
|
|
struct iovec *iov;
|
2021-02-24 18:53:34 +00:00
|
|
|
int iov_cnt, iov_bidi_cnt;
|
|
|
|
uint32_t cmd_id, cmd_head;
|
2014-10-01 23:07:05 +00:00
|
|
|
uint64_t cdb_off;
|
2020-09-10 15:50:39 +00:00
|
|
|
/* size of data buffer needed */
|
|
|
|
size_t data_length = (size_t)tcmu_cmd->dbi_cnt * DATA_BLOCK_SIZE;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:37 +00:00
|
|
|
*scsi_err = TCM_NO_SENSE;
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags)) {
|
|
|
|
*scsi_err = TCM_LUN_BUSY;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:37 +00:00
|
|
|
if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
|
|
|
|
*scsi_err = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
|
|
|
|
return -1;
|
|
|
|
}
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
if (!list_empty(&udev->qfull_queue))
|
|
|
|
goto queue;
|
|
|
|
|
|
|
|
if (data_length > udev->data_size) {
|
|
|
|
pr_warn("TCMU: Request of size %zu is too big for %zu data area\n",
|
|
|
|
data_length, udev->data_size);
|
|
|
|
*scsi_err = TCM_INVALID_CDB_FIELD;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
iov_cnt = tcmu_alloc_data_space(udev, tcmu_cmd, &iov_bidi_cnt);
|
|
|
|
if (iov_cnt < 0)
|
|
|
|
goto free_and_queue;
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
/*
|
|
|
|
* Must be a certain minimum size for response sense info, but
|
|
|
|
* also may be larger if the iov array is large.
|
2017-05-02 07:54:29 +00:00
|
|
|
*/
|
2020-09-10 15:50:40 +00:00
|
|
|
base_command_size = tcmu_cmd_get_base_cmd_size(iov_cnt);
|
2017-05-02 07:54:29 +00:00
|
|
|
command_size = tcmu_cmd_get_cmd_size(tcmu_cmd, base_command_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
if (command_size > (udev->cmdr_size / 2)) {
|
|
|
|
pr_warn("TCMU: Request of size %zu is too big for %u cmd ring\n",
|
|
|
|
command_size, udev->cmdr_size);
|
|
|
|
tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
|
2017-11-28 18:40:37 +00:00
|
|
|
*scsi_err = TCM_INVALID_CDB_FIELD;
|
|
|
|
return -1;
|
2016-08-25 15:55:53 +00:00
|
|
|
}
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
if (!is_ring_space_avail(udev, command_size))
|
2017-11-28 18:40:33 +00:00
|
|
|
/*
|
|
|
|
* Don't leave commands partially setup because the unmap
|
|
|
|
* thread might need the blocks to make forward progress.
|
|
|
|
*/
|
2020-09-10 15:50:40 +00:00
|
|
|
goto free_and_queue;
|
|
|
|
|
2021-02-24 18:53:34 +00:00
|
|
|
if (xa_alloc(&udev->commands, &cmd_id, tcmu_cmd, XA_LIMIT(1, 0xffff),
|
|
|
|
GFP_NOWAIT) < 0) {
|
2020-09-10 15:50:40 +00:00
|
|
|
pr_err("tcmu: Could not allocate cmd id.\n");
|
|
|
|
|
|
|
|
tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cnt);
|
|
|
|
*scsi_err = TCM_OUT_OF_RESOURCES;
|
|
|
|
return -1;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
2020-09-10 15:50:40 +00:00
|
|
|
tcmu_cmd->cmd_id = cmd_id;
|
|
|
|
|
|
|
|
pr_debug("allocated cmd id %u for cmd %p dev %s\n", tcmu_cmd->cmd_id,
|
|
|
|
tcmu_cmd, udev->name);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-07-26 15:35:07 +00:00
|
|
|
cmd_head = ring_insert_padding(udev, command_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
entry = (void *) mb + CMDR_OFF + cmd_head;
|
2017-07-11 09:59:43 +00:00
|
|
|
memset(entry, 0, command_size);
|
2015-04-15 00:30:04 +00:00
|
|
|
tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_CMD);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
/* prepare iov list and copy data to data area if necessary */
|
2017-05-02 03:38:06 +00:00
|
|
|
tcmu_cmd_reset_dbi_cur(tcmu_cmd);
|
2014-10-01 23:07:05 +00:00
|
|
|
iov = &entry->req.iov[0];
|
2020-09-10 15:50:41 +00:00
|
|
|
|
|
|
|
if (se_cmd->data_direction == DMA_TO_DEVICE ||
|
|
|
|
se_cmd->se_cmd_flags & SCF_BIDI)
|
|
|
|
scatter_data_area(udev, tcmu_cmd, &iov);
|
|
|
|
else
|
|
|
|
tcmu_setup_iovs(udev, tcmu_cmd, &iov, se_cmd->data_length);
|
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
entry->req.iov_cnt = iov_cnt - iov_bidi_cnt;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2015-04-23 18:30:09 +00:00
|
|
|
/* Handle BIDI commands */
|
2017-03-27 09:07:40 +00:00
|
|
|
if (se_cmd->se_cmd_flags & SCF_BIDI) {
|
|
|
|
iov++;
|
2020-09-10 15:50:41 +00:00
|
|
|
tcmu_setup_iovs(udev, tcmu_cmd, &iov, tcmu_cmd->data_len_bidi);
|
2020-09-10 15:50:40 +00:00
|
|
|
entry->req.iov_bidi_cnt = iov_bidi_cnt;
|
2017-03-27 09:07:40 +00:00
|
|
|
}
|
2020-05-18 16:48:33 +00:00
|
|
|
|
|
|
|
tcmu_setup_cmd_timer(tcmu_cmd, udev->cmd_time_out, &udev->cmd_timer);
|
|
|
|
|
2017-10-25 16:47:15 +00:00
|
|
|
entry->hdr.cmd_id = tcmu_cmd->cmd_id;
|
|
|
|
|
2017-05-02 07:54:29 +00:00
|
|
|
tcmu_hdr_set_len(&entry->hdr.len_op, command_size);
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
/* All offsets relative to mb_addr, not start of entry! */
|
|
|
|
cdb_off = CMDR_OFF + cmd_head + base_command_size;
|
|
|
|
memcpy((void *) mb + cdb_off, se_cmd->t_task_cdb, scsi_command_size(se_cmd->t_task_cdb));
|
|
|
|
entry->req.cdb_off = cdb_off;
|
2020-05-28 19:31:08 +00:00
|
|
|
tcmu_flush_dcache_range(entry, command_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
UPDATE_HEAD(mb->cmd_head, command_size, udev->cmdr_size);
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
list_add_tail(&tcmu_cmd->queue_entry, &udev->inflight_queue);
|
|
|
|
|
2021-02-27 17:00:04 +00:00
|
|
|
if (!test_bit(TCM_DEV_BIT_PLUGGED, &udev->flags))
|
|
|
|
uio_event_notify(&udev->uio_info);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:37 +00:00
|
|
|
return 0;
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2020-09-10 15:50:40 +00:00
|
|
|
free_and_queue:
|
|
|
|
tcmu_cmd_free_data(tcmu_cmd, tcmu_cmd->dbi_cur);
|
|
|
|
tcmu_cmd_reset_dbi_cur(tcmu_cmd);
|
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
queue:
|
2018-11-23 01:15:30 +00:00
|
|
|
if (add_to_qfull_queue(tcmu_cmd)) {
|
2017-11-28 18:40:39 +00:00
|
|
|
*scsi_err = TCM_OUT_OF_RESOURCES;
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
/**
|
|
|
|
* queue_tmr_ring - queue tmr info to ring or internally
|
|
|
|
* @udev: related tcmu_dev
|
|
|
|
* @tmr: tcmu_tmr containing tmr info to queue
|
|
|
|
*
|
|
|
|
* Returns:
|
|
|
|
* 0 success
|
|
|
|
* 1 internally queued to wait for ring memory to free.
|
|
|
|
*/
|
|
|
|
static int
|
|
|
|
queue_tmr_ring(struct tcmu_dev *udev, struct tcmu_tmr *tmr)
|
|
|
|
{
|
|
|
|
struct tcmu_tmr_entry *entry;
|
|
|
|
int cmd_size;
|
|
|
|
int id_list_sz;
|
|
|
|
struct tcmu_mailbox *mb = udev->mb_addr;
|
|
|
|
uint32_t cmd_head;
|
|
|
|
|
|
|
|
if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags))
|
|
|
|
goto out_free;
|
|
|
|
|
|
|
|
id_list_sz = sizeof(tmr->tmr_cmd_ids[0]) * tmr->tmr_cmd_cnt;
|
|
|
|
cmd_size = round_up(sizeof(*entry) + id_list_sz, TCMU_OP_ALIGN_SIZE);
|
|
|
|
|
|
|
|
if (!list_empty(&udev->tmr_queue) ||
|
2020-09-10 15:50:40 +00:00
|
|
|
!is_ring_space_avail(udev, cmd_size)) {
|
2020-07-26 15:35:09 +00:00
|
|
|
list_add_tail(&tmr->queue_entry, &udev->tmr_queue);
|
|
|
|
pr_debug("adding tmr %p on dev %s to TMR ring space wait queue\n",
|
|
|
|
tmr, udev->name);
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
cmd_head = ring_insert_padding(udev, cmd_size);
|
|
|
|
|
|
|
|
entry = (void *)mb + CMDR_OFF + cmd_head;
|
|
|
|
memset(entry, 0, cmd_size);
|
|
|
|
tcmu_hdr_set_op(&entry->hdr.len_op, TCMU_OP_TMR);
|
|
|
|
tcmu_hdr_set_len(&entry->hdr.len_op, cmd_size);
|
|
|
|
entry->tmr_type = tmr->tmr_type;
|
|
|
|
entry->cmd_cnt = tmr->tmr_cmd_cnt;
|
|
|
|
memcpy(&entry->cmd_ids[0], &tmr->tmr_cmd_ids[0], id_list_sz);
|
|
|
|
tcmu_flush_dcache_range(entry, cmd_size);
|
|
|
|
|
|
|
|
UPDATE_HEAD(mb->cmd_head, cmd_size, udev->cmdr_size);
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
|
|
|
|
uio_event_notify(&udev->uio_info);
|
|
|
|
|
|
|
|
out_free:
|
|
|
|
kfree(tmr);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-10-06 15:07:07 +00:00
|
|
|
static sense_reason_t
|
|
|
|
tcmu_queue_cmd(struct se_cmd *se_cmd)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
2017-11-28 18:40:39 +00:00
|
|
|
struct se_device *se_dev = se_cmd->se_dev;
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
2014-10-01 23:07:05 +00:00
|
|
|
struct tcmu_cmd *tcmu_cmd;
|
2020-07-26 15:35:06 +00:00
|
|
|
sense_reason_t scsi_ret = TCM_CHECK_CONDITION_ABORT_CMD;
|
|
|
|
int ret = -1;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
tcmu_cmd = tcmu_alloc_cmd(se_cmd);
|
|
|
|
if (!tcmu_cmd)
|
2016-10-06 15:07:07 +00:00
|
|
|
return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2020-07-26 15:35:06 +00:00
|
|
|
if (!(se_cmd->transport_state & CMD_T_ABORTED))
|
|
|
|
ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
|
2017-11-28 18:40:39 +00:00
|
|
|
if (ret < 0)
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
tcmu_free_cmd(tcmu_cmd);
|
2021-01-13 02:45:08 +00:00
|
|
|
else
|
|
|
|
se_cmd->priv = tcmu_cmd;
|
2020-07-26 15:35:05 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
2017-11-28 18:40:37 +00:00
|
|
|
return scsi_ret;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
static void tcmu_set_next_deadline(struct list_head *queue,
|
|
|
|
struct timer_list *timer)
|
|
|
|
{
|
|
|
|
struct tcmu_cmd *cmd;
|
|
|
|
|
|
|
|
if (!list_empty(queue)) {
|
|
|
|
cmd = list_first_entry(queue, struct tcmu_cmd, queue_entry);
|
|
|
|
mod_timer(timer, cmd->deadline);
|
|
|
|
} else
|
|
|
|
del_timer(timer);
|
|
|
|
}
|
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
static int
|
|
|
|
tcmu_tmr_type(enum tcm_tmreq_table tmf)
|
|
|
|
{
|
|
|
|
switch (tmf) {
|
|
|
|
case TMR_ABORT_TASK: return TCMU_TMR_ABORT_TASK;
|
|
|
|
case TMR_ABORT_TASK_SET: return TCMU_TMR_ABORT_TASK_SET;
|
|
|
|
case TMR_CLEAR_ACA: return TCMU_TMR_CLEAR_ACA;
|
|
|
|
case TMR_CLEAR_TASK_SET: return TCMU_TMR_CLEAR_TASK_SET;
|
|
|
|
case TMR_LUN_RESET: return TCMU_TMR_LUN_RESET;
|
|
|
|
case TMR_TARGET_WARM_RESET: return TCMU_TMR_TARGET_WARM_RESET;
|
|
|
|
case TMR_TARGET_COLD_RESET: return TCMU_TMR_TARGET_COLD_RESET;
|
|
|
|
case TMR_LUN_RESET_PRO: return TCMU_TMR_LUN_RESET_PRO;
|
|
|
|
default: return TCMU_TMR_UNKNOWN;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void
|
|
|
|
tcmu_tmr_notify(struct se_device *se_dev, enum tcm_tmreq_table tmf,
|
|
|
|
struct list_head *cmd_list)
|
|
|
|
{
|
|
|
|
int i = 0, cmd_cnt = 0;
|
|
|
|
bool unqueued = false;
|
|
|
|
uint16_t *cmd_ids = NULL;
|
|
|
|
struct tcmu_cmd *cmd;
|
|
|
|
struct se_cmd *se_cmd;
|
|
|
|
struct tcmu_tmr *tmr;
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
|
|
|
|
/* First we check for aborted commands in qfull_queue */
|
|
|
|
list_for_each_entry(se_cmd, cmd_list, state_list) {
|
|
|
|
i++;
|
|
|
|
if (!se_cmd->priv)
|
|
|
|
continue;
|
|
|
|
cmd = se_cmd->priv;
|
|
|
|
/* Commands on qfull queue have no id yet */
|
|
|
|
if (cmd->cmd_id) {
|
|
|
|
cmd_cnt++;
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
pr_debug("Removing aborted command %p from queue on dev %s.\n",
|
|
|
|
cmd, udev->name);
|
|
|
|
|
|
|
|
list_del_init(&cmd->queue_entry);
|
|
|
|
tcmu_free_cmd(cmd);
|
2021-01-13 02:45:08 +00:00
|
|
|
se_cmd->priv = NULL;
|
2020-07-26 15:35:09 +00:00
|
|
|
target_complete_cmd(se_cmd, SAM_STAT_TASK_ABORTED);
|
|
|
|
unqueued = true;
|
|
|
|
}
|
|
|
|
if (unqueued)
|
|
|
|
tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
|
|
|
|
|
2020-07-26 15:35:10 +00:00
|
|
|
if (!test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags))
|
|
|
|
goto unlock;
|
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
pr_debug("TMR event %d on dev %s, aborted cmds %d, afflicted cmd_ids %d\n",
|
|
|
|
tcmu_tmr_type(tmf), udev->name, i, cmd_cnt);
|
|
|
|
|
2021-03-05 19:00:09 +00:00
|
|
|
tmr = kmalloc(sizeof(*tmr) + cmd_cnt * sizeof(*cmd_ids), GFP_NOIO);
|
2020-07-26 15:35:09 +00:00
|
|
|
if (!tmr)
|
|
|
|
goto unlock;
|
|
|
|
|
|
|
|
tmr->tmr_type = tcmu_tmr_type(tmf);
|
|
|
|
tmr->tmr_cmd_cnt = cmd_cnt;
|
|
|
|
|
|
|
|
if (cmd_cnt != 0) {
|
|
|
|
cmd_cnt = 0;
|
|
|
|
list_for_each_entry(se_cmd, cmd_list, state_list) {
|
|
|
|
if (!se_cmd->priv)
|
|
|
|
continue;
|
|
|
|
cmd = se_cmd->priv;
|
|
|
|
if (cmd->cmd_id)
|
|
|
|
tmr->tmr_cmd_ids[cmd_cnt++] = cmd->cmd_id;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
queue_tmr_ring(udev, tmr);
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static void tcmu_handle_completion(struct tcmu_cmd *cmd, struct tcmu_cmd_entry *entry)
|
|
|
|
{
|
|
|
|
struct se_cmd *se_cmd = cmd->se_cmd;
|
|
|
|
struct tcmu_dev *udev = cmd->tcmu_dev;
|
2018-05-24 16:49:41 +00:00
|
|
|
bool read_len_valid = false;
|
2019-08-11 18:25:10 +00:00
|
|
|
uint32_t read_len;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
/*
|
|
|
|
* cmd has been completed already from timeout, just reclaim
|
|
|
|
* data area space and free cmd
|
|
|
|
*/
|
2019-08-11 18:25:10 +00:00
|
|
|
if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
|
|
|
|
WARN_ON_ONCE(se_cmd);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
goto out;
|
2019-08-11 18:25:10 +00:00
|
|
|
}
|
2016-02-26 22:59:58 +00:00
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
list_del_init(&cmd->queue_entry);
|
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
tcmu_cmd_reset_dbi_cur(cmd);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2015-04-15 00:30:04 +00:00
|
|
|
if (entry->hdr.uflags & TCMU_UFLAG_UNKNOWN_OP) {
|
|
|
|
pr_warn("TCMU: Userspace set UNKNOWN_OP flag on se_cmd %p\n",
|
|
|
|
cmd->se_cmd);
|
2015-09-03 23:03:44 +00:00
|
|
|
entry->rsp.scsi_status = SAM_STAT_CHECK_CONDITION;
|
2018-05-24 16:49:41 +00:00
|
|
|
goto done;
|
|
|
|
}
|
|
|
|
|
2019-08-11 18:25:10 +00:00
|
|
|
read_len = se_cmd->data_length;
|
2018-05-24 16:49:41 +00:00
|
|
|
if (se_cmd->data_direction == DMA_FROM_DEVICE &&
|
|
|
|
(entry->hdr.uflags & TCMU_UFLAG_READ_LEN) && entry->rsp.read_len) {
|
|
|
|
read_len_valid = true;
|
|
|
|
if (entry->rsp.read_len < read_len)
|
|
|
|
read_len = entry->rsp.read_len;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (entry->rsp.scsi_status == SAM_STAT_CHECK_CONDITION) {
|
2017-05-31 20:52:40 +00:00
|
|
|
transport_copy_sense_to_cmd(se_cmd, entry->rsp.sense_buffer);
|
2018-05-24 16:49:41 +00:00
|
|
|
if (!read_len_valid )
|
|
|
|
goto done;
|
|
|
|
else
|
|
|
|
se_cmd->se_cmd_flags |= SCF_TREAT_READ_AS_NORMAL;
|
|
|
|
}
|
|
|
|
if (se_cmd->se_cmd_flags & SCF_BIDI) {
|
2016-02-26 22:59:57 +00:00
|
|
|
/* Get Data-In buffer before clean up */
|
2018-05-24 16:49:41 +00:00
|
|
|
gather_data_area(udev, cmd, true, read_len);
|
2015-04-23 18:30:09 +00:00
|
|
|
} else if (se_cmd->data_direction == DMA_FROM_DEVICE) {
|
2018-05-24 16:49:41 +00:00
|
|
|
gather_data_area(udev, cmd, false, read_len);
|
2014-10-01 23:07:05 +00:00
|
|
|
} else if (se_cmd->data_direction == DMA_TO_DEVICE) {
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
/* TODO: */
|
2015-04-23 18:30:05 +00:00
|
|
|
} else if (se_cmd->data_direction != DMA_NONE) {
|
|
|
|
pr_warn("TCMU: data direction was %d!\n",
|
|
|
|
se_cmd->data_direction);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2018-05-24 16:49:41 +00:00
|
|
|
done:
|
2021-01-13 02:45:08 +00:00
|
|
|
se_cmd->priv = NULL;
|
2018-05-24 16:49:41 +00:00
|
|
|
if (read_len_valid) {
|
|
|
|
pr_debug("read_len = %d\n", read_len);
|
|
|
|
target_complete_cmd_with_length(cmd->se_cmd,
|
|
|
|
entry->rsp.scsi_status, read_len);
|
|
|
|
} else
|
|
|
|
target_complete_cmd(cmd->se_cmd, entry->rsp.scsi_status);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
out:
|
2017-05-02 03:38:06 +00:00
|
|
|
tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
tcmu_free_cmd(cmd);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
static int tcmu_run_tmr_queue(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct tcmu_tmr *tmr, *tmp;
|
|
|
|
LIST_HEAD(tmrs);
|
|
|
|
|
|
|
|
if (list_empty(&udev->tmr_queue))
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
pr_debug("running %s's tmr queue\n", udev->name);
|
|
|
|
|
|
|
|
list_splice_init(&udev->tmr_queue, &tmrs);
|
|
|
|
|
|
|
|
list_for_each_entry_safe(tmr, tmp, &tmrs, queue_entry) {
|
|
|
|
list_del_init(&tmr->queue_entry);
|
|
|
|
|
|
|
|
pr_debug("removing tmr %p on dev %s from queue\n",
|
|
|
|
tmr, udev->name);
|
|
|
|
|
|
|
|
if (queue_tmr_ring(udev, tmr)) {
|
|
|
|
pr_debug("ran out of space during tmr queue run\n");
|
|
|
|
/*
|
|
|
|
* tmr was requeued, so just put all tmrs back in
|
|
|
|
* the queue
|
|
|
|
*/
|
|
|
|
list_splice_tail(&tmrs, &udev->tmr_queue);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static unsigned int tcmu_handle_completions(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct tcmu_mailbox *mb;
|
2018-11-23 01:15:30 +00:00
|
|
|
struct tcmu_cmd *cmd;
|
2020-07-26 15:35:09 +00:00
|
|
|
bool free_space = false;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
if (test_bit(TCMU_DEV_BIT_BROKEN, &udev->flags)) {
|
|
|
|
pr_err("ring broken, not handling completions\n");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
mb = udev->mb_addr;
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
|
locking/atomics: COCCINELLE/treewide: Convert trivial ACCESS_ONCE() patterns to READ_ONCE()/WRITE_ONCE()
Please do not apply this to mainline directly, instead please re-run the
coccinelle script shown below and apply its output.
For several reasons, it is desirable to use {READ,WRITE}_ONCE() in
preference to ACCESS_ONCE(), and new code is expected to use one of the
former. So far, there's been no reason to change most existing uses of
ACCESS_ONCE(), as these aren't harmful, and changing them results in
churn.
However, for some features, the read/write distinction is critical to
correct operation. To distinguish these cases, separate read/write
accessors must be used. This patch migrates (most) remaining
ACCESS_ONCE() instances to {READ,WRITE}_ONCE(), using the following
coccinelle script:
----
// Convert trivial ACCESS_ONCE() uses to equivalent READ_ONCE() and
// WRITE_ONCE()
// $ make coccicheck COCCI=/home/mark/once.cocci SPFLAGS="--include-headers" MODE=patch
virtual patch
@ depends on patch @
expression E1, E2;
@@
- ACCESS_ONCE(E1) = E2
+ WRITE_ONCE(E1, E2)
@ depends on patch @
expression E;
@@
- ACCESS_ONCE(E)
+ READ_ONCE(E)
----
Signed-off-by: Mark Rutland <mark.rutland@arm.com>
Signed-off-by: Paul E. McKenney <paulmck@linux.vnet.ibm.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Cc: davem@davemloft.net
Cc: linux-arch@vger.kernel.org
Cc: mpe@ellerman.id.au
Cc: shuah@kernel.org
Cc: snitzer@redhat.com
Cc: thor.thayer@linux.intel.com
Cc: tj@kernel.org
Cc: viro@zeniv.linux.org.uk
Cc: will.deacon@arm.com
Link: http://lkml.kernel.org/r/1508792849-3115-19-git-send-email-paulmck@linux.vnet.ibm.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>
2017-10-23 21:07:29 +00:00
|
|
|
while (udev->cmdr_last_cleaned != READ_ONCE(mb->cmd_tail)) {
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
struct tcmu_cmd_entry *entry = (void *) mb + CMDR_OFF + udev->cmdr_last_cleaned;
|
|
|
|
|
2020-06-29 09:37:56 +00:00
|
|
|
/*
|
|
|
|
* Flush max. up to end of cmd ring since current entry might
|
|
|
|
* be a padding that is shorter than sizeof(*entry)
|
|
|
|
*/
|
|
|
|
size_t ring_left = head_to_end(udev->cmdr_last_cleaned,
|
|
|
|
udev->cmdr_size);
|
|
|
|
tcmu_flush_dcache_range(entry, ring_left < sizeof(*entry) ?
|
|
|
|
ring_left : sizeof(*entry));
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
free_space = true;
|
|
|
|
|
|
|
|
if (tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_PAD ||
|
|
|
|
tcmu_hdr_get_op(entry->hdr.len_op) == TCMU_OP_TMR) {
|
2015-04-15 00:30:04 +00:00
|
|
|
UPDATE_HEAD(udev->cmdr_last_cleaned,
|
|
|
|
tcmu_hdr_get_len(entry->hdr.len_op),
|
|
|
|
udev->cmdr_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
continue;
|
|
|
|
}
|
2015-04-15 00:30:04 +00:00
|
|
|
WARN_ON(tcmu_hdr_get_op(entry->hdr.len_op) != TCMU_OP_CMD);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2021-02-24 18:53:34 +00:00
|
|
|
cmd = xa_erase(&udev->commands, entry->hdr.cmd_id);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (!cmd) {
|
2017-12-19 10:03:56 +00:00
|
|
|
pr_err("cmd_id %u not found, ring is broken\n",
|
|
|
|
entry->hdr.cmd_id);
|
2014-10-01 23:07:05 +00:00
|
|
|
set_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
tcmu_handle_completion(cmd, entry);
|
|
|
|
|
2015-04-15 00:30:04 +00:00
|
|
|
UPDATE_HEAD(udev->cmdr_last_cleaned,
|
|
|
|
tcmu_hdr_get_len(entry->hdr.len_op),
|
|
|
|
udev->cmdr_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
2020-07-26 15:35:09 +00:00
|
|
|
if (free_space)
|
|
|
|
free_space = tcmu_run_tmr_queue(udev);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
if (atomic_read(&global_db_count) > tcmu_global_max_blocks &&
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_empty(&udev->commands) && list_empty(&udev->qfull_queue)) {
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
/*
|
|
|
|
* Allocated blocks exceeded global block limit, currently no
|
|
|
|
* more pending or waiting commands so try to reclaim blocks.
|
|
|
|
*/
|
|
|
|
schedule_delayed_work(&tcmu_unmap_work, 0);
|
2017-11-28 18:40:39 +00:00
|
|
|
}
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
if (udev->cmd_time_out)
|
|
|
|
tcmu_set_next_deadline(&udev->inflight_queue, &udev->cmd_timer);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
return free_space;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
static void tcmu_check_expired_ring_cmd(struct tcmu_cmd *cmd)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
2017-11-28 18:40:39 +00:00
|
|
|
struct se_cmd *se_cmd;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
if (!time_after_eq(jiffies, cmd->deadline))
|
2020-05-18 16:48:33 +00:00
|
|
|
return;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
set_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags);
|
|
|
|
list_del_init(&cmd->queue_entry);
|
2018-01-15 20:37:59 +00:00
|
|
|
se_cmd = cmd->se_cmd;
|
2020-07-26 15:35:05 +00:00
|
|
|
se_cmd->priv = NULL;
|
2020-05-18 16:48:33 +00:00
|
|
|
cmd->se_cmd = NULL;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
pr_debug("Timing out inflight cmd %u on dev %s.\n",
|
|
|
|
cmd->cmd_id, cmd->tcmu_dev->name);
|
2017-11-28 18:40:40 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
target_complete_cmd(se_cmd, SAM_STAT_CHECK_CONDITION);
|
|
|
|
}
|
2017-11-28 18:40:40 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
static void tcmu_check_expired_queue_cmd(struct tcmu_cmd *cmd)
|
|
|
|
{
|
|
|
|
struct se_cmd *se_cmd;
|
2017-11-28 18:40:40 +00:00
|
|
|
|
scsi: target: tcmu: Fix and simplify timeout handling
During cmd timeout handling in check_timedout_devices(), due to a race, it
can happen that tcmu_set_next_deadline() does not start a timer as
expected:
1) Either tcmu_check_expired_ring_cmd() checks the inflight_queue or
tcmu_check_expired_queue_cmd() checks the qfull_queue while jiffies has
the value X
2) At the end of the check the queue contains one remaining command with
deadline X (time_after(X, X) is false and thus the command is not
handled as being timed out).
3) After tcmu_check_expired_xxxxx_cmd() a timer interrupt happens and
jiffies is incremented to X+1.
4) Now tcmu_set_next_deadline() is called, but it skips the command, since
time_after(X+1, X) is true. Therefore tcmu_set_next_deadline() finds no
new deadline and stops the timer, which it shouldn't.
Since commands that time out are removed from inflight_queue or
qfull_queue, we don't need the check with time_after() in
tcmu_set_next_deadline() but can use the deadline from the first cmd in
the queue.
Additionally, replace the remaining time_after() calls in
tcmu_check_expired_xxxxx_cmd() with time_after_eq(), because it is not
useful to set the timeout to deadline but then check for jiffies being
greater than deadline.
Simplify the end of tcmu_handle_completions() and change the check for no
more pending commands from
mb->cmd_tail == mb->cmd_head
to
idr_is_empty(&udev->commands)
because the old check doesn't work correctly if paddings or in the future
TMRs are in the ring.
Finally tcmu_set_next_deadline() was shifted in the source as
preparation for later implementation of tmr_notify callback.
Link: https://lore.kernel.org/r/20200726153510.13077-7-bstroesser@ts.fujitsu.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bstroesser@ts.fujitsu.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2020-07-26 15:35:08 +00:00
|
|
|
if (!time_after_eq(jiffies, cmd->deadline))
|
2020-05-18 16:48:33 +00:00
|
|
|
return;
|
|
|
|
|
2020-05-23 10:11:29 +00:00
|
|
|
pr_debug("Timing out queued cmd %p on dev %s.\n",
|
|
|
|
cmd, cmd->tcmu_dev->name);
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
list_del_init(&cmd->queue_entry);
|
|
|
|
se_cmd = cmd->se_cmd;
|
|
|
|
tcmu_free_cmd(cmd);
|
|
|
|
|
2021-01-13 02:45:08 +00:00
|
|
|
se_cmd->priv = NULL;
|
2020-05-18 16:48:33 +00:00
|
|
|
target_complete_cmd(se_cmd, SAM_STAT_TASK_SET_FULL);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
static void tcmu_device_timedout(struct tcmu_dev *udev)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
2017-11-28 18:40:31 +00:00
|
|
|
spin_lock(&timed_out_udevs_lock);
|
|
|
|
if (list_empty(&udev->timedout_entry))
|
|
|
|
list_add_tail(&udev->timedout_entry, &timed_out_udevs);
|
|
|
|
spin_unlock(&timed_out_udevs_lock);
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
schedule_delayed_work(&tcmu_unmap_work, 0);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
static void tcmu_cmd_timedout(struct timer_list *t)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = from_timer(udev, t, cmd_timer);
|
|
|
|
|
|
|
|
pr_debug("%s cmd timeout has expired\n", udev->name);
|
|
|
|
tcmu_device_timedout(udev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_qfull_timedout(struct timer_list *t)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = from_timer(udev, t, qfull_timer);
|
|
|
|
|
|
|
|
pr_debug("%s qfull timeout has expired\n", udev->name);
|
|
|
|
tcmu_device_timedout(udev);
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static int tcmu_attach_hba(struct se_hba *hba, u32 host_id)
|
|
|
|
{
|
|
|
|
struct tcmu_hba *tcmu_hba;
|
|
|
|
|
|
|
|
tcmu_hba = kzalloc(sizeof(struct tcmu_hba), GFP_KERNEL);
|
|
|
|
if (!tcmu_hba)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
tcmu_hba->host_id = host_id;
|
|
|
|
hba->hba_ptr = tcmu_hba;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_detach_hba(struct se_hba *hba)
|
|
|
|
{
|
|
|
|
kfree(hba->hba_ptr);
|
|
|
|
hba->hba_ptr = NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct se_device *tcmu_alloc_device(struct se_hba *hba, const char *name)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev;
|
|
|
|
|
|
|
|
udev = kzalloc(sizeof(struct tcmu_dev), GFP_KERNEL);
|
|
|
|
if (!udev)
|
|
|
|
return NULL;
|
2017-05-17 09:34:37 +00:00
|
|
|
kref_init(&udev->kref);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
udev->name = kstrdup(name, GFP_KERNEL);
|
|
|
|
if (!udev->name) {
|
|
|
|
kfree(udev);
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
|
|
|
udev->hba = hba;
|
2017-03-09 08:42:09 +00:00
|
|
|
udev->cmd_time_out = TCMU_TIME_OUT;
|
2017-11-28 18:40:40 +00:00
|
|
|
udev->qfull_time_out = -1;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
udev->max_blocks = DATA_BLOCK_BITS_DEF;
|
2017-05-02 03:38:06 +00:00
|
|
|
mutex_init(&udev->cmdr_lock);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2018-07-23 19:07:44 +00:00
|
|
|
INIT_LIST_HEAD(&udev->node);
|
2017-11-28 18:40:31 +00:00
|
|
|
INIT_LIST_HEAD(&udev->timedout_entry);
|
2018-11-23 01:15:30 +00:00
|
|
|
INIT_LIST_HEAD(&udev->qfull_queue);
|
2020-07-26 15:35:09 +00:00
|
|
|
INIT_LIST_HEAD(&udev->tmr_queue);
|
2018-11-23 01:15:30 +00:00
|
|
|
INIT_LIST_HEAD(&udev->inflight_queue);
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_init_flags(&udev->commands, XA_FLAGS_ALLOC1);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
timer_setup(&udev->qfull_timer, tcmu_qfull_timedout, 0);
|
|
|
|
timer_setup(&udev->cmd_timer, tcmu_cmd_timedout, 0);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2021-02-24 18:53:35 +00:00
|
|
|
xa_init(&udev->data_blocks);
|
2017-09-14 01:30:05 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
return &udev->se_dev;
|
|
|
|
}
|
|
|
|
|
2021-02-18 17:50:38 +00:00
|
|
|
static void tcmu_dev_call_rcu(struct rcu_head *p)
|
|
|
|
{
|
|
|
|
struct se_device *dev = container_of(p, struct se_device, rcu_head);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
|
|
|
|
kfree(udev->uio_info.name);
|
|
|
|
kfree(udev->name);
|
|
|
|
kfree(udev);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_check_and_free_pending_cmd(struct tcmu_cmd *cmd)
|
|
|
|
{
|
|
|
|
if (test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
|
|
|
|
kmem_cache_free(tcmu_cmd_cache, cmd);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2021-02-24 18:53:35 +00:00
|
|
|
static void tcmu_blocks_release(struct xarray *blocks, unsigned long first,
|
|
|
|
unsigned long last)
|
2021-02-18 17:50:38 +00:00
|
|
|
{
|
2021-02-24 18:53:35 +00:00
|
|
|
XA_STATE(xas, blocks, first);
|
2021-02-18 17:50:38 +00:00
|
|
|
struct page *page;
|
|
|
|
|
2021-02-24 18:53:35 +00:00
|
|
|
xas_lock(&xas);
|
|
|
|
xas_for_each(&xas, page, last) {
|
|
|
|
xas_store(&xas, NULL);
|
|
|
|
__free_page(page);
|
|
|
|
atomic_dec(&global_db_count);
|
2021-02-18 17:50:38 +00:00
|
|
|
}
|
2021-02-24 18:53:35 +00:00
|
|
|
xas_unlock(&xas);
|
2021-02-18 17:50:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_remove_all_queued_tmr(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct tcmu_tmr *tmr, *tmp;
|
|
|
|
|
|
|
|
list_for_each_entry_safe(tmr, tmp, &udev->tmr_queue, queue_entry) {
|
|
|
|
list_del_init(&tmr->queue_entry);
|
|
|
|
kfree(tmr);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_dev_kref_release(struct kref *kref)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = container_of(kref, struct tcmu_dev, kref);
|
|
|
|
struct se_device *dev = &udev->se_dev;
|
|
|
|
struct tcmu_cmd *cmd;
|
|
|
|
bool all_expired = true;
|
2021-02-24 18:53:34 +00:00
|
|
|
unsigned long i;
|
2021-02-18 17:50:38 +00:00
|
|
|
|
|
|
|
vfree(udev->mb_addr);
|
|
|
|
udev->mb_addr = NULL;
|
|
|
|
|
|
|
|
spin_lock_bh(&timed_out_udevs_lock);
|
|
|
|
if (!list_empty(&udev->timedout_entry))
|
|
|
|
list_del(&udev->timedout_entry);
|
|
|
|
spin_unlock_bh(&timed_out_udevs_lock);
|
|
|
|
|
|
|
|
/* Upper layer should drain all requests before calling this */
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_for_each(&udev->commands, i, cmd) {
|
2021-02-18 17:50:38 +00:00
|
|
|
if (tcmu_check_and_free_pending_cmd(cmd) != 0)
|
|
|
|
all_expired = false;
|
|
|
|
}
|
|
|
|
/* There can be left over TMR cmds. Remove them. */
|
|
|
|
tcmu_remove_all_queued_tmr(udev);
|
|
|
|
if (!list_empty(&udev->qfull_queue))
|
|
|
|
all_expired = false;
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_destroy(&udev->commands);
|
2021-02-18 17:50:38 +00:00
|
|
|
WARN_ON(!all_expired);
|
|
|
|
|
|
|
|
tcmu_blocks_release(&udev->data_blocks, 0, udev->dbi_max + 1);
|
|
|
|
bitmap_free(udev->data_bitmap);
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
|
scsi: target: tcmu: Fix memory leak caused by wrong uio usage
When user deletes a tcmu device via configFS, tcmu calls
uio_unregister_device(). During that call uio resets its pointer to struct
uio_info provided by tcmu. That means, after uio_unregister_device() uio
will no longer execute any of the callbacks tcmu had set in uio_info.
Especially, if userspace daemon still holds the corresponding uio device
open or mmap'ed while tcmu calls uio_unregister_device(), uio will not call
tcmu_release() when userspace finally closes and munmaps the uio device.
Since tcmu does refcounting for the tcmu device in tcmu_open() and
tcmu_release(), in the decribed case refcount does not drop to 0 and tcmu
does not free tcmu device's resources. In extreme cases this can cause
memory leaking of up to 1 GB for a single tcmu device.
After uio_unregister_device(), uio will reject every open, read, write,
mmap from userspace with -EOI. But userspace daemon can still access the
mmap'ed command ring and data area. Therefore tcmu should wait until
userspace munmaps the uio device before it frees the resources, as we don't
want to cause SIGSEGV or SIGBUS to user space.
That said, current refcounting during tcmu_open and tcmu_release does not
work correctly, and refcounting better should be done in the open and close
callouts of the vm_operations_struct, which tcmu assigns to each mmap of
the uio device (because it wants its own page fault handler).
This patch fixes the memory leak by removing refcounting from tcmu_open and
tcmu_close, and instead adding new tcmu_vma_open() and tcmu_vma_close()
handlers that only do refcounting.
Link: https://lore.kernel.org/r/20210218175039.7829-3-bostroesser@gmail.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bostroesser@gmail.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2021-02-18 17:50:39 +00:00
|
|
|
pr_debug("dev_kref_release\n");
|
|
|
|
|
2021-02-18 17:50:38 +00:00
|
|
|
call_rcu(&dev->rcu_head, tcmu_dev_call_rcu);
|
|
|
|
}
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
static void run_qfull_queue(struct tcmu_dev *udev, bool fail)
|
2017-11-28 18:40:39 +00:00
|
|
|
{
|
|
|
|
struct tcmu_cmd *tcmu_cmd, *tmp_cmd;
|
|
|
|
LIST_HEAD(cmds);
|
|
|
|
sense_reason_t scsi_ret;
|
|
|
|
int ret;
|
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
if (list_empty(&udev->qfull_queue))
|
2020-05-18 16:48:33 +00:00
|
|
|
return;
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
pr_debug("running %s's cmdr queue forcefail %d\n", udev->name, fail);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
list_splice_init(&udev->qfull_queue, &cmds);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2018-11-23 01:15:30 +00:00
|
|
|
list_for_each_entry_safe(tcmu_cmd, tmp_cmd, &cmds, queue_entry) {
|
|
|
|
list_del_init(&tcmu_cmd->queue_entry);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
pr_debug("removing cmd %p on dev %s from queue\n",
|
|
|
|
tcmu_cmd, udev->name);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
if (fail) {
|
|
|
|
/*
|
|
|
|
* We were not able to even start the command, so
|
|
|
|
* fail with busy to allow a retry in case runner
|
|
|
|
* was only temporarily down. If the device is being
|
|
|
|
* removed then LIO core will do the right thing and
|
|
|
|
* fail the retry.
|
|
|
|
*/
|
2021-01-13 02:45:08 +00:00
|
|
|
tcmu_cmd->se_cmd->priv = NULL;
|
2017-12-19 10:03:58 +00:00
|
|
|
target_complete_cmd(tcmu_cmd->se_cmd, SAM_STAT_BUSY);
|
|
|
|
tcmu_free_cmd(tcmu_cmd);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
ret = queue_cmd_ring(tcmu_cmd, &scsi_ret);
|
|
|
|
if (ret < 0) {
|
2020-05-18 16:48:33 +00:00
|
|
|
pr_debug("cmd %p on dev %s failed with %u\n",
|
|
|
|
tcmu_cmd, udev->name, scsi_ret);
|
2017-11-28 18:40:39 +00:00
|
|
|
/*
|
|
|
|
* Ignore scsi_ret for now. target_complete_cmd
|
|
|
|
* drops it.
|
|
|
|
*/
|
2021-01-13 02:45:08 +00:00
|
|
|
tcmu_cmd->se_cmd->priv = NULL;
|
2017-11-28 18:40:39 +00:00
|
|
|
target_complete_cmd(tcmu_cmd->se_cmd,
|
|
|
|
SAM_STAT_CHECK_CONDITION);
|
|
|
|
tcmu_free_cmd(tcmu_cmd);
|
|
|
|
} else if (ret > 0) {
|
|
|
|
pr_debug("ran out of space during cmdr queue run\n");
|
|
|
|
/*
|
|
|
|
* cmd was requeued, so just put all cmds back in
|
|
|
|
* the queue
|
|
|
|
*/
|
2018-11-23 01:15:30 +00:00
|
|
|
list_splice_tail(&cmds, &udev->qfull_queue);
|
|
|
|
break;
|
2017-11-28 18:40:39 +00:00
|
|
|
}
|
|
|
|
}
|
2018-11-23 01:15:30 +00:00
|
|
|
|
|
|
|
tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
|
2017-11-28 18:40:39 +00:00
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static int tcmu_irqcontrol(struct uio_info *info, s32 irq_on)
|
|
|
|
{
|
2017-11-28 18:40:39 +00:00
|
|
|
struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2020-07-26 15:35:09 +00:00
|
|
|
if (tcmu_handle_completions(udev))
|
|
|
|
run_qfull_queue(udev, false);
|
2017-11-28 18:40:39 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* mmap code from uio.c. Copied here because we want to hook mmap()
|
|
|
|
* and this stuff must come along.
|
|
|
|
*/
|
|
|
|
static int tcmu_find_mem_index(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = vma->vm_private_data;
|
|
|
|
struct uio_info *info = &udev->uio_info;
|
|
|
|
|
|
|
|
if (vma->vm_pgoff < MAX_UIO_MAPS) {
|
|
|
|
if (info->mem[vma->vm_pgoff].size == 0)
|
|
|
|
return -1;
|
|
|
|
return (int)vma->vm_pgoff;
|
|
|
|
}
|
|
|
|
return -1;
|
|
|
|
}
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
static struct page *tcmu_try_get_block_page(struct tcmu_dev *udev, uint32_t dbi)
|
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
page = tcmu_get_block_page(udev, dbi);
|
|
|
|
if (likely(page)) {
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
2017-12-19 10:03:54 +00:00
|
|
|
* Userspace messed up and passed in a address not in the
|
|
|
|
* data iov passed to it.
|
2017-05-02 03:38:06 +00:00
|
|
|
*/
|
2017-12-19 10:03:54 +00:00
|
|
|
pr_err("Invalid addr to data block mapping (dbi %u) on device %s\n",
|
|
|
|
dbi, udev->name);
|
|
|
|
page = NULL;
|
2017-05-02 03:38:06 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
|
|
|
|
return page;
|
|
|
|
}
|
|
|
|
|
scsi: target: tcmu: Fix memory leak caused by wrong uio usage
When user deletes a tcmu device via configFS, tcmu calls
uio_unregister_device(). During that call uio resets its pointer to struct
uio_info provided by tcmu. That means, after uio_unregister_device() uio
will no longer execute any of the callbacks tcmu had set in uio_info.
Especially, if userspace daemon still holds the corresponding uio device
open or mmap'ed while tcmu calls uio_unregister_device(), uio will not call
tcmu_release() when userspace finally closes and munmaps the uio device.
Since tcmu does refcounting for the tcmu device in tcmu_open() and
tcmu_release(), in the decribed case refcount does not drop to 0 and tcmu
does not free tcmu device's resources. In extreme cases this can cause
memory leaking of up to 1 GB for a single tcmu device.
After uio_unregister_device(), uio will reject every open, read, write,
mmap from userspace with -EOI. But userspace daemon can still access the
mmap'ed command ring and data area. Therefore tcmu should wait until
userspace munmaps the uio device before it frees the resources, as we don't
want to cause SIGSEGV or SIGBUS to user space.
That said, current refcounting during tcmu_open and tcmu_release does not
work correctly, and refcounting better should be done in the open and close
callouts of the vm_operations_struct, which tcmu assigns to each mmap of
the uio device (because it wants its own page fault handler).
This patch fixes the memory leak by removing refcounting from tcmu_open and
tcmu_close, and instead adding new tcmu_vma_open() and tcmu_vma_close()
handlers that only do refcounting.
Link: https://lore.kernel.org/r/20210218175039.7829-3-bostroesser@gmail.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bostroesser@gmail.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2021-02-18 17:50:39 +00:00
|
|
|
static void tcmu_vma_open(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = vma->vm_private_data;
|
|
|
|
|
|
|
|
pr_debug("vma_open\n");
|
|
|
|
|
|
|
|
kref_get(&udev->kref);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_vma_close(struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = vma->vm_private_data;
|
|
|
|
|
|
|
|
pr_debug("vma_close\n");
|
|
|
|
|
|
|
|
/* release ref from tcmu_vma_open */
|
|
|
|
kref_put(&udev->kref, tcmu_dev_kref_release);
|
|
|
|
}
|
|
|
|
|
2018-04-14 18:55:57 +00:00
|
|
|
static vm_fault_t tcmu_vma_fault(struct vm_fault *vmf)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
2017-02-24 22:56:41 +00:00
|
|
|
struct tcmu_dev *udev = vmf->vma->vm_private_data;
|
2014-10-01 23:07:05 +00:00
|
|
|
struct uio_info *info = &udev->uio_info;
|
|
|
|
struct page *page;
|
|
|
|
unsigned long offset;
|
|
|
|
void *addr;
|
|
|
|
|
2017-02-24 22:56:41 +00:00
|
|
|
int mi = tcmu_find_mem_index(vmf->vma);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (mi < 0)
|
|
|
|
return VM_FAULT_SIGBUS;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We need to subtract mi because userspace uses offset = N*PAGE_SIZE
|
|
|
|
* to use mem[N].
|
|
|
|
*/
|
|
|
|
offset = (vmf->pgoff - mi) << PAGE_SHIFT;
|
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
if (offset < udev->data_off) {
|
|
|
|
/* For the vmalloc()ed cmd area pages */
|
|
|
|
addr = (void *)(unsigned long)info->mem[mi].addr + offset;
|
2014-10-01 23:07:05 +00:00
|
|
|
page = vmalloc_to_page(addr);
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
} else {
|
|
|
|
uint32_t dbi;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
/* For the dynamically growing data area pages */
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
dbi = (offset - udev->data_off) / DATA_BLOCK_SIZE;
|
2017-05-02 03:38:06 +00:00
|
|
|
page = tcmu_try_get_block_page(udev, dbi);
|
|
|
|
if (!page)
|
2017-12-19 10:03:54 +00:00
|
|
|
return VM_FAULT_SIGBUS;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
get_page(page);
|
|
|
|
vmf->page = page;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct vm_operations_struct tcmu_vm_ops = {
|
scsi: target: tcmu: Fix memory leak caused by wrong uio usage
When user deletes a tcmu device via configFS, tcmu calls
uio_unregister_device(). During that call uio resets its pointer to struct
uio_info provided by tcmu. That means, after uio_unregister_device() uio
will no longer execute any of the callbacks tcmu had set in uio_info.
Especially, if userspace daemon still holds the corresponding uio device
open or mmap'ed while tcmu calls uio_unregister_device(), uio will not call
tcmu_release() when userspace finally closes and munmaps the uio device.
Since tcmu does refcounting for the tcmu device in tcmu_open() and
tcmu_release(), in the decribed case refcount does not drop to 0 and tcmu
does not free tcmu device's resources. In extreme cases this can cause
memory leaking of up to 1 GB for a single tcmu device.
After uio_unregister_device(), uio will reject every open, read, write,
mmap from userspace with -EOI. But userspace daemon can still access the
mmap'ed command ring and data area. Therefore tcmu should wait until
userspace munmaps the uio device before it frees the resources, as we don't
want to cause SIGSEGV or SIGBUS to user space.
That said, current refcounting during tcmu_open and tcmu_release does not
work correctly, and refcounting better should be done in the open and close
callouts of the vm_operations_struct, which tcmu assigns to each mmap of
the uio device (because it wants its own page fault handler).
This patch fixes the memory leak by removing refcounting from tcmu_open and
tcmu_close, and instead adding new tcmu_vma_open() and tcmu_vma_close()
handlers that only do refcounting.
Link: https://lore.kernel.org/r/20210218175039.7829-3-bostroesser@gmail.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bostroesser@gmail.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2021-02-18 17:50:39 +00:00
|
|
|
.open = tcmu_vma_open,
|
|
|
|
.close = tcmu_vma_close,
|
2014-10-01 23:07:05 +00:00
|
|
|
.fault = tcmu_vma_fault,
|
|
|
|
};
|
|
|
|
|
|
|
|
static int tcmu_mmap(struct uio_info *info, struct vm_area_struct *vma)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
|
|
|
|
|
|
|
|
vma->vm_flags |= VM_DONTEXPAND | VM_DONTDUMP;
|
|
|
|
vma->vm_ops = &tcmu_vm_ops;
|
|
|
|
|
|
|
|
vma->vm_private_data = udev;
|
|
|
|
|
|
|
|
/* Ensure the mmap is exactly the right size */
|
2017-11-28 18:40:41 +00:00
|
|
|
if (vma_pages(vma) != (udev->ring_size >> PAGE_SHIFT))
|
2014-10-01 23:07:05 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
scsi: target: tcmu: Fix memory leak caused by wrong uio usage
When user deletes a tcmu device via configFS, tcmu calls
uio_unregister_device(). During that call uio resets its pointer to struct
uio_info provided by tcmu. That means, after uio_unregister_device() uio
will no longer execute any of the callbacks tcmu had set in uio_info.
Especially, if userspace daemon still holds the corresponding uio device
open or mmap'ed while tcmu calls uio_unregister_device(), uio will not call
tcmu_release() when userspace finally closes and munmaps the uio device.
Since tcmu does refcounting for the tcmu device in tcmu_open() and
tcmu_release(), in the decribed case refcount does not drop to 0 and tcmu
does not free tcmu device's resources. In extreme cases this can cause
memory leaking of up to 1 GB for a single tcmu device.
After uio_unregister_device(), uio will reject every open, read, write,
mmap from userspace with -EOI. But userspace daemon can still access the
mmap'ed command ring and data area. Therefore tcmu should wait until
userspace munmaps the uio device before it frees the resources, as we don't
want to cause SIGSEGV or SIGBUS to user space.
That said, current refcounting during tcmu_open and tcmu_release does not
work correctly, and refcounting better should be done in the open and close
callouts of the vm_operations_struct, which tcmu assigns to each mmap of
the uio device (because it wants its own page fault handler).
This patch fixes the memory leak by removing refcounting from tcmu_open and
tcmu_close, and instead adding new tcmu_vma_open() and tcmu_vma_close()
handlers that only do refcounting.
Link: https://lore.kernel.org/r/20210218175039.7829-3-bostroesser@gmail.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bostroesser@gmail.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2021-02-18 17:50:39 +00:00
|
|
|
tcmu_vma_open(vma);
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_open(struct uio_info *info, struct inode *inode)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
|
|
|
|
|
|
|
|
/* O_EXCL not supported for char devs, so fake it? */
|
|
|
|
if (test_and_set_bit(TCMU_DEV_BIT_OPEN, &udev->flags))
|
|
|
|
return -EBUSY;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
udev->inode = inode;
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
pr_debug("open\n");
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_release(struct uio_info *info, struct inode *inode)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = container_of(info, struct tcmu_dev, uio_info);
|
|
|
|
|
|
|
|
clear_bit(TCMU_DEV_BIT_OPEN, &udev->flags);
|
|
|
|
|
|
|
|
pr_debug("close\n");
|
scsi: target: tcmu: Fix memory leak caused by wrong uio usage
When user deletes a tcmu device via configFS, tcmu calls
uio_unregister_device(). During that call uio resets its pointer to struct
uio_info provided by tcmu. That means, after uio_unregister_device() uio
will no longer execute any of the callbacks tcmu had set in uio_info.
Especially, if userspace daemon still holds the corresponding uio device
open or mmap'ed while tcmu calls uio_unregister_device(), uio will not call
tcmu_release() when userspace finally closes and munmaps the uio device.
Since tcmu does refcounting for the tcmu device in tcmu_open() and
tcmu_release(), in the decribed case refcount does not drop to 0 and tcmu
does not free tcmu device's resources. In extreme cases this can cause
memory leaking of up to 1 GB for a single tcmu device.
After uio_unregister_device(), uio will reject every open, read, write,
mmap from userspace with -EOI. But userspace daemon can still access the
mmap'ed command ring and data area. Therefore tcmu should wait until
userspace munmaps the uio device before it frees the resources, as we don't
want to cause SIGSEGV or SIGBUS to user space.
That said, current refcounting during tcmu_open and tcmu_release does not
work correctly, and refcounting better should be done in the open and close
callouts of the vm_operations_struct, which tcmu assigns to each mmap of
the uio device (because it wants its own page fault handler).
This patch fixes the memory leak by removing refcounting from tcmu_open and
tcmu_close, and instead adding new tcmu_vma_open() and tcmu_vma_close()
handlers that only do refcounting.
Link: https://lore.kernel.org/r/20210218175039.7829-3-bostroesser@gmail.com
Reviewed-by: Mike Christie <michael.christie@oracle.com>
Signed-off-by: Bodo Stroesser <bostroesser@gmail.com>
Signed-off-by: Martin K. Petersen <martin.petersen@oracle.com>
2021-02-18 17:50:39 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-06-22 21:40:20 +00:00
|
|
|
static int tcmu_init_genl_cmd_reply(struct tcmu_dev *udev, int cmd)
|
2017-06-23 06:18:15 +00:00
|
|
|
{
|
|
|
|
struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
|
|
|
|
|
|
|
|
if (!tcmu_kern_cmd_reply_supported)
|
2018-06-22 21:40:20 +00:00
|
|
|
return 0;
|
2017-09-13 05:01:22 +00:00
|
|
|
|
|
|
|
if (udev->nl_reply_supported <= 0)
|
2018-06-22 21:40:20 +00:00
|
|
|
return 0;
|
2017-09-13 05:01:22 +00:00
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_lock(&tcmu_nl_cmd_mutex);
|
2017-09-13 05:01:22 +00:00
|
|
|
|
2018-06-22 21:40:22 +00:00
|
|
|
if (tcmu_netlink_blocked) {
|
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
|
|
|
pr_warn("Failing nl cmd %d on %s. Interface is blocked.\n", cmd,
|
|
|
|
udev->name);
|
|
|
|
return -EAGAIN;
|
|
|
|
}
|
2017-06-23 06:18:15 +00:00
|
|
|
|
|
|
|
if (nl_cmd->cmd != TCMU_CMD_UNSPEC) {
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
2018-06-22 21:40:20 +00:00
|
|
|
pr_warn("netlink cmd %d already executing on %s\n",
|
|
|
|
nl_cmd->cmd, udev->name);
|
|
|
|
return -EBUSY;
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
memset(nl_cmd, 0, sizeof(*nl_cmd));
|
|
|
|
nl_cmd->cmd = cmd;
|
2018-06-22 21:40:19 +00:00
|
|
|
nl_cmd->udev = udev;
|
2017-06-23 06:18:15 +00:00
|
|
|
init_completion(&nl_cmd->complete);
|
2018-06-22 21:40:19 +00:00
|
|
|
INIT_LIST_HEAD(&nl_cmd->nl_list);
|
|
|
|
|
|
|
|
list_add_tail(&nl_cmd->nl_list, &tcmu_nl_cmd_list);
|
2017-06-23 06:18:15 +00:00
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
2018-06-22 21:40:20 +00:00
|
|
|
return 0;
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
2019-08-05 00:43:36 +00:00
|
|
|
static void tcmu_destroy_genl_cmd_reply(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
|
|
|
|
|
|
|
|
if (!tcmu_kern_cmd_reply_supported)
|
|
|
|
return;
|
|
|
|
|
|
|
|
if (udev->nl_reply_supported <= 0)
|
|
|
|
return;
|
|
|
|
|
|
|
|
mutex_lock(&tcmu_nl_cmd_mutex);
|
|
|
|
|
|
|
|
list_del(&nl_cmd->nl_list);
|
|
|
|
memset(nl_cmd, 0, sizeof(*nl_cmd));
|
|
|
|
|
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
|
|
|
}
|
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
static int tcmu_wait_genl_cmd_reply(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct tcmu_nl_cmd *nl_cmd = &udev->curr_nl_cmd;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (!tcmu_kern_cmd_reply_supported)
|
|
|
|
return 0;
|
|
|
|
|
2017-09-13 05:01:22 +00:00
|
|
|
if (udev->nl_reply_supported <= 0)
|
|
|
|
return 0;
|
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
pr_debug("sleeping for nl reply\n");
|
|
|
|
wait_for_completion(&nl_cmd->complete);
|
|
|
|
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_lock(&tcmu_nl_cmd_mutex);
|
2017-06-23 06:18:15 +00:00
|
|
|
nl_cmd->cmd = TCMU_CMD_UNSPEC;
|
|
|
|
ret = nl_cmd->status;
|
2018-06-22 21:40:19 +00:00
|
|
|
mutex_unlock(&tcmu_nl_cmd_mutex);
|
2017-06-23 06:18:15 +00:00
|
|
|
|
2018-01-16 15:34:32 +00:00
|
|
|
return ret;
|
2017-06-23 06:18:15 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:39 +00:00
|
|
|
static int tcmu_netlink_event_init(struct tcmu_dev *udev,
|
|
|
|
enum tcmu_genl_cmd cmd,
|
|
|
|
struct sk_buff **buf, void **hdr)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
|
|
|
struct sk_buff *skb;
|
|
|
|
void *msg_header;
|
2014-10-02 06:01:15 +00:00
|
|
|
int ret = -ENOMEM;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
skb = genlmsg_new(NLMSG_GOODSIZE, GFP_KERNEL);
|
|
|
|
if (!skb)
|
2014-10-02 06:01:15 +00:00
|
|
|
return ret;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
msg_header = genlmsg_put(skb, 0, 0, &tcmu_genl_family, 0, cmd);
|
2014-10-02 06:01:15 +00:00
|
|
|
if (!msg_header)
|
|
|
|
goto free_skb;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
ret = nla_put_string(skb, TCMU_ATTR_DEVICE, udev->uio_info.name);
|
2014-10-02 06:01:15 +00:00
|
|
|
if (ret < 0)
|
|
|
|
goto free_skb;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-06-23 06:18:15 +00:00
|
|
|
ret = nla_put_u32(skb, TCMU_ATTR_MINOR, udev->uio_info.uio_dev->minor);
|
|
|
|
if (ret < 0)
|
|
|
|
goto free_skb;
|
|
|
|
|
|
|
|
ret = nla_put_u32(skb, TCMU_ATTR_DEVICE_ID, udev->se_dev.dev_index);
|
2014-10-02 06:01:15 +00:00
|
|
|
if (ret < 0)
|
|
|
|
goto free_skb;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2018-05-02 03:13:39 +00:00
|
|
|
*buf = skb;
|
|
|
|
*hdr = msg_header;
|
|
|
|
return ret;
|
2017-06-12 06:34:28 +00:00
|
|
|
|
2018-05-02 03:13:39 +00:00
|
|
|
free_skb:
|
|
|
|
nlmsg_free(skb);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_netlink_event_send(struct tcmu_dev *udev,
|
|
|
|
enum tcmu_genl_cmd cmd,
|
2018-06-22 21:40:21 +00:00
|
|
|
struct sk_buff *skb, void *msg_header)
|
2018-05-02 03:13:39 +00:00
|
|
|
{
|
2018-06-22 21:40:21 +00:00
|
|
|
int ret;
|
2017-06-06 14:28:52 +00:00
|
|
|
|
2015-01-16 21:09:00 +00:00
|
|
|
genlmsg_end(skb, msg_header);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2018-06-22 21:40:20 +00:00
|
|
|
ret = tcmu_init_genl_cmd_reply(udev, cmd);
|
|
|
|
if (ret) {
|
|
|
|
nlmsg_free(skb);
|
|
|
|
return ret;
|
|
|
|
}
|
2017-06-23 06:18:15 +00:00
|
|
|
|
2016-01-14 01:26:13 +00:00
|
|
|
ret = genlmsg_multicast_allns(&tcmu_genl_family, skb, 0,
|
2018-05-02 03:13:39 +00:00
|
|
|
TCMU_MCGRP_CONFIG, GFP_KERNEL);
|
2019-02-28 19:28:24 +00:00
|
|
|
|
|
|
|
/* Wait during an add as the listener may not be up yet */
|
|
|
|
if (ret == 0 ||
|
|
|
|
(ret == -ESRCH && cmd == TCMU_CMD_ADDED_DEVICE))
|
|
|
|
return tcmu_wait_genl_cmd_reply(udev);
|
2019-08-05 00:43:36 +00:00
|
|
|
else
|
|
|
|
tcmu_destroy_genl_cmd_reply(udev);
|
2019-02-28 19:28:24 +00:00
|
|
|
|
2014-10-02 06:01:15 +00:00
|
|
|
return ret;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:40 +00:00
|
|
|
static int tcmu_send_dev_add_event(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
void *msg_header = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_netlink_event_init(udev, TCMU_CMD_ADDED_DEVICE, &skb,
|
|
|
|
&msg_header);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
2018-06-22 21:40:21 +00:00
|
|
|
return tcmu_netlink_event_send(udev, TCMU_CMD_ADDED_DEVICE, skb,
|
|
|
|
msg_header);
|
2018-05-02 03:13:40 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:41 +00:00
|
|
|
static int tcmu_send_dev_remove_event(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
void *msg_header = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_netlink_event_init(udev, TCMU_CMD_REMOVED_DEVICE,
|
|
|
|
&skb, &msg_header);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
return tcmu_netlink_event_send(udev, TCMU_CMD_REMOVED_DEVICE,
|
2018-06-22 21:40:21 +00:00
|
|
|
skb, msg_header);
|
2018-05-02 03:13:41 +00:00
|
|
|
}
|
|
|
|
|
2017-07-07 19:20:00 +00:00
|
|
|
static int tcmu_update_uio_info(struct tcmu_dev *udev)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
|
|
|
struct tcmu_hba *hba = udev->hba->hba_ptr;
|
|
|
|
struct uio_info *info;
|
|
|
|
char *str;
|
|
|
|
|
|
|
|
info = &udev->uio_info;
|
|
|
|
|
|
|
|
if (udev->dev_config[0])
|
2019-06-16 07:02:20 +00:00
|
|
|
str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s/%s", hba->host_id,
|
|
|
|
udev->name, udev->dev_config);
|
|
|
|
else
|
|
|
|
str = kasprintf(GFP_KERNEL, "tcm-user/%u/%s", hba->host_id,
|
|
|
|
udev->name);
|
|
|
|
if (!str)
|
|
|
|
return -ENOMEM;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-07-14 13:11:04 +00:00
|
|
|
/* If the old string exists, free it */
|
|
|
|
kfree(info->name);
|
2014-10-01 23:07:05 +00:00
|
|
|
info->name = str;
|
|
|
|
|
2017-07-07 19:20:00 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int tcmu_configure_device(struct se_device *dev)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
struct uio_info *info;
|
|
|
|
struct tcmu_mailbox *mb;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_update_uio_info(udev);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
info = &udev->uio_info;
|
|
|
|
|
2018-07-23 19:07:49 +00:00
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2019-03-04 09:44:34 +00:00
|
|
|
udev->data_bitmap = bitmap_zalloc(udev->max_blocks, GFP_KERNEL);
|
2018-07-23 19:07:49 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
2018-01-11 11:12:25 +00:00
|
|
|
if (!udev->data_bitmap) {
|
|
|
|
ret = -ENOMEM;
|
2017-11-28 18:40:41 +00:00
|
|
|
goto err_bitmap_alloc;
|
2018-01-11 11:12:25 +00:00
|
|
|
}
|
2017-11-28 18:40:41 +00:00
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
udev->mb_addr = vzalloc(CMDR_SIZE);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (!udev->mb_addr) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto err_vzalloc;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* mailbox fits in first part of CMDR space */
|
|
|
|
udev->cmdr_size = CMDR_SIZE - CMDR_OFF;
|
|
|
|
udev->data_off = CMDR_SIZE;
|
2017-11-28 18:40:41 +00:00
|
|
|
udev->data_size = udev->max_blocks * DATA_BLOCK_SIZE;
|
2017-05-02 03:38:06 +00:00
|
|
|
udev->dbi_thresh = 0; /* Default in Idle state */
|
2014-10-01 23:07:05 +00:00
|
|
|
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
/* Initialise the mailbox of the ring buffer */
|
2014-10-01 23:07:05 +00:00
|
|
|
mb = udev->mb_addr;
|
2015-04-15 00:30:04 +00:00
|
|
|
mb->version = TCMU_MAILBOX_VERSION;
|
2020-07-26 15:35:09 +00:00
|
|
|
mb->flags = TCMU_MAILBOX_FLAG_CAP_OOOC |
|
|
|
|
TCMU_MAILBOX_FLAG_CAP_READ_LEN |
|
|
|
|
TCMU_MAILBOX_FLAG_CAP_TMR;
|
2014-10-01 23:07:05 +00:00
|
|
|
mb->cmdr_off = CMDR_OFF;
|
|
|
|
mb->cmdr_size = udev->cmdr_size;
|
|
|
|
|
|
|
|
WARN_ON(!PAGE_ALIGNED(udev->data_off));
|
|
|
|
WARN_ON(udev->data_size % PAGE_SIZE);
|
2016-02-26 22:59:57 +00:00
|
|
|
WARN_ON(udev->data_size % DATA_BLOCK_SIZE);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2015-09-03 23:39:56 +00:00
|
|
|
info->version = __stringify(TCMU_MAILBOX_VERSION);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
info->mem[0].name = "tcm-user command & data buffer";
|
2016-02-01 16:29:45 +00:00
|
|
|
info->mem[0].addr = (phys_addr_t)(uintptr_t)udev->mb_addr;
|
2017-11-28 18:40:41 +00:00
|
|
|
info->mem[0].size = udev->ring_size = udev->data_size + CMDR_SIZE;
|
tcmu: Add dynamic growing data area feature support
Currently for the TCMU, the ring buffer size is fixed to 64K cmd
area + 1M data area, and this will be bottlenecks for high iops.
The struct tcmu_cmd_entry {} size is fixed about 112 bytes with
iovec[N] & N <= 4, and the size of struct iovec is about 16 bytes.
If N == 0, the ratio will be sizeof(cmd entry) : sizeof(datas) ==
112Bytes : (N * 4096)Bytes = 28 : 0, no data area is need.
If 0 < N <=4, the ratio will be sizeof(cmd entry) : sizeof(datas)
== 112Bytes : (N * 4096)Bytes = 28 : (N * 1024), so the max will
be 28 : 1024.
If N > 4, the sizeof(cmd entry) will be [(N - 4) *16 + 112] bytes,
and its corresponding data size will be [N * 4096], so the ratio
of sizeof(cmd entry) : sizeof(datas) == [(N - 4) * 16 + 112)Bytes
: (N * 4096)Bytes == 4/1024 - 12/(N * 1024), so the max is about
4 : 1024.
When N is bigger, the ratio will be smaller.
As the initial patch, we will set the cmd area size to 2M, and
the cmd area size to 32M. The TCMU will dynamically grows the data
area from 0 to max 32M size as needed.
The cmd area memory will be allocated through vmalloc(), and the
data area's blocks will be allocated individually later when needed.
The allocated data area block memory will be managed via radix tree.
For now the bitmap still be the most efficient way to search and
manage the block index, this could be update later.
Signed-off-by: Xiubo Li <lixiubo@cmss.chinamobile.com>
Signed-off-by: Jianfei Hu <hujianfei@cmss.chinamobile.com>
Acked-by: Mike Christie <mchristi@redhat.com>
Signed-off-by: Nicholas Bellinger <nab@linux-iscsi.org>
2017-05-02 03:38:05 +00:00
|
|
|
info->mem[0].memtype = UIO_MEM_NONE;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
info->irqcontrol = tcmu_irqcontrol;
|
|
|
|
info->irq = UIO_IRQ_CUSTOM;
|
|
|
|
|
|
|
|
info->mmap = tcmu_mmap;
|
|
|
|
info->open = tcmu_open;
|
|
|
|
info->release = tcmu_release;
|
|
|
|
|
|
|
|
ret = uio_register_device(tcmu_root_device, info);
|
|
|
|
if (ret)
|
|
|
|
goto err_register;
|
|
|
|
|
2015-12-28 19:57:39 +00:00
|
|
|
/* User can set hw_block_size before enable the device */
|
|
|
|
if (dev->dev_attrib.hw_block_size == 0)
|
|
|
|
dev->dev_attrib.hw_block_size = 512;
|
2014-10-01 23:07:05 +00:00
|
|
|
/* Other attributes can be configured in userspace */
|
2017-03-02 05:14:39 +00:00
|
|
|
if (!dev->dev_attrib.hw_max_sectors)
|
|
|
|
dev->dev_attrib.hw_max_sectors = 128;
|
2017-06-06 14:28:48 +00:00
|
|
|
if (!dev->dev_attrib.emulate_write_cache)
|
|
|
|
dev->dev_attrib.emulate_write_cache = 0;
|
2014-10-01 23:07:05 +00:00
|
|
|
dev->dev_attrib.hw_queue_depth = 128;
|
|
|
|
|
2017-09-13 05:01:22 +00:00
|
|
|
/* If user didn't explicitly disable netlink reply support, use
|
|
|
|
* module scope setting.
|
|
|
|
*/
|
|
|
|
if (udev->nl_reply_supported >= 0)
|
|
|
|
udev->nl_reply_supported = tcmu_kern_cmd_reply_supported;
|
|
|
|
|
2017-05-17 09:34:37 +00:00
|
|
|
/*
|
|
|
|
* Get a ref incase userspace does a close on the uio device before
|
|
|
|
* LIO has initiated tcmu_free_device.
|
|
|
|
*/
|
|
|
|
kref_get(&udev->kref);
|
|
|
|
|
2018-05-02 03:13:40 +00:00
|
|
|
ret = tcmu_send_dev_add_event(udev);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (ret)
|
|
|
|
goto err_netlink;
|
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
mutex_lock(&root_udev_mutex);
|
|
|
|
list_add(&udev->node, &root_udev);
|
|
|
|
mutex_unlock(&root_udev_mutex);
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
err_netlink:
|
2017-05-17 09:34:37 +00:00
|
|
|
kref_put(&udev->kref, tcmu_dev_kref_release);
|
2014-10-01 23:07:05 +00:00
|
|
|
uio_unregister_device(&udev->uio_info);
|
|
|
|
err_register:
|
|
|
|
vfree(udev->mb_addr);
|
2017-09-14 01:30:05 +00:00
|
|
|
udev->mb_addr = NULL;
|
2014-10-01 23:07:05 +00:00
|
|
|
err_vzalloc:
|
2019-03-04 09:44:34 +00:00
|
|
|
bitmap_free(udev->data_bitmap);
|
2017-11-28 18:40:41 +00:00
|
|
|
udev->data_bitmap = NULL;
|
|
|
|
err_bitmap_alloc:
|
2014-10-01 23:07:05 +00:00
|
|
|
kfree(info->name);
|
2017-05-17 09:34:37 +00:00
|
|
|
info->name = NULL;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_free_device(struct se_device *dev)
|
2017-06-23 06:18:12 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
|
|
|
|
/* release ref from init */
|
|
|
|
kref_put(&udev->kref, tcmu_dev_kref_release);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_destroy_device(struct se_device *dev)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
del_timer_sync(&udev->cmd_timer);
|
|
|
|
del_timer_sync(&udev->qfull_timer);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-05-02 03:38:06 +00:00
|
|
|
mutex_lock(&root_udev_mutex);
|
|
|
|
list_del(&udev->node);
|
|
|
|
mutex_unlock(&root_udev_mutex);
|
|
|
|
|
2018-05-02 03:13:41 +00:00
|
|
|
tcmu_send_dev_remove_event(udev);
|
2014-10-01 23:07:05 +00:00
|
|
|
|
2017-06-23 06:18:19 +00:00
|
|
|
uio_unregister_device(&udev->uio_info);
|
2017-06-23 06:18:20 +00:00
|
|
|
|
|
|
|
/* release ref from configure */
|
|
|
|
kref_put(&udev->kref, tcmu_dev_kref_release);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
static void tcmu_unblock_dev(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
clear_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags);
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_block_dev(struct tcmu_dev *udev)
|
|
|
|
{
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
|
|
|
|
if (test_and_set_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
|
|
|
|
goto unlock;
|
|
|
|
|
|
|
|
/* complete IO that has executed successfully */
|
|
|
|
tcmu_handle_completions(udev);
|
|
|
|
/* fail IO waiting to be queued */
|
2018-11-23 01:15:30 +00:00
|
|
|
run_qfull_queue(udev, true);
|
2017-12-19 10:03:58 +00:00
|
|
|
|
|
|
|
unlock:
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void tcmu_reset_ring(struct tcmu_dev *udev, u8 err_level)
|
|
|
|
{
|
|
|
|
struct tcmu_mailbox *mb;
|
|
|
|
struct tcmu_cmd *cmd;
|
2021-02-24 18:53:34 +00:00
|
|
|
unsigned long i;
|
2017-12-19 10:03:58 +00:00
|
|
|
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_for_each(&udev->commands, i, cmd) {
|
2017-12-19 10:03:58 +00:00
|
|
|
pr_debug("removing cmd %u on dev %s from ring (is expired %d)\n",
|
|
|
|
cmd->cmd_id, udev->name,
|
|
|
|
test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags));
|
|
|
|
|
2021-02-24 18:53:34 +00:00
|
|
|
xa_erase(&udev->commands, i);
|
2017-12-19 10:03:58 +00:00
|
|
|
if (!test_bit(TCMU_CMD_BIT_EXPIRED, &cmd->flags)) {
|
2019-08-11 18:25:10 +00:00
|
|
|
WARN_ON(!cmd->se_cmd);
|
2018-11-23 01:15:30 +00:00
|
|
|
list_del_init(&cmd->queue_entry);
|
2021-01-13 02:45:08 +00:00
|
|
|
cmd->se_cmd->priv = NULL;
|
2017-12-19 10:03:58 +00:00
|
|
|
if (err_level == 1) {
|
|
|
|
/*
|
|
|
|
* Userspace was not able to start the
|
|
|
|
* command or it is retryable.
|
|
|
|
*/
|
|
|
|
target_complete_cmd(cmd->se_cmd, SAM_STAT_BUSY);
|
|
|
|
} else {
|
|
|
|
/* hard failure */
|
|
|
|
target_complete_cmd(cmd->se_cmd,
|
|
|
|
SAM_STAT_CHECK_CONDITION);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
tcmu_cmd_free_data(cmd, cmd->dbi_cnt);
|
|
|
|
tcmu_free_cmd(cmd);
|
|
|
|
}
|
|
|
|
|
|
|
|
mb = udev->mb_addr;
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
|
|
|
pr_debug("mb last %u head %u tail %u\n", udev->cmdr_last_cleaned,
|
|
|
|
mb->cmd_tail, mb->cmd_head);
|
|
|
|
|
|
|
|
udev->cmdr_last_cleaned = 0;
|
|
|
|
mb->cmd_tail = 0;
|
|
|
|
mb->cmd_head = 0;
|
|
|
|
tcmu_flush_dcache_range(mb, sizeof(*mb));
|
2020-04-09 10:10:26 +00:00
|
|
|
clear_bit(TCMU_DEV_BIT_BROKEN, &udev->flags);
|
2017-12-19 10:03:58 +00:00
|
|
|
|
|
|
|
del_timer(&udev->cmd_timer);
|
|
|
|
|
2020-07-26 15:35:09 +00:00
|
|
|
/*
|
|
|
|
* ring is empty and qfull queue never contains aborted commands.
|
|
|
|
* So TMRs in tmr queue do not contain relevant cmd_ids.
|
|
|
|
* After a ring reset userspace should do a fresh start, so
|
|
|
|
* even LUN RESET message is no longer relevant.
|
|
|
|
* Therefore remove all TMRs from qfull queue
|
|
|
|
*/
|
|
|
|
tcmu_remove_all_queued_tmr(udev);
|
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
run_qfull_queue(udev, false);
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
enum {
|
2017-03-02 05:14:39 +00:00
|
|
|
Opt_dev_config, Opt_dev_size, Opt_hw_block_size, Opt_hw_max_sectors,
|
2017-11-28 18:40:41 +00:00
|
|
|
Opt_nl_reply_supported, Opt_max_data_area_mb, Opt_err,
|
2014-10-01 23:07:05 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
static match_table_t tokens = {
|
|
|
|
{Opt_dev_config, "dev_config=%s"},
|
2018-07-23 19:07:51 +00:00
|
|
|
{Opt_dev_size, "dev_size=%s"},
|
2018-07-23 19:07:50 +00:00
|
|
|
{Opt_hw_block_size, "hw_block_size=%d"},
|
|
|
|
{Opt_hw_max_sectors, "hw_max_sectors=%d"},
|
2017-09-13 05:01:22 +00:00
|
|
|
{Opt_nl_reply_supported, "nl_reply_supported=%d"},
|
2018-07-23 19:07:49 +00:00
|
|
|
{Opt_max_data_area_mb, "max_data_area_mb=%d"},
|
2014-10-01 23:07:05 +00:00
|
|
|
{Opt_err, NULL}
|
|
|
|
};
|
|
|
|
|
2017-03-02 05:14:39 +00:00
|
|
|
static int tcmu_set_dev_attrib(substring_t *arg, u32 *dev_attrib)
|
|
|
|
{
|
2018-07-23 19:07:50 +00:00
|
|
|
int val, ret;
|
2017-03-02 05:14:39 +00:00
|
|
|
|
2018-07-23 19:07:50 +00:00
|
|
|
ret = match_int(arg, &val);
|
2017-03-02 05:14:39 +00:00
|
|
|
if (ret < 0) {
|
2018-07-23 19:07:50 +00:00
|
|
|
pr_err("match_int() failed for dev attrib. Error %d.\n",
|
|
|
|
ret);
|
2017-03-02 05:14:39 +00:00
|
|
|
return ret;
|
|
|
|
}
|
2018-07-23 19:07:50 +00:00
|
|
|
|
|
|
|
if (val <= 0) {
|
|
|
|
pr_err("Invalid dev attrib value %d. Must be greater than zero.\n",
|
|
|
|
val);
|
2017-03-02 05:14:39 +00:00
|
|
|
return -EINVAL;
|
|
|
|
}
|
2018-07-23 19:07:50 +00:00
|
|
|
*dev_attrib = val;
|
2017-03-02 05:14:39 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2018-07-23 19:07:49 +00:00
|
|
|
static int tcmu_set_max_blocks_param(struct tcmu_dev *udev, substring_t *arg)
|
|
|
|
{
|
|
|
|
int val, ret;
|
|
|
|
|
|
|
|
ret = match_int(arg, &val);
|
|
|
|
if (ret < 0) {
|
|
|
|
pr_err("match_int() failed for max_data_area_mb=. Error %d.\n",
|
|
|
|
ret);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (val <= 0) {
|
|
|
|
pr_err("Invalid max_data_area %d.\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
|
|
|
if (udev->data_bitmap) {
|
|
|
|
pr_err("Cannot set max_data_area_mb after it has been enabled.\n");
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto unlock;
|
|
|
|
}
|
|
|
|
|
|
|
|
udev->max_blocks = TCMU_MBS_TO_BLOCKS(val);
|
|
|
|
if (udev->max_blocks > tcmu_global_max_blocks) {
|
|
|
|
pr_err("%d is too large. Adjusting max_data_area_mb to global limit of %u\n",
|
|
|
|
val, TCMU_BLOCKS_TO_MBS(tcmu_global_max_blocks));
|
|
|
|
udev->max_blocks = tcmu_global_max_blocks;
|
|
|
|
}
|
|
|
|
|
|
|
|
unlock:
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static ssize_t tcmu_set_configfs_dev_params(struct se_device *dev,
|
|
|
|
const char *page, ssize_t count)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
2018-07-23 19:07:51 +00:00
|
|
|
char *orig, *ptr, *opts;
|
2014-10-01 23:07:05 +00:00
|
|
|
substring_t args[MAX_OPT_ARGS];
|
2018-07-23 19:07:49 +00:00
|
|
|
int ret = 0, token;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
opts = kstrdup(page, GFP_KERNEL);
|
|
|
|
if (!opts)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
orig = opts;
|
|
|
|
|
|
|
|
while ((ptr = strsep(&opts, ",\n")) != NULL) {
|
|
|
|
if (!*ptr)
|
|
|
|
continue;
|
|
|
|
|
|
|
|
token = match_token(ptr, tokens, args);
|
|
|
|
switch (token) {
|
|
|
|
case Opt_dev_config:
|
|
|
|
if (match_strlcpy(udev->dev_config, &args[0],
|
|
|
|
TCMU_CONFIG_LEN) == 0) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
pr_debug("TCMU: Referencing Path: %s\n", udev->dev_config);
|
|
|
|
break;
|
|
|
|
case Opt_dev_size:
|
2018-07-23 19:07:51 +00:00
|
|
|
ret = match_u64(&args[0], &udev->dev_size);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (ret < 0)
|
2018-07-23 19:07:51 +00:00
|
|
|
pr_err("match_u64() failed for dev_size=. Error %d.\n",
|
|
|
|
ret);
|
2014-10-01 23:07:05 +00:00
|
|
|
break;
|
2015-05-19 21:44:39 +00:00
|
|
|
case Opt_hw_block_size:
|
2017-03-02 05:14:39 +00:00
|
|
|
ret = tcmu_set_dev_attrib(&args[0],
|
|
|
|
&(dev->dev_attrib.hw_block_size));
|
|
|
|
break;
|
|
|
|
case Opt_hw_max_sectors:
|
|
|
|
ret = tcmu_set_dev_attrib(&args[0],
|
|
|
|
&(dev->dev_attrib.hw_max_sectors));
|
2015-05-19 21:44:39 +00:00
|
|
|
break;
|
2017-09-13 05:01:22 +00:00
|
|
|
case Opt_nl_reply_supported:
|
2018-07-23 19:07:50 +00:00
|
|
|
ret = match_int(&args[0], &udev->nl_reply_supported);
|
2017-09-13 05:01:22 +00:00
|
|
|
if (ret < 0)
|
2018-07-23 19:07:50 +00:00
|
|
|
pr_err("match_int() failed for nl_reply_supported=. Error %d.\n",
|
|
|
|
ret);
|
2017-09-13 05:01:22 +00:00
|
|
|
break;
|
2017-11-28 18:40:41 +00:00
|
|
|
case Opt_max_data_area_mb:
|
2018-07-23 19:07:49 +00:00
|
|
|
ret = tcmu_set_max_blocks_param(udev, &args[0]);
|
2017-11-28 18:40:41 +00:00
|
|
|
break;
|
2014-10-01 23:07:05 +00:00
|
|
|
default:
|
|
|
|
break;
|
|
|
|
}
|
2017-03-02 05:14:40 +00:00
|
|
|
|
|
|
|
if (ret)
|
|
|
|
break;
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
kfree(orig);
|
|
|
|
return (!ret) ? count : ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_show_configfs_dev_params(struct se_device *dev, char *b)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
ssize_t bl = 0;
|
|
|
|
|
|
|
|
bl = sprintf(b + bl, "Config: %s ",
|
|
|
|
udev->dev_config[0] ? udev->dev_config : "NULL");
|
2018-07-23 19:07:51 +00:00
|
|
|
bl += sprintf(b + bl, "Size: %llu ", udev->dev_size);
|
2017-11-28 18:40:41 +00:00
|
|
|
bl += sprintf(b + bl, "MaxDataAreaMB: %u\n",
|
|
|
|
TCMU_BLOCKS_TO_MBS(udev->max_blocks));
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
return bl;
|
|
|
|
}
|
|
|
|
|
|
|
|
static sector_t tcmu_get_blocks(struct se_device *dev)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(dev);
|
|
|
|
|
|
|
|
return div_u64(udev->dev_size - dev->dev_attrib.block_size,
|
|
|
|
dev->dev_attrib.block_size);
|
|
|
|
}
|
|
|
|
|
|
|
|
static sense_reason_t
|
2015-05-19 21:44:39 +00:00
|
|
|
tcmu_parse_cdb(struct se_cmd *cmd)
|
2014-10-01 23:07:05 +00:00
|
|
|
{
|
2016-10-06 15:07:07 +00:00
|
|
|
return passthrough_parse_cdb(cmd, tcmu_queue_cmd);
|
2014-10-01 23:07:05 +00:00
|
|
|
}
|
|
|
|
|
2017-03-18 22:04:13 +00:00
|
|
|
static ssize_t tcmu_cmd_time_out_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
2017-09-15 05:44:55 +00:00
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
2017-03-18 22:04:13 +00:00
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%lu\n", udev->cmd_time_out / MSEC_PER_SEC);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_cmd_time_out_store(struct config_item *item, const char *page,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = container_of(da->da_dev,
|
|
|
|
struct tcmu_dev, se_dev);
|
|
|
|
u32 val;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
if (da->da_dev->export_count) {
|
|
|
|
pr_err("Unable to set tcmu cmd_time_out while exports exist\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = kstrtou32(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
udev->cmd_time_out = val * MSEC_PER_SEC;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, cmd_time_out);
|
|
|
|
|
2017-11-28 18:40:40 +00:00
|
|
|
static ssize_t tcmu_qfull_time_out_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%ld\n", udev->qfull_time_out <= 0 ?
|
|
|
|
udev->qfull_time_out :
|
|
|
|
udev->qfull_time_out / MSEC_PER_SEC);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_qfull_time_out_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
s32 val;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = kstrtos32(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (val >= 0) {
|
|
|
|
udev->qfull_time_out = val * MSEC_PER_SEC;
|
2018-05-10 13:42:18 +00:00
|
|
|
} else if (val == -1) {
|
|
|
|
udev->qfull_time_out = val;
|
2017-11-28 18:40:40 +00:00
|
|
|
} else {
|
|
|
|
printk(KERN_ERR "Invalid qfull timeout value %d\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, qfull_time_out);
|
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
static ssize_t tcmu_max_data_area_mb_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%u\n",
|
|
|
|
TCMU_BLOCKS_TO_MBS(udev->max_blocks));
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR_RO(tcmu_, max_data_area_mb);
|
|
|
|
|
2017-06-12 06:34:28 +00:00
|
|
|
static ssize_t tcmu_dev_config_show(struct config_item *item, char *page)
|
2017-06-06 14:28:51 +00:00
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%s\n", udev->dev_config);
|
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:42 +00:00
|
|
|
static int tcmu_send_dev_config_event(struct tcmu_dev *udev,
|
|
|
|
const char *reconfig_data)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
void *msg_header = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
|
|
|
|
&skb, &msg_header);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
ret = nla_put_string(skb, TCMU_ATTR_DEV_CFG, reconfig_data);
|
|
|
|
if (ret < 0) {
|
|
|
|
nlmsg_free(skb);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
|
2018-06-22 21:40:21 +00:00
|
|
|
skb, msg_header);
|
2018-05-02 03:13:42 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
2017-06-12 06:34:28 +00:00
|
|
|
static ssize_t tcmu_dev_config_store(struct config_item *item, const char *page,
|
|
|
|
size_t count)
|
2017-06-06 14:28:51 +00:00
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
2017-06-12 06:34:28 +00:00
|
|
|
int ret, len;
|
2017-06-06 14:28:51 +00:00
|
|
|
|
2017-06-12 06:34:28 +00:00
|
|
|
len = strlen(page);
|
|
|
|
if (!len || len > TCMU_CONFIG_LEN - 1)
|
2017-06-06 14:28:51 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
/* Check if device has been configured before */
|
2018-07-23 19:07:46 +00:00
|
|
|
if (target_dev_configured(&udev->se_dev)) {
|
2018-05-02 03:13:42 +00:00
|
|
|
ret = tcmu_send_dev_config_event(udev, page);
|
2017-06-06 14:28:51 +00:00
|
|
|
if (ret) {
|
|
|
|
pr_err("Unable to reconfigure device\n");
|
|
|
|
return ret;
|
|
|
|
}
|
2017-07-07 19:20:00 +00:00
|
|
|
strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
|
|
|
|
|
|
|
|
ret = tcmu_update_uio_info(udev);
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
return count;
|
2017-06-06 14:28:51 +00:00
|
|
|
}
|
2017-06-12 06:34:28 +00:00
|
|
|
strlcpy(udev->dev_config, page, TCMU_CONFIG_LEN);
|
2017-06-06 14:28:51 +00:00
|
|
|
|
|
|
|
return count;
|
|
|
|
}
|
2017-06-12 06:34:28 +00:00
|
|
|
CONFIGFS_ATTR(tcmu_, dev_config);
|
2017-06-06 14:28:51 +00:00
|
|
|
|
2017-06-06 14:28:50 +00:00
|
|
|
static ssize_t tcmu_dev_size_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
2018-07-23 19:07:51 +00:00
|
|
|
return snprintf(page, PAGE_SIZE, "%llu\n", udev->dev_size);
|
2017-06-06 14:28:50 +00:00
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:43 +00:00
|
|
|
static int tcmu_send_dev_size_event(struct tcmu_dev *udev, u64 size)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
void *msg_header = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
|
|
|
|
&skb, &msg_header);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
ret = nla_put_u64_64bit(skb, TCMU_ATTR_DEV_SIZE,
|
|
|
|
size, TCMU_ATTR_PAD);
|
|
|
|
if (ret < 0) {
|
|
|
|
nlmsg_free(skb);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
|
2018-06-22 21:40:21 +00:00
|
|
|
skb, msg_header);
|
2018-05-02 03:13:43 +00:00
|
|
|
}
|
|
|
|
|
2017-06-06 14:28:50 +00:00
|
|
|
static ssize_t tcmu_dev_size_store(struct config_item *item, const char *page,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
2017-06-12 06:34:28 +00:00
|
|
|
u64 val;
|
2017-06-06 14:28:50 +00:00
|
|
|
int ret;
|
|
|
|
|
2017-06-12 06:34:28 +00:00
|
|
|
ret = kstrtou64(page, 0, &val);
|
2017-06-06 14:28:50 +00:00
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
/* Check if device has been configured before */
|
2018-07-23 19:07:46 +00:00
|
|
|
if (target_dev_configured(&udev->se_dev)) {
|
2018-05-02 03:13:43 +00:00
|
|
|
ret = tcmu_send_dev_size_event(udev, val);
|
2017-06-06 14:28:50 +00:00
|
|
|
if (ret) {
|
|
|
|
pr_err("Unable to reconfigure device\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
2017-06-12 06:34:28 +00:00
|
|
|
udev->dev_size = val;
|
2017-06-06 14:28:50 +00:00
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, dev_size);
|
|
|
|
|
2017-09-13 05:01:22 +00:00
|
|
|
static ssize_t tcmu_nl_reply_supported_show(struct config_item *item,
|
|
|
|
char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%d\n", udev->nl_reply_supported);
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_nl_reply_supported_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
s8 val;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = kstrtos8(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
udev->nl_reply_supported = val;
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, nl_reply_supported);
|
|
|
|
|
2017-06-06 14:28:48 +00:00
|
|
|
static ssize_t tcmu_emulate_write_cache_show(struct config_item *item,
|
|
|
|
char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%i\n", da->emulate_write_cache);
|
|
|
|
}
|
|
|
|
|
2018-05-02 03:13:44 +00:00
|
|
|
static int tcmu_send_emulate_write_cache(struct tcmu_dev *udev, u8 val)
|
|
|
|
{
|
|
|
|
struct sk_buff *skb = NULL;
|
|
|
|
void *msg_header = NULL;
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
ret = tcmu_netlink_event_init(udev, TCMU_CMD_RECONFIG_DEVICE,
|
|
|
|
&skb, &msg_header);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
ret = nla_put_u8(skb, TCMU_ATTR_WRITECACHE, val);
|
|
|
|
if (ret < 0) {
|
|
|
|
nlmsg_free(skb);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
return tcmu_netlink_event_send(udev, TCMU_CMD_RECONFIG_DEVICE,
|
2018-06-22 21:40:21 +00:00
|
|
|
skb, msg_header);
|
2018-05-02 03:13:44 +00:00
|
|
|
}
|
|
|
|
|
2017-06-06 14:28:48 +00:00
|
|
|
static ssize_t tcmu_emulate_write_cache_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
2017-06-06 14:28:49 +00:00
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
2017-06-12 06:34:28 +00:00
|
|
|
u8 val;
|
2017-06-06 14:28:48 +00:00
|
|
|
int ret;
|
|
|
|
|
2017-06-12 06:34:28 +00:00
|
|
|
ret = kstrtou8(page, 0, &val);
|
2017-06-06 14:28:48 +00:00
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
2017-06-06 14:28:49 +00:00
|
|
|
/* Check if device has been configured before */
|
2018-07-23 19:07:46 +00:00
|
|
|
if (target_dev_configured(&udev->se_dev)) {
|
2018-05-02 03:13:44 +00:00
|
|
|
ret = tcmu_send_emulate_write_cache(udev, val);
|
2017-06-06 14:28:49 +00:00
|
|
|
if (ret) {
|
|
|
|
pr_err("Unable to reconfigure device\n");
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
}
|
2017-06-12 06:34:28 +00:00
|
|
|
|
|
|
|
da->emulate_write_cache = val;
|
2017-06-06 14:28:48 +00:00
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, emulate_write_cache);
|
|
|
|
|
2020-07-26 15:35:10 +00:00
|
|
|
static ssize_t tcmu_tmr_notification_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
|
|
|
|
return snprintf(page, PAGE_SIZE, "%i\n",
|
|
|
|
test_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags));
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_tmr_notification_store(struct config_item *item,
|
|
|
|
const char *page, size_t count)
|
|
|
|
{
|
|
|
|
struct se_dev_attrib *da = container_of(to_config_group(item),
|
|
|
|
struct se_dev_attrib, da_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(da->da_dev);
|
|
|
|
u8 val;
|
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = kstrtou8(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
if (val > 1)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (val)
|
|
|
|
set_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
|
|
|
|
else
|
|
|
|
clear_bit(TCMU_DEV_BIT_TMR_NOTIFY, &udev->flags);
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, tmr_notification);
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
static ssize_t tcmu_block_dev_show(struct config_item *item, char *page)
|
|
|
|
{
|
|
|
|
struct se_device *se_dev = container_of(to_config_group(item),
|
|
|
|
struct se_device,
|
|
|
|
dev_action_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
|
|
|
|
if (test_bit(TCMU_DEV_BIT_BLOCKED, &udev->flags))
|
|
|
|
return snprintf(page, PAGE_SIZE, "%s\n", "blocked");
|
|
|
|
else
|
|
|
|
return snprintf(page, PAGE_SIZE, "%s\n", "unblocked");
|
|
|
|
}
|
|
|
|
|
|
|
|
static ssize_t tcmu_block_dev_store(struct config_item *item, const char *page,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
struct se_device *se_dev = container_of(to_config_group(item),
|
|
|
|
struct se_device,
|
|
|
|
dev_action_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
u8 val;
|
|
|
|
int ret;
|
|
|
|
|
2018-07-23 19:07:47 +00:00
|
|
|
if (!target_dev_configured(&udev->se_dev)) {
|
|
|
|
pr_err("Device is not configured.\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
ret = kstrtou8(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (val > 1) {
|
|
|
|
pr_err("Invalid block value %d\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!val)
|
|
|
|
tcmu_unblock_dev(udev);
|
|
|
|
else
|
|
|
|
tcmu_block_dev(udev);
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR(tcmu_, block_dev);
|
|
|
|
|
|
|
|
static ssize_t tcmu_reset_ring_store(struct config_item *item, const char *page,
|
|
|
|
size_t count)
|
|
|
|
{
|
|
|
|
struct se_device *se_dev = container_of(to_config_group(item),
|
|
|
|
struct se_device,
|
|
|
|
dev_action_group);
|
|
|
|
struct tcmu_dev *udev = TCMU_DEV(se_dev);
|
|
|
|
u8 val;
|
|
|
|
int ret;
|
|
|
|
|
2018-07-23 19:07:47 +00:00
|
|
|
if (!target_dev_configured(&udev->se_dev)) {
|
|
|
|
pr_err("Device is not configured.\n");
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
ret = kstrtou8(page, 0, &val);
|
|
|
|
if (ret < 0)
|
|
|
|
return ret;
|
|
|
|
|
|
|
|
if (val != 1 && val != 2) {
|
|
|
|
pr_err("Invalid reset ring value %d\n", val);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
tcmu_reset_ring(udev, val);
|
|
|
|
return count;
|
|
|
|
}
|
|
|
|
CONFIGFS_ATTR_WO(tcmu_, reset_ring);
|
|
|
|
|
2017-06-13 13:29:09 +00:00
|
|
|
static struct configfs_attribute *tcmu_attrib_attrs[] = {
|
2017-06-06 14:28:50 +00:00
|
|
|
&tcmu_attr_cmd_time_out,
|
2017-11-28 18:40:40 +00:00
|
|
|
&tcmu_attr_qfull_time_out,
|
2017-11-28 18:40:41 +00:00
|
|
|
&tcmu_attr_max_data_area_mb,
|
2017-06-12 06:34:28 +00:00
|
|
|
&tcmu_attr_dev_config,
|
2017-06-06 14:28:50 +00:00
|
|
|
&tcmu_attr_dev_size,
|
|
|
|
&tcmu_attr_emulate_write_cache,
|
2020-07-26 15:35:10 +00:00
|
|
|
&tcmu_attr_tmr_notification,
|
2017-09-13 05:01:22 +00:00
|
|
|
&tcmu_attr_nl_reply_supported,
|
2017-06-06 14:28:50 +00:00
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
2017-03-18 22:04:13 +00:00
|
|
|
static struct configfs_attribute **tcmu_attrs;
|
|
|
|
|
2017-12-19 10:03:58 +00:00
|
|
|
static struct configfs_attribute *tcmu_action_attrs[] = {
|
|
|
|
&tcmu_attr_block_dev,
|
|
|
|
&tcmu_attr_reset_ring,
|
|
|
|
NULL,
|
|
|
|
};
|
|
|
|
|
2017-03-18 22:04:13 +00:00
|
|
|
static struct target_backend_ops tcmu_ops = {
|
2014-10-01 23:07:05 +00:00
|
|
|
.name = "user",
|
|
|
|
.owner = THIS_MODULE,
|
2020-04-27 15:08:22 +00:00
|
|
|
.transport_flags_default = TRANSPORT_FLAG_PASSTHROUGH,
|
2020-04-27 15:08:23 +00:00
|
|
|
.transport_flags_changeable = TRANSPORT_FLAG_PASSTHROUGH_PGR |
|
|
|
|
TRANSPORT_FLAG_PASSTHROUGH_ALUA,
|
2014-10-01 23:07:05 +00:00
|
|
|
.attach_hba = tcmu_attach_hba,
|
|
|
|
.detach_hba = tcmu_detach_hba,
|
|
|
|
.alloc_device = tcmu_alloc_device,
|
|
|
|
.configure_device = tcmu_configure_device,
|
2017-06-23 06:18:12 +00:00
|
|
|
.destroy_device = tcmu_destroy_device,
|
2014-10-01 23:07:05 +00:00
|
|
|
.free_device = tcmu_free_device,
|
2021-02-27 17:00:04 +00:00
|
|
|
.unplug_device = tcmu_unplug_device,
|
|
|
|
.plug_device = tcmu_plug_device,
|
2014-10-01 23:07:05 +00:00
|
|
|
.parse_cdb = tcmu_parse_cdb,
|
2020-07-26 15:35:09 +00:00
|
|
|
.tmr_notify = tcmu_tmr_notify,
|
2014-10-01 23:07:05 +00:00
|
|
|
.set_configfs_dev_params = tcmu_set_configfs_dev_params,
|
|
|
|
.show_configfs_dev_params = tcmu_show_configfs_dev_params,
|
|
|
|
.get_device_type = sbc_get_device_type,
|
|
|
|
.get_blocks = tcmu_get_blocks,
|
2017-12-19 10:03:58 +00:00
|
|
|
.tb_dev_action_attrs = tcmu_action_attrs,
|
2014-10-01 23:07:05 +00:00
|
|
|
};
|
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
static void find_free_blocks(void)
|
2017-05-02 03:38:06 +00:00
|
|
|
{
|
|
|
|
struct tcmu_dev *udev;
|
|
|
|
loff_t off;
|
2017-11-28 18:40:39 +00:00
|
|
|
u32 start, end, block, total_freed = 0;
|
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
if (atomic_read(&global_db_count) <= tcmu_global_max_blocks)
|
2017-11-28 18:40:39 +00:00
|
|
|
return;
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
mutex_lock(&root_udev_mutex);
|
|
|
|
list_for_each_entry(udev, &root_udev, node) {
|
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2018-07-23 19:07:48 +00:00
|
|
|
if (!target_dev_configured(&udev->se_dev)) {
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
/* Try to complete the finished commands first */
|
2020-07-26 15:35:09 +00:00
|
|
|
if (tcmu_handle_completions(udev))
|
|
|
|
run_qfull_queue(udev, false);
|
2017-05-03 04:57:05 +00:00
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
/* Skip the udevs in idle */
|
|
|
|
if (!udev->dbi_thresh) {
|
2017-11-28 18:40:29 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
continue;
|
|
|
|
}
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
end = udev->dbi_max + 1;
|
|
|
|
block = find_last_bit(udev->data_bitmap, end);
|
|
|
|
if (block == udev->dbi_max) {
|
|
|
|
/*
|
2017-11-28 18:40:39 +00:00
|
|
|
* The last bit is dbi_max, so it is not possible
|
|
|
|
* reclaim any blocks.
|
2017-11-28 18:40:29 +00:00
|
|
|
*/
|
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
|
|
|
continue;
|
|
|
|
} else if (block == end) {
|
|
|
|
/* The current udev will goto idle state */
|
|
|
|
udev->dbi_thresh = start = 0;
|
|
|
|
udev->dbi_max = 0;
|
|
|
|
} else {
|
|
|
|
udev->dbi_thresh = start = block + 1;
|
|
|
|
udev->dbi_max = block;
|
|
|
|
}
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
/* Here will truncate the data area from off */
|
|
|
|
off = udev->data_off + start * DATA_BLOCK_SIZE;
|
|
|
|
unmap_mapping_range(udev->inode->i_mapping, off, 0, 1);
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:29 +00:00
|
|
|
/* Release the block pages */
|
2021-02-24 18:53:35 +00:00
|
|
|
tcmu_blocks_release(&udev->data_blocks, start, end - 1);
|
2017-11-28 18:40:29 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
2017-05-02 03:38:06 +00:00
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
total_freed += end - start;
|
|
|
|
pr_debug("Freed %u blocks (total %u) from %s.\n", end - start,
|
|
|
|
total_freed, udev->name);
|
2017-11-28 18:40:29 +00:00
|
|
|
}
|
|
|
|
mutex_unlock(&root_udev_mutex);
|
2017-11-28 18:40:39 +00:00
|
|
|
|
2017-11-28 18:40:41 +00:00
|
|
|
if (atomic_read(&global_db_count) > tcmu_global_max_blocks)
|
2017-11-28 18:40:39 +00:00
|
|
|
schedule_delayed_work(&tcmu_unmap_work, msecs_to_jiffies(5000));
|
2017-11-28 18:40:29 +00:00
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:31 +00:00
|
|
|
static void check_timedout_devices(void)
|
|
|
|
{
|
|
|
|
struct tcmu_dev *udev, *tmp_dev;
|
2020-05-18 16:48:33 +00:00
|
|
|
struct tcmu_cmd *cmd, *tmp_cmd;
|
2017-11-28 18:40:31 +00:00
|
|
|
LIST_HEAD(devs);
|
|
|
|
|
|
|
|
spin_lock_bh(&timed_out_udevs_lock);
|
|
|
|
list_splice_init(&timed_out_udevs, &devs);
|
|
|
|
|
|
|
|
list_for_each_entry_safe(udev, tmp_dev, &devs, timedout_entry) {
|
|
|
|
list_del_init(&udev->timedout_entry);
|
|
|
|
spin_unlock_bh(&timed_out_udevs_lock);
|
|
|
|
|
2017-11-28 18:40:32 +00:00
|
|
|
mutex_lock(&udev->cmdr_lock);
|
2018-11-23 01:15:30 +00:00
|
|
|
|
2020-05-18 16:48:33 +00:00
|
|
|
/*
|
|
|
|
* If cmd_time_out is disabled but qfull is set deadline
|
|
|
|
* will only reflect the qfull timeout. Ignore it.
|
|
|
|
*/
|
|
|
|
if (udev->cmd_time_out) {
|
|
|
|
list_for_each_entry_safe(cmd, tmp_cmd,
|
|
|
|
&udev->inflight_queue,
|
|
|
|
queue_entry) {
|
|
|
|
tcmu_check_expired_ring_cmd(cmd);
|
|
|
|
}
|
|
|
|
tcmu_set_next_deadline(&udev->inflight_queue,
|
|
|
|
&udev->cmd_timer);
|
|
|
|
}
|
|
|
|
list_for_each_entry_safe(cmd, tmp_cmd, &udev->qfull_queue,
|
|
|
|
queue_entry) {
|
|
|
|
tcmu_check_expired_queue_cmd(cmd);
|
|
|
|
}
|
2018-11-23 01:15:30 +00:00
|
|
|
tcmu_set_next_deadline(&udev->qfull_queue, &udev->qfull_timer);
|
|
|
|
|
2017-11-28 18:40:32 +00:00
|
|
|
mutex_unlock(&udev->cmdr_lock);
|
2017-11-28 18:40:31 +00:00
|
|
|
|
|
|
|
spin_lock_bh(&timed_out_udevs_lock);
|
|
|
|
}
|
|
|
|
|
|
|
|
spin_unlock_bh(&timed_out_udevs_lock);
|
|
|
|
}
|
|
|
|
|
2017-11-28 18:40:30 +00:00
|
|
|
static void tcmu_unmap_work_fn(struct work_struct *work)
|
2017-11-28 18:40:29 +00:00
|
|
|
{
|
2017-11-28 18:40:31 +00:00
|
|
|
check_timedout_devices();
|
2017-11-28 18:40:30 +00:00
|
|
|
find_free_blocks();
|
2017-05-02 03:38:06 +00:00
|
|
|
}
|
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
static int __init tcmu_module_init(void)
|
|
|
|
{
|
2017-06-06 14:28:50 +00:00
|
|
|
int ret, i, k, len = 0;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
BUILD_BUG_ON((sizeof(struct tcmu_cmd_entry) % TCMU_OP_ALIGN_SIZE) != 0);
|
|
|
|
|
2017-11-28 18:40:39 +00:00
|
|
|
INIT_DELAYED_WORK(&tcmu_unmap_work, tcmu_unmap_work_fn);
|
2017-11-28 18:40:30 +00:00
|
|
|
|
2014-10-01 23:07:05 +00:00
|
|
|
tcmu_cmd_cache = kmem_cache_create("tcmu_cmd_cache",
|
|
|
|
sizeof(struct tcmu_cmd),
|
|
|
|
__alignof__(struct tcmu_cmd),
|
|
|
|
0, NULL);
|
|
|
|
if (!tcmu_cmd_cache)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
tcmu_root_device = root_device_register("tcm_user");
|
|
|
|
if (IS_ERR(tcmu_root_device)) {
|
|
|
|
ret = PTR_ERR(tcmu_root_device);
|
|
|
|
goto out_free_cache;
|
|
|
|
}
|
|
|
|
|
|
|
|
ret = genl_register_family(&tcmu_genl_family);
|
|
|
|
if (ret < 0) {
|
|
|
|
goto out_unreg_device;
|
|
|
|
}
|
|
|
|
|
2020-04-27 15:08:21 +00:00
|
|
|
for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
|
2017-03-18 22:04:13 +00:00
|
|
|
len += sizeof(struct configfs_attribute *);
|
2020-04-27 15:08:21 +00:00
|
|
|
for (i = 0; passthrough_pr_attrib_attrs[i] != NULL; i++)
|
|
|
|
len += sizeof(struct configfs_attribute *);
|
|
|
|
for (i = 0; tcmu_attrib_attrs[i] != NULL; i++)
|
2017-06-06 14:28:50 +00:00
|
|
|
len += sizeof(struct configfs_attribute *);
|
|
|
|
len += sizeof(struct configfs_attribute *);
|
2017-03-18 22:04:13 +00:00
|
|
|
|
|
|
|
tcmu_attrs = kzalloc(len, GFP_KERNEL);
|
|
|
|
if (!tcmu_attrs) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out_unreg_genl;
|
|
|
|
}
|
|
|
|
|
2020-04-27 15:08:21 +00:00
|
|
|
for (i = 0; passthrough_attrib_attrs[i] != NULL; i++)
|
2017-03-18 22:04:13 +00:00
|
|
|
tcmu_attrs[i] = passthrough_attrib_attrs[i];
|
2020-04-27 15:08:21 +00:00
|
|
|
for (k = 0; passthrough_pr_attrib_attrs[k] != NULL; k++)
|
|
|
|
tcmu_attrs[i++] = passthrough_pr_attrib_attrs[k];
|
|
|
|
for (k = 0; tcmu_attrib_attrs[k] != NULL; k++)
|
|
|
|
tcmu_attrs[i++] = tcmu_attrib_attrs[k];
|
2017-03-18 22:04:13 +00:00
|
|
|
tcmu_ops.tb_dev_attrib_attrs = tcmu_attrs;
|
|
|
|
|
2015-05-10 16:14:56 +00:00
|
|
|
ret = transport_backend_register(&tcmu_ops);
|
2014-10-01 23:07:05 +00:00
|
|
|
if (ret)
|
2017-03-18 22:04:13 +00:00
|
|
|
goto out_attrs;
|
2014-10-01 23:07:05 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
2017-03-18 22:04:13 +00:00
|
|
|
out_attrs:
|
|
|
|
kfree(tcmu_attrs);
|
2014-10-01 23:07:05 +00:00
|
|
|
out_unreg_genl:
|
|
|
|
genl_unregister_family(&tcmu_genl_family);
|
|
|
|
out_unreg_device:
|
|
|
|
root_device_unregister(tcmu_root_device);
|
|
|
|
out_free_cache:
|
|
|
|
kmem_cache_destroy(tcmu_cmd_cache);
|
|
|
|
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __exit tcmu_module_exit(void)
|
|
|
|
{
|
2017-11-28 18:40:39 +00:00
|
|
|
cancel_delayed_work_sync(&tcmu_unmap_work);
|
2015-05-10 16:14:56 +00:00
|
|
|
target_backend_unregister(&tcmu_ops);
|
2017-03-18 22:04:13 +00:00
|
|
|
kfree(tcmu_attrs);
|
2014-10-01 23:07:05 +00:00
|
|
|
genl_unregister_family(&tcmu_genl_family);
|
|
|
|
root_device_unregister(tcmu_root_device);
|
|
|
|
kmem_cache_destroy(tcmu_cmd_cache);
|
|
|
|
}
|
|
|
|
|
|
|
|
MODULE_DESCRIPTION("TCM USER subsystem plugin");
|
|
|
|
MODULE_AUTHOR("Shaohua Li <shli@kernel.org>");
|
|
|
|
MODULE_AUTHOR("Andy Grover <agrover@redhat.com>");
|
|
|
|
MODULE_LICENSE("GPL");
|
|
|
|
|
|
|
|
module_init(tcmu_module_init);
|
|
|
|
module_exit(tcmu_module_exit);
|