From 98e2eeb31e4a0cd248af6a26290d5246b28488d8 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 4 Mar 2013 13:52:07 -0800 Subject: [PATCH 01/25] target/iscsi: Remove chap_set_random() The result from get_random_bytes should already be random, so further manipulation and mixing should not be needed. Signed-off-by: Andy Grover Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target_auth.c | 28 +----------------------- 1 file changed, 1 insertion(+), 27 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target_auth.c b/drivers/target/iscsi/iscsi_target_auth.c index a0fc7b9eea65..cee17543278c 100644 --- a/drivers/target/iscsi/iscsi_target_auth.c +++ b/drivers/target/iscsi/iscsi_target_auth.c @@ -49,32 +49,6 @@ static void chap_binaryhex_to_asciihex(char *dst, char *src, int src_len) } } -static void chap_set_random(char *data, int length) -{ - long r; - unsigned n; - - while (length > 0) { - get_random_bytes(&r, sizeof(long)); - r = r ^ (r >> 8); - r = r ^ (r >> 4); - n = r & 0x7; - - get_random_bytes(&r, sizeof(long)); - r = r ^ (r >> 8); - r = r ^ (r >> 5); - n = (n << 3) | (r & 0x7); - - get_random_bytes(&r, sizeof(long)); - r = r ^ (r >> 8); - r = r ^ (r >> 5); - n = (n << 2) | (r & 0x3); - - *data++ = n; - length--; - } -} - static void chap_gen_challenge( struct iscsi_conn *conn, int caller, @@ -86,7 +60,7 @@ static void chap_gen_challenge( memset(challenge_asciihex, 0, CHAP_CHALLENGE_LENGTH * 2 + 1); - chap_set_random(chap->challenge, CHAP_CHALLENGE_LENGTH); + get_random_bytes(chap->challenge, CHAP_CHALLENGE_LENGTH); chap_binaryhex_to_asciihex(challenge_asciihex, chap->challenge, CHAP_CHALLENGE_LENGTH); /* From 5d358065daaed54e8cacf911ab9f9554a845bd00 Mon Sep 17 00:00:00 2001 From: Andy Grover Date: Mon, 4 Mar 2013 13:52:08 -0800 Subject: [PATCH 02/25] target/iscsi: Use ISCSI_LOGIN_CURRENT/NEXT_STAGE macros Fix bit-clearing in login_rsp->flags for case 0. Signed-off-by: Andy Grover Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target_nego.c | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c index 9d902aefe01a..3cbdccd77e0b 100644 --- a/drivers/target/iscsi/iscsi_target_nego.c +++ b/drivers/target/iscsi/iscsi_target_nego.c @@ -200,8 +200,8 @@ static int iscsi_target_check_login_request( return -1; } - req_csg = (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2; - req_nsg = (login_req->flags & ISCSI_FLAG_LOGIN_NEXT_STAGE_MASK); + req_csg = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags); + req_nsg = ISCSI_LOGIN_NEXT_STAGE(login_req->flags); if (req_csg != login->current_stage) { pr_err("Initiator unexpectedly changed login stage" @@ -681,9 +681,9 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo return -1; } - switch ((login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2) { + switch (ISCSI_LOGIN_CURRENT_STAGE(login_req->flags)) { case 0: - login_rsp->flags |= (0 & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK); + login_rsp->flags &= ~ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK; if (iscsi_target_handle_csg_zero(conn, login) < 0) return -1; break; @@ -702,8 +702,7 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo default: pr_err("Illegal CSG: %d received from" " Initiator, protocol error.\n", - (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) - >> 2); + ISCSI_LOGIN_CURRENT_STAGE(login_req->flags)); break; } @@ -755,8 +754,7 @@ static int iscsi_target_locate_portal( login->first_request = 1; login->leading_connection = (!login_req->tsih) ? 1 : 0; - login->current_stage = - (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2; + login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags); login->version_min = login_req->min_version; login->version_max = login_req->max_version; memcpy(login->isid, login_req->isid, 6); From a6ad57efc39ca0ed58733f13f658ac14317a90ba Mon Sep 17 00:00:00 2001 From: Wei Yongjun Date: Mon, 11 Mar 2013 21:48:28 +0800 Subject: [PATCH 03/25] tcm_fc: using kfree_rcu() to simplify the code The callback function of call_rcu() just calls a kfree(), so we can use kfree_rcu() instead of call_rcu() + callback function. Signed-off-by: Wei Yongjun Signed-off-by: Nicholas Bellinger --- drivers/target/tcm_fc/tfc_sess.c | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/drivers/target/tcm_fc/tfc_sess.c b/drivers/target/tcm_fc/tfc_sess.c index 113f33598b9f..4859505ae2ed 100644 --- a/drivers/target/tcm_fc/tfc_sess.c +++ b/drivers/target/tcm_fc/tfc_sess.c @@ -428,19 +428,12 @@ static int ft_prli(struct fc_rport_priv *rdata, u32 spp_len, return ret; } -static void ft_sess_rcu_free(struct rcu_head *rcu) -{ - struct ft_sess *sess = container_of(rcu, struct ft_sess, rcu); - - kfree(sess); -} - static void ft_sess_free(struct kref *kref) { struct ft_sess *sess = container_of(kref, struct ft_sess, kref); transport_deregister_session(sess->se_sess); - call_rcu(&sess->rcu, ft_sess_rcu_free); + kfree_rcu(sess, rcu); } void ft_sess_put(struct ft_sess *sess) From c0c2dd499b7ae3d4d314ef2965ee2387848a39d9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Engel?= Date: Mon, 18 Mar 2013 18:29:53 -0400 Subject: [PATCH 04/25] qla2xxx: Remove unused function It was already unused when first introduced in 2d70c103. Signed-off-by: Joern Engel Signed-off-by: Nicholas Bellinger --- drivers/scsi/qla2xxx/qla_target.c | 19 ------------------- drivers/scsi/qla2xxx/qla_target.h | 1 - 2 files changed, 20 deletions(-) diff --git a/drivers/scsi/qla2xxx/qla_target.c b/drivers/scsi/qla2xxx/qla_target.c index 61b5d8c2b5da..fcdc22306cab 100644 --- a/drivers/scsi/qla2xxx/qla_target.c +++ b/drivers/scsi/qla2xxx/qla_target.c @@ -2585,25 +2585,6 @@ static void qlt_do_ctio_completion(struct scsi_qla_host *vha, uint32_t handle, ha->tgt.tgt_ops->free_cmd(cmd); } -/* ha->hardware_lock supposed to be held on entry */ -/* called via callback from qla2xxx */ -void qlt_ctio_completion(struct scsi_qla_host *vha, uint32_t handle) -{ - struct qla_hw_data *ha = vha->hw; - struct qla_tgt *tgt = ha->tgt.qla_tgt; - - if (likely(tgt == NULL)) { - ql_dbg(ql_dbg_tgt, vha, 0xe021, - "CTIO, but target mode not enabled" - " (ha %d %p handle %#x)", vha->vp_idx, ha, handle); - return; - } - - tgt->irq_cmd_count++; - qlt_do_ctio_completion(vha, handle, CTIO_SUCCESS, NULL); - tgt->irq_cmd_count--; -} - static inline int qlt_get_fcp_task_attr(struct scsi_qla_host *vha, uint8_t task_codes) { diff --git a/drivers/scsi/qla2xxx/qla_target.h b/drivers/scsi/qla2xxx/qla_target.h index ff9ccb9fd036..b33e411f28a0 100644 --- a/drivers/scsi/qla2xxx/qla_target.h +++ b/drivers/scsi/qla2xxx/qla_target.h @@ -980,7 +980,6 @@ extern int qlt_xmit_response(struct qla_tgt_cmd *, int, uint8_t); extern void qlt_xmit_tm_rsp(struct qla_tgt_mgmt_cmd *); extern void qlt_free_mcmd(struct qla_tgt_mgmt_cmd *); extern void qlt_free_cmd(struct qla_tgt_cmd *cmd); -extern void qlt_ctio_completion(struct scsi_qla_host *, uint32_t); extern void qlt_async_event(uint16_t, struct scsi_qla_host *, uint16_t *); extern void qlt_enable_vha(struct scsi_qla_host *); extern void qlt_vport_create(struct scsi_qla_host *, struct qla_hw_data *); From 70d3ae5c80758cb2bf3cf1e0d5b2c89d040de90b Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 25 Feb 2013 14:03:42 +0800 Subject: [PATCH 05/25] target/file: Add WRITE_SAME w/ UNMAP=1 emulation support This patch adds support for emulation of WRITE_SAME w/ UNMAP=1 within fd_execute_write_same_unmap() backend code. If the FILEIO backend is normal file, the emulation uses fallocate to punch hole to reclaim the free space used by the file. If the FILEIO backend is block device, the emulation uses blkdev_issue_discard(). Tested with 512, 1k, 2k, and 4k block_sizes. Changes in v2: - Set the various dev->dev_attrib.*unmap* values (nab) Cc: Christoph Hellwig Cc: Martin K. Petersen Cc: Nicholas Bellinger Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_file.c | 94 +++++++++++++++++++++++++++++-- 1 file changed, 89 insertions(+), 5 deletions(-) diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index 17a6acbc3ab0..e1878bfd97b7 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -30,6 +30,7 @@ #include #include #include +#include #include #include @@ -166,6 +167,30 @@ static int fd_configure_device(struct se_device *dev) " block_device blocks: %llu logical_block_size: %d\n", dev_size, div_u64(dev_size, fd_dev->fd_block_size), fd_dev->fd_block_size); + /* + * Check if the underlying struct block_device request_queue supports + * the QUEUE_FLAG_DISCARD bit for UNMAP/WRITE_SAME in SCSI + TRIM + * in ATA and we need to set TPE=1 + */ + if (blk_queue_discard(q)) { + dev->dev_attrib.max_unmap_lba_count = + q->limits.max_discard_sectors; + /* + * Currently hardcoded to 1 in Linux/SCSI code.. + */ + dev->dev_attrib.max_unmap_block_desc_count = 1; + dev->dev_attrib.unmap_granularity = + q->limits.discard_granularity >> 9; + dev->dev_attrib.unmap_granularity_alignment = + q->limits.discard_alignment; + pr_debug("IFILE: BLOCK Discard support available," + " disabled by default\n"); + } + /* + * Enable write same emulation for IBLOCK and use 0xFFFF as + * the smaller WRITE_SAME(10) only has a two-byte block count. + */ + dev->dev_attrib.max_write_same_len = 0xFFFF; } else { if (!(fd_dev->fbd_flags & FBDF_HAS_SIZE)) { pr_err("FILEIO: Missing fd_dev_size=" @@ -176,6 +201,23 @@ static int fd_configure_device(struct se_device *dev) dev->dev_attrib.hw_block_size = FD_BLOCKSIZE; dev->dev_attrib.hw_max_sectors = FD_MAX_SECTORS; + + /* + * Limit UNMAP emulation to 8k Number of LBAs (NoLB) + */ + dev->dev_attrib.max_unmap_lba_count = 0x2000; + /* + * Currently hardcoded to 1 in Linux/SCSI code.. + */ + dev->dev_attrib.max_unmap_block_desc_count = 1; + dev->dev_attrib.unmap_granularity = 1; + dev->dev_attrib.unmap_granularity_alignment = 0; + + /* + * Limit WRITE_SAME w/ UNMAP=0 emulation to 8k Number of LBAs (NoLB) + * based upon struct iovec limit for vfs_writev() + */ + dev->dev_attrib.max_write_same_len = 0x1000; } fd_dev->fd_block_size = dev->dev_attrib.hw_block_size; @@ -190,11 +232,6 @@ static int fd_configure_device(struct se_device *dev) fd_dev->fd_dev_id = fd_host->fd_host_dev_id_count++; fd_dev->fd_queue_depth = dev->queue_depth; - /* - * Limit WRITE_SAME w/ UNMAP=0 emulation to 8k Number of LBAs (NoLB) - * based upon struct iovec limit for vfs_writev() - */ - dev->dev_attrib.max_write_same_len = 0x1000; pr_debug("CORE_FILE[%u] - Added TCM FILEIO Device ID: %u at %s," " %llu total bytes\n", fd_host->fd_host_id, fd_dev->fd_dev_id, @@ -441,6 +478,52 @@ fd_execute_write_same(struct se_cmd *cmd) return 0; } +static sense_reason_t +fd_execute_write_same_unmap(struct se_cmd *cmd) +{ + struct se_device *se_dev = cmd->se_dev; + struct fd_dev *fd_dev = FD_DEV(se_dev); + struct file *file = fd_dev->fd_file; + struct inode *inode = file->f_mapping->host; + sector_t nolb = sbc_get_write_same_sectors(cmd); + int ret; + + if (!nolb) { + target_complete_cmd(cmd, SAM_STAT_GOOD); + return 0; + } + + if (S_ISBLK(inode->i_mode)) { + /* The backend is block device, use discard */ + struct block_device *bdev = inode->i_bdev; + + ret = blkdev_issue_discard(bdev, cmd->t_task_lba, + nolb, GFP_KERNEL, 0); + if (ret < 0) { + pr_warn("FILEIO: blkdev_issue_discard() failed: %d\n", + ret); + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } + } else { + /* The backend is normal file, use fallocate */ + loff_t pos = cmd->t_task_lba * se_dev->dev_attrib.block_size; + unsigned int len = nolb * se_dev->dev_attrib.block_size; + int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; + + if (!file->f_op->fallocate) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + + ret = file->f_op->fallocate(file, mode, pos, len); + if (ret < 0) { + pr_warn("FILEIO: fallocate() failed: %d\n", ret); + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } + } + + target_complete_cmd(cmd, GOOD); + return 0; +} + static sense_reason_t fd_execute_rw(struct se_cmd *cmd) { @@ -600,6 +683,7 @@ static struct sbc_ops fd_sbc_ops = { .execute_rw = fd_execute_rw, .execute_sync_cache = fd_execute_sync_cache, .execute_write_same = fd_execute_write_same, + .execute_write_same_unmap = fd_execute_write_same_unmap, }; static sense_reason_t From 506427628afeb5d70eefdac1fe4c2a1ffb27df5f Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 25 Feb 2013 14:03:43 +0800 Subject: [PATCH 06/25] target/file: Add UNMAP emulation support This patch adds support for emulation of UNMAP within fd_execute_unmap() backend code. If the FILEIO backend is normal file, the emulation uses fallocate to punch hole to reclaim the free space used by the file. If the FILEIO backend is block device, the emulation uses blkdev_issue_discard(). Cc: Christoph Hellwig Cc: Martin K. Petersen Cc: Nicholas Bellinger Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_file.c | 111 ++++++++++++++++++++++++++++++ 1 file changed, 111 insertions(+) diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index e1878bfd97b7..7e02304a1024 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -33,6 +33,7 @@ #include #include #include +#include #include #include @@ -524,6 +525,115 @@ fd_execute_write_same_unmap(struct se_cmd *cmd) return 0; } +static sense_reason_t +fd_execute_unmap(struct se_cmd *cmd) +{ + struct se_device *dev = cmd->se_dev; + struct fd_dev *fd_dev = FD_DEV(dev); + struct file *file = fd_dev->fd_file; + struct inode *inode = file->f_mapping->host; + unsigned char *buf, *ptr = NULL; + sector_t lba; + int size; + u32 range; + sense_reason_t ret = 0; + int dl, bd_dl, err; + + /* We never set ANC_SUP */ + if (cmd->t_task_cdb[1]) + return TCM_INVALID_CDB_FIELD; + + if (cmd->data_length == 0) { + target_complete_cmd(cmd, SAM_STAT_GOOD); + return 0; + } + + if (cmd->data_length < 8) { + pr_warn("UNMAP parameter list length %u too small\n", + cmd->data_length); + return TCM_PARAMETER_LIST_LENGTH_ERROR; + } + + buf = transport_kmap_data_sg(cmd); + if (!buf) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + + dl = get_unaligned_be16(&buf[0]); + bd_dl = get_unaligned_be16(&buf[2]); + + size = cmd->data_length - 8; + if (bd_dl > size) + pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n", + cmd->data_length, bd_dl); + else + size = bd_dl; + + if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) { + ret = TCM_INVALID_PARAMETER_LIST; + goto err; + } + + /* First UNMAP block descriptor starts at 8 byte offset */ + ptr = &buf[8]; + pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u" + " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr); + + while (size >= 16) { + lba = get_unaligned_be64(&ptr[0]); + range = get_unaligned_be32(&ptr[8]); + pr_debug("UNMAP: Using lba: %llu and range: %u\n", + (unsigned long long)lba, range); + + if (range > dev->dev_attrib.max_unmap_lba_count) { + ret = TCM_INVALID_PARAMETER_LIST; + goto err; + } + + if (lba + range > dev->transport->get_blocks(dev) + 1) { + ret = TCM_ADDRESS_OUT_OF_RANGE; + goto err; + } + + if (S_ISBLK(inode->i_mode)) { + /* The backend is block device, use discard */ + struct block_device *bdev = inode->i_bdev; + + err = blkdev_issue_discard(bdev, lba, range, GFP_KERNEL, 0); + if (err < 0) { + pr_err("FILEIO: blkdev_issue_discard() failed: %d\n", + err); + ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + goto err; + } + } else { + /* The backend is normal file, use fallocate */ + loff_t pos = lba * dev->dev_attrib.block_size; + unsigned int len = range * dev->dev_attrib.block_size; + int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; + + if (!file->f_op->fallocate) { + ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + goto err; + } + err = file->f_op->fallocate(file, mode, pos, len); + if (err < 0) { + pr_warn("FILEIO: fallocate() failed: %d\n", err); + ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + goto err; + } + } + + ptr += 16; + size -= 16; + } + +err: + transport_kunmap_data_sg(cmd); + if (!ret) + target_complete_cmd(cmd, GOOD); + return ret; +} + static sense_reason_t fd_execute_rw(struct se_cmd *cmd) { @@ -684,6 +794,7 @@ static struct sbc_ops fd_sbc_ops = { .execute_sync_cache = fd_execute_sync_cache, .execute_write_same = fd_execute_write_same, .execute_write_same_unmap = fd_execute_write_same_unmap, + .execute_unmap = fd_execute_unmap, }; static sense_reason_t From 43f55bbb144acca55cea50326388d1b648b6fd0f Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 25 Feb 2013 14:03:44 +0800 Subject: [PATCH 07/25] target/file: Add fd_do_unmap() helper Add helper fd_do_unmap() to remove duplicated code in fd_execute_write_same_unmap() and fd_execute_unmap(). Cc: Christoph Hellwig Cc: Martin K. Petersen Cc: Nicholas Bellinger Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_file.c | 72 +++++++++++++------------------ 1 file changed, 30 insertions(+), 42 deletions(-) diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index 7e02304a1024..d5b84fac31c7 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -480,25 +480,16 @@ fd_execute_write_same(struct se_cmd *cmd) } static sense_reason_t -fd_execute_write_same_unmap(struct se_cmd *cmd) +fd_do_unmap(struct se_cmd *cmd, struct file *file, sector_t lba, sector_t nolb) { - struct se_device *se_dev = cmd->se_dev; - struct fd_dev *fd_dev = FD_DEV(se_dev); - struct file *file = fd_dev->fd_file; struct inode *inode = file->f_mapping->host; - sector_t nolb = sbc_get_write_same_sectors(cmd); int ret; - if (!nolb) { - target_complete_cmd(cmd, SAM_STAT_GOOD); - return 0; - } - if (S_ISBLK(inode->i_mode)) { /* The backend is block device, use discard */ struct block_device *bdev = inode->i_bdev; - ret = blkdev_issue_discard(bdev, cmd->t_task_lba, + ret = blkdev_issue_discard(bdev, lba, nolb, GFP_KERNEL, 0); if (ret < 0) { pr_warn("FILEIO: blkdev_issue_discard() failed: %d\n", @@ -507,7 +498,8 @@ fd_execute_write_same_unmap(struct se_cmd *cmd) } } else { /* The backend is normal file, use fallocate */ - loff_t pos = cmd->t_task_lba * se_dev->dev_attrib.block_size; + struct se_device *se_dev = cmd->se_dev; + loff_t pos = lba * se_dev->dev_attrib.block_size; unsigned int len = nolb * se_dev->dev_attrib.block_size; int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; @@ -521,6 +513,28 @@ fd_execute_write_same_unmap(struct se_cmd *cmd) } } + return 0; +} + +static sense_reason_t +fd_execute_write_same_unmap(struct se_cmd *cmd) +{ + struct se_device *se_dev = cmd->se_dev; + struct fd_dev *fd_dev = FD_DEV(se_dev); + struct file *file = fd_dev->fd_file; + sector_t lba = cmd->t_task_lba; + sector_t nolb = sbc_get_write_same_sectors(cmd); + int ret; + + if (!nolb) { + target_complete_cmd(cmd, SAM_STAT_GOOD); + return 0; + } + + ret = fd_do_unmap(cmd, file, lba, nolb); + if (ret) + return ret; + target_complete_cmd(cmd, GOOD); return 0; } @@ -531,13 +545,12 @@ fd_execute_unmap(struct se_cmd *cmd) struct se_device *dev = cmd->se_dev; struct fd_dev *fd_dev = FD_DEV(dev); struct file *file = fd_dev->fd_file; - struct inode *inode = file->f_mapping->host; unsigned char *buf, *ptr = NULL; sector_t lba; int size; u32 range; sense_reason_t ret = 0; - int dl, bd_dl, err; + int dl, bd_dl; /* We never set ANC_SUP */ if (cmd->t_task_cdb[1]) @@ -594,34 +607,9 @@ fd_execute_unmap(struct se_cmd *cmd) goto err; } - if (S_ISBLK(inode->i_mode)) { - /* The backend is block device, use discard */ - struct block_device *bdev = inode->i_bdev; - - err = blkdev_issue_discard(bdev, lba, range, GFP_KERNEL, 0); - if (err < 0) { - pr_err("FILEIO: blkdev_issue_discard() failed: %d\n", - err); - ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - goto err; - } - } else { - /* The backend is normal file, use fallocate */ - loff_t pos = lba * dev->dev_attrib.block_size; - unsigned int len = range * dev->dev_attrib.block_size; - int mode = FALLOC_FL_PUNCH_HOLE | FALLOC_FL_KEEP_SIZE; - - if (!file->f_op->fallocate) { - ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - goto err; - } - err = file->f_op->fallocate(file, mode, pos, len); - if (err < 0) { - pr_warn("FILEIO: fallocate() failed: %d\n", err); - ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - goto err; - } - } + ret = fd_do_unmap(cmd, file, lba, range); + if (ret) + goto err; ptr += 16; size -= 16; From dbc21c5abbeb4419da5a54157c4b7a0ec0dee185 Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 25 Feb 2013 14:03:45 +0800 Subject: [PATCH 08/25] target/iblock: Add iblock_do_unmap() helper Add helper iblock_do_unmap() to remove duplicated code in iblock_execute_write_same_unmap() and iblock_execute_unmap(). Cc: Christoph Hellwig Cc: Martin K. Petersen Cc: Nicholas Bellinger Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_iblock.c | 41 ++++++++++++++++++----------- 1 file changed, 25 insertions(+), 16 deletions(-) diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 8bcc514ec8b6..96a9b620e89a 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -379,6 +379,21 @@ iblock_execute_sync_cache(struct se_cmd *cmd) return 0; } +static sense_reason_t +iblock_do_unmap(struct se_cmd *cmd, struct block_device *bdev, + sector_t lba, sector_t nolb) +{ + int ret; + + ret = blkdev_issue_discard(bdev, lba, nolb, GFP_KERNEL, 0); + if (ret < 0) { + pr_err("blkdev_issue_discard() failed: %d\n", ret); + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + } + + return 0; +} + static sense_reason_t iblock_execute_unmap(struct se_cmd *cmd) { @@ -389,7 +404,7 @@ iblock_execute_unmap(struct se_cmd *cmd) int size; u32 range; sense_reason_t ret = 0; - int dl, bd_dl, err; + int dl, bd_dl; /* We never set ANC_SUP */ if (cmd->t_task_cdb[1]) @@ -446,14 +461,9 @@ iblock_execute_unmap(struct se_cmd *cmd) goto err; } - err = blkdev_issue_discard(ib_dev->ibd_bd, lba, range, - GFP_KERNEL, 0); - if (err < 0) { - pr_err("blkdev_issue_discard() failed: %d\n", - err); - ret = TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + ret = iblock_do_unmap(cmd, ib_dev->ibd_bd, lba, range); + if (ret) goto err; - } ptr += 16; size -= 16; @@ -469,15 +479,14 @@ iblock_execute_unmap(struct se_cmd *cmd) static sense_reason_t iblock_execute_write_same_unmap(struct se_cmd *cmd) { - struct iblock_dev *ib_dev = IBLOCK_DEV(cmd->se_dev); - int rc; + struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd; + sector_t lba = cmd->t_task_lba; + sector_t nolb = sbc_get_write_same_sectors(cmd); + int ret; - rc = blkdev_issue_discard(ib_dev->ibd_bd, cmd->t_task_lba, - sbc_get_write_same_sectors(cmd), GFP_KERNEL, 0); - if (rc < 0) { - pr_warn("blkdev_issue_discard() failed: %d\n", rc); - return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - } + ret = iblock_do_unmap(cmd, bdev, lba, nolb); + if (ret) + return ret; target_complete_cmd(cmd, GOOD); return 0; From 86d7182985d25900929adce14fffd729cc8c6fb8 Mon Sep 17 00:00:00 2001 From: Asias He Date: Mon, 25 Feb 2013 14:03:46 +0800 Subject: [PATCH 09/25] target: Add sbc_execute_unmap() helper iblock_execute_unmap() and fd_execute_unmap share a lot of code. Add sbc_execute_unmap() helper to remove duplicated code for iblock_execute_unmap() and fd_execute_unmap(). Cc: Christoph Hellwig Cc: Martin K. Petersen Cc: Nicholas Bellinger Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_file.c | 82 ++------------------------- drivers/target/target_core_iblock.c | 81 ++------------------------ drivers/target/target_core_sbc.c | 85 ++++++++++++++++++++++++++++ include/target/target_core_backend.h | 4 ++ 4 files changed, 97 insertions(+), 155 deletions(-) diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index d5b84fac31c7..9063c31df9e1 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -480,8 +480,9 @@ fd_execute_write_same(struct se_cmd *cmd) } static sense_reason_t -fd_do_unmap(struct se_cmd *cmd, struct file *file, sector_t lba, sector_t nolb) +fd_do_unmap(struct se_cmd *cmd, void *priv, sector_t lba, sector_t nolb) { + struct file *file = priv; struct inode *inode = file->f_mapping->host; int ret; @@ -542,84 +543,9 @@ fd_execute_write_same_unmap(struct se_cmd *cmd) static sense_reason_t fd_execute_unmap(struct se_cmd *cmd) { - struct se_device *dev = cmd->se_dev; - struct fd_dev *fd_dev = FD_DEV(dev); - struct file *file = fd_dev->fd_file; - unsigned char *buf, *ptr = NULL; - sector_t lba; - int size; - u32 range; - sense_reason_t ret = 0; - int dl, bd_dl; + struct file *file = FD_DEV(cmd->se_dev)->fd_file; - /* We never set ANC_SUP */ - if (cmd->t_task_cdb[1]) - return TCM_INVALID_CDB_FIELD; - - if (cmd->data_length == 0) { - target_complete_cmd(cmd, SAM_STAT_GOOD); - return 0; - } - - if (cmd->data_length < 8) { - pr_warn("UNMAP parameter list length %u too small\n", - cmd->data_length); - return TCM_PARAMETER_LIST_LENGTH_ERROR; - } - - buf = transport_kmap_data_sg(cmd); - if (!buf) - return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - - dl = get_unaligned_be16(&buf[0]); - bd_dl = get_unaligned_be16(&buf[2]); - - size = cmd->data_length - 8; - if (bd_dl > size) - pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n", - cmd->data_length, bd_dl); - else - size = bd_dl; - - if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) { - ret = TCM_INVALID_PARAMETER_LIST; - goto err; - } - - /* First UNMAP block descriptor starts at 8 byte offset */ - ptr = &buf[8]; - pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u" - " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr); - - while (size >= 16) { - lba = get_unaligned_be64(&ptr[0]); - range = get_unaligned_be32(&ptr[8]); - pr_debug("UNMAP: Using lba: %llu and range: %u\n", - (unsigned long long)lba, range); - - if (range > dev->dev_attrib.max_unmap_lba_count) { - ret = TCM_INVALID_PARAMETER_LIST; - goto err; - } - - if (lba + range > dev->transport->get_blocks(dev) + 1) { - ret = TCM_ADDRESS_OUT_OF_RANGE; - goto err; - } - - ret = fd_do_unmap(cmd, file, lba, range); - if (ret) - goto err; - - ptr += 16; - size -= 16; - } - -err: - transport_kunmap_data_sg(cmd); - if (!ret) - target_complete_cmd(cmd, GOOD); - return ret; + return sbc_execute_unmap(cmd, fd_do_unmap, file); } static sense_reason_t diff --git a/drivers/target/target_core_iblock.c b/drivers/target/target_core_iblock.c index 96a9b620e89a..07f5f94634bb 100644 --- a/drivers/target/target_core_iblock.c +++ b/drivers/target/target_core_iblock.c @@ -380,9 +380,10 @@ iblock_execute_sync_cache(struct se_cmd *cmd) } static sense_reason_t -iblock_do_unmap(struct se_cmd *cmd, struct block_device *bdev, +iblock_do_unmap(struct se_cmd *cmd, void *priv, sector_t lba, sector_t nolb) { + struct block_device *bdev = priv; int ret; ret = blkdev_issue_discard(bdev, lba, nolb, GFP_KERNEL, 0); @@ -397,83 +398,9 @@ iblock_do_unmap(struct se_cmd *cmd, struct block_device *bdev, static sense_reason_t iblock_execute_unmap(struct se_cmd *cmd) { - struct se_device *dev = cmd->se_dev; - struct iblock_dev *ib_dev = IBLOCK_DEV(dev); - unsigned char *buf, *ptr = NULL; - sector_t lba; - int size; - u32 range; - sense_reason_t ret = 0; - int dl, bd_dl; + struct block_device *bdev = IBLOCK_DEV(cmd->se_dev)->ibd_bd; - /* We never set ANC_SUP */ - if (cmd->t_task_cdb[1]) - return TCM_INVALID_CDB_FIELD; - - if (cmd->data_length == 0) { - target_complete_cmd(cmd, SAM_STAT_GOOD); - return 0; - } - - if (cmd->data_length < 8) { - pr_warn("UNMAP parameter list length %u too small\n", - cmd->data_length); - return TCM_PARAMETER_LIST_LENGTH_ERROR; - } - - buf = transport_kmap_data_sg(cmd); - if (!buf) - return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; - - dl = get_unaligned_be16(&buf[0]); - bd_dl = get_unaligned_be16(&buf[2]); - - size = cmd->data_length - 8; - if (bd_dl > size) - pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n", - cmd->data_length, bd_dl); - else - size = bd_dl; - - if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) { - ret = TCM_INVALID_PARAMETER_LIST; - goto err; - } - - /* First UNMAP block descriptor starts at 8 byte offset */ - ptr = &buf[8]; - pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u" - " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr); - - while (size >= 16) { - lba = get_unaligned_be64(&ptr[0]); - range = get_unaligned_be32(&ptr[8]); - pr_debug("UNMAP: Using lba: %llu and range: %u\n", - (unsigned long long)lba, range); - - if (range > dev->dev_attrib.max_unmap_lba_count) { - ret = TCM_INVALID_PARAMETER_LIST; - goto err; - } - - if (lba + range > dev->transport->get_blocks(dev) + 1) { - ret = TCM_ADDRESS_OUT_OF_RANGE; - goto err; - } - - ret = iblock_do_unmap(cmd, ib_dev->ibd_bd, lba, range); - if (ret) - goto err; - - ptr += 16; - size -= 16; - } - -err: - transport_kunmap_data_sg(cmd); - if (!ret) - target_complete_cmd(cmd, GOOD); - return ret; + return sbc_execute_unmap(cmd, iblock_do_unmap, bdev); } static sense_reason_t diff --git a/drivers/target/target_core_sbc.c b/drivers/target/target_core_sbc.c index 60d4b5185f32..bbc5b0ee2bdc 100644 --- a/drivers/target/target_core_sbc.c +++ b/drivers/target/target_core_sbc.c @@ -596,3 +596,88 @@ u32 sbc_get_device_type(struct se_device *dev) return TYPE_DISK; } EXPORT_SYMBOL(sbc_get_device_type); + +sense_reason_t +sbc_execute_unmap(struct se_cmd *cmd, + sense_reason_t (*do_unmap_fn)(struct se_cmd *, void *, + sector_t, sector_t), + void *priv) +{ + struct se_device *dev = cmd->se_dev; + unsigned char *buf, *ptr = NULL; + sector_t lba; + int size; + u32 range; + sense_reason_t ret = 0; + int dl, bd_dl; + + /* We never set ANC_SUP */ + if (cmd->t_task_cdb[1]) + return TCM_INVALID_CDB_FIELD; + + if (cmd->data_length == 0) { + target_complete_cmd(cmd, SAM_STAT_GOOD); + return 0; + } + + if (cmd->data_length < 8) { + pr_warn("UNMAP parameter list length %u too small\n", + cmd->data_length); + return TCM_PARAMETER_LIST_LENGTH_ERROR; + } + + buf = transport_kmap_data_sg(cmd); + if (!buf) + return TCM_LOGICAL_UNIT_COMMUNICATION_FAILURE; + + dl = get_unaligned_be16(&buf[0]); + bd_dl = get_unaligned_be16(&buf[2]); + + size = cmd->data_length - 8; + if (bd_dl > size) + pr_warn("UNMAP parameter list length %u too small, ignoring bd_dl %u\n", + cmd->data_length, bd_dl); + else + size = bd_dl; + + if (size / 16 > dev->dev_attrib.max_unmap_block_desc_count) { + ret = TCM_INVALID_PARAMETER_LIST; + goto err; + } + + /* First UNMAP block descriptor starts at 8 byte offset */ + ptr = &buf[8]; + pr_debug("UNMAP: Sub: %s Using dl: %u bd_dl: %u size: %u" + " ptr: %p\n", dev->transport->name, dl, bd_dl, size, ptr); + + while (size >= 16) { + lba = get_unaligned_be64(&ptr[0]); + range = get_unaligned_be32(&ptr[8]); + pr_debug("UNMAP: Using lba: %llu and range: %u\n", + (unsigned long long)lba, range); + + if (range > dev->dev_attrib.max_unmap_lba_count) { + ret = TCM_INVALID_PARAMETER_LIST; + goto err; + } + + if (lba + range > dev->transport->get_blocks(dev) + 1) { + ret = TCM_ADDRESS_OUT_OF_RANGE; + goto err; + } + + ret = do_unmap_fn(cmd, priv, lba, range); + if (ret) + goto err; + + ptr += 16; + size -= 16; + } + +err: + transport_kunmap_data_sg(cmd); + if (!ret) + target_complete_cmd(cmd, GOOD); + return ret; +} +EXPORT_SYMBOL(sbc_execute_unmap); diff --git a/include/target/target_core_backend.h b/include/target/target_core_backend.h index b128c20770bc..ffa2696d64dc 100644 --- a/include/target/target_core_backend.h +++ b/include/target/target_core_backend.h @@ -60,6 +60,10 @@ sense_reason_t sbc_parse_cdb(struct se_cmd *cmd, struct sbc_ops *ops); u32 sbc_get_device_rev(struct se_device *dev); u32 sbc_get_device_type(struct se_device *dev); sector_t sbc_get_write_same_sectors(struct se_cmd *cmd); +sense_reason_t sbc_execute_unmap(struct se_cmd *cmd, + sense_reason_t (*do_unmap_fn)(struct se_cmd *cmd, void *priv, + sector_t lba, sector_t nolb), + void *priv); void transport_set_vpd_proto_id(struct t10_vpd *, unsigned char *); int transport_set_vpd_assoc(struct t10_vpd *, unsigned char *); From 0463a3fefda4cf0591edc685d4fb215a33315e01 Mon Sep 17 00:00:00 2001 From: Asias He Date: Wed, 24 Apr 2013 19:38:11 -0700 Subject: [PATCH 10/25] target/file: Set is_nonrot attribute Set is_nonrot attribute according to the block queue if the backend device is a block device. Signed-off-by: Asias He Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_file.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/drivers/target/target_core_file.c b/drivers/target/target_core_file.c index 9063c31df9e1..58ed683e04ae 100644 --- a/drivers/target/target_core_file.c +++ b/drivers/target/target_core_file.c @@ -192,6 +192,9 @@ static int fd_configure_device(struct se_device *dev) * the smaller WRITE_SAME(10) only has a two-byte block count. */ dev->dev_attrib.max_write_same_len = 0xFFFF; + + if (blk_queue_nonrot(q)) + dev->dev_attrib.is_nonrot = 1; } else { if (!(fd_dev->fbd_flags & FBDF_HAS_SIZE)) { pr_err("FILEIO: Missing fd_dev_size=" From ad6732820c56664be7d022002907ba62b981bd85 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Engel?= Date: Mon, 18 Mar 2013 18:30:31 -0400 Subject: [PATCH 11/25] target: Change default sense key of NOT_READY As the comment sais, this allows Solaris initiators to survive intermittent errors. The comment from someone reading the Solaris sources seem to imply that multipathing would be broken without this patch as well. Signed-off-by: Joern Engel Cc: Brian Bunker Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_transport.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 3243ea790eab..493e9e589059 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -2765,8 +2765,13 @@ transport_send_check_condition_and_sense(struct se_cmd *cmd, /* CURRENT ERROR */ buffer[0] = 0x70; buffer[SPC_ADD_SENSE_LEN_OFFSET] = 10; - /* ILLEGAL REQUEST */ - buffer[SPC_SENSE_KEY_OFFSET] = ILLEGAL_REQUEST; + /* + * Returning ILLEGAL REQUEST would cause immediate IO errors on + * Solaris initiators. Returning NOT READY instead means the + * operations will be retried a finite number of times and we + * can survive intermittent errors. + */ + buffer[SPC_SENSE_KEY_OFFSET] = NOT_READY; /* LOGICAL UNIT COMMUNICATION FAILURE */ buffer[SPC_ASC_KEY_OFFSET] = 0x08; break; From 20361e69ba0b52880f9a692bb80aceb989bf0e77 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Thu, 21 Mar 2013 22:54:28 -0700 Subject: [PATCH 12/25] target: Add export of target_get_sess_cmd symbol Export target_get_sess_cmd() symbol so that it can be used by iscsi-target. Signed-off-by: Nicholas Bellinger --- drivers/target/target_core_transport.c | 4 ++-- include/target/target_core_fabric.h | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/drivers/target/target_core_transport.c b/drivers/target/target_core_transport.c index 493e9e589059..f8388b4024aa 100644 --- a/drivers/target/target_core_transport.c +++ b/drivers/target/target_core_transport.c @@ -65,7 +65,6 @@ static void transport_complete_task_attr(struct se_cmd *cmd); static void transport_handle_queue_full(struct se_cmd *cmd, struct se_device *dev); static int transport_generic_get_mem(struct se_cmd *cmd); -static int target_get_sess_cmd(struct se_session *, struct se_cmd *, bool); static void transport_put_cmd(struct se_cmd *cmd); static void target_complete_ok_work(struct work_struct *work); @@ -2179,7 +2178,7 @@ EXPORT_SYMBOL(transport_generic_free_cmd); * @se_cmd: command descriptor to add * @ack_kref: Signal that fabric will perform an ack target_put_sess_cmd() */ -static int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd, +int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd, bool ack_kref) { unsigned long flags; @@ -2208,6 +2207,7 @@ static int target_get_sess_cmd(struct se_session *se_sess, struct se_cmd *se_cmd spin_unlock_irqrestore(&se_sess->sess_cmd_lock, flags); return ret; } +EXPORT_SYMBOL(target_get_sess_cmd); static void target_release_cmd_kref(struct kref *kref) { diff --git a/include/target/target_core_fabric.h b/include/target/target_core_fabric.h index aaa1ee6ab391..ba3471b73c07 100644 --- a/include/target/target_core_fabric.h +++ b/include/target/target_core_fabric.h @@ -120,7 +120,7 @@ bool transport_wait_for_tasks(struct se_cmd *); int transport_check_aborted_status(struct se_cmd *, int); int transport_send_check_condition_and_sense(struct se_cmd *, sense_reason_t, int); - +int target_get_sess_cmd(struct se_session *, struct se_cmd *, bool); int target_put_sess_cmd(struct se_session *, struct se_cmd *); void target_sess_cmd_list_set_waiting(struct se_session *); void target_wait_for_sess_cmds(struct se_session *, int); From 3f99306337ba1871afc8568e8b36004ce3cee039 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 21:53:26 -0800 Subject: [PATCH 13/25] iscsi-target: Add iscsit_transport API template Add basic struct iscsit_transport API template to allow iscsi-target for running with external transport modules using existing iscsi_target_core.h code. For all external modules, this calls try_module_get() and module_put() to obtain + release an external iscsit_transport module reference count. Also include the iscsi-target symbols necessary in iscsi_transport.h to allow external transport modules to function. v3 changes: - Add iscsit_build_reject export for ISTATE_SEND_REJECT usage v2 changes: - Drop unnecessary export of iscsit_get_transport + iscsit_put_transport (roland) - Add ->iscsit_queue_data_in() to remove extra context switch on RDMA_WRITE - Add ->iscsit_queue_status() to remove extra context switch on IB_SEND status - Add ->iscsit_get_dataout() to remove extra context switch on RDMA_READ - Drop ->iscsit_free_cmd() - Drop ->iscsit_unmap_cmd() - Rename iscsit_create_transport() -> iscsit_register_transport() (andy) - Rename iscsit_destroy_transport() -> iscsit_unregister_transport() (andy) Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/Makefile | 3 +- drivers/target/iscsi/iscsi_target_transport.c | 55 ++++++++++++ include/target/iscsi/iscsi_transport.h | 83 +++++++++++++++++++ 3 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 drivers/target/iscsi/iscsi_target_transport.c create mode 100644 include/target/iscsi/iscsi_transport.h diff --git a/drivers/target/iscsi/Makefile b/drivers/target/iscsi/Makefile index 5b9a2cf7f0a9..13a92403fe3e 100644 --- a/drivers/target/iscsi/Makefile +++ b/drivers/target/iscsi/Makefile @@ -15,6 +15,7 @@ iscsi_target_mod-y += iscsi_target_parameters.o \ iscsi_target_util.o \ iscsi_target.o \ iscsi_target_configfs.o \ - iscsi_target_stat.o + iscsi_target_stat.o \ + iscsi_target_transport.o obj-$(CONFIG_ISCSI_TARGET) += iscsi_target_mod.o diff --git a/drivers/target/iscsi/iscsi_target_transport.c b/drivers/target/iscsi/iscsi_target_transport.c new file mode 100644 index 000000000000..882728fac30c --- /dev/null +++ b/drivers/target/iscsi/iscsi_target_transport.c @@ -0,0 +1,55 @@ +#include +#include +#include + +static LIST_HEAD(g_transport_list); +static DEFINE_MUTEX(transport_mutex); + +struct iscsit_transport *iscsit_get_transport(int type) +{ + struct iscsit_transport *t; + + mutex_lock(&transport_mutex); + list_for_each_entry(t, &g_transport_list, t_node) { + if (t->transport_type == type) { + if (t->owner && !try_module_get(t->owner)) { + t = NULL; + } + mutex_unlock(&transport_mutex); + return t; + } + } + mutex_unlock(&transport_mutex); + + return NULL; +} + +void iscsit_put_transport(struct iscsit_transport *t) +{ + if (t->owner) + module_put(t->owner); +} + +int iscsit_register_transport(struct iscsit_transport *t) +{ + INIT_LIST_HEAD(&t->t_node); + + mutex_lock(&transport_mutex); + list_add_tail(&t->t_node, &g_transport_list); + mutex_unlock(&transport_mutex); + + pr_debug("Registered iSCSI transport: %s\n", t->name); + + return 0; +} +EXPORT_SYMBOL(iscsit_register_transport); + +void iscsit_unregister_transport(struct iscsit_transport *t) +{ + mutex_lock(&transport_mutex); + list_del(&t->t_node); + mutex_unlock(&transport_mutex); + + pr_debug("Unregistered iSCSI transport: %s\n", t->name); +} +EXPORT_SYMBOL(iscsit_unregister_transport); diff --git a/include/target/iscsi/iscsi_transport.h b/include/target/iscsi/iscsi_transport.h new file mode 100644 index 000000000000..23a87d0cd72c --- /dev/null +++ b/include/target/iscsi/iscsi_transport.h @@ -0,0 +1,83 @@ +#include +#include +#include "../../../drivers/target/iscsi/iscsi_target_core.h" + +struct iscsit_transport { +#define ISCSIT_TRANSPORT_NAME 16 + char name[ISCSIT_TRANSPORT_NAME]; + int transport_type; + struct module *owner; + struct list_head t_node; + int (*iscsit_setup_np)(struct iscsi_np *, struct __kernel_sockaddr_storage *); + int (*iscsit_accept_np)(struct iscsi_np *, struct iscsi_conn *); + void (*iscsit_free_np)(struct iscsi_np *); + void (*iscsit_free_conn)(struct iscsi_conn *); + struct iscsi_cmd *(*iscsit_alloc_cmd)(struct iscsi_conn *, gfp_t); + int (*iscsit_get_login_rx)(struct iscsi_conn *, struct iscsi_login *); + int (*iscsit_put_login_tx)(struct iscsi_conn *, struct iscsi_login *, u32); + int (*iscsit_immediate_queue)(struct iscsi_conn *, struct iscsi_cmd *, int); + int (*iscsit_response_queue)(struct iscsi_conn *, struct iscsi_cmd *, int); + int (*iscsit_get_dataout)(struct iscsi_conn *, struct iscsi_cmd *, bool); + int (*iscsit_queue_data_in)(struct iscsi_conn *, struct iscsi_cmd *); + int (*iscsit_queue_status)(struct iscsi_conn *, struct iscsi_cmd *); +}; + +/* + * From iscsi_target_transport.c + */ + +extern int iscsit_register_transport(struct iscsit_transport *); +extern void iscsit_unregister_transport(struct iscsit_transport *); +extern struct iscsit_transport *iscsit_get_transport(int); +extern void iscsit_put_transport(struct iscsit_transport *); + +/* + * From iscsi_target.c + */ +extern int iscsit_add_reject_from_cmd(u8, int, int, unsigned char *, + struct iscsi_cmd *); +extern int iscsit_setup_scsi_cmd(struct iscsi_conn *, struct iscsi_cmd *, + unsigned char *); +extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *); +extern int iscsit_process_scsi_cmd(struct iscsi_conn *, struct iscsi_cmd *, + struct iscsi_scsi_req *); +extern int iscsit_check_dataout_hdr(struct iscsi_conn *, unsigned char *, + struct iscsi_cmd **); +extern int iscsit_check_dataout_payload(struct iscsi_cmd *, struct iscsi_data *, + bool); +extern int iscsit_handle_nop_out(struct iscsi_conn *, struct iscsi_cmd *, + unsigned char *); +extern int iscsit_handle_logout_cmd(struct iscsi_conn *, struct iscsi_cmd *, + unsigned char *); +extern int iscsit_handle_task_mgt_cmd(struct iscsi_conn *, struct iscsi_cmd *, + unsigned char *); +extern void iscsit_build_rsp_pdu(struct iscsi_cmd *, struct iscsi_conn *, + bool, struct iscsi_scsi_rsp *); +extern void iscsit_build_nopin_rsp(struct iscsi_cmd *, struct iscsi_conn *, + struct iscsi_nopin *, bool); +extern void iscsit_build_task_mgt_rsp(struct iscsi_cmd *, struct iscsi_conn *, + struct iscsi_tm_rsp *); +extern void iscsit_build_reject(struct iscsi_cmd *, struct iscsi_conn *, + struct iscsi_reject *); +extern int iscsit_build_logout_rsp(struct iscsi_cmd *, struct iscsi_conn *, + struct iscsi_logout_rsp *); +extern int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *); +/* + * From iscsi_target_device.c + */ +extern void iscsit_increment_maxcmdsn(struct iscsi_cmd *, struct iscsi_session *); +/* + * From iscsi_target_erl1.c + */ +extern void iscsit_stop_dataout_timer(struct iscsi_cmd *); + +/* + * From iscsi_target_tmr.c + */ +extern int iscsit_tmr_post_handler(struct iscsi_cmd *, struct iscsi_conn *); + +/* + * From iscsi_target_util.c + */ +extern struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *, gfp_t); +extern int iscsit_sequence_cmd(struct iscsi_conn *, struct iscsi_cmd *, __be32); From baa4d64b147a4064626f6597646ae8a330d9e2ed Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 21:54:13 -0800 Subject: [PATCH 14/25] iscsi-target: Initial traditional TCP conversion to iscsit_transport This patch performs the initial conversion of existing traditional iscsi to use iscsit_transport API callers. This includes: - iscsi-np cleanups for iscsit_transport_type - Add iscsi-np transport calls w/ ->iscsit_setup_up() and ->iscsit_free_np() - Convert login thread process context to use ->iscsit_accept_np() for connections with pre-allocated struct iscsi_conn - Convert existing socket accept code to iscsit_accept_np() - Convert login RX/TX callers to use ->iscsit_get_login_rx() and ->iscsit_put_login_tx() to exchange request/response PDUs - Convert existing socket login RX/TX calls into iscsit_get_login_rx() and iscsit_put_login_tx() - Change iscsit_close_connection() to invoke ->iscsit_free_conn() + iscsit_put_transport() calls. - Add iscsit_register_transport() + iscsit_unregister_transport() calls to module init/exit v4 changes: - Add missing iscsit_put_transport() call in iscsi_target_setup_login_socket() failure case v2 changes: - Update module init/exit to use register_transport() + unregister_transport() Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target.c | 35 +- drivers/target/iscsi/iscsi_target_core.h | 15 +- drivers/target/iscsi/iscsi_target_login.c | 416 ++++++++++++------ drivers/target/iscsi/iscsi_target_login.h | 6 + drivers/target/iscsi/iscsi_target_nego.c | 167 +------ drivers/target/iscsi/iscsi_target_nego.h | 11 +- .../target/iscsi/iscsi_target_parameters.c | 12 +- drivers/target/iscsi/iscsi_target_tpg.c | 6 +- drivers/target/iscsi/iscsi_target_util.c | 27 +- 9 files changed, 380 insertions(+), 315 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 7ea246a07731..8203bf3de848 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -49,6 +49,8 @@ #include "iscsi_target_device.h" #include "iscsi_target_stat.h" +#include + static LIST_HEAD(g_tiqn_list); static LIST_HEAD(g_np_list); static DEFINE_SPINLOCK(tiqn_lock); @@ -401,8 +403,7 @@ struct iscsi_np *iscsit_add_np( spin_unlock_bh(&np_lock); pr_debug("CORE[0] - Added Network Portal: %s:%hu on %s\n", - np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? - "TCP" : "SCTP"); + np->np_ip, np->np_port, np->np_transport->name); return np; } @@ -441,11 +442,10 @@ int iscsit_reset_np_thread( return 0; } -static int iscsit_del_np_comm(struct iscsi_np *np) +static void iscsit_free_np(struct iscsi_np *np) { if (np->np_socket) sock_release(np->np_socket); - return 0; } int iscsit_del_np(struct iscsi_np *np) @@ -467,20 +467,32 @@ int iscsit_del_np(struct iscsi_np *np) send_sig(SIGINT, np->np_thread, 1); kthread_stop(np->np_thread); } - iscsit_del_np_comm(np); + + np->np_transport->iscsit_free_np(np); spin_lock_bh(&np_lock); list_del(&np->np_list); spin_unlock_bh(&np_lock); pr_debug("CORE[0] - Removed Network Portal: %s:%hu on %s\n", - np->np_ip, np->np_port, (np->np_network_transport == ISCSI_TCP) ? - "TCP" : "SCTP"); + np->np_ip, np->np_port, np->np_transport->name); + iscsit_put_transport(np->np_transport); kfree(np); return 0; } +static struct iscsit_transport iscsi_target_transport = { + .name = "iSCSI/TCP", + .transport_type = ISCSI_TCP, + .owner = NULL, + .iscsit_setup_np = iscsit_setup_np, + .iscsit_accept_np = iscsit_accept_np, + .iscsit_free_np = iscsit_free_np, + .iscsit_get_login_rx = iscsit_get_login_rx, + .iscsit_put_login_tx = iscsit_put_login_tx, +}; + static int __init iscsi_target_init_module(void) { int ret = 0; @@ -557,6 +569,8 @@ static int __init iscsi_target_init_module(void) goto ooo_out; } + iscsit_register_transport(&iscsi_target_transport); + if (iscsit_load_discovery_tpg() < 0) goto r2t_out; @@ -587,6 +601,7 @@ static void __exit iscsi_target_cleanup_module(void) iscsi_deallocate_thread_sets(); iscsi_thread_set_free(); iscsit_release_discovery_tpg(); + iscsit_unregister_transport(&iscsi_target_transport); kmem_cache_destroy(lio_cmd_cache); kmem_cache_destroy(lio_qr_cache); kmem_cache_destroy(lio_dr_cache); @@ -4053,6 +4068,12 @@ int iscsit_close_connection( if (conn->sock) sock_release(conn->sock); + + if (conn->conn_transport->iscsit_free_conn) + conn->conn_transport->iscsit_free_conn(conn); + + iscsit_put_transport(conn->conn_transport); + conn->thread_set = NULL; pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h index 7a333d28d9a2..258767776090 100644 --- a/drivers/target/iscsi/iscsi_target_core.h +++ b/drivers/target/iscsi/iscsi_target_core.h @@ -60,7 +60,7 @@ #define ISCSI_IOV_DATA_BUFFER 5 -enum tpg_np_network_transport_table { +enum iscsit_transport_type { ISCSI_TCP = 0, ISCSI_SCTP_TCP = 1, ISCSI_SCTP_UDP = 2, @@ -503,6 +503,7 @@ struct iscsi_conn { u16 login_port; u16 local_port; int net_size; + int login_family; u32 auth_id; u32 conn_flags; /* Used for iscsi_tx_login_rsp() */ @@ -562,9 +563,12 @@ struct iscsi_conn { struct list_head immed_queue_list; struct list_head response_queue_list; struct iscsi_conn_ops *conn_ops; + struct iscsi_login *conn_login; + struct iscsit_transport *conn_transport; struct iscsi_param_list *param_list; /* Used for per connection auth state machine */ void *auth_protocol; + void *context; struct iscsi_login_thread_s *login_thread; struct iscsi_portal_group *tpg; /* Pointer to parent session */ @@ -663,6 +667,8 @@ struct iscsi_login { u8 first_request; u8 version_min; u8 version_max; + u8 login_complete; + u8 login_failed; char isid[6]; u32 cmd_sn; itt_t init_task_tag; @@ -670,10 +676,11 @@ struct iscsi_login { u32 rsp_length; u16 cid; u16 tsih; - char *req; - char *rsp; + char req[ISCSI_HDR_LEN]; + char rsp[ISCSI_HDR_LEN]; char *req_buf; char *rsp_buf; + struct iscsi_conn *conn; } ____cacheline_aligned; struct iscsi_node_attrib { @@ -754,6 +761,8 @@ struct iscsi_np { struct task_struct *np_thread; struct timer_list np_login_timer; struct iscsi_portal_group *np_login_tpg; + void *np_context; + struct iscsit_transport *np_transport; struct list_head np_list; } ____cacheline_aligned; diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index 2535d4d46c0e..0de5c47d1c81 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -39,8 +39,39 @@ #include "iscsi_target.h" #include "iscsi_target_parameters.h" -static int iscsi_login_init_conn(struct iscsi_conn *conn) +#include + +static struct iscsi_login *iscsi_login_init_conn(struct iscsi_conn *conn) { + struct iscsi_login *login; + + login = kzalloc(sizeof(struct iscsi_login), GFP_KERNEL); + if (!login) { + pr_err("Unable to allocate memory for struct iscsi_login.\n"); + return NULL; + } + login->conn = conn; + login->first_request = 1; + + login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); + if (!login->req_buf) { + pr_err("Unable to allocate memory for response buffer.\n"); + goto out_login; + } + + login->rsp_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); + if (!login->rsp_buf) { + pr_err("Unable to allocate memory for request buffer.\n"); + goto out_req_buf; + } + + conn->conn_ops = kzalloc(sizeof(struct iscsi_conn_ops), GFP_KERNEL); + if (!conn->conn_ops) { + pr_err("Unable to allocate memory for" + " struct iscsi_conn_ops.\n"); + goto out_rsp_buf; + } + init_waitqueue_head(&conn->queues_wq); INIT_LIST_HEAD(&conn->conn_list); INIT_LIST_HEAD(&conn->conn_cmd_list); @@ -62,10 +93,21 @@ static int iscsi_login_init_conn(struct iscsi_conn *conn) if (!zalloc_cpumask_var(&conn->conn_cpumask, GFP_KERNEL)) { pr_err("Unable to allocate conn->conn_cpumask\n"); - return -ENOMEM; + goto out_conn_ops; } + conn->conn_login = login; - return 0; + return login; + +out_conn_ops: + kfree(conn->conn_ops); +out_rsp_buf: + kfree(login->rsp_buf); +out_req_buf: + kfree(login->req_buf); +out_login: + kfree(login); + return NULL; } /* @@ -573,10 +615,13 @@ int iscsi_login_post_auth_non_zero_tsih( static void iscsi_post_login_start_timers(struct iscsi_conn *conn) { +#warning FIXME: Reenable iscsit_start_nopin_timer +#if 0 struct iscsi_session *sess = conn->sess; if (!sess->sess_ops->SessionType) iscsit_start_nopin_timer(conn); +#endif } static int iscsi_post_login_handler( @@ -632,7 +677,13 @@ static int iscsi_post_login_handler( spin_unlock_bh(&sess->conn_lock); iscsi_post_login_start_timers(conn); - iscsi_activate_thread_set(conn, ts); + + if (conn->conn_transport == ISCSI_TCP) { + iscsi_activate_thread_set(conn, ts); + } else { + printk("Not calling iscsi_activate_thread_set....\n"); + dump_stack(); + } /* * Determine CPU mask to ensure connection's RX and TX kthreads * are scheduled on the same CPU. @@ -761,11 +812,11 @@ static void iscsi_stop_login_thread_timer(struct iscsi_np *np) spin_unlock_bh(&np->np_thread_lock); } -int iscsi_target_setup_login_socket( +int iscsit_setup_np( struct iscsi_np *np, struct __kernel_sockaddr_storage *sockaddr) { - struct socket *sock; + struct socket *sock = NULL; int backlog = 5, ret, opt = 0, len; switch (np->np_network_transport) { @@ -781,15 +832,15 @@ int iscsi_target_setup_login_socket( np->np_ip_proto = IPPROTO_SCTP; np->np_sock_type = SOCK_SEQPACKET; break; - case ISCSI_IWARP_TCP: - case ISCSI_IWARP_SCTP: - case ISCSI_INFINIBAND: default: pr_err("Unsupported network_transport: %d\n", np->np_network_transport); return -EINVAL; } + np->np_ip_proto = IPPROTO_TCP; + np->np_sock_type = SOCK_STREAM; + ret = sock_create(sockaddr->ss_family, np->np_sock_type, np->np_ip_proto, &sock); if (ret < 0) { @@ -853,7 +904,6 @@ int iscsi_target_setup_login_socket( } return 0; - fail: np->np_socket = NULL; if (sock) @@ -861,21 +911,170 @@ int iscsi_target_setup_login_socket( return ret; } +int iscsi_target_setup_login_socket( + struct iscsi_np *np, + struct __kernel_sockaddr_storage *sockaddr) +{ + struct iscsit_transport *t; + int rc; + + t = iscsit_get_transport(np->np_network_transport); + if (!t) + return -EINVAL; + + rc = t->iscsit_setup_np(np, sockaddr); + if (rc < 0) { + iscsit_put_transport(t); + return rc; + } + + np->np_transport = t; + printk("Set np->np_transport to %p -> %s\n", np->np_transport, + np->np_transport->name); + return 0; +} + +int iscsit_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) +{ + struct socket *new_sock, *sock = np->np_socket; + struct sockaddr_in sock_in; + struct sockaddr_in6 sock_in6; + int rc, err; + + rc = kernel_accept(sock, &new_sock, 0); + if (rc < 0) + return rc; + + conn->sock = new_sock; + conn->login_family = np->np_sockaddr.ss_family; + printk("iSCSI/TCP: Setup conn->sock from new_sock: %p\n", new_sock); + + if (np->np_sockaddr.ss_family == AF_INET6) { + memset(&sock_in6, 0, sizeof(struct sockaddr_in6)); + + rc = conn->sock->ops->getname(conn->sock, + (struct sockaddr *)&sock_in6, &err, 1); + if (!rc) { + snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", + &sock_in6.sin6_addr.in6_u); + conn->login_port = ntohs(sock_in6.sin6_port); + } + + rc = conn->sock->ops->getname(conn->sock, + (struct sockaddr *)&sock_in6, &err, 0); + if (!rc) { + snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", + &sock_in6.sin6_addr.in6_u); + conn->local_port = ntohs(sock_in6.sin6_port); + } + } else { + memset(&sock_in, 0, sizeof(struct sockaddr_in)); + + rc = conn->sock->ops->getname(conn->sock, + (struct sockaddr *)&sock_in, &err, 1); + if (!rc) { + sprintf(conn->login_ip, "%pI4", + &sock_in.sin_addr.s_addr); + conn->login_port = ntohs(sock_in.sin_port); + } + + rc = conn->sock->ops->getname(conn->sock, + (struct sockaddr *)&sock_in, &err, 0); + if (!rc) { + sprintf(conn->local_ip, "%pI4", + &sock_in.sin_addr.s_addr); + conn->local_port = ntohs(sock_in.sin_port); + } + } + + return 0; +} + +int iscsit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) +{ + struct iscsi_login_req *login_req; + u32 padding = 0, payload_length; + + if (iscsi_login_rx_data(conn, login->req, ISCSI_HDR_LEN) < 0) + return -1; + + login_req = (struct iscsi_login_req *)login->req; + payload_length = ntoh24(login_req->dlength); + padding = ((-payload_length) & 3); + + pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," + " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", + login_req->flags, login_req->itt, login_req->cmdsn, + login_req->exp_statsn, login_req->cid, payload_length); + /* + * Setup the initial iscsi_login values from the leading + * login request PDU. + */ + if (login->first_request) { + login_req = (struct iscsi_login_req *)login->req; + login->leading_connection = (!login_req->tsih) ? 1 : 0; + login->current_stage = + (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2; + login->version_min = login_req->min_version; + login->version_max = login_req->max_version; + memcpy(login->isid, login_req->isid, 6); + login->cmd_sn = be32_to_cpu(login_req->cmdsn); + login->init_task_tag = login_req->itt; + login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); + login->cid = be16_to_cpu(login_req->cid); + login->tsih = be16_to_cpu(login_req->tsih); + } + + if (iscsi_target_check_login_request(conn, login) < 0) + return -1; + + memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); + if (iscsi_login_rx_data(conn, login->req_buf, + payload_length + padding) < 0) + return -1; + + return 0; +} + +int iscsit_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, + u32 length) +{ + if (iscsi_login_tx_data(conn, login->rsp, login->rsp_buf, length) < 0) + return -1; + + return 0; +} + +static int +iscsit_conn_set_transport(struct iscsi_conn *conn, struct iscsit_transport *t) +{ + int rc; + + if (!t->owner) { + conn->conn_transport = t; + return 0; + } + + rc = try_module_get(t->owner); + if (!rc) { + pr_err("try_module_get() failed for %s\n", t->name); + return -EINVAL; + } + + conn->conn_transport = t; + return 0; +} + static int __iscsi_target_login_thread(struct iscsi_np *np) { - u8 buffer[ISCSI_HDR_LEN], iscsi_opcode, zero_tsih = 0; - int err, ret = 0, stop; + u8 *buffer, zero_tsih = 0; + int ret = 0, rc, stop; struct iscsi_conn *conn = NULL; struct iscsi_login *login; struct iscsi_portal_group *tpg = NULL; - struct socket *new_sock, *sock; - struct kvec iov; struct iscsi_login_req *pdu; - struct sockaddr_in sock_in; - struct sockaddr_in6 sock_in6; flush_signals(current); - sock = np->np_socket; spin_lock_bh(&np->np_thread_lock); if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { @@ -886,75 +1085,76 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) } spin_unlock_bh(&np->np_thread_lock); - if (kernel_accept(sock, &new_sock, 0) < 0) { - spin_lock_bh(&np->np_thread_lock); - if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { - spin_unlock_bh(&np->np_thread_lock); - complete(&np->np_restart_comp); - /* Get another socket */ - return 1; - } - spin_unlock_bh(&np->np_thread_lock); - goto out; - } - iscsi_start_login_thread_timer(np); - conn = kzalloc(sizeof(struct iscsi_conn), GFP_KERNEL); if (!conn) { pr_err("Could not allocate memory for" " new connection\n"); - sock_release(new_sock); /* Get another socket */ return 1; } - pr_debug("Moving to TARG_CONN_STATE_FREE.\n"); conn->conn_state = TARG_CONN_STATE_FREE; - conn->sock = new_sock; - pr_debug("Moving to TARG_CONN_STATE_XPT_UP.\n"); - conn->conn_state = TARG_CONN_STATE_XPT_UP; + if (iscsit_conn_set_transport(conn, np->np_transport) < 0) { + kfree(conn); + return 1; + } - /* - * Allocate conn->conn_ops early as a failure calling - * iscsit_tx_login_rsp() below will call tx_data(). - */ - conn->conn_ops = kzalloc(sizeof(struct iscsi_conn_ops), GFP_KERNEL); - if (!conn->conn_ops) { - pr_err("Unable to allocate memory for" - " struct iscsi_conn_ops.\n"); - goto new_sess_out; + rc = np->np_transport->iscsit_accept_np(np, conn); + if (rc == -ENOSYS) { + complete(&np->np_restart_comp); + iscsit_put_transport(conn->conn_transport); + kfree(conn); + conn = NULL; + goto exit; + } else if (rc < 0) { + spin_lock_bh(&np->np_thread_lock); + if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { + spin_unlock_bh(&np->np_thread_lock); + complete(&np->np_restart_comp); + if (ret == -ENODEV) { + iscsit_put_transport(conn->conn_transport); + kfree(conn); + conn = NULL; + goto out; + } + /* Get another socket */ + return 1; + } + spin_unlock_bh(&np->np_thread_lock); + iscsit_put_transport(conn->conn_transport); + kfree(conn); + conn = NULL; + goto out; } /* * Perform the remaining iSCSI connection initialization items.. */ - if (iscsi_login_init_conn(conn) < 0) - goto new_sess_out; - - memset(buffer, 0, ISCSI_HDR_LEN); - memset(&iov, 0, sizeof(struct kvec)); - iov.iov_base = buffer; - iov.iov_len = ISCSI_HDR_LEN; - - if (rx_data(conn, &iov, 1, ISCSI_HDR_LEN) <= 0) { - pr_err("rx_data() returned an error.\n"); + login = iscsi_login_init_conn(conn); + if (!login) { goto new_sess_out; } - iscsi_opcode = (buffer[0] & ISCSI_OPCODE_MASK); - if (!(iscsi_opcode & ISCSI_OP_LOGIN)) { - pr_err("First opcode is not login request," - " failing login request.\n"); + iscsi_start_login_thread_timer(np); + + pr_debug("Moving to TARG_CONN_STATE_XPT_UP.\n"); + conn->conn_state = TARG_CONN_STATE_XPT_UP; + /* + * This will process the first login request + payload.. + */ + rc = np->np_transport->iscsit_get_login_rx(conn, login); + if (rc == 1) + return 1; + else if (rc < 0) goto new_sess_out; - } - - pdu = (struct iscsi_login_req *) buffer; + buffer = &login->req[0]; + pdu = (struct iscsi_login_req *)buffer; /* * Used by iscsit_tx_login_rsp() for Login Resonses PDUs * when Status-Class != 0. */ - conn->login_itt = pdu->itt; + conn->login_itt = pdu->itt; spin_lock_bh(&np->np_thread_lock); if (np->np_thread_state != ISCSI_NP_THREAD_ACTIVE) { @@ -967,61 +1167,11 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) } spin_unlock_bh(&np->np_thread_lock); - if (np->np_sockaddr.ss_family == AF_INET6) { - memset(&sock_in6, 0, sizeof(struct sockaddr_in6)); - - if (conn->sock->ops->getname(conn->sock, - (struct sockaddr *)&sock_in6, &err, 1) < 0) { - pr_err("sock_ops->getname() failed.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_TARGET_ERROR); - goto new_sess_out; - } - snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", - &sock_in6.sin6_addr.in6_u); - conn->login_port = ntohs(sock_in6.sin6_port); - - if (conn->sock->ops->getname(conn->sock, - (struct sockaddr *)&sock_in6, &err, 0) < 0) { - pr_err("sock_ops->getname() failed.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_TARGET_ERROR); - goto new_sess_out; - } - snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", - &sock_in6.sin6_addr.in6_u); - conn->local_port = ntohs(sock_in6.sin6_port); - - } else { - memset(&sock_in, 0, sizeof(struct sockaddr_in)); - - if (conn->sock->ops->getname(conn->sock, - (struct sockaddr *)&sock_in, &err, 1) < 0) { - pr_err("sock_ops->getname() failed.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_TARGET_ERROR); - goto new_sess_out; - } - sprintf(conn->login_ip, "%pI4", &sock_in.sin_addr.s_addr); - conn->login_port = ntohs(sock_in.sin_port); - - if (conn->sock->ops->getname(conn->sock, - (struct sockaddr *)&sock_in, &err, 0) < 0) { - pr_err("sock_ops->getname() failed.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_TARGET_ERROR); - goto new_sess_out; - } - sprintf(conn->local_ip, "%pI4", &sock_in.sin_addr.s_addr); - conn->local_port = ntohs(sock_in.sin_port); - } - conn->network_transport = np->np_network_transport; pr_debug("Received iSCSI login request from %s on %s Network" - " Portal %s:%hu\n", conn->login_ip, - (conn->network_transport == ISCSI_TCP) ? "TCP" : "SCTP", - conn->local_ip, conn->local_port); + " Portal %s:%hu\n", conn->login_ip, np->np_transport->name, + conn->local_ip, conn->local_port); pr_debug("Moving to TARG_CONN_STATE_IN_LOGIN.\n"); conn->conn_state = TARG_CONN_STATE_IN_LOGIN; @@ -1050,13 +1200,17 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) if (iscsi_login_non_zero_tsih_s1(conn, buffer) < 0) goto new_sess_out; } - /* - * This will process the first login request, and call - * iscsi_target_locate_portal(), and return a valid struct iscsi_login. + * SessionType: Discovery + * + * Locates Default Portal + * + * SessionType: Normal + * + * Locates Target Portal from NP -> Target IQN */ - login = iscsi_target_init_negotiation(np, conn, buffer); - if (!login) { + rc = iscsi_target_locate_portal(np, conn, login); + if (rc < 0) { tpg = conn->tpg; goto new_sess_out; } @@ -1068,15 +1222,11 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) } if (zero_tsih) { - if (iscsi_login_zero_tsih_s2(conn) < 0) { - iscsi_target_nego_release(login, conn); + if (iscsi_login_zero_tsih_s2(conn) < 0) goto new_sess_out; - } } else { - if (iscsi_login_non_zero_tsih_s2(conn, buffer) < 0) { - iscsi_target_nego_release(login, conn); + if (iscsi_login_non_zero_tsih_s2(conn, buffer) < 0) goto old_sess_out; - } } if (iscsi_target_start_negotiation(login, conn) < 0) @@ -1153,8 +1303,18 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) iscsi_release_param_list(conn->param_list); conn->param_list = NULL; } - if (conn->sock) + iscsi_target_nego_release(conn); + + if (conn->sock) { sock_release(conn->sock); + conn->sock = NULL; + } + + if (conn->conn_transport->iscsit_free_conn) + conn->conn_transport->iscsit_free_conn(conn); + + iscsit_put_transport(conn->conn_transport); + kfree(conn); if (tpg) { @@ -1172,11 +1332,13 @@ static int __iscsi_target_login_thread(struct iscsi_np *np) /* Wait for another socket.. */ if (!stop) return 1; - +exit: iscsi_stop_login_thread_timer(np); spin_lock_bh(&np->np_thread_lock); np->np_thread_state = ISCSI_NP_THREAD_EXIT; + np->np_thread = NULL; spin_unlock_bh(&np->np_thread_lock); + return 0; } diff --git a/drivers/target/iscsi/iscsi_target_login.h b/drivers/target/iscsi/iscsi_target_login.h index 091dcae2532b..63efd2878451 100644 --- a/drivers/target/iscsi/iscsi_target_login.h +++ b/drivers/target/iscsi/iscsi_target_login.h @@ -4,8 +4,14 @@ extern int iscsi_login_setup_crypto(struct iscsi_conn *); extern int iscsi_check_for_session_reinstatement(struct iscsi_conn *); extern int iscsi_login_post_auth_non_zero_tsih(struct iscsi_conn *, u16, u32); +extern int iscsit_setup_np(struct iscsi_np *, + struct __kernel_sockaddr_storage *); extern int iscsi_target_setup_login_socket(struct iscsi_np *, struct __kernel_sockaddr_storage *); +extern int iscsit_accept_np(struct iscsi_np *, struct iscsi_conn *); +extern int iscsit_get_login_rx(struct iscsi_conn *, struct iscsi_login *); +extern int iscsit_put_login_tx(struct iscsi_conn *, struct iscsi_login *, u32); +extern void iscsit_free_conn(struct iscsi_np *, struct iscsi_conn *); extern int iscsi_target_login_thread(void *); extern int iscsi_login_disable_FIM_keys(struct iscsi_param_list *, struct iscsi_conn *); diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c index 3cbdccd77e0b..879a0cb44bc3 100644 --- a/drivers/target/iscsi/iscsi_target_nego.c +++ b/drivers/target/iscsi/iscsi_target_nego.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "iscsi_target_core.h" #include "iscsi_target_parameters.h" @@ -169,7 +170,7 @@ static void iscsi_remove_failed_auth_entry(struct iscsi_conn *conn) kfree(conn->auth_protocol); } -static int iscsi_target_check_login_request( +int iscsi_target_check_login_request( struct iscsi_conn *conn, struct iscsi_login *login) { @@ -352,11 +353,8 @@ static int iscsi_target_do_tx_login_io(struct iscsi_conn *conn, struct iscsi_log padding = ((-login->rsp_length) & 3); - if (iscsi_login_tx_data( - conn, - login->rsp, - login->rsp_buf, - login->rsp_length + padding) < 0) + if (conn->conn_transport->iscsit_put_login_tx(conn, login, + login->rsp_length + padding) < 0) return -1; login->rsp_length = 0; @@ -368,72 +366,12 @@ static int iscsi_target_do_tx_login_io(struct iscsi_conn *conn, struct iscsi_log return 0; } -static int iscsi_target_do_rx_login_io(struct iscsi_conn *conn, struct iscsi_login *login) -{ - u32 padding = 0, payload_length; - struct iscsi_login_req *login_req; - - if (iscsi_login_rx_data(conn, login->req, ISCSI_HDR_LEN) < 0) - return -1; - - login_req = (struct iscsi_login_req *) login->req; - payload_length = ntoh24(login_req->dlength); - - pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," - " CmdSN: 0x%08x, ExpStatSN: 0x%08x, CID: %hu, Length: %u\n", - login_req->flags, login_req->itt, login_req->cmdsn, - login_req->exp_statsn, login_req->cid, payload_length); - - if (iscsi_target_check_login_request(conn, login) < 0) - return -1; - - padding = ((-payload_length) & 3); - memset(login->req_buf, 0, MAX_KEY_VALUE_PAIRS); - - if (iscsi_login_rx_data( - conn, - login->req_buf, - payload_length + padding) < 0) - return -1; - - return 0; -} - static int iscsi_target_do_login_io(struct iscsi_conn *conn, struct iscsi_login *login) { if (iscsi_target_do_tx_login_io(conn, login) < 0) return -1; - if (iscsi_target_do_rx_login_io(conn, login) < 0) - return -1; - - return 0; -} - -static int iscsi_target_get_initial_payload( - struct iscsi_conn *conn, - struct iscsi_login *login) -{ - u32 padding = 0, payload_length; - struct iscsi_login_req *login_req; - - login_req = (struct iscsi_login_req *) login->req; - payload_length = ntoh24(login_req->dlength); - - pr_debug("Got Login Command, Flags 0x%02x, ITT: 0x%08x," - " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", - login_req->flags, login_req->itt, login_req->cmdsn, - login_req->exp_statsn, payload_length); - - if (iscsi_target_check_login_request(conn, login) < 0) - return -1; - - padding = ((-payload_length) & 3); - - if (iscsi_login_rx_data( - conn, - login->req_buf, - payload_length + padding) < 0) + if (conn->conn_transport->iscsit_get_login_rx(conn, login) < 0) return -1; return 0; @@ -693,6 +631,7 @@ static int iscsi_target_do_login(struct iscsi_conn *conn, struct iscsi_login *lo return -1; if (login_rsp->flags & ISCSI_FLAG_LOGIN_TRANSIT) { login->tsih = conn->sess->tsih; + login->login_complete = 1; if (iscsi_target_do_tx_login_io(conn, login) < 0) return -1; @@ -736,7 +675,7 @@ static void iscsi_initiatorname_tolower( /* * Processes the first Login Request.. */ -static int iscsi_target_locate_portal( +int iscsi_target_locate_portal( struct iscsi_np *np, struct iscsi_conn *conn, struct iscsi_login *login) @@ -798,6 +737,8 @@ static int iscsi_target_locate_portal( start += strlen(key) + strlen(value) + 2; } + printk("i_buf: %s, s_buf: %s, t_buf: %s\n", i_buf, s_buf, t_buf); + /* * See 5.3. Login Phase. */ @@ -956,100 +897,30 @@ static int iscsi_target_locate_portal( return ret; } -struct iscsi_login *iscsi_target_init_negotiation( - struct iscsi_np *np, - struct iscsi_conn *conn, - char *login_pdu) -{ - struct iscsi_login *login; - - login = kzalloc(sizeof(struct iscsi_login), GFP_KERNEL); - if (!login) { - pr_err("Unable to allocate memory for struct iscsi_login.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_NO_RESOURCES); - return NULL; - } - - login->req = kmemdup(login_pdu, ISCSI_HDR_LEN, GFP_KERNEL); - if (!login->req) { - pr_err("Unable to allocate memory for Login Request.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_NO_RESOURCES); - goto out; - } - - login->req_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); - if (!login->req_buf) { - pr_err("Unable to allocate memory for response buffer.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_NO_RESOURCES); - goto out; - } - /* - * SessionType: Discovery - * - * Locates Default Portal - * - * SessionType: Normal - * - * Locates Target Portal from NP -> Target IQN - */ - if (iscsi_target_locate_portal(np, conn, login) < 0) { - goto out; - } - - return login; -out: - kfree(login->req); - kfree(login->req_buf); - kfree(login); - - return NULL; -} - int iscsi_target_start_negotiation( struct iscsi_login *login, struct iscsi_conn *conn) { - int ret = -1; - - login->rsp = kzalloc(ISCSI_HDR_LEN, GFP_KERNEL); - if (!login->rsp) { - pr_err("Unable to allocate memory for" - " Login Response.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_NO_RESOURCES); - ret = -1; - goto out; - } - - login->rsp_buf = kzalloc(MAX_KEY_VALUE_PAIRS, GFP_KERNEL); - if (!login->rsp_buf) { - pr_err("Unable to allocate memory for" - " request buffer.\n"); - iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, - ISCSI_LOGIN_STATUS_NO_RESOURCES); - ret = -1; - goto out; - } + int ret; ret = iscsi_target_do_login(conn, login); -out: if (ret != 0) iscsi_remove_failed_auth_entry(conn); - iscsi_target_nego_release(login, conn); + iscsi_target_nego_release(conn); return ret; } -void iscsi_target_nego_release( - struct iscsi_login *login, - struct iscsi_conn *conn) +void iscsi_target_nego_release(struct iscsi_conn *conn) { - kfree(login->req); - kfree(login->rsp); + struct iscsi_login *login = conn->conn_login; + + if (!login) + return; + kfree(login->req_buf); kfree(login->rsp_buf); kfree(login); + + conn->conn_login = NULL; } diff --git a/drivers/target/iscsi/iscsi_target_nego.h b/drivers/target/iscsi/iscsi_target_nego.h index 92e133a5158f..f021cbd330e5 100644 --- a/drivers/target/iscsi/iscsi_target_nego.h +++ b/drivers/target/iscsi/iscsi_target_nego.h @@ -7,11 +7,14 @@ extern void convert_null_to_semi(char *, int); extern int extract_param(const char *, const char *, unsigned int, char *, unsigned char *); -extern struct iscsi_login *iscsi_target_init_negotiation( - struct iscsi_np *, struct iscsi_conn *, char *); +extern int iscsi_target_check_login_request(struct iscsi_conn *, + struct iscsi_login *); +extern int iscsi_target_get_initial_payload(struct iscsi_conn *, + struct iscsi_login *); +extern int iscsi_target_locate_portal(struct iscsi_np *, struct iscsi_conn *, + struct iscsi_login *); extern int iscsi_target_start_negotiation( struct iscsi_login *, struct iscsi_conn *); -extern void iscsi_target_nego_release( - struct iscsi_login *, struct iscsi_conn *); +extern void iscsi_target_nego_release(struct iscsi_conn *); #endif /* ISCSI_TARGET_NEGO_H */ diff --git a/drivers/target/iscsi/iscsi_target_parameters.c b/drivers/target/iscsi/iscsi_target_parameters.c index ca2be406f141..84ce94a14948 100644 --- a/drivers/target/iscsi/iscsi_target_parameters.c +++ b/drivers/target/iscsi/iscsi_target_parameters.c @@ -59,7 +59,7 @@ int iscsi_login_tx_data( char *text_buf, int text_length) { - int length, tx_sent; + int length, tx_sent, iov_cnt = 1; struct kvec iov[2]; length = (ISCSI_HDR_LEN + text_length); @@ -67,8 +67,12 @@ int iscsi_login_tx_data( memset(&iov[0], 0, 2 * sizeof(struct kvec)); iov[0].iov_len = ISCSI_HDR_LEN; iov[0].iov_base = pdu_buf; - iov[1].iov_len = text_length; - iov[1].iov_base = text_buf; + + if (text_buf && text_length) { + iov[1].iov_len = text_length; + iov[1].iov_base = text_buf; + iov_cnt++; + } /* * Initial Marker-less Interval. @@ -77,7 +81,7 @@ int iscsi_login_tx_data( */ conn->if_marker += length; - tx_sent = tx_data(conn, &iov[0], 2, length); + tx_sent = tx_data(conn, &iov[0], iov_cnt, length); if (tx_sent != length) { pr_err("tx_data returned %d, expecting %d.\n", tx_sent, length); diff --git a/drivers/target/iscsi/iscsi_target_tpg.c b/drivers/target/iscsi/iscsi_target_tpg.c index ee8f8c66248d..439260b7d87f 100644 --- a/drivers/target/iscsi/iscsi_target_tpg.c +++ b/drivers/target/iscsi/iscsi_target_tpg.c @@ -31,6 +31,8 @@ #include "iscsi_target.h" #include "iscsi_target_parameters.h" +#include + struct iscsi_portal_group *iscsit_alloc_portal_group(struct iscsi_tiqn *tiqn, u16 tpgt) { struct iscsi_portal_group *tpg; @@ -508,7 +510,7 @@ struct iscsi_tpg_np *iscsit_tpg_add_network_portal( pr_debug("CORE[%s] - Added Network Portal: %s:%hu,%hu on %s\n", tpg->tpg_tiqn->tiqn, np->np_ip, np->np_port, tpg->tpgt, - (np->np_network_transport == ISCSI_TCP) ? "TCP" : "SCTP"); + np->np_transport->name); return tpg_np; } @@ -522,7 +524,7 @@ static int iscsit_tpg_release_np( pr_debug("CORE[%s] - Removed Network Portal: %s:%hu,%hu on %s\n", tpg->tpg_tiqn->tiqn, np->np_ip, np->np_port, tpg->tpgt, - (np->np_network_transport == ISCSI_TCP) ? "TCP" : "SCTP"); + np->np_transport->name); tpg_np->tpg_np = NULL; tpg_np->tpg = NULL; diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c index 7ce350578c82..4cf1e7f94ac3 100644 --- a/drivers/target/iscsi/iscsi_target_util.c +++ b/drivers/target/iscsi/iscsi_target_util.c @@ -24,6 +24,7 @@ #include #include #include +#include #include "iscsi_target_core.h" #include "iscsi_target_parameters.h" @@ -1226,34 +1227,19 @@ int iscsit_fe_sendpage_sg( */ int iscsit_tx_login_rsp(struct iscsi_conn *conn, u8 status_class, u8 status_detail) { - u8 iscsi_hdr[ISCSI_HDR_LEN]; - int err; - struct kvec iov; struct iscsi_login_rsp *hdr; + struct iscsi_login *login = conn->conn_login; + login->login_failed = 1; iscsit_collect_login_stats(conn, status_class, status_detail); - memset(&iov, 0, sizeof(struct kvec)); - memset(&iscsi_hdr, 0x0, ISCSI_HDR_LEN); - - hdr = (struct iscsi_login_rsp *)&iscsi_hdr; + hdr = (struct iscsi_login_rsp *)&login->rsp[0]; hdr->opcode = ISCSI_OP_LOGIN_RSP; hdr->status_class = status_class; hdr->status_detail = status_detail; hdr->itt = conn->login_itt; - iov.iov_base = &iscsi_hdr; - iov.iov_len = ISCSI_HDR_LEN; - - PRINT_BUFF(iscsi_hdr, ISCSI_HDR_LEN); - - err = tx_data(conn, &iov, 1, ISCSI_HDR_LEN); - if (err != ISCSI_HDR_LEN) { - pr_err("tx_data returned less than expected\n"); - return -1; - } - - return 0; + return conn->conn_transport->iscsit_put_login_tx(conn, login, 0); } void iscsit_print_session_params(struct iscsi_session *sess) @@ -1432,7 +1418,8 @@ void iscsit_collect_login_stats( strcpy(ls->last_intr_fail_name, (intrname ? intrname->value : "Unknown")); - ls->last_intr_fail_ip_family = conn->sock->sk->sk_family; + ls->last_intr_fail_ip_family = conn->login_family; + snprintf(ls->last_intr_fail_ip_addr, IPV6_ADDRESS_SPACE, "%s", conn->login_ip); ls->last_fail_time = get_jiffies_64(); From 03aa207060e395282c5ea0c055e2050b35803b4d Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 21:54:38 -0800 Subject: [PATCH 15/25] iscsi-target: Add iser-target parameter keys + setup during login This patch adds RDMAExtensions, InitiatorRecvDataSegmentLength and TargetRecvDataSegmentLength parameters keys necessary for iser-target login to occur. This includes setting the necessary parameters during login path code within iscsi_login_zero_tsih_s2(), and currently PAGE_SIZE aligning the target's advertised MRDSL for immediate data and unsolicited data-out incoming payloads. v3 changes: - Add iscsi_post_login_start_timers FIXME for ISER v2 changes: - Fix RDMAExtentions -> RDMAExtensions typo (andy) - Drop unnecessary '== true' conditional checks for type bool Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target_core.h | 10 +++ drivers/target/iscsi/iscsi_target_login.c | 77 ++++++++++++++++--- .../target/iscsi/iscsi_target_parameters.c | 75 ++++++++++++++++-- .../target/iscsi/iscsi_target_parameters.h | 16 +++- 4 files changed, 161 insertions(+), 17 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h index 258767776090..53400b0bf711 100644 --- a/drivers/target/iscsi/iscsi_target_core.h +++ b/drivers/target/iscsi/iscsi_target_core.h @@ -244,6 +244,11 @@ struct iscsi_conn_ops { u8 IFMarker; /* [0,1] == [No,Yes] */ u32 OFMarkInt; /* [1..65535] */ u32 IFMarkInt; /* [1..65535] */ + /* + * iSER specific connection parameters + */ + u32 InitiatorRecvDataSegmentLength; /* [512..2**24-1] */ + u32 TargetRecvDataSegmentLength; /* [512..2**24-1] */ }; struct iscsi_sess_ops { @@ -265,6 +270,10 @@ struct iscsi_sess_ops { u8 DataSequenceInOrder; /* [0,1] == [No,Yes] */ u8 ErrorRecoveryLevel; /* [0..2] */ u8 SessionType; /* [0,1] == [Normal,Discovery]*/ + /* + * iSER specific session parameters + */ + u8 RDMAExtensions; /* [0,1] == [No,Yes] */ }; struct iscsi_queue_req { @@ -284,6 +293,7 @@ struct iscsi_data_count { }; struct iscsi_param_list { + bool iser; struct list_head param_list; struct list_head extra_response_list; }; diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index 0de5c47d1c81..e6a826927a40 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -340,6 +340,7 @@ static int iscsi_login_zero_tsih_s2( struct iscsi_node_attrib *na; struct iscsi_session *sess = conn->sess; unsigned char buf[32]; + bool iser = false; sess->tpg = conn->tpg; @@ -361,7 +362,10 @@ static int iscsi_login_zero_tsih_s2( return -1; } - iscsi_set_keys_to_negotiate(0, conn->param_list); + if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) + iser = true; + + iscsi_set_keys_to_negotiate(conn->param_list, iser); if (sess->sess_ops->SessionType) return iscsi_set_keys_irrelevant_for_discovery( @@ -399,6 +403,56 @@ static int iscsi_login_zero_tsih_s2( if (iscsi_login_disable_FIM_keys(conn->param_list, conn) < 0) return -1; + /* + * Set RDMAExtensions=Yes by default for iSER enabled network portals + */ + if (iser) { + struct iscsi_param *param; + unsigned long mrdsl, off; + int rc; + + sprintf(buf, "RDMAExtensions=Yes"); + if (iscsi_change_param_value(buf, conn->param_list, 0) < 0) { + iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, + ISCSI_LOGIN_STATUS_NO_RESOURCES); + return -1; + } + /* + * Make MaxRecvDataSegmentLength PAGE_SIZE aligned for + * Immediate Data + Unsolicitied Data-OUT if necessary.. + */ + param = iscsi_find_param_from_key("MaxRecvDataSegmentLength", + conn->param_list); + if (!param) { + iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, + ISCSI_LOGIN_STATUS_NO_RESOURCES); + return -1; + } + rc = strict_strtoul(param->value, 0, &mrdsl); + if (rc < 0) { + iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, + ISCSI_LOGIN_STATUS_NO_RESOURCES); + return -1; + } + off = mrdsl % PAGE_SIZE; + if (!off) + return 0; + + if (mrdsl < PAGE_SIZE) + mrdsl = PAGE_SIZE; + else + mrdsl -= off; + + pr_warn("Aligning ISER MaxRecvDataSegmentLength: %lu down" + " to PAGE_SIZE\n", mrdsl); + + sprintf(buf, "MaxRecvDataSegmentLength=%lu\n", mrdsl); + if (iscsi_change_param_value(buf, conn->param_list, 0) < 0) { + iscsit_tx_login_rsp(conn, ISCSI_STATUS_CLS_TARGET_ERR, + ISCSI_LOGIN_STATUS_NO_RESOURCES); + return -1; + } + } return 0; } @@ -478,6 +532,7 @@ static int iscsi_login_non_zero_tsih_s2( struct se_portal_group *se_tpg = &tpg->tpg_se_tpg; struct se_session *se_sess, *se_sess_tmp; struct iscsi_login_req *pdu = (struct iscsi_login_req *)buf; + bool iser = false; spin_lock_bh(&se_tpg->session_lock); list_for_each_entry_safe(se_sess, se_sess_tmp, &se_tpg->tpg_sess_list, @@ -527,7 +582,10 @@ static int iscsi_login_non_zero_tsih_s2( return -1; } - iscsi_set_keys_to_negotiate(0, conn->param_list); + if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) + iser = true; + + iscsi_set_keys_to_negotiate(conn->param_list, iser); /* * Need to send TargetPortalGroupTag back in first login response * on any iSCSI connection where the Initiator provides TargetName. @@ -615,13 +673,15 @@ int iscsi_login_post_auth_non_zero_tsih( static void iscsi_post_login_start_timers(struct iscsi_conn *conn) { -#warning FIXME: Reenable iscsit_start_nopin_timer -#if 0 struct iscsi_session *sess = conn->sess; + /* + * FIXME: Unsolicitied NopIN support for ISER + */ + if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) + return; if (!sess->sess_ops->SessionType) iscsit_start_nopin_timer(conn); -#endif } static int iscsi_post_login_handler( @@ -678,12 +738,7 @@ static int iscsi_post_login_handler( iscsi_post_login_start_timers(conn); - if (conn->conn_transport == ISCSI_TCP) { - iscsi_activate_thread_set(conn, ts); - } else { - printk("Not calling iscsi_activate_thread_set....\n"); - dump_stack(); - } + iscsi_activate_thread_set(conn, ts); /* * Determine CPU mask to ensure connection's RX and TX kthreads * are scheduled on the same CPU. diff --git a/drivers/target/iscsi/iscsi_target_parameters.c b/drivers/target/iscsi/iscsi_target_parameters.c index 84ce94a14948..f690be9e5293 100644 --- a/drivers/target/iscsi/iscsi_target_parameters.c +++ b/drivers/target/iscsi/iscsi_target_parameters.c @@ -433,6 +433,28 @@ int iscsi_create_default_params(struct iscsi_param_list **param_list_ptr) TYPERANGE_MARKINT, USE_INITIAL_ONLY); if (!param) goto out; + /* + * Extra parameters for ISER from RFC-5046 + */ + param = iscsi_set_default_param(pl, RDMAEXTENTIONS, INITIAL_RDMAEXTENTIONS, + PHASE_OPERATIONAL, SCOPE_SESSION_WIDE, SENDER_BOTH, + TYPERANGE_BOOL_AND, USE_LEADING_ONLY); + if (!param) + goto out; + + param = iscsi_set_default_param(pl, INITIATORRECVDATASEGMENTLENGTH, + INITIAL_INITIATORRECVDATASEGMENTLENGTH, + PHASE_OPERATIONAL, SCOPE_CONNECTION_ONLY, SENDER_BOTH, + TYPERANGE_512_TO_16777215, USE_ALL); + if (!param) + goto out; + + param = iscsi_set_default_param(pl, TARGETRECVDATASEGMENTLENGTH, + INITIAL_TARGETRECVDATASEGMENTLENGTH, + PHASE_OPERATIONAL, SCOPE_CONNECTION_ONLY, SENDER_BOTH, + TYPERANGE_512_TO_16777215, USE_ALL); + if (!param) + goto out; *param_list_ptr = pl; return 0; @@ -442,19 +464,23 @@ int iscsi_create_default_params(struct iscsi_param_list **param_list_ptr) } int iscsi_set_keys_to_negotiate( - int sessiontype, - struct iscsi_param_list *param_list) + struct iscsi_param_list *param_list, + bool iser) { struct iscsi_param *param; + param_list->iser = iser; + list_for_each_entry(param, ¶m_list->param_list, p_list) { param->state = 0; if (!strcmp(param->name, AUTHMETHOD)) { SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, HEADERDIGEST)) { - SET_PSTATE_NEGOTIATE(param); + if (iser == false) + SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, DATADIGEST)) { - SET_PSTATE_NEGOTIATE(param); + if (iser == false) + SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, MAXCONNECTIONS)) { SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, TARGETNAME)) { @@ -473,7 +499,8 @@ int iscsi_set_keys_to_negotiate( } else if (!strcmp(param->name, IMMEDIATEDATA)) { SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, MAXRECVDATASEGMENTLENGTH)) { - SET_PSTATE_NEGOTIATE(param); + if (iser == false) + SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, MAXXMITDATASEGMENTLENGTH)) { continue; } else if (!strcmp(param->name, MAXBURSTLENGTH)) { @@ -502,6 +529,15 @@ int iscsi_set_keys_to_negotiate( SET_PSTATE_NEGOTIATE(param); } else if (!strcmp(param->name, OFMARKINT)) { SET_PSTATE_NEGOTIATE(param); + } else if (!strcmp(param->name, RDMAEXTENTIONS)) { + if (iser == true) + SET_PSTATE_NEGOTIATE(param); + } else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH)) { + if (iser == true) + SET_PSTATE_NEGOTIATE(param); + } else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH)) { + if (iser == true) + SET_PSTATE_NEGOTIATE(param); } } @@ -544,6 +580,12 @@ int iscsi_set_keys_irrelevant_for_discovery( param->state &= ~PSTATE_NEGOTIATE; else if (!strcmp(param->name, OFMARKINT)) param->state &= ~PSTATE_NEGOTIATE; + else if (!strcmp(param->name, RDMAEXTENTIONS)) + param->state &= ~PSTATE_NEGOTIATE; + else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH)) + param->state &= ~PSTATE_NEGOTIATE; + else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH)) + param->state &= ~PSTATE_NEGOTIATE; } return 0; @@ -1759,6 +1801,9 @@ void iscsi_set_connection_parameters( * this key is not sent over the wire. */ if (!strcmp(param->name, MAXXMITDATASEGMENTLENGTH)) { + if (param_list->iser == true) + continue; + ops->MaxXmitDataSegmentLength = simple_strtoul(param->value, &tmpptr, 0); pr_debug("MaxXmitDataSegmentLength: %s\n", @@ -1804,6 +1849,22 @@ void iscsi_set_connection_parameters( simple_strtoul(param->value, &tmpptr, 0); pr_debug("IFMarkInt: %s\n", param->value); + } else if (!strcmp(param->name, INITIATORRECVDATASEGMENTLENGTH)) { + ops->InitiatorRecvDataSegmentLength = + simple_strtoul(param->value, &tmpptr, 0); + pr_debug("InitiatorRecvDataSegmentLength: %s\n", + param->value); + ops->MaxRecvDataSegmentLength = + ops->InitiatorRecvDataSegmentLength; + pr_debug("Set MRDSL from InitiatorRecvDataSegmentLength\n"); + } else if (!strcmp(param->name, TARGETRECVDATASEGMENTLENGTH)) { + ops->TargetRecvDataSegmentLength = + simple_strtoul(param->value, &tmpptr, 0); + pr_debug("TargetRecvDataSegmentLength: %s\n", + param->value); + ops->MaxXmitDataSegmentLength = + ops->TargetRecvDataSegmentLength; + pr_debug("Set MXDSL from TargetRecvDataSegmentLength\n"); } } pr_debug("----------------------------------------------------" @@ -1916,6 +1977,10 @@ void iscsi_set_session_parameters( ops->SessionType = !strcmp(param->value, DISCOVERY); pr_debug("SessionType: %s\n", param->value); + } else if (!strcmp(param->name, RDMAEXTENTIONS)) { + ops->RDMAExtensions = !strcmp(param->value, YES); + pr_debug("RDMAExtensions: %s\n", + param->value); } } pr_debug("----------------------------------------------------" diff --git a/drivers/target/iscsi/iscsi_target_parameters.h b/drivers/target/iscsi/iscsi_target_parameters.h index 1e1b7504a76b..f31b9c4b83f2 100644 --- a/drivers/target/iscsi/iscsi_target_parameters.h +++ b/drivers/target/iscsi/iscsi_target_parameters.h @@ -27,7 +27,7 @@ extern void iscsi_dump_conn_ops(struct iscsi_conn_ops *); extern void iscsi_dump_sess_ops(struct iscsi_sess_ops *); extern void iscsi_print_params(struct iscsi_param_list *); extern int iscsi_create_default_params(struct iscsi_param_list **); -extern int iscsi_set_keys_to_negotiate(int, struct iscsi_param_list *); +extern int iscsi_set_keys_to_negotiate(struct iscsi_param_list *, bool); extern int iscsi_set_keys_irrelevant_for_discovery(struct iscsi_param_list *); extern int iscsi_copy_param_list(struct iscsi_param_list **, struct iscsi_param_list *, int); @@ -88,6 +88,13 @@ extern void iscsi_set_session_parameters(struct iscsi_sess_ops *, #define X_EXTENSIONKEY_CISCO_NEW "X-com.cisco.protocol" #define X_EXTENSIONKEY_CISCO_OLD "X-com.cisco.iscsi.draft" +/* + * Parameter names of iSCSI Extentions for RDMA (iSER). See RFC-5046 + */ +#define RDMAEXTENTIONS "RDMAExtensions" +#define INITIATORRECVDATASEGMENTLENGTH "InitiatorRecvDataSegmentLength" +#define TARGETRECVDATASEGMENTLENGTH "TargetRecvDataSegmentLength" + /* * For AuthMethod. */ @@ -132,6 +139,13 @@ extern void iscsi_set_session_parameters(struct iscsi_sess_ops *, #define INITIAL_IFMARKINT "2048~65535" #define INITIAL_OFMARKINT "2048~65535" +/* + * Initial values for iSER parameters following RFC-5046 Section 6 + */ +#define INITIAL_RDMAEXTENTIONS NO +#define INITIAL_INITIATORRECVDATASEGMENTLENGTH "262144" +#define INITIAL_TARGETRECVDATASEGMENTLENGTH "8192" + /* * For [Header,Data]Digests. */ From cdb726651c357c19c02b17ee3c99a224b03022b6 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 22:09:17 -0800 Subject: [PATCH 16/25] iscsi-target: Add per transport iscsi_cmd alloc/free This patch converts struct iscsi_cmd memory allocation + free to use ->iscsit_alloc_cmd() iscsit_transport API caller, and export iscsit_allocate_cmd() symbols Also add iscsi_cmd->release_cmd() to be used seperately from iscsit_transport for connection/session shutdown. v2 changes: - Remove unnecessary checks in iscsit_alloc_cmd (asias) - Drop iscsit_transport->iscsit_free_cmd() usage - Drop iscsit_transport->iscsit_unmap_cmd() usage - Add iscsi_cmd->release_cmd() - Convert lio_release_cmd() to use iscsi_cmd->release_cmd() Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target.c | 1 + drivers/target/iscsi/iscsi_target_configfs.c | 3 ++- drivers/target/iscsi/iscsi_target_core.h | 1 + drivers/target/iscsi/iscsi_target_util.c | 25 ++++++++++++++++---- drivers/target/iscsi/iscsi_target_util.h | 1 + 5 files changed, 26 insertions(+), 5 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 8203bf3de848..b01a10eae365 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -489,6 +489,7 @@ static struct iscsit_transport iscsi_target_transport = { .iscsit_setup_np = iscsit_setup_np, .iscsit_accept_np = iscsit_accept_np, .iscsit_free_np = iscsit_free_np, + .iscsit_alloc_cmd = iscsit_alloc_cmd, .iscsit_get_login_rx = iscsit_get_login_rx, .iscsit_put_login_tx = iscsit_put_login_tx, }; diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c index 78d75c8567d0..c78b824194e6 100644 --- a/drivers/target/iscsi/iscsi_target_configfs.c +++ b/drivers/target/iscsi/iscsi_target_configfs.c @@ -1700,7 +1700,8 @@ static void lio_release_cmd(struct se_cmd *se_cmd) { struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd); - iscsit_release_cmd(cmd); + pr_debug("Entering lio_release_cmd for se_cmd: %p\n", se_cmd); + cmd->release_cmd(cmd); } /* End functions for target_core_fabric_ops */ diff --git a/drivers/target/iscsi/iscsi_target_core.h b/drivers/target/iscsi/iscsi_target_core.h index 53400b0bf711..60ec4b92be03 100644 --- a/drivers/target/iscsi/iscsi_target_core.h +++ b/drivers/target/iscsi/iscsi_target_core.h @@ -485,6 +485,7 @@ struct iscsi_cmd { u32 first_data_sg_off; u32 kmapped_nents; sense_reason_t sense_reason; + void (*release_cmd)(struct iscsi_cmd *); } ____cacheline_aligned; struct iscsi_tmr_req { diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c index 4cf1e7f94ac3..0b73f90c9c54 100644 --- a/drivers/target/iscsi/iscsi_target_util.c +++ b/drivers/target/iscsi/iscsi_target_util.c @@ -149,6 +149,18 @@ void iscsit_free_r2ts_from_list(struct iscsi_cmd *cmd) spin_unlock_bh(&cmd->r2t_lock); } +struct iscsi_cmd *iscsit_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp_mask) +{ + struct iscsi_cmd *cmd; + + cmd = kmem_cache_zalloc(lio_cmd_cache, gfp_mask); + if (!cmd) + return NULL; + + cmd->release_cmd = &iscsit_release_cmd; + return cmd; +} + /* * May be called from software interrupt (timer) context for allocating * iSCSI NopINs. @@ -157,13 +169,12 @@ struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp_mask) { struct iscsi_cmd *cmd; - cmd = kmem_cache_zalloc(lio_cmd_cache, gfp_mask); + cmd = conn->conn_transport->iscsit_alloc_cmd(conn, gfp_mask); if (!cmd) { pr_err("Unable to allocate memory for struct iscsi_cmd.\n"); return NULL; } - - cmd->conn = conn; + cmd->conn = conn; INIT_LIST_HEAD(&cmd->i_conn_node); INIT_LIST_HEAD(&cmd->datain_list); INIT_LIST_HEAD(&cmd->cmd_r2t_list); @@ -176,6 +187,7 @@ struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *conn, gfp_t gfp_mask) return cmd; } +EXPORT_SYMBOL(iscsit_allocate_cmd); struct iscsi_seq *iscsit_get_seq_holder_for_datain( struct iscsi_cmd *cmd, @@ -690,6 +702,11 @@ void iscsit_free_cmd(struct iscsi_cmd *cmd) */ switch (cmd->iscsi_opcode) { case ISCSI_OP_SCSI_CMD: + if (cmd->data_direction == DMA_TO_DEVICE) + iscsit_stop_dataout_timer(cmd); + /* + * Fallthrough + */ case ISCSI_OP_SCSI_TMFUNC: transport_generic_free_cmd(&cmd->se_cmd, 1); break; @@ -705,7 +722,7 @@ void iscsit_free_cmd(struct iscsi_cmd *cmd) } /* Fall-through */ default: - iscsit_release_cmd(cmd); + cmd->release_cmd(cmd); break; } } diff --git a/drivers/target/iscsi/iscsi_target_util.h b/drivers/target/iscsi/iscsi_target_util.h index 894d0f837924..4f8e01a47081 100644 --- a/drivers/target/iscsi/iscsi_target_util.h +++ b/drivers/target/iscsi/iscsi_target_util.h @@ -8,6 +8,7 @@ extern struct iscsi_r2t *iscsit_get_r2t_for_eos(struct iscsi_cmd *, u32, u32); extern struct iscsi_r2t *iscsit_get_r2t_from_list(struct iscsi_cmd *); extern void iscsit_free_r2t(struct iscsi_r2t *, struct iscsi_cmd *); extern void iscsit_free_r2ts_from_list(struct iscsi_cmd *); +extern struct iscsi_cmd *iscsit_alloc_cmd(struct iscsi_conn *, gfp_t); extern struct iscsi_cmd *iscsit_allocate_cmd(struct iscsi_conn *, gfp_t); extern struct iscsi_seq *iscsit_get_seq_holder_for_datain(struct iscsi_cmd *, u32); extern struct iscsi_seq *iscsit_get_seq_holder_for_r2t(struct iscsi_cmd *); From 3e1c81a95f0d776cea68a36cfc78a0ce8f6d3a77 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 22:18:24 -0800 Subject: [PATCH 17/25] iscsi-target: Refactor RX PDU logic + export request PDU handling This patch refactors existing traditional iscsi RX side PDU handling to use iscsit_transport, and exports the necessary logic for external transport modules. This includes: - Refactor iscsit_handle_scsi_cmd() into PDU setup / processing - Add updated iscsit_handle_scsi_cmd() for tradtional iscsi code - Add iscsit_set_unsoliticed_dataout() wrapper - Refactor iscsit_handle_data_out() into PDU check / processing - Add updated iscsit_handle_data_out() for tradtional iscsi code - Add iscsit_handle_nop_out() + iscsit_handle_task_mgt_cmd() to accept pre-allocated struct iscsi_cmd - Add iscsit_build_r2ts_for_cmd() caller for iscsi_target_transport to handle ISTATE_SEND_R2T for TX immediate queue - Refactor main traditional iscsi iscsi_target_rx_thread() PDU switch into iscsi_target_rx_opcode() using iscsit_allocate_cmd() - Turn iscsi_target_rx_thread() process context into NOP for ib_isert side work-queue. v5 changes: - Make iscsit_handle_scsi_cmd() static (Fengguang) - Fix iscsit_handle_scsi_cmd() exception se_cmd leak (nab) v3 changes: - Add extra target_put_sess_cmd call in iscsit_add_reject_from_cmd after completion v2 changes: - Disable iscsit_ack_from_expstatsn() usage for RDMAExtentions=Yes - Disable iscsit_allocate_datain_req() usage for RDMAExtentions=Yes - Add target_get_sess_cmd() reference counting to iscsit_setup_scsi_cmd() - Add TFO->lio_check_stop_free() fabric API caller - Add export of iscsit_stop_dataout_timer() symbol - Add iscsit_build_r2ts_for_cmd() for iscsit_transport->iscsit_get_dataout() - Convert existing usage of iscsit_build_r2ts_for_cmd() to ->iscsit_get_dataout() - Drop RDMAExtentions=Yes specific check in iscsit_build_r2ts_for_cmd() - Fix RDMAExtentions -> RDMAExtensions typo (andy) - Pass correct dump_payload value into iscsit_get_immediate_data() for iscsit_handle_scsi_cmd() Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target.c | 505 ++++++++++++------- drivers/target/iscsi/iscsi_target.h | 3 +- drivers/target/iscsi/iscsi_target_configfs.c | 9 +- drivers/target/iscsi/iscsi_target_erl1.c | 13 +- drivers/target/iscsi/iscsi_target_login.c | 3 +- drivers/target/iscsi/iscsi_target_nego.c | 15 - drivers/target/iscsi/iscsi_target_tmr.c | 3 +- drivers/target/iscsi/iscsi_target_util.c | 1 + 8 files changed, 347 insertions(+), 205 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index b01a10eae365..40bb916f57ff 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -492,6 +492,7 @@ static struct iscsit_transport iscsi_target_transport = { .iscsit_alloc_cmd = iscsit_alloc_cmd, .iscsit_get_login_rx = iscsit_get_login_rx, .iscsit_put_login_tx = iscsit_put_login_tx, + .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, }; static int __init iscsi_target_init_module(void) @@ -650,6 +651,14 @@ static int iscsit_add_reject( iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); ret = wait_for_completion_interruptible(&cmd->reject_comp); + /* + * Perform the kref_put now if se_cmd has been setup by + * iscsit_setup_scsi_cmd() + */ + if (cmd->se_cmd.se_tfo != NULL) { + pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); + } if (ret != 0) return -1; @@ -698,11 +707,20 @@ int iscsit_add_reject_from_cmd( iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); ret = wait_for_completion_interruptible(&cmd->reject_comp); + /* + * Perform the kref_put now if se_cmd has already been setup by + * scsit_setup_scsi_cmd() + */ + if (cmd->se_cmd.se_tfo != NULL) { + pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); + } if (ret != 0) return -1; return (!fail_conn) ? 0 : -1; } +EXPORT_SYMBOL(iscsit_add_reject_from_cmd); /* * Map some portion of the allocated scatterlist to an iovec, suitable for @@ -761,6 +779,9 @@ static void iscsit_ack_from_expstatsn(struct iscsi_conn *conn, u32 exp_statsn) conn->exp_statsn = exp_statsn; + if (conn->sess->sess_ops->RDMAExtensions) + return; + spin_lock_bh(&conn->cmd_lock); list_for_each_entry(cmd, &conn->conn_cmd_list, i_conn_node) { spin_lock(&cmd->istate_lock); @@ -793,12 +814,10 @@ static int iscsit_allocate_iovecs(struct iscsi_cmd *cmd) return 0; } -static int iscsit_handle_scsi_cmd( - struct iscsi_conn *conn, - unsigned char *buf) +int iscsit_setup_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + unsigned char *buf) { - int data_direction, payload_length, cmdsn_ret = 0, immed_ret; - struct iscsi_cmd *cmd = NULL; + int data_direction, payload_length; struct iscsi_scsi_req *hdr; int iscsi_task_attr; int sam_task_attr; @@ -821,8 +840,8 @@ static int iscsit_handle_scsi_cmd( !(hdr->flags & ISCSI_FLAG_CMD_FINAL)) { pr_err("ISCSI_FLAG_CMD_WRITE & ISCSI_FLAG_CMD_FINAL" " not set. Bad iSCSI Initiator.\n"); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } if (((hdr->flags & ISCSI_FLAG_CMD_READ) || @@ -842,8 +861,8 @@ static int iscsit_handle_scsi_cmd( pr_err("ISCSI_FLAG_CMD_READ or ISCSI_FLAG_CMD_WRITE" " set when Expected Data Transfer Length is 0 for" " CDB: 0x%02x. Bad iSCSI Initiator.\n", hdr->cdb[0]); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } done: @@ -852,29 +871,29 @@ static int iscsit_handle_scsi_cmd( pr_err("ISCSI_FLAG_CMD_READ and/or ISCSI_FLAG_CMD_WRITE" " MUST be set if Expected Data Transfer Length is not 0." " Bad iSCSI Initiator\n"); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } if ((hdr->flags & ISCSI_FLAG_CMD_READ) && (hdr->flags & ISCSI_FLAG_CMD_WRITE)) { pr_err("Bidirectional operations not supported!\n"); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } if (hdr->opcode & ISCSI_OP_IMMEDIATE) { pr_err("Illegally set Immediate Bit in iSCSI Initiator" " Scsi Command PDU.\n"); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } if (payload_length && !conn->sess->sess_ops->ImmediateData) { pr_err("ImmediateData=No but DataSegmentLength=%u," " protocol error.\n", payload_length); - return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, + 1, 1, buf, cmd); } if ((be32_to_cpu(hdr->data_length )== payload_length) && @@ -882,43 +901,38 @@ static int iscsit_handle_scsi_cmd( pr_err("Expected Data Transfer Length and Length of" " Immediate Data are the same, but ISCSI_FLAG_CMD_FINAL" " bit is not set protocol error\n"); - return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, + 1, 1, buf, cmd); } if (payload_length > be32_to_cpu(hdr->data_length)) { pr_err("DataSegmentLength: %u is greater than" " EDTL: %u, protocol error.\n", payload_length, hdr->data_length); - return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, + 1, 1, buf, cmd); } if (payload_length > conn->conn_ops->MaxXmitDataSegmentLength) { pr_err("DataSegmentLength: %u is greater than" " MaxXmitDataSegmentLength: %u, protocol error.\n", payload_length, conn->conn_ops->MaxXmitDataSegmentLength); - return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, + 1, 1, buf, cmd); } if (payload_length > conn->sess->sess_ops->FirstBurstLength) { pr_err("DataSegmentLength: %u is greater than" " FirstBurstLength: %u, protocol error.\n", payload_length, conn->sess->sess_ops->FirstBurstLength); - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_INVALID, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_BOOKMARK_INVALID, + 1, 1, buf, cmd); } data_direction = (hdr->flags & ISCSI_FLAG_CMD_WRITE) ? DMA_TO_DEVICE : (hdr->flags & ISCSI_FLAG_CMD_READ) ? DMA_FROM_DEVICE : DMA_NONE; - cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); - if (!cmd) - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, - buf, conn); - cmd->data_direction = data_direction; iscsi_task_attr = hdr->flags & ISCSI_FLAG_CMD_ATTR_MASK; /* @@ -961,7 +975,8 @@ static int iscsit_handle_scsi_cmd( cmd->exp_stat_sn = be32_to_cpu(hdr->exp_statsn); cmd->first_burst_len = payload_length; - if (cmd->data_direction == DMA_FROM_DEVICE) { + if (!conn->sess->sess_ops->RDMAExtensions && + cmd->data_direction == DMA_FROM_DEVICE) { struct iscsi_datain_req *dr; dr = iscsit_allocate_datain_req(); @@ -983,7 +998,10 @@ static int iscsit_handle_scsi_cmd( pr_debug("Got SCSI Command, ITT: 0x%08x, CmdSN: 0x%08x," " ExpXferLen: %u, Length: %u, CID: %hu\n", hdr->itt, - hdr->cmdsn, hdr->data_length, payload_length, conn->cid); + hdr->cmdsn, be32_to_cpu(hdr->data_length), payload_length, + conn->cid); + + target_get_sess_cmd(conn->sess->se_sess, &cmd->se_cmd, true); cmd->sense_reason = transport_lookup_cmd_lun(&cmd->se_cmd, scsilun_to_int(&hdr->lun)); @@ -1017,12 +1035,24 @@ static int iscsit_handle_scsi_cmd( */ core_alua_check_nonop_delay(&cmd->se_cmd); - if (iscsit_allocate_iovecs(cmd) < 0) { - return iscsit_add_reject_from_cmd( - ISCSI_REASON_BOOKMARK_NO_RESOURCES, - 1, 0, buf, cmd); - } + return 0; +} +EXPORT_SYMBOL(iscsit_setup_scsi_cmd); +void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *cmd) +{ + iscsit_set_dataout_sequence_values(cmd); + + spin_lock_bh(&cmd->dataout_timeout_lock); + iscsit_start_dataout_timer(cmd, cmd->conn); + spin_unlock_bh(&cmd->dataout_timeout_lock); +} +EXPORT_SYMBOL(iscsit_set_unsoliticed_dataout); + +int iscsit_process_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + struct iscsi_scsi_req *hdr) +{ + int cmdsn_ret = 0; /* * Check the CmdSN against ExpCmdSN/MaxCmdSN here if * the Immediate Bit is not set, and no Immediate @@ -1035,12 +1065,17 @@ static int iscsit_handle_scsi_cmd( */ if (!cmd->immediate_data) { cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); - if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) + if (cmdsn_ret == CMDSN_LOWER_THAN_EXP) { + if (!cmd->sense_reason) + return 0; + + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); return 0; - else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) + } else if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) { return iscsit_add_reject_from_cmd( ISCSI_REASON_PROTOCOL_ERROR, - 1, 0, buf, cmd); + 1, 0, (unsigned char *)hdr, cmd); + } } iscsit_ack_from_expstatsn(conn, be32_to_cpu(hdr->exp_statsn)); @@ -1049,25 +1084,23 @@ static int iscsit_handle_scsi_cmd( * If no Immediate Data is attached, it's OK to return now. */ if (!cmd->immediate_data) { - if (!cmd->sense_reason && cmd->unsolicited_data) { - iscsit_set_dataout_sequence_values(cmd); - - spin_lock_bh(&cmd->dataout_timeout_lock); - iscsit_start_dataout_timer(cmd, cmd->conn); - spin_unlock_bh(&cmd->dataout_timeout_lock); - } + if (!cmd->sense_reason && cmd->unsolicited_data) + iscsit_set_unsoliticed_dataout(cmd); + if (!cmd->sense_reason) + return 0; + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); return 0; } /* - * Early CHECK_CONDITIONs never make it to the transport processing - * thread. They are processed in CmdSN order by - * iscsit_check_received_cmdsn() below. + * Early CHECK_CONDITIONs with ImmediateData never make it to command + * execution. These exceptions are processed in CmdSN order using + * iscsit_check_received_cmdsn() in iscsit_get_immediate_data() below. */ if (cmd->sense_reason) { - immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; - goto after_immediate_data; + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); + return 1; } /* * Call directly into transport_generic_new_cmd() to perform @@ -1075,11 +1108,27 @@ static int iscsit_handle_scsi_cmd( */ cmd->sense_reason = transport_generic_new_cmd(&cmd->se_cmd); if (cmd->sense_reason) { - immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; - goto after_immediate_data; + target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); + return 1; } - immed_ret = iscsit_handle_immediate_data(cmd, buf, payload_length); + return 0; +} +EXPORT_SYMBOL(iscsit_process_scsi_cmd); + +static int +iscsit_get_immediate_data(struct iscsi_cmd *cmd, struct iscsi_scsi_req *hdr, + bool dump_payload) +{ + int cmdsn_ret = 0, immed_ret = IMMEDIATE_DATA_NORMAL_OPERATION; + /* + * Special case for Unsupported SAM WRITE Opcodes and ImmediateData=Yes. + */ + if (dump_payload == true) + goto after_immediate_data; + + immed_ret = iscsit_handle_immediate_data(cmd, hdr, + cmd->first_burst_len); after_immediate_data: if (immed_ret == IMMEDIATE_DATA_NORMAL_OPERATION) { /* @@ -1087,26 +1136,19 @@ static int iscsit_handle_scsi_cmd( * DataCRC, check against ExpCmdSN/MaxCmdSN if * Immediate Bit is not set. */ - cmdsn_ret = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); - /* - * Special case for Unsupported SAM WRITE Opcodes - * and ImmediateData=Yes. - */ - if (cmd->sense_reason) { - if (iscsit_dump_data_payload(conn, payload_length, 1) < 0) - return -1; - } else if (cmd->unsolicited_data) { - iscsit_set_dataout_sequence_values(cmd); + cmdsn_ret = iscsit_sequence_cmd(cmd->conn, cmd, hdr->cmdsn); - spin_lock_bh(&cmd->dataout_timeout_lock); - iscsit_start_dataout_timer(cmd, cmd->conn); - spin_unlock_bh(&cmd->dataout_timeout_lock); - } + if (cmd->sense_reason) { + if (iscsit_dump_data_payload(cmd->conn, + cmd->first_burst_len, 1) < 0) + return -1; + } else if (cmd->unsolicited_data) + iscsit_set_unsoliticed_dataout(cmd); if (cmdsn_ret == CMDSN_ERROR_CANNOT_RECOVER) return iscsit_add_reject_from_cmd( ISCSI_REASON_PROTOCOL_ERROR, - 1, 0, buf, cmd); + 1, 0, (unsigned char *)hdr, cmd); } else if (immed_ret == IMMEDIATE_DATA_ERL1_CRC_FAILURE) { /* @@ -1121,13 +1163,47 @@ static int iscsit_handle_scsi_cmd( * CmdSN and issue a retry to plug the sequence. */ cmd->i_state = ISTATE_REMOVE; - iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); + iscsit_add_cmd_to_immediate_queue(cmd, cmd->conn, cmd->i_state); } else /* immed_ret == IMMEDIATE_DATA_CANNOT_RECOVER */ return -1; return 0; } +static int +iscsit_handle_scsi_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + unsigned char *buf) +{ + struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; + int rc, immed_data; + bool dump_payload = false; + + rc = iscsit_setup_scsi_cmd(conn, cmd, buf); + if (rc < 0) + return rc; + /* + * Allocation iovecs needed for struct socket operations for + * traditional iSCSI block I/O. + */ + if (iscsit_allocate_iovecs(cmd) < 0) { + return iscsit_add_reject_from_cmd( + ISCSI_REASON_BOOKMARK_NO_RESOURCES, + 1, 0, buf, cmd); + } + immed_data = cmd->immediate_data; + + rc = iscsit_process_scsi_cmd(conn, cmd, hdr); + if (rc < 0) + return rc; + else if (rc > 0) + dump_payload = true; + + if (!immed_data) + return 0; + + return iscsit_get_immediate_data(cmd, hdr, dump_payload); +} + static u32 iscsit_do_crypto_hash_sg( struct hash_desc *hash, struct iscsi_cmd *cmd, @@ -1190,20 +1266,16 @@ static void iscsit_do_crypto_hash_buf( crypto_hash_final(hash, data_crc); } -static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) +int +iscsit_check_dataout_hdr(struct iscsi_conn *conn, unsigned char *buf, + struct iscsi_cmd **out_cmd) { - int iov_ret, ooo_cmdsn = 0, ret; - u8 data_crc_failed = 0; - u32 checksum, iov_count = 0, padding = 0, rx_got = 0; - u32 rx_size = 0, payload_length; + struct iscsi_data *hdr = (struct iscsi_data *)buf; struct iscsi_cmd *cmd = NULL; struct se_cmd *se_cmd; - struct iscsi_data *hdr; - struct kvec *iov; unsigned long flags; - - hdr = (struct iscsi_data *) buf; - payload_length = ntoh24(hdr->dlength); + u32 payload_length = ntoh24(hdr->dlength); + int rc; if (!payload_length) { pr_err("DataOUT payload is ZERO, protocol error.\n"); @@ -1236,7 +1308,7 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) pr_debug("Got DataOut ITT: 0x%08x, TTT: 0x%08x," " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", - hdr->itt, hdr->ttt, hdr->datasn, hdr->offset, + hdr->itt, hdr->ttt, hdr->datasn, ntohl(hdr->offset), payload_length, conn->cid); if (cmd->cmd_flags & ICF_GOT_LAST_DATAOUT) { @@ -1328,12 +1400,26 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) * Preform DataSN, DataSequenceInOrder, DataPDUInOrder, and * within-command recovery checks before receiving the payload. */ - ret = iscsit_check_pre_dataout(cmd, buf); - if (ret == DATAOUT_WITHIN_COMMAND_RECOVERY) + rc = iscsit_check_pre_dataout(cmd, buf); + if (rc == DATAOUT_WITHIN_COMMAND_RECOVERY) return 0; - else if (ret == DATAOUT_CANNOT_RECOVER) + else if (rc == DATAOUT_CANNOT_RECOVER) return -1; + *out_cmd = cmd; + return 0; +} +EXPORT_SYMBOL(iscsit_check_dataout_hdr); + +static int +iscsit_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + struct iscsi_data *hdr) +{ + struct kvec *iov; + u32 checksum, iov_count = 0, padding = 0, rx_got = 0, rx_size = 0; + u32 payload_length = ntoh24(hdr->dlength); + int iov_ret, data_crc_failed = 0; + rx_size += payload_length; iov = &cmd->iov_data[0]; @@ -1386,17 +1472,27 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) payload_length); } } + + return data_crc_failed; +} + +int +iscsit_check_dataout_payload(struct iscsi_cmd *cmd, struct iscsi_data *hdr, + bool data_crc_failed) +{ + struct iscsi_conn *conn = cmd->conn; + int rc, ooo_cmdsn; /* * Increment post receive data and CRC values or perform * within-command recovery. */ - ret = iscsit_check_post_dataout(cmd, buf, data_crc_failed); - if ((ret == DATAOUT_NORMAL) || (ret == DATAOUT_WITHIN_COMMAND_RECOVERY)) + rc = iscsit_check_post_dataout(cmd, (unsigned char *)hdr, data_crc_failed); + if ((rc == DATAOUT_NORMAL) || (rc == DATAOUT_WITHIN_COMMAND_RECOVERY)) return 0; - else if (ret == DATAOUT_SEND_R2T) { + else if (rc == DATAOUT_SEND_R2T) { iscsit_set_dataout_sequence_values(cmd); - iscsit_build_r2ts_for_cmd(cmd, conn, false); - } else if (ret == DATAOUT_SEND_TO_TRANSPORT) { + conn->conn_transport->iscsit_get_dataout(conn, cmd, false); + } else if (rc == DATAOUT_SEND_TO_TRANSPORT) { /* * Handle extra special case for out of order * Unsolicited Data Out. @@ -1417,15 +1513,37 @@ static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) return 0; } +EXPORT_SYMBOL(iscsit_check_dataout_payload); -static int iscsit_handle_nop_out( - struct iscsi_conn *conn, - unsigned char *buf) +static int iscsit_handle_data_out(struct iscsi_conn *conn, unsigned char *buf) +{ + struct iscsi_cmd *cmd; + struct iscsi_data *hdr = (struct iscsi_data *)buf; + int rc; + bool data_crc_failed = false; + + rc = iscsit_check_dataout_hdr(conn, buf, &cmd); + if (rc < 0) + return rc; + else if (!cmd) + return 0; + + rc = iscsit_get_dataout(conn, cmd, hdr); + if (rc < 0) + return rc; + else if (rc > 0) + data_crc_failed = true; + + return iscsit_check_dataout_payload(cmd, hdr, data_crc_failed); +} + +int iscsit_handle_nop_out(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + unsigned char *buf) { unsigned char *ping_data = NULL; int cmdsn_ret, niov = 0, ret = 0, rx_got, rx_size; u32 checksum, data_crc, padding = 0, payload_length; - struct iscsi_cmd *cmd = NULL; + struct iscsi_cmd *cmd_p = NULL; struct kvec *iov = NULL; struct iscsi_nopout *hdr; @@ -1448,7 +1566,7 @@ static int iscsit_handle_nop_out( buf, conn); } - pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%09x," + pr_debug("Got NOPOUT Ping %s ITT: 0x%08x, TTT: 0x%08x," " CmdSN: 0x%08x, ExpStatSN: 0x%08x, Length: %u\n", hdr->itt == RESERVED_ITT ? "Response" : "Request", hdr->itt, hdr->ttt, hdr->cmdsn, hdr->exp_statsn, @@ -1461,7 +1579,6 @@ static int iscsit_handle_nop_out( * can contain ping data. */ if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { - cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); if (!cmd) return iscsit_add_reject( ISCSI_REASON_BOOKMARK_NO_RESOURCES, @@ -1596,14 +1713,14 @@ static int iscsit_handle_nop_out( /* * This was a response to a unsolicited NOPIN ping. */ - cmd = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt)); - if (!cmd) + cmd_p = iscsit_find_cmd_from_ttt(conn, be32_to_cpu(hdr->ttt)); + if (!cmd_p) return -1; iscsit_stop_nopin_response_timer(conn); - cmd->i_state = ISTATE_REMOVE; - iscsit_add_cmd_to_immediate_queue(cmd, conn, cmd->i_state); + cmd_p->i_state = ISTATE_REMOVE; + iscsit_add_cmd_to_immediate_queue(cmd_p, conn, cmd_p->i_state); iscsit_start_nopin_timer(conn); } else { /* @@ -1627,12 +1744,12 @@ static int iscsit_handle_nop_out( kfree(ping_data); return ret; } +EXPORT_SYMBOL(iscsit_handle_nop_out); -static int iscsit_handle_task_mgt_cmd( - struct iscsi_conn *conn, - unsigned char *buf) +int +iscsit_handle_task_mgt_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + unsigned char *buf) { - struct iscsi_cmd *cmd; struct se_tmr_req *se_tmr; struct iscsi_tmr_req *tmr_req; struct iscsi_tm *hdr; @@ -1661,18 +1778,13 @@ static int iscsit_handle_task_mgt_cmd( pr_err("Task Management Request TASK_REASSIGN not" " issued as immediate command, bad iSCSI Initiator" "implementation\n"); - return iscsit_add_reject(ISCSI_REASON_PROTOCOL_ERROR, 1, - buf, conn); + return iscsit_add_reject_from_cmd(ISCSI_REASON_PROTOCOL_ERROR, + 1, 1, buf, cmd); } if ((function != ISCSI_TM_FUNC_ABORT_TASK) && be32_to_cpu(hdr->refcmdsn) != ISCSI_RESERVED_TAG) hdr->refcmdsn = cpu_to_be32(ISCSI_RESERVED_TAG); - cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); - if (!cmd) - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, - 1, buf, conn); - cmd->data_direction = DMA_NONE; cmd->tmr_req = kzalloc(sizeof(struct iscsi_tmr_req), GFP_KERNEL); @@ -1843,6 +1955,7 @@ static int iscsit_handle_task_mgt_cmd( iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); return 0; } +EXPORT_SYMBOL(iscsit_handle_task_mgt_cmd); /* #warning FIXME: Support Text Command parameters besides SendTargets */ static int iscsit_handle_text_cmd( @@ -2105,13 +2218,12 @@ int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *cmd, struct iscsi_conn return 0; } -static int iscsit_handle_logout_cmd( - struct iscsi_conn *conn, - unsigned char *buf) +int +iscsit_handle_logout_cmd(struct iscsi_conn *conn, struct iscsi_cmd *cmd, + unsigned char *buf) { int cmdsn_ret, logout_remove = 0; u8 reason_code = 0; - struct iscsi_cmd *cmd; struct iscsi_logout *hdr; struct iscsi_tiqn *tiqn = iscsit_snmp_get_tiqn(conn); @@ -2135,14 +2247,10 @@ static int iscsit_handle_logout_cmd( if (conn->conn_state != TARG_CONN_STATE_LOGGED_IN) { pr_err("Received logout request on connection that" " is not in logged in state, ignoring request.\n"); + iscsit_release_cmd(cmd); return 0; } - cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); - if (!cmd) - return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, 1, - buf, conn); - cmd->iscsi_opcode = ISCSI_OP_LOGOUT; cmd->i_state = ISTATE_SEND_LOGOUTRSP; cmd->immediate_cmd = ((hdr->opcode & ISCSI_OP_IMMEDIATE) ? 1 : 0); @@ -2192,6 +2300,7 @@ static int iscsit_handle_logout_cmd( return logout_remove; } +EXPORT_SYMBOL(iscsit_handle_logout_cmd); static int iscsit_handle_snack( struct iscsi_conn *conn, @@ -2259,7 +2368,7 @@ static void iscsit_rx_thread_wait_for_tcp(struct iscsi_conn *conn) static int iscsit_handle_immediate_data( struct iscsi_cmd *cmd, - unsigned char *buf, + struct iscsi_scsi_req *hdr, u32 length) { int iov_ret, rx_got = 0, rx_size = 0; @@ -2315,12 +2424,12 @@ static int iscsit_handle_immediate_data( " in ERL=0.\n"); iscsit_add_reject_from_cmd( ISCSI_REASON_DATA_DIGEST_ERROR, - 1, 0, buf, cmd); + 1, 0, (unsigned char *)hdr, cmd); return IMMEDIATE_DATA_CANNOT_RECOVER; } else { iscsit_add_reject_from_cmd( ISCSI_REASON_DATA_DIGEST_ERROR, - 0, 0, buf, cmd); + 0, 0, (unsigned char *)hdr, cmd); return IMMEDIATE_DATA_ERL1_CRC_FAILURE; } } else { @@ -2955,8 +3064,8 @@ static int iscsit_send_r2t( * connection recovery. */ int iscsit_build_r2ts_for_cmd( - struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_cmd *cmd, bool recovery) { int first_r2t = 1; @@ -3758,6 +3867,83 @@ int iscsi_target_tx_thread(void *arg) return 0; } +static int iscsi_target_rx_opcode(struct iscsi_conn *conn, unsigned char *buf) +{ + struct iscsi_hdr *hdr = (struct iscsi_hdr *)buf; + struct iscsi_cmd *cmd; + int ret = 0; + + switch (hdr->opcode & ISCSI_OPCODE_MASK) { + case ISCSI_OP_SCSI_CMD: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, + 1, buf, conn); + + ret = iscsit_handle_scsi_cmd(conn, cmd, buf); + break; + case ISCSI_OP_SCSI_DATA_OUT: + ret = iscsit_handle_data_out(conn, buf); + break; + case ISCSI_OP_NOOP_OUT: + cmd = NULL; + if (hdr->ttt == cpu_to_be32(0xFFFFFFFF)) { + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, + 1, buf, conn); + } + ret = iscsit_handle_nop_out(conn, cmd, buf); + break; + case ISCSI_OP_SCSI_TMFUNC: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, + 1, buf, conn); + + ret = iscsit_handle_task_mgt_cmd(conn, cmd, buf); + break; + case ISCSI_OP_TEXT: + ret = iscsit_handle_text_cmd(conn, buf); + break; + case ISCSI_OP_LOGOUT: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + return iscsit_add_reject(ISCSI_REASON_BOOKMARK_NO_RESOURCES, + 1, buf, conn); + + ret = iscsit_handle_logout_cmd(conn, cmd, buf); + if (ret > 0) + wait_for_completion_timeout(&conn->conn_logout_comp, + SECONDS_FOR_LOGOUT_COMP * HZ); + break; + case ISCSI_OP_SNACK: + ret = iscsit_handle_snack(conn, buf); + break; + default: + pr_err("Got unknown iSCSI OpCode: 0x%02x\n", hdr->opcode); + if (!conn->sess->sess_ops->ErrorRecoveryLevel) { + pr_err("Cannot recover from unknown" + " opcode while ERL=0, closing iSCSI connection.\n"); + return -1; + } + if (!conn->conn_ops->OFMarker) { + pr_err("Unable to recover from unknown" + " opcode while OFMarker=No, closing iSCSI" + " connection.\n"); + return -1; + } + if (iscsit_recover_from_unknown_opcode(conn) < 0) { + pr_err("Unable to recover from unknown" + " opcode, closing iSCSI connection.\n"); + return -1; + } + break; + } + + return ret; +} + int iscsi_target_rx_thread(void *arg) { int ret; @@ -3777,6 +3963,18 @@ int iscsi_target_rx_thread(void *arg) if (!conn) goto out; + if (conn->conn_transport->transport_type == ISCSI_INFINIBAND) { + struct completion comp; + int rc; + + init_completion(&comp); + rc = wait_for_completion_interruptible(&comp); + if (rc < 0) + goto transport_err; + + goto out; + } + while (!kthread_should_stop()) { /* * Ensure that both TX and RX per connection kthreads @@ -3848,62 +4046,9 @@ int iscsi_target_rx_thread(void *arg) goto transport_err; } - switch (opcode) { - case ISCSI_OP_SCSI_CMD: - if (iscsit_handle_scsi_cmd(conn, buffer) < 0) - goto transport_err; - break; - case ISCSI_OP_SCSI_DATA_OUT: - if (iscsit_handle_data_out(conn, buffer) < 0) - goto transport_err; - break; - case ISCSI_OP_NOOP_OUT: - if (iscsit_handle_nop_out(conn, buffer) < 0) - goto transport_err; - break; - case ISCSI_OP_SCSI_TMFUNC: - if (iscsit_handle_task_mgt_cmd(conn, buffer) < 0) - goto transport_err; - break; - case ISCSI_OP_TEXT: - if (iscsit_handle_text_cmd(conn, buffer) < 0) - goto transport_err; - break; - case ISCSI_OP_LOGOUT: - ret = iscsit_handle_logout_cmd(conn, buffer); - if (ret > 0) { - wait_for_completion_timeout(&conn->conn_logout_comp, - SECONDS_FOR_LOGOUT_COMP * HZ); - goto transport_err; - } else if (ret < 0) - goto transport_err; - break; - case ISCSI_OP_SNACK: - if (iscsit_handle_snack(conn, buffer) < 0) - goto transport_err; - break; - default: - pr_err("Got unknown iSCSI OpCode: 0x%02x\n", - opcode); - if (!conn->sess->sess_ops->ErrorRecoveryLevel) { - pr_err("Cannot recover from unknown" - " opcode while ERL=0, closing iSCSI connection" - ".\n"); - goto transport_err; - } - if (!conn->conn_ops->OFMarker) { - pr_err("Unable to recover from unknown" - " opcode while OFMarker=No, closing iSCSI" - " connection.\n"); - goto transport_err; - } - if (iscsit_recover_from_unknown_opcode(conn) < 0) { - pr_err("Unable to recover from unknown" - " opcode, closing iSCSI connection.\n"); - goto transport_err; - } - break; - } + ret = iscsi_target_rx_opcode(conn, buffer); + if (ret < 0) + goto transport_err; } transport_err: diff --git a/drivers/target/iscsi/iscsi_target.h b/drivers/target/iscsi/iscsi_target.h index b1a1e6350707..a0050b2f294e 100644 --- a/drivers/target/iscsi/iscsi_target.h +++ b/drivers/target/iscsi/iscsi_target.h @@ -16,11 +16,12 @@ extern int iscsit_reset_np_thread(struct iscsi_np *, struct iscsi_tpg_np *, struct iscsi_portal_group *); extern int iscsit_del_np(struct iscsi_np *); extern int iscsit_add_reject_from_cmd(u8, int, int, unsigned char *, struct iscsi_cmd *); +extern void iscsit_set_unsoliticed_dataout(struct iscsi_cmd *); extern int iscsit_logout_closesession(struct iscsi_cmd *, struct iscsi_conn *); extern int iscsit_logout_closeconnection(struct iscsi_cmd *, struct iscsi_conn *); extern int iscsit_logout_removeconnforrecovery(struct iscsi_cmd *, struct iscsi_conn *); extern int iscsit_send_async_msg(struct iscsi_conn *, u16, u8, u8); -extern int iscsit_build_r2ts_for_cmd(struct iscsi_cmd *, struct iscsi_conn *, bool recovery); +extern int iscsit_build_r2ts_for_cmd(struct iscsi_conn *, struct iscsi_cmd *, bool recovery); extern void iscsit_thread_get_cpumask(struct iscsi_conn *); extern int iscsi_target_tx_thread(void *); extern int iscsi_target_rx_thread(void *); diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c index c78b824194e6..ce3d3210fe41 100644 --- a/drivers/target/iscsi/iscsi_target_configfs.c +++ b/drivers/target/iscsi/iscsi_target_configfs.c @@ -1543,9 +1543,10 @@ static int lio_queue_data_in(struct se_cmd *se_cmd) static int lio_write_pending(struct se_cmd *se_cmd) { struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd); + struct iscsi_conn *conn = cmd->conn; if (!cmd->immediate_data && !cmd->unsolicited_data) - return iscsit_build_r2ts_for_cmd(cmd, cmd->conn, false); + return conn->conn_transport->iscsit_get_dataout(conn, cmd, false); return 0; } @@ -1696,6 +1697,11 @@ static void lio_set_default_node_attributes(struct se_node_acl *se_acl) iscsit_set_default_node_attribues(acl); } +static int lio_check_stop_free(struct se_cmd *se_cmd) +{ + return target_put_sess_cmd(se_cmd->se_sess, se_cmd); +} + static void lio_release_cmd(struct se_cmd *se_cmd) { struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd); @@ -1741,6 +1747,7 @@ int iscsi_target_register_configfs(void) fabric->tf_ops.tpg_alloc_fabric_acl = &lio_tpg_alloc_fabric_acl; fabric->tf_ops.tpg_release_fabric_acl = &lio_tpg_release_fabric_acl; fabric->tf_ops.tpg_get_inst_index = &lio_tpg_get_inst_index; + fabric->tf_ops.check_stop_free = &lio_check_stop_free, fabric->tf_ops.release_cmd = &lio_release_cmd; fabric->tf_ops.shutdown_session = &lio_tpg_shutdown_session; fabric->tf_ops.close_session = &lio_tpg_close_session; diff --git a/drivers/target/iscsi/iscsi_target_erl1.c b/drivers/target/iscsi/iscsi_target_erl1.c index 0b52a2371305..7816af6cdd12 100644 --- a/drivers/target/iscsi/iscsi_target_erl1.c +++ b/drivers/target/iscsi/iscsi_target_erl1.c @@ -22,6 +22,7 @@ #include #include #include +#include #include "iscsi_target_core.h" #include "iscsi_target_seq_pdu_list.h" @@ -53,6 +54,9 @@ int iscsit_dump_data_payload( u32 length, padding, offset = 0, size; struct kvec iov; + if (conn->sess->sess_ops->RDMAExtensions) + return 0; + length = (buf_len > OFFLOAD_BUF_SIZE) ? OFFLOAD_BUF_SIZE : buf_len; buf = kzalloc(length, GFP_ATOMIC); @@ -919,6 +923,7 @@ int iscsit_execute_ooo_cmdsns(struct iscsi_session *sess) int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo) { struct se_cmd *se_cmd = &cmd->se_cmd; + struct iscsi_conn *conn = cmd->conn; int lr = 0; spin_lock_bh(&cmd->istate_lock); @@ -981,7 +986,7 @@ int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo) return 0; iscsit_set_dataout_sequence_values(cmd); - iscsit_build_r2ts_for_cmd(cmd, cmd->conn, false); + conn->conn_transport->iscsit_get_dataout(conn, cmd, false); } return 0; } @@ -999,10 +1004,7 @@ int iscsit_execute_cmd(struct iscsi_cmd *cmd, int ooo) if (transport_check_aborted_status(se_cmd, 1) != 0) return 0; - iscsit_set_dataout_sequence_values(cmd); - spin_lock_bh(&cmd->dataout_timeout_lock); - iscsit_start_dataout_timer(cmd, cmd->conn); - spin_unlock_bh(&cmd->dataout_timeout_lock); + iscsit_set_unsoliticed_dataout(cmd); } return transport_handle_cdb_direct(&cmd->se_cmd); @@ -1290,3 +1292,4 @@ void iscsit_stop_dataout_timer(struct iscsi_cmd *cmd) cmd->init_task_tag); spin_unlock_bh(&cmd->dataout_timeout_lock); } +EXPORT_SYMBOL(iscsit_stop_dataout_timer); diff --git a/drivers/target/iscsi/iscsi_target_login.c b/drivers/target/iscsi/iscsi_target_login.c index e6a826927a40..bb5d5c5bce65 100644 --- a/drivers/target/iscsi/iscsi_target_login.c +++ b/drivers/target/iscsi/iscsi_target_login.c @@ -1068,8 +1068,7 @@ int iscsit_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) if (login->first_request) { login_req = (struct iscsi_login_req *)login->req; login->leading_connection = (!login_req->tsih) ? 1 : 0; - login->current_stage = - (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) >> 2; + login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags); login->version_min = login_req->min_version; login->version_max = login_req->max_version; memcpy(login->isid, login_req->isid, 6); diff --git a/drivers/target/iscsi/iscsi_target_nego.c b/drivers/target/iscsi/iscsi_target_nego.c index 879a0cb44bc3..7ad912060e21 100644 --- a/drivers/target/iscsi/iscsi_target_nego.c +++ b/drivers/target/iscsi/iscsi_target_nego.c @@ -691,21 +691,6 @@ int iscsi_target_locate_portal( login_req = (struct iscsi_login_req *) login->req; payload_length = ntoh24(login_req->dlength); - login->first_request = 1; - login->leading_connection = (!login_req->tsih) ? 1 : 0; - login->current_stage = ISCSI_LOGIN_CURRENT_STAGE(login_req->flags); - login->version_min = login_req->min_version; - login->version_max = login_req->max_version; - memcpy(login->isid, login_req->isid, 6); - login->cmd_sn = be32_to_cpu(login_req->cmdsn); - login->init_task_tag = login_req->itt; - login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); - login->cid = be16_to_cpu(login_req->cid); - login->tsih = be16_to_cpu(login_req->tsih); - - if (iscsi_target_get_initial_payload(conn, login) < 0) - return -1; - tmpbuf = kzalloc(payload_length + 1, GFP_KERNEL); if (!tmpbuf) { pr_err("Unable to allocate memory for tmpbuf.\n"); diff --git a/drivers/target/iscsi/iscsi_target_tmr.c b/drivers/target/iscsi/iscsi_target_tmr.c index 9d4417aae921..16ca0ef9266f 100644 --- a/drivers/target/iscsi/iscsi_target_tmr.c +++ b/drivers/target/iscsi/iscsi_target_tmr.c @@ -23,6 +23,7 @@ #include #include #include +#include #include "iscsi_target_core.h" #include "iscsi_target_seq_pdu_list.h" @@ -301,7 +302,7 @@ static int iscsit_task_reassign_complete_write( /* * iscsit_build_r2ts_for_cmd() can handle the rest from here. */ - return iscsit_build_r2ts_for_cmd(cmd, conn, true); + return conn->conn_transport->iscsit_get_dataout(conn, cmd, true); } static int iscsit_task_reassign_complete_read( diff --git a/drivers/target/iscsi/iscsi_target_util.c b/drivers/target/iscsi/iscsi_target_util.c index 0b73f90c9c54..2cc6c9a3ffb8 100644 --- a/drivers/target/iscsi/iscsi_target_util.c +++ b/drivers/target/iscsi/iscsi_target_util.c @@ -317,6 +317,7 @@ int iscsit_sequence_cmd( return ret; } +EXPORT_SYMBOL(iscsit_sequence_cmd); int iscsit_check_unsolicited_dataout(struct iscsi_cmd *cmd, unsigned char *buf) { From 2ec5a8c118139e756d4d39844550ba77ec3543cc Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 20 Mar 2013 15:29:15 -0700 Subject: [PATCH 18/25] iscsi-target: Refactor TX queue logic + export response PDU creation This patch refactors TX immediate + response queue handling to use the new iscsit_transport API callers, and exports the necessary traditional iscsi PDU response creation functions for iser-target to utilize. This includes: - Add iscsit_build_datain_pdu() for DATAIN PDU init + convert iscsit_build_datain_pdu() - Add iscsit_build_logout_rsp() for LOGOUT_RSP PDU init + convert iscsit_send_logout() - Add iscsit_build_nopin_rsp() for NOPIN_RSP PDU init + convert iscsit_send_nopin() - Add iscsit_build_rsp_pdu() for SCSI_RSP PDU init + convert iscsit_send_response() - Add iscsit_build_task_mgt_rsp for TM_RSP PDU init + convert iscsit_send_task_mgt_rsp() - Refactor immediate queue state switch into iscsit_immediate_queue() - Convert handle_immediate_queue() to use iscsit_transport caller - Refactor response queue state switch into iscsit_response_queue() - Convert handle_response_queue to use iscsit_transport caller - Export iscsit_logout_post_handler(), iscsit_increment_maxcmdsn() and iscsit_tmr_post_handler() for external transport module usage v5 changes: - Fix solicited NopIN handling with RDMAExtensions=No (nab) v3 changes: - Add iscsit_build_reject for REJECT PDU init + convert iscsit_send_reject() v2 changes: - Add iscsit_queue_rsp() for iscsit_transport->iscsit_queue_data_in() and iscsit_transport->iscsit_queue_status() - Update lio_queue_data_in() to use ->iscsit_queue_data_in() - Update lio_queue_status() to use ->iscsit_queue_status() - Use mutex_trylock() in iscsit_increment_maxcmdsn() Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target.c | 667 ++++++++++--------- drivers/target/iscsi/iscsi_target_configfs.c | 7 +- drivers/target/iscsi/iscsi_target_device.c | 7 +- drivers/target/iscsi/iscsi_target_tmr.c | 1 + 4 files changed, 374 insertions(+), 308 deletions(-) diff --git a/drivers/target/iscsi/iscsi_target.c b/drivers/target/iscsi/iscsi_target.c index 40bb916f57ff..ffbc6a94be52 100644 --- a/drivers/target/iscsi/iscsi_target.c +++ b/drivers/target/iscsi/iscsi_target.c @@ -70,8 +70,7 @@ struct kmem_cache *lio_ooo_cache; struct kmem_cache *lio_r2t_cache; static int iscsit_handle_immediate_data(struct iscsi_cmd *, - unsigned char *buf, u32); -static int iscsit_logout_post_handler(struct iscsi_cmd *, struct iscsi_conn *); + struct iscsi_scsi_req *, u32); struct iscsi_tiqn *iscsit_get_tiqn_for_login(unsigned char *buf) { @@ -482,6 +481,15 @@ int iscsit_del_np(struct iscsi_np *np) return 0; } +static int iscsit_immediate_queue(struct iscsi_conn *, struct iscsi_cmd *, int); +static int iscsit_response_queue(struct iscsi_conn *, struct iscsi_cmd *, int); + +static int iscsit_queue_rsp(struct iscsi_conn *conn, struct iscsi_cmd *cmd) +{ + iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); + return 0; +} + static struct iscsit_transport iscsi_target_transport = { .name = "iSCSI/TCP", .transport_type = ISCSI_TCP, @@ -493,6 +501,10 @@ static struct iscsit_transport iscsi_target_transport = { .iscsit_get_login_rx = iscsit_get_login_rx, .iscsit_put_login_tx = iscsit_put_login_tx, .iscsit_get_dataout = iscsit_build_r2ts_for_cmd, + .iscsit_immediate_queue = iscsit_immediate_queue, + .iscsit_response_queue = iscsit_response_queue, + .iscsit_queue_data_in = iscsit_queue_rsp, + .iscsit_queue_status = iscsit_queue_rsp, }; static int __init iscsi_target_init_module(void) @@ -651,14 +663,6 @@ static int iscsit_add_reject( iscsit_add_cmd_to_response_queue(cmd, conn, cmd->i_state); ret = wait_for_completion_interruptible(&cmd->reject_comp); - /* - * Perform the kref_put now if se_cmd has been setup by - * iscsit_setup_scsi_cmd() - */ - if (cmd->se_cmd.se_tfo != NULL) { - pr_debug("iscsi reject: calling target_put_sess_cmd >>>>>>\n"); - target_put_sess_cmd(conn->sess->se_sess, &cmd->se_cmd); - } if (ret != 0) return -1; @@ -2549,18 +2553,60 @@ static void iscsit_tx_thread_wait_for_tcp(struct iscsi_conn *conn) } } -static int iscsit_send_data_in( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) +static void +iscsit_build_datain_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_datain *datain, struct iscsi_data_rsp *hdr, + bool set_statsn) { - int iov_ret = 0, set_statsn = 0; - u32 iov_count = 0, tx_size = 0; + hdr->opcode = ISCSI_OP_SCSI_DATA_IN; + hdr->flags = datain->flags; + if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { + if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { + hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; + hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); + } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { + hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; + hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); + } + } + hton24(hdr->dlength, datain->length); + if (hdr->flags & ISCSI_FLAG_DATA_ACK) + int_to_scsilun(cmd->se_cmd.orig_fe_lun, + (struct scsi_lun *)&hdr->lun); + else + put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); + + hdr->itt = cmd->init_task_tag; + + if (hdr->flags & ISCSI_FLAG_DATA_ACK) + hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); + else + hdr->ttt = cpu_to_be32(0xFFFFFFFF); + if (set_statsn) + hdr->statsn = cpu_to_be32(cmd->stat_sn); + else + hdr->statsn = cpu_to_be32(0xFFFFFFFF); + + hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); + hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + hdr->datasn = cpu_to_be32(datain->data_sn); + hdr->offset = cpu_to_be32(datain->offset); + + pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," + " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", + cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), + ntohl(hdr->offset), datain->length, conn->cid); +} + +static int iscsit_send_datain(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct iscsi_data_rsp *hdr = (struct iscsi_data_rsp *)&cmd->pdu[0]; struct iscsi_datain datain; struct iscsi_datain_req *dr; - struct iscsi_data_rsp *hdr; struct kvec *iov; - int eodr = 0; - int ret; + u32 iov_count = 0, tx_size = 0; + int eodr = 0, ret, iov_ret; + bool set_statsn = false; memset(&datain, 0, sizeof(struct iscsi_datain)); dr = iscsit_get_datain_values(cmd, &datain); @@ -2569,7 +2615,6 @@ static int iscsit_send_data_in( cmd->init_task_tag); return -1; } - /* * Be paranoid and double check the logic for now. */ @@ -2577,7 +2622,7 @@ static int iscsit_send_data_in( pr_err("Command ITT: 0x%08x, datain.offset: %u and" " datain.length: %u exceeds cmd->data_length: %u\n", cmd->init_task_tag, datain.offset, datain.length, - cmd->se_cmd.data_length); + cmd->se_cmd.data_length); return -1; } @@ -2601,47 +2646,13 @@ static int iscsit_send_data_in( (dr->dr_complete == DATAIN_COMPLETE_CONNECTION_RECOVERY)) { iscsit_increment_maxcmdsn(cmd, conn->sess); cmd->stat_sn = conn->stat_sn++; - set_statsn = 1; + set_statsn = true; } else if (dr->dr_complete == - DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) - set_statsn = 1; + DATAIN_COMPLETE_WITHIN_COMMAND_RECOVERY) + set_statsn = true; } - hdr = (struct iscsi_data_rsp *) cmd->pdu; - memset(hdr, 0, ISCSI_HDR_LEN); - hdr->opcode = ISCSI_OP_SCSI_DATA_IN; - hdr->flags = datain.flags; - if (hdr->flags & ISCSI_FLAG_DATA_STATUS) { - if (cmd->se_cmd.se_cmd_flags & SCF_OVERFLOW_BIT) { - hdr->flags |= ISCSI_FLAG_DATA_OVERFLOW; - hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); - } else if (cmd->se_cmd.se_cmd_flags & SCF_UNDERFLOW_BIT) { - hdr->flags |= ISCSI_FLAG_DATA_UNDERFLOW; - hdr->residual_count = cpu_to_be32(cmd->se_cmd.residual_count); - } - } - hton24(hdr->dlength, datain.length); - if (hdr->flags & ISCSI_FLAG_DATA_ACK) - int_to_scsilun(cmd->se_cmd.orig_fe_lun, - (struct scsi_lun *)&hdr->lun); - else - put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); - - hdr->itt = cmd->init_task_tag; - - if (hdr->flags & ISCSI_FLAG_DATA_ACK) - hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); - else - hdr->ttt = cpu_to_be32(0xFFFFFFFF); - if (set_statsn) - hdr->statsn = cpu_to_be32(cmd->stat_sn); - else - hdr->statsn = cpu_to_be32(0xFFFFFFFF); - - hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); - hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); - hdr->datasn = cpu_to_be32(datain.data_sn); - hdr->offset = cpu_to_be32(datain.offset); + iscsit_build_datain_pdu(cmd, conn, &datain, hdr, set_statsn); iov = &cmd->iov_data[0]; iov[iov_count].iov_base = cmd->pdu; @@ -2652,7 +2663,7 @@ static int iscsit_send_data_in( u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, - (unsigned char *)hdr, ISCSI_HDR_LEN, + (unsigned char *)cmd->pdu, ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); iov[0].iov_len += ISCSI_CRC_LEN; @@ -2662,7 +2673,8 @@ static int iscsit_send_data_in( " for DataIN PDU 0x%08x\n", *header_digest); } - iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], datain.offset, datain.length); + iov_ret = iscsit_map_iovec(cmd, &cmd->iov_data[1], + datain.offset, datain.length); if (iov_ret < 0) return -1; @@ -2693,11 +2705,6 @@ static int iscsit_send_data_in( cmd->iov_data_count = iov_count; cmd->tx_size = tx_size; - pr_debug("Built DataIN ITT: 0x%08x, StatSN: 0x%08x," - " DataSN: 0x%08x, Offset: %u, Length: %u, CID: %hu\n", - cmd->init_task_tag, ntohl(hdr->statsn), ntohl(hdr->datasn), - ntohl(hdr->offset), datain.length, conn->cid); - /* sendpage is preferred but can't insert markers */ if (!conn->conn_ops->IFMarker) ret = iscsit_fe_sendpage_sg(cmd, conn); @@ -2720,16 +2727,13 @@ static int iscsit_send_data_in( return eodr; } -static int iscsit_send_logout_response( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) +int +iscsit_build_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_logout_rsp *hdr) { - int niov = 0, tx_size; struct iscsi_conn *logout_conn = NULL; struct iscsi_conn_recovery *cr = NULL; struct iscsi_session *sess = conn->sess; - struct kvec *iov; - struct iscsi_logout_rsp *hdr; /* * The actual shutting down of Sessions and/or Connections * for CLOSESESSION and CLOSECONNECTION Logout Requests @@ -2798,9 +2802,6 @@ static int iscsit_send_logout_response( return -1; } - tx_size = ISCSI_HDR_LEN; - hdr = (struct iscsi_logout_rsp *)cmd->pdu; - memset(hdr, 0, ISCSI_HDR_LEN); hdr->opcode = ISCSI_OP_LOGOUT_RSP; hdr->flags |= ISCSI_FLAG_CMD_FINAL; hdr->response = cmd->logout_response; @@ -2812,6 +2813,27 @@ static int iscsit_send_logout_response( hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + pr_debug("Built Logout Response ITT: 0x%08x StatSN:" + " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", + cmd->init_task_tag, cmd->stat_sn, hdr->response, + cmd->logout_cid, conn->cid); + + return 0; +} +EXPORT_SYMBOL(iscsit_build_logout_rsp); + +static int +iscsit_send_logout(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct kvec *iov; + int niov = 0, tx_size, rc; + + rc = iscsit_build_logout_rsp(cmd, conn, + (struct iscsi_logout_rsp *)&cmd->pdu[0]); + if (rc < 0) + return rc; + + tx_size = ISCSI_HDR_LEN; iov = &cmd->iov_misc[0]; iov[niov].iov_base = cmd->pdu; iov[niov++].iov_len = ISCSI_HDR_LEN; @@ -2820,7 +2842,7 @@ static int iscsit_send_logout_response( u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, - (unsigned char *)hdr, ISCSI_HDR_LEN, + (unsigned char *)&cmd->pdu[0], ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); iov[0].iov_len += ISCSI_CRC_LEN; @@ -2831,14 +2853,37 @@ static int iscsit_send_logout_response( cmd->iov_misc_count = niov; cmd->tx_size = tx_size; - pr_debug("Sending Logout Response ITT: 0x%08x StatSN:" - " 0x%08x Response: 0x%02x CID: %hu on CID: %hu\n", - cmd->init_task_tag, cmd->stat_sn, hdr->response, - cmd->logout_cid, conn->cid); - return 0; } +void +iscsit_build_nopin_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_nopin *hdr, bool nopout_response) +{ + hdr->opcode = ISCSI_OP_NOOP_IN; + hdr->flags |= ISCSI_FLAG_CMD_FINAL; + hton24(hdr->dlength, cmd->buf_ptr_size); + if (nopout_response) + put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); + hdr->itt = cmd->init_task_tag; + hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); + cmd->stat_sn = (nopout_response) ? conn->stat_sn++ : + conn->stat_sn; + hdr->statsn = cpu_to_be32(cmd->stat_sn); + + if (nopout_response) + iscsit_increment_maxcmdsn(cmd, conn->sess); + + hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); + hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + + pr_debug("Built NOPIN %s Response ITT: 0x%08x, TTT: 0x%08x," + " StatSN: 0x%08x, Length %u\n", (nopout_response) ? + "Solicitied" : "Unsolicitied", cmd->init_task_tag, + cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); +} +EXPORT_SYMBOL(iscsit_build_nopin_rsp); + /* * Unsolicited NOPIN, either requesting a response or not. */ @@ -2847,20 +2892,10 @@ static int iscsit_send_unsolicited_nopin( struct iscsi_conn *conn, int want_response) { - int tx_size = ISCSI_HDR_LEN; - struct iscsi_nopin *hdr; - int ret; + struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; + int tx_size = ISCSI_HDR_LEN, ret; - hdr = (struct iscsi_nopin *) cmd->pdu; - memset(hdr, 0, ISCSI_HDR_LEN); - hdr->opcode = ISCSI_OP_NOOP_IN; - hdr->flags |= ISCSI_FLAG_CMD_FINAL; - hdr->itt = cmd->init_task_tag; - hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); - cmd->stat_sn = conn->stat_sn; - hdr->statsn = cpu_to_be32(cmd->stat_sn); - hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); - hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + iscsit_build_nopin_rsp(cmd, conn, hdr, false); if (conn->conn_ops->HeaderDigest) { u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; @@ -2896,31 +2931,17 @@ static int iscsit_send_unsolicited_nopin( return 0; } -static int iscsit_send_nopin_response( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) +static int +iscsit_send_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn) { - int niov = 0, tx_size; - u32 padding = 0; + struct iscsi_nopin *hdr = (struct iscsi_nopin *)&cmd->pdu[0]; struct kvec *iov; - struct iscsi_nopin *hdr; + u32 padding = 0; + int niov = 0, tx_size; + + iscsit_build_nopin_rsp(cmd, conn, hdr, true); tx_size = ISCSI_HDR_LEN; - hdr = (struct iscsi_nopin *) cmd->pdu; - memset(hdr, 0, ISCSI_HDR_LEN); - hdr->opcode = ISCSI_OP_NOOP_IN; - hdr->flags |= ISCSI_FLAG_CMD_FINAL; - hton24(hdr->dlength, cmd->buf_ptr_size); - put_unaligned_le64(0xFFFFFFFFFFFFFFFFULL, &hdr->lun); - hdr->itt = cmd->init_task_tag; - hdr->ttt = cpu_to_be32(cmd->targ_xfer_tag); - cmd->stat_sn = conn->stat_sn++; - hdr->statsn = cpu_to_be32(cmd->stat_sn); - - iscsit_increment_maxcmdsn(cmd, conn->sess); - hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); - hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); - iov = &cmd->iov_misc[0]; iov[niov].iov_base = cmd->pdu; iov[niov++].iov_len = ISCSI_HDR_LEN; @@ -2976,10 +2997,6 @@ static int iscsit_send_nopin_response( cmd->iov_misc_count = niov; cmd->tx_size = tx_size; - pr_debug("Sending NOPIN Response ITT: 0x%08x, TTT:" - " 0x%08x, StatSN: 0x%08x, Length %u\n", cmd->init_task_tag, - cmd->targ_xfer_tag, cmd->stat_sn, cmd->buf_ptr_size); - return 0; } @@ -3140,24 +3157,16 @@ int iscsit_build_r2ts_for_cmd( return 0; } -static int iscsit_send_status( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) +void iscsit_build_rsp_pdu(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + bool inc_stat_sn, struct iscsi_scsi_rsp *hdr) { - u8 iov_count = 0, recovery; - u32 padding = 0, tx_size = 0; - struct iscsi_scsi_rsp *hdr; - struct kvec *iov; - - recovery = (cmd->i_state != ISTATE_SEND_STATUS); - if (!recovery) + if (inc_stat_sn) cmd->stat_sn = conn->stat_sn++; spin_lock_bh(&conn->sess->session_stats_lock); conn->sess->rsp_pdus++; spin_unlock_bh(&conn->sess->session_stats_lock); - hdr = (struct iscsi_scsi_rsp *) cmd->pdu; memset(hdr, 0, ISCSI_HDR_LEN); hdr->opcode = ISCSI_OP_SCSI_CMD_RSP; hdr->flags |= ISCSI_FLAG_CMD_FINAL; @@ -3177,6 +3186,23 @@ static int iscsit_send_status( hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + pr_debug("Built SCSI Response, ITT: 0x%08x, StatSN: 0x%08x," + " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", + cmd->init_task_tag, cmd->stat_sn, cmd->se_cmd.scsi_status, + cmd->se_cmd.scsi_status, conn->cid); +} +EXPORT_SYMBOL(iscsit_build_rsp_pdu); + +static int iscsit_send_response(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *)&cmd->pdu[0]; + struct kvec *iov; + u32 padding = 0, tx_size = 0; + int iov_count = 0; + bool inc_stat_sn = (cmd->i_state == ISTATE_SEND_STATUS); + + iscsit_build_rsp_pdu(cmd, conn, inc_stat_sn, hdr); + iov = &cmd->iov_misc[0]; iov[iov_count].iov_base = cmd->pdu; iov[iov_count++].iov_len = ISCSI_HDR_LEN; @@ -3231,7 +3257,7 @@ static int iscsit_send_status( u32 *header_digest = (u32 *)&cmd->pdu[ISCSI_HDR_LEN]; iscsit_do_crypto_hash_buf(&conn->conn_tx_hash, - (unsigned char *)hdr, ISCSI_HDR_LEN, + (unsigned char *)cmd->pdu, ISCSI_HDR_LEN, 0, NULL, (u8 *)header_digest); iov[0].iov_len += ISCSI_CRC_LEN; @@ -3243,11 +3269,6 @@ static int iscsit_send_status( cmd->iov_misc_count = iov_count; cmd->tx_size = tx_size; - pr_debug("Built %sSCSI Response, ITT: 0x%08x, StatSN: 0x%08x," - " Response: 0x%02x, SAM Status: 0x%02x, CID: %hu\n", - (!recovery) ? "" : "Recovery ", cmd->init_task_tag, - cmd->stat_sn, 0x00, cmd->se_cmd.scsi_status, conn->cid); - return 0; } @@ -3270,16 +3291,12 @@ static u8 iscsit_convert_tcm_tmr_rsp(struct se_tmr_req *se_tmr) } } -static int iscsit_send_task_mgt_rsp( - struct iscsi_cmd *cmd, - struct iscsi_conn *conn) +void +iscsit_build_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_tm_rsp *hdr) { struct se_tmr_req *se_tmr = cmd->se_cmd.se_tmr_req; - struct iscsi_tm_rsp *hdr; - u32 tx_size = 0; - hdr = (struct iscsi_tm_rsp *) cmd->pdu; - memset(hdr, 0, ISCSI_HDR_LEN); hdr->opcode = ISCSI_OP_SCSI_TMFUNC_RSP; hdr->flags = ISCSI_FLAG_CMD_FINAL; hdr->response = iscsit_convert_tcm_tmr_rsp(se_tmr); @@ -3291,6 +3308,20 @@ static int iscsit_send_task_mgt_rsp( hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + pr_debug("Built Task Management Response ITT: 0x%08x," + " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", + cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); +} +EXPORT_SYMBOL(iscsit_build_task_mgt_rsp); + +static int +iscsit_send_task_mgt_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct iscsi_tm_rsp *hdr = (struct iscsi_tm_rsp *)&cmd->pdu[0]; + u32 tx_size = 0; + + iscsit_build_task_mgt_rsp(cmd, conn, hdr); + cmd->iov_misc[0].iov_base = cmd->pdu; cmd->iov_misc[0].iov_len = ISCSI_HDR_LEN; tx_size += ISCSI_HDR_LEN; @@ -3311,10 +3342,6 @@ static int iscsit_send_task_mgt_rsp( cmd->iov_misc_count = 1; cmd->tx_size = tx_size; - pr_debug("Built Task Management Response ITT: 0x%08x," - " StatSN: 0x%08x, Response: 0x%02x, CID: %hu\n", - cmd->init_task_tag, cmd->stat_sn, hdr->response, conn->cid); - return 0; } @@ -3510,6 +3537,22 @@ static int iscsit_send_text_rsp( return 0; } +void +iscsit_build_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + struct iscsi_reject *hdr) +{ + hdr->opcode = ISCSI_OP_REJECT; + hdr->flags |= ISCSI_FLAG_CMD_FINAL; + hton24(hdr->dlength, ISCSI_HDR_LEN); + hdr->ffffffff = cpu_to_be32(0xffffffff); + cmd->stat_sn = conn->stat_sn++; + hdr->statsn = cpu_to_be32(cmd->stat_sn); + hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); + hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + +} +EXPORT_SYMBOL(iscsit_build_reject); + static int iscsit_send_reject( struct iscsi_cmd *cmd, struct iscsi_conn *conn) @@ -3518,18 +3561,9 @@ static int iscsit_send_reject( struct iscsi_reject *hdr; struct kvec *iov; - hdr = (struct iscsi_reject *) cmd->pdu; - hdr->opcode = ISCSI_OP_REJECT; - hdr->flags |= ISCSI_FLAG_CMD_FINAL; - hton24(hdr->dlength, ISCSI_HDR_LEN); - hdr->ffffffff = cpu_to_be32(0xffffffff); - cmd->stat_sn = conn->stat_sn++; - hdr->statsn = cpu_to_be32(cmd->stat_sn); - hdr->exp_cmdsn = cpu_to_be32(conn->sess->exp_cmd_sn); - hdr->max_cmdsn = cpu_to_be32(conn->sess->max_cmd_sn); + iscsit_build_reject(cmd, conn, (struct iscsi_reject *)&cmd->pdu[0]); iov = &cmd->iov_misc[0]; - iov[iov_count].iov_base = cmd->pdu; iov[iov_count++].iov_len = ISCSI_HDR_LEN; iov[iov_count].iov_base = cmd->buf_ptr; @@ -3626,8 +3660,53 @@ static inline void iscsit_thread_check_cpumask( set_cpus_allowed_ptr(p, conn->conn_cpumask); } -static int handle_immediate_queue(struct iscsi_conn *conn) +static int +iscsit_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) { + int ret; + + switch (state) { + case ISTATE_SEND_R2T: + ret = iscsit_send_r2t(cmd, conn); + if (ret < 0) + goto err; + break; + case ISTATE_REMOVE: + spin_lock_bh(&conn->cmd_lock); + list_del(&cmd->i_conn_node); + spin_unlock_bh(&conn->cmd_lock); + + iscsit_free_cmd(cmd); + break; + case ISTATE_SEND_NOPIN_WANT_RESPONSE: + iscsit_mod_nopin_response_timer(conn); + ret = iscsit_send_unsolicited_nopin(cmd, conn, 1); + if (ret < 0) + goto err; + break; + case ISTATE_SEND_NOPIN_NO_RESPONSE: + ret = iscsit_send_unsolicited_nopin(cmd, conn, 0); + if (ret < 0) + goto err; + break; + default: + pr_err("Unknown Opcode: 0x%02x ITT:" + " 0x%08x, i_state: %d on CID: %hu\n", + cmd->iscsi_opcode, cmd->init_task_tag, state, + conn->cid); + goto err; + } + + return 0; + +err: + return -1; +} + +static int +iscsit_handle_immediate_queue(struct iscsi_conn *conn) +{ + struct iscsit_transport *t = conn->conn_transport; struct iscsi_queue_req *qr; struct iscsi_cmd *cmd; u8 state; @@ -3639,52 +3718,139 @@ static int handle_immediate_queue(struct iscsi_conn *conn) state = qr->state; kmem_cache_free(lio_qr_cache, qr); - switch (state) { - case ISTATE_SEND_R2T: - ret = iscsit_send_r2t(cmd, conn); - if (ret < 0) - goto err; - break; - case ISTATE_REMOVE: - if (cmd->data_direction == DMA_TO_DEVICE) - iscsit_stop_dataout_timer(cmd); + ret = t->iscsit_immediate_queue(conn, cmd, state); + if (ret < 0) + return ret; + } - spin_lock_bh(&conn->cmd_lock); - list_del(&cmd->i_conn_node); - spin_unlock_bh(&conn->cmd_lock); + return 0; +} - iscsit_free_cmd(cmd); - continue; - case ISTATE_SEND_NOPIN_WANT_RESPONSE: - iscsit_mod_nopin_response_timer(conn); - ret = iscsit_send_unsolicited_nopin(cmd, - conn, 1); - if (ret < 0) - goto err; +static int +iscsit_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) +{ + int ret; + +check_rsp_state: + switch (state) { + case ISTATE_SEND_DATAIN: + ret = iscsit_send_datain(cmd, conn); + if (ret < 0) + goto err; + else if (!ret) + /* more drs */ + goto check_rsp_state; + else if (ret == 1) { + /* all done */ + spin_lock_bh(&cmd->istate_lock); + cmd->i_state = ISTATE_SENT_STATUS; + spin_unlock_bh(&cmd->istate_lock); + + if (atomic_read(&conn->check_immediate_queue)) + return 1; + + return 0; + } else if (ret == 2) { + /* Still must send status, + SCF_TRANSPORT_TASK_SENSE was set */ + spin_lock_bh(&cmd->istate_lock); + cmd->i_state = ISTATE_SEND_STATUS; + spin_unlock_bh(&cmd->istate_lock); + state = ISTATE_SEND_STATUS; + goto check_rsp_state; + } + + break; + case ISTATE_SEND_STATUS: + case ISTATE_SEND_STATUS_RECOVERY: + ret = iscsit_send_response(cmd, conn); + break; + case ISTATE_SEND_LOGOUTRSP: + ret = iscsit_send_logout(cmd, conn); + break; + case ISTATE_SEND_ASYNCMSG: + ret = iscsit_send_conn_drop_async_message( + cmd, conn); + break; + case ISTATE_SEND_NOPIN: + ret = iscsit_send_nopin(cmd, conn); + break; + case ISTATE_SEND_REJECT: + ret = iscsit_send_reject(cmd, conn); + break; + case ISTATE_SEND_TASKMGTRSP: + ret = iscsit_send_task_mgt_rsp(cmd, conn); + if (ret != 0) break; - case ISTATE_SEND_NOPIN_NO_RESPONSE: - ret = iscsit_send_unsolicited_nopin(cmd, - conn, 0); - if (ret < 0) - goto err; - break; - default: - pr_err("Unknown Opcode: 0x%02x ITT:" - " 0x%08x, i_state: %d on CID: %hu\n", - cmd->iscsi_opcode, cmd->init_task_tag, state, - conn->cid); + ret = iscsit_tmr_post_handler(cmd, conn); + if (ret != 0) + iscsit_fall_back_to_erl0(conn->sess); + break; + case ISTATE_SEND_TEXTRSP: + ret = iscsit_send_text_rsp(cmd, conn); + break; + default: + pr_err("Unknown Opcode: 0x%02x ITT:" + " 0x%08x, i_state: %d on CID: %hu\n", + cmd->iscsi_opcode, cmd->init_task_tag, + state, conn->cid); + goto err; + } + if (ret < 0) + goto err; + + if (iscsit_send_tx_data(cmd, conn, 1) < 0) { + iscsit_tx_thread_wait_for_tcp(conn); + iscsit_unmap_iovec(cmd); + goto err; + } + iscsit_unmap_iovec(cmd); + + switch (state) { + case ISTATE_SEND_LOGOUTRSP: + if (!iscsit_logout_post_handler(cmd, conn)) + goto restart; + /* fall through */ + case ISTATE_SEND_STATUS: + case ISTATE_SEND_ASYNCMSG: + case ISTATE_SEND_NOPIN: + case ISTATE_SEND_STATUS_RECOVERY: + case ISTATE_SEND_TEXTRSP: + case ISTATE_SEND_TASKMGTRSP: + spin_lock_bh(&cmd->istate_lock); + cmd->i_state = ISTATE_SENT_STATUS; + spin_unlock_bh(&cmd->istate_lock); + break; + case ISTATE_SEND_REJECT: + if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) { + cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN; + complete(&cmd->reject_comp); goto err; } + complete(&cmd->reject_comp); + break; + default: + pr_err("Unknown Opcode: 0x%02x ITT:" + " 0x%08x, i_state: %d on CID: %hu\n", + cmd->iscsi_opcode, cmd->init_task_tag, + cmd->i_state, conn->cid); + goto err; } + if (atomic_read(&conn->check_immediate_queue)) + return 1; + return 0; err: return -1; +restart: + return -EAGAIN; } -static int handle_response_queue(struct iscsi_conn *conn) +static int iscsit_handle_response_queue(struct iscsi_conn *conn) { + struct iscsit_transport *t = conn->conn_transport; struct iscsi_queue_req *qr; struct iscsi_cmd *cmd; u8 state; @@ -3695,122 +3861,12 @@ static int handle_response_queue(struct iscsi_conn *conn) state = qr->state; kmem_cache_free(lio_qr_cache, qr); -check_rsp_state: - switch (state) { - case ISTATE_SEND_DATAIN: - ret = iscsit_send_data_in(cmd, conn); - if (ret < 0) - goto err; - else if (!ret) - /* more drs */ - goto check_rsp_state; - else if (ret == 1) { - /* all done */ - spin_lock_bh(&cmd->istate_lock); - cmd->i_state = ISTATE_SENT_STATUS; - spin_unlock_bh(&cmd->istate_lock); - - if (atomic_read(&conn->check_immediate_queue)) - return 1; - - continue; - } else if (ret == 2) { - /* Still must send status, - SCF_TRANSPORT_TASK_SENSE was set */ - spin_lock_bh(&cmd->istate_lock); - cmd->i_state = ISTATE_SEND_STATUS; - spin_unlock_bh(&cmd->istate_lock); - state = ISTATE_SEND_STATUS; - goto check_rsp_state; - } - - break; - case ISTATE_SEND_STATUS: - case ISTATE_SEND_STATUS_RECOVERY: - ret = iscsit_send_status(cmd, conn); - break; - case ISTATE_SEND_LOGOUTRSP: - ret = iscsit_send_logout_response(cmd, conn); - break; - case ISTATE_SEND_ASYNCMSG: - ret = iscsit_send_conn_drop_async_message( - cmd, conn); - break; - case ISTATE_SEND_NOPIN: - ret = iscsit_send_nopin_response(cmd, conn); - break; - case ISTATE_SEND_REJECT: - ret = iscsit_send_reject(cmd, conn); - break; - case ISTATE_SEND_TASKMGTRSP: - ret = iscsit_send_task_mgt_rsp(cmd, conn); - if (ret != 0) - break; - ret = iscsit_tmr_post_handler(cmd, conn); - if (ret != 0) - iscsit_fall_back_to_erl0(conn->sess); - break; - case ISTATE_SEND_TEXTRSP: - ret = iscsit_send_text_rsp(cmd, conn); - break; - default: - pr_err("Unknown Opcode: 0x%02x ITT:" - " 0x%08x, i_state: %d on CID: %hu\n", - cmd->iscsi_opcode, cmd->init_task_tag, - state, conn->cid); - goto err; - } - if (ret < 0) - goto err; - - if (iscsit_send_tx_data(cmd, conn, 1) < 0) { - iscsit_tx_thread_wait_for_tcp(conn); - iscsit_unmap_iovec(cmd); - goto err; - } - iscsit_unmap_iovec(cmd); - - switch (state) { - case ISTATE_SEND_LOGOUTRSP: - if (!iscsit_logout_post_handler(cmd, conn)) - goto restart; - /* fall through */ - case ISTATE_SEND_STATUS: - case ISTATE_SEND_ASYNCMSG: - case ISTATE_SEND_NOPIN: - case ISTATE_SEND_STATUS_RECOVERY: - case ISTATE_SEND_TEXTRSP: - case ISTATE_SEND_TASKMGTRSP: - spin_lock_bh(&cmd->istate_lock); - cmd->i_state = ISTATE_SENT_STATUS; - spin_unlock_bh(&cmd->istate_lock); - break; - case ISTATE_SEND_REJECT: - if (cmd->cmd_flags & ICF_REJECT_FAIL_CONN) { - cmd->cmd_flags &= ~ICF_REJECT_FAIL_CONN; - complete(&cmd->reject_comp); - goto err; - } - complete(&cmd->reject_comp); - break; - default: - pr_err("Unknown Opcode: 0x%02x ITT:" - " 0x%08x, i_state: %d on CID: %hu\n", - cmd->iscsi_opcode, cmd->init_task_tag, - cmd->i_state, conn->cid); - goto err; - } - - if (atomic_read(&conn->check_immediate_queue)) - return 1; + ret = t->iscsit_response_queue(conn, cmd, state); + if (ret == 1 || ret < 0) + return ret; } return 0; - -err: - return -1; -restart: - return -EAGAIN; } int iscsi_target_tx_thread(void *arg) @@ -3847,11 +3903,11 @@ int iscsi_target_tx_thread(void *arg) goto transport_err; get_immediate: - ret = handle_immediate_queue(conn); + ret = iscsit_handle_immediate_queue(conn); if (ret < 0) goto transport_err; - ret = handle_response_queue(conn); + ret = iscsit_handle_response_queue(conn); if (ret == 1) goto get_immediate; else if (ret == -EAGAIN) @@ -4451,7 +4507,7 @@ static void iscsit_logout_post_handler_diffcid( /* * Return of 0 causes the TX thread to restart. */ -static int iscsit_logout_post_handler( +int iscsit_logout_post_handler( struct iscsi_cmd *cmd, struct iscsi_conn *conn) { @@ -4509,6 +4565,7 @@ static int iscsit_logout_post_handler( } return ret; } +EXPORT_SYMBOL(iscsit_logout_post_handler); void iscsit_fail_session(struct iscsi_session *sess) { diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c index ce3d3210fe41..2704daf6ba74 100644 --- a/drivers/target/iscsi/iscsi_target_configfs.c +++ b/drivers/target/iscsi/iscsi_target_configfs.c @@ -27,6 +27,7 @@ #include #include #include +#include #include "iscsi_target_core.h" #include "iscsi_target_parameters.h" @@ -1536,7 +1537,8 @@ static int lio_queue_data_in(struct se_cmd *se_cmd) struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd); cmd->i_state = ISTATE_SEND_DATAIN; - iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); + cmd->conn->conn_transport->iscsit_queue_data_in(cmd->conn, cmd); + return 0; } @@ -1568,7 +1570,8 @@ static int lio_queue_status(struct se_cmd *se_cmd) struct iscsi_cmd *cmd = container_of(se_cmd, struct iscsi_cmd, se_cmd); cmd->i_state = ISTATE_SEND_STATUS; - iscsit_add_cmd_to_response_queue(cmd, cmd->conn, cmd->i_state); + cmd->conn->conn_transport->iscsit_queue_status(cmd->conn, cmd); + return 0; } diff --git a/drivers/target/iscsi/iscsi_target_device.c b/drivers/target/iscsi/iscsi_target_device.c index bcc409853a67..1b74033510a0 100644 --- a/drivers/target/iscsi/iscsi_target_device.c +++ b/drivers/target/iscsi/iscsi_target_device.c @@ -60,8 +60,13 @@ void iscsit_increment_maxcmdsn(struct iscsi_cmd *cmd, struct iscsi_session *sess cmd->maxcmdsn_inc = 1; - mutex_lock(&sess->cmdsn_mutex); + if (!mutex_trylock(&sess->cmdsn_mutex)) { + sess->max_cmd_sn += 1; + pr_debug("Updated MaxCmdSN to 0x%08x\n", sess->max_cmd_sn); + return; + } sess->max_cmd_sn += 1; pr_debug("Updated MaxCmdSN to 0x%08x\n", sess->max_cmd_sn); mutex_unlock(&sess->cmdsn_mutex); } +EXPORT_SYMBOL(iscsit_increment_maxcmdsn); diff --git a/drivers/target/iscsi/iscsi_target_tmr.c b/drivers/target/iscsi/iscsi_target_tmr.c index 16ca0ef9266f..b997e5da47d3 100644 --- a/drivers/target/iscsi/iscsi_target_tmr.c +++ b/drivers/target/iscsi/iscsi_target_tmr.c @@ -472,6 +472,7 @@ int iscsit_tmr_post_handler(struct iscsi_cmd *cmd, struct iscsi_conn *conn) return 0; } +EXPORT_SYMBOL(iscsit_tmr_post_handler); /* * Nothing to do here, but leave it for good measure. :-) From 72438cddd6d39b293dbc37836a4d3e691d26c356 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Wed, 6 Mar 2013 22:20:36 -0800 Subject: [PATCH 19/25] iscsi-target: Add iser network portal attribute This patch adds a new network portal attribute for iser, that lives under existing iscsi-target configfs layout at: /sys/kernel/config/target/iscsi/$TARGETNAME/$TPGT/np/$PORTAL/iser When lio_target_np_store_iser() is enabled, iscsit_tpg_add_network_portal() will attempt to start an rdma_cma network portal for iser-target, only if the external ib_isert module transport has been loaded. When disabled, iscsit_tpg_del_network_portal() will cease iser login service on the network portal, and release any external ib_isert module reference. v4 changes: - Add request_module for ib_isert to lio_target_np_store_iser() Signed-off-by: Nicholas Bellinger --- drivers/target/iscsi/iscsi_target_configfs.c | 79 ++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/drivers/target/iscsi/iscsi_target_configfs.c b/drivers/target/iscsi/iscsi_target_configfs.c index 2704daf6ba74..13e9e715ad2e 100644 --- a/drivers/target/iscsi/iscsi_target_configfs.c +++ b/drivers/target/iscsi/iscsi_target_configfs.c @@ -125,8 +125,87 @@ static ssize_t lio_target_np_store_sctp( TF_NP_BASE_ATTR(lio_target, sctp, S_IRUGO | S_IWUSR); +static ssize_t lio_target_np_show_iser( + struct se_tpg_np *se_tpg_np, + char *page) +{ + struct iscsi_tpg_np *tpg_np = container_of(se_tpg_np, + struct iscsi_tpg_np, se_tpg_np); + struct iscsi_tpg_np *tpg_np_iser; + ssize_t rb; + + tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND); + if (tpg_np_iser) + rb = sprintf(page, "1\n"); + else + rb = sprintf(page, "0\n"); + + return rb; +} + +static ssize_t lio_target_np_store_iser( + struct se_tpg_np *se_tpg_np, + const char *page, + size_t count) +{ + struct iscsi_np *np; + struct iscsi_portal_group *tpg; + struct iscsi_tpg_np *tpg_np = container_of(se_tpg_np, + struct iscsi_tpg_np, se_tpg_np); + struct iscsi_tpg_np *tpg_np_iser = NULL; + char *endptr; + u32 op; + int rc; + + op = simple_strtoul(page, &endptr, 0); + if ((op != 1) && (op != 0)) { + pr_err("Illegal value for tpg_enable: %u\n", op); + return -EINVAL; + } + np = tpg_np->tpg_np; + if (!np) { + pr_err("Unable to locate struct iscsi_np from" + " struct iscsi_tpg_np\n"); + return -EINVAL; + } + + tpg = tpg_np->tpg; + if (iscsit_get_tpg(tpg) < 0) + return -EINVAL; + + if (op) { + int rc = request_module("ib_isert"); + if (rc != 0) + pr_warn("Unable to request_module for ib_isert\n"); + + tpg_np_iser = iscsit_tpg_add_network_portal(tpg, &np->np_sockaddr, + np->np_ip, tpg_np, ISCSI_INFINIBAND); + if (!tpg_np_iser || IS_ERR(tpg_np_iser)) + goto out; + } else { + tpg_np_iser = iscsit_tpg_locate_child_np(tpg_np, ISCSI_INFINIBAND); + if (!tpg_np_iser) + goto out; + + rc = iscsit_tpg_del_network_portal(tpg, tpg_np_iser); + if (rc < 0) + goto out; + } + + printk("lio_target_np_store_iser() done, op: %d\n", op); + + iscsit_put_tpg(tpg); + return count; +out: + iscsit_put_tpg(tpg); + return -EINVAL; +} + +TF_NP_BASE_ATTR(lio_target, iser, S_IRUGO | S_IWUSR); + static struct configfs_attribute *lio_target_portal_attrs[] = { &lio_target_np_sctp.attr, + &lio_target_np_iser.attr, NULL, }; From d3682b1aeba088957a210e3abc2c71dba63d6def Mon Sep 17 00:00:00 2001 From: Mark Rustad Date: Mon, 22 Apr 2013 09:49:55 -0700 Subject: [PATCH 20/25] tcm_fc: Check for aborted sequence Add a check for an aborted sequence, which has a NULL sequence pointer, to avoid target crashes. The most relevant messages from the crash (entered from video capture) include: BUG: unable to handle kernel paging request at ffffffffffffffdf IP: [] fc_seq_send+0x3c/0x150 [libfc] ... Call Trace: [] ft_queue_data_in+0x266/0x560 [tcm_fc] Signed-off-by: Mark Rustad Signed-off-by: Nicholas Bellinger --- drivers/target/tcm_fc/tfc_io.c | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/drivers/target/tcm_fc/tfc_io.c b/drivers/target/tcm_fc/tfc_io.c index b6fd4cf42840..e415af32115a 100644 --- a/drivers/target/tcm_fc/tfc_io.c +++ b/drivers/target/tcm_fc/tfc_io.c @@ -103,6 +103,13 @@ int ft_queue_data_in(struct se_cmd *se_cmd) use_sg = !(remaining % 4); while (remaining) { + struct fc_seq *seq = cmd->seq; + + if (!seq) { + pr_debug("%s: Command aborted, xid 0x%x\n", + __func__, ep->xid); + break; + } if (!mem_len) { sg = sg_next(sg); mem_len = min((size_t)sg->length, remaining); @@ -169,7 +176,7 @@ int ft_queue_data_in(struct se_cmd *se_cmd) f_ctl |= FC_FC_END_SEQ; fc_fill_fc_hdr(fp, FC_RCTL_DD_SOL_DATA, ep->did, ep->sid, FC_TYPE_FCP, f_ctl, fh_off); - error = lport->tt.seq_send(lport, cmd->seq, fp); + error = lport->tt.seq_send(lport, seq, fp); if (error) { /* XXX For now, initiator will retry */ pr_err_ratelimited("%s: Failed to send frame %p, " From f2b7daf5b13f69ac09e706fb5b906478212a4100 Mon Sep 17 00:00:00 2001 From: Asias He Date: Thu, 25 Apr 2013 15:35:20 +0800 Subject: [PATCH 21/25] tcm_vhost: Refactor the lock nesting rule We want to use tcm_vhost_mutex to make sure hotplug/hotunplug will not happen when set_endpoint/clear_endpoint is in process. Signed-off-by: Asias He Acked-by: Michael S. Tsirkin Signed-off-by: Nicholas Bellinger --- drivers/vhost/tcm_vhost.c | 32 +++++++++++++++++++------------- 1 file changed, 19 insertions(+), 13 deletions(-) diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c index 957a0b98a5d9..822cd1fbff5e 100644 --- a/drivers/vhost/tcm_vhost.c +++ b/drivers/vhost/tcm_vhost.c @@ -808,6 +808,9 @@ static void vhost_scsi_flush(struct vhost_scsi *vs) /* * Called from vhost_scsi_ioctl() context to walk the list of available * tcm_vhost_tpg with an active struct tcm_vhost_nexus + * + * The lock nesting rule is: + * tcm_vhost_mutex -> vs->dev.mutex -> tpg->tv_tpg_mutex -> vq->mutex */ static int vhost_scsi_set_endpoint( struct vhost_scsi *vs, @@ -820,26 +823,27 @@ static int vhost_scsi_set_endpoint( int index, ret, i, len; bool match = false; + mutex_lock(&tcm_vhost_mutex); mutex_lock(&vs->dev.mutex); + /* Verify that ring has been setup correctly. */ for (index = 0; index < vs->dev.nvqs; ++index) { /* Verify that ring has been setup correctly. */ if (!vhost_vq_access_ok(&vs->vqs[index])) { - mutex_unlock(&vs->dev.mutex); - return -EFAULT; + ret = -EFAULT; + goto out; } } len = sizeof(vs_tpg[0]) * VHOST_SCSI_MAX_TARGET; vs_tpg = kzalloc(len, GFP_KERNEL); if (!vs_tpg) { - mutex_unlock(&vs->dev.mutex); - return -ENOMEM; + ret = -ENOMEM; + goto out; } if (vs->vs_tpg) memcpy(vs_tpg, vs->vs_tpg, len); - mutex_lock(&tcm_vhost_mutex); list_for_each_entry(tv_tpg, &tcm_vhost_list, tv_tpg_list) { mutex_lock(&tv_tpg->tv_tpg_mutex); if (!tv_tpg->tpg_nexus) { @@ -854,11 +858,10 @@ static int vhost_scsi_set_endpoint( if (!strcmp(tv_tport->tport_name, t->vhost_wwpn)) { if (vs->vs_tpg && vs->vs_tpg[tv_tpg->tport_tpgt]) { - mutex_unlock(&tv_tpg->tv_tpg_mutex); - mutex_unlock(&tcm_vhost_mutex); - mutex_unlock(&vs->dev.mutex); kfree(vs_tpg); - return -EEXIST; + mutex_unlock(&tv_tpg->tv_tpg_mutex); + ret = -EEXIST; + goto out; } tv_tpg->tv_tpg_vhost_count++; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg; @@ -867,7 +870,6 @@ static int vhost_scsi_set_endpoint( } mutex_unlock(&tv_tpg->tv_tpg_mutex); } - mutex_unlock(&tcm_vhost_mutex); if (match) { memcpy(vs->vs_vhost_wwpn, t->vhost_wwpn, @@ -893,7 +895,9 @@ static int vhost_scsi_set_endpoint( kfree(vs->vs_tpg); vs->vs_tpg = vs_tpg; +out: mutex_unlock(&vs->dev.mutex); + mutex_unlock(&tcm_vhost_mutex); return ret; } @@ -908,6 +912,7 @@ static int vhost_scsi_clear_endpoint( int index, ret, i; u8 target; + mutex_lock(&tcm_vhost_mutex); mutex_lock(&vs->dev.mutex); /* Verify that ring has been setup correctly. */ for (index = 0; index < vs->dev.nvqs; ++index) { @@ -918,8 +923,8 @@ static int vhost_scsi_clear_endpoint( } if (!vs->vs_tpg) { - mutex_unlock(&vs->dev.mutex); - return 0; + ret = 0; + goto err_dev; } for (i = 0; i < VHOST_SCSI_MAX_TARGET; i++) { @@ -965,13 +970,14 @@ static int vhost_scsi_clear_endpoint( kfree(vs->vs_tpg); vs->vs_tpg = NULL; mutex_unlock(&vs->dev.mutex); - + mutex_unlock(&tcm_vhost_mutex); return 0; err_tpg: mutex_unlock(&tv_tpg->tv_tpg_mutex); err_dev: mutex_unlock(&vs->dev.mutex); + mutex_unlock(&tcm_vhost_mutex); return ret; } From a6c9af87363c5e964d981fe0ec4aa73061a0c793 Mon Sep 17 00:00:00 2001 From: Asias He Date: Thu, 25 Apr 2013 15:35:21 +0800 Subject: [PATCH 22/25] tcm_vhost: Add hotplug/hotunplug support In commit 365a7150094 ([SCSI] virtio-scsi: hotplug support for virtio-scsi), hotplug support is added to virtio-scsi. This patch adds hotplug and hotunplug support to tcm_vhost. You can create or delete a LUN in targetcli to hotplug or hotunplug a LUN in guest. Signed-off-by: Asias He Reviewed-by: Stefan Hajnoczi Acked-by: Michael S. Tsirkin Signed-off-by: Nicholas Bellinger --- drivers/vhost/tcm_vhost.c | 210 +++++++++++++++++++++++++++++++++++++- drivers/vhost/tcm_vhost.h | 10 ++ 2 files changed, 218 insertions(+), 2 deletions(-) diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c index 822cd1fbff5e..5340fd759c6e 100644 --- a/drivers/vhost/tcm_vhost.c +++ b/drivers/vhost/tcm_vhost.c @@ -71,6 +71,7 @@ enum { #define VHOST_SCSI_MAX_TARGET 256 #define VHOST_SCSI_MAX_VQ 128 +#define VHOST_SCSI_MAX_EVENT 128 struct vhost_scsi { /* Protected by vhost_scsi->dev.mutex */ @@ -82,6 +83,12 @@ struct vhost_scsi { struct vhost_work vs_completion_work; /* cmd completion work item */ struct llist_head vs_completion_list; /* cmd completion queue */ + + struct vhost_work vs_event_work; /* evt injection work item */ + struct llist_head vs_event_list; /* evt injection queue */ + + bool vs_events_missed; /* any missed events, protected by vq->mutex */ + int vs_events_nr; /* num of pending events, protected by vq->mutex */ }; /* Local pointer to allocated TCM configfs fabric module */ @@ -349,6 +356,37 @@ static int tcm_vhost_queue_tm_rsp(struct se_cmd *se_cmd) return 0; } +static void tcm_vhost_free_evt(struct vhost_scsi *vs, struct tcm_vhost_evt *evt) +{ + vs->vs_events_nr--; + kfree(evt); +} + +static struct tcm_vhost_evt *tcm_vhost_allocate_evt(struct vhost_scsi *vs, + u32 event, u32 reason) +{ + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + struct tcm_vhost_evt *evt; + + if (vs->vs_events_nr > VHOST_SCSI_MAX_EVENT) { + vs->vs_events_missed = true; + return NULL; + } + + evt = kzalloc(sizeof(*evt), GFP_KERNEL); + if (!evt) { + vq_err(vq, "Failed to allocate tcm_vhost_evt\n"); + vs->vs_events_missed = true; + return NULL; + } + + evt->event.event = event; + evt->event.reason = reason; + vs->vs_events_nr++; + + return evt; +} + static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) { struct se_cmd *se_cmd = &tv_cmd->tvc_se_cmd; @@ -367,6 +405,75 @@ static void vhost_scsi_free_cmd(struct tcm_vhost_cmd *tv_cmd) kfree(tv_cmd); } +static void tcm_vhost_do_evt_work(struct vhost_scsi *vs, + struct tcm_vhost_evt *evt) +{ + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + struct virtio_scsi_event *event = &evt->event; + struct virtio_scsi_event __user *eventp; + unsigned out, in; + int head, ret; + + if (!vq->private_data) { + vs->vs_events_missed = true; + return; + } + +again: + vhost_disable_notify(&vs->dev, vq); + head = vhost_get_vq_desc(&vs->dev, vq, vq->iov, + ARRAY_SIZE(vq->iov), &out, &in, + NULL, NULL); + if (head < 0) { + vs->vs_events_missed = true; + return; + } + if (head == vq->num) { + if (vhost_enable_notify(&vs->dev, vq)) + goto again; + vs->vs_events_missed = true; + return; + } + + if ((vq->iov[out].iov_len != sizeof(struct virtio_scsi_event))) { + vq_err(vq, "Expecting virtio_scsi_event, got %zu bytes\n", + vq->iov[out].iov_len); + vs->vs_events_missed = true; + return; + } + + if (vs->vs_events_missed) { + event->event |= VIRTIO_SCSI_T_EVENTS_MISSED; + vs->vs_events_missed = false; + } + + eventp = vq->iov[out].iov_base; + ret = __copy_to_user(eventp, event, sizeof(*event)); + if (!ret) + vhost_add_used_and_signal(&vs->dev, vq, head, 0); + else + vq_err(vq, "Faulted on tcm_vhost_send_event\n"); +} + +static void tcm_vhost_evt_work(struct vhost_work *work) +{ + struct vhost_scsi *vs = container_of(work, struct vhost_scsi, + vs_event_work); + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + struct tcm_vhost_evt *evt; + struct llist_node *llnode; + + mutex_lock(&vq->mutex); + llnode = llist_del_all(&vs->vs_event_list); + while (llnode) { + evt = llist_entry(llnode, struct tcm_vhost_evt, list); + llnode = llist_next(llnode); + tcm_vhost_do_evt_work(vs, evt); + tcm_vhost_free_evt(vs, evt); + } + mutex_unlock(&vq->mutex); +} + /* Fill in status and signal that we are done processing this command * * This is scheduled in the vhost work queue so we are called with the owner @@ -777,9 +884,46 @@ static void vhost_scsi_ctl_handle_kick(struct vhost_work *work) pr_debug("%s: The handling func for control queue.\n", __func__); } +static void tcm_vhost_send_evt(struct vhost_scsi *vs, struct tcm_vhost_tpg *tpg, + struct se_lun *lun, u32 event, u32 reason) +{ + struct tcm_vhost_evt *evt; + + evt = tcm_vhost_allocate_evt(vs, event, reason); + if (!evt) + return; + + if (tpg && lun) { + /* TODO: share lun setup code with virtio-scsi.ko */ + /* + * Note: evt->event is zeroed when we allocate it and + * lun[4-7] need to be zero according to virtio-scsi spec. + */ + evt->event.lun[0] = 0x01; + evt->event.lun[1] = tpg->tport_tpgt & 0xFF; + if (lun->unpacked_lun >= 256) + evt->event.lun[2] = lun->unpacked_lun >> 8 | 0x40 ; + evt->event.lun[3] = lun->unpacked_lun & 0xFF; + } + + llist_add(&evt->list, &vs->vs_event_list); + vhost_work_queue(&vs->dev, &vs->vs_event_work); +} + static void vhost_scsi_evt_handle_kick(struct vhost_work *work) { - pr_debug("%s: The handling func for event queue.\n", __func__); + struct vhost_virtqueue *vq = container_of(work, struct vhost_virtqueue, + poll.work); + struct vhost_scsi *vs = container_of(vq->dev, struct vhost_scsi, dev); + + mutex_lock(&vq->mutex); + if (!vq->private_data) + goto out; + + if (vs->vs_events_missed) + tcm_vhost_send_evt(vs, NULL, NULL, VIRTIO_SCSI_T_NO_EVENT, 0); +out: + mutex_unlock(&vq->mutex); } static void vhost_scsi_handle_kick(struct vhost_work *work) @@ -803,6 +947,7 @@ static void vhost_scsi_flush(struct vhost_scsi *vs) for (i = 0; i < VHOST_SCSI_MAX_VQ; i++) vhost_scsi_flush_vq(vs, i); vhost_work_flush(&vs->dev, &vs->vs_completion_work); + vhost_work_flush(&vs->dev, &vs->vs_event_work); } /* @@ -864,6 +1009,7 @@ static int vhost_scsi_set_endpoint( goto out; } tv_tpg->tv_tpg_vhost_count++; + tv_tpg->vhost_scsi = vs; vs_tpg[tv_tpg->tport_tpgt] = tv_tpg; smp_mb__after_atomic_inc(); match = true; @@ -949,6 +1095,7 @@ static int vhost_scsi_clear_endpoint( goto err_tpg; } tv_tpg->tv_tpg_vhost_count--; + tv_tpg->vhost_scsi = NULL; vs->vs_tpg[target] = NULL; match = true; mutex_unlock(&tv_tpg->tv_tpg_mutex); @@ -969,6 +1116,7 @@ static int vhost_scsi_clear_endpoint( vhost_scsi_flush(vs); kfree(vs->vs_tpg); vs->vs_tpg = NULL; + WARN_ON(vs->vs_events_nr); mutex_unlock(&vs->dev.mutex); mutex_unlock(&tcm_vhost_mutex); return 0; @@ -1009,6 +1157,10 @@ static int vhost_scsi_open(struct inode *inode, struct file *f) return -ENOMEM; vhost_work_init(&s->vs_completion_work, vhost_scsi_complete_cmd_work); + vhost_work_init(&s->vs_event_work, tcm_vhost_evt_work); + + s->vs_events_nr = 0; + s->vs_events_missed = false; s->vqs[VHOST_SCSI_VQ_CTL].handle_kick = vhost_scsi_ctl_handle_kick; s->vqs[VHOST_SCSI_VQ_EVT].handle_kick = vhost_scsi_evt_handle_kick; @@ -1035,6 +1187,8 @@ static int vhost_scsi_release(struct inode *inode, struct file *f) vhost_scsi_clear_endpoint(s, &t); vhost_dev_stop(&s->dev); vhost_dev_cleanup(&s->dev, false); + /* Jobs can re-queue themselves in evt kick handler. Do extra flush. */ + vhost_scsi_flush(s); kfree(s); return 0; } @@ -1139,28 +1293,80 @@ static char *tcm_vhost_dump_proto_id(struct tcm_vhost_tport *tport) return "Unknown"; } +static void tcm_vhost_do_plug(struct tcm_vhost_tpg *tpg, + struct se_lun *lun, bool plug) +{ + + struct vhost_scsi *vs = tpg->vhost_scsi; + struct vhost_virtqueue *vq; + u32 reason; + + if (!vs) + return; + + mutex_lock(&vs->dev.mutex); + if (!vhost_has_feature(&vs->dev, VIRTIO_SCSI_F_HOTPLUG)) { + mutex_unlock(&vs->dev.mutex); + return; + } + + if (plug) + reason = VIRTIO_SCSI_EVT_RESET_RESCAN; + else + reason = VIRTIO_SCSI_EVT_RESET_REMOVED; + + vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; + mutex_lock(&vq->mutex); + tcm_vhost_send_evt(vs, tpg, lun, + VIRTIO_SCSI_T_TRANSPORT_RESET, reason); + mutex_unlock(&vq->mutex); + mutex_unlock(&vs->dev.mutex); +} + +static void tcm_vhost_hotplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun) +{ + tcm_vhost_do_plug(tpg, lun, true); +} + +static void tcm_vhost_hotunplug(struct tcm_vhost_tpg *tpg, struct se_lun *lun) +{ + tcm_vhost_do_plug(tpg, lun, false); +} + static int tcm_vhost_port_link(struct se_portal_group *se_tpg, struct se_lun *lun) { struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); + mutex_lock(&tcm_vhost_mutex); + mutex_lock(&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count++; mutex_unlock(&tv_tpg->tv_tpg_mutex); + tcm_vhost_hotplug(tv_tpg, lun); + + mutex_unlock(&tcm_vhost_mutex); + return 0; } static void tcm_vhost_port_unlink(struct se_portal_group *se_tpg, - struct se_lun *se_lun) + struct se_lun *lun) { struct tcm_vhost_tpg *tv_tpg = container_of(se_tpg, struct tcm_vhost_tpg, se_tpg); + mutex_lock(&tcm_vhost_mutex); + mutex_lock(&tv_tpg->tv_tpg_mutex); tv_tpg->tv_tpg_port_count--; mutex_unlock(&tv_tpg->tv_tpg_mutex); + + tcm_vhost_hotunplug(tv_tpg, lun); + + mutex_unlock(&tcm_vhost_mutex); } static struct se_node_acl *tcm_vhost_make_nodeacl( diff --git a/drivers/vhost/tcm_vhost.h b/drivers/vhost/tcm_vhost.h index 1d2ae7a60e11..a545a5b766a3 100644 --- a/drivers/vhost/tcm_vhost.h +++ b/drivers/vhost/tcm_vhost.h @@ -53,6 +53,7 @@ struct tcm_vhost_nacl { struct se_node_acl se_node_acl; }; +struct vhost_scsi; struct tcm_vhost_tpg { /* Vhost port target portal group tag for TCM */ u16 tport_tpgt; @@ -70,6 +71,8 @@ struct tcm_vhost_tpg { struct tcm_vhost_tport *tport; /* Returned by tcm_vhost_make_tpg() */ struct se_portal_group se_tpg; + /* Pointer back to vhost_scsi, protected by tv_tpg_mutex */ + struct vhost_scsi *vhost_scsi; }; struct tcm_vhost_tport { @@ -83,6 +86,13 @@ struct tcm_vhost_tport { struct se_wwn tport_wwn; }; +struct tcm_vhost_evt { + /* event to be sent to guest */ + struct virtio_scsi_event event; + /* event list, serviced from vhost worker thread */ + struct llist_node list; +}; + /* * As per request from MST, keep TCM_VHOST related ioctl defines out of * linux/vhost.h (user-space) for now.. From 11c634183961c1f22a44167a676d0610cee3edaa Mon Sep 17 00:00:00 2001 From: Asias He Date: Thu, 25 Apr 2013 15:35:22 +0800 Subject: [PATCH 23/25] tcm_vhost: Add ioctl to get and set events missed flag Signed-off-by: Asias He Acked-by: Michael S. Tsirkin Signed-off-by: Nicholas Bellinger --- drivers/vhost/tcm_vhost.c | 17 +++++++++++++++++ drivers/vhost/tcm_vhost.h | 3 +++ 2 files changed, 20 insertions(+) diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c index 5340fd759c6e..07217d818ad7 100644 --- a/drivers/vhost/tcm_vhost.c +++ b/drivers/vhost/tcm_vhost.c @@ -1200,8 +1200,11 @@ static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl, struct vhost_scsi_target backend; void __user *argp = (void __user *)arg; u64 __user *featurep = argp; + u32 __user *eventsp = argp; + u32 events_missed; u64 features; int r, abi_version = VHOST_SCSI_ABI_VERSION; + struct vhost_virtqueue *vq = &vs->vqs[VHOST_SCSI_VQ_EVT]; switch (ioctl) { case VHOST_SCSI_SET_ENDPOINT: @@ -1222,6 +1225,20 @@ static long vhost_scsi_ioctl(struct file *f, unsigned int ioctl, if (copy_to_user(argp, &abi_version, sizeof abi_version)) return -EFAULT; return 0; + case VHOST_SCSI_SET_EVENTS_MISSED: + if (get_user(events_missed, eventsp)) + return -EFAULT; + mutex_lock(&vq->mutex); + vs->vs_events_missed = events_missed; + mutex_unlock(&vq->mutex); + return 0; + case VHOST_SCSI_GET_EVENTS_MISSED: + mutex_lock(&vq->mutex); + events_missed = vs->vs_events_missed; + mutex_unlock(&vq->mutex); + if (put_user(events_missed, eventsp)) + return -EFAULT; + return 0; case VHOST_GET_FEATURES: features = VHOST_SCSI_FEATURES; if (copy_to_user(featurep, &features, sizeof features)) diff --git a/drivers/vhost/tcm_vhost.h b/drivers/vhost/tcm_vhost.h index a545a5b766a3..514b9fda230e 100644 --- a/drivers/vhost/tcm_vhost.h +++ b/drivers/vhost/tcm_vhost.h @@ -123,3 +123,6 @@ struct vhost_scsi_target { #define VHOST_SCSI_CLEAR_ENDPOINT _IOW(VHOST_VIRTIO, 0x41, struct vhost_scsi_target) /* Changing this breaks userspace. */ #define VHOST_SCSI_GET_ABI_VERSION _IOW(VHOST_VIRTIO, 0x42, int) +/* Set and get the events missed flag */ +#define VHOST_SCSI_SET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x43, __u32) +#define VHOST_SCSI_GET_EVENTS_MISSED _IOW(VHOST_VIRTIO, 0x44, __u32) From 04b59babc0ddd2adaebe6ee3064b4d1b4eeecb7f Mon Sep 17 00:00:00 2001 From: Asias He Date: Thu, 25 Apr 2013 15:35:23 +0800 Subject: [PATCH 24/25] tcm_vhost: Enable VIRTIO_SCSI_F_HOTPLUG Everything for hotplug is ready. Let's enable the feature bit. Signed-off-by: Asias He Acked-by: Michael S. Tsirkin Signed-off-by: Nicholas Bellinger --- drivers/vhost/tcm_vhost.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/vhost/tcm_vhost.c b/drivers/vhost/tcm_vhost.c index 07217d818ad7..1677238d281f 100644 --- a/drivers/vhost/tcm_vhost.c +++ b/drivers/vhost/tcm_vhost.c @@ -66,7 +66,8 @@ enum { * TODO: debug and remove the workaround. */ enum { - VHOST_SCSI_FEATURES = VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX) + VHOST_SCSI_FEATURES = (VHOST_FEATURES & (~VIRTIO_RING_F_EVENT_IDX)) | + (1ULL << VIRTIO_SCSI_F_HOTPLUG) }; #define VHOST_SCSI_MAX_TARGET 256 From b8d26b3be8b33682cf163274ed07479a70554633 Mon Sep 17 00:00:00 2001 From: Nicholas Bellinger Date: Thu, 7 Mar 2013 00:56:19 -0800 Subject: [PATCH 25/25] iser-target: Add iSCSI Extensions for RDMA (iSER) target driver This patch adds support for iSCSI Extensions for RDMA target mode, and includes CQ pooling per isert_device context distributed across multiple active iser target sessions. It also uses cmwq process context for RX / TX ib_post_cq() polling via isert_cq_desc->cq_[rx,tx]_work invoked by isert_cq_[rx,tx]_callback() hardIRQ context callbacks. v5 changes: - Use ISER_RECV_DATA_SEG_LEN instead of hardcoded value in ISER_RX_PAD_SIZE (Or) - Fix make W=1 warnings (Or) - Add missing depends on NET && INFINIBAND_ADDR_TRANS in Kconfig (Randy + Or) - Make isert_device_find_by_ib_dev() return proper ERR_PTR (Wei Yongjun) - Properly setup iscsi_np->np_sockaddr in isert_setup_np() (Shlomi + nab) - Add special case for early ISCSI_OP_SCSI_CMD exception handling (nab) v4 changes: - Mark isert_cq_rx_work as static (Or) - Drop unnecessary ib_dma_sync_single_for_cpu + ib_dma_sync_single_for_device calls for isert_cmd->sense_buf_dma from isert_put_response (Or) - Use 12288 for ISER_RX_PAD_SIZE base to save extra page per struct iser_rx_desc (Or + nab) - Drop now unnecessary isert_rx_desc usage, and convert RX users to iser_rx_desc (Or + nab) - Move isert_[alloc,free]_rx_descriptors() ahead of isert_create_device_ib_res() usage (nab) - Mark isert_cq_[rx,tx]_callback() + prototypes as static - Fix 'warning: 'ret' may be used uninitialized' warning for isert_create_device_ib_res on powerpc allmodconfig (fengguang + nab) - Fix 'warning: 'ret' may be used uninitialized' warning for isert_connect_request on i386 allyesconfig (fengguang + nab) - Fix pr_debug conversion specification in isert_rx_completion() (fengguang + nab) - Drop unnecessary isert_conn->conn_cm_id != NULL check in isert_connect_release causing the build warning: "variable dereferenced before check 'isert_conn->conn_cm_id'" - Fix isert_lid + isert_np leak in isert_setup_np failure path - Add isert_conn->conn_wait_comp_err usage in isert_free_conn() for isert_cq_comp_err completion path - Add isert_conn->logout_posted bit to determine decrement of isert_conn->post_send_buf_count from logout response completion - Always set ISER_CONN_DOWN from isert_disconnect_work() callback v3 changes: - Convert to use per isert_cq_desc->cq_[rx,tx]_work + drop tasklets (Or + nab) - Move IB_EVENT_QP_LAST_WQE_REACHED warn into correct isert_qp_event_callback (Or) - Drop unnecessary IB_ACCESS_REMOTE_* access flag usage in isert_create_device_ib_res (Or) - Add common isert_init_send_wr(), and convert isert_put_* calls (Or) - Move to verbs+core logic to single ib_isert.[c,h] (Or + nab) - Add kmem_cache isert_cmd_cache usage for descriptor allocation (nab) - Move common ib_post_send() logic used by isert_put_*() to isert_post_response() (nab) - Add isert_put_reject call in isert_response_queue() for posting ISCSI_REJECT response. (nab) - Add ISTATE_SEND_REJECT checking in isert_do_control_comp. (nab) v2 changes: - Drop unused ISERT_ADDR_ROUTE_TIMEOUT define - Add rdma_notify() call for IB_EVENT_COMM_EST in isert_qp_event_callback() - Make isert_query_device() less verbose - Drop unused RDMA_CM_EVENT_ADDR_ERROR and RDMA_CM_EVENT_ROUTE_ERROR cases from isert_cma_handler() - Drop unused rdma/ib_fmr_pool.h include - Update isert_conn_setup_qp() to assign cq based upon least used - Add isert_create_device_ib_res() to setup PD, CQs and MRs for each underlying struct ib_device, instead of using per isert_conn resources. - Add isert_free_device_ib_res() to release PD, CQs and MRs for each underlying struct ib_device. - Add isert_device_find_by_ib_dev() - Change isert_connect_request() to drop PD, CQs and MRs allocation, and use isert_device_find_by_ib_dev() instead. - Add isert_device_try_release() - Change isert_connect_release() to decrement cq_active_qps, and drop PD, CQs and MRs resource release. - Update isert_connect_release() to call isert_device_try_release() - Make isert_create_device_ib_res() determine device->cqs_used based upon num_online_cpus() - Drop misleading isert_dump_ib_wc() usage - Drop unused rdma/ib_fmr_pool.h include - Use proper xfer_len for login PDUs in isert_rx_completion() - Add isert_release_cmd() usage - Change isert_alloc_cmd() to setup iscsi_cmd.release_cmd() pointer - Change isert_put_cmd() to perform per iscsi_opcode specific release logic - Add isert_unmap_cmd() call for ISCSI_OP_SCSI_CMD from isert_put_cmd() - Change isert_send_completion() to call atomic_dec(&isert_conn->post_send_buf_count) based upon per iscsi_opcode logic - Drop ISTATE_REMOVE processing from isert_immediate_queue() - Drop ISTATE_SEND_DATAIN processing from isert_response_queue() - Drop ISTATE_SEND_STATUS processing from isert_response_queue() - Drop iscsit_transport->iscsit_unmap_cmd() and ->iscsit_free_cmd() - Convert iser_cq_tx_tasklet() to use struct isert_cq_desc pooling logic - Convert isert_cq_tx_callback() to use struct isert_cq_desc pooling logic - Convert iser_cq_rx_tasklet() to use struct isert_cq_desc pooling logic - Convert isert_cq_rx_callback() to use struct isert_cq_desc pooling logic - Add explict iscsit_stop_dataout_timer() call to isert_do_rdma_read_comp() - Use isert_get_dataout() for iscsit_transport->iscsit_get_dataout() caller - Drop ISTATE_SEND_R2T processing from isert_immediate_queue() - Drop unused rdma/ib_fmr_pool.h include - Drop isert_cmd->cmd_kref in favor of se_cmd->cmd_kref usage - Add struct isert_device in order to support multiple EQs + CQ pooling - Add struct isert_cq_desc - Drop tasklets and cqs from isert_conn - Bump ISERT_MAX_CQ to 64 - Various minor checkpatch fixes Signed-off-by: Or Gerlitz Signed-off-by: Nicholas Bellinger --- drivers/infiniband/Kconfig | 1 + drivers/infiniband/Makefile | 1 + drivers/infiniband/ulp/isert/Kconfig | 5 + drivers/infiniband/ulp/isert/Makefile | 2 + drivers/infiniband/ulp/isert/ib_isert.c | 2281 ++++++++++++++++++++ drivers/infiniband/ulp/isert/ib_isert.h | 138 ++ drivers/infiniband/ulp/isert/isert_proto.h | 47 + 7 files changed, 2475 insertions(+) create mode 100644 drivers/infiniband/ulp/isert/Kconfig create mode 100644 drivers/infiniband/ulp/isert/Makefile create mode 100644 drivers/infiniband/ulp/isert/ib_isert.c create mode 100644 drivers/infiniband/ulp/isert/ib_isert.h create mode 100644 drivers/infiniband/ulp/isert/isert_proto.h diff --git a/drivers/infiniband/Kconfig b/drivers/infiniband/Kconfig index a0f29c1d03bc..c85b56c28099 100644 --- a/drivers/infiniband/Kconfig +++ b/drivers/infiniband/Kconfig @@ -59,5 +59,6 @@ source "drivers/infiniband/ulp/srp/Kconfig" source "drivers/infiniband/ulp/srpt/Kconfig" source "drivers/infiniband/ulp/iser/Kconfig" +source "drivers/infiniband/ulp/isert/Kconfig" endif # INFINIBAND diff --git a/drivers/infiniband/Makefile b/drivers/infiniband/Makefile index bf846a14b9d3..b126fefe0b1c 100644 --- a/drivers/infiniband/Makefile +++ b/drivers/infiniband/Makefile @@ -13,3 +13,4 @@ obj-$(CONFIG_INFINIBAND_IPOIB) += ulp/ipoib/ obj-$(CONFIG_INFINIBAND_SRP) += ulp/srp/ obj-$(CONFIG_INFINIBAND_SRPT) += ulp/srpt/ obj-$(CONFIG_INFINIBAND_ISER) += ulp/iser/ +obj-$(CONFIG_INFINIBAND_ISERT) += ulp/isert/ diff --git a/drivers/infiniband/ulp/isert/Kconfig b/drivers/infiniband/ulp/isert/Kconfig new file mode 100644 index 000000000000..ce3fd32167dc --- /dev/null +++ b/drivers/infiniband/ulp/isert/Kconfig @@ -0,0 +1,5 @@ +config INFINIBAND_ISERT + tristate "iSCSI Extentions for RDMA (iSER) target support" + depends on INET && INFINIBAND_ADDR_TRANS && TARGET_CORE && ISCSI_TARGET + ---help--- + Support for iSCSI Extentions for RDMA (iSER) Target on Infiniband fabrics. diff --git a/drivers/infiniband/ulp/isert/Makefile b/drivers/infiniband/ulp/isert/Makefile new file mode 100644 index 000000000000..c8bf2421f5bc --- /dev/null +++ b/drivers/infiniband/ulp/isert/Makefile @@ -0,0 +1,2 @@ +ccflags-y := -Idrivers/target -Idrivers/target/iscsi +obj-$(CONFIG_INFINIBAND_ISERT) += ib_isert.o diff --git a/drivers/infiniband/ulp/isert/ib_isert.c b/drivers/infiniband/ulp/isert/ib_isert.c new file mode 100644 index 000000000000..41712f096515 --- /dev/null +++ b/drivers/infiniband/ulp/isert/ib_isert.c @@ -0,0 +1,2281 @@ +/******************************************************************************* + * This file contains iSCSI extentions for RDMA (iSER) Verbs + * + * (c) Copyright 2013 RisingTide Systems LLC. + * + * Nicholas A. Bellinger + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation; either version 2 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + ****************************************************************************/ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include "isert_proto.h" +#include "ib_isert.h" + +#define ISERT_MAX_CONN 8 +#define ISER_MAX_RX_CQ_LEN (ISERT_QP_MAX_RECV_DTOS * ISERT_MAX_CONN) +#define ISER_MAX_TX_CQ_LEN (ISERT_QP_MAX_REQ_DTOS * ISERT_MAX_CONN) + +static DEFINE_MUTEX(device_list_mutex); +static LIST_HEAD(device_list); +static struct workqueue_struct *isert_rx_wq; +static struct workqueue_struct *isert_comp_wq; +static struct kmem_cache *isert_cmd_cache; + +static void +isert_qp_event_callback(struct ib_event *e, void *context) +{ + struct isert_conn *isert_conn = (struct isert_conn *)context; + + pr_err("isert_qp_event_callback event: %d\n", e->event); + switch (e->event) { + case IB_EVENT_COMM_EST: + rdma_notify(isert_conn->conn_cm_id, IB_EVENT_COMM_EST); + break; + case IB_EVENT_QP_LAST_WQE_REACHED: + pr_warn("Reached TX IB_EVENT_QP_LAST_WQE_REACHED:\n"); + break; + default: + break; + } +} + +static int +isert_query_device(struct ib_device *ib_dev, struct ib_device_attr *devattr) +{ + int ret; + + ret = ib_query_device(ib_dev, devattr); + if (ret) { + pr_err("ib_query_device() failed: %d\n", ret); + return ret; + } + pr_debug("devattr->max_sge: %d\n", devattr->max_sge); + pr_debug("devattr->max_sge_rd: %d\n", devattr->max_sge_rd); + + return 0; +} + +static int +isert_conn_setup_qp(struct isert_conn *isert_conn, struct rdma_cm_id *cma_id) +{ + struct isert_device *device = isert_conn->conn_device; + struct ib_qp_init_attr attr; + struct ib_device_attr devattr; + int ret, index, min_index = 0; + + memset(&devattr, 0, sizeof(struct ib_device_attr)); + ret = isert_query_device(cma_id->device, &devattr); + if (ret) + return ret; + + mutex_lock(&device_list_mutex); + for (index = 0; index < device->cqs_used; index++) + if (device->cq_active_qps[index] < + device->cq_active_qps[min_index]) + min_index = index; + device->cq_active_qps[min_index]++; + pr_debug("isert_conn_setup_qp: Using min_index: %d\n", min_index); + mutex_unlock(&device_list_mutex); + + memset(&attr, 0, sizeof(struct ib_qp_init_attr)); + attr.event_handler = isert_qp_event_callback; + attr.qp_context = isert_conn; + attr.send_cq = device->dev_tx_cq[min_index]; + attr.recv_cq = device->dev_rx_cq[min_index]; + attr.cap.max_send_wr = ISERT_QP_MAX_REQ_DTOS; + attr.cap.max_recv_wr = ISERT_QP_MAX_RECV_DTOS; + /* + * FIXME: Use devattr.max_sge - 2 for max_send_sge as + * work-around for RDMA_READ.. + */ + attr.cap.max_send_sge = devattr.max_sge - 2; + isert_conn->max_sge = attr.cap.max_send_sge; + + attr.cap.max_recv_sge = 1; + attr.sq_sig_type = IB_SIGNAL_REQ_WR; + attr.qp_type = IB_QPT_RC; + + pr_debug("isert_conn_setup_qp cma_id->device: %p\n", + cma_id->device); + pr_debug("isert_conn_setup_qp conn_pd->device: %p\n", + isert_conn->conn_pd->device); + + ret = rdma_create_qp(cma_id, isert_conn->conn_pd, &attr); + if (ret) { + pr_err("rdma_create_qp failed for cma_id %d\n", ret); + return ret; + } + isert_conn->conn_qp = cma_id->qp; + pr_debug("rdma_create_qp() returned success >>>>>>>>>>>>>>>>>>>>>>>>>.\n"); + + return 0; +} + +static void +isert_cq_event_callback(struct ib_event *e, void *context) +{ + pr_debug("isert_cq_event_callback event: %d\n", e->event); +} + +static int +isert_alloc_rx_descriptors(struct isert_conn *isert_conn) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct iser_rx_desc *rx_desc; + struct ib_sge *rx_sg; + u64 dma_addr; + int i, j; + + isert_conn->conn_rx_descs = kzalloc(ISERT_QP_MAX_RECV_DTOS * + sizeof(struct iser_rx_desc), GFP_KERNEL); + if (!isert_conn->conn_rx_descs) + goto fail; + + rx_desc = isert_conn->conn_rx_descs; + + for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { + dma_addr = ib_dma_map_single(ib_dev, (void *)rx_desc, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); + if (ib_dma_mapping_error(ib_dev, dma_addr)) + goto dma_map_fail; + + rx_desc->dma_addr = dma_addr; + + rx_sg = &rx_desc->rx_sg; + rx_sg->addr = rx_desc->dma_addr; + rx_sg->length = ISER_RX_PAYLOAD_SIZE; + rx_sg->lkey = isert_conn->conn_mr->lkey; + } + + isert_conn->conn_rx_desc_head = 0; + return 0; + +dma_map_fail: + rx_desc = isert_conn->conn_rx_descs; + for (j = 0; j < i; j++, rx_desc++) { + ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); + } + kfree(isert_conn->conn_rx_descs); + isert_conn->conn_rx_descs = NULL; +fail: + return -ENOMEM; +} + +static void +isert_free_rx_descriptors(struct isert_conn *isert_conn) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct iser_rx_desc *rx_desc; + int i; + + if (!isert_conn->conn_rx_descs) + return; + + rx_desc = isert_conn->conn_rx_descs; + for (i = 0; i < ISERT_QP_MAX_RECV_DTOS; i++, rx_desc++) { + ib_dma_unmap_single(ib_dev, rx_desc->dma_addr, + ISER_RX_PAYLOAD_SIZE, DMA_FROM_DEVICE); + } + + kfree(isert_conn->conn_rx_descs); + isert_conn->conn_rx_descs = NULL; +} + +static void isert_cq_tx_callback(struct ib_cq *, void *); +static void isert_cq_rx_callback(struct ib_cq *, void *); + +static int +isert_create_device_ib_res(struct isert_device *device) +{ + struct ib_device *ib_dev = device->ib_device; + struct isert_cq_desc *cq_desc; + int ret = 0, i, j; + + device->cqs_used = min_t(int, num_online_cpus(), + device->ib_device->num_comp_vectors); + device->cqs_used = min(ISERT_MAX_CQ, device->cqs_used); + pr_debug("Using %d CQs, device %s supports %d vectors\n", + device->cqs_used, device->ib_device->name, + device->ib_device->num_comp_vectors); + device->cq_desc = kzalloc(sizeof(struct isert_cq_desc) * + device->cqs_used, GFP_KERNEL); + if (!device->cq_desc) { + pr_err("Unable to allocate device->cq_desc\n"); + return -ENOMEM; + } + cq_desc = device->cq_desc; + + device->dev_pd = ib_alloc_pd(ib_dev); + if (IS_ERR(device->dev_pd)) { + ret = PTR_ERR(device->dev_pd); + pr_err("ib_alloc_pd failed for dev_pd: %d\n", ret); + goto out_cq_desc; + } + + for (i = 0; i < device->cqs_used; i++) { + cq_desc[i].device = device; + cq_desc[i].cq_index = i; + + device->dev_rx_cq[i] = ib_create_cq(device->ib_device, + isert_cq_rx_callback, + isert_cq_event_callback, + (void *)&cq_desc[i], + ISER_MAX_RX_CQ_LEN, i); + if (IS_ERR(device->dev_rx_cq[i])) + goto out_cq; + + device->dev_tx_cq[i] = ib_create_cq(device->ib_device, + isert_cq_tx_callback, + isert_cq_event_callback, + (void *)&cq_desc[i], + ISER_MAX_TX_CQ_LEN, i); + if (IS_ERR(device->dev_tx_cq[i])) + goto out_cq; + + if (ib_req_notify_cq(device->dev_rx_cq[i], IB_CQ_NEXT_COMP)) + goto out_cq; + + if (ib_req_notify_cq(device->dev_tx_cq[i], IB_CQ_NEXT_COMP)) + goto out_cq; + } + + device->dev_mr = ib_get_dma_mr(device->dev_pd, IB_ACCESS_LOCAL_WRITE); + if (IS_ERR(device->dev_mr)) { + ret = PTR_ERR(device->dev_mr); + pr_err("ib_get_dma_mr failed for dev_mr: %d\n", ret); + goto out_cq; + } + + return 0; + +out_cq: + for (j = 0; j < i; j++) { + cq_desc = &device->cq_desc[j]; + + if (device->dev_rx_cq[j]) { + cancel_work_sync(&cq_desc->cq_rx_work); + ib_destroy_cq(device->dev_rx_cq[j]); + } + if (device->dev_tx_cq[j]) { + cancel_work_sync(&cq_desc->cq_tx_work); + ib_destroy_cq(device->dev_tx_cq[j]); + } + } + ib_dealloc_pd(device->dev_pd); + +out_cq_desc: + kfree(device->cq_desc); + + return ret; +} + +static void +isert_free_device_ib_res(struct isert_device *device) +{ + struct isert_cq_desc *cq_desc; + int i; + + for (i = 0; i < device->cqs_used; i++) { + cq_desc = &device->cq_desc[i]; + + cancel_work_sync(&cq_desc->cq_rx_work); + cancel_work_sync(&cq_desc->cq_tx_work); + ib_destroy_cq(device->dev_rx_cq[i]); + ib_destroy_cq(device->dev_tx_cq[i]); + device->dev_rx_cq[i] = NULL; + device->dev_tx_cq[i] = NULL; + } + + ib_dereg_mr(device->dev_mr); + ib_dealloc_pd(device->dev_pd); + kfree(device->cq_desc); +} + +static void +isert_device_try_release(struct isert_device *device) +{ + mutex_lock(&device_list_mutex); + device->refcount--; + if (!device->refcount) { + isert_free_device_ib_res(device); + list_del(&device->dev_node); + kfree(device); + } + mutex_unlock(&device_list_mutex); +} + +static struct isert_device * +isert_device_find_by_ib_dev(struct rdma_cm_id *cma_id) +{ + struct isert_device *device; + int ret; + + mutex_lock(&device_list_mutex); + list_for_each_entry(device, &device_list, dev_node) { + if (device->ib_device->node_guid == cma_id->device->node_guid) { + device->refcount++; + mutex_unlock(&device_list_mutex); + return device; + } + } + + device = kzalloc(sizeof(struct isert_device), GFP_KERNEL); + if (!device) { + mutex_unlock(&device_list_mutex); + return ERR_PTR(-ENOMEM); + } + + INIT_LIST_HEAD(&device->dev_node); + + device->ib_device = cma_id->device; + ret = isert_create_device_ib_res(device); + if (ret) { + kfree(device); + mutex_unlock(&device_list_mutex); + return ERR_PTR(ret); + } + + device->refcount++; + list_add_tail(&device->dev_node, &device_list); + mutex_unlock(&device_list_mutex); + + return device; +} + +static int +isert_connect_request(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) +{ + struct iscsi_np *np = cma_id->context; + struct isert_np *isert_np = np->np_context; + struct isert_conn *isert_conn; + struct isert_device *device; + struct ib_device *ib_dev = cma_id->device; + int ret = 0; + + pr_debug("Entering isert_connect_request cma_id: %p, context: %p\n", + cma_id, cma_id->context); + + isert_conn = kzalloc(sizeof(struct isert_conn), GFP_KERNEL); + if (!isert_conn) { + pr_err("Unable to allocate isert_conn\n"); + return -ENOMEM; + } + isert_conn->state = ISER_CONN_INIT; + INIT_LIST_HEAD(&isert_conn->conn_accept_node); + init_completion(&isert_conn->conn_login_comp); + init_waitqueue_head(&isert_conn->conn_wait); + init_waitqueue_head(&isert_conn->conn_wait_comp_err); + kref_init(&isert_conn->conn_kref); + kref_get(&isert_conn->conn_kref); + + cma_id->context = isert_conn; + isert_conn->conn_cm_id = cma_id; + isert_conn->responder_resources = event->param.conn.responder_resources; + isert_conn->initiator_depth = event->param.conn.initiator_depth; + pr_debug("Using responder_resources: %u initiator_depth: %u\n", + isert_conn->responder_resources, isert_conn->initiator_depth); + + isert_conn->login_buf = kzalloc(ISCSI_DEF_MAX_RECV_SEG_LEN + + ISER_RX_LOGIN_SIZE, GFP_KERNEL); + if (!isert_conn->login_buf) { + pr_err("Unable to allocate isert_conn->login_buf\n"); + ret = -ENOMEM; + goto out; + } + + isert_conn->login_req_buf = isert_conn->login_buf; + isert_conn->login_rsp_buf = isert_conn->login_buf + + ISCSI_DEF_MAX_RECV_SEG_LEN; + pr_debug("Set login_buf: %p login_req_buf: %p login_rsp_buf: %p\n", + isert_conn->login_buf, isert_conn->login_req_buf, + isert_conn->login_rsp_buf); + + isert_conn->login_req_dma = ib_dma_map_single(ib_dev, + (void *)isert_conn->login_req_buf, + ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); + + ret = ib_dma_mapping_error(ib_dev, isert_conn->login_req_dma); + if (ret) { + pr_err("ib_dma_mapping_error failed for login_req_dma: %d\n", + ret); + isert_conn->login_req_dma = 0; + goto out_login_buf; + } + + isert_conn->login_rsp_dma = ib_dma_map_single(ib_dev, + (void *)isert_conn->login_rsp_buf, + ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); + + ret = ib_dma_mapping_error(ib_dev, isert_conn->login_rsp_dma); + if (ret) { + pr_err("ib_dma_mapping_error failed for login_rsp_dma: %d\n", + ret); + isert_conn->login_rsp_dma = 0; + goto out_req_dma_map; + } + + device = isert_device_find_by_ib_dev(cma_id); + if (IS_ERR(device)) { + ret = PTR_ERR(device); + goto out_rsp_dma_map; + } + + isert_conn->conn_device = device; + isert_conn->conn_pd = device->dev_pd; + isert_conn->conn_mr = device->dev_mr; + + ret = isert_conn_setup_qp(isert_conn, cma_id); + if (ret) + goto out_conn_dev; + + mutex_lock(&isert_np->np_accept_mutex); + list_add_tail(&isert_np->np_accept_list, &isert_conn->conn_accept_node); + mutex_unlock(&isert_np->np_accept_mutex); + + pr_debug("isert_connect_request() waking up np_accept_wq: %p\n", np); + wake_up(&isert_np->np_accept_wq); + return 0; + +out_conn_dev: + isert_device_try_release(device); +out_rsp_dma_map: + ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, + ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); +out_req_dma_map: + ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, + ISCSI_DEF_MAX_RECV_SEG_LEN, DMA_FROM_DEVICE); +out_login_buf: + kfree(isert_conn->login_buf); +out: + kfree(isert_conn); + return ret; +} + +static void +isert_connect_release(struct isert_conn *isert_conn) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct isert_device *device = isert_conn->conn_device; + int cq_index; + + pr_debug("Entering isert_connect_release(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); + + if (isert_conn->conn_qp) { + cq_index = ((struct isert_cq_desc *) + isert_conn->conn_qp->recv_cq->cq_context)->cq_index; + pr_debug("isert_connect_release: cq_index: %d\n", cq_index); + isert_conn->conn_device->cq_active_qps[cq_index]--; + + rdma_destroy_qp(isert_conn->conn_cm_id); + } + + isert_free_rx_descriptors(isert_conn); + rdma_destroy_id(isert_conn->conn_cm_id); + + if (isert_conn->login_buf) { + ib_dma_unmap_single(ib_dev, isert_conn->login_rsp_dma, + ISER_RX_LOGIN_SIZE, DMA_TO_DEVICE); + ib_dma_unmap_single(ib_dev, isert_conn->login_req_dma, + ISCSI_DEF_MAX_RECV_SEG_LEN, + DMA_FROM_DEVICE); + kfree(isert_conn->login_buf); + } + kfree(isert_conn); + + if (device) + isert_device_try_release(device); + + pr_debug("Leaving isert_connect_release >>>>>>>>>>>>\n"); +} + +static void +isert_connected_handler(struct rdma_cm_id *cma_id) +{ + return; +} + +static void +isert_release_conn_kref(struct kref *kref) +{ + struct isert_conn *isert_conn = container_of(kref, + struct isert_conn, conn_kref); + + pr_debug("Calling isert_connect_release for final kref %s/%d\n", + current->comm, current->pid); + + isert_connect_release(isert_conn); +} + +static void +isert_put_conn(struct isert_conn *isert_conn) +{ + kref_put(&isert_conn->conn_kref, isert_release_conn_kref); +} + +static void +isert_disconnect_work(struct work_struct *work) +{ + struct isert_conn *isert_conn = container_of(work, + struct isert_conn, conn_logout_work); + + pr_debug("isert_disconnect_work(): >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); + + isert_conn->state = ISER_CONN_DOWN; + + if (isert_conn->post_recv_buf_count == 0 && + atomic_read(&isert_conn->post_send_buf_count) == 0) { + pr_debug("Calling wake_up(&isert_conn->conn_wait);\n"); + wake_up(&isert_conn->conn_wait); + } + + isert_put_conn(isert_conn); +} + +static void +isert_disconnected_handler(struct rdma_cm_id *cma_id) +{ + struct isert_conn *isert_conn = (struct isert_conn *)cma_id->context; + + INIT_WORK(&isert_conn->conn_logout_work, isert_disconnect_work); + schedule_work(&isert_conn->conn_logout_work); +} + +static int +isert_cma_handler(struct rdma_cm_id *cma_id, struct rdma_cm_event *event) +{ + int ret = 0; + + pr_debug("isert_cma_handler: event %d status %d conn %p id %p\n", + event->event, event->status, cma_id->context, cma_id); + + switch (event->event) { + case RDMA_CM_EVENT_CONNECT_REQUEST: + pr_debug("RDMA_CM_EVENT_CONNECT_REQUEST: >>>>>>>>>>>>>>>\n"); + ret = isert_connect_request(cma_id, event); + break; + case RDMA_CM_EVENT_ESTABLISHED: + pr_debug("RDMA_CM_EVENT_ESTABLISHED >>>>>>>>>>>>>>\n"); + isert_connected_handler(cma_id); + break; + case RDMA_CM_EVENT_DISCONNECTED: + pr_debug("RDMA_CM_EVENT_DISCONNECTED: >>>>>>>>>>>>>>\n"); + isert_disconnected_handler(cma_id); + break; + case RDMA_CM_EVENT_DEVICE_REMOVAL: + case RDMA_CM_EVENT_ADDR_CHANGE: + break; + case RDMA_CM_EVENT_CONNECT_ERROR: + default: + pr_err("Unknown RDMA CMA event: %d\n", event->event); + break; + } + + if (ret != 0) { + pr_err("isert_cma_handler failed RDMA_CM_EVENT: 0x%08x %d\n", + event->event, ret); + dump_stack(); + } + + return ret; +} + +static int +isert_post_recv(struct isert_conn *isert_conn, u32 count) +{ + struct ib_recv_wr *rx_wr, *rx_wr_failed; + int i, ret; + unsigned int rx_head = isert_conn->conn_rx_desc_head; + struct iser_rx_desc *rx_desc; + + for (rx_wr = isert_conn->conn_rx_wr, i = 0; i < count; i++, rx_wr++) { + rx_desc = &isert_conn->conn_rx_descs[rx_head]; + rx_wr->wr_id = (unsigned long)rx_desc; + rx_wr->sg_list = &rx_desc->rx_sg; + rx_wr->num_sge = 1; + rx_wr->next = rx_wr + 1; + rx_head = (rx_head + 1) & (ISERT_QP_MAX_RECV_DTOS - 1); + } + + rx_wr--; + rx_wr->next = NULL; /* mark end of work requests list */ + + isert_conn->post_recv_buf_count += count; + ret = ib_post_recv(isert_conn->conn_qp, isert_conn->conn_rx_wr, + &rx_wr_failed); + if (ret) { + pr_err("ib_post_recv() failed with ret: %d\n", ret); + isert_conn->post_recv_buf_count -= count; + } else { + pr_debug("isert_post_recv(): Posted %d RX buffers\n", count); + isert_conn->conn_rx_desc_head = rx_head; + } + return ret; +} + +static int +isert_post_send(struct isert_conn *isert_conn, struct iser_tx_desc *tx_desc) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct ib_send_wr send_wr, *send_wr_failed; + int ret; + + ib_dma_sync_single_for_device(ib_dev, tx_desc->dma_addr, + ISER_HEADERS_LEN, DMA_TO_DEVICE); + + send_wr.next = NULL; + send_wr.wr_id = (unsigned long)tx_desc; + send_wr.sg_list = tx_desc->tx_sg; + send_wr.num_sge = tx_desc->num_sge; + send_wr.opcode = IB_WR_SEND; + send_wr.send_flags = IB_SEND_SIGNALED; + + atomic_inc(&isert_conn->post_send_buf_count); + + ret = ib_post_send(isert_conn->conn_qp, &send_wr, &send_wr_failed); + if (ret) { + pr_err("ib_post_send() failed, ret: %d\n", ret); + atomic_dec(&isert_conn->post_send_buf_count); + } + + return ret; +} + +static void +isert_create_send_desc(struct isert_conn *isert_conn, + struct isert_cmd *isert_cmd, + struct iser_tx_desc *tx_desc) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + + ib_dma_sync_single_for_cpu(ib_dev, tx_desc->dma_addr, + ISER_HEADERS_LEN, DMA_TO_DEVICE); + + memset(&tx_desc->iser_header, 0, sizeof(struct iser_hdr)); + tx_desc->iser_header.flags = ISER_VER; + + tx_desc->num_sge = 1; + tx_desc->isert_cmd = isert_cmd; + + if (tx_desc->tx_sg[0].lkey != isert_conn->conn_mr->lkey) { + tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; + pr_debug("tx_desc %p lkey mismatch, fixing\n", tx_desc); + } +} + +static int +isert_init_tx_hdrs(struct isert_conn *isert_conn, + struct iser_tx_desc *tx_desc) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + u64 dma_addr; + + dma_addr = ib_dma_map_single(ib_dev, (void *)tx_desc, + ISER_HEADERS_LEN, DMA_TO_DEVICE); + if (ib_dma_mapping_error(ib_dev, dma_addr)) { + pr_err("ib_dma_mapping_error() failed\n"); + return -ENOMEM; + } + + tx_desc->dma_addr = dma_addr; + tx_desc->tx_sg[0].addr = tx_desc->dma_addr; + tx_desc->tx_sg[0].length = ISER_HEADERS_LEN; + tx_desc->tx_sg[0].lkey = isert_conn->conn_mr->lkey; + + pr_debug("isert_init_tx_hdrs: Setup tx_sg[0].addr: 0x%llx length: %u" + " lkey: 0x%08x\n", tx_desc->tx_sg[0].addr, + tx_desc->tx_sg[0].length, tx_desc->tx_sg[0].lkey); + + return 0; +} + +static void +isert_init_send_wr(struct isert_cmd *isert_cmd, struct ib_send_wr *send_wr) +{ + isert_cmd->rdma_wr.iser_ib_op = ISER_IB_SEND; + send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; + send_wr->opcode = IB_WR_SEND; + send_wr->send_flags = IB_SEND_SIGNALED; + send_wr->sg_list = &isert_cmd->tx_desc.tx_sg[0]; + send_wr->num_sge = isert_cmd->tx_desc.num_sge; +} + +static int +isert_rdma_post_recvl(struct isert_conn *isert_conn) +{ + struct ib_recv_wr rx_wr, *rx_wr_fail; + struct ib_sge sge; + int ret; + + memset(&sge, 0, sizeof(struct ib_sge)); + sge.addr = isert_conn->login_req_dma; + sge.length = ISER_RX_LOGIN_SIZE; + sge.lkey = isert_conn->conn_mr->lkey; + + pr_debug("Setup sge: addr: %llx length: %d 0x%08x\n", + sge.addr, sge.length, sge.lkey); + + memset(&rx_wr, 0, sizeof(struct ib_recv_wr)); + rx_wr.wr_id = (unsigned long)isert_conn->login_req_buf; + rx_wr.sg_list = &sge; + rx_wr.num_sge = 1; + + isert_conn->post_recv_buf_count++; + ret = ib_post_recv(isert_conn->conn_qp, &rx_wr, &rx_wr_fail); + if (ret) { + pr_err("ib_post_recv() failed: %d\n", ret); + isert_conn->post_recv_buf_count--; + } + + pr_debug("ib_post_recv(): returned success >>>>>>>>>>>>>>>>>>>>>>>>\n"); + return ret; +} + +static int +isert_put_login_tx(struct iscsi_conn *conn, struct iscsi_login *login, + u32 length) +{ + struct isert_conn *isert_conn = conn->context; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct iser_tx_desc *tx_desc = &isert_conn->conn_login_tx_desc; + int ret; + + isert_create_send_desc(isert_conn, NULL, tx_desc); + + memcpy(&tx_desc->iscsi_header, &login->rsp[0], + sizeof(struct iscsi_hdr)); + + isert_init_tx_hdrs(isert_conn, tx_desc); + + if (length > 0) { + struct ib_sge *tx_dsg = &tx_desc->tx_sg[1]; + + ib_dma_sync_single_for_cpu(ib_dev, isert_conn->login_rsp_dma, + length, DMA_TO_DEVICE); + + memcpy(isert_conn->login_rsp_buf, login->rsp_buf, length); + + ib_dma_sync_single_for_device(ib_dev, isert_conn->login_rsp_dma, + length, DMA_TO_DEVICE); + + tx_dsg->addr = isert_conn->login_rsp_dma; + tx_dsg->length = length; + tx_dsg->lkey = isert_conn->conn_mr->lkey; + tx_desc->num_sge = 2; + } + if (!login->login_failed) { + if (login->login_complete) { + ret = isert_alloc_rx_descriptors(isert_conn); + if (ret) + return ret; + + ret = isert_post_recv(isert_conn, ISERT_MIN_POSTED_RX); + if (ret) + return ret; + + isert_conn->state = ISER_CONN_UP; + goto post_send; + } + + ret = isert_rdma_post_recvl(isert_conn); + if (ret) + return ret; + } +post_send: + ret = isert_post_send(isert_conn, tx_desc); + if (ret) + return ret; + + return 0; +} + +static void +isert_rx_login_req(struct iser_rx_desc *rx_desc, int rx_buflen, + struct isert_conn *isert_conn) +{ + struct iscsi_conn *conn = isert_conn->conn; + struct iscsi_login *login = conn->conn_login; + int size; + + if (!login) { + pr_err("conn->conn_login is NULL\n"); + dump_stack(); + return; + } + + if (login->first_request) { + struct iscsi_login_req *login_req = + (struct iscsi_login_req *)&rx_desc->iscsi_header; + /* + * Setup the initial iscsi_login values from the leading + * login request PDU. + */ + login->leading_connection = (!login_req->tsih) ? 1 : 0; + login->current_stage = + (login_req->flags & ISCSI_FLAG_LOGIN_CURRENT_STAGE_MASK) + >> 2; + login->version_min = login_req->min_version; + login->version_max = login_req->max_version; + memcpy(login->isid, login_req->isid, 6); + login->cmd_sn = be32_to_cpu(login_req->cmdsn); + login->init_task_tag = login_req->itt; + login->initial_exp_statsn = be32_to_cpu(login_req->exp_statsn); + login->cid = be16_to_cpu(login_req->cid); + login->tsih = be16_to_cpu(login_req->tsih); + } + + memcpy(&login->req[0], (void *)&rx_desc->iscsi_header, ISCSI_HDR_LEN); + + size = min(rx_buflen, MAX_KEY_VALUE_PAIRS); + pr_debug("Using login payload size: %d, rx_buflen: %d MAX_KEY_VALUE_PAIRS: %d\n", + size, rx_buflen, MAX_KEY_VALUE_PAIRS); + memcpy(login->req_buf, &rx_desc->data[0], size); + + complete(&isert_conn->conn_login_comp); +} + +static void +isert_release_cmd(struct iscsi_cmd *cmd) +{ + struct isert_cmd *isert_cmd = container_of(cmd, struct isert_cmd, + iscsi_cmd); + + pr_debug("Entering isert_release_cmd %p >>>>>>>>>>>>>>>.\n", isert_cmd); + + kfree(cmd->buf_ptr); + kfree(cmd->tmr_req); + + kmem_cache_free(isert_cmd_cache, isert_cmd); +} + +static struct iscsi_cmd +*isert_alloc_cmd(struct iscsi_conn *conn, gfp_t gfp) +{ + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct isert_cmd *isert_cmd; + + isert_cmd = kmem_cache_zalloc(isert_cmd_cache, gfp); + if (!isert_cmd) { + pr_err("Unable to allocate isert_cmd\n"); + return NULL; + } + isert_cmd->conn = isert_conn; + isert_cmd->iscsi_cmd.release_cmd = &isert_release_cmd; + + return &isert_cmd->iscsi_cmd; +} + +static int +isert_handle_scsi_cmd(struct isert_conn *isert_conn, + struct isert_cmd *isert_cmd, struct iser_rx_desc *rx_desc, + unsigned char *buf) +{ + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + struct iscsi_conn *conn = isert_conn->conn; + struct iscsi_scsi_req *hdr = (struct iscsi_scsi_req *)buf; + struct scatterlist *sg; + int imm_data, imm_data_len, unsol_data, sg_nents, rc; + bool dump_payload = false; + + rc = iscsit_setup_scsi_cmd(conn, cmd, buf); + if (rc < 0) + return rc; + + imm_data = cmd->immediate_data; + imm_data_len = cmd->first_burst_len; + unsol_data = cmd->unsolicited_data; + + rc = iscsit_process_scsi_cmd(conn, cmd, hdr); + if (rc < 0) { + return 0; + } else if (rc > 0) { + dump_payload = true; + goto sequence_cmd; + } + + if (!imm_data) + return 0; + + sg = &cmd->se_cmd.t_data_sg[0]; + sg_nents = max(1UL, DIV_ROUND_UP(imm_data_len, PAGE_SIZE)); + + pr_debug("Copying Immediate SG: %p sg_nents: %u from %p imm_data_len: %d\n", + sg, sg_nents, &rx_desc->data[0], imm_data_len); + + sg_copy_from_buffer(sg, sg_nents, &rx_desc->data[0], imm_data_len); + + cmd->write_data_done += imm_data_len; + + if (cmd->write_data_done == cmd->se_cmd.data_length) { + spin_lock_bh(&cmd->istate_lock); + cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; + cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; + spin_unlock_bh(&cmd->istate_lock); + } + +sequence_cmd: + rc = iscsit_sequence_cmd(conn, cmd, hdr->cmdsn); + + if (!rc && dump_payload == false && unsol_data) + iscsit_set_unsoliticed_dataout(cmd); + + if (rc == CMDSN_ERROR_CANNOT_RECOVER) + return iscsit_add_reject_from_cmd( + ISCSI_REASON_PROTOCOL_ERROR, + 1, 0, (unsigned char *)hdr, cmd); + + return 0; +} + +static int +isert_handle_iscsi_dataout(struct isert_conn *isert_conn, + struct iser_rx_desc *rx_desc, unsigned char *buf) +{ + struct scatterlist *sg_start; + struct iscsi_conn *conn = isert_conn->conn; + struct iscsi_cmd *cmd = NULL; + struct iscsi_data *hdr = (struct iscsi_data *)buf; + u32 unsol_data_len = ntoh24(hdr->dlength); + int rc, sg_nents, sg_off, page_off; + + rc = iscsit_check_dataout_hdr(conn, buf, &cmd); + if (rc < 0) + return rc; + else if (!cmd) + return 0; + /* + * FIXME: Unexpected unsolicited_data out + */ + if (!cmd->unsolicited_data) { + pr_err("Received unexpected solicited data payload\n"); + dump_stack(); + return -1; + } + + pr_debug("Unsolicited DataOut unsol_data_len: %u, write_data_done: %u, data_length: %u\n", + unsol_data_len, cmd->write_data_done, cmd->se_cmd.data_length); + + sg_off = cmd->write_data_done / PAGE_SIZE; + sg_start = &cmd->se_cmd.t_data_sg[sg_off]; + sg_nents = max(1UL, DIV_ROUND_UP(unsol_data_len, PAGE_SIZE)); + page_off = cmd->write_data_done % PAGE_SIZE; + /* + * FIXME: Non page-aligned unsolicited_data out + */ + if (page_off) { + pr_err("Received unexpected non-page aligned data payload\n"); + dump_stack(); + return -1; + } + pr_debug("Copying DataOut: sg_start: %p, sg_off: %u sg_nents: %u from %p %u\n", + sg_start, sg_off, sg_nents, &rx_desc->data[0], unsol_data_len); + + sg_copy_from_buffer(sg_start, sg_nents, &rx_desc->data[0], + unsol_data_len); + + rc = iscsit_check_dataout_payload(cmd, hdr, false); + if (rc < 0) + return rc; + + return 0; +} + +static int +isert_rx_opcode(struct isert_conn *isert_conn, struct iser_rx_desc *rx_desc, + uint32_t read_stag, uint64_t read_va, + uint32_t write_stag, uint64_t write_va) +{ + struct iscsi_hdr *hdr = &rx_desc->iscsi_header; + struct iscsi_conn *conn = isert_conn->conn; + struct iscsi_cmd *cmd; + struct isert_cmd *isert_cmd; + int ret = -EINVAL; + u8 opcode = (hdr->opcode & ISCSI_OPCODE_MASK); + + switch (opcode) { + case ISCSI_OP_SCSI_CMD: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + break; + + isert_cmd = container_of(cmd, struct isert_cmd, iscsi_cmd); + isert_cmd->read_stag = read_stag; + isert_cmd->read_va = read_va; + isert_cmd->write_stag = write_stag; + isert_cmd->write_va = write_va; + + ret = isert_handle_scsi_cmd(isert_conn, isert_cmd, + rx_desc, (unsigned char *)hdr); + break; + case ISCSI_OP_NOOP_OUT: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + break; + + ret = iscsit_handle_nop_out(conn, cmd, (unsigned char *)hdr); + break; + case ISCSI_OP_SCSI_DATA_OUT: + ret = isert_handle_iscsi_dataout(isert_conn, rx_desc, + (unsigned char *)hdr); + break; + case ISCSI_OP_SCSI_TMFUNC: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + break; + + ret = iscsit_handle_task_mgt_cmd(conn, cmd, + (unsigned char *)hdr); + break; + case ISCSI_OP_LOGOUT: + cmd = iscsit_allocate_cmd(conn, GFP_KERNEL); + if (!cmd) + break; + + ret = iscsit_handle_logout_cmd(conn, cmd, (unsigned char *)hdr); + if (ret > 0) + wait_for_completion_timeout(&conn->conn_logout_comp, + SECONDS_FOR_LOGOUT_COMP * + HZ); + break; + default: + pr_err("Got unknown iSCSI OpCode: 0x%02x\n", opcode); + dump_stack(); + break; + } + + return ret; +} + +static void +isert_rx_do_work(struct iser_rx_desc *rx_desc, struct isert_conn *isert_conn) +{ + struct iser_hdr *iser_hdr = &rx_desc->iser_header; + uint64_t read_va = 0, write_va = 0; + uint32_t read_stag = 0, write_stag = 0; + int rc; + + switch (iser_hdr->flags & 0xF0) { + case ISCSI_CTRL: + if (iser_hdr->flags & ISER_RSV) { + read_stag = be32_to_cpu(iser_hdr->read_stag); + read_va = be64_to_cpu(iser_hdr->read_va); + pr_debug("ISER_RSV: read_stag: 0x%08x read_va: 0x%16llx\n", + read_stag, (unsigned long long)read_va); + } + if (iser_hdr->flags & ISER_WSV) { + write_stag = be32_to_cpu(iser_hdr->write_stag); + write_va = be64_to_cpu(iser_hdr->write_va); + pr_debug("ISER_WSV: write__stag: 0x%08x write_va: 0x%16llx\n", + write_stag, (unsigned long long)write_va); + } + + pr_debug("ISER ISCSI_CTRL PDU\n"); + break; + case ISER_HELLO: + pr_err("iSER Hello message\n"); + break; + default: + pr_warn("Unknown iSER hdr flags: 0x%02x\n", iser_hdr->flags); + break; + } + + rc = isert_rx_opcode(isert_conn, rx_desc, + read_stag, read_va, write_stag, write_va); +} + +static void +isert_rx_completion(struct iser_rx_desc *desc, struct isert_conn *isert_conn, + unsigned long xfer_len) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct iscsi_hdr *hdr; + u64 rx_dma; + int rx_buflen, outstanding; + + if ((char *)desc == isert_conn->login_req_buf) { + rx_dma = isert_conn->login_req_dma; + rx_buflen = ISER_RX_LOGIN_SIZE; + pr_debug("ISER login_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", + rx_dma, rx_buflen); + } else { + rx_dma = desc->dma_addr; + rx_buflen = ISER_RX_PAYLOAD_SIZE; + pr_debug("ISER req_buf: Using rx_dma: 0x%llx, rx_buflen: %d\n", + rx_dma, rx_buflen); + } + + ib_dma_sync_single_for_cpu(ib_dev, rx_dma, rx_buflen, DMA_FROM_DEVICE); + + hdr = &desc->iscsi_header; + pr_debug("iSCSI opcode: 0x%02x, ITT: 0x%08x, flags: 0x%02x dlen: %d\n", + hdr->opcode, hdr->itt, hdr->flags, + (int)(xfer_len - ISER_HEADERS_LEN)); + + if ((char *)desc == isert_conn->login_req_buf) + isert_rx_login_req(desc, xfer_len - ISER_HEADERS_LEN, + isert_conn); + else + isert_rx_do_work(desc, isert_conn); + + ib_dma_sync_single_for_device(ib_dev, rx_dma, rx_buflen, + DMA_FROM_DEVICE); + + isert_conn->post_recv_buf_count--; + pr_debug("iSERT: Decremented post_recv_buf_count: %d\n", + isert_conn->post_recv_buf_count); + + if ((char *)desc == isert_conn->login_req_buf) + return; + + outstanding = isert_conn->post_recv_buf_count; + if (outstanding + ISERT_MIN_POSTED_RX <= ISERT_QP_MAX_RECV_DTOS) { + int err, count = min(ISERT_QP_MAX_RECV_DTOS - outstanding, + ISERT_MIN_POSTED_RX); + err = isert_post_recv(isert_conn, count); + if (err) { + pr_err("isert_post_recv() count: %d failed, %d\n", + count, err); + } + } +} + +static void +isert_unmap_cmd(struct isert_cmd *isert_cmd, struct isert_conn *isert_conn) +{ + struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + + pr_debug("isert_unmap_cmd >>>>>>>>>>>>>>>>>>>>>>>\n"); + + if (wr->sge) { + ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); + wr->sge = NULL; + } + + kfree(wr->send_wr); + wr->send_wr = NULL; + + kfree(isert_cmd->ib_sge); + isert_cmd->ib_sge = NULL; +} + +static void +isert_put_cmd(struct isert_cmd *isert_cmd) +{ + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + struct isert_conn *isert_conn = isert_cmd->conn; + struct iscsi_conn *conn; + + pr_debug("Entering isert_put_cmd: %p\n", isert_cmd); + + switch (cmd->iscsi_opcode) { + case ISCSI_OP_SCSI_CMD: + conn = isert_conn->conn; + + spin_lock_bh(&conn->cmd_lock); + if (!list_empty(&cmd->i_conn_node)) + list_del(&cmd->i_conn_node); + spin_unlock_bh(&conn->cmd_lock); + + if (cmd->data_direction == DMA_TO_DEVICE) + iscsit_stop_dataout_timer(cmd); + + isert_unmap_cmd(isert_cmd, isert_conn); + /* + * Fall-through + */ + case ISCSI_OP_SCSI_TMFUNC: + transport_generic_free_cmd(&cmd->se_cmd, 0); + break; + case ISCSI_OP_REJECT: + case ISCSI_OP_NOOP_OUT: + conn = isert_conn->conn; + + spin_lock_bh(&conn->cmd_lock); + if (!list_empty(&cmd->i_conn_node)) + list_del(&cmd->i_conn_node); + spin_unlock_bh(&conn->cmd_lock); + + /* + * Handle special case for REJECT when iscsi_add_reject*() has + * overwritten the original iscsi_opcode assignment, and the + * associated cmd->se_cmd needs to be released. + */ + if (cmd->se_cmd.se_tfo != NULL) { + transport_generic_free_cmd(&cmd->se_cmd, 0); + break; + } + /* + * Fall-through + */ + default: + isert_release_cmd(cmd); + break; + } +} + +static void +isert_unmap_tx_desc(struct iser_tx_desc *tx_desc, struct ib_device *ib_dev) +{ + if (tx_desc->dma_addr != 0) { + pr_debug("Calling ib_dma_unmap_single for tx_desc->dma_addr\n"); + ib_dma_unmap_single(ib_dev, tx_desc->dma_addr, + ISER_HEADERS_LEN, DMA_TO_DEVICE); + tx_desc->dma_addr = 0; + } +} + +static void +isert_completion_put(struct iser_tx_desc *tx_desc, struct isert_cmd *isert_cmd, + struct ib_device *ib_dev) +{ + if (isert_cmd->sense_buf_dma != 0) { + pr_debug("Calling ib_dma_unmap_single for isert_cmd->sense_buf_dma\n"); + ib_dma_unmap_single(ib_dev, isert_cmd->sense_buf_dma, + isert_cmd->sense_buf_len, DMA_TO_DEVICE); + isert_cmd->sense_buf_dma = 0; + } + + isert_unmap_tx_desc(tx_desc, ib_dev); + isert_put_cmd(isert_cmd); +} + +static void +isert_completion_rdma_read(struct iser_tx_desc *tx_desc, + struct isert_cmd *isert_cmd) +{ + struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + struct se_cmd *se_cmd = &cmd->se_cmd; + struct ib_device *ib_dev = isert_cmd->conn->conn_cm_id->device; + + iscsit_stop_dataout_timer(cmd); + + if (wr->sge) { + pr_debug("isert_do_rdma_read_comp: Unmapping wr->sge from t_data_sg\n"); + ib_dma_unmap_sg(ib_dev, wr->sge, wr->num_sge, DMA_TO_DEVICE); + wr->sge = NULL; + } + + if (isert_cmd->ib_sge) { + pr_debug("isert_do_rdma_read_comp: Freeing isert_cmd->ib_sge\n"); + kfree(isert_cmd->ib_sge); + isert_cmd->ib_sge = NULL; + } + + cmd->write_data_done = se_cmd->data_length; + + pr_debug("isert_do_rdma_read_comp, calling target_execute_cmd\n"); + spin_lock_bh(&cmd->istate_lock); + cmd->cmd_flags |= ICF_GOT_LAST_DATAOUT; + cmd->i_state = ISTATE_RECEIVED_LAST_DATAOUT; + spin_unlock_bh(&cmd->istate_lock); + + target_execute_cmd(se_cmd); +} + +static void +isert_do_control_comp(struct work_struct *work) +{ + struct isert_cmd *isert_cmd = container_of(work, + struct isert_cmd, comp_work); + struct isert_conn *isert_conn = isert_cmd->conn; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + + switch (cmd->i_state) { + case ISTATE_SEND_TASKMGTRSP: + pr_debug("Calling iscsit_tmr_post_handler >>>>>>>>>>>>>>>>>\n"); + + atomic_dec(&isert_conn->post_send_buf_count); + iscsit_tmr_post_handler(cmd, cmd->conn); + + cmd->i_state = ISTATE_SENT_STATUS; + isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); + break; + case ISTATE_SEND_REJECT: + pr_debug("Got isert_do_control_comp ISTATE_SEND_REJECT: >>>\n"); + atomic_dec(&isert_conn->post_send_buf_count); + + cmd->i_state = ISTATE_SENT_STATUS; + complete(&cmd->reject_comp); + isert_completion_put(&isert_cmd->tx_desc, isert_cmd, ib_dev); + case ISTATE_SEND_LOGOUTRSP: + pr_debug("Calling iscsit_logout_post_handler >>>>>>>>>>>>>>\n"); + /* + * Call atomic_dec(&isert_conn->post_send_buf_count) + * from isert_free_conn() + */ + isert_conn->logout_posted = true; + iscsit_logout_post_handler(cmd, cmd->conn); + break; + default: + pr_err("Unknown do_control_comp i_state %d\n", cmd->i_state); + dump_stack(); + break; + } +} + +static void +isert_response_completion(struct iser_tx_desc *tx_desc, + struct isert_cmd *isert_cmd, + struct isert_conn *isert_conn, + struct ib_device *ib_dev) +{ + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + + if (cmd->i_state == ISTATE_SEND_TASKMGTRSP || + cmd->i_state == ISTATE_SEND_LOGOUTRSP) { + isert_unmap_tx_desc(tx_desc, ib_dev); + + INIT_WORK(&isert_cmd->comp_work, isert_do_control_comp); + queue_work(isert_comp_wq, &isert_cmd->comp_work); + return; + } + atomic_dec(&isert_conn->post_send_buf_count); + + cmd->i_state = ISTATE_SENT_STATUS; + isert_completion_put(tx_desc, isert_cmd, ib_dev); +} + +static void +isert_send_completion(struct iser_tx_desc *tx_desc, + struct isert_conn *isert_conn) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct isert_cmd *isert_cmd = tx_desc->isert_cmd; + struct isert_rdma_wr *wr; + + if (!isert_cmd) { + atomic_dec(&isert_conn->post_send_buf_count); + isert_unmap_tx_desc(tx_desc, ib_dev); + return; + } + wr = &isert_cmd->rdma_wr; + + switch (wr->iser_ib_op) { + case ISER_IB_RECV: + pr_err("isert_send_completion: Got ISER_IB_RECV\n"); + dump_stack(); + break; + case ISER_IB_SEND: + pr_debug("isert_send_completion: Got ISER_IB_SEND\n"); + isert_response_completion(tx_desc, isert_cmd, + isert_conn, ib_dev); + break; + case ISER_IB_RDMA_WRITE: + pr_err("isert_send_completion: Got ISER_IB_RDMA_WRITE\n"); + dump_stack(); + break; + case ISER_IB_RDMA_READ: + pr_debug("isert_send_completion: Got ISER_IB_RDMA_READ:\n"); + + atomic_dec(&isert_conn->post_send_buf_count); + isert_completion_rdma_read(tx_desc, isert_cmd); + break; + default: + pr_err("Unknown wr->iser_ib_op: 0x%02x\n", wr->iser_ib_op); + dump_stack(); + break; + } +} + +static void +isert_cq_comp_err(struct iser_tx_desc *tx_desc, struct isert_conn *isert_conn) +{ + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + + if (tx_desc) { + struct isert_cmd *isert_cmd = tx_desc->isert_cmd; + + if (!isert_cmd) + isert_unmap_tx_desc(tx_desc, ib_dev); + else + isert_completion_put(tx_desc, isert_cmd, ib_dev); + } + + if (isert_conn->post_recv_buf_count == 0 && + atomic_read(&isert_conn->post_send_buf_count) == 0) { + pr_debug("isert_cq_comp_err >>>>>>>>>>>>>>>>>>>>>>>>>>>>>>\n"); + pr_debug("Calling wake_up from isert_cq_comp_err\n"); + + isert_conn->state = ISER_CONN_TERMINATING; + wake_up(&isert_conn->conn_wait_comp_err); + } +} + +static void +isert_cq_tx_work(struct work_struct *work) +{ + struct isert_cq_desc *cq_desc = container_of(work, + struct isert_cq_desc, cq_tx_work); + struct isert_device *device = cq_desc->device; + int cq_index = cq_desc->cq_index; + struct ib_cq *tx_cq = device->dev_tx_cq[cq_index]; + struct isert_conn *isert_conn; + struct iser_tx_desc *tx_desc; + struct ib_wc wc; + + while (ib_poll_cq(tx_cq, 1, &wc) == 1) { + tx_desc = (struct iser_tx_desc *)(unsigned long)wc.wr_id; + isert_conn = wc.qp->qp_context; + + if (wc.status == IB_WC_SUCCESS) { + isert_send_completion(tx_desc, isert_conn); + } else { + pr_debug("TX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); + pr_debug("TX wc.status: 0x%08x\n", wc.status); + atomic_dec(&isert_conn->post_send_buf_count); + isert_cq_comp_err(tx_desc, isert_conn); + } + } + + ib_req_notify_cq(tx_cq, IB_CQ_NEXT_COMP); +} + +static void +isert_cq_tx_callback(struct ib_cq *cq, void *context) +{ + struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; + + INIT_WORK(&cq_desc->cq_tx_work, isert_cq_tx_work); + queue_work(isert_comp_wq, &cq_desc->cq_tx_work); +} + +static void +isert_cq_rx_work(struct work_struct *work) +{ + struct isert_cq_desc *cq_desc = container_of(work, + struct isert_cq_desc, cq_rx_work); + struct isert_device *device = cq_desc->device; + int cq_index = cq_desc->cq_index; + struct ib_cq *rx_cq = device->dev_rx_cq[cq_index]; + struct isert_conn *isert_conn; + struct iser_rx_desc *rx_desc; + struct ib_wc wc; + unsigned long xfer_len; + + while (ib_poll_cq(rx_cq, 1, &wc) == 1) { + rx_desc = (struct iser_rx_desc *)(unsigned long)wc.wr_id; + isert_conn = wc.qp->qp_context; + + if (wc.status == IB_WC_SUCCESS) { + xfer_len = (unsigned long)wc.byte_len; + isert_rx_completion(rx_desc, isert_conn, xfer_len); + } else { + pr_debug("RX wc.status != IB_WC_SUCCESS >>>>>>>>>>>>>>\n"); + if (wc.status != IB_WC_WR_FLUSH_ERR) + pr_debug("RX wc.status: 0x%08x\n", wc.status); + + isert_conn->post_recv_buf_count--; + isert_cq_comp_err(NULL, isert_conn); + } + } + + ib_req_notify_cq(rx_cq, IB_CQ_NEXT_COMP); +} + +static void +isert_cq_rx_callback(struct ib_cq *cq, void *context) +{ + struct isert_cq_desc *cq_desc = (struct isert_cq_desc *)context; + + INIT_WORK(&cq_desc->cq_rx_work, isert_cq_rx_work); + queue_work(isert_rx_wq, &cq_desc->cq_rx_work); +} + +static int +isert_post_response(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd) +{ + struct ib_send_wr *wr_failed; + int ret; + + atomic_inc(&isert_conn->post_send_buf_count); + + ret = ib_post_send(isert_conn->conn_qp, &isert_cmd->tx_desc.send_wr, + &wr_failed); + if (ret) { + pr_err("ib_post_send failed with %d\n", ret); + atomic_dec(&isert_conn->post_send_buf_count); + return ret; + } + return ret; +} + +static int +isert_put_response(struct iscsi_conn *conn, struct iscsi_cmd *cmd) +{ + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; + struct iscsi_scsi_rsp *hdr = (struct iscsi_scsi_rsp *) + &isert_cmd->tx_desc.iscsi_header; + + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_rsp_pdu(cmd, conn, true, hdr); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + /* + * Attach SENSE DATA payload to iSCSI Response PDU + */ + if (cmd->se_cmd.sense_buffer && + ((cmd->se_cmd.se_cmd_flags & SCF_TRANSPORT_TASK_SENSE) || + (cmd->se_cmd.se_cmd_flags & SCF_EMULATED_TASK_SENSE))) { + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct ib_sge *tx_dsg = &isert_cmd->tx_desc.tx_sg[1]; + u32 padding, sense_len; + + put_unaligned_be16(cmd->se_cmd.scsi_sense_length, + cmd->sense_buffer); + cmd->se_cmd.scsi_sense_length += sizeof(__be16); + + padding = -(cmd->se_cmd.scsi_sense_length) & 3; + hton24(hdr->dlength, (u32)cmd->se_cmd.scsi_sense_length); + sense_len = cmd->se_cmd.scsi_sense_length + padding; + + isert_cmd->sense_buf_dma = ib_dma_map_single(ib_dev, + (void *)cmd->sense_buffer, sense_len, + DMA_TO_DEVICE); + + isert_cmd->sense_buf_len = sense_len; + tx_dsg->addr = isert_cmd->sense_buf_dma; + tx_dsg->length = sense_len; + tx_dsg->lkey = isert_conn->conn_mr->lkey; + isert_cmd->tx_desc.num_sge = 2; + } + + isert_init_send_wr(isert_cmd, send_wr); + + pr_debug("Posting SCSI Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); + + return isert_post_response(isert_conn, isert_cmd); +} + +static int +isert_put_nopin(struct iscsi_cmd *cmd, struct iscsi_conn *conn, + bool nopout_response) +{ + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; + + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_nopin_rsp(cmd, conn, (struct iscsi_nopin *) + &isert_cmd->tx_desc.iscsi_header, + nopout_response); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + isert_init_send_wr(isert_cmd, send_wr); + + pr_debug("Posting NOPIN Reponse IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); + + return isert_post_response(isert_conn, isert_cmd); +} + +static int +isert_put_logout_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; + + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_logout_rsp(cmd, conn, (struct iscsi_logout_rsp *) + &isert_cmd->tx_desc.iscsi_header); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + isert_init_send_wr(isert_cmd, send_wr); + + pr_debug("Posting Logout Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); + + return isert_post_response(isert_conn, isert_cmd); +} + +static int +isert_put_tm_rsp(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; + + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_task_mgt_rsp(cmd, conn, (struct iscsi_tm_rsp *) + &isert_cmd->tx_desc.iscsi_header); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + isert_init_send_wr(isert_cmd, send_wr); + + pr_debug("Posting Task Management Response IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); + + return isert_post_response(isert_conn, isert_cmd); +} + +static int +isert_put_reject(struct iscsi_cmd *cmd, struct iscsi_conn *conn) +{ + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *send_wr = &isert_cmd->tx_desc.send_wr; + + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_reject(cmd, conn, (struct iscsi_reject *) + &isert_cmd->tx_desc.iscsi_header); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + isert_init_send_wr(isert_cmd, send_wr); + + pr_debug("Posting Reject IB_WR_SEND >>>>>>>>>>>>>>>>>>>>>>\n"); + + return isert_post_response(isert_conn, isert_cmd); +} + +static int +isert_build_rdma_wr(struct isert_conn *isert_conn, struct isert_cmd *isert_cmd, + struct ib_sge *ib_sge, struct ib_send_wr *send_wr, + u32 data_left, u32 offset) +{ + struct iscsi_cmd *cmd = &isert_cmd->iscsi_cmd; + struct scatterlist *sg_start, *tmp_sg; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + u32 sg_off, page_off; + int i = 0, sg_nents; + + sg_off = offset / PAGE_SIZE; + sg_start = &cmd->se_cmd.t_data_sg[sg_off]; + sg_nents = min(cmd->se_cmd.t_data_nents - sg_off, isert_conn->max_sge); + page_off = offset % PAGE_SIZE; + + send_wr->sg_list = ib_sge; + send_wr->num_sge = sg_nents; + send_wr->wr_id = (unsigned long)&isert_cmd->tx_desc; + /* + * Perform mapping of TCM scatterlist memory ib_sge dma_addr. + */ + for_each_sg(sg_start, tmp_sg, sg_nents, i) { + pr_debug("ISER RDMA from SGL dma_addr: 0x%16llx dma_len: %u, page_off: %u\n", + (unsigned long long)tmp_sg->dma_address, + tmp_sg->length, page_off); + + ib_sge->addr = ib_sg_dma_address(ib_dev, tmp_sg) + page_off; + ib_sge->length = min_t(u32, data_left, + ib_sg_dma_len(ib_dev, tmp_sg) - page_off); + ib_sge->lkey = isert_conn->conn_mr->lkey; + + pr_debug("RDMA ib_sge: addr: 0x%16llx length: %u\n", + ib_sge->addr, ib_sge->length); + page_off = 0; + data_left -= ib_sge->length; + ib_sge++; + pr_debug("Incrementing ib_sge pointer to %p\n", ib_sge); + } + + pr_debug("Set outgoing sg_list: %p num_sg: %u from TCM SGLs\n", + send_wr->sg_list, send_wr->num_sge); + + return sg_nents; +} + +static int +isert_put_datain(struct iscsi_conn *conn, struct iscsi_cmd *cmd) +{ + struct se_cmd *se_cmd = &cmd->se_cmd; + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *wr_failed, *send_wr; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct ib_sge *ib_sge; + struct scatterlist *sg; + u32 offset = 0, data_len, data_left, rdma_write_max; + int rc, ret = 0, count, sg_nents, i, ib_sge_cnt; + + pr_debug("RDMA_WRITE: data_length: %u\n", se_cmd->data_length); + + sg = &se_cmd->t_data_sg[0]; + sg_nents = se_cmd->t_data_nents; + + count = ib_dma_map_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); + if (unlikely(!count)) { + pr_err("Unable to map put_datain SGs\n"); + return -EINVAL; + } + wr->sge = sg; + wr->num_sge = sg_nents; + pr_debug("Mapped IB count: %u sg: %p sg_nents: %u for RDMA_WRITE\n", + count, sg, sg_nents); + + ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); + if (!ib_sge) { + pr_warn("Unable to allocate datain ib_sge\n"); + ret = -ENOMEM; + goto unmap_sg; + } + isert_cmd->ib_sge = ib_sge; + + pr_debug("Allocated ib_sge: %p from t_data_ents: %d for RDMA_WRITE\n", + ib_sge, se_cmd->t_data_nents); + + wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); + wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, + GFP_KERNEL); + if (!wr->send_wr) { + pr_err("Unable to allocate wr->send_wr\n"); + ret = -ENOMEM; + goto unmap_sg; + } + pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", + wr->send_wr, wr->send_wr_num); + + iscsit_increment_maxcmdsn(cmd, conn->sess); + cmd->stat_sn = conn->stat_sn++; + + wr->isert_cmd = isert_cmd; + rdma_write_max = isert_conn->max_sge * PAGE_SIZE; + data_left = se_cmd->data_length; + + for (i = 0; i < wr->send_wr_num; i++) { + send_wr = &isert_cmd->rdma_wr.send_wr[i]; + data_len = min(data_left, rdma_write_max); + + send_wr->opcode = IB_WR_RDMA_WRITE; + send_wr->send_flags = 0; + send_wr->wr.rdma.remote_addr = isert_cmd->read_va + offset; + send_wr->wr.rdma.rkey = isert_cmd->read_stag; + + ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, + send_wr, data_len, offset); + ib_sge += ib_sge_cnt; + + if (i + 1 == wr->send_wr_num) + send_wr->next = &isert_cmd->tx_desc.send_wr; + else + send_wr->next = &wr->send_wr[i + 1]; + + offset += data_len; + data_left -= data_len; + } + /* + * Build isert_conn->tx_desc for iSCSI response PDU and attach + */ + isert_create_send_desc(isert_conn, isert_cmd, &isert_cmd->tx_desc); + iscsit_build_rsp_pdu(cmd, conn, false, (struct iscsi_scsi_rsp *) + &isert_cmd->tx_desc.iscsi_header); + isert_init_tx_hdrs(isert_conn, &isert_cmd->tx_desc); + isert_init_send_wr(isert_cmd, &isert_cmd->tx_desc.send_wr); + + atomic_inc(&isert_conn->post_send_buf_count); + + rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); + if (rc) { + pr_warn("ib_post_send() failed for IB_WR_RDMA_WRITE\n"); + atomic_dec(&isert_conn->post_send_buf_count); + } + pr_debug("Posted RDMA_WRITE + Response for iSER Data READ\n"); + return 1; + +unmap_sg: + ib_dma_unmap_sg(ib_dev, sg, sg_nents, DMA_TO_DEVICE); + return ret; +} + +static int +isert_get_dataout(struct iscsi_conn *conn, struct iscsi_cmd *cmd, bool recovery) +{ + struct se_cmd *se_cmd = &cmd->se_cmd; + struct isert_cmd *isert_cmd = container_of(cmd, + struct isert_cmd, iscsi_cmd); + struct isert_rdma_wr *wr = &isert_cmd->rdma_wr; + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + struct ib_send_wr *wr_failed, *send_wr; + struct ib_sge *ib_sge; + struct ib_device *ib_dev = isert_conn->conn_cm_id->device; + struct scatterlist *sg_start; + u32 sg_off, sg_nents, page_off, va_offset = 0; + u32 offset = 0, data_len, data_left, rdma_write_max; + int rc, ret = 0, count, i, ib_sge_cnt; + + pr_debug("RDMA_READ: data_length: %u write_data_done: %u\n", + se_cmd->data_length, cmd->write_data_done); + + sg_off = cmd->write_data_done / PAGE_SIZE; + sg_start = &cmd->se_cmd.t_data_sg[sg_off]; + page_off = cmd->write_data_done % PAGE_SIZE; + + pr_debug("RDMA_READ: sg_off: %d, sg_start: %p page_off: %d\n", + sg_off, sg_start, page_off); + + data_left = se_cmd->data_length - cmd->write_data_done; + sg_nents = se_cmd->t_data_nents - sg_off; + + pr_debug("RDMA_READ: data_left: %d, sg_nents: %d\n", + data_left, sg_nents); + + count = ib_dma_map_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); + if (unlikely(!count)) { + pr_err("Unable to map get_dataout SGs\n"); + return -EINVAL; + } + wr->sge = sg_start; + wr->num_sge = sg_nents; + pr_debug("Mapped IB count: %u sg_start: %p sg_nents: %u for RDMA_READ\n", + count, sg_start, sg_nents); + + ib_sge = kzalloc(sizeof(struct ib_sge) * sg_nents, GFP_KERNEL); + if (!ib_sge) { + pr_warn("Unable to allocate dataout ib_sge\n"); + ret = -ENOMEM; + goto unmap_sg; + } + isert_cmd->ib_sge = ib_sge; + + pr_debug("Using ib_sge: %p from sg_ents: %d for RDMA_READ\n", + ib_sge, sg_nents); + + wr->send_wr_num = DIV_ROUND_UP(sg_nents, isert_conn->max_sge); + wr->send_wr = kzalloc(sizeof(struct ib_send_wr) * wr->send_wr_num, + GFP_KERNEL); + if (!wr->send_wr) { + pr_debug("Unable to allocate wr->send_wr\n"); + ret = -ENOMEM; + goto unmap_sg; + } + pr_debug("Allocated wr->send_wr: %p wr->send_wr_num: %u\n", + wr->send_wr, wr->send_wr_num); + + isert_cmd->tx_desc.isert_cmd = isert_cmd; + + wr->iser_ib_op = ISER_IB_RDMA_READ; + wr->isert_cmd = isert_cmd; + rdma_write_max = isert_conn->max_sge * PAGE_SIZE; + offset = cmd->write_data_done; + + for (i = 0; i < wr->send_wr_num; i++) { + send_wr = &isert_cmd->rdma_wr.send_wr[i]; + data_len = min(data_left, rdma_write_max); + + send_wr->opcode = IB_WR_RDMA_READ; + send_wr->wr.rdma.remote_addr = isert_cmd->write_va + va_offset; + send_wr->wr.rdma.rkey = isert_cmd->write_stag; + + ib_sge_cnt = isert_build_rdma_wr(isert_conn, isert_cmd, ib_sge, + send_wr, data_len, offset); + ib_sge += ib_sge_cnt; + + if (i + 1 == wr->send_wr_num) + send_wr->send_flags = IB_SEND_SIGNALED; + else + send_wr->next = &wr->send_wr[i + 1]; + + offset += data_len; + va_offset += data_len; + data_left -= data_len; + } + + atomic_inc(&isert_conn->post_send_buf_count); + + rc = ib_post_send(isert_conn->conn_qp, wr->send_wr, &wr_failed); + if (rc) { + pr_warn("ib_post_send() failed for IB_WR_RDMA_READ\n"); + atomic_dec(&isert_conn->post_send_buf_count); + } + pr_debug("Posted RDMA_READ memory for ISER Data WRITE\n"); + return 0; + +unmap_sg: + ib_dma_unmap_sg(ib_dev, sg_start, sg_nents, DMA_FROM_DEVICE); + return ret; +} + +static int +isert_immediate_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) +{ + int ret; + + switch (state) { + case ISTATE_SEND_NOPIN_WANT_RESPONSE: + ret = isert_put_nopin(cmd, conn, false); + break; + default: + pr_err("Unknown immediate state: 0x%02x\n", state); + ret = -EINVAL; + break; + } + + return ret; +} + +static int +isert_response_queue(struct iscsi_conn *conn, struct iscsi_cmd *cmd, int state) +{ + int ret; + + switch (state) { + case ISTATE_SEND_LOGOUTRSP: + ret = isert_put_logout_rsp(cmd, conn); + if (!ret) { + pr_debug("Returning iSER Logout -EAGAIN\n"); + ret = -EAGAIN; + } + break; + case ISTATE_SEND_NOPIN: + ret = isert_put_nopin(cmd, conn, true); + break; + case ISTATE_SEND_TASKMGTRSP: + ret = isert_put_tm_rsp(cmd, conn); + break; + case ISTATE_SEND_REJECT: + ret = isert_put_reject(cmd, conn); + break; + case ISTATE_SEND_STATUS: + /* + * Special case for sending non GOOD SCSI status from TX thread + * context during pre se_cmd excecution failure. + */ + ret = isert_put_response(conn, cmd); + break; + default: + pr_err("Unknown response state: 0x%02x\n", state); + ret = -EINVAL; + break; + } + + return ret; +} + +static int +isert_setup_np(struct iscsi_np *np, + struct __kernel_sockaddr_storage *ksockaddr) +{ + struct isert_np *isert_np; + struct rdma_cm_id *isert_lid; + struct sockaddr *sa; + int ret; + + isert_np = kzalloc(sizeof(struct isert_np), GFP_KERNEL); + if (!isert_np) { + pr_err("Unable to allocate struct isert_np\n"); + return -ENOMEM; + } + init_waitqueue_head(&isert_np->np_accept_wq); + mutex_init(&isert_np->np_accept_mutex); + INIT_LIST_HEAD(&isert_np->np_accept_list); + init_completion(&isert_np->np_login_comp); + + sa = (struct sockaddr *)ksockaddr; + pr_debug("ksockaddr: %p, sa: %p\n", ksockaddr, sa); + /* + * Setup the np->np_sockaddr from the passed sockaddr setup + * in iscsi_target_configfs.c code.. + */ + memcpy(&np->np_sockaddr, ksockaddr, + sizeof(struct __kernel_sockaddr_storage)); + + isert_lid = rdma_create_id(isert_cma_handler, np, RDMA_PS_TCP, + IB_QPT_RC); + if (IS_ERR(isert_lid)) { + pr_err("rdma_create_id() for isert_listen_handler failed: %ld\n", + PTR_ERR(isert_lid)); + ret = PTR_ERR(isert_lid); + goto out; + } + + ret = rdma_bind_addr(isert_lid, sa); + if (ret) { + pr_err("rdma_bind_addr() for isert_lid failed: %d\n", ret); + goto out_lid; + } + + ret = rdma_listen(isert_lid, ISERT_RDMA_LISTEN_BACKLOG); + if (ret) { + pr_err("rdma_listen() for isert_lid failed: %d\n", ret); + goto out_lid; + } + + isert_np->np_cm_id = isert_lid; + np->np_context = isert_np; + pr_debug("Setup isert_lid->context: %p\n", isert_lid->context); + + return 0; + +out_lid: + rdma_destroy_id(isert_lid); +out: + kfree(isert_np); + return ret; +} + +static int +isert_check_accept_queue(struct isert_np *isert_np) +{ + int empty; + + mutex_lock(&isert_np->np_accept_mutex); + empty = list_empty(&isert_np->np_accept_list); + mutex_unlock(&isert_np->np_accept_mutex); + + return empty; +} + +static int +isert_rdma_accept(struct isert_conn *isert_conn) +{ + struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; + struct rdma_conn_param cp; + int ret; + + memset(&cp, 0, sizeof(struct rdma_conn_param)); + cp.responder_resources = isert_conn->responder_resources; + cp.initiator_depth = isert_conn->initiator_depth; + cp.retry_count = 7; + cp.rnr_retry_count = 7; + + pr_debug("Before rdma_accept >>>>>>>>>>>>>>>>>>>>.\n"); + + ret = rdma_accept(cm_id, &cp); + if (ret) { + pr_err("rdma_accept() failed with: %d\n", ret); + return ret; + } + + pr_debug("After rdma_accept >>>>>>>>>>>>>>>>>>>>>.\n"); + + return 0; +} + +static int +isert_get_login_rx(struct iscsi_conn *conn, struct iscsi_login *login) +{ + struct isert_conn *isert_conn = (struct isert_conn *)conn->context; + int ret; + + pr_debug("isert_get_login_rx before conn_login_comp conn: %p\n", conn); + + ret = wait_for_completion_interruptible(&isert_conn->conn_login_comp); + if (ret) + return ret; + + pr_debug("isert_get_login_rx processing login->req: %p\n", login->req); + return 0; +} + +static void +isert_set_conn_info(struct iscsi_np *np, struct iscsi_conn *conn, + struct isert_conn *isert_conn) +{ + struct rdma_cm_id *cm_id = isert_conn->conn_cm_id; + struct rdma_route *cm_route = &cm_id->route; + struct sockaddr_in *sock_in; + struct sockaddr_in6 *sock_in6; + + conn->login_family = np->np_sockaddr.ss_family; + + if (np->np_sockaddr.ss_family == AF_INET6) { + sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.dst_addr; + snprintf(conn->login_ip, sizeof(conn->login_ip), "%pI6c", + &sock_in6->sin6_addr.in6_u); + conn->login_port = ntohs(sock_in6->sin6_port); + + sock_in6 = (struct sockaddr_in6 *)&cm_route->addr.src_addr; + snprintf(conn->local_ip, sizeof(conn->local_ip), "%pI6c", + &sock_in6->sin6_addr.in6_u); + conn->local_port = ntohs(sock_in6->sin6_port); + } else { + sock_in = (struct sockaddr_in *)&cm_route->addr.dst_addr; + sprintf(conn->login_ip, "%pI4", + &sock_in->sin_addr.s_addr); + conn->login_port = ntohs(sock_in->sin_port); + + sock_in = (struct sockaddr_in *)&cm_route->addr.src_addr; + sprintf(conn->local_ip, "%pI4", + &sock_in->sin_addr.s_addr); + conn->local_port = ntohs(sock_in->sin_port); + } +} + +static int +isert_accept_np(struct iscsi_np *np, struct iscsi_conn *conn) +{ + struct isert_np *isert_np = (struct isert_np *)np->np_context; + struct isert_conn *isert_conn; + int max_accept = 0, ret; + +accept_wait: + ret = wait_event_interruptible(isert_np->np_accept_wq, + !isert_check_accept_queue(isert_np) || + np->np_thread_state == ISCSI_NP_THREAD_RESET); + if (max_accept > 5) + return -ENODEV; + + spin_lock_bh(&np->np_thread_lock); + if (np->np_thread_state == ISCSI_NP_THREAD_RESET) { + spin_unlock_bh(&np->np_thread_lock); + pr_err("ISCSI_NP_THREAD_RESET for isert_accept_np\n"); + return -ENODEV; + } + spin_unlock_bh(&np->np_thread_lock); + + mutex_lock(&isert_np->np_accept_mutex); + if (list_empty(&isert_np->np_accept_list)) { + mutex_unlock(&isert_np->np_accept_mutex); + max_accept++; + goto accept_wait; + } + isert_conn = list_first_entry(&isert_np->np_accept_list, + struct isert_conn, conn_accept_node); + list_del_init(&isert_conn->conn_accept_node); + mutex_unlock(&isert_np->np_accept_mutex); + + conn->context = isert_conn; + isert_conn->conn = conn; + max_accept = 0; + + ret = isert_rdma_post_recvl(isert_conn); + if (ret) + return ret; + + ret = isert_rdma_accept(isert_conn); + if (ret) + return ret; + + isert_set_conn_info(np, conn, isert_conn); + + pr_debug("Processing isert_accept_np: isert_conn: %p\n", isert_conn); + return 0; +} + +static void +isert_free_np(struct iscsi_np *np) +{ + struct isert_np *isert_np = (struct isert_np *)np->np_context; + + rdma_destroy_id(isert_np->np_cm_id); + + np->np_context = NULL; + kfree(isert_np); +} + +static void isert_free_conn(struct iscsi_conn *conn) +{ + struct isert_conn *isert_conn = conn->context; + + pr_debug("isert_free_conn: Starting \n"); + /* + * Decrement post_send_buf_count for special case when called + * from isert_do_control_comp() -> iscsit_logout_post_handler() + */ + if (isert_conn->logout_posted) + atomic_dec(&isert_conn->post_send_buf_count); + + if (isert_conn->conn_cm_id) + rdma_disconnect(isert_conn->conn_cm_id); + /* + * Only wait for conn_wait_comp_err if the isert_conn made it + * into full feature phase.. + */ + if (isert_conn->state > ISER_CONN_INIT) { + pr_debug("isert_free_conn: Before wait_event comp_err %d\n", + isert_conn->state); + wait_event(isert_conn->conn_wait_comp_err, + isert_conn->state == ISER_CONN_TERMINATING); + pr_debug("isert_free_conn: After wait_event #1 >>>>>>>>>>>>\n"); + } + + pr_debug("isert_free_conn: wait_event conn_wait %d\n", isert_conn->state); + wait_event(isert_conn->conn_wait, isert_conn->state == ISER_CONN_DOWN); + pr_debug("isert_free_conn: After wait_event #2 >>>>>>>>>>>>>>>>>>>>\n"); + + isert_put_conn(isert_conn); +} + +static struct iscsit_transport iser_target_transport = { + .name = "IB/iSER", + .transport_type = ISCSI_INFINIBAND, + .owner = THIS_MODULE, + .iscsit_setup_np = isert_setup_np, + .iscsit_accept_np = isert_accept_np, + .iscsit_free_np = isert_free_np, + .iscsit_free_conn = isert_free_conn, + .iscsit_alloc_cmd = isert_alloc_cmd, + .iscsit_get_login_rx = isert_get_login_rx, + .iscsit_put_login_tx = isert_put_login_tx, + .iscsit_immediate_queue = isert_immediate_queue, + .iscsit_response_queue = isert_response_queue, + .iscsit_get_dataout = isert_get_dataout, + .iscsit_queue_data_in = isert_put_datain, + .iscsit_queue_status = isert_put_response, +}; + +static int __init isert_init(void) +{ + int ret; + + isert_rx_wq = alloc_workqueue("isert_rx_wq", 0, 0); + if (!isert_rx_wq) { + pr_err("Unable to allocate isert_rx_wq\n"); + return -ENOMEM; + } + + isert_comp_wq = alloc_workqueue("isert_comp_wq", 0, 0); + if (!isert_comp_wq) { + pr_err("Unable to allocate isert_comp_wq\n"); + ret = -ENOMEM; + goto destroy_rx_wq; + } + + isert_cmd_cache = kmem_cache_create("isert_cmd_cache", + sizeof(struct isert_cmd), __alignof__(struct isert_cmd), + 0, NULL); + if (!isert_cmd_cache) { + pr_err("Unable to create isert_cmd_cache\n"); + ret = -ENOMEM; + goto destroy_tx_cq; + } + + iscsit_register_transport(&iser_target_transport); + pr_debug("iSER_TARGET[0] - Loaded iser_target_transport\n"); + return 0; + +destroy_tx_cq: + destroy_workqueue(isert_comp_wq); +destroy_rx_wq: + destroy_workqueue(isert_rx_wq); + return ret; +} + +static void __exit isert_exit(void) +{ + kmem_cache_destroy(isert_cmd_cache); + destroy_workqueue(isert_comp_wq); + destroy_workqueue(isert_rx_wq); + iscsit_unregister_transport(&iser_target_transport); + pr_debug("iSER_TARGET[0] - Released iser_target_transport\n"); +} + +MODULE_DESCRIPTION("iSER-Target for mainline target infrastructure"); +MODULE_VERSION("0.1"); +MODULE_AUTHOR("nab@Linux-iSCSI.org"); +MODULE_LICENSE("GPL"); + +module_init(isert_init); +module_exit(isert_exit); diff --git a/drivers/infiniband/ulp/isert/ib_isert.h b/drivers/infiniband/ulp/isert/ib_isert.h new file mode 100644 index 000000000000..b104f4c2cd38 --- /dev/null +++ b/drivers/infiniband/ulp/isert/ib_isert.h @@ -0,0 +1,138 @@ +#include +#include +#include +#include +#include + +#define ISERT_RDMA_LISTEN_BACKLOG 10 + +enum isert_desc_type { + ISCSI_TX_CONTROL, + ISCSI_TX_DATAIN +}; + +enum iser_ib_op_code { + ISER_IB_RECV, + ISER_IB_SEND, + ISER_IB_RDMA_WRITE, + ISER_IB_RDMA_READ, +}; + +enum iser_conn_state { + ISER_CONN_INIT, + ISER_CONN_UP, + ISER_CONN_TERMINATING, + ISER_CONN_DOWN, +}; + +struct iser_rx_desc { + struct iser_hdr iser_header; + struct iscsi_hdr iscsi_header; + char data[ISER_RECV_DATA_SEG_LEN]; + u64 dma_addr; + struct ib_sge rx_sg; + char pad[ISER_RX_PAD_SIZE]; +} __packed; + +struct iser_tx_desc { + struct iser_hdr iser_header; + struct iscsi_hdr iscsi_header; + enum isert_desc_type type; + u64 dma_addr; + struct ib_sge tx_sg[2]; + int num_sge; + struct isert_cmd *isert_cmd; + struct ib_send_wr send_wr; +} __packed; + +struct isert_rdma_wr { + struct list_head wr_list; + struct isert_cmd *isert_cmd; + enum iser_ib_op_code iser_ib_op; + struct ib_sge *ib_sge; + int num_sge; + struct scatterlist *sge; + int send_wr_num; + struct ib_send_wr *send_wr; +}; + +struct isert_cmd { + uint32_t read_stag; + uint32_t write_stag; + uint64_t read_va; + uint64_t write_va; + u64 sense_buf_dma; + u32 sense_buf_len; + u32 read_va_off; + u32 write_va_off; + u32 rdma_wr_num; + struct isert_conn *conn; + struct iscsi_cmd iscsi_cmd; + struct ib_sge *ib_sge; + struct iser_tx_desc tx_desc; + struct isert_rdma_wr rdma_wr; + struct work_struct comp_work; +}; + +struct isert_device; + +struct isert_conn { + enum iser_conn_state state; + bool logout_posted; + int post_recv_buf_count; + atomic_t post_send_buf_count; + u32 responder_resources; + u32 initiator_depth; + u32 max_sge; + char *login_buf; + char *login_req_buf; + char *login_rsp_buf; + u64 login_req_dma; + u64 login_rsp_dma; + unsigned int conn_rx_desc_head; + struct iser_rx_desc *conn_rx_descs; + struct ib_recv_wr conn_rx_wr[ISERT_MIN_POSTED_RX]; + struct iscsi_conn *conn; + struct list_head conn_accept_node; + struct completion conn_login_comp; + struct iser_tx_desc conn_login_tx_desc; + struct rdma_cm_id *conn_cm_id; + struct ib_pd *conn_pd; + struct ib_mr *conn_mr; + struct ib_qp *conn_qp; + struct isert_device *conn_device; + struct work_struct conn_logout_work; + wait_queue_head_t conn_wait; + wait_queue_head_t conn_wait_comp_err; + struct kref conn_kref; +}; + +#define ISERT_MAX_CQ 64 + +struct isert_cq_desc { + struct isert_device *device; + int cq_index; + struct work_struct cq_rx_work; + struct work_struct cq_tx_work; +}; + +struct isert_device { + int cqs_used; + int refcount; + int cq_active_qps[ISERT_MAX_CQ]; + struct ib_device *ib_device; + struct ib_pd *dev_pd; + struct ib_mr *dev_mr; + struct ib_cq *dev_rx_cq[ISERT_MAX_CQ]; + struct ib_cq *dev_tx_cq[ISERT_MAX_CQ]; + struct isert_cq_desc *cq_desc; + struct list_head dev_node; +}; + +struct isert_np { + wait_queue_head_t np_accept_wq; + struct rdma_cm_id *np_cm_id; + struct mutex np_accept_mutex; + struct list_head np_accept_list; + struct completion np_login_comp; +}; diff --git a/drivers/infiniband/ulp/isert/isert_proto.h b/drivers/infiniband/ulp/isert/isert_proto.h new file mode 100644 index 000000000000..4dccd313b777 --- /dev/null +++ b/drivers/infiniband/ulp/isert/isert_proto.h @@ -0,0 +1,47 @@ +/* From iscsi_iser.h */ + +struct iser_hdr { + u8 flags; + u8 rsvd[3]; + __be32 write_stag; /* write rkey */ + __be64 write_va; + __be32 read_stag; /* read rkey */ + __be64 read_va; +} __packed; + +/*Constant PDU lengths calculations */ +#define ISER_HEADERS_LEN (sizeof(struct iser_hdr) + sizeof(struct iscsi_hdr)) + +#define ISER_RECV_DATA_SEG_LEN 8192 +#define ISER_RX_PAYLOAD_SIZE (ISER_HEADERS_LEN + ISER_RECV_DATA_SEG_LEN) +#define ISER_RX_LOGIN_SIZE (ISER_HEADERS_LEN + ISCSI_DEF_MAX_RECV_SEG_LEN) + +/* QP settings */ +/* Maximal bounds on received asynchronous PDUs */ +#define ISERT_MAX_TX_MISC_PDUS 4 /* NOOP_IN(2) , ASYNC_EVENT(2) */ + +#define ISERT_MAX_RX_MISC_PDUS 6 /* NOOP_OUT(2), TEXT(1), * + * SCSI_TMFUNC(2), LOGOUT(1) */ + +#define ISCSI_DEF_XMIT_CMDS_MAX 128 /* from libiscsi.h, must be power of 2 */ + +#define ISERT_QP_MAX_RECV_DTOS (ISCSI_DEF_XMIT_CMDS_MAX) + +#define ISERT_MIN_POSTED_RX (ISCSI_DEF_XMIT_CMDS_MAX >> 2) + +#define ISERT_INFLIGHT_DATAOUTS 8 + +#define ISERT_QP_MAX_REQ_DTOS (ISCSI_DEF_XMIT_CMDS_MAX * \ + (1 + ISERT_INFLIGHT_DATAOUTS) + \ + ISERT_MAX_TX_MISC_PDUS + \ + ISERT_MAX_RX_MISC_PDUS) + +#define ISER_RX_PAD_SIZE (ISER_RECV_DATA_SEG_LEN + 4096 - \ + (ISER_RX_PAYLOAD_SIZE + sizeof(u64) + sizeof(struct ib_sge))) + +#define ISER_VER 0x10 +#define ISER_WSV 0x08 +#define ISER_RSV 0x04 +#define ISCSI_CTRL 0x10 +#define ISER_HELLO 0x20 +#define ISER_HELLORPLY 0x30