From 05f1d8ed03f547054efbc4d29bb7991c958ede95 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Fri, 21 Jul 2023 21:36:44 +0200 Subject: [PATCH 1/8] s390/dasd: fix hanging device after quiesce/resume Quiesce and resume are functions that tell the DASD driver to stop/resume issuing I/Os to a specific DASD. On resume dasd_schedule_block_bh() is called to kick handling of IO requests again. This does unfortunately not cover internal requests which are used for path verification for example. This could lead to a hanging device when a path event or anything else that triggers internal requests occurs on a quiesced device. Fix by also calling dasd_schedule_device_bh() which triggers handling of internal requests on resume. Fixes: 8e09f21574ea ("[S390] dasd: add hyper PAV support to DASD device driver, part 1") Cc: stable@vger.kernel.org Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20230721193647.3889634-2-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_ioctl.c | 1 + 1 file changed, 1 insertion(+) diff --git a/drivers/s390/block/dasd_ioctl.c b/drivers/s390/block/dasd_ioctl.c index 513a7e6eee63..d55862605b82 100644 --- a/drivers/s390/block/dasd_ioctl.c +++ b/drivers/s390/block/dasd_ioctl.c @@ -131,6 +131,7 @@ static int dasd_ioctl_resume(struct dasd_block *block) spin_unlock_irqrestore(get_ccwdev_lock(base->cdev), flags); dasd_schedule_block_bh(block); + dasd_schedule_device_bh(base); return 0; } From acea28a6b74f458defda7417d2217b051ba7d444 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Fri, 21 Jul 2023 21:36:45 +0200 Subject: [PATCH 2/8] s390/dasd: use correct number of retries for ERP requests If a DASD request fails an error recovery procedure (ERP) request might be built as a copy of the original request to do error recovery. The ERP request gets a number of retries assigned. This number is always 256 no matter what other value might have been set for the original request. This is not what is expected when a user specifies a certain amount of retries for the device via sysfs. Correctly use the number of retries of the original request for ERP requests. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20230721193647.3889634-3-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_3990_erp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 9fd36c468706..91e9a17b848e 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -2441,7 +2441,7 @@ static struct dasd_ccw_req *dasd_3990_erp_add_erp(struct dasd_ccw_req *cqr) erp->block = cqr->block; erp->magic = cqr->magic; erp->expires = cqr->expires; - erp->retries = 256; + erp->retries = device->default_retries; erp->buildclk = get_tod_clock(); erp->status = DASD_CQR_FILLED; From 8a2278ce9c25048d999fe1a3561def75d963f471 Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Fri, 21 Jul 2023 21:36:46 +0200 Subject: [PATCH 3/8] s390/dasd: fix hanging device after request requeue The DASD device driver has a function to requeue requests to the blocklayer. This function is used in various cases when basic settings for the device have to be changed like High Performance Ficon related parameters or copy pair settings. The functions iterates over the device->ccw_queue and also removes the requests from the block->ccw_queue. In case the device is started on an alias device instead of the base device it might be removed from the block->ccw_queue without having it canceled properly before. This might lead to a hanging device since the request is no longer on a queue and can not be handled properly. Fix by iterating over the block->ccw_queue instead of the device->ccw_queue. This will take care of all blocklayer related requests and handle them on all associated DASD devices. Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20230721193647.3889634-4-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd.c | 127 +++++++++++++++----------------------- 1 file changed, 49 insertions(+), 78 deletions(-) diff --git a/drivers/s390/block/dasd.c b/drivers/s390/block/dasd.c index edcbf77852c3..50a5ff70814a 100644 --- a/drivers/s390/block/dasd.c +++ b/drivers/s390/block/dasd.c @@ -2943,41 +2943,32 @@ static void _dasd_wake_block_flush_cb(struct dasd_ccw_req *cqr, void *data) * Requeue a request back to the block request queue * only works for block requests */ -static int _dasd_requeue_request(struct dasd_ccw_req *cqr) +static void _dasd_requeue_request(struct dasd_ccw_req *cqr) { - struct dasd_block *block = cqr->block; struct request *req; - if (!block) - return -EINVAL; /* * If the request is an ERP request there is nothing to requeue. * This will be done with the remaining original request. */ if (cqr->refers) - return 0; + return; spin_lock_irq(&cqr->dq->lock); req = (struct request *) cqr->callback_data; blk_mq_requeue_request(req, true); spin_unlock_irq(&cqr->dq->lock); - return 0; + return; } -/* - * Go through all request on the dasd_block request queue, cancel them - * on the respective dasd_device, and return them to the generic - * block layer. - */ -static int dasd_flush_block_queue(struct dasd_block *block) +static int _dasd_requests_to_flushqueue(struct dasd_block *block, + struct list_head *flush_queue) { struct dasd_ccw_req *cqr, *n; - int rc, i; - struct list_head flush_queue; unsigned long flags; + int rc, i; - INIT_LIST_HEAD(&flush_queue); - spin_lock_bh(&block->queue_lock); + spin_lock_irqsave(&block->queue_lock, flags); rc = 0; restart: list_for_each_entry_safe(cqr, n, &block->ccw_queue, blocklist) { @@ -2992,13 +2983,32 @@ static int dasd_flush_block_queue(struct dasd_block *block) * is returned from the dasd_device layer. */ cqr->callback = _dasd_wake_block_flush_cb; - for (i = 0; cqr != NULL; cqr = cqr->refers, i++) - list_move_tail(&cqr->blocklist, &flush_queue); + for (i = 0; cqr; cqr = cqr->refers, i++) + list_move_tail(&cqr->blocklist, flush_queue); if (i > 1) /* moved more than one request - need to restart */ goto restart; } - spin_unlock_bh(&block->queue_lock); + spin_unlock_irqrestore(&block->queue_lock, flags); + + return rc; +} + +/* + * Go through all request on the dasd_block request queue, cancel them + * on the respective dasd_device, and return them to the generic + * block layer. + */ +static int dasd_flush_block_queue(struct dasd_block *block) +{ + struct dasd_ccw_req *cqr, *n; + struct list_head flush_queue; + unsigned long flags; + int rc; + + INIT_LIST_HEAD(&flush_queue); + rc = _dasd_requests_to_flushqueue(block, &flush_queue); + /* Now call the callback function of flushed requests */ restart_cb: list_for_each_entry_safe(cqr, n, &flush_queue, blocklist) { @@ -3881,75 +3891,36 @@ EXPORT_SYMBOL_GPL(dasd_generic_space_avail); */ int dasd_generic_requeue_all_requests(struct dasd_device *device) { + struct dasd_block *block = device->block; struct list_head requeue_queue; struct dasd_ccw_req *cqr, *n; - struct dasd_ccw_req *refers; int rc; + if (!block) + return 0; + INIT_LIST_HEAD(&requeue_queue); - spin_lock_irq(get_ccwdev_lock(device->cdev)); - rc = 0; - list_for_each_entry_safe(cqr, n, &device->ccw_queue, devlist) { - /* Check status and move request to flush_queue */ - if (cqr->status == DASD_CQR_IN_IO) { - rc = device->discipline->term_IO(cqr); - if (rc) { - /* unable to terminate requeust */ - dev_err(&device->cdev->dev, - "Unable to terminate request %p " - "on suspend\n", cqr); - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - dasd_put_device(device); - return rc; - } + rc = _dasd_requests_to_flushqueue(block, &requeue_queue); + + /* Now call the callback function of flushed requests */ +restart_cb: + list_for_each_entry_safe(cqr, n, &requeue_queue, blocklist) { + wait_event(dasd_flush_wq, (cqr->status < DASD_CQR_QUEUED)); + /* Process finished ERP request. */ + if (cqr->refers) { + spin_lock_bh(&block->queue_lock); + __dasd_process_erp(block->base, cqr); + spin_unlock_bh(&block->queue_lock); + /* restart list_for_xx loop since dasd_process_erp + * might remove multiple elements + */ + goto restart_cb; } - list_move_tail(&cqr->devlist, &requeue_queue); - } - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - - list_for_each_entry_safe(cqr, n, &requeue_queue, devlist) { - wait_event(dasd_flush_wq, - (cqr->status != DASD_CQR_CLEAR_PENDING)); - - /* - * requeue requests to blocklayer will only work - * for block device requests - */ - if (_dasd_requeue_request(cqr)) - continue; - - /* remove requests from device and block queue */ - list_del_init(&cqr->devlist); - while (cqr->refers != NULL) { - refers = cqr->refers; - /* remove the request from the block queue */ - list_del(&cqr->blocklist); - /* free the finished erp request */ - dasd_free_erp_request(cqr, cqr->memdev); - cqr = refers; - } - - /* - * _dasd_requeue_request already checked for a valid - * blockdevice, no need to check again - * all erp requests (cqr->refers) have a cqr->block - * pointer copy from the original cqr - */ + _dasd_requeue_request(cqr); list_del_init(&cqr->blocklist); cqr->block->base->discipline->free_cp( cqr, (struct request *) cqr->callback_data); } - - /* - * if requests remain then they are internal request - * and go back to the device queue - */ - if (!list_empty(&requeue_queue)) { - /* move freeze_queue to start of the ccw_queue */ - spin_lock_irq(get_ccwdev_lock(device->cdev)); - list_splice_tail(&requeue_queue, &device->ccw_queue); - spin_unlock_irq(get_ccwdev_lock(device->cdev)); - } dasd_schedule_device_bh(device); return rc; } From 856d8e3c633b183df23549ce760ae84478a7098d Mon Sep 17 00:00:00 2001 From: Stefan Haberland Date: Fri, 21 Jul 2023 21:36:47 +0200 Subject: [PATCH 4/8] s390/dasd: print copy pair message only for the correct error The DASD driver has certain types of requests that might be rejected by the storage server or z/VM because they are not supported. Since the missing support of the command is not a real issue there is no user visible kernel error message for this. For copy pair setups there is a specific error that IO is not allowed on secondary devices. This error case is explicitly handled and an error message is printed. The code checking for the error did use a bitwise 'and' that is used to check for specific bits. But in this case the whole sense byte has to match. This leads to the problem that the copy pair related error message is erroneously printed for other error cases that are usually not reported. This might heavily confuse users and lead to follow on actions that might disrupt application processing. Fix by checking the sense byte for the exact value and not single bits. Cc: stable@vger.kernel.org # 6.1+ Fixes: 1fca631a1185 ("s390/dasd: suppress generic error messages for PPRC secondary devices") Signed-off-by: Stefan Haberland Reviewed-by: Jan Hoeppner Link: https://lore.kernel.org/r/20230721193647.3889634-5-sth@linux.ibm.com Signed-off-by: Jens Axboe --- drivers/s390/block/dasd_3990_erp.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/drivers/s390/block/dasd_3990_erp.c b/drivers/s390/block/dasd_3990_erp.c index 91e9a17b848e..89957bb7244d 100644 --- a/drivers/s390/block/dasd_3990_erp.c +++ b/drivers/s390/block/dasd_3990_erp.c @@ -1050,7 +1050,7 @@ dasd_3990_erp_com_rej(struct dasd_ccw_req * erp, char *sense) dev_err(&device->cdev->dev, "An I/O request was rejected" " because writing is inhibited\n"); erp = dasd_3990_erp_cleanup(erp, DASD_CQR_FAILED); - } else if (sense[7] & SNS7_INVALID_ON_SEC) { + } else if (sense[7] == SNS7_INVALID_ON_SEC) { dev_err(&device->cdev->dev, "An I/O request was rejected on a copy pair secondary device\n"); /* suppress dump of sense data for this error */ set_bit(DASD_CQR_SUPPRESS_CR, &erp->refers->flags); From e0933b526fbfd937c4a8f4e35fcdd49f0e22d411 Mon Sep 17 00:00:00 2001 From: Bart Van Assche Date: Thu, 6 Jul 2023 13:14:12 -0700 Subject: [PATCH 5/8] block: Fix a source code comment in include/uapi/linux/blkzoned.h Fix the symbolic names for zone conditions in the blkzoned.h header file. Cc: Hannes Reinecke Cc: Damien Le Moal Fixes: 6a0cb1bc106f ("block: Implement support for zoned block devices") Signed-off-by: Bart Van Assche Reviewed-by: Damien Le Moal Link: https://lore.kernel.org/r/20230706201422.3987341-1-bvanassche@acm.org Signed-off-by: Jens Axboe --- include/uapi/linux/blkzoned.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/uapi/linux/blkzoned.h b/include/uapi/linux/blkzoned.h index b80fcc9ea525..f85743ef6e7d 100644 --- a/include/uapi/linux/blkzoned.h +++ b/include/uapi/linux/blkzoned.h @@ -51,13 +51,13 @@ enum blk_zone_type { * * The Zone Condition state machine in the ZBC/ZAC standards maps the above * deinitions as: - * - ZC1: Empty | BLK_ZONE_EMPTY + * - ZC1: Empty | BLK_ZONE_COND_EMPTY * - ZC2: Implicit Open | BLK_ZONE_COND_IMP_OPEN * - ZC3: Explicit Open | BLK_ZONE_COND_EXP_OPEN - * - ZC4: Closed | BLK_ZONE_CLOSED - * - ZC5: Full | BLK_ZONE_FULL - * - ZC6: Read Only | BLK_ZONE_READONLY - * - ZC7: Offline | BLK_ZONE_OFFLINE + * - ZC4: Closed | BLK_ZONE_COND_CLOSED + * - ZC5: Full | BLK_ZONE_COND_FULL + * - ZC6: Read Only | BLK_ZONE_COND_READONLY + * - ZC7: Offline | BLK_ZONE_COND_OFFLINE * * Conditions 0x5 to 0xC are reserved by the current ZBC/ZAC spec and should * be considered invalid. From 53e7d08f6d6e214c40db1f51291bb2975c789dc2 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 26 Jul 2023 22:45:00 +0800 Subject: [PATCH 6/8] ublk: fail to start device if queue setup is interrupted In ublk_ctrl_start_dev(), if wait_for_completion_interruptible() is interrupted by signal, queues aren't setup successfully yet, so we have to fail UBLK_CMD_START_DEV, otherwise kernel oops can be triggered. Reported by German when working on qemu-storage-deamon which requires single thread ublk daemon. Fixes: 71f28f3136af ("ublk_drv: add io_uring based userspace block driver") Reported-by: German Maglione Signed-off-by: Ming Lei Link: https://lore.kernel.org/r/20230726144502.566785-2-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 1c823750c95a..7938221f4f7e 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -1847,7 +1847,8 @@ static int ublk_ctrl_start_dev(struct ublk_device *ub, struct io_uring_cmd *cmd) if (ublksrv_pid <= 0) return -EINVAL; - wait_for_completion_interruptible(&ub->completion); + if (wait_for_completion_interruptible(&ub->completion) != 0) + return -EINTR; schedule_delayed_work(&ub->monitor_work, UBLK_DAEMON_MONITOR_PERIOD); From 0c0cbd4ebc375ceebc75c89df04b74f215fab23a Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 26 Jul 2023 22:45:01 +0800 Subject: [PATCH 7/8] ublk: fail to recover device if queue setup is interrupted In ublk_ctrl_end_recovery(), if wait_for_completion_interruptible() is interrupted by signal, queues aren't setup successfully yet, so we have to fail UBLK_CMD_END_USER_RECOVERY, otherwise kernel oops can be triggered. Fixes: c732a852b419 ("ublk_drv: add START_USER_RECOVERY and END_USER_RECOVERY support") Reported-by: Stefano Garzarella Signed-off-by: Ming Lei Reviewed-by: Stefano Garzarella Link: https://lore.kernel.org/r/20230726144502.566785-3-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 7938221f4f7e..9fcba3834e8d 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2324,7 +2324,9 @@ static int ublk_ctrl_end_recovery(struct ublk_device *ub, pr_devel("%s: Waiting for new ubq_daemons(nr: %d) are ready, dev id %d...\n", __func__, ub->dev_info.nr_hw_queues, header->dev_id); /* wait until new ubq_daemon sending all FETCH_REQ */ - wait_for_completion_interruptible(&ub->completion); + if (wait_for_completion_interruptible(&ub->completion)) + return -EINTR; + pr_devel("%s: All new ubq_daemons(nr: %d) are ready, dev id %d\n", __func__, ub->dev_info.nr_hw_queues, header->dev_id); From 3e9dce80dbf91972aed972c743f539c396a34312 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Wed, 26 Jul 2023 22:45:02 +0800 Subject: [PATCH 8/8] ublk: return -EINTR if breaking from waiting for existed users in DEL_DEV If user interrupts wait_event_interruptible() in ublk_ctrl_del_dev(), return -EINTR and let user know what happens. Fixes: 0abe39dec065 ("block: ublk: improve handling device deletion") Reported-by: Stefano Garzarella Signed-off-by: Ming Lei Reviewed-by: Stefano Garzarella Link: https://lore.kernel.org/r/20230726144502.566785-4-ming.lei@redhat.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 9fcba3834e8d..21d2e71c5514 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -2126,8 +2126,8 @@ static int ublk_ctrl_del_dev(struct ublk_device **p_ub) * - the device number is freed already, we will not find this * device via ublk_get_device_from_id() */ - wait_event_interruptible(ublk_idr_wq, ublk_idr_freed(idx)); - + if (wait_event_interruptible(ublk_idr_wq, ublk_idr_freed(idx))) + return -EINTR; return 0; }