From 92cb6e2e5dbaea02c2fa317f3543c8918db25e89 Mon Sep 17 00:00:00 2001 From: ZiyangZhang Date: Wed, 10 Aug 2022 13:52:12 +0800 Subject: [PATCH 1/6] ublk_drv: update iod->addr for UBLK_IO_NEED_GET_DATA If ublksrv sends UBLK_IO_NEED_GET_DATA with new allocated io buffer, we have to update iod->addr in task_work before calling io_uring_cmd_done(). Then usersapce target can handle (write)io request with the new io buffer reading from updated iod. Without this change, userspace target may touch a wrong io buffer! Signed-off-by: ZiyangZhang Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20220810055212.66417-1-ZiyangZhang@linux.alibaba.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 2b7d1db5c4a7..5d8c7234639c 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -680,6 +680,11 @@ static inline void __ublk_rq_task_work(struct request *req) * do the copy work. */ io->flags &= ~UBLK_IO_FLAG_NEED_GET_DATA; + /* update iod->addr because ublksrv may have passed a new io buffer */ + ublk_get_iod(ubq, req->tag)->addr = io->addr; + pr_devel("%s: update iod->addr: op %d, qid %d tag %d io_flags %x addr %llx\n", + __func__, io->cmd->cmd_op, ubq->q_id, req->tag, io->flags, + ublk_get_iod(ubq, req->tag)->addr); } mapped_bytes = ublk_map_io(ubq, req, io); From 966120b51a245c9ff5857c5b169310c248e0ae87 Mon Sep 17 00:00:00 2001 From: ZiyangZhang Date: Mon, 15 Aug 2022 10:36:31 +0800 Subject: [PATCH 2/6] ublk_drv: check ubq_daemon_is_dying() in __ublk_rq_task_work() Replace direct check on PF_EXITING in __ublk_rq_task_work() by the existing wrapper. Also inline ubq_daemon_is_dying(). Reviewed-by: Ming Lei Signed-off-by: ZiyangZhang Link: https://lore.kernel.org/r/20220815023633.259825-2-ZiyangZhang@linux.alibaba.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 5d8c7234639c..17896172b0fe 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -555,7 +555,7 @@ static inline struct ublk_uring_cmd_pdu *ublk_get_uring_cmd_pdu( return (struct ublk_uring_cmd_pdu *)&ioucmd->pdu; } -static bool ubq_daemon_is_dying(struct ublk_queue *ubq) +static inline bool ubq_daemon_is_dying(struct ublk_queue *ubq) { return ubq->ubq_daemon->flags & PF_EXITING; } @@ -644,8 +644,7 @@ static inline void __ublk_rq_task_work(struct request *req) struct ublk_device *ub = ubq->dev; int tag = req->tag; struct ublk_io *io = &ubq->ios[tag]; - bool task_exiting = current != ubq->ubq_daemon || - (current->flags & PF_EXITING); + bool task_exiting = current != ubq->ubq_daemon || ubq_daemon_is_dying(ubq); unsigned int mapped_bytes; pr_devel("%s: complete: op %d, qid %d tag %d io_flags %x addr %llx\n", From bb24174754afc5a7d185ca5406dcfbc608cdf157 Mon Sep 17 00:00:00 2001 From: ZiyangZhang Date: Mon, 15 Aug 2022 10:36:32 +0800 Subject: [PATCH 3/6] ublk_drv: update comment for __ublk_fail_req() Since __ublk_rq_task_work always fails requests immediately during exiting, __ublk_fail_req() is only called from abort context during exiting. So lock is unnecessary. Signed-off-by: ZiyangZhang Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20220815023633.259825-3-ZiyangZhang@linux.alibaba.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 17896172b0fe..685a43b7ae6e 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -605,8 +605,9 @@ static void ublk_complete_rq(struct request *req) } /* - * __ublk_fail_req() may be called from abort context or ->ubq_daemon - * context during exiting, so lock is required. + * Since __ublk_rq_task_work always fails requests immediately during + * exiting, __ublk_fail_req() is only called from abort context during + * exiting. So lock is unnecessary. * * Also aborting may not be started yet, keep in mind that one failed * request may be issued by block layer again. From e6190dd0031d335c22586d34ef898301ed20f230 Mon Sep 17 00:00:00 2001 From: ZiyangZhang Date: Mon, 15 Aug 2022 10:36:33 +0800 Subject: [PATCH 4/6] ublk_drv: do not add a re-issued request aborted previously to ioucmd's task_work In ublk_queue_rq(), Assume current request is a re-issued request aborted previously in monitor_work because the ubq_daemon(ioucmd's task) is PF_EXITING. For this request, we cannot call io_uring_cmd_complete_in_task() anymore because at that moment io_uring context may be freed in case that no inflight ioucmd exists. Otherwise, we may cause null-deref in ctx->fallback_work. Add a check on UBLK_IO_FLAG_ABORTED to prevent the above situation. This check is safe and makes sense. Note: monitor_work sets UBLK_IO_FLAG_ABORTED and ends this request (releasing the tag). Then the request is restarted(allocating the tag) and we are here. Since releasing/allocating a tag implies smp_mb(), finding UBLK_IO_FLAG_ABORTED guarantees that here is a re-issued request aborted previously. Suggested-by: Ming Lei Signed-off-by: ZiyangZhang Reviewed-by: Ming Lei Link: https://lore.kernel.org/r/20220815023633.259825-4-ZiyangZhang@linux.alibaba.com Signed-off-by: Jens Axboe --- drivers/block/ublk_drv.c | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/drivers/block/ublk_drv.c b/drivers/block/ublk_drv.c index 685a43b7ae6e..6a4a94b4cdf4 100644 --- a/drivers/block/ublk_drv.c +++ b/drivers/block/ublk_drv.c @@ -756,9 +756,25 @@ static blk_status_t ublk_queue_rq(struct blk_mq_hw_ctx *hctx, if (task_work_add(ubq->ubq_daemon, &data->work, notify_mode)) goto fail; } else { - struct io_uring_cmd *cmd = ubq->ios[rq->tag].cmd; + struct ublk_io *io = &ubq->ios[rq->tag]; + struct io_uring_cmd *cmd = io->cmd; struct ublk_uring_cmd_pdu *pdu = ublk_get_uring_cmd_pdu(cmd); + /* + * If the check pass, we know that this is a re-issued request aborted + * previously in monitor_work because the ubq_daemon(cmd's task) is + * PF_EXITING. We cannot call io_uring_cmd_complete_in_task() anymore + * because this ioucmd's io_uring context may be freed now if no inflight + * ioucmd exists. Otherwise we may cause null-deref in ctx->fallback_work. + * + * Note: monitor_work sets UBLK_IO_FLAG_ABORTED and ends this request(releasing + * the tag). Then the request is re-started(allocating the tag) and we are here. + * Since releasing/allocating a tag implies smp_mb(), finding UBLK_IO_FLAG_ABORTED + * guarantees that here is a re-issued request aborted previously. + */ + if ((io->flags & UBLK_IO_FLAG_ABORTED)) + goto fail; + pdu->req = rq; io_uring_cmd_complete_in_task(cmd, ublk_rq_task_work_cb); } From a8239f0342bae5a51acca967ba95b9a8ad56dd62 Mon Sep 17 00:00:00 2001 From: Yu Kuai Date: Thu, 18 Aug 2022 14:35:55 +0800 Subject: [PATCH 5/6] blk-mq: remove unused function blk_mq_queue_stopped() blk_mq_queue_stopped() doesn't have any caller, which was found by code coverage test, thus remove it. Signed-off-by: Yu Kuai Link: https://lore.kernel.org/r/20220818063555.3741222-1-yukuai1@huaweicloud.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 20 -------------------- include/linux/blk-mq.h | 1 - 2 files changed, 21 deletions(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 5ee62b95f3e5..5568c7d09114 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2229,26 +2229,6 @@ void blk_mq_delay_run_hw_queues(struct request_queue *q, unsigned long msecs) } EXPORT_SYMBOL(blk_mq_delay_run_hw_queues); -/** - * blk_mq_queue_stopped() - check whether one or more hctxs have been stopped - * @q: request queue. - * - * The caller is responsible for serializing this function against - * blk_mq_{start,stop}_hw_queue(). - */ -bool blk_mq_queue_stopped(struct request_queue *q) -{ - struct blk_mq_hw_ctx *hctx; - unsigned long i; - - queue_for_each_hw_ctx(q, hctx, i) - if (blk_mq_hctx_stopped(hctx)) - return true; - - return false; -} -EXPORT_SYMBOL(blk_mq_queue_stopped); - /* * This function is often used for pausing .queue_rq() by driver when * there isn't enough resource or some conditions aren't satisfied, and diff --git a/include/linux/blk-mq.h b/include/linux/blk-mq.h index effee1dc715a..92294a5fb083 100644 --- a/include/linux/blk-mq.h +++ b/include/linux/blk-mq.h @@ -857,7 +857,6 @@ void blk_mq_kick_requeue_list(struct request_queue *q); void blk_mq_delay_kick_requeue_list(struct request_queue *q, unsigned long msecs); void blk_mq_complete_request(struct request *rq); bool blk_mq_complete_request_remote(struct request *rq); -bool blk_mq_queue_stopped(struct request_queue *q); void blk_mq_stop_hw_queue(struct blk_mq_hw_ctx *hctx); void blk_mq_start_hw_queue(struct blk_mq_hw_ctx *hctx); void blk_mq_stop_hw_queues(struct request_queue *q); From d3b38596875dbc709b4e721a5873f4663d8a9ea2 Mon Sep 17 00:00:00 2001 From: Yufen Yu Date: Wed, 3 Aug 2022 10:33:55 +0800 Subject: [PATCH 6/6] blk-mq: run queue no matter whether the request is the last request We do test on a virtio scsi device (/dev/sda) and the default mq scheduler is 'none'. We found a IO hung as following: blk_finish_plug blk_mq_plug_issue_direct scsi_mq_get_budget //get budget_token fail and sdev->restarts=1 scsi_end_request scsi_run_queue_async //sdev->restart=0 and run queue blk_mq_request_bypass_insert //add request to hctx->dispatch list //continue to dispath plug list blk_mq_dispatch_plug_list blk_mq_try_issue_list_directly //success issue all requests from plug list After .get_budget fail, scsi_mq_get_budget will increase 'restarts'. Normally, it will run hw queue when io complete and set 'restarts' as 0. But if we run queue before adding request to the dispatch list and blk_mq_dispatch_plug_list also success issue all requests, then on one will run queue, and the request will be stall in the dispatch list and cannot complete forever. It is wrong to use last request of plug list to decide if run queue is needed since all the remained requests in plug list may be from other hctxs. To fix the bug, pass run_queue as true always to blk_mq_request_bypass_insert(). Fix-suggested-by: Ming Lei Signed-off-by: Yufen Yu Reviewed-by: Ming Lei Fixes: dc5fc361d891 ("block: attempt direct issue of plug list") Link: https://lore.kernel.org/r/20220803023355.3687360-1-yuyufen@huaweicloud.com Signed-off-by: Jens Axboe --- block/blk-mq.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/block/blk-mq.c b/block/blk-mq.c index 5568c7d09114..3c1e6b6d991d 100644 --- a/block/blk-mq.c +++ b/block/blk-mq.c @@ -2550,7 +2550,7 @@ static void blk_mq_plug_issue_direct(struct blk_plug *plug, bool from_schedule) break; case BLK_STS_RESOURCE: case BLK_STS_DEV_RESOURCE: - blk_mq_request_bypass_insert(rq, false, last); + blk_mq_request_bypass_insert(rq, false, true); blk_mq_commit_rqs(hctx, &queued, from_schedule); return; default: