block: move update request helpers into blk-mq.c
For some reason we still have them in blk-core, with the rest of the request completion being in blk-mq. That causes and out-of-line call for each completion. Move them into blk-mq.c instead, where they belong. Reviewed-by: Christoph Hellwig <hch@lst.de> Signed-off-by: Jens Axboe <axboe@kernel.dk>
This commit is contained in:
Jens Axboe
parent
c477b79778
commit
9be3e06fb7
146
block/blk-core.c
146
block/blk-core.c
|
|
@ -216,7 +216,7 @@ int blk_status_to_errno(blk_status_t status)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(blk_status_to_errno);
|
||||
|
||||
static void print_req_error(struct request *req, blk_status_t status)
|
||||
void blk_print_req_error(struct request *req, blk_status_t status)
|
||||
{
|
||||
int idx = (__force int)status;
|
||||
|
||||
|
|
@ -234,33 +234,6 @@ static void print_req_error(struct request *req, blk_status_t status)
|
|||
IOPRIO_PRIO_CLASS(req->ioprio));
|
||||
}
|
||||
|
||||
static void req_bio_endio(struct request *rq, struct bio *bio,
|
||||
unsigned int nbytes, blk_status_t error)
|
||||
{
|
||||
if (error)
|
||||
bio->bi_status = error;
|
||||
|
||||
if (unlikely(rq->rq_flags & RQF_QUIET))
|
||||
bio_set_flag(bio, BIO_QUIET);
|
||||
|
||||
bio_advance(bio, nbytes);
|
||||
|
||||
if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
|
||||
/*
|
||||
* Partial zone append completions cannot be supported as the
|
||||
* BIO fragments may end up not being written sequentially.
|
||||
*/
|
||||
if (bio->bi_iter.bi_size)
|
||||
bio->bi_status = BLK_STS_IOERR;
|
||||
else
|
||||
bio->bi_iter.bi_sector = rq->__sector;
|
||||
}
|
||||
|
||||
/* don't actually finish bio if it's part of flush sequence */
|
||||
if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
|
||||
bio_endio(bio);
|
||||
}
|
||||
|
||||
void blk_dump_rq_flags(struct request *rq, char *msg)
|
||||
{
|
||||
printk(KERN_INFO "%s: dev %s: flags=%llx\n", msg,
|
||||
|
|
@ -1311,17 +1284,6 @@ again:
|
|||
}
|
||||
}
|
||||
|
||||
static void blk_account_io_completion(struct request *req, unsigned int bytes)
|
||||
{
|
||||
if (req->part && blk_do_io_stat(req)) {
|
||||
const int sgrp = op_stat_group(req_op(req));
|
||||
|
||||
part_stat_lock();
|
||||
part_stat_add(req->part, sectors[sgrp], bytes >> 9);
|
||||
part_stat_unlock();
|
||||
}
|
||||
}
|
||||
|
||||
void __blk_account_io_done(struct request *req, u64 now)
|
||||
{
|
||||
const int sgrp = op_stat_group(req_op(req));
|
||||
|
|
@ -1430,112 +1392,6 @@ void blk_steal_bios(struct bio_list *list, struct request *rq)
|
|||
}
|
||||
EXPORT_SYMBOL_GPL(blk_steal_bios);
|
||||
|
||||
/**
|
||||
* blk_update_request - Complete multiple bytes without completing the request
|
||||
* @req: the request being processed
|
||||
* @error: block status code
|
||||
* @nr_bytes: number of bytes to complete for @req
|
||||
*
|
||||
* Description:
|
||||
* Ends I/O on a number of bytes attached to @req, but doesn't complete
|
||||
* the request structure even if @req doesn't have leftover.
|
||||
* If @req has leftover, sets it up for the next range of segments.
|
||||
*
|
||||
* Passing the result of blk_rq_bytes() as @nr_bytes guarantees
|
||||
* %false return from this function.
|
||||
*
|
||||
* Note:
|
||||
* The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
|
||||
* except in the consistency check at the end of this function.
|
||||
*
|
||||
* Return:
|
||||
* %false - this request doesn't have any more data
|
||||
* %true - this request has more data
|
||||
**/
|
||||
bool blk_update_request(struct request *req, blk_status_t error,
|
||||
unsigned int nr_bytes)
|
||||
{
|
||||
int total_bytes;
|
||||
|
||||
trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
|
||||
|
||||
if (!req->bio)
|
||||
return false;
|
||||
|
||||
#ifdef CONFIG_BLK_DEV_INTEGRITY
|
||||
if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
|
||||
error == BLK_STS_OK)
|
||||
req->q->integrity.profile->complete_fn(req, nr_bytes);
|
||||
#endif
|
||||
|
||||
if (unlikely(error && !blk_rq_is_passthrough(req) &&
|
||||
!(req->rq_flags & RQF_QUIET)))
|
||||
print_req_error(req, error);
|
||||
|
||||
blk_account_io_completion(req, nr_bytes);
|
||||
|
||||
total_bytes = 0;
|
||||
while (req->bio) {
|
||||
struct bio *bio = req->bio;
|
||||
unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
|
||||
|
||||
if (bio_bytes == bio->bi_iter.bi_size)
|
||||
req->bio = bio->bi_next;
|
||||
|
||||
/* Completion has already been traced */
|
||||
bio_clear_flag(bio, BIO_TRACE_COMPLETION);
|
||||
req_bio_endio(req, bio, bio_bytes, error);
|
||||
|
||||
total_bytes += bio_bytes;
|
||||
nr_bytes -= bio_bytes;
|
||||
|
||||
if (!nr_bytes)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* completely done
|
||||
*/
|
||||
if (!req->bio) {
|
||||
/*
|
||||
* Reset counters so that the request stacking driver
|
||||
* can find how many bytes remain in the request
|
||||
* later.
|
||||
*/
|
||||
req->__data_len = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
req->__data_len -= total_bytes;
|
||||
|
||||
/* update sector only for requests with clear definition of sector */
|
||||
if (!blk_rq_is_passthrough(req))
|
||||
req->__sector += total_bytes >> 9;
|
||||
|
||||
/* mixed attributes always follow the first bio */
|
||||
if (req->rq_flags & RQF_MIXED_MERGE) {
|
||||
req->cmd_flags &= ~REQ_FAILFAST_MASK;
|
||||
req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
|
||||
}
|
||||
|
||||
if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
|
||||
/*
|
||||
* If total number of sectors is less than the first segment
|
||||
* size, something has gone terribly wrong.
|
||||
*/
|
||||
if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
|
||||
blk_dump_rq_flags(req, "request botched");
|
||||
req->__data_len = blk_rq_cur_bytes(req);
|
||||
}
|
||||
|
||||
/* recalculate the number of segments */
|
||||
req->nr_phys_segments = blk_recalc_rq_segments(req);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blk_update_request);
|
||||
|
||||
#if ARCH_IMPLEMENTS_FLUSH_DCACHE_PAGE
|
||||
/**
|
||||
* rq_flush_dcache_pages - Helper function to flush all pages in a request
|
||||
|
|
|
|||
144
block/blk-mq.c
144
block/blk-mq.c
|
|
@ -628,6 +628,150 @@ void blk_mq_free_plug_rqs(struct blk_plug *plug)
|
|||
}
|
||||
}
|
||||
|
||||
static void req_bio_endio(struct request *rq, struct bio *bio,
|
||||
unsigned int nbytes, blk_status_t error)
|
||||
{
|
||||
if (error)
|
||||
bio->bi_status = error;
|
||||
|
||||
if (unlikely(rq->rq_flags & RQF_QUIET))
|
||||
bio_set_flag(bio, BIO_QUIET);
|
||||
|
||||
bio_advance(bio, nbytes);
|
||||
|
||||
if (req_op(rq) == REQ_OP_ZONE_APPEND && error == BLK_STS_OK) {
|
||||
/*
|
||||
* Partial zone append completions cannot be supported as the
|
||||
* BIO fragments may end up not being written sequentially.
|
||||
*/
|
||||
if (bio->bi_iter.bi_size)
|
||||
bio->bi_status = BLK_STS_IOERR;
|
||||
else
|
||||
bio->bi_iter.bi_sector = rq->__sector;
|
||||
}
|
||||
|
||||
/* don't actually finish bio if it's part of flush sequence */
|
||||
if (bio->bi_iter.bi_size == 0 && !(rq->rq_flags & RQF_FLUSH_SEQ))
|
||||
bio_endio(bio);
|
||||
}
|
||||
|
||||
static void blk_account_io_completion(struct request *req, unsigned int bytes)
|
||||
{
|
||||
if (req->part && blk_do_io_stat(req)) {
|
||||
const int sgrp = op_stat_group(req_op(req));
|
||||
|
||||
part_stat_lock();
|
||||
part_stat_add(req->part, sectors[sgrp], bytes >> 9);
|
||||
part_stat_unlock();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* blk_update_request - Complete multiple bytes without completing the request
|
||||
* @req: the request being processed
|
||||
* @error: block status code
|
||||
* @nr_bytes: number of bytes to complete for @req
|
||||
*
|
||||
* Description:
|
||||
* Ends I/O on a number of bytes attached to @req, but doesn't complete
|
||||
* the request structure even if @req doesn't have leftover.
|
||||
* If @req has leftover, sets it up for the next range of segments.
|
||||
*
|
||||
* Passing the result of blk_rq_bytes() as @nr_bytes guarantees
|
||||
* %false return from this function.
|
||||
*
|
||||
* Note:
|
||||
* The RQF_SPECIAL_PAYLOAD flag is ignored on purpose in this function
|
||||
* except in the consistency check at the end of this function.
|
||||
*
|
||||
* Return:
|
||||
* %false - this request doesn't have any more data
|
||||
* %true - this request has more data
|
||||
**/
|
||||
bool blk_update_request(struct request *req, blk_status_t error,
|
||||
unsigned int nr_bytes)
|
||||
{
|
||||
int total_bytes;
|
||||
|
||||
trace_block_rq_complete(req, blk_status_to_errno(error), nr_bytes);
|
||||
|
||||
if (!req->bio)
|
||||
return false;
|
||||
|
||||
#ifdef CONFIG_BLK_DEV_INTEGRITY
|
||||
if (blk_integrity_rq(req) && req_op(req) == REQ_OP_READ &&
|
||||
error == BLK_STS_OK)
|
||||
req->q->integrity.profile->complete_fn(req, nr_bytes);
|
||||
#endif
|
||||
|
||||
if (unlikely(error && !blk_rq_is_passthrough(req) &&
|
||||
!(req->rq_flags & RQF_QUIET)))
|
||||
blk_print_req_error(req, error);
|
||||
|
||||
blk_account_io_completion(req, nr_bytes);
|
||||
|
||||
total_bytes = 0;
|
||||
while (req->bio) {
|
||||
struct bio *bio = req->bio;
|
||||
unsigned bio_bytes = min(bio->bi_iter.bi_size, nr_bytes);
|
||||
|
||||
if (bio_bytes == bio->bi_iter.bi_size)
|
||||
req->bio = bio->bi_next;
|
||||
|
||||
/* Completion has already been traced */
|
||||
bio_clear_flag(bio, BIO_TRACE_COMPLETION);
|
||||
req_bio_endio(req, bio, bio_bytes, error);
|
||||
|
||||
total_bytes += bio_bytes;
|
||||
nr_bytes -= bio_bytes;
|
||||
|
||||
if (!nr_bytes)
|
||||
break;
|
||||
}
|
||||
|
||||
/*
|
||||
* completely done
|
||||
*/
|
||||
if (!req->bio) {
|
||||
/*
|
||||
* Reset counters so that the request stacking driver
|
||||
* can find how many bytes remain in the request
|
||||
* later.
|
||||
*/
|
||||
req->__data_len = 0;
|
||||
return false;
|
||||
}
|
||||
|
||||
req->__data_len -= total_bytes;
|
||||
|
||||
/* update sector only for requests with clear definition of sector */
|
||||
if (!blk_rq_is_passthrough(req))
|
||||
req->__sector += total_bytes >> 9;
|
||||
|
||||
/* mixed attributes always follow the first bio */
|
||||
if (req->rq_flags & RQF_MIXED_MERGE) {
|
||||
req->cmd_flags &= ~REQ_FAILFAST_MASK;
|
||||
req->cmd_flags |= req->bio->bi_opf & REQ_FAILFAST_MASK;
|
||||
}
|
||||
|
||||
if (!(req->rq_flags & RQF_SPECIAL_PAYLOAD)) {
|
||||
/*
|
||||
* If total number of sectors is less than the first segment
|
||||
* size, something has gone terribly wrong.
|
||||
*/
|
||||
if (blk_rq_bytes(req) < blk_rq_cur_bytes(req)) {
|
||||
blk_dump_rq_flags(req, "request botched");
|
||||
req->__data_len = blk_rq_cur_bytes(req);
|
||||
}
|
||||
|
||||
/* recalculate the number of segments */
|
||||
req->nr_phys_segments = blk_recalc_rq_segments(req);
|
||||
}
|
||||
|
||||
return true;
|
||||
}
|
||||
EXPORT_SYMBOL_GPL(blk_update_request);
|
||||
|
||||
inline void __blk_mq_end_request(struct request *rq, blk_status_t error)
|
||||
{
|
||||
if (blk_mq_need_time_stamp(rq)) {
|
||||
|
|
|
|||
|
|
@ -215,6 +215,7 @@ static inline void blk_integrity_del(struct gendisk *disk)
|
|||
|
||||
unsigned long blk_rq_timeout(unsigned long timeout);
|
||||
void blk_add_timer(struct request *req);
|
||||
void blk_print_req_error(struct request *req, blk_status_t status);
|
||||
|
||||
bool blk_attempt_plug_merge(struct request_queue *q, struct bio *bio,
|
||||
unsigned int nr_segs, struct request **same_queue_rq);
|
||||
|
|
|
|||
Loading…
Reference in New Issue