mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-22 17:01:14 +00:00
block: add queue flag for disabling SG merging
If devices are not SG starved, we waste a lot of time potentially collapsing SG segments. Enough that 1.5% of the CPU time goes to this, at only 400K IOPS. Add a queue flag, QUEUE_FLAG_NO_SG_MERGE, which just returns the number of vectors in a bio instead of looping over all segments and checking for collapsible ones. Add a BLK_MQ_F_SG_MERGE flag so that drivers can opt-in on the sg merging, if they so desire. Signed-off-by: Jens Axboe <axboe@fb.com>
This commit is contained in:
parent
4d92a9beb3
commit
05f1dd5315
4 changed files with 26 additions and 7 deletions
|
@ -13,7 +13,7 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
|
|||
struct bio *bio)
|
||||
{
|
||||
struct bio_vec bv, bvprv = { NULL };
|
||||
int cluster, high, highprv = 1;
|
||||
int cluster, high, highprv = 1, no_sg_merge;
|
||||
unsigned int seg_size, nr_phys_segs;
|
||||
struct bio *fbio, *bbio;
|
||||
struct bvec_iter iter;
|
||||
|
@ -35,12 +35,21 @@ static unsigned int __blk_recalc_rq_segments(struct request_queue *q,
|
|||
cluster = blk_queue_cluster(q);
|
||||
seg_size = 0;
|
||||
nr_phys_segs = 0;
|
||||
no_sg_merge = test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
|
||||
high = 0;
|
||||
for_each_bio(bio) {
|
||||
bio_for_each_segment(bv, bio, iter) {
|
||||
/*
|
||||
* If SG merging is disabled, each bio vector is
|
||||
* a segment
|
||||
*/
|
||||
if (no_sg_merge)
|
||||
goto new_segment;
|
||||
|
||||
/*
|
||||
* the trick here is making sure that a high page is
|
||||
* never considered part of another segment, since that
|
||||
* might change with the bounce page.
|
||||
* never considered part of another segment, since
|
||||
* that might change with the bounce page.
|
||||
*/
|
||||
high = page_to_pfn(bv.bv_page) > queue_bounce_pfn(q);
|
||||
if (!high && !highprv && cluster) {
|
||||
|
@ -84,11 +93,16 @@ void blk_recalc_rq_segments(struct request *rq)
|
|||
|
||||
void blk_recount_segments(struct request_queue *q, struct bio *bio)
|
||||
{
|
||||
struct bio *nxt = bio->bi_next;
|
||||
if (test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags))
|
||||
bio->bi_phys_segments = bio->bi_vcnt;
|
||||
else {
|
||||
struct bio *nxt = bio->bi_next;
|
||||
|
||||
bio->bi_next = NULL;
|
||||
bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
|
||||
bio->bi_next = nxt;
|
||||
}
|
||||
|
||||
bio->bi_next = NULL;
|
||||
bio->bi_phys_segments = __blk_recalc_rq_segments(q, bio);
|
||||
bio->bi_next = nxt;
|
||||
bio->bi_flags |= (1 << BIO_SEG_VALID);
|
||||
}
|
||||
EXPORT_SYMBOL(blk_recount_segments);
|
||||
|
|
|
@ -1829,6 +1829,9 @@ struct request_queue *blk_mq_init_queue(struct blk_mq_tag_set *set)
|
|||
q->mq_ops = set->ops;
|
||||
q->queue_flags |= QUEUE_FLAG_MQ_DEFAULT;
|
||||
|
||||
if (!(set->flags & BLK_MQ_F_SG_MERGE))
|
||||
q->queue_flags |= 1 << QUEUE_FLAG_NO_SG_MERGE;
|
||||
|
||||
q->sg_reserved_size = INT_MAX;
|
||||
|
||||
INIT_WORK(&q->requeue_work, blk_mq_requeue_work);
|
||||
|
|
|
@ -129,6 +129,7 @@ enum {
|
|||
BLK_MQ_F_SHOULD_MERGE = 1 << 0,
|
||||
BLK_MQ_F_SHOULD_SORT = 1 << 1,
|
||||
BLK_MQ_F_TAG_SHARED = 1 << 2,
|
||||
BLK_MQ_F_SG_MERGE = 1 << 3,
|
||||
|
||||
BLK_MQ_S_STOPPED = 0,
|
||||
BLK_MQ_S_TAG_ACTIVE = 1,
|
||||
|
|
|
@ -510,6 +510,7 @@ struct request_queue {
|
|||
#define QUEUE_FLAG_SAME_FORCE 18 /* force complete on same CPU */
|
||||
#define QUEUE_FLAG_DEAD 19 /* queue tear-down finished */
|
||||
#define QUEUE_FLAG_INIT_DONE 20 /* queue is initialized */
|
||||
#define QUEUE_FLAG_NO_SG_MERGE 21 /* don't attempt to merge SG segments*/
|
||||
|
||||
#define QUEUE_FLAG_DEFAULT ((1 << QUEUE_FLAG_IO_STAT) | \
|
||||
(1 << QUEUE_FLAG_STACKABLE) | \
|
||||
|
|
Loading…
Reference in a new issue