mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 19:29:37 +00:00
205 lines
4.6 KiB
C
205 lines
4.6 KiB
C
![]() |
#include <linux/kernel.h>
|
||
|
#include <linux/module.h>
|
||
|
#include <linux/percpu_ida.h>
|
||
|
|
||
|
#include <linux/blk-mq.h>
|
||
|
#include "blk.h"
|
||
|
#include "blk-mq.h"
|
||
|
#include "blk-mq-tag.h"
|
||
|
|
||
|
/*
|
||
|
* Per tagged queue (tag address space) map
|
||
|
*/
|
||
|
struct blk_mq_tags {
|
||
|
unsigned int nr_tags;
|
||
|
unsigned int nr_reserved_tags;
|
||
|
unsigned int nr_batch_move;
|
||
|
unsigned int nr_max_cache;
|
||
|
|
||
|
struct percpu_ida free_tags;
|
||
|
struct percpu_ida reserved_tags;
|
||
|
};
|
||
|
|
||
|
void blk_mq_wait_for_tags(struct blk_mq_tags *tags)
|
||
|
{
|
||
|
int tag = blk_mq_get_tag(tags, __GFP_WAIT, false);
|
||
|
blk_mq_put_tag(tags, tag);
|
||
|
}
|
||
|
|
||
|
bool blk_mq_has_free_tags(struct blk_mq_tags *tags)
|
||
|
{
|
||
|
return !tags ||
|
||
|
percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids) != 0;
|
||
|
}
|
||
|
|
||
|
static unsigned int __blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp)
|
||
|
{
|
||
|
int tag;
|
||
|
|
||
|
tag = percpu_ida_alloc(&tags->free_tags, gfp);
|
||
|
if (tag < 0)
|
||
|
return BLK_MQ_TAG_FAIL;
|
||
|
return tag + tags->nr_reserved_tags;
|
||
|
}
|
||
|
|
||
|
static unsigned int __blk_mq_get_reserved_tag(struct blk_mq_tags *tags,
|
||
|
gfp_t gfp)
|
||
|
{
|
||
|
int tag;
|
||
|
|
||
|
if (unlikely(!tags->nr_reserved_tags)) {
|
||
|
WARN_ON_ONCE(1);
|
||
|
return BLK_MQ_TAG_FAIL;
|
||
|
}
|
||
|
|
||
|
tag = percpu_ida_alloc(&tags->reserved_tags, gfp);
|
||
|
if (tag < 0)
|
||
|
return BLK_MQ_TAG_FAIL;
|
||
|
return tag;
|
||
|
}
|
||
|
|
||
|
unsigned int blk_mq_get_tag(struct blk_mq_tags *tags, gfp_t gfp, bool reserved)
|
||
|
{
|
||
|
if (!reserved)
|
||
|
return __blk_mq_get_tag(tags, gfp);
|
||
|
|
||
|
return __blk_mq_get_reserved_tag(tags, gfp);
|
||
|
}
|
||
|
|
||
|
static void __blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
|
||
|
{
|
||
|
BUG_ON(tag >= tags->nr_tags);
|
||
|
|
||
|
percpu_ida_free(&tags->free_tags, tag - tags->nr_reserved_tags);
|
||
|
}
|
||
|
|
||
|
static void __blk_mq_put_reserved_tag(struct blk_mq_tags *tags,
|
||
|
unsigned int tag)
|
||
|
{
|
||
|
BUG_ON(tag >= tags->nr_reserved_tags);
|
||
|
|
||
|
percpu_ida_free(&tags->reserved_tags, tag);
|
||
|
}
|
||
|
|
||
|
void blk_mq_put_tag(struct blk_mq_tags *tags, unsigned int tag)
|
||
|
{
|
||
|
if (tag >= tags->nr_reserved_tags)
|
||
|
__blk_mq_put_tag(tags, tag);
|
||
|
else
|
||
|
__blk_mq_put_reserved_tag(tags, tag);
|
||
|
}
|
||
|
|
||
|
static int __blk_mq_tag_iter(unsigned id, void *data)
|
||
|
{
|
||
|
unsigned long *tag_map = data;
|
||
|
__set_bit(id, tag_map);
|
||
|
return 0;
|
||
|
}
|
||
|
|
||
|
void blk_mq_tag_busy_iter(struct blk_mq_tags *tags,
|
||
|
void (*fn)(void *, unsigned long *), void *data)
|
||
|
{
|
||
|
unsigned long *tag_map;
|
||
|
size_t map_size;
|
||
|
|
||
|
map_size = ALIGN(tags->nr_tags, BITS_PER_LONG) / BITS_PER_LONG;
|
||
|
tag_map = kzalloc(map_size * sizeof(unsigned long), GFP_ATOMIC);
|
||
|
if (!tag_map)
|
||
|
return;
|
||
|
|
||
|
percpu_ida_for_each_free(&tags->free_tags, __blk_mq_tag_iter, tag_map);
|
||
|
if (tags->nr_reserved_tags)
|
||
|
percpu_ida_for_each_free(&tags->reserved_tags, __blk_mq_tag_iter,
|
||
|
tag_map);
|
||
|
|
||
|
fn(data, tag_map);
|
||
|
kfree(tag_map);
|
||
|
}
|
||
|
|
||
|
struct blk_mq_tags *blk_mq_init_tags(unsigned int total_tags,
|
||
|
unsigned int reserved_tags, int node)
|
||
|
{
|
||
|
unsigned int nr_tags, nr_cache;
|
||
|
struct blk_mq_tags *tags;
|
||
|
int ret;
|
||
|
|
||
|
if (total_tags > BLK_MQ_TAG_MAX) {
|
||
|
pr_err("blk-mq: tag depth too large\n");
|
||
|
return NULL;
|
||
|
}
|
||
|
|
||
|
tags = kzalloc_node(sizeof(*tags), GFP_KERNEL, node);
|
||
|
if (!tags)
|
||
|
return NULL;
|
||
|
|
||
|
nr_tags = total_tags - reserved_tags;
|
||
|
nr_cache = nr_tags / num_possible_cpus();
|
||
|
|
||
|
if (nr_cache < BLK_MQ_TAG_CACHE_MIN)
|
||
|
nr_cache = BLK_MQ_TAG_CACHE_MIN;
|
||
|
else if (nr_cache > BLK_MQ_TAG_CACHE_MAX)
|
||
|
nr_cache = BLK_MQ_TAG_CACHE_MAX;
|
||
|
|
||
|
tags->nr_tags = total_tags;
|
||
|
tags->nr_reserved_tags = reserved_tags;
|
||
|
tags->nr_max_cache = nr_cache;
|
||
|
tags->nr_batch_move = max(1u, nr_cache / 2);
|
||
|
|
||
|
ret = __percpu_ida_init(&tags->free_tags, tags->nr_tags -
|
||
|
tags->nr_reserved_tags,
|
||
|
tags->nr_max_cache,
|
||
|
tags->nr_batch_move);
|
||
|
if (ret)
|
||
|
goto err_free_tags;
|
||
|
|
||
|
if (reserved_tags) {
|
||
|
/*
|
||
|
* With max_cahe and batch set to 1, the allocator fallbacks to
|
||
|
* no cached. It's fine reserved tags allocation is slow.
|
||
|
*/
|
||
|
ret = __percpu_ida_init(&tags->reserved_tags, reserved_tags,
|
||
|
1, 1);
|
||
|
if (ret)
|
||
|
goto err_reserved_tags;
|
||
|
}
|
||
|
|
||
|
return tags;
|
||
|
|
||
|
err_reserved_tags:
|
||
|
percpu_ida_destroy(&tags->free_tags);
|
||
|
err_free_tags:
|
||
|
kfree(tags);
|
||
|
return NULL;
|
||
|
}
|
||
|
|
||
|
void blk_mq_free_tags(struct blk_mq_tags *tags)
|
||
|
{
|
||
|
percpu_ida_destroy(&tags->free_tags);
|
||
|
percpu_ida_destroy(&tags->reserved_tags);
|
||
|
kfree(tags);
|
||
|
}
|
||
|
|
||
|
ssize_t blk_mq_tag_sysfs_show(struct blk_mq_tags *tags, char *page)
|
||
|
{
|
||
|
char *orig_page = page;
|
||
|
int cpu;
|
||
|
|
||
|
if (!tags)
|
||
|
return 0;
|
||
|
|
||
|
page += sprintf(page, "nr_tags=%u, reserved_tags=%u, batch_move=%u,"
|
||
|
" max_cache=%u\n", tags->nr_tags, tags->nr_reserved_tags,
|
||
|
tags->nr_batch_move, tags->nr_max_cache);
|
||
|
|
||
|
page += sprintf(page, "nr_free=%u, nr_reserved=%u\n",
|
||
|
percpu_ida_free_tags(&tags->free_tags, nr_cpu_ids),
|
||
|
percpu_ida_free_tags(&tags->reserved_tags, nr_cpu_ids));
|
||
|
|
||
|
for_each_possible_cpu(cpu) {
|
||
|
page += sprintf(page, " cpu%02u: nr_free=%u\n", cpu,
|
||
|
percpu_ida_free_tags(&tags->free_tags, cpu));
|
||
|
}
|
||
|
|
||
|
return page - orig_page;
|
||
|
}
|