mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-27 11:19:37 +00:00
net/mlx5: Add flow counter pool
Add a pool of flow counters, based on flow counter bulks, removing the need to allocate a new counter via a costly FW command during the flow creation process. The time it takes to acquire/release a flow counter is cut from ~50 [us] to ~50 [ns]. The pool is part of the mlx5 driver instance, and provides flow counters for aging flows. mlx5_fc_create() was modified to provide counters for aging flows from the pool by default, and mlx5_destroy_fc() was modified to release counters back to the pool for later reuse. If bulk allocation is not supported or fails, and for non-aging flows, the fallback behavior is to allocate and free individual counters. The pool is comprised of three lists of flow counter bulks, one of fully used bulks, one of partially used bulks, and one of unused bulks. Counters are provided from the partially used bulks first, to help limit bulk fragmentation. The pool maintains a threshold, and strives to maintain the amount of available counters below it. The pool is increased in size when a counter acquisition request is made and there are no available counters, and it is decreased in size when the last counter in a bulk is released and there are more available counters than the threshold. All pool size changes are done in the context of the acquiring/releasing process. The value of the threshold is directly correlated to the amount of used counters the pool is providing, while constrained by a hard maximum, and is recalculated every time a bulk is allocated/freed. This ensures that the pool only consumes large amounts of memory for available counters if the pool is being used heavily. When fully populated and at the hard maximum, the buffer of available counters consumes ~40 [MB]. Signed-off-by: Gavi Teitz <gavi@mellanox.com> Reviewed-by: Vlad Buslov <vladbu@mellanox.com> Signed-off-by: Saeed Mahameed <saeedm@mellanox.com>
This commit is contained in:
parent
5d8a025365
commit
558101f1b9
2 changed files with 218 additions and 25 deletions
|
@ -40,6 +40,8 @@
|
|||
#define MLX5_FC_STATS_PERIOD msecs_to_jiffies(1000)
|
||||
/* Max number of counters to query in bulk read is 32K */
|
||||
#define MLX5_SW_MAX_COUNTERS_BULK BIT(15)
|
||||
#define MLX5_FC_POOL_MAX_THRESHOLD BIT(18)
|
||||
#define MLX5_FC_POOL_USED_BUFF_RATIO 10
|
||||
|
||||
struct mlx5_fc_cache {
|
||||
u64 packets;
|
||||
|
@ -65,6 +67,11 @@ struct mlx5_fc {
|
|||
struct mlx5_fc_cache cache ____cacheline_aligned_in_smp;
|
||||
};
|
||||
|
||||
static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev);
|
||||
static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool);
|
||||
static struct mlx5_fc *mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool);
|
||||
static void mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc);
|
||||
|
||||
/* locking scheme:
|
||||
*
|
||||
* It is the responsibility of the user to prevent concurrent calls or bad
|
||||
|
@ -202,13 +209,22 @@ static void mlx5_fc_stats_query_counter_range(struct mlx5_core_dev *dev,
|
|||
}
|
||||
}
|
||||
|
||||
static void mlx5_free_fc(struct mlx5_core_dev *dev,
|
||||
struct mlx5_fc *counter)
|
||||
static void mlx5_fc_free(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
|
||||
{
|
||||
mlx5_cmd_fc_free(dev, counter->id);
|
||||
kfree(counter);
|
||||
}
|
||||
|
||||
static void mlx5_fc_release(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
|
||||
{
|
||||
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
|
||||
|
||||
if (counter->bulk)
|
||||
mlx5_fc_pool_release_counter(&fc_stats->fc_pool, counter);
|
||||
else
|
||||
mlx5_fc_free(dev, counter);
|
||||
}
|
||||
|
||||
static void mlx5_fc_stats_work(struct work_struct *work)
|
||||
{
|
||||
struct mlx5_core_dev *dev = container_of(work, struct mlx5_core_dev,
|
||||
|
@ -232,7 +248,7 @@ static void mlx5_fc_stats_work(struct work_struct *work)
|
|||
llist_for_each_entry_safe(counter, tmp, dellist, dellist) {
|
||||
mlx5_fc_stats_remove(dev, counter);
|
||||
|
||||
mlx5_free_fc(dev, counter);
|
||||
mlx5_fc_release(dev, counter);
|
||||
}
|
||||
|
||||
if (time_before(now, fc_stats->next_query) ||
|
||||
|
@ -248,26 +264,56 @@ static void mlx5_fc_stats_work(struct work_struct *work)
|
|||
fc_stats->next_query = now + fc_stats->sampling_interval;
|
||||
}
|
||||
|
||||
struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging)
|
||||
static struct mlx5_fc *mlx5_fc_single_alloc(struct mlx5_core_dev *dev)
|
||||
{
|
||||
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
|
||||
struct mlx5_fc *counter;
|
||||
int err;
|
||||
|
||||
counter = kzalloc(sizeof(*counter), GFP_KERNEL);
|
||||
if (!counter)
|
||||
return ERR_PTR(-ENOMEM);
|
||||
INIT_LIST_HEAD(&counter->list);
|
||||
|
||||
err = mlx5_cmd_fc_alloc(dev, &counter->id);
|
||||
if (err)
|
||||
goto err_out;
|
||||
if (err) {
|
||||
kfree(counter);
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
|
||||
return counter;
|
||||
}
|
||||
|
||||
static struct mlx5_fc *mlx5_fc_acquire(struct mlx5_core_dev *dev, bool aging)
|
||||
{
|
||||
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
|
||||
struct mlx5_fc *counter;
|
||||
|
||||
if (aging && MLX5_CAP_GEN(dev, flow_counter_bulk_alloc) != 0) {
|
||||
counter = mlx5_fc_pool_acquire_counter(&fc_stats->fc_pool);
|
||||
if (!IS_ERR(counter))
|
||||
return counter;
|
||||
}
|
||||
|
||||
return mlx5_fc_single_alloc(dev);
|
||||
}
|
||||
|
||||
struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging)
|
||||
{
|
||||
struct mlx5_fc *counter = mlx5_fc_acquire(dev, aging);
|
||||
struct mlx5_fc_stats *fc_stats = &dev->priv.fc_stats;
|
||||
int err;
|
||||
|
||||
if (IS_ERR(counter))
|
||||
return counter;
|
||||
|
||||
INIT_LIST_HEAD(&counter->list);
|
||||
counter->aging = aging;
|
||||
|
||||
if (aging) {
|
||||
u32 id = counter->id;
|
||||
|
||||
counter->cache.lastuse = jiffies;
|
||||
counter->aging = true;
|
||||
counter->lastbytes = counter->cache.bytes;
|
||||
counter->lastpackets = counter->cache.packets;
|
||||
|
||||
idr_preload(GFP_KERNEL);
|
||||
spin_lock(&fc_stats->counters_idr_lock);
|
||||
|
@ -288,10 +334,7 @@ struct mlx5_fc *mlx5_fc_create(struct mlx5_core_dev *dev, bool aging)
|
|||
return counter;
|
||||
|
||||
err_out_alloc:
|
||||
mlx5_cmd_fc_free(dev, counter->id);
|
||||
err_out:
|
||||
kfree(counter);
|
||||
|
||||
mlx5_fc_release(dev, counter);
|
||||
return ERR_PTR(err);
|
||||
}
|
||||
EXPORT_SYMBOL(mlx5_fc_create);
|
||||
|
@ -315,7 +358,7 @@ void mlx5_fc_destroy(struct mlx5_core_dev *dev, struct mlx5_fc *counter)
|
|||
return;
|
||||
}
|
||||
|
||||
mlx5_free_fc(dev, counter);
|
||||
mlx5_fc_release(dev, counter);
|
||||
}
|
||||
EXPORT_SYMBOL(mlx5_fc_destroy);
|
||||
|
||||
|
@ -344,6 +387,7 @@ int mlx5_init_fc_stats(struct mlx5_core_dev *dev)
|
|||
fc_stats->sampling_interval = MLX5_FC_STATS_PERIOD;
|
||||
INIT_DELAYED_WORK(&fc_stats->work, mlx5_fc_stats_work);
|
||||
|
||||
mlx5_fc_pool_init(&fc_stats->fc_pool, dev);
|
||||
return 0;
|
||||
|
||||
err_wq_create:
|
||||
|
@ -358,6 +402,7 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
|
|||
struct mlx5_fc *counter;
|
||||
struct mlx5_fc *tmp;
|
||||
|
||||
mlx5_fc_pool_cleanup(&fc_stats->fc_pool);
|
||||
cancel_delayed_work_sync(&dev->priv.fc_stats.work);
|
||||
destroy_workqueue(dev->priv.fc_stats.wq);
|
||||
dev->priv.fc_stats.wq = NULL;
|
||||
|
@ -368,10 +413,10 @@ void mlx5_cleanup_fc_stats(struct mlx5_core_dev *dev)
|
|||
|
||||
tmplist = llist_del_all(&fc_stats->addlist);
|
||||
llist_for_each_entry_safe(counter, tmp, tmplist, addlist)
|
||||
mlx5_free_fc(dev, counter);
|
||||
mlx5_fc_release(dev, counter);
|
||||
|
||||
list_for_each_entry_safe(counter, tmp, &fc_stats->counters, list)
|
||||
mlx5_free_fc(dev, counter);
|
||||
mlx5_fc_release(dev, counter);
|
||||
}
|
||||
|
||||
int mlx5_fc_query(struct mlx5_core_dev *dev, struct mlx5_fc *counter,
|
||||
|
@ -417,14 +462,15 @@ void mlx5_fc_update_sampling_interval(struct mlx5_core_dev *dev,
|
|||
/* Flow counter bluks */
|
||||
|
||||
struct mlx5_fc_bulk {
|
||||
struct list_head pool_list;
|
||||
u32 base_id;
|
||||
int bulk_len;
|
||||
unsigned long *bitmask;
|
||||
struct mlx5_fc fcs[0];
|
||||
};
|
||||
|
||||
static void
|
||||
mlx5_fc_init(struct mlx5_fc *counter, struct mlx5_fc_bulk *bulk, u32 id)
|
||||
static void mlx5_fc_init(struct mlx5_fc *counter, struct mlx5_fc_bulk *bulk,
|
||||
u32 id)
|
||||
{
|
||||
counter->bulk = bulk;
|
||||
counter->id = id;
|
||||
|
@ -435,8 +481,7 @@ static int mlx5_fc_bulk_get_free_fcs_amount(struct mlx5_fc_bulk *bulk)
|
|||
return bitmap_weight(bulk->bitmask, bulk->bulk_len);
|
||||
}
|
||||
|
||||
static struct mlx5_fc_bulk __attribute__((unused))
|
||||
*mlx5_fc_bulk_create(struct mlx5_core_dev *dev)
|
||||
static struct mlx5_fc_bulk *mlx5_fc_bulk_create(struct mlx5_core_dev *dev)
|
||||
{
|
||||
enum mlx5_fc_bulk_alloc_bitmask alloc_bitmask;
|
||||
struct mlx5_fc_bulk *bulk;
|
||||
|
@ -479,7 +524,7 @@ static struct mlx5_fc_bulk __attribute__((unused))
|
|||
return ERR_PTR(err);
|
||||
}
|
||||
|
||||
static int __attribute__((unused))
|
||||
static int
|
||||
mlx5_fc_bulk_destroy(struct mlx5_core_dev *dev, struct mlx5_fc_bulk *bulk)
|
||||
{
|
||||
if (mlx5_fc_bulk_get_free_fcs_amount(bulk) < bulk->bulk_len) {
|
||||
|
@ -494,8 +539,7 @@ mlx5_fc_bulk_destroy(struct mlx5_core_dev *dev, struct mlx5_fc_bulk *bulk)
|
|||
return 0;
|
||||
}
|
||||
|
||||
static struct mlx5_fc __attribute__((unused))
|
||||
*mlx5_fc_bulk_acquire_fc(struct mlx5_fc_bulk *bulk)
|
||||
static struct mlx5_fc *mlx5_fc_bulk_acquire_fc(struct mlx5_fc_bulk *bulk)
|
||||
{
|
||||
int free_fc_index = find_first_bit(bulk->bitmask, bulk->bulk_len);
|
||||
|
||||
|
@ -506,8 +550,7 @@ static struct mlx5_fc __attribute__((unused))
|
|||
return &bulk->fcs[free_fc_index];
|
||||
}
|
||||
|
||||
static int __attribute__((unused))
|
||||
mlx5_fc_bulk_release_fc(struct mlx5_fc_bulk *bulk, struct mlx5_fc *fc)
|
||||
static int mlx5_fc_bulk_release_fc(struct mlx5_fc_bulk *bulk, struct mlx5_fc *fc)
|
||||
{
|
||||
int fc_index = fc->id - bulk->base_id;
|
||||
|
||||
|
@ -517,3 +560,141 @@ mlx5_fc_bulk_release_fc(struct mlx5_fc_bulk *bulk, struct mlx5_fc *fc)
|
|||
set_bit(fc_index, bulk->bitmask);
|
||||
return 0;
|
||||
}
|
||||
|
||||
/* Flow counters pool API */
|
||||
|
||||
static void mlx5_fc_pool_init(struct mlx5_fc_pool *fc_pool, struct mlx5_core_dev *dev)
|
||||
{
|
||||
fc_pool->dev = dev;
|
||||
mutex_init(&fc_pool->pool_lock);
|
||||
INIT_LIST_HEAD(&fc_pool->fully_used);
|
||||
INIT_LIST_HEAD(&fc_pool->partially_used);
|
||||
INIT_LIST_HEAD(&fc_pool->unused);
|
||||
fc_pool->available_fcs = 0;
|
||||
fc_pool->used_fcs = 0;
|
||||
fc_pool->threshold = 0;
|
||||
}
|
||||
|
||||
static void mlx5_fc_pool_cleanup(struct mlx5_fc_pool *fc_pool)
|
||||
{
|
||||
struct mlx5_core_dev *dev = fc_pool->dev;
|
||||
struct mlx5_fc_bulk *bulk;
|
||||
struct mlx5_fc_bulk *tmp;
|
||||
|
||||
list_for_each_entry_safe(bulk, tmp, &fc_pool->fully_used, pool_list)
|
||||
mlx5_fc_bulk_destroy(dev, bulk);
|
||||
list_for_each_entry_safe(bulk, tmp, &fc_pool->partially_used, pool_list)
|
||||
mlx5_fc_bulk_destroy(dev, bulk);
|
||||
list_for_each_entry_safe(bulk, tmp, &fc_pool->unused, pool_list)
|
||||
mlx5_fc_bulk_destroy(dev, bulk);
|
||||
}
|
||||
|
||||
static void mlx5_fc_pool_update_threshold(struct mlx5_fc_pool *fc_pool)
|
||||
{
|
||||
fc_pool->threshold = min_t(int, MLX5_FC_POOL_MAX_THRESHOLD,
|
||||
fc_pool->used_fcs / MLX5_FC_POOL_USED_BUFF_RATIO);
|
||||
}
|
||||
|
||||
static struct mlx5_fc_bulk *
|
||||
mlx5_fc_pool_alloc_new_bulk(struct mlx5_fc_pool *fc_pool)
|
||||
{
|
||||
struct mlx5_core_dev *dev = fc_pool->dev;
|
||||
struct mlx5_fc_bulk *new_bulk;
|
||||
|
||||
new_bulk = mlx5_fc_bulk_create(dev);
|
||||
if (!IS_ERR(new_bulk))
|
||||
fc_pool->available_fcs += new_bulk->bulk_len;
|
||||
mlx5_fc_pool_update_threshold(fc_pool);
|
||||
return new_bulk;
|
||||
}
|
||||
|
||||
static void
|
||||
mlx5_fc_pool_free_bulk(struct mlx5_fc_pool *fc_pool, struct mlx5_fc_bulk *bulk)
|
||||
{
|
||||
struct mlx5_core_dev *dev = fc_pool->dev;
|
||||
|
||||
fc_pool->available_fcs -= bulk->bulk_len;
|
||||
mlx5_fc_bulk_destroy(dev, bulk);
|
||||
mlx5_fc_pool_update_threshold(fc_pool);
|
||||
}
|
||||
|
||||
static struct mlx5_fc *
|
||||
mlx5_fc_pool_acquire_from_list(struct list_head *src_list,
|
||||
struct list_head *next_list,
|
||||
bool move_non_full_bulk)
|
||||
{
|
||||
struct mlx5_fc_bulk *bulk;
|
||||
struct mlx5_fc *fc;
|
||||
|
||||
if (list_empty(src_list))
|
||||
return ERR_PTR(-ENODATA);
|
||||
|
||||
bulk = list_first_entry(src_list, struct mlx5_fc_bulk, pool_list);
|
||||
fc = mlx5_fc_bulk_acquire_fc(bulk);
|
||||
if (move_non_full_bulk || mlx5_fc_bulk_get_free_fcs_amount(bulk) == 0)
|
||||
list_move(&bulk->pool_list, next_list);
|
||||
return fc;
|
||||
}
|
||||
|
||||
static struct mlx5_fc *
|
||||
mlx5_fc_pool_acquire_counter(struct mlx5_fc_pool *fc_pool)
|
||||
{
|
||||
struct mlx5_fc_bulk *new_bulk;
|
||||
struct mlx5_fc *fc;
|
||||
|
||||
mutex_lock(&fc_pool->pool_lock);
|
||||
|
||||
fc = mlx5_fc_pool_acquire_from_list(&fc_pool->partially_used,
|
||||
&fc_pool->fully_used, false);
|
||||
if (IS_ERR(fc))
|
||||
fc = mlx5_fc_pool_acquire_from_list(&fc_pool->unused,
|
||||
&fc_pool->partially_used,
|
||||
true);
|
||||
if (IS_ERR(fc)) {
|
||||
new_bulk = mlx5_fc_pool_alloc_new_bulk(fc_pool);
|
||||
if (IS_ERR(new_bulk)) {
|
||||
fc = ERR_CAST(new_bulk);
|
||||
goto out;
|
||||
}
|
||||
fc = mlx5_fc_bulk_acquire_fc(new_bulk);
|
||||
list_add(&new_bulk->pool_list, &fc_pool->partially_used);
|
||||
}
|
||||
fc_pool->available_fcs--;
|
||||
fc_pool->used_fcs++;
|
||||
|
||||
out:
|
||||
mutex_unlock(&fc_pool->pool_lock);
|
||||
return fc;
|
||||
}
|
||||
|
||||
static void
|
||||
mlx5_fc_pool_release_counter(struct mlx5_fc_pool *fc_pool, struct mlx5_fc *fc)
|
||||
{
|
||||
struct mlx5_core_dev *dev = fc_pool->dev;
|
||||
struct mlx5_fc_bulk *bulk = fc->bulk;
|
||||
int bulk_free_fcs_amount;
|
||||
|
||||
mutex_lock(&fc_pool->pool_lock);
|
||||
|
||||
if (mlx5_fc_bulk_release_fc(bulk, fc)) {
|
||||
mlx5_core_warn(dev, "Attempted to release a counter which is not acquired\n");
|
||||
goto unlock;
|
||||
}
|
||||
|
||||
fc_pool->available_fcs++;
|
||||
fc_pool->used_fcs--;
|
||||
|
||||
bulk_free_fcs_amount = mlx5_fc_bulk_get_free_fcs_amount(bulk);
|
||||
if (bulk_free_fcs_amount == 1)
|
||||
list_move_tail(&bulk->pool_list, &fc_pool->partially_used);
|
||||
if (bulk_free_fcs_amount == bulk->bulk_len) {
|
||||
list_del(&bulk->pool_list);
|
||||
if (fc_pool->available_fcs > fc_pool->threshold)
|
||||
mlx5_fc_pool_free_bulk(fc_pool, bulk);
|
||||
else
|
||||
list_add(&bulk->pool_list, &fc_pool->unused);
|
||||
}
|
||||
|
||||
unlock:
|
||||
mutex_unlock(&fc_pool->pool_lock);
|
||||
}
|
||||
|
|
|
@ -477,6 +477,17 @@ struct mlx5_core_sriov {
|
|||
u16 max_vfs;
|
||||
};
|
||||
|
||||
struct mlx5_fc_pool {
|
||||
struct mlx5_core_dev *dev;
|
||||
struct mutex pool_lock; /* protects pool lists */
|
||||
struct list_head fully_used;
|
||||
struct list_head partially_used;
|
||||
struct list_head unused;
|
||||
int available_fcs;
|
||||
int used_fcs;
|
||||
int threshold;
|
||||
};
|
||||
|
||||
struct mlx5_fc_stats {
|
||||
spinlock_t counters_idr_lock; /* protects counters_idr */
|
||||
struct idr counters_idr;
|
||||
|
@ -489,6 +500,7 @@ struct mlx5_fc_stats {
|
|||
unsigned long next_query;
|
||||
unsigned long sampling_interval; /* jiffies */
|
||||
u32 *bulk_query_out;
|
||||
struct mlx5_fc_pool fc_pool;
|
||||
};
|
||||
|
||||
struct mlx5_events;
|
||||
|
|
Loading…
Reference in a new issue