2019-06-10 23:38:37 +00:00
|
|
|
// SPDX-License-Identifier: GPL-2.0 OR Linux-OpenIB
|
|
|
|
/* Copyright (c) 2019 Mellanox Technologies. */
|
|
|
|
|
2023-04-02 12:17:09 +00:00
|
|
|
#include <linux/pci.h>
|
2019-06-10 23:38:37 +00:00
|
|
|
#include <linux/interrupt.h>
|
|
|
|
#include <linux/notifier.h>
|
|
|
|
#include <linux/mlx5/driver.h>
|
2022-12-06 18:51:16 +00:00
|
|
|
#include <linux/mlx5/vport.h>
|
2019-06-10 23:38:37 +00:00
|
|
|
#include "mlx5_core.h"
|
2021-04-06 18:42:17 +00:00
|
|
|
#include "mlx5_irq.h"
|
2021-11-23 08:48:07 +00:00
|
|
|
#include "pci_irq.h"
|
2021-02-23 09:57:32 +00:00
|
|
|
#include "lib/sf.h"
|
2023-01-01 06:16:23 +00:00
|
|
|
#include "lib/eq.h"
|
2019-06-10 23:38:37 +00:00
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
|
#include <linux/cpu_rmap.h>
|
|
|
|
#endif
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
#define MLX5_SFS_PER_CTRL_IRQ 64
|
|
|
|
#define MLX5_IRQ_CTRL_SF_MAX 8
|
2021-07-30 03:03:00 +00:00
|
|
|
/* min num of vectors for SFs to be enabled */
|
2021-02-23 09:48:17 +00:00
|
|
|
#define MLX5_IRQ_VEC_COMP_BASE_SF 2
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2021-02-23 09:57:32 +00:00
|
|
|
#define MLX5_EQ_SHARE_IRQ_MAX_COMP (8)
|
|
|
|
#define MLX5_EQ_SHARE_IRQ_MAX_CTRL (UINT_MAX)
|
|
|
|
#define MLX5_EQ_SHARE_IRQ_MIN_COMP (1)
|
|
|
|
#define MLX5_EQ_SHARE_IRQ_MIN_CTRL (4)
|
|
|
|
|
2019-06-10 23:38:39 +00:00
|
|
|
struct mlx5_irq {
|
2019-06-10 23:38:37 +00:00
|
|
|
struct atomic_notifier_head nh;
|
|
|
|
cpumask_var_t mask;
|
|
|
|
char name[MLX5_MAX_IRQ_NAME];
|
2021-06-16 15:58:26 +00:00
|
|
|
struct mlx5_irq_pool *pool;
|
2021-06-22 11:20:16 +00:00
|
|
|
int refcount;
|
2022-12-28 14:30:52 +00:00
|
|
|
struct msi_map map;
|
2023-04-17 07:57:50 +00:00
|
|
|
u32 pool_index;
|
2019-06-10 23:38:37 +00:00
|
|
|
};
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_irq_table {
|
2023-01-01 07:22:34 +00:00
|
|
|
struct mlx5_irq_pool *pcif_pool;
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_irq_pool *sf_ctrl_pool;
|
|
|
|
struct mlx5_irq_pool *sf_comp_pool;
|
|
|
|
};
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2023-03-07 17:13:43 +00:00
|
|
|
static int mlx5_core_func_to_vport(const struct mlx5_core_dev *dev,
|
|
|
|
int func,
|
|
|
|
bool ec_vf_func)
|
|
|
|
{
|
|
|
|
if (!ec_vf_func)
|
|
|
|
return func;
|
|
|
|
return mlx5_core_ec_vf_vport_base(dev) + func - 1;
|
|
|
|
}
|
|
|
|
|
2021-03-14 12:42:55 +00:00
|
|
|
/**
|
|
|
|
* mlx5_get_default_msix_vec_count - Get the default number of MSI-X vectors
|
|
|
|
* to be ssigned to each VF.
|
|
|
|
* @dev: PF to work on
|
|
|
|
* @num_vfs: Number of enabled VFs
|
|
|
|
*/
|
|
|
|
int mlx5_get_default_msix_vec_count(struct mlx5_core_dev *dev, int num_vfs)
|
|
|
|
{
|
|
|
|
int num_vf_msix, min_msix, max_msix;
|
|
|
|
|
|
|
|
num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
|
|
|
|
if (!num_vf_msix)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
|
|
|
|
max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
|
|
|
|
|
|
|
|
/* Limit maximum number of MSI-X vectors so the default configuration
|
|
|
|
* has some available in the pool. This will allow the user to increase
|
|
|
|
* the number of vectors in a VF without having to first size-down other
|
|
|
|
* VFs.
|
|
|
|
*/
|
|
|
|
return max(min(num_vf_msix / num_vfs, max_msix / 2), min_msix);
|
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mlx5_set_msix_vec_count - Set dynamically allocated MSI-X on the VF
|
|
|
|
* @dev: PF to work on
|
|
|
|
* @function_id: Internal PCI VF function IDd
|
|
|
|
* @msix_vec_count: Number of MSI-X vectors to set
|
|
|
|
*/
|
|
|
|
int mlx5_set_msix_vec_count(struct mlx5_core_dev *dev, int function_id,
|
|
|
|
int msix_vec_count)
|
|
|
|
{
|
2021-04-25 10:28:10 +00:00
|
|
|
int query_sz = MLX5_ST_SZ_BYTES(query_hca_cap_out);
|
|
|
|
int set_sz = MLX5_ST_SZ_BYTES(set_hca_cap_in);
|
|
|
|
void *hca_cap = NULL, *query_cap = NULL, *cap;
|
2021-03-14 12:42:55 +00:00
|
|
|
int num_vf_msix, min_msix, max_msix;
|
2023-03-07 17:13:43 +00:00
|
|
|
bool ec_vf_function;
|
|
|
|
int vport;
|
2021-03-14 12:42:55 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
num_vf_msix = MLX5_CAP_GEN_MAX(dev, num_total_dynamic_vf_msix);
|
|
|
|
if (!num_vf_msix)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
if (!MLX5_CAP_GEN(dev, vport_group_manager) || !mlx5_core_is_pf(dev))
|
|
|
|
return -EOPNOTSUPP;
|
|
|
|
|
|
|
|
min_msix = MLX5_CAP_GEN(dev, min_dynamic_vf_msix_table_size);
|
|
|
|
max_msix = MLX5_CAP_GEN(dev, max_dynamic_vf_msix_table_size);
|
|
|
|
|
|
|
|
if (msix_vec_count < min_msix)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (msix_vec_count > max_msix)
|
|
|
|
return -EOVERFLOW;
|
|
|
|
|
2021-04-27 14:23:16 +00:00
|
|
|
query_cap = kvzalloc(query_sz, GFP_KERNEL);
|
|
|
|
hca_cap = kvzalloc(set_sz, GFP_KERNEL);
|
2021-04-25 10:28:10 +00:00
|
|
|
if (!hca_cap || !query_cap) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
2023-03-07 17:13:43 +00:00
|
|
|
ec_vf_function = mlx5_core_ec_sriov_enabled(dev);
|
|
|
|
vport = mlx5_core_func_to_vport(dev, function_id, ec_vf_function);
|
|
|
|
ret = mlx5_vport_get_other_func_general_cap(dev, vport, query_cap);
|
2021-04-25 10:28:10 +00:00
|
|
|
if (ret)
|
|
|
|
goto out;
|
2021-03-14 12:42:55 +00:00
|
|
|
|
|
|
|
cap = MLX5_ADDR_OF(set_hca_cap_in, hca_cap, capability);
|
2021-04-25 10:28:10 +00:00
|
|
|
memcpy(cap, MLX5_ADDR_OF(query_hca_cap_out, query_cap, capability),
|
|
|
|
MLX5_UN_SZ_BYTES(hca_cap_union));
|
2021-03-14 12:42:55 +00:00
|
|
|
MLX5_SET(cmd_hca_cap, cap, dynamic_msix_table_size, msix_vec_count);
|
|
|
|
|
|
|
|
MLX5_SET(set_hca_cap_in, hca_cap, opcode, MLX5_CMD_OP_SET_HCA_CAP);
|
|
|
|
MLX5_SET(set_hca_cap_in, hca_cap, other_function, 1);
|
2023-03-07 17:13:43 +00:00
|
|
|
MLX5_SET(set_hca_cap_in, hca_cap, ec_vf_function, ec_vf_function);
|
2021-03-14 12:42:55 +00:00
|
|
|
MLX5_SET(set_hca_cap_in, hca_cap, function_id, function_id);
|
|
|
|
|
|
|
|
MLX5_SET(set_hca_cap_in, hca_cap, op_mod,
|
|
|
|
MLX5_SET_HCA_CAP_OP_MOD_GENERAL_DEVICE << 1);
|
|
|
|
ret = mlx5_cmd_exec_in(dev, set_hca_cap, hca_cap);
|
2021-04-25 10:28:10 +00:00
|
|
|
out:
|
2021-04-27 14:23:16 +00:00
|
|
|
kvfree(hca_cap);
|
|
|
|
kvfree(query_cap);
|
2021-03-14 12:42:55 +00:00
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2023-06-08 19:00:54 +00:00
|
|
|
/* mlx5_system_free_irq - Free an IRQ
|
|
|
|
* @irq: IRQ to free
|
|
|
|
*
|
|
|
|
* Free the IRQ and other resources such as rmap from the system.
|
|
|
|
* BUT doesn't free or remove reference from mlx5.
|
|
|
|
* This function is very important for the shutdown flow, where we need to
|
|
|
|
* cleanup system resoruces but keep mlx5 objects alive,
|
|
|
|
* see mlx5_irq_table_free_irqs().
|
|
|
|
*/
|
|
|
|
static void mlx5_system_free_irq(struct mlx5_irq *irq)
|
2021-02-23 07:32:21 +00:00
|
|
|
{
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_irq_pool *pool = irq->pool;
|
2023-01-01 06:16:23 +00:00
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
|
struct cpu_rmap *rmap;
|
|
|
|
#endif
|
2021-02-23 07:32:21 +00:00
|
|
|
|
2023-01-01 06:16:23 +00:00
|
|
|
/* free_irq requires that affinity_hint and rmap will be cleared before
|
|
|
|
* calling it. To satisfy this requirement, we call
|
|
|
|
* irq_cpu_rmap_remove() to remove the notifier
|
2021-02-23 09:08:26 +00:00
|
|
|
*/
|
2022-12-28 14:30:52 +00:00
|
|
|
irq_update_affinity_hint(irq->map.virq, NULL);
|
2023-01-01 06:16:23 +00:00
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
|
rmap = mlx5_eq_table_get_rmap(pool->dev);
|
2023-05-30 08:59:34 +00:00
|
|
|
if (rmap)
|
2023-01-01 06:16:23 +00:00
|
|
|
irq_cpu_rmap_remove(rmap, irq->map.virq);
|
|
|
|
#endif
|
|
|
|
|
2022-12-28 14:30:52 +00:00
|
|
|
free_irq(irq->map.virq, &irq->nh);
|
2023-01-01 06:16:23 +00:00
|
|
|
if (irq->map.index && pci_msix_can_alloc_dyn(pool->dev->pdev))
|
|
|
|
pci_msix_free_irq(pool->dev->pdev, irq->map);
|
2023-06-08 19:00:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void irq_release(struct mlx5_irq *irq)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_pool *pool = irq->pool;
|
|
|
|
|
|
|
|
xa_erase(&pool->irqs, irq->pool_index);
|
|
|
|
mlx5_system_free_irq(irq);
|
|
|
|
free_cpumask_var(irq->mask);
|
2021-02-23 09:38:52 +00:00
|
|
|
kfree(irq);
|
2021-02-23 07:32:21 +00:00
|
|
|
}
|
|
|
|
|
2021-11-23 10:50:19 +00:00
|
|
|
int mlx5_irq_put(struct mlx5_irq *irq)
|
2021-02-23 07:32:21 +00:00
|
|
|
{
|
2021-02-23 09:57:32 +00:00
|
|
|
struct mlx5_irq_pool *pool = irq->pool;
|
2021-11-23 10:50:19 +00:00
|
|
|
int ret = 0;
|
2021-02-23 09:57:32 +00:00
|
|
|
|
|
|
|
mutex_lock(&pool->lock);
|
2021-06-22 11:20:16 +00:00
|
|
|
irq->refcount--;
|
2021-11-23 10:50:19 +00:00
|
|
|
if (!irq->refcount) {
|
2021-06-22 11:20:16 +00:00
|
|
|
irq_release(irq);
|
2021-11-23 10:50:19 +00:00
|
|
|
ret = 1;
|
|
|
|
}
|
2021-02-23 09:57:32 +00:00
|
|
|
mutex_unlock(&pool->lock);
|
2021-11-23 10:50:19 +00:00
|
|
|
return ret;
|
2021-02-23 07:32:21 +00:00
|
|
|
}
|
|
|
|
|
2021-11-23 08:48:07 +00:00
|
|
|
int mlx5_irq_read_locked(struct mlx5_irq *irq)
|
|
|
|
{
|
|
|
|
lockdep_assert_held(&irq->pool->lock);
|
|
|
|
return irq->refcount;
|
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_irq_get_locked(struct mlx5_irq *irq)
|
2021-06-22 11:20:16 +00:00
|
|
|
{
|
|
|
|
lockdep_assert_held(&irq->pool->lock);
|
|
|
|
if (WARN_ON_ONCE(!irq->refcount))
|
|
|
|
return 0;
|
|
|
|
irq->refcount++;
|
|
|
|
return 1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int irq_get(struct mlx5_irq *irq)
|
|
|
|
{
|
|
|
|
int err;
|
|
|
|
|
|
|
|
mutex_lock(&irq->pool->lock);
|
2021-11-23 08:48:07 +00:00
|
|
|
err = mlx5_irq_get_locked(irq);
|
2021-06-22 11:20:16 +00:00
|
|
|
mutex_unlock(&irq->pool->lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:15:43 +00:00
|
|
|
static irqreturn_t irq_int_handler(int irq, void *nh)
|
2019-06-10 23:38:37 +00:00
|
|
|
{
|
|
|
|
atomic_notifier_call_chain(nh, 0, NULL);
|
|
|
|
return IRQ_HANDLED;
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
static void irq_sf_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
|
|
|
|
{
|
|
|
|
snprintf(name, MLX5_MAX_IRQ_NAME, "%s%d", pool->name, vecidx);
|
|
|
|
}
|
|
|
|
|
2021-08-19 13:18:57 +00:00
|
|
|
static void irq_set_name(struct mlx5_irq_pool *pool, char *name, int vecidx)
|
2019-06-10 23:38:37 +00:00
|
|
|
{
|
2021-08-01 09:08:49 +00:00
|
|
|
if (!pool->xa_num_irqs.max) {
|
|
|
|
/* in case we only have a single irq for the device */
|
|
|
|
snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_combined%d", vecidx);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2023-01-01 06:16:23 +00:00
|
|
|
if (!vecidx) {
|
2021-02-23 09:48:17 +00:00
|
|
|
snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_async%d", vecidx);
|
2019-06-10 23:38:41 +00:00
|
|
|
return;
|
2019-06-10 23:38:37 +00:00
|
|
|
}
|
2019-06-10 23:38:41 +00:00
|
|
|
|
2021-08-19 13:18:57 +00:00
|
|
|
snprintf(name, MLX5_MAX_IRQ_NAME, "mlx5_comp%d", vecidx);
|
2019-06-10 23:38:37 +00:00
|
|
|
}
|
|
|
|
|
2021-11-23 08:48:07 +00:00
|
|
|
struct mlx5_irq *mlx5_irq_alloc(struct mlx5_irq_pool *pool, int i,
|
2023-01-01 06:16:23 +00:00
|
|
|
struct irq_affinity_desc *af_desc,
|
|
|
|
struct cpu_rmap **rmap)
|
2019-06-10 23:38:37 +00:00
|
|
|
{
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_core_dev *dev = pool->dev;
|
2019-06-10 23:38:37 +00:00
|
|
|
char name[MLX5_MAX_IRQ_NAME];
|
2021-02-23 09:24:47 +00:00
|
|
|
struct mlx5_irq *irq;
|
2019-06-10 23:38:37 +00:00
|
|
|
int err;
|
2021-02-23 09:15:43 +00:00
|
|
|
|
2021-02-23 09:38:52 +00:00
|
|
|
irq = kzalloc(sizeof(*irq), GFP_KERNEL);
|
|
|
|
if (!irq)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
2023-01-01 06:16:23 +00:00
|
|
|
if (!i || !pci_msix_can_alloc_dyn(dev->pdev)) {
|
2023-05-31 08:48:56 +00:00
|
|
|
/* The vector at index 0 is always statically allocated. If
|
|
|
|
* dynamic irq is not supported all vectors are statically
|
|
|
|
* allocated. In both cases just get the irq number and set
|
|
|
|
* the index.
|
2023-01-01 06:16:23 +00:00
|
|
|
*/
|
|
|
|
irq->map.virq = pci_irq_vector(dev->pdev, i);
|
2023-05-31 08:48:56 +00:00
|
|
|
irq->map.index = i;
|
2023-01-01 06:16:23 +00:00
|
|
|
} else {
|
|
|
|
irq->map = pci_msix_alloc_irq_at(dev->pdev, MSI_ANY_INDEX, af_desc);
|
|
|
|
if (!irq->map.virq) {
|
|
|
|
err = irq->map.index;
|
|
|
|
goto err_alloc_irq;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (i && rmap && *rmap) {
|
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
|
err = irq_cpu_rmap_add(*rmap, irq->map.virq);
|
|
|
|
if (err)
|
|
|
|
goto err_irq_rmap;
|
|
|
|
#endif
|
|
|
|
}
|
2021-11-23 10:50:19 +00:00
|
|
|
if (!mlx5_irq_pool_is_sf_pool(pool))
|
2021-08-19 13:18:57 +00:00
|
|
|
irq_set_name(pool, name, i);
|
2021-02-23 09:48:17 +00:00
|
|
|
else
|
|
|
|
irq_sf_set_name(pool, name, i);
|
2021-02-23 09:15:43 +00:00
|
|
|
ATOMIC_INIT_NOTIFIER_HEAD(&irq->nh);
|
|
|
|
snprintf(irq->name, MLX5_MAX_IRQ_NAME,
|
|
|
|
"%s@pci:%s", name, pci_name(dev->pdev));
|
2022-12-28 14:30:52 +00:00
|
|
|
err = request_irq(irq->map.virq, irq_int_handler, 0, irq->name,
|
2021-02-23 09:15:43 +00:00
|
|
|
&irq->nh);
|
|
|
|
if (err) {
|
|
|
|
mlx5_core_err(dev, "Failed to request irq. err = %d\n", err);
|
2021-02-23 09:24:47 +00:00
|
|
|
goto err_req_irq;
|
2019-06-10 23:38:37 +00:00
|
|
|
}
|
2021-02-23 09:15:43 +00:00
|
|
|
if (!zalloc_cpumask_var(&irq->mask, GFP_KERNEL)) {
|
|
|
|
mlx5_core_warn(dev, "zalloc_cpumask_var failed\n");
|
2021-02-23 09:24:47 +00:00
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_cpumask;
|
2021-02-23 09:15:43 +00:00
|
|
|
}
|
2022-12-29 09:02:19 +00:00
|
|
|
if (af_desc) {
|
|
|
|
cpumask_copy(irq->mask, &af_desc->mask);
|
2022-12-28 14:30:52 +00:00
|
|
|
irq_set_affinity_and_hint(irq->map.virq, irq->mask);
|
2021-11-23 07:14:58 +00:00
|
|
|
}
|
2021-06-16 15:59:05 +00:00
|
|
|
irq->pool = pool;
|
2021-06-22 11:20:16 +00:00
|
|
|
irq->refcount = 1;
|
2023-04-17 07:57:50 +00:00
|
|
|
irq->pool_index = i;
|
|
|
|
err = xa_err(xa_store(&pool->irqs, irq->pool_index, irq, GFP_KERNEL));
|
2021-02-23 09:38:52 +00:00
|
|
|
if (err) {
|
|
|
|
mlx5_core_err(dev, "Failed to alloc xa entry for irq(%u). err = %d\n",
|
2023-04-17 07:57:50 +00:00
|
|
|
irq->pool_index, err);
|
2021-02-23 09:38:52 +00:00
|
|
|
goto err_xa;
|
|
|
|
}
|
|
|
|
return irq;
|
|
|
|
err_xa:
|
2022-12-29 09:02:19 +00:00
|
|
|
if (af_desc)
|
|
|
|
irq_update_affinity_hint(irq->map.virq, NULL);
|
2021-02-23 09:38:52 +00:00
|
|
|
free_cpumask_var(irq->mask);
|
2021-02-23 09:24:47 +00:00
|
|
|
err_cpumask:
|
2022-12-28 14:30:52 +00:00
|
|
|
free_irq(irq->map.virq, &irq->nh);
|
2021-02-23 09:24:47 +00:00
|
|
|
err_req_irq:
|
2023-01-01 06:16:23 +00:00
|
|
|
#ifdef CONFIG_RFS_ACCEL
|
|
|
|
if (i && rmap && *rmap) {
|
|
|
|
free_irq_cpu_rmap(*rmap);
|
|
|
|
*rmap = NULL;
|
|
|
|
}
|
|
|
|
err_irq_rmap:
|
|
|
|
#endif
|
|
|
|
if (i && pci_msix_can_alloc_dyn(dev->pdev))
|
|
|
|
pci_msix_free_irq(dev->pdev, irq->map);
|
|
|
|
err_alloc_irq:
|
2021-02-23 09:38:52 +00:00
|
|
|
kfree(irq);
|
|
|
|
return ERR_PTR(err);
|
2021-02-23 09:15:43 +00:00
|
|
|
}
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
int mlx5_irq_attach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
|
|
|
|
{
|
2021-06-22 11:20:16 +00:00
|
|
|
int ret;
|
2021-02-23 09:48:17 +00:00
|
|
|
|
2021-06-22 11:20:16 +00:00
|
|
|
ret = irq_get(irq);
|
|
|
|
if (!ret)
|
2021-02-23 09:48:17 +00:00
|
|
|
/* Something very bad happens here, we are enabling EQ
|
|
|
|
* on non-existing IRQ.
|
|
|
|
*/
|
|
|
|
return -ENOENT;
|
2021-06-22 11:20:16 +00:00
|
|
|
ret = atomic_notifier_chain_register(&irq->nh, nb);
|
|
|
|
if (ret)
|
2021-11-23 10:50:19 +00:00
|
|
|
mlx5_irq_put(irq);
|
2021-06-22 11:20:16 +00:00
|
|
|
return ret;
|
2021-02-23 09:48:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_irq_detach_nb(struct mlx5_irq *irq, struct notifier_block *nb)
|
|
|
|
{
|
2021-06-16 15:54:05 +00:00
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
err = atomic_notifier_chain_unregister(&irq->nh, nb);
|
2021-11-23 10:50:19 +00:00
|
|
|
mlx5_irq_put(irq);
|
2021-06-16 15:54:05 +00:00
|
|
|
return err;
|
2021-02-23 09:48:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
struct cpumask *mlx5_irq_get_affinity_mask(struct mlx5_irq *irq)
|
|
|
|
{
|
|
|
|
return irq->mask;
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:57:32 +00:00
|
|
|
int mlx5_irq_get_index(struct mlx5_irq *irq)
|
|
|
|
{
|
2022-12-28 14:30:52 +00:00
|
|
|
return irq->map.index;
|
2021-02-23 09:57:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* irq_pool API */
|
|
|
|
|
|
|
|
/* requesting an irq from a given pool according to given index */
|
|
|
|
static struct mlx5_irq *
|
|
|
|
irq_pool_request_vector(struct mlx5_irq_pool *pool, int vecidx,
|
2023-01-01 06:16:23 +00:00
|
|
|
struct irq_affinity_desc *af_desc,
|
|
|
|
struct cpu_rmap **rmap)
|
2021-02-23 09:57:32 +00:00
|
|
|
{
|
|
|
|
struct mlx5_irq *irq;
|
|
|
|
|
|
|
|
mutex_lock(&pool->lock);
|
|
|
|
irq = xa_load(&pool->irqs, vecidx);
|
|
|
|
if (irq) {
|
2021-11-23 08:48:07 +00:00
|
|
|
mlx5_irq_get_locked(irq);
|
2021-02-23 09:57:32 +00:00
|
|
|
goto unlock;
|
|
|
|
}
|
2023-01-01 06:16:23 +00:00
|
|
|
irq = mlx5_irq_alloc(pool, vecidx, af_desc, rmap);
|
2021-02-23 09:57:32 +00:00
|
|
|
unlock:
|
|
|
|
mutex_unlock(&pool->lock);
|
|
|
|
return irq;
|
|
|
|
}
|
|
|
|
|
2021-11-14 11:01:21 +00:00
|
|
|
static struct mlx5_irq_pool *sf_ctrl_irq_pool_get(struct mlx5_irq_table *irq_table)
|
|
|
|
{
|
|
|
|
return irq_table->sf_ctrl_pool;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct mlx5_irq_pool *sf_irq_pool_get(struct mlx5_irq_table *irq_table)
|
2021-02-23 09:57:32 +00:00
|
|
|
{
|
|
|
|
return irq_table->sf_comp_pool;
|
|
|
|
}
|
|
|
|
|
2021-11-23 10:50:19 +00:00
|
|
|
struct mlx5_irq_pool *mlx5_irq_pool_get(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev);
|
|
|
|
struct mlx5_irq_pool *pool = NULL;
|
|
|
|
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
pool = sf_irq_pool_get(irq_table);
|
|
|
|
|
|
|
|
/* In some configs, there won't be a pool of SFs IRQs. Hence, returning
|
|
|
|
* the PF IRQs pool in case the SF pool doesn't exist.
|
|
|
|
*/
|
2023-01-01 07:22:34 +00:00
|
|
|
return pool ? pool : irq_table->pcif_pool;
|
2021-11-23 10:50:19 +00:00
|
|
|
}
|
|
|
|
|
2021-11-14 11:01:21 +00:00
|
|
|
static struct mlx5_irq_pool *ctrl_irq_pool_get(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev);
|
|
|
|
struct mlx5_irq_pool *pool = NULL;
|
|
|
|
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
pool = sf_ctrl_irq_pool_get(irq_table);
|
|
|
|
|
|
|
|
/* In some configs, there won't be a pool of SFs IRQs. Hence, returning
|
|
|
|
* the PF IRQs pool in case the SF pool doesn't exist.
|
|
|
|
*/
|
2023-01-01 07:22:34 +00:00
|
|
|
return pool ? pool : irq_table->pcif_pool;
|
2021-11-14 11:01:21 +00:00
|
|
|
}
|
|
|
|
|
2023-06-11 11:35:36 +00:00
|
|
|
static void _mlx5_irq_release(struct mlx5_irq *irq)
|
2021-02-23 09:15:43 +00:00
|
|
|
{
|
2023-06-11 11:35:36 +00:00
|
|
|
synchronize_irq(irq->map.virq);
|
|
|
|
mlx5_irq_put(irq);
|
2021-12-12 12:51:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mlx5_ctrl_irq_release - release a ctrl IRQ back to the system.
|
|
|
|
* @ctrl_irq: ctrl IRQ to be released.
|
|
|
|
*/
|
|
|
|
void mlx5_ctrl_irq_release(struct mlx5_irq *ctrl_irq)
|
|
|
|
{
|
2023-06-11 11:35:36 +00:00
|
|
|
_mlx5_irq_release(ctrl_irq);
|
2021-02-23 09:15:43 +00:00
|
|
|
}
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2021-11-14 11:01:21 +00:00
|
|
|
/**
|
|
|
|
* mlx5_ctrl_irq_request - request a ctrl IRQ for mlx5 device.
|
|
|
|
* @dev: mlx5 device that requesting the IRQ.
|
|
|
|
*
|
|
|
|
* This function returns a pointer to IRQ, or ERR_PTR in case of error.
|
|
|
|
*/
|
|
|
|
struct mlx5_irq *mlx5_ctrl_irq_request(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_pool *pool = ctrl_irq_pool_get(dev);
|
2022-12-29 09:02:19 +00:00
|
|
|
struct irq_affinity_desc af_desc;
|
2021-11-14 11:01:21 +00:00
|
|
|
struct mlx5_irq *irq;
|
|
|
|
|
2022-12-29 09:02:19 +00:00
|
|
|
cpumask_copy(&af_desc.mask, cpu_online_mask);
|
|
|
|
af_desc.is_managed = false;
|
2021-11-23 10:50:19 +00:00
|
|
|
if (!mlx5_irq_pool_is_sf_pool(pool)) {
|
2022-12-28 13:51:29 +00:00
|
|
|
/* In case we are allocating a control IRQ from a pci device's pool.
|
|
|
|
* This can happen also for a SF if the SFs pool is empty.
|
|
|
|
*/
|
2021-11-14 11:01:21 +00:00
|
|
|
if (!pool->xa_num_irqs.max) {
|
2022-12-29 09:02:19 +00:00
|
|
|
cpumask_clear(&af_desc.mask);
|
2021-11-14 11:01:21 +00:00
|
|
|
/* In case we only have a single IRQ for PF/VF */
|
2022-12-29 09:02:19 +00:00
|
|
|
cpumask_set_cpu(cpumask_first(cpu_online_mask), &af_desc.mask);
|
2021-11-14 11:01:21 +00:00
|
|
|
}
|
2023-01-01 06:16:23 +00:00
|
|
|
/* Allocate the IRQ in index 0. The vector was already allocated */
|
|
|
|
irq = irq_pool_request_vector(pool, 0, &af_desc, NULL);
|
2021-11-14 11:01:21 +00:00
|
|
|
} else {
|
2022-12-29 09:02:19 +00:00
|
|
|
irq = mlx5_irq_affinity_request(pool, &af_desc);
|
2021-11-14 11:01:21 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return irq;
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:15:43 +00:00
|
|
|
/**
|
2021-11-23 10:50:19 +00:00
|
|
|
* mlx5_irq_request - request an IRQ for mlx5 PF/VF device.
|
2021-02-23 09:15:43 +00:00
|
|
|
* @dev: mlx5 device that requesting the IRQ.
|
|
|
|
* @vecidx: vector index of the IRQ. This argument is ignore if affinity is
|
|
|
|
* provided.
|
2022-12-29 09:02:19 +00:00
|
|
|
* @af_desc: affinity descriptor for this IRQ.
|
2023-01-01 06:16:23 +00:00
|
|
|
* @rmap: pointer to reverse map pointer for completion interrupts
|
2021-02-23 09:15:43 +00:00
|
|
|
*
|
|
|
|
* This function returns a pointer to IRQ, or ERR_PTR in case of error.
|
|
|
|
*/
|
2021-02-23 09:57:32 +00:00
|
|
|
struct mlx5_irq *mlx5_irq_request(struct mlx5_core_dev *dev, u16 vecidx,
|
2023-01-01 06:16:23 +00:00
|
|
|
struct irq_affinity_desc *af_desc,
|
|
|
|
struct cpu_rmap **rmap)
|
2021-02-23 09:15:43 +00:00
|
|
|
{
|
2021-02-23 09:38:52 +00:00
|
|
|
struct mlx5_irq_table *irq_table = mlx5_irq_table_get(dev);
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_irq_pool *pool;
|
2021-02-23 09:38:52 +00:00
|
|
|
struct mlx5_irq *irq;
|
2021-02-23 09:15:43 +00:00
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
pool = irq_table->pcif_pool;
|
2023-01-01 06:16:23 +00:00
|
|
|
irq = irq_pool_request_vector(pool, vecidx, af_desc, rmap);
|
2021-02-23 09:38:52 +00:00
|
|
|
if (IS_ERR(irq))
|
2021-02-23 09:15:43 +00:00
|
|
|
return irq;
|
2021-02-23 09:57:32 +00:00
|
|
|
mlx5_core_dbg(dev, "irq %u mapped to cpu %*pbl, %u EQs on this irq\n",
|
2022-12-29 09:02:19 +00:00
|
|
|
irq->map.virq, cpumask_pr_args(&af_desc->mask),
|
2021-06-22 11:20:16 +00:00
|
|
|
irq->refcount / MLX5_EQ_REFS_PER_IRQ);
|
2021-02-23 09:15:43 +00:00
|
|
|
return irq;
|
2019-06-10 23:38:37 +00:00
|
|
|
}
|
|
|
|
|
2023-01-05 09:31:46 +00:00
|
|
|
/**
|
|
|
|
* mlx5_msix_alloc - allocate msix interrupt
|
|
|
|
* @dev: mlx5 device from which to request
|
|
|
|
* @handler: interrupt handler
|
|
|
|
* @affdesc: affinity descriptor
|
|
|
|
* @name: interrupt name
|
|
|
|
*
|
|
|
|
* Returns: struct msi_map with result encoded.
|
|
|
|
* Note: the caller must make sure to release the irq by calling
|
|
|
|
* mlx5_msix_free() if shutdown was initiated.
|
|
|
|
*/
|
|
|
|
struct msi_map mlx5_msix_alloc(struct mlx5_core_dev *dev,
|
|
|
|
irqreturn_t (*handler)(int, void *),
|
|
|
|
const struct irq_affinity_desc *affdesc,
|
|
|
|
const char *name)
|
|
|
|
{
|
|
|
|
struct msi_map map;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
if (!dev->pdev) {
|
|
|
|
map.virq = 0;
|
|
|
|
map.index = -EINVAL;
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
|
|
|
|
map = pci_msix_alloc_irq_at(dev->pdev, MSI_ANY_INDEX, affdesc);
|
|
|
|
if (!map.virq)
|
|
|
|
return map;
|
|
|
|
|
|
|
|
err = request_irq(map.virq, handler, 0, name, NULL);
|
|
|
|
if (err) {
|
|
|
|
mlx5_core_warn(dev, "err %d\n", err);
|
|
|
|
pci_msix_free_irq(dev->pdev, map);
|
|
|
|
map.virq = 0;
|
|
|
|
map.index = -ENOMEM;
|
|
|
|
}
|
|
|
|
return map;
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mlx5_msix_alloc);
|
|
|
|
|
|
|
|
/**
|
|
|
|
* mlx5_msix_free - free a previously allocated msix interrupt
|
|
|
|
* @dev: mlx5 device associated with interrupt
|
|
|
|
* @map: map previously returned by mlx5_msix_alloc()
|
|
|
|
*/
|
|
|
|
void mlx5_msix_free(struct mlx5_core_dev *dev, struct msi_map map)
|
|
|
|
{
|
|
|
|
free_irq(map.virq, NULL);
|
|
|
|
pci_msix_free_irq(dev->pdev, map);
|
|
|
|
}
|
|
|
|
EXPORT_SYMBOL(mlx5_msix_free);
|
|
|
|
|
2021-12-12 12:51:27 +00:00
|
|
|
/**
|
2023-06-11 11:35:36 +00:00
|
|
|
* mlx5_irq_release_vector - release one IRQ back to the system.
|
|
|
|
* @irq: the irq to release.
|
2021-12-12 12:51:27 +00:00
|
|
|
*/
|
2023-06-11 11:35:36 +00:00
|
|
|
void mlx5_irq_release_vector(struct mlx5_irq *irq)
|
2021-12-12 12:51:27 +00:00
|
|
|
{
|
2023-06-11 11:35:36 +00:00
|
|
|
_mlx5_irq_release(irq);
|
2021-12-12 12:51:27 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/**
|
2023-06-11 11:35:36 +00:00
|
|
|
* mlx5_irq_request_vector - request one IRQ for mlx5 device.
|
|
|
|
* @dev: mlx5 device that is requesting the IRQ.
|
|
|
|
* @cpu: CPU to bind the IRQ to.
|
|
|
|
* @vecidx: vector index to request an IRQ for.
|
2023-01-01 06:16:23 +00:00
|
|
|
* @rmap: pointer to reverse map pointer for completion interrupts
|
2021-12-12 12:51:27 +00:00
|
|
|
*
|
|
|
|
* Each IRQ is bound to at most 1 CPU.
|
2023-06-11 11:35:36 +00:00
|
|
|
* This function is requests one IRQ, for the given @vecidx.
|
2021-12-12 12:51:27 +00:00
|
|
|
*
|
2023-06-11 11:35:36 +00:00
|
|
|
* This function returns a pointer to the irq on success, or an error pointer
|
|
|
|
* in case of an error.
|
2021-12-12 12:51:27 +00:00
|
|
|
*/
|
2023-06-11 11:35:36 +00:00
|
|
|
struct mlx5_irq *mlx5_irq_request_vector(struct mlx5_core_dev *dev, u16 cpu,
|
|
|
|
u16 vecidx, struct cpu_rmap **rmap)
|
2021-12-12 12:51:27 +00:00
|
|
|
{
|
2023-05-03 12:10:05 +00:00
|
|
|
struct mlx5_irq_table *table = mlx5_irq_table_get(dev);
|
|
|
|
struct mlx5_irq_pool *pool = table->pcif_pool;
|
2022-12-29 09:02:19 +00:00
|
|
|
struct irq_affinity_desc af_desc;
|
2023-05-03 12:10:05 +00:00
|
|
|
int offset = 1;
|
2021-12-12 12:51:27 +00:00
|
|
|
|
2023-05-03 12:10:05 +00:00
|
|
|
if (!pool->xa_num_irqs.max)
|
|
|
|
offset = 0;
|
|
|
|
|
2023-04-16 05:54:04 +00:00
|
|
|
af_desc.is_managed = false;
|
2023-06-11 11:35:36 +00:00
|
|
|
cpumask_clear(&af_desc.mask);
|
|
|
|
cpumask_set_cpu(cpu, &af_desc.mask);
|
|
|
|
return mlx5_irq_request(dev, vecidx + offset, &af_desc, rmap);
|
2021-12-12 12:51:27 +00:00
|
|
|
}
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
static struct mlx5_irq_pool *
|
2021-02-23 09:57:32 +00:00
|
|
|
irq_pool_alloc(struct mlx5_core_dev *dev, int start, int size, char *name,
|
|
|
|
u32 min_threshold, u32 max_threshold)
|
2019-06-10 23:38:37 +00:00
|
|
|
{
|
2021-02-23 09:48:17 +00:00
|
|
|
struct mlx5_irq_pool *pool = kvzalloc(sizeof(*pool), GFP_KERNEL);
|
|
|
|
|
|
|
|
if (!pool)
|
|
|
|
return ERR_PTR(-ENOMEM);
|
|
|
|
pool->dev = dev;
|
2021-06-16 22:51:00 +00:00
|
|
|
mutex_init(&pool->lock);
|
2021-02-23 09:48:17 +00:00
|
|
|
xa_init_flags(&pool->irqs, XA_FLAGS_ALLOC);
|
|
|
|
pool->xa_num_irqs.min = start;
|
|
|
|
pool->xa_num_irqs.max = start + size - 1;
|
|
|
|
if (name)
|
|
|
|
snprintf(pool->name, MLX5_MAX_IRQ_NAME - MLX5_MAX_IRQ_IDX_CHARS,
|
2021-11-03 21:01:05 +00:00
|
|
|
"%s", name);
|
2021-02-23 09:57:32 +00:00
|
|
|
pool->min_threshold = min_threshold * MLX5_EQ_REFS_PER_IRQ;
|
|
|
|
pool->max_threshold = max_threshold * MLX5_EQ_REFS_PER_IRQ;
|
2021-02-23 09:48:17 +00:00
|
|
|
mlx5_core_dbg(dev, "pool->name = %s, pool->size = %d, pool->start = %d",
|
|
|
|
name, size, start);
|
|
|
|
return pool;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void irq_pool_free(struct mlx5_irq_pool *pool)
|
|
|
|
{
|
|
|
|
struct mlx5_irq *irq;
|
|
|
|
unsigned long index;
|
|
|
|
|
2021-06-22 11:20:16 +00:00
|
|
|
/* There are cases in which we are destrying the irq_table before
|
|
|
|
* freeing all the IRQs, fast teardown for example. Hence, free the irqs
|
|
|
|
* which might not have been freed.
|
|
|
|
*/
|
2021-02-23 09:48:17 +00:00
|
|
|
xa_for_each(&pool->irqs, index, irq)
|
2021-06-22 11:20:16 +00:00
|
|
|
irq_release(irq);
|
2021-02-23 09:48:17 +00:00
|
|
|
xa_destroy(&pool->irqs);
|
2021-06-16 22:51:00 +00:00
|
|
|
mutex_destroy(&pool->lock);
|
2021-11-23 10:50:19 +00:00
|
|
|
kfree(pool->irqs_per_cpu);
|
2021-02-23 09:48:17 +00:00
|
|
|
kvfree(pool);
|
|
|
|
}
|
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
static int irq_pools_init(struct mlx5_core_dev *dev, int sf_vec, int pcif_vec)
|
2021-02-23 09:48:17 +00:00
|
|
|
{
|
|
|
|
struct mlx5_irq_table *table = dev->priv.irq_table;
|
|
|
|
int num_sf_ctrl_by_msix;
|
|
|
|
int num_sf_ctrl_by_sfs;
|
|
|
|
int num_sf_ctrl;
|
|
|
|
int err;
|
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
/* init pcif_pool */
|
|
|
|
table->pcif_pool = irq_pool_alloc(dev, 0, pcif_vec, NULL,
|
|
|
|
MLX5_EQ_SHARE_IRQ_MIN_COMP,
|
|
|
|
MLX5_EQ_SHARE_IRQ_MAX_COMP);
|
|
|
|
if (IS_ERR(table->pcif_pool))
|
|
|
|
return PTR_ERR(table->pcif_pool);
|
2021-02-23 09:48:17 +00:00
|
|
|
if (!mlx5_sf_max_functions(dev))
|
|
|
|
return 0;
|
|
|
|
if (sf_vec < MLX5_IRQ_VEC_COMP_BASE_SF) {
|
2021-06-29 11:47:30 +00:00
|
|
|
mlx5_core_dbg(dev, "Not enught IRQs for SFs. SF may run at lower performance\n");
|
2021-02-23 09:48:17 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* init sf_ctrl_pool */
|
|
|
|
num_sf_ctrl_by_msix = DIV_ROUND_UP(sf_vec, MLX5_COMP_EQS_PER_SF);
|
|
|
|
num_sf_ctrl_by_sfs = DIV_ROUND_UP(mlx5_sf_max_functions(dev),
|
|
|
|
MLX5_SFS_PER_CTRL_IRQ);
|
|
|
|
num_sf_ctrl = min_t(int, num_sf_ctrl_by_msix, num_sf_ctrl_by_sfs);
|
|
|
|
num_sf_ctrl = min_t(int, MLX5_IRQ_CTRL_SF_MAX, num_sf_ctrl);
|
2023-01-01 07:22:34 +00:00
|
|
|
table->sf_ctrl_pool = irq_pool_alloc(dev, pcif_vec, num_sf_ctrl,
|
2021-02-23 09:57:32 +00:00
|
|
|
"mlx5_sf_ctrl",
|
|
|
|
MLX5_EQ_SHARE_IRQ_MIN_CTRL,
|
|
|
|
MLX5_EQ_SHARE_IRQ_MAX_CTRL);
|
2021-02-23 09:48:17 +00:00
|
|
|
if (IS_ERR(table->sf_ctrl_pool)) {
|
|
|
|
err = PTR_ERR(table->sf_ctrl_pool);
|
|
|
|
goto err_pf;
|
|
|
|
}
|
|
|
|
/* init sf_comp_pool */
|
2023-01-01 07:22:34 +00:00
|
|
|
table->sf_comp_pool = irq_pool_alloc(dev, pcif_vec + num_sf_ctrl,
|
2021-02-23 09:57:32 +00:00
|
|
|
sf_vec - num_sf_ctrl, "mlx5_sf_comp",
|
|
|
|
MLX5_EQ_SHARE_IRQ_MIN_COMP,
|
|
|
|
MLX5_EQ_SHARE_IRQ_MAX_COMP);
|
2021-02-23 09:48:17 +00:00
|
|
|
if (IS_ERR(table->sf_comp_pool)) {
|
|
|
|
err = PTR_ERR(table->sf_comp_pool);
|
|
|
|
goto err_sf_ctrl;
|
|
|
|
}
|
2021-11-23 10:50:19 +00:00
|
|
|
|
|
|
|
table->sf_comp_pool->irqs_per_cpu = kcalloc(nr_cpu_ids, sizeof(u16), GFP_KERNEL);
|
|
|
|
if (!table->sf_comp_pool->irqs_per_cpu) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto err_irqs_per_cpu;
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
return 0;
|
2021-11-23 10:50:19 +00:00
|
|
|
|
|
|
|
err_irqs_per_cpu:
|
|
|
|
irq_pool_free(table->sf_comp_pool);
|
2021-02-23 09:48:17 +00:00
|
|
|
err_sf_ctrl:
|
|
|
|
irq_pool_free(table->sf_ctrl_pool);
|
|
|
|
err_pf:
|
2023-01-01 07:22:34 +00:00
|
|
|
irq_pool_free(table->pcif_pool);
|
2021-02-23 09:48:17 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void irq_pools_destroy(struct mlx5_irq_table *table)
|
|
|
|
{
|
|
|
|
if (table->sf_ctrl_pool) {
|
|
|
|
irq_pool_free(table->sf_comp_pool);
|
|
|
|
irq_pool_free(table->sf_ctrl_pool);
|
|
|
|
}
|
2023-01-01 07:22:34 +00:00
|
|
|
irq_pool_free(table->pcif_pool);
|
2021-02-23 09:48:17 +00:00
|
|
|
}
|
|
|
|
|
2023-04-13 19:15:31 +00:00
|
|
|
static void mlx5_irq_pool_free_irqs(struct mlx5_irq_pool *pool)
|
|
|
|
{
|
|
|
|
struct mlx5_irq *irq;
|
|
|
|
unsigned long index;
|
|
|
|
|
|
|
|
xa_for_each(&pool->irqs, index, irq)
|
2023-06-08 19:00:54 +00:00
|
|
|
mlx5_system_free_irq(irq);
|
|
|
|
|
2023-04-13 19:15:31 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void mlx5_irq_pools_free_irqs(struct mlx5_irq_table *table)
|
|
|
|
{
|
|
|
|
if (table->sf_ctrl_pool) {
|
|
|
|
mlx5_irq_pool_free_irqs(table->sf_comp_pool);
|
|
|
|
mlx5_irq_pool_free_irqs(table->sf_ctrl_pool);
|
|
|
|
}
|
|
|
|
mlx5_irq_pool_free_irqs(table->pcif_pool);
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:48:17 +00:00
|
|
|
/* irq_table API */
|
|
|
|
|
|
|
|
int mlx5_irq_table_init(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_table *irq_table;
|
|
|
|
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return 0;
|
|
|
|
|
2020-12-30 09:59:55 +00:00
|
|
|
irq_table = kvzalloc_node(sizeof(*irq_table), GFP_KERNEL,
|
|
|
|
dev->priv.numa_node);
|
2021-02-23 09:48:17 +00:00
|
|
|
if (!irq_table)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
dev->priv.irq_table = irq_table;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
void mlx5_irq_table_cleanup(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return;
|
|
|
|
|
|
|
|
kvfree(dev->priv.irq_table);
|
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_irq_table_get_num_comp(struct mlx5_irq_table *table)
|
|
|
|
{
|
2023-01-01 07:22:34 +00:00
|
|
|
if (!table->pcif_pool->xa_num_irqs.max)
|
2021-08-01 09:08:49 +00:00
|
|
|
return 1;
|
2023-01-01 07:22:34 +00:00
|
|
|
return table->pcif_pool->xa_num_irqs.max - table->pcif_pool->xa_num_irqs.min;
|
2019-06-10 23:38:37 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
int mlx5_irq_table_create(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
int num_eqs = MLX5_CAP_GEN(dev, max_num_eqs) ?
|
|
|
|
MLX5_CAP_GEN(dev, max_num_eqs) :
|
|
|
|
1 << MLX5_CAP_GEN(dev, log_max_eq);
|
2021-02-23 09:48:17 +00:00
|
|
|
int total_vec;
|
2023-01-01 07:22:34 +00:00
|
|
|
int pcif_vec;
|
2023-01-01 06:16:23 +00:00
|
|
|
int req_vec;
|
2019-06-10 23:38:37 +00:00
|
|
|
int err;
|
2023-01-01 06:16:23 +00:00
|
|
|
int n;
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2020-12-12 06:12:18 +00:00
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return 0;
|
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
pcif_vec = MLX5_CAP_GEN(dev, num_ports) * num_online_cpus() + 1;
|
|
|
|
pcif_vec = min_t(int, pcif_vec, num_eqs);
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
total_vec = pcif_vec;
|
2021-02-23 09:48:17 +00:00
|
|
|
if (mlx5_sf_max_functions(dev))
|
|
|
|
total_vec += MLX5_IRQ_CTRL_SF_MAX +
|
|
|
|
MLX5_COMP_EQS_PER_SF * mlx5_sf_max_functions(dev);
|
2023-01-01 06:16:23 +00:00
|
|
|
total_vec = min_t(int, total_vec, pci_msix_vec_count(dev->pdev));
|
|
|
|
pcif_vec = min_t(int, pcif_vec, pci_msix_vec_count(dev->pdev));
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2023-01-01 06:16:23 +00:00
|
|
|
req_vec = pci_msix_can_alloc_dyn(dev->pdev) ? 1 : total_vec;
|
|
|
|
n = pci_alloc_irq_vectors(dev->pdev, 1, req_vec, PCI_IRQ_MSIX);
|
|
|
|
if (n < 0)
|
|
|
|
return n;
|
2019-06-10 23:38:37 +00:00
|
|
|
|
2023-01-01 07:22:34 +00:00
|
|
|
err = irq_pools_init(dev, total_vec - pcif_vec, pcif_vec);
|
2021-02-23 09:48:17 +00:00
|
|
|
if (err)
|
|
|
|
pci_free_irq_vectors(dev->pdev);
|
2019-06-10 23:38:37 +00:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
void mlx5_irq_table_destroy(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_table *table = dev->priv.irq_table;
|
|
|
|
|
2020-12-12 06:12:18 +00:00
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return;
|
|
|
|
|
2021-02-23 09:38:52 +00:00
|
|
|
/* There are cases where IRQs still will be in used when we reaching
|
2021-07-30 03:03:00 +00:00
|
|
|
* to here. Hence, making sure all the irqs are released.
|
2021-02-23 09:38:52 +00:00
|
|
|
*/
|
2021-02-23 09:48:17 +00:00
|
|
|
irq_pools_destroy(table);
|
2019-06-10 23:38:37 +00:00
|
|
|
pci_free_irq_vectors(dev->pdev);
|
|
|
|
}
|
|
|
|
|
2023-04-13 19:15:31 +00:00
|
|
|
void mlx5_irq_table_free_irqs(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
struct mlx5_irq_table *table = dev->priv.irq_table;
|
|
|
|
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return;
|
|
|
|
|
|
|
|
mlx5_irq_pools_free_irqs(table);
|
|
|
|
pci_free_irq_vectors(dev->pdev);
|
|
|
|
}
|
|
|
|
|
2021-02-23 09:57:32 +00:00
|
|
|
int mlx5_irq_table_get_sfs_vec(struct mlx5_irq_table *table)
|
|
|
|
{
|
|
|
|
if (table->sf_comp_pool)
|
2021-09-14 07:13:02 +00:00
|
|
|
return min_t(int, num_online_cpus(),
|
|
|
|
table->sf_comp_pool->xa_num_irqs.max -
|
|
|
|
table->sf_comp_pool->xa_num_irqs.min + 1);
|
2021-02-23 09:57:32 +00:00
|
|
|
else
|
|
|
|
return mlx5_irq_table_get_num_comp(table);
|
|
|
|
}
|
|
|
|
|
2020-12-12 06:12:18 +00:00
|
|
|
struct mlx5_irq_table *mlx5_irq_table_get(struct mlx5_core_dev *dev)
|
|
|
|
{
|
|
|
|
#ifdef CONFIG_MLX5_SF
|
|
|
|
if (mlx5_core_is_sf(dev))
|
|
|
|
return dev->priv.parent_mdev->priv.irq_table;
|
|
|
|
#endif
|
|
|
|
return dev->priv.irq_table;
|
|
|
|
}
|