From 25d1270b6e9ea893c2fe19a4bd52ebfde22812f4 Mon Sep 17 00:00:00 2001 From: Shannon Nelson Date: Fri, 19 May 2023 14:56:27 -0700 Subject: [PATCH] pds_vdpa: get vdpa management info Find the vDPA management information from the DSC in order to advertise it to the vdpa subsystem. Signed-off-by: Shannon Nelson Acked-by: Jason Wang Message-Id: <20230519215632.12343-7-shannon.nelson@amd.com> Signed-off-by: Michael S. Tsirkin --- drivers/vdpa/pds/Makefile | 3 +- drivers/vdpa/pds/aux_drv.c | 17 ++++++ drivers/vdpa/pds/aux_drv.h | 7 +++ drivers/vdpa/pds/debugfs.c | 1 + drivers/vdpa/pds/vdpa_dev.c | 108 ++++++++++++++++++++++++++++++++++++ drivers/vdpa/pds/vdpa_dev.h | 15 +++++ 6 files changed, 150 insertions(+), 1 deletion(-) create mode 100644 drivers/vdpa/pds/vdpa_dev.c create mode 100644 drivers/vdpa/pds/vdpa_dev.h diff --git a/drivers/vdpa/pds/Makefile b/drivers/vdpa/pds/Makefile index a9cd2f450ae1..13b50394ec64 100644 --- a/drivers/vdpa/pds/Makefile +++ b/drivers/vdpa/pds/Makefile @@ -3,6 +3,7 @@ obj-$(CONFIG_PDS_VDPA) := pds_vdpa.o -pds_vdpa-y := aux_drv.o +pds_vdpa-y := aux_drv.o \ + vdpa_dev.o pds_vdpa-$(CONFIG_DEBUG_FS) += debugfs.o diff --git a/drivers/vdpa/pds/aux_drv.c b/drivers/vdpa/pds/aux_drv.c index e4a0ad61ea22..aa748cf55d2b 100644 --- a/drivers/vdpa/pds/aux_drv.c +++ b/drivers/vdpa/pds/aux_drv.c @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -11,6 +12,7 @@ #include "aux_drv.h" #include "debugfs.h" +#include "vdpa_dev.h" static const struct auxiliary_device_id pds_vdpa_id_table[] = { { .name = PDS_VDPA_DEV_NAME, }, @@ -24,15 +26,28 @@ static int pds_vdpa_probe(struct auxiliary_device *aux_dev, struct pds_auxiliary_dev *padev = container_of(aux_dev, struct pds_auxiliary_dev, aux_dev); struct pds_vdpa_aux *vdpa_aux; + int err; vdpa_aux = kzalloc(sizeof(*vdpa_aux), GFP_KERNEL); if (!vdpa_aux) return -ENOMEM; vdpa_aux->padev = padev; + vdpa_aux->vf_id = pci_iov_vf_id(padev->vf_pdev); auxiliary_set_drvdata(aux_dev, vdpa_aux); + /* Get device ident info and set up the vdpa_mgmt_dev */ + err = pds_vdpa_get_mgmt_info(vdpa_aux); + if (err) + goto err_free_mem; + return 0; + +err_free_mem: + kfree(vdpa_aux); + auxiliary_set_drvdata(aux_dev, NULL); + + return err; } static void pds_vdpa_remove(struct auxiliary_device *aux_dev) @@ -40,6 +55,8 @@ static void pds_vdpa_remove(struct auxiliary_device *aux_dev) struct pds_vdpa_aux *vdpa_aux = auxiliary_get_drvdata(aux_dev); struct device *dev = &aux_dev->dev; + pci_free_irq_vectors(vdpa_aux->padev->vf_pdev); + kfree(vdpa_aux); auxiliary_set_drvdata(aux_dev, NULL); diff --git a/drivers/vdpa/pds/aux_drv.h b/drivers/vdpa/pds/aux_drv.h index f1e99359424e..dcec782e79eb 100644 --- a/drivers/vdpa/pds/aux_drv.h +++ b/drivers/vdpa/pds/aux_drv.h @@ -10,6 +10,13 @@ struct pds_vdpa_aux { struct pds_auxiliary_dev *padev; + struct vdpa_mgmt_dev vdpa_mdev; + + struct pds_vdpa_ident ident; + + int vf_id; struct dentry *dentry; + + int nintrs; }; #endif /* _AUX_DRV_H_ */ diff --git a/drivers/vdpa/pds/debugfs.c b/drivers/vdpa/pds/debugfs.c index 5be22fb7a76a..d91dceb07380 100644 --- a/drivers/vdpa/pds/debugfs.c +++ b/drivers/vdpa/pds/debugfs.c @@ -2,6 +2,7 @@ /* Copyright(c) 2023 Advanced Micro Devices, Inc */ #include +#include #include #include diff --git a/drivers/vdpa/pds/vdpa_dev.c b/drivers/vdpa/pds/vdpa_dev.c new file mode 100644 index 000000000000..0f0f0ab8b811 --- /dev/null +++ b/drivers/vdpa/pds/vdpa_dev.c @@ -0,0 +1,108 @@ +// SPDX-License-Identifier: GPL-2.0-only +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#include +#include +#include + +#include +#include +#include +#include + +#include "vdpa_dev.h" +#include "aux_drv.h" + +static struct virtio_device_id pds_vdpa_id_table[] = { + {VIRTIO_ID_NET, VIRTIO_DEV_ANY_ID}, + {0}, +}; + +static int pds_vdpa_dev_add(struct vdpa_mgmt_dev *mdev, const char *name, + const struct vdpa_dev_set_config *add_config) +{ + return -EOPNOTSUPP; +} + +static void pds_vdpa_dev_del(struct vdpa_mgmt_dev *mdev, + struct vdpa_device *vdpa_dev) +{ +} + +static const struct vdpa_mgmtdev_ops pds_vdpa_mgmt_dev_ops = { + .dev_add = pds_vdpa_dev_add, + .dev_del = pds_vdpa_dev_del +}; + +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux) +{ + union pds_core_adminq_cmd cmd = { + .vdpa_ident.opcode = PDS_VDPA_CMD_IDENT, + .vdpa_ident.vf_id = cpu_to_le16(vdpa_aux->vf_id), + }; + union pds_core_adminq_comp comp = {}; + struct vdpa_mgmt_dev *mgmt; + struct pci_dev *pf_pdev; + struct device *pf_dev; + struct pci_dev *pdev; + dma_addr_t ident_pa; + struct device *dev; + u16 dev_intrs; + u16 max_vqs; + int err; + + dev = &vdpa_aux->padev->aux_dev.dev; + pdev = vdpa_aux->padev->vf_pdev; + mgmt = &vdpa_aux->vdpa_mdev; + + /* Get resource info through the PF's adminq. It is a block of info, + * so we need to map some memory for PF to make available to the + * firmware for writing the data. + */ + pf_pdev = pci_physfn(vdpa_aux->padev->vf_pdev); + pf_dev = &pf_pdev->dev; + ident_pa = dma_map_single(pf_dev, &vdpa_aux->ident, + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); + if (dma_mapping_error(pf_dev, ident_pa)) { + dev_err(dev, "Failed to map ident space\n"); + return -ENOMEM; + } + + cmd.vdpa_ident.ident_pa = cpu_to_le64(ident_pa); + cmd.vdpa_ident.len = cpu_to_le32(sizeof(vdpa_aux->ident)); + err = pds_client_adminq_cmd(vdpa_aux->padev, &cmd, + sizeof(cmd.vdpa_ident), &comp, 0); + dma_unmap_single(pf_dev, ident_pa, + sizeof(vdpa_aux->ident), DMA_FROM_DEVICE); + if (err) { + dev_err(dev, "Failed to ident hw, status %d: %pe\n", + comp.status, ERR_PTR(err)); + return err; + } + + max_vqs = le16_to_cpu(vdpa_aux->ident.max_vqs); + dev_intrs = pci_msix_vec_count(pdev); + dev_dbg(dev, "ident.max_vqs %d dev_intrs %d\n", max_vqs, dev_intrs); + + max_vqs = min_t(u16, dev_intrs, max_vqs); + mgmt->max_supported_vqs = min_t(u16, PDS_VDPA_MAX_QUEUES, max_vqs); + vdpa_aux->nintrs = mgmt->max_supported_vqs; + + mgmt->ops = &pds_vdpa_mgmt_dev_ops; + mgmt->id_table = pds_vdpa_id_table; + mgmt->device = dev; + mgmt->supported_features = le64_to_cpu(vdpa_aux->ident.hw_features); + mgmt->config_attr_mask = BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MACADDR); + mgmt->config_attr_mask |= BIT_ULL(VDPA_ATTR_DEV_NET_CFG_MAX_VQP); + + err = pci_alloc_irq_vectors(pdev, vdpa_aux->nintrs, vdpa_aux->nintrs, + PCI_IRQ_MSIX); + if (err < 0) { + dev_err(dev, "Couldn't get %d msix vectors: %pe\n", + vdpa_aux->nintrs, ERR_PTR(err)); + return err; + } + vdpa_aux->nintrs = err; + + return 0; +} diff --git a/drivers/vdpa/pds/vdpa_dev.h b/drivers/vdpa/pds/vdpa_dev.h new file mode 100644 index 000000000000..97fab833a0aa --- /dev/null +++ b/drivers/vdpa/pds/vdpa_dev.h @@ -0,0 +1,15 @@ +/* SPDX-License-Identifier: GPL-2.0-only */ +/* Copyright(c) 2023 Advanced Micro Devices, Inc */ + +#ifndef _VDPA_DEV_H_ +#define _VDPA_DEV_H_ + +#define PDS_VDPA_MAX_QUEUES 65 + +struct pds_vdpa_device { + struct vdpa_device vdpa_dev; + struct pds_vdpa_aux *vdpa_aux; +}; + +int pds_vdpa_get_mgmt_info(struct pds_vdpa_aux *vdpa_aux); +#endif /* _VDPA_DEV_H_ */