mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-08-23 09:19:51 +00:00
PCI/MSI: Protect MSI operations
To prepare for dynamic extension of MSI-X vectors, protect the MSI operations for MSI and MSI-X. This requires to move the invocation of irq_create_affinity_masks() out of the descriptor lock section to avoid reverse lock ordering vs. CPU hotplug lock as some callers of the PCI/MSI allocation interfaces already hold it. Signed-off-by: Thomas Gleixner <tglx@linutronix.de> Tested-by: Michael Kelley <mikelley@microsoft.com> Tested-by: Nishanth Menon <nm@ti.com> Reviewed-by: Jason Gunthorpe <jgg@nvidia.com> Acked-by: Bjorn Helgaas <bhelgaas@google.com> Link: https://lore.kernel.org/r/20211206210747.982292705@linutronix.de
This commit is contained in:
parent
645474e2ce
commit
5512c5eaf5
2 changed files with 63 additions and 41 deletions
|
@ -14,7 +14,7 @@ int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
|
||||||
|
|
||||||
domain = dev_get_msi_domain(&dev->dev);
|
domain = dev_get_msi_domain(&dev->dev);
|
||||||
if (domain && irq_domain_is_hierarchy(domain))
|
if (domain && irq_domain_is_hierarchy(domain))
|
||||||
return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
|
return msi_domain_alloc_irqs_descs_locked(domain, &dev->dev, nvec);
|
||||||
|
|
||||||
return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);
|
return pci_msi_legacy_setup_msi_irqs(dev, nvec, type);
|
||||||
}
|
}
|
||||||
|
@ -25,7 +25,7 @@ void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
|
||||||
|
|
||||||
domain = dev_get_msi_domain(&dev->dev);
|
domain = dev_get_msi_domain(&dev->dev);
|
||||||
if (domain && irq_domain_is_hierarchy(domain))
|
if (domain && irq_domain_is_hierarchy(domain))
|
||||||
msi_domain_free_irqs(domain, &dev->dev);
|
msi_domain_free_irqs_descs_locked(domain, &dev->dev);
|
||||||
else
|
else
|
||||||
pci_msi_legacy_teardown_msi_irqs(dev);
|
pci_msi_legacy_teardown_msi_irqs(dev);
|
||||||
}
|
}
|
||||||
|
|
|
@ -320,11 +320,13 @@ static void __pci_restore_msix_state(struct pci_dev *dev)
|
||||||
|
|
||||||
write_msg = arch_restore_msi_irqs(dev);
|
write_msg = arch_restore_msi_irqs(dev);
|
||||||
|
|
||||||
|
msi_lock_descs(&dev->dev);
|
||||||
for_each_pci_msi_entry(entry, dev) {
|
for_each_pci_msi_entry(entry, dev) {
|
||||||
if (write_msg)
|
if (write_msg)
|
||||||
__pci_write_msi_msg(entry, &entry->msg);
|
__pci_write_msi_msg(entry, &entry->msg);
|
||||||
pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl);
|
pci_msix_write_vector_ctrl(entry, entry->pci.msix_ctrl);
|
||||||
}
|
}
|
||||||
|
msi_unlock_descs(&dev->dev);
|
||||||
|
|
||||||
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
||||||
}
|
}
|
||||||
|
@ -375,19 +377,15 @@ static int pci_setup_msi_context(struct pci_dev *dev)
|
||||||
}
|
}
|
||||||
|
|
||||||
static struct msi_desc *
|
static struct msi_desc *
|
||||||
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
|
msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity_desc *masks)
|
||||||
{
|
{
|
||||||
struct irq_affinity_desc *masks = NULL;
|
|
||||||
struct msi_desc *entry;
|
struct msi_desc *entry;
|
||||||
u16 control;
|
u16 control;
|
||||||
|
|
||||||
if (affd)
|
|
||||||
masks = irq_create_affinity_masks(nvec, affd);
|
|
||||||
|
|
||||||
/* MSI Entry Initialization */
|
/* MSI Entry Initialization */
|
||||||
entry = alloc_msi_entry(&dev->dev, nvec, masks);
|
entry = alloc_msi_entry(&dev->dev, nvec, masks);
|
||||||
if (!entry)
|
if (!entry)
|
||||||
goto out;
|
return NULL;
|
||||||
|
|
||||||
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
||||||
/* Lies, damned lies, and MSIs */
|
/* Lies, damned lies, and MSIs */
|
||||||
|
@ -410,8 +408,7 @@ msi_setup_entry(struct pci_dev *dev, int nvec, struct irq_affinity *affd)
|
||||||
if (entry->pci.msi_attrib.can_mask)
|
if (entry->pci.msi_attrib.can_mask)
|
||||||
pci_read_config_dword(dev, entry->pci.mask_pos, &entry->pci.msi_mask);
|
pci_read_config_dword(dev, entry->pci.mask_pos, &entry->pci.msi_mask);
|
||||||
|
|
||||||
out:
|
|
||||||
kfree(masks);
|
|
||||||
return entry;
|
return entry;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -447,6 +444,7 @@ static int msi_verify_entries(struct pci_dev *dev)
|
||||||
static int msi_capability_init(struct pci_dev *dev, int nvec,
|
static int msi_capability_init(struct pci_dev *dev, int nvec,
|
||||||
struct irq_affinity *affd)
|
struct irq_affinity *affd)
|
||||||
{
|
{
|
||||||
|
struct irq_affinity_desc *masks = NULL;
|
||||||
struct msi_desc *entry;
|
struct msi_desc *entry;
|
||||||
int ret;
|
int ret;
|
||||||
|
|
||||||
|
@ -457,7 +455,11 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
|
||||||
pci_msi_set_enable(dev, 0);
|
pci_msi_set_enable(dev, 0);
|
||||||
dev->msi_enabled = 1;
|
dev->msi_enabled = 1;
|
||||||
|
|
||||||
entry = msi_setup_entry(dev, nvec, affd);
|
if (affd)
|
||||||
|
masks = irq_create_affinity_masks(nvec, affd);
|
||||||
|
|
||||||
|
msi_lock_descs(&dev->dev);
|
||||||
|
entry = msi_setup_entry(dev, nvec, masks);
|
||||||
if (!entry) {
|
if (!entry) {
|
||||||
ret = -ENOMEM;
|
ret = -ENOMEM;
|
||||||
goto fail;
|
goto fail;
|
||||||
|
@ -483,13 +485,16 @@ static int msi_capability_init(struct pci_dev *dev, int nvec,
|
||||||
|
|
||||||
pcibios_free_irq(dev);
|
pcibios_free_irq(dev);
|
||||||
dev->irq = entry->irq;
|
dev->irq = entry->irq;
|
||||||
return 0;
|
goto unlock;
|
||||||
|
|
||||||
err:
|
err:
|
||||||
pci_msi_unmask(entry, msi_multi_mask(entry));
|
pci_msi_unmask(entry, msi_multi_mask(entry));
|
||||||
free_msi_irqs(dev);
|
free_msi_irqs(dev);
|
||||||
fail:
|
fail:
|
||||||
dev->msi_enabled = 0;
|
dev->msi_enabled = 0;
|
||||||
|
unlock:
|
||||||
|
msi_unlock_descs(&dev->dev);
|
||||||
|
kfree(masks);
|
||||||
return ret;
|
return ret;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -516,23 +521,18 @@ static void __iomem *msix_map_region(struct pci_dev *dev,
|
||||||
|
|
||||||
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
|
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
|
||||||
struct msix_entry *entries, int nvec,
|
struct msix_entry *entries, int nvec,
|
||||||
struct irq_affinity *affd)
|
struct irq_affinity_desc *masks)
|
||||||
{
|
{
|
||||||
struct irq_affinity_desc *curmsk, *masks = NULL;
|
int i, vec_count = pci_msix_vec_count(dev);
|
||||||
|
struct irq_affinity_desc *curmsk;
|
||||||
struct msi_desc *entry;
|
struct msi_desc *entry;
|
||||||
void __iomem *addr;
|
void __iomem *addr;
|
||||||
int ret, i;
|
|
||||||
int vec_count = pci_msix_vec_count(dev);
|
|
||||||
|
|
||||||
if (affd)
|
|
||||||
masks = irq_create_affinity_masks(nvec, affd);
|
|
||||||
|
|
||||||
for (i = 0, curmsk = masks; i < nvec; i++) {
|
for (i = 0, curmsk = masks; i < nvec; i++) {
|
||||||
entry = alloc_msi_entry(&dev->dev, 1, curmsk);
|
entry = alloc_msi_entry(&dev->dev, 1, curmsk);
|
||||||
if (!entry) {
|
if (!entry) {
|
||||||
/* No enough memory. Don't try again */
|
/* No enough memory. Don't try again */
|
||||||
ret = -ENOMEM;
|
return -ENOMEM;
|
||||||
goto out;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
entry->pci.msi_attrib.is_msix = 1;
|
entry->pci.msi_attrib.is_msix = 1;
|
||||||
|
@ -560,10 +560,7 @@ static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
|
||||||
if (masks)
|
if (masks)
|
||||||
curmsk++;
|
curmsk++;
|
||||||
}
|
}
|
||||||
ret = 0;
|
return 0;
|
||||||
out:
|
|
||||||
kfree(masks);
|
|
||||||
return ret;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries)
|
static void msix_update_entries(struct pci_dev *dev, struct msix_entry *entries)
|
||||||
|
@ -590,6 +587,41 @@ static void msix_mask_all(void __iomem *base, int tsize)
|
||||||
writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL);
|
writel(ctrl, base + PCI_MSIX_ENTRY_VECTOR_CTRL);
|
||||||
}
|
}
|
||||||
|
|
||||||
|
static int msix_setup_interrupts(struct pci_dev *dev, void __iomem *base,
|
||||||
|
struct msix_entry *entries, int nvec,
|
||||||
|
struct irq_affinity *affd)
|
||||||
|
{
|
||||||
|
struct irq_affinity_desc *masks = NULL;
|
||||||
|
int ret;
|
||||||
|
|
||||||
|
if (affd)
|
||||||
|
masks = irq_create_affinity_masks(nvec, affd);
|
||||||
|
|
||||||
|
msi_lock_descs(&dev->dev);
|
||||||
|
ret = msix_setup_entries(dev, base, entries, nvec, masks);
|
||||||
|
if (ret)
|
||||||
|
goto out_free;
|
||||||
|
|
||||||
|
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
|
||||||
|
if (ret)
|
||||||
|
goto out_free;
|
||||||
|
|
||||||
|
/* Check if all MSI entries honor device restrictions */
|
||||||
|
ret = msi_verify_entries(dev);
|
||||||
|
if (ret)
|
||||||
|
goto out_free;
|
||||||
|
|
||||||
|
msix_update_entries(dev, entries);
|
||||||
|
goto out_unlock;
|
||||||
|
|
||||||
|
out_free:
|
||||||
|
free_msi_irqs(dev);
|
||||||
|
out_unlock:
|
||||||
|
msi_unlock_descs(&dev->dev);
|
||||||
|
kfree(masks);
|
||||||
|
return ret;
|
||||||
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* msix_capability_init - configure device's MSI-X capability
|
* msix_capability_init - configure device's MSI-X capability
|
||||||
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
||||||
|
@ -630,20 +662,9 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
|
||||||
|
|
||||||
dev->msix_base = base;
|
dev->msix_base = base;
|
||||||
|
|
||||||
ret = msix_setup_entries(dev, base, entries, nvec, affd);
|
ret = msix_setup_interrupts(dev, base, entries, nvec, affd);
|
||||||
if (ret)
|
if (ret)
|
||||||
goto out_free;
|
goto out_disable;
|
||||||
|
|
||||||
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
|
|
||||||
if (ret)
|
|
||||||
goto out_free;
|
|
||||||
|
|
||||||
/* Check if all MSI entries honor device restrictions */
|
|
||||||
ret = msi_verify_entries(dev);
|
|
||||||
if (ret)
|
|
||||||
goto out_free;
|
|
||||||
|
|
||||||
msix_update_entries(dev, entries);
|
|
||||||
|
|
||||||
/* Disable INTX */
|
/* Disable INTX */
|
||||||
pci_intx_for_msi(dev, 0);
|
pci_intx_for_msi(dev, 0);
|
||||||
|
@ -662,9 +683,6 @@ static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
|
||||||
pcibios_free_irq(dev);
|
pcibios_free_irq(dev);
|
||||||
return 0;
|
return 0;
|
||||||
|
|
||||||
out_free:
|
|
||||||
free_msi_irqs(dev);
|
|
||||||
|
|
||||||
out_disable:
|
out_disable:
|
||||||
dev->msix_enabled = 0;
|
dev->msix_enabled = 0;
|
||||||
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0);
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE, 0);
|
||||||
|
@ -773,8 +791,10 @@ void pci_disable_msi(struct pci_dev *dev)
|
||||||
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
msi_lock_descs(&dev->dev);
|
||||||
pci_msi_shutdown(dev);
|
pci_msi_shutdown(dev);
|
||||||
free_msi_irqs(dev);
|
free_msi_irqs(dev);
|
||||||
|
msi_unlock_descs(&dev->dev);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(pci_disable_msi);
|
EXPORT_SYMBOL(pci_disable_msi);
|
||||||
|
|
||||||
|
@ -860,8 +880,10 @@ void pci_disable_msix(struct pci_dev *dev)
|
||||||
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
||||||
return;
|
return;
|
||||||
|
|
||||||
|
msi_lock_descs(&dev->dev);
|
||||||
pci_msix_shutdown(dev);
|
pci_msix_shutdown(dev);
|
||||||
free_msi_irqs(dev);
|
free_msi_irqs(dev);
|
||||||
|
msi_unlock_descs(&dev->dev);
|
||||||
}
|
}
|
||||||
EXPORT_SYMBOL(pci_disable_msix);
|
EXPORT_SYMBOL(pci_disable_msix);
|
||||||
|
|
||||||
|
|
Loading…
Reference in a new issue