mirror of
https://git.kernel.org/pub/scm/linux/kernel/git/stable/linux.git
synced 2024-09-04 18:09:57 +00:00
![Linus Torvalds](/assets/img/avatar_default.png)
-----BEGIN PGP SIGNATURE----- Version: GnuPG v1 iQIcBAABAgAGBQJZsr8cAAoJEFmIoMA60/r8lXYQAKViYIRMJDD4n3NhjMeLOsnJ vwaBmWlLRjSFIEpag5kMjS1RJE17qAvmkBZnDvSNZ6cT28INkkZnVM2IW96WECVq 64MIvDijVPcvqGuWePCfWdDiSXApiDWwJuw55BOhmvV996wGy0gYgzpPY+1g0Knh XzH9IOzDL79hZleLfsxX0MLV6FGBVtOsr0jvQ04k4IgEMIxEDTlbw85rnrvzQUtc 0Vj2koaxWIESZsq7G/wiZb2n6ekaFdXO/VlVvvhmTSDLCBaJ63Hb/gfOhwMuVkS6 B3cVprNrCT0dSzWmU4ZXf+wpOyDpBexlemW/OR/6CQUkC6AUS6kQ5si1X44dbGmJ nBPh414tdlm/6V4h/A3UFPOajSGa/ZWZ/uQZPfvKs1R6WfjUerWVBfUpAzPbgjam c/mhJ19HYT1J7vFBfhekBMeY2Px3JgSJ9rNsrFl48ynAALaX5GEwdpo4aqBfscKz 4/f9fU4ysumopvCEuKD2SsJvsPKd5gMQGGtvAhXM1TxvAoQ5V4cc99qEetAPXXPf h2EqWm4ph7YP4a+n/OZBjzluHCmZJn1CntH5+//6wpUk6HnmzsftGELuO9n12cLE GGkreI3T9ctV1eOkzVVa0l0QTE1X/VLyEyKCtb9obXsDaG4Ud7uKQoZgB19DwyTJ EG76ridTolUFVV+wzJD9 =9cLP -----END PGP SIGNATURE----- Merge tag 'pci-v4.14-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci Pull PCI updates from Bjorn Helgaas: - add enhanced Downstream Port Containment support, which prints more details about Root Port Programmed I/O errors (Dongdong Liu) - add Layerscape ls1088a and ls2088a support (Hou Zhiqiang) - add MediaTek MT2712 and MT7622 support (Ryder Lee) - add MediaTek MT2712 and MT7622 MSI support (Honghui Zhang) - add Qualcom IPQ8074 support (Varadarajan Narayanan) - add R-Car r8a7743/5 device tree support (Biju Das) - add Rockchip per-lane PHY support for better power management (Shawn Lin) - fix IRQ mapping for hot-added devices by replacing the pci_fixup_irqs() boot-time design with a host bridge hook called at probe-time (Lorenzo Pieralisi, Matthew Minter) - fix race when enabling two devices that results in upstream bridge not being enabled correctly (Srinath Mannam) - fix pciehp power fault infinite loop (Keith Busch) - fix SHPC bridge MSI hotplug events by enabling bus mastering (Aleksandr Bezzubikov) - fix a VFIO issue by correcting PCIe capability sizes (Alex Williamson) - fix an INTD issue on Xilinx and possibly other drivers by unifying INTx IRQ domain support (Paul Burton) - avoid IOMMU stalls by marking AMD Stoney GPU ATS as broken (Joerg Roedel) - allow APM X-Gene device assignment to guests by adding an ACS quirk (Feng Kan) - fix driver crashes by disabling Extended Tags on Broadcom HT2100 (Extended Tags support is required for PCIe Receivers but not Requesters, and we now enable them by default when Requesters support them) (Sinan Kaya) - fix MSIs for devices that use phantom RIDs for DMA by assuming MSIs use the real Requester ID (not a phantom RID) (Robin Murphy) - prevent assignment of Intel VMD children to guests (which may be supported eventually, but isn't yet) by not associating an IOMMU with them (Jon Derrick) - fix Intel VMD suspend/resume by releasing IRQs on suspend (Scott Bauer) - fix a Function-Level Reset issue with Intel 750 NVMe by waiting longer (up to 60sec instead of 1sec) for device to become ready (Sinan Kaya) - fix a Function-Level Reset issue on iProc Stingray by working around hardware defects in the CRS implementation (Oza Pawandeep) - fix an issue with Intel NVMe P3700 after an iProc reset by adding a delay during shutdown (Oza Pawandeep) - fix a Microsoft Hyper-V lockdep issue by polling instead of blocking in compose_msi_msg() (Stephen Hemminger) - fix a wireless LAN driver timeout by clearing DesignWare MSI interrupt status after it is handled, not before (Faiz Abbas) - fix DesignWare ATU enable checking (Jisheng Zhang) - reduce Layerscape dependencies on the bootloader by doing more initialization in the driver (Hou Zhiqiang) - improve Intel VMD performance allowing allocation of more IRQ vectors than present CPUs (Keith Busch) - improve endpoint framework support for initial DMA mask, different BAR sizes, configurable page sizes, MSI, test driver, etc (Kishon Vijay Abraham I, Stan Drozd) - rework CRS support to add periodic messages while we poll during enumeration and after Function-Level Reset and prepare for possible other uses of CRS (Sinan Kaya) - clean up Root Port AER handling by removing unnecessary code and moving error handler methods to struct pcie_port_service_driver (Christoph Hellwig) - clean up error handling paths in various drivers (Bjorn Andersson, Fabio Estevam, Gustavo A. R. Silva, Harunobu Kurokawa, Jeffy Chen, Lorenzo Pieralisi, Sergei Shtylyov) - clean up SR-IOV resource handling by disabling VF decoding before updating the corresponding resource structs (Gavin Shan) - clean up DesignWare-based drivers by unifying quirks to update Class Code and Interrupt Pin and related handling of write-protected registers (Hou Zhiqiang) - clean up by adding empty generic pcibios_align_resource() and pcibios_fixup_bus() and removing empty arch-specific implementations (Palmer Dabbelt) - request exclusive reset control for several drivers to allow cleanup elsewhere (Philipp Zabel) - constify various structures (Arvind Yadav, Bhumika Goyal) - convert from full_name() to %pOF (Rob Herring) - remove unused variables from iProc, HiSi, Altera, Keystone (Shawn Lin) * tag 'pci-v4.14-changes' of git://git.kernel.org/pub/scm/linux/kernel/git/helgaas/pci: (170 commits) PCI: xgene: Clean up whitespace PCI: xgene: Define XGENE_PCI_EXP_CAP and use generic PCI_EXP_RTCTL offset PCI: xgene: Fix platform_get_irq() error handling PCI: xilinx-nwl: Fix platform_get_irq() error handling PCI: rockchip: Fix platform_get_irq() error handling PCI: altera: Fix platform_get_irq() error handling PCI: spear13xx: Fix platform_get_irq() error handling PCI: artpec6: Fix platform_get_irq() error handling PCI: armada8k: Fix platform_get_irq() error handling PCI: dra7xx: Fix platform_get_irq() error handling PCI: exynos: Fix platform_get_irq() error handling PCI: iproc: Clean up whitespace PCI: iproc: Rename PCI_EXP_CAP to IPROC_PCI_EXP_CAP PCI: iproc: Add 500ms delay during device shutdown PCI: Fix typos and whitespace errors PCI: Remove unused "res" variable from pci_resource_io() PCI: Correct kernel-doc of pci_vpd_srdt_size(), pci_vpd_srdt_tag() PCI/AER: Reformat AER register definitions iommu/vt-d: Prevent VMD child devices from being remapping targets x86/PCI: Use is_vmd() rather than relying on the domain number ...
1522 lines
38 KiB
C
1522 lines
38 KiB
C
/*
|
|
* File: msi.c
|
|
* Purpose: PCI Message Signaled Interrupt (MSI)
|
|
*
|
|
* Copyright (C) 2003-2004 Intel
|
|
* Copyright (C) Tom Long Nguyen (tom.l.nguyen@intel.com)
|
|
* Copyright (C) 2016 Christoph Hellwig.
|
|
*/
|
|
|
|
#include <linux/err.h>
|
|
#include <linux/mm.h>
|
|
#include <linux/irq.h>
|
|
#include <linux/interrupt.h>
|
|
#include <linux/export.h>
|
|
#include <linux/ioport.h>
|
|
#include <linux/pci.h>
|
|
#include <linux/proc_fs.h>
|
|
#include <linux/msi.h>
|
|
#include <linux/smp.h>
|
|
#include <linux/errno.h>
|
|
#include <linux/io.h>
|
|
#include <linux/acpi_iort.h>
|
|
#include <linux/slab.h>
|
|
#include <linux/irqdomain.h>
|
|
#include <linux/of_irq.h>
|
|
|
|
#include "pci.h"
|
|
|
|
static int pci_msi_enable = 1;
|
|
int pci_msi_ignore_mask;
|
|
|
|
#define msix_table_size(flags) ((flags & PCI_MSIX_FLAGS_QSIZE) + 1)
|
|
|
|
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
|
|
static int pci_msi_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
|
|
{
|
|
struct irq_domain *domain;
|
|
|
|
domain = dev_get_msi_domain(&dev->dev);
|
|
if (domain && irq_domain_is_hierarchy(domain))
|
|
return msi_domain_alloc_irqs(domain, &dev->dev, nvec);
|
|
|
|
return arch_setup_msi_irqs(dev, nvec, type);
|
|
}
|
|
|
|
static void pci_msi_teardown_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
struct irq_domain *domain;
|
|
|
|
domain = dev_get_msi_domain(&dev->dev);
|
|
if (domain && irq_domain_is_hierarchy(domain))
|
|
msi_domain_free_irqs(domain, &dev->dev);
|
|
else
|
|
arch_teardown_msi_irqs(dev);
|
|
}
|
|
#else
|
|
#define pci_msi_setup_msi_irqs arch_setup_msi_irqs
|
|
#define pci_msi_teardown_msi_irqs arch_teardown_msi_irqs
|
|
#endif
|
|
|
|
/* Arch hooks */
|
|
|
|
int __weak arch_setup_msi_irq(struct pci_dev *dev, struct msi_desc *desc)
|
|
{
|
|
struct msi_controller *chip = dev->bus->msi;
|
|
int err;
|
|
|
|
if (!chip || !chip->setup_irq)
|
|
return -EINVAL;
|
|
|
|
err = chip->setup_irq(chip, dev, desc);
|
|
if (err < 0)
|
|
return err;
|
|
|
|
irq_set_chip_data(desc->irq, chip);
|
|
|
|
return 0;
|
|
}
|
|
|
|
void __weak arch_teardown_msi_irq(unsigned int irq)
|
|
{
|
|
struct msi_controller *chip = irq_get_chip_data(irq);
|
|
|
|
if (!chip || !chip->teardown_irq)
|
|
return;
|
|
|
|
chip->teardown_irq(chip, irq);
|
|
}
|
|
|
|
int __weak arch_setup_msi_irqs(struct pci_dev *dev, int nvec, int type)
|
|
{
|
|
struct msi_controller *chip = dev->bus->msi;
|
|
struct msi_desc *entry;
|
|
int ret;
|
|
|
|
if (chip && chip->setup_irqs)
|
|
return chip->setup_irqs(chip, dev, nvec, type);
|
|
/*
|
|
* If an architecture wants to support multiple MSI, it needs to
|
|
* override arch_setup_msi_irqs()
|
|
*/
|
|
if (type == PCI_CAP_ID_MSI && nvec > 1)
|
|
return 1;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
ret = arch_setup_msi_irq(dev, entry);
|
|
if (ret < 0)
|
|
return ret;
|
|
if (ret > 0)
|
|
return -ENOSPC;
|
|
}
|
|
|
|
return 0;
|
|
}
|
|
|
|
/*
|
|
* We have a default implementation available as a separate non-weak
|
|
* function, as it is used by the Xen x86 PCI code
|
|
*/
|
|
void default_teardown_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
int i;
|
|
struct msi_desc *entry;
|
|
|
|
for_each_pci_msi_entry(entry, dev)
|
|
if (entry->irq)
|
|
for (i = 0; i < entry->nvec_used; i++)
|
|
arch_teardown_msi_irq(entry->irq + i);
|
|
}
|
|
|
|
void __weak arch_teardown_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
return default_teardown_msi_irqs(dev);
|
|
}
|
|
|
|
static void default_restore_msi_irq(struct pci_dev *dev, int irq)
|
|
{
|
|
struct msi_desc *entry;
|
|
|
|
entry = NULL;
|
|
if (dev->msix_enabled) {
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (irq == entry->irq)
|
|
break;
|
|
}
|
|
} else if (dev->msi_enabled) {
|
|
entry = irq_get_msi_desc(irq);
|
|
}
|
|
|
|
if (entry)
|
|
__pci_write_msi_msg(entry, &entry->msg);
|
|
}
|
|
|
|
void __weak arch_restore_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
return default_restore_msi_irqs(dev);
|
|
}
|
|
|
|
static inline __attribute_const__ u32 msi_mask(unsigned x)
|
|
{
|
|
/* Don't shift by >= width of type */
|
|
if (x >= 5)
|
|
return 0xffffffff;
|
|
return (1 << (1 << x)) - 1;
|
|
}
|
|
|
|
/*
|
|
* PCI 2.3 does not specify mask bits for each MSI interrupt. Attempting to
|
|
* mask all MSI interrupts by clearing the MSI enable bit does not work
|
|
* reliably as devices without an INTx disable bit will then generate a
|
|
* level IRQ which will never be cleared.
|
|
*/
|
|
u32 __pci_msi_desc_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
|
|
{
|
|
u32 mask_bits = desc->masked;
|
|
|
|
if (pci_msi_ignore_mask || !desc->msi_attrib.maskbit)
|
|
return 0;
|
|
|
|
mask_bits &= ~mask;
|
|
mask_bits |= flag;
|
|
pci_write_config_dword(msi_desc_to_pci_dev(desc), desc->mask_pos,
|
|
mask_bits);
|
|
|
|
return mask_bits;
|
|
}
|
|
|
|
static void msi_mask_irq(struct msi_desc *desc, u32 mask, u32 flag)
|
|
{
|
|
desc->masked = __pci_msi_desc_mask_irq(desc, mask, flag);
|
|
}
|
|
|
|
static void __iomem *pci_msix_desc_addr(struct msi_desc *desc)
|
|
{
|
|
return desc->mask_base +
|
|
desc->msi_attrib.entry_nr * PCI_MSIX_ENTRY_SIZE;
|
|
}
|
|
|
|
/*
|
|
* This internal function does not flush PCI writes to the device.
|
|
* All users must ensure that they read from the device before either
|
|
* assuming that the device state is up to date, or returning out of this
|
|
* file. This saves a few milliseconds when initialising devices with lots
|
|
* of MSI-X interrupts.
|
|
*/
|
|
u32 __pci_msix_desc_mask_irq(struct msi_desc *desc, u32 flag)
|
|
{
|
|
u32 mask_bits = desc->masked;
|
|
|
|
if (pci_msi_ignore_mask)
|
|
return 0;
|
|
|
|
mask_bits &= ~PCI_MSIX_ENTRY_CTRL_MASKBIT;
|
|
if (flag)
|
|
mask_bits |= PCI_MSIX_ENTRY_CTRL_MASKBIT;
|
|
writel(mask_bits, pci_msix_desc_addr(desc) + PCI_MSIX_ENTRY_VECTOR_CTRL);
|
|
|
|
return mask_bits;
|
|
}
|
|
|
|
static void msix_mask_irq(struct msi_desc *desc, u32 flag)
|
|
{
|
|
desc->masked = __pci_msix_desc_mask_irq(desc, flag);
|
|
}
|
|
|
|
static void msi_set_mask_bit(struct irq_data *data, u32 flag)
|
|
{
|
|
struct msi_desc *desc = irq_data_get_msi_desc(data);
|
|
|
|
if (desc->msi_attrib.is_msix) {
|
|
msix_mask_irq(desc, flag);
|
|
readl(desc->mask_base); /* Flush write to device */
|
|
} else {
|
|
unsigned offset = data->irq - desc->irq;
|
|
msi_mask_irq(desc, 1 << offset, flag << offset);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* pci_msi_mask_irq - Generic irq chip callback to mask PCI/MSI interrupts
|
|
* @data: pointer to irqdata associated to that interrupt
|
|
*/
|
|
void pci_msi_mask_irq(struct irq_data *data)
|
|
{
|
|
msi_set_mask_bit(data, 1);
|
|
}
|
|
EXPORT_SYMBOL_GPL(pci_msi_mask_irq);
|
|
|
|
/**
|
|
* pci_msi_unmask_irq - Generic irq chip callback to unmask PCI/MSI interrupts
|
|
* @data: pointer to irqdata associated to that interrupt
|
|
*/
|
|
void pci_msi_unmask_irq(struct irq_data *data)
|
|
{
|
|
msi_set_mask_bit(data, 0);
|
|
}
|
|
EXPORT_SYMBOL_GPL(pci_msi_unmask_irq);
|
|
|
|
void default_restore_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
struct msi_desc *entry;
|
|
|
|
for_each_pci_msi_entry(entry, dev)
|
|
default_restore_msi_irq(dev, entry->irq);
|
|
}
|
|
|
|
void __pci_read_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
|
|
{
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(entry);
|
|
|
|
BUG_ON(dev->current_state != PCI_D0);
|
|
|
|
if (entry->msi_attrib.is_msix) {
|
|
void __iomem *base = pci_msix_desc_addr(entry);
|
|
|
|
msg->address_lo = readl(base + PCI_MSIX_ENTRY_LOWER_ADDR);
|
|
msg->address_hi = readl(base + PCI_MSIX_ENTRY_UPPER_ADDR);
|
|
msg->data = readl(base + PCI_MSIX_ENTRY_DATA);
|
|
} else {
|
|
int pos = dev->msi_cap;
|
|
u16 data;
|
|
|
|
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
|
|
&msg->address_lo);
|
|
if (entry->msi_attrib.is_64) {
|
|
pci_read_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
|
|
&msg->address_hi);
|
|
pci_read_config_word(dev, pos + PCI_MSI_DATA_64, &data);
|
|
} else {
|
|
msg->address_hi = 0;
|
|
pci_read_config_word(dev, pos + PCI_MSI_DATA_32, &data);
|
|
}
|
|
msg->data = data;
|
|
}
|
|
}
|
|
|
|
void __pci_write_msi_msg(struct msi_desc *entry, struct msi_msg *msg)
|
|
{
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(entry);
|
|
|
|
if (dev->current_state != PCI_D0 || pci_dev_is_disconnected(dev)) {
|
|
/* Don't touch the hardware now */
|
|
} else if (entry->msi_attrib.is_msix) {
|
|
void __iomem *base = pci_msix_desc_addr(entry);
|
|
|
|
writel(msg->address_lo, base + PCI_MSIX_ENTRY_LOWER_ADDR);
|
|
writel(msg->address_hi, base + PCI_MSIX_ENTRY_UPPER_ADDR);
|
|
writel(msg->data, base + PCI_MSIX_ENTRY_DATA);
|
|
} else {
|
|
int pos = dev->msi_cap;
|
|
u16 msgctl;
|
|
|
|
pci_read_config_word(dev, pos + PCI_MSI_FLAGS, &msgctl);
|
|
msgctl &= ~PCI_MSI_FLAGS_QSIZE;
|
|
msgctl |= entry->msi_attrib.multiple << 4;
|
|
pci_write_config_word(dev, pos + PCI_MSI_FLAGS, msgctl);
|
|
|
|
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_LO,
|
|
msg->address_lo);
|
|
if (entry->msi_attrib.is_64) {
|
|
pci_write_config_dword(dev, pos + PCI_MSI_ADDRESS_HI,
|
|
msg->address_hi);
|
|
pci_write_config_word(dev, pos + PCI_MSI_DATA_64,
|
|
msg->data);
|
|
} else {
|
|
pci_write_config_word(dev, pos + PCI_MSI_DATA_32,
|
|
msg->data);
|
|
}
|
|
}
|
|
entry->msg = *msg;
|
|
}
|
|
|
|
void pci_write_msi_msg(unsigned int irq, struct msi_msg *msg)
|
|
{
|
|
struct msi_desc *entry = irq_get_msi_desc(irq);
|
|
|
|
__pci_write_msi_msg(entry, msg);
|
|
}
|
|
EXPORT_SYMBOL_GPL(pci_write_msi_msg);
|
|
|
|
static void free_msi_irqs(struct pci_dev *dev)
|
|
{
|
|
struct list_head *msi_list = dev_to_msi_list(&dev->dev);
|
|
struct msi_desc *entry, *tmp;
|
|
struct attribute **msi_attrs;
|
|
struct device_attribute *dev_attr;
|
|
int i, count = 0;
|
|
|
|
for_each_pci_msi_entry(entry, dev)
|
|
if (entry->irq)
|
|
for (i = 0; i < entry->nvec_used; i++)
|
|
BUG_ON(irq_has_action(entry->irq + i));
|
|
|
|
pci_msi_teardown_msi_irqs(dev);
|
|
|
|
list_for_each_entry_safe(entry, tmp, msi_list, list) {
|
|
if (entry->msi_attrib.is_msix) {
|
|
if (list_is_last(&entry->list, msi_list))
|
|
iounmap(entry->mask_base);
|
|
}
|
|
|
|
list_del(&entry->list);
|
|
free_msi_entry(entry);
|
|
}
|
|
|
|
if (dev->msi_irq_groups) {
|
|
sysfs_remove_groups(&dev->dev.kobj, dev->msi_irq_groups);
|
|
msi_attrs = dev->msi_irq_groups[0]->attrs;
|
|
while (msi_attrs[count]) {
|
|
dev_attr = container_of(msi_attrs[count],
|
|
struct device_attribute, attr);
|
|
kfree(dev_attr->attr.name);
|
|
kfree(dev_attr);
|
|
++count;
|
|
}
|
|
kfree(msi_attrs);
|
|
kfree(dev->msi_irq_groups[0]);
|
|
kfree(dev->msi_irq_groups);
|
|
dev->msi_irq_groups = NULL;
|
|
}
|
|
}
|
|
|
|
static void pci_intx_for_msi(struct pci_dev *dev, int enable)
|
|
{
|
|
if (!(dev->dev_flags & PCI_DEV_FLAGS_MSI_INTX_DISABLE_BUG))
|
|
pci_intx(dev, enable);
|
|
}
|
|
|
|
static void __pci_restore_msi_state(struct pci_dev *dev)
|
|
{
|
|
u16 control;
|
|
struct msi_desc *entry;
|
|
|
|
if (!dev->msi_enabled)
|
|
return;
|
|
|
|
entry = irq_get_msi_desc(dev->irq);
|
|
|
|
pci_intx_for_msi(dev, 0);
|
|
pci_msi_set_enable(dev, 0);
|
|
arch_restore_msi_irqs(dev);
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
|
msi_mask_irq(entry, msi_mask(entry->msi_attrib.multi_cap),
|
|
entry->masked);
|
|
control &= ~PCI_MSI_FLAGS_QSIZE;
|
|
control |= (entry->msi_attrib.multiple << 4) | PCI_MSI_FLAGS_ENABLE;
|
|
pci_write_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, control);
|
|
}
|
|
|
|
static void __pci_restore_msix_state(struct pci_dev *dev)
|
|
{
|
|
struct msi_desc *entry;
|
|
|
|
if (!dev->msix_enabled)
|
|
return;
|
|
BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
|
|
|
|
/* route the table */
|
|
pci_intx_for_msi(dev, 0);
|
|
pci_msix_clear_and_set_ctrl(dev, 0,
|
|
PCI_MSIX_FLAGS_ENABLE | PCI_MSIX_FLAGS_MASKALL);
|
|
|
|
arch_restore_msi_irqs(dev);
|
|
for_each_pci_msi_entry(entry, dev)
|
|
msix_mask_irq(entry, entry->masked);
|
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
|
}
|
|
|
|
void pci_restore_msi_state(struct pci_dev *dev)
|
|
{
|
|
__pci_restore_msi_state(dev);
|
|
__pci_restore_msix_state(dev);
|
|
}
|
|
EXPORT_SYMBOL_GPL(pci_restore_msi_state);
|
|
|
|
static ssize_t msi_mode_show(struct device *dev, struct device_attribute *attr,
|
|
char *buf)
|
|
{
|
|
struct msi_desc *entry;
|
|
unsigned long irq;
|
|
int retval;
|
|
|
|
retval = kstrtoul(attr->attr.name, 10, &irq);
|
|
if (retval)
|
|
return retval;
|
|
|
|
entry = irq_get_msi_desc(irq);
|
|
if (entry)
|
|
return sprintf(buf, "%s\n",
|
|
entry->msi_attrib.is_msix ? "msix" : "msi");
|
|
|
|
return -ENODEV;
|
|
}
|
|
|
|
static int populate_msi_sysfs(struct pci_dev *pdev)
|
|
{
|
|
struct attribute **msi_attrs;
|
|
struct attribute *msi_attr;
|
|
struct device_attribute *msi_dev_attr;
|
|
struct attribute_group *msi_irq_group;
|
|
const struct attribute_group **msi_irq_groups;
|
|
struct msi_desc *entry;
|
|
int ret = -ENOMEM;
|
|
int num_msi = 0;
|
|
int count = 0;
|
|
int i;
|
|
|
|
/* Determine how many msi entries we have */
|
|
for_each_pci_msi_entry(entry, pdev)
|
|
num_msi += entry->nvec_used;
|
|
if (!num_msi)
|
|
return 0;
|
|
|
|
/* Dynamically create the MSI attributes for the PCI device */
|
|
msi_attrs = kzalloc(sizeof(void *) * (num_msi + 1), GFP_KERNEL);
|
|
if (!msi_attrs)
|
|
return -ENOMEM;
|
|
for_each_pci_msi_entry(entry, pdev) {
|
|
for (i = 0; i < entry->nvec_used; i++) {
|
|
msi_dev_attr = kzalloc(sizeof(*msi_dev_attr), GFP_KERNEL);
|
|
if (!msi_dev_attr)
|
|
goto error_attrs;
|
|
msi_attrs[count] = &msi_dev_attr->attr;
|
|
|
|
sysfs_attr_init(&msi_dev_attr->attr);
|
|
msi_dev_attr->attr.name = kasprintf(GFP_KERNEL, "%d",
|
|
entry->irq + i);
|
|
if (!msi_dev_attr->attr.name)
|
|
goto error_attrs;
|
|
msi_dev_attr->attr.mode = S_IRUGO;
|
|
msi_dev_attr->show = msi_mode_show;
|
|
++count;
|
|
}
|
|
}
|
|
|
|
msi_irq_group = kzalloc(sizeof(*msi_irq_group), GFP_KERNEL);
|
|
if (!msi_irq_group)
|
|
goto error_attrs;
|
|
msi_irq_group->name = "msi_irqs";
|
|
msi_irq_group->attrs = msi_attrs;
|
|
|
|
msi_irq_groups = kzalloc(sizeof(void *) * 2, GFP_KERNEL);
|
|
if (!msi_irq_groups)
|
|
goto error_irq_group;
|
|
msi_irq_groups[0] = msi_irq_group;
|
|
|
|
ret = sysfs_create_groups(&pdev->dev.kobj, msi_irq_groups);
|
|
if (ret)
|
|
goto error_irq_groups;
|
|
pdev->msi_irq_groups = msi_irq_groups;
|
|
|
|
return 0;
|
|
|
|
error_irq_groups:
|
|
kfree(msi_irq_groups);
|
|
error_irq_group:
|
|
kfree(msi_irq_group);
|
|
error_attrs:
|
|
count = 0;
|
|
msi_attr = msi_attrs[count];
|
|
while (msi_attr) {
|
|
msi_dev_attr = container_of(msi_attr, struct device_attribute, attr);
|
|
kfree(msi_attr->name);
|
|
kfree(msi_dev_attr);
|
|
++count;
|
|
msi_attr = msi_attrs[count];
|
|
}
|
|
kfree(msi_attrs);
|
|
return ret;
|
|
}
|
|
|
|
static struct msi_desc *
|
|
msi_setup_entry(struct pci_dev *dev, int nvec, const struct irq_affinity *affd)
|
|
{
|
|
struct cpumask *masks = NULL;
|
|
struct msi_desc *entry;
|
|
u16 control;
|
|
|
|
if (affd)
|
|
masks = irq_create_affinity_masks(nvec, affd);
|
|
|
|
|
|
/* MSI Entry Initialization */
|
|
entry = alloc_msi_entry(&dev->dev, nvec, masks);
|
|
if (!entry)
|
|
goto out;
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &control);
|
|
|
|
entry->msi_attrib.is_msix = 0;
|
|
entry->msi_attrib.is_64 = !!(control & PCI_MSI_FLAGS_64BIT);
|
|
entry->msi_attrib.entry_nr = 0;
|
|
entry->msi_attrib.maskbit = !!(control & PCI_MSI_FLAGS_MASKBIT);
|
|
entry->msi_attrib.default_irq = dev->irq; /* Save IOAPIC IRQ */
|
|
entry->msi_attrib.multi_cap = (control & PCI_MSI_FLAGS_QMASK) >> 1;
|
|
entry->msi_attrib.multiple = ilog2(__roundup_pow_of_two(nvec));
|
|
|
|
if (control & PCI_MSI_FLAGS_64BIT)
|
|
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_64;
|
|
else
|
|
entry->mask_pos = dev->msi_cap + PCI_MSI_MASK_32;
|
|
|
|
/* Save the initial mask status */
|
|
if (entry->msi_attrib.maskbit)
|
|
pci_read_config_dword(dev, entry->mask_pos, &entry->masked);
|
|
|
|
out:
|
|
kfree(masks);
|
|
return entry;
|
|
}
|
|
|
|
static int msi_verify_entries(struct pci_dev *dev)
|
|
{
|
|
struct msi_desc *entry;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (!dev->no_64bit_msi || !entry->msg.address_hi)
|
|
continue;
|
|
dev_err(&dev->dev, "Device has broken 64-bit MSI but arch"
|
|
" tried to assign one above 4G\n");
|
|
return -EIO;
|
|
}
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* msi_capability_init - configure device's MSI capability structure
|
|
* @dev: pointer to the pci_dev data structure of MSI device function
|
|
* @nvec: number of interrupts to allocate
|
|
* @affd: description of automatic irq affinity assignments (may be %NULL)
|
|
*
|
|
* Setup the MSI capability structure of the device with the requested
|
|
* number of interrupts. A return value of zero indicates the successful
|
|
* setup of an entry with the new MSI irq. A negative return value indicates
|
|
* an error, and a positive return value indicates the number of interrupts
|
|
* which could have been allocated.
|
|
*/
|
|
static int msi_capability_init(struct pci_dev *dev, int nvec,
|
|
const struct irq_affinity *affd)
|
|
{
|
|
struct msi_desc *entry;
|
|
int ret;
|
|
unsigned mask;
|
|
|
|
pci_msi_set_enable(dev, 0); /* Disable MSI during set up */
|
|
|
|
entry = msi_setup_entry(dev, nvec, affd);
|
|
if (!entry)
|
|
return -ENOMEM;
|
|
|
|
/* All MSIs are unmasked by default, Mask them all */
|
|
mask = msi_mask(entry->msi_attrib.multi_cap);
|
|
msi_mask_irq(entry, mask, mask);
|
|
|
|
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
|
|
|
|
/* Configure MSI capability structure */
|
|
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSI);
|
|
if (ret) {
|
|
msi_mask_irq(entry, mask, ~mask);
|
|
free_msi_irqs(dev);
|
|
return ret;
|
|
}
|
|
|
|
ret = msi_verify_entries(dev);
|
|
if (ret) {
|
|
msi_mask_irq(entry, mask, ~mask);
|
|
free_msi_irqs(dev);
|
|
return ret;
|
|
}
|
|
|
|
ret = populate_msi_sysfs(dev);
|
|
if (ret) {
|
|
msi_mask_irq(entry, mask, ~mask);
|
|
free_msi_irqs(dev);
|
|
return ret;
|
|
}
|
|
|
|
/* Set MSI enabled bits */
|
|
pci_intx_for_msi(dev, 0);
|
|
pci_msi_set_enable(dev, 1);
|
|
dev->msi_enabled = 1;
|
|
|
|
pcibios_free_irq(dev);
|
|
dev->irq = entry->irq;
|
|
return 0;
|
|
}
|
|
|
|
static void __iomem *msix_map_region(struct pci_dev *dev, unsigned nr_entries)
|
|
{
|
|
resource_size_t phys_addr;
|
|
u32 table_offset;
|
|
unsigned long flags;
|
|
u8 bir;
|
|
|
|
pci_read_config_dword(dev, dev->msix_cap + PCI_MSIX_TABLE,
|
|
&table_offset);
|
|
bir = (u8)(table_offset & PCI_MSIX_TABLE_BIR);
|
|
flags = pci_resource_flags(dev, bir);
|
|
if (!flags || (flags & IORESOURCE_UNSET))
|
|
return NULL;
|
|
|
|
table_offset &= PCI_MSIX_TABLE_OFFSET;
|
|
phys_addr = pci_resource_start(dev, bir) + table_offset;
|
|
|
|
return ioremap_nocache(phys_addr, nr_entries * PCI_MSIX_ENTRY_SIZE);
|
|
}
|
|
|
|
static int msix_setup_entries(struct pci_dev *dev, void __iomem *base,
|
|
struct msix_entry *entries, int nvec,
|
|
const struct irq_affinity *affd)
|
|
{
|
|
struct cpumask *curmsk, *masks = NULL;
|
|
struct msi_desc *entry;
|
|
int ret, i;
|
|
|
|
if (affd)
|
|
masks = irq_create_affinity_masks(nvec, affd);
|
|
|
|
for (i = 0, curmsk = masks; i < nvec; i++) {
|
|
entry = alloc_msi_entry(&dev->dev, 1, curmsk);
|
|
if (!entry) {
|
|
if (!i)
|
|
iounmap(base);
|
|
else
|
|
free_msi_irqs(dev);
|
|
/* No enough memory. Don't try again */
|
|
ret = -ENOMEM;
|
|
goto out;
|
|
}
|
|
|
|
entry->msi_attrib.is_msix = 1;
|
|
entry->msi_attrib.is_64 = 1;
|
|
if (entries)
|
|
entry->msi_attrib.entry_nr = entries[i].entry;
|
|
else
|
|
entry->msi_attrib.entry_nr = i;
|
|
entry->msi_attrib.default_irq = dev->irq;
|
|
entry->mask_base = base;
|
|
|
|
list_add_tail(&entry->list, dev_to_msi_list(&dev->dev));
|
|
if (masks)
|
|
curmsk++;
|
|
}
|
|
ret = 0;
|
|
out:
|
|
kfree(masks);
|
|
return ret;
|
|
}
|
|
|
|
static void msix_program_entries(struct pci_dev *dev,
|
|
struct msix_entry *entries)
|
|
{
|
|
struct msi_desc *entry;
|
|
int i = 0;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (entries)
|
|
entries[i++].vector = entry->irq;
|
|
entry->masked = readl(pci_msix_desc_addr(entry) +
|
|
PCI_MSIX_ENTRY_VECTOR_CTRL);
|
|
msix_mask_irq(entry, 1);
|
|
}
|
|
}
|
|
|
|
/**
|
|
* msix_capability_init - configure device's MSI-X capability
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
|
* @entries: pointer to an array of struct msix_entry entries
|
|
* @nvec: number of @entries
|
|
* @affd: Optional pointer to enable automatic affinity assignement
|
|
*
|
|
* Setup the MSI-X capability structure of device function with a
|
|
* single MSI-X irq. A return of zero indicates the successful setup of
|
|
* requested MSI-X entries with allocated irqs or non-zero for otherwise.
|
|
**/
|
|
static int msix_capability_init(struct pci_dev *dev, struct msix_entry *entries,
|
|
int nvec, const struct irq_affinity *affd)
|
|
{
|
|
int ret;
|
|
u16 control;
|
|
void __iomem *base;
|
|
|
|
/* Ensure MSI-X is disabled while it is set up */
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
|
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
|
|
/* Request & Map MSI-X table region */
|
|
base = msix_map_region(dev, msix_table_size(control));
|
|
if (!base)
|
|
return -ENOMEM;
|
|
|
|
ret = msix_setup_entries(dev, base, entries, nvec, affd);
|
|
if (ret)
|
|
return ret;
|
|
|
|
ret = pci_msi_setup_msi_irqs(dev, nvec, PCI_CAP_ID_MSIX);
|
|
if (ret)
|
|
goto out_avail;
|
|
|
|
/* Check if all MSI entries honor device restrictions */
|
|
ret = msi_verify_entries(dev);
|
|
if (ret)
|
|
goto out_free;
|
|
|
|
/*
|
|
* Some devices require MSI-X to be enabled before we can touch the
|
|
* MSI-X registers. We need to mask all the vectors to prevent
|
|
* interrupts coming in before they're fully set up.
|
|
*/
|
|
pci_msix_clear_and_set_ctrl(dev, 0,
|
|
PCI_MSIX_FLAGS_MASKALL | PCI_MSIX_FLAGS_ENABLE);
|
|
|
|
msix_program_entries(dev, entries);
|
|
|
|
ret = populate_msi_sysfs(dev);
|
|
if (ret)
|
|
goto out_free;
|
|
|
|
/* Set MSI-X enabled bits and unmask the function */
|
|
pci_intx_for_msi(dev, 0);
|
|
dev->msix_enabled = 1;
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_MASKALL, 0);
|
|
|
|
pcibios_free_irq(dev);
|
|
return 0;
|
|
|
|
out_avail:
|
|
if (ret < 0) {
|
|
/*
|
|
* If we had some success, report the number of irqs
|
|
* we succeeded in setting up.
|
|
*/
|
|
struct msi_desc *entry;
|
|
int avail = 0;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (entry->irq != 0)
|
|
avail++;
|
|
}
|
|
if (avail != 0)
|
|
ret = avail;
|
|
}
|
|
|
|
out_free:
|
|
free_msi_irqs(dev);
|
|
|
|
return ret;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_supported - check whether MSI may be enabled on a device
|
|
* @dev: pointer to the pci_dev data structure of MSI device function
|
|
* @nvec: how many MSIs have been requested ?
|
|
*
|
|
* Look at global flags, the device itself, and its parent buses
|
|
* to determine if MSI/-X are supported for the device. If MSI/-X is
|
|
* supported return 1, else return 0.
|
|
**/
|
|
static int pci_msi_supported(struct pci_dev *dev, int nvec)
|
|
{
|
|
struct pci_bus *bus;
|
|
|
|
/* MSI must be globally enabled and supported by the device */
|
|
if (!pci_msi_enable)
|
|
return 0;
|
|
|
|
if (!dev || dev->no_msi || dev->current_state != PCI_D0)
|
|
return 0;
|
|
|
|
/*
|
|
* You can't ask to have 0 or less MSIs configured.
|
|
* a) it's stupid ..
|
|
* b) the list manipulation code assumes nvec >= 1.
|
|
*/
|
|
if (nvec < 1)
|
|
return 0;
|
|
|
|
/*
|
|
* Any bridge which does NOT route MSI transactions from its
|
|
* secondary bus to its primary bus must set NO_MSI flag on
|
|
* the secondary pci_bus.
|
|
* We expect only arch-specific PCI host bus controller driver
|
|
* or quirks for specific PCI bridges to be setting NO_MSI.
|
|
*/
|
|
for (bus = dev->bus; bus; bus = bus->parent)
|
|
if (bus->bus_flags & PCI_BUS_FLAGS_NO_MSI)
|
|
return 0;
|
|
|
|
return 1;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_vec_count - Return the number of MSI vectors a device can send
|
|
* @dev: device to report about
|
|
*
|
|
* This function returns the number of MSI vectors a device requested via
|
|
* Multiple Message Capable register. It returns a negative errno if the
|
|
* device is not capable sending MSI interrupts. Otherwise, the call succeeds
|
|
* and returns a power of two, up to a maximum of 2^5 (32), according to the
|
|
* MSI specification.
|
|
**/
|
|
int pci_msi_vec_count(struct pci_dev *dev)
|
|
{
|
|
int ret;
|
|
u16 msgctl;
|
|
|
|
if (!dev->msi_cap)
|
|
return -EINVAL;
|
|
|
|
pci_read_config_word(dev, dev->msi_cap + PCI_MSI_FLAGS, &msgctl);
|
|
ret = 1 << ((msgctl & PCI_MSI_FLAGS_QMASK) >> 1);
|
|
|
|
return ret;
|
|
}
|
|
EXPORT_SYMBOL(pci_msi_vec_count);
|
|
|
|
static void pci_msi_shutdown(struct pci_dev *dev)
|
|
{
|
|
struct msi_desc *desc;
|
|
u32 mask;
|
|
|
|
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
|
return;
|
|
|
|
BUG_ON(list_empty(dev_to_msi_list(&dev->dev)));
|
|
desc = first_pci_msi_entry(dev);
|
|
|
|
pci_msi_set_enable(dev, 0);
|
|
pci_intx_for_msi(dev, 1);
|
|
dev->msi_enabled = 0;
|
|
|
|
/* Return the device with MSI unmasked as initial states */
|
|
mask = msi_mask(desc->msi_attrib.multi_cap);
|
|
/* Keep cached state to be restored */
|
|
__pci_msi_desc_mask_irq(desc, mask, ~mask);
|
|
|
|
/* Restore dev->irq to its default pin-assertion irq */
|
|
dev->irq = desc->msi_attrib.default_irq;
|
|
pcibios_alloc_irq(dev);
|
|
}
|
|
|
|
void pci_disable_msi(struct pci_dev *dev)
|
|
{
|
|
if (!pci_msi_enable || !dev || !dev->msi_enabled)
|
|
return;
|
|
|
|
pci_msi_shutdown(dev);
|
|
free_msi_irqs(dev);
|
|
}
|
|
EXPORT_SYMBOL(pci_disable_msi);
|
|
|
|
/**
|
|
* pci_msix_vec_count - return the number of device's MSI-X table entries
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
|
* This function returns the number of device's MSI-X table entries and
|
|
* therefore the number of MSI-X vectors device is capable of sending.
|
|
* It returns a negative errno if the device is not capable of sending MSI-X
|
|
* interrupts.
|
|
**/
|
|
int pci_msix_vec_count(struct pci_dev *dev)
|
|
{
|
|
u16 control;
|
|
|
|
if (!dev->msix_cap)
|
|
return -EINVAL;
|
|
|
|
pci_read_config_word(dev, dev->msix_cap + PCI_MSIX_FLAGS, &control);
|
|
return msix_table_size(control);
|
|
}
|
|
EXPORT_SYMBOL(pci_msix_vec_count);
|
|
|
|
static int __pci_enable_msix(struct pci_dev *dev, struct msix_entry *entries,
|
|
int nvec, const struct irq_affinity *affd)
|
|
{
|
|
int nr_entries;
|
|
int i, j;
|
|
|
|
if (!pci_msi_supported(dev, nvec))
|
|
return -EINVAL;
|
|
|
|
nr_entries = pci_msix_vec_count(dev);
|
|
if (nr_entries < 0)
|
|
return nr_entries;
|
|
if (nvec > nr_entries)
|
|
return nr_entries;
|
|
|
|
if (entries) {
|
|
/* Check for any invalid entries */
|
|
for (i = 0; i < nvec; i++) {
|
|
if (entries[i].entry >= nr_entries)
|
|
return -EINVAL; /* invalid entry */
|
|
for (j = i + 1; j < nvec; j++) {
|
|
if (entries[i].entry == entries[j].entry)
|
|
return -EINVAL; /* duplicate entry */
|
|
}
|
|
}
|
|
}
|
|
WARN_ON(!!dev->msix_enabled);
|
|
|
|
/* Check whether driver already requested for MSI irq */
|
|
if (dev->msi_enabled) {
|
|
dev_info(&dev->dev, "can't enable MSI-X (MSI IRQ already assigned)\n");
|
|
return -EINVAL;
|
|
}
|
|
return msix_capability_init(dev, entries, nvec, affd);
|
|
}
|
|
|
|
static void pci_msix_shutdown(struct pci_dev *dev)
|
|
{
|
|
struct msi_desc *entry;
|
|
|
|
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
|
return;
|
|
|
|
if (pci_dev_is_disconnected(dev)) {
|
|
dev->msix_enabled = 0;
|
|
return;
|
|
}
|
|
|
|
/* Return the device with MSI-X masked as initial states */
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
/* Keep cached states to be restored */
|
|
__pci_msix_desc_mask_irq(entry, 1);
|
|
}
|
|
|
|
pci_msix_clear_and_set_ctrl(dev, PCI_MSIX_FLAGS_ENABLE, 0);
|
|
pci_intx_for_msi(dev, 1);
|
|
dev->msix_enabled = 0;
|
|
pcibios_alloc_irq(dev);
|
|
}
|
|
|
|
void pci_disable_msix(struct pci_dev *dev)
|
|
{
|
|
if (!pci_msi_enable || !dev || !dev->msix_enabled)
|
|
return;
|
|
|
|
pci_msix_shutdown(dev);
|
|
free_msi_irqs(dev);
|
|
}
|
|
EXPORT_SYMBOL(pci_disable_msix);
|
|
|
|
void pci_no_msi(void)
|
|
{
|
|
pci_msi_enable = 0;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_enabled - is MSI enabled?
|
|
*
|
|
* Returns true if MSI has not been disabled by the command-line option
|
|
* pci=nomsi.
|
|
**/
|
|
int pci_msi_enabled(void)
|
|
{
|
|
return pci_msi_enable;
|
|
}
|
|
EXPORT_SYMBOL(pci_msi_enabled);
|
|
|
|
static int __pci_enable_msi_range(struct pci_dev *dev, int minvec, int maxvec,
|
|
const struct irq_affinity *affd)
|
|
{
|
|
int nvec;
|
|
int rc;
|
|
|
|
if (!pci_msi_supported(dev, minvec))
|
|
return -EINVAL;
|
|
|
|
WARN_ON(!!dev->msi_enabled);
|
|
|
|
/* Check whether driver already requested MSI-X irqs */
|
|
if (dev->msix_enabled) {
|
|
dev_info(&dev->dev,
|
|
"can't enable MSI (MSI-X already enabled)\n");
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (maxvec < minvec)
|
|
return -ERANGE;
|
|
|
|
nvec = pci_msi_vec_count(dev);
|
|
if (nvec < 0)
|
|
return nvec;
|
|
if (nvec < minvec)
|
|
return -ENOSPC;
|
|
|
|
if (nvec > maxvec)
|
|
nvec = maxvec;
|
|
|
|
for (;;) {
|
|
if (affd) {
|
|
nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
|
|
if (nvec < minvec)
|
|
return -ENOSPC;
|
|
}
|
|
|
|
rc = msi_capability_init(dev, nvec, affd);
|
|
if (rc == 0)
|
|
return nvec;
|
|
|
|
if (rc < 0)
|
|
return rc;
|
|
if (rc < minvec)
|
|
return -ENOSPC;
|
|
|
|
nvec = rc;
|
|
}
|
|
}
|
|
|
|
/* deprecated, don't use */
|
|
int pci_enable_msi(struct pci_dev *dev)
|
|
{
|
|
int rc = __pci_enable_msi_range(dev, 1, 1, NULL);
|
|
if (rc < 0)
|
|
return rc;
|
|
return 0;
|
|
}
|
|
EXPORT_SYMBOL(pci_enable_msi);
|
|
|
|
static int __pci_enable_msix_range(struct pci_dev *dev,
|
|
struct msix_entry *entries, int minvec,
|
|
int maxvec, const struct irq_affinity *affd)
|
|
{
|
|
int rc, nvec = maxvec;
|
|
|
|
if (maxvec < minvec)
|
|
return -ERANGE;
|
|
|
|
for (;;) {
|
|
if (affd) {
|
|
nvec = irq_calc_affinity_vectors(minvec, nvec, affd);
|
|
if (nvec < minvec)
|
|
return -ENOSPC;
|
|
}
|
|
|
|
rc = __pci_enable_msix(dev, entries, nvec, affd);
|
|
if (rc == 0)
|
|
return nvec;
|
|
|
|
if (rc < 0)
|
|
return rc;
|
|
if (rc < minvec)
|
|
return -ENOSPC;
|
|
|
|
nvec = rc;
|
|
}
|
|
}
|
|
|
|
/**
|
|
* pci_enable_msix_range - configure device's MSI-X capability structure
|
|
* @dev: pointer to the pci_dev data structure of MSI-X device function
|
|
* @entries: pointer to an array of MSI-X entries
|
|
* @minvec: minimum number of MSI-X irqs requested
|
|
* @maxvec: maximum number of MSI-X irqs requested
|
|
*
|
|
* Setup the MSI-X capability structure of device function with a maximum
|
|
* possible number of interrupts in the range between @minvec and @maxvec
|
|
* upon its software driver call to request for MSI-X mode enabled on its
|
|
* hardware device function. It returns a negative errno if an error occurs.
|
|
* If it succeeds, it returns the actual number of interrupts allocated and
|
|
* indicates the successful configuration of MSI-X capability structure
|
|
* with new allocated MSI-X interrupts.
|
|
**/
|
|
int pci_enable_msix_range(struct pci_dev *dev, struct msix_entry *entries,
|
|
int minvec, int maxvec)
|
|
{
|
|
return __pci_enable_msix_range(dev, entries, minvec, maxvec, NULL);
|
|
}
|
|
EXPORT_SYMBOL(pci_enable_msix_range);
|
|
|
|
/**
|
|
* pci_alloc_irq_vectors_affinity - allocate multiple IRQs for a device
|
|
* @dev: PCI device to operate on
|
|
* @min_vecs: minimum number of vectors required (must be >= 1)
|
|
* @max_vecs: maximum (desired) number of vectors
|
|
* @flags: flags or quirks for the allocation
|
|
* @affd: optional description of the affinity requirements
|
|
*
|
|
* Allocate up to @max_vecs interrupt vectors for @dev, using MSI-X or MSI
|
|
* vectors if available, and fall back to a single legacy vector
|
|
* if neither is available. Return the number of vectors allocated,
|
|
* (which might be smaller than @max_vecs) if successful, or a negative
|
|
* error code on error. If less than @min_vecs interrupt vectors are
|
|
* available for @dev the function will fail with -ENOSPC.
|
|
*
|
|
* To get the Linux IRQ number used for a vector that can be passed to
|
|
* request_irq() use the pci_irq_vector() helper.
|
|
*/
|
|
int pci_alloc_irq_vectors_affinity(struct pci_dev *dev, unsigned int min_vecs,
|
|
unsigned int max_vecs, unsigned int flags,
|
|
const struct irq_affinity *affd)
|
|
{
|
|
static const struct irq_affinity msi_default_affd;
|
|
int vecs = -ENOSPC;
|
|
|
|
if (flags & PCI_IRQ_AFFINITY) {
|
|
if (!affd)
|
|
affd = &msi_default_affd;
|
|
} else {
|
|
if (WARN_ON(affd))
|
|
affd = NULL;
|
|
}
|
|
|
|
if (flags & PCI_IRQ_MSIX) {
|
|
vecs = __pci_enable_msix_range(dev, NULL, min_vecs, max_vecs,
|
|
affd);
|
|
if (vecs > 0)
|
|
return vecs;
|
|
}
|
|
|
|
if (flags & PCI_IRQ_MSI) {
|
|
vecs = __pci_enable_msi_range(dev, min_vecs, max_vecs, affd);
|
|
if (vecs > 0)
|
|
return vecs;
|
|
}
|
|
|
|
/* use legacy irq if allowed */
|
|
if (flags & PCI_IRQ_LEGACY) {
|
|
if (min_vecs == 1 && dev->irq) {
|
|
pci_intx(dev, 1);
|
|
return 1;
|
|
}
|
|
}
|
|
|
|
return vecs;
|
|
}
|
|
EXPORT_SYMBOL(pci_alloc_irq_vectors_affinity);
|
|
|
|
/**
|
|
* pci_free_irq_vectors - free previously allocated IRQs for a device
|
|
* @dev: PCI device to operate on
|
|
*
|
|
* Undoes the allocations and enabling in pci_alloc_irq_vectors().
|
|
*/
|
|
void pci_free_irq_vectors(struct pci_dev *dev)
|
|
{
|
|
pci_disable_msix(dev);
|
|
pci_disable_msi(dev);
|
|
}
|
|
EXPORT_SYMBOL(pci_free_irq_vectors);
|
|
|
|
/**
|
|
* pci_irq_vector - return Linux IRQ number of a device vector
|
|
* @dev: PCI device to operate on
|
|
* @nr: device-relative interrupt vector index (0-based).
|
|
*/
|
|
int pci_irq_vector(struct pci_dev *dev, unsigned int nr)
|
|
{
|
|
if (dev->msix_enabled) {
|
|
struct msi_desc *entry;
|
|
int i = 0;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (i == nr)
|
|
return entry->irq;
|
|
i++;
|
|
}
|
|
WARN_ON_ONCE(1);
|
|
return -EINVAL;
|
|
}
|
|
|
|
if (dev->msi_enabled) {
|
|
struct msi_desc *entry = first_pci_msi_entry(dev);
|
|
|
|
if (WARN_ON_ONCE(nr >= entry->nvec_used))
|
|
return -EINVAL;
|
|
} else {
|
|
if (WARN_ON_ONCE(nr > 0))
|
|
return -EINVAL;
|
|
}
|
|
|
|
return dev->irq + nr;
|
|
}
|
|
EXPORT_SYMBOL(pci_irq_vector);
|
|
|
|
/**
|
|
* pci_irq_get_affinity - return the affinity of a particular msi vector
|
|
* @dev: PCI device to operate on
|
|
* @nr: device-relative interrupt vector index (0-based).
|
|
*/
|
|
const struct cpumask *pci_irq_get_affinity(struct pci_dev *dev, int nr)
|
|
{
|
|
if (dev->msix_enabled) {
|
|
struct msi_desc *entry;
|
|
int i = 0;
|
|
|
|
for_each_pci_msi_entry(entry, dev) {
|
|
if (i == nr)
|
|
return entry->affinity;
|
|
i++;
|
|
}
|
|
WARN_ON_ONCE(1);
|
|
return NULL;
|
|
} else if (dev->msi_enabled) {
|
|
struct msi_desc *entry = first_pci_msi_entry(dev);
|
|
|
|
if (WARN_ON_ONCE(!entry || !entry->affinity ||
|
|
nr >= entry->nvec_used))
|
|
return NULL;
|
|
|
|
return &entry->affinity[nr];
|
|
} else {
|
|
return cpu_possible_mask;
|
|
}
|
|
}
|
|
EXPORT_SYMBOL(pci_irq_get_affinity);
|
|
|
|
/**
|
|
* pci_irq_get_node - return the numa node of a particular msi vector
|
|
* @pdev: PCI device to operate on
|
|
* @vec: device-relative interrupt vector index (0-based).
|
|
*/
|
|
int pci_irq_get_node(struct pci_dev *pdev, int vec)
|
|
{
|
|
const struct cpumask *mask;
|
|
|
|
mask = pci_irq_get_affinity(pdev, vec);
|
|
if (mask)
|
|
return local_memory_node(cpu_to_node(cpumask_first(mask)));
|
|
return dev_to_node(&pdev->dev);
|
|
}
|
|
EXPORT_SYMBOL(pci_irq_get_node);
|
|
|
|
struct pci_dev *msi_desc_to_pci_dev(struct msi_desc *desc)
|
|
{
|
|
return to_pci_dev(desc->dev);
|
|
}
|
|
EXPORT_SYMBOL(msi_desc_to_pci_dev);
|
|
|
|
void *msi_desc_to_pci_sysdata(struct msi_desc *desc)
|
|
{
|
|
struct pci_dev *dev = msi_desc_to_pci_dev(desc);
|
|
|
|
return dev->bus->sysdata;
|
|
}
|
|
EXPORT_SYMBOL_GPL(msi_desc_to_pci_sysdata);
|
|
|
|
#ifdef CONFIG_PCI_MSI_IRQ_DOMAIN
|
|
/**
|
|
* pci_msi_domain_write_msg - Helper to write MSI message to PCI config space
|
|
* @irq_data: Pointer to interrupt data of the MSI interrupt
|
|
* @msg: Pointer to the message
|
|
*/
|
|
void pci_msi_domain_write_msg(struct irq_data *irq_data, struct msi_msg *msg)
|
|
{
|
|
struct msi_desc *desc = irq_data_get_msi_desc(irq_data);
|
|
|
|
/*
|
|
* For MSI-X desc->irq is always equal to irq_data->irq. For
|
|
* MSI only the first interrupt of MULTI MSI passes the test.
|
|
*/
|
|
if (desc->irq == irq_data->irq)
|
|
__pci_write_msi_msg(desc, msg);
|
|
}
|
|
|
|
/**
|
|
* pci_msi_domain_calc_hwirq - Generate a unique ID for an MSI source
|
|
* @dev: Pointer to the PCI device
|
|
* @desc: Pointer to the msi descriptor
|
|
*
|
|
* The ID number is only used within the irqdomain.
|
|
*/
|
|
irq_hw_number_t pci_msi_domain_calc_hwirq(struct pci_dev *dev,
|
|
struct msi_desc *desc)
|
|
{
|
|
return (irq_hw_number_t)desc->msi_attrib.entry_nr |
|
|
PCI_DEVID(dev->bus->number, dev->devfn) << 11 |
|
|
(pci_domain_nr(dev->bus) & 0xFFFFFFFF) << 27;
|
|
}
|
|
|
|
static inline bool pci_msi_desc_is_multi_msi(struct msi_desc *desc)
|
|
{
|
|
return !desc->msi_attrib.is_msix && desc->nvec_used > 1;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_domain_check_cap - Verify that @domain supports the capabilities for @dev
|
|
* @domain: The interrupt domain to check
|
|
* @info: The domain info for verification
|
|
* @dev: The device to check
|
|
*
|
|
* Returns:
|
|
* 0 if the functionality is supported
|
|
* 1 if Multi MSI is requested, but the domain does not support it
|
|
* -ENOTSUPP otherwise
|
|
*/
|
|
int pci_msi_domain_check_cap(struct irq_domain *domain,
|
|
struct msi_domain_info *info, struct device *dev)
|
|
{
|
|
struct msi_desc *desc = first_pci_msi_entry(to_pci_dev(dev));
|
|
|
|
/* Special handling to support __pci_enable_msi_range() */
|
|
if (pci_msi_desc_is_multi_msi(desc) &&
|
|
!(info->flags & MSI_FLAG_MULTI_PCI_MSI))
|
|
return 1;
|
|
else if (desc->msi_attrib.is_msix && !(info->flags & MSI_FLAG_PCI_MSIX))
|
|
return -ENOTSUPP;
|
|
|
|
return 0;
|
|
}
|
|
|
|
static int pci_msi_domain_handle_error(struct irq_domain *domain,
|
|
struct msi_desc *desc, int error)
|
|
{
|
|
/* Special handling to support __pci_enable_msi_range() */
|
|
if (pci_msi_desc_is_multi_msi(desc) && error == -ENOSPC)
|
|
return 1;
|
|
|
|
return error;
|
|
}
|
|
|
|
#ifdef GENERIC_MSI_DOMAIN_OPS
|
|
static void pci_msi_domain_set_desc(msi_alloc_info_t *arg,
|
|
struct msi_desc *desc)
|
|
{
|
|
arg->desc = desc;
|
|
arg->hwirq = pci_msi_domain_calc_hwirq(msi_desc_to_pci_dev(desc),
|
|
desc);
|
|
}
|
|
#else
|
|
#define pci_msi_domain_set_desc NULL
|
|
#endif
|
|
|
|
static struct msi_domain_ops pci_msi_domain_ops_default = {
|
|
.set_desc = pci_msi_domain_set_desc,
|
|
.msi_check = pci_msi_domain_check_cap,
|
|
.handle_error = pci_msi_domain_handle_error,
|
|
};
|
|
|
|
static void pci_msi_domain_update_dom_ops(struct msi_domain_info *info)
|
|
{
|
|
struct msi_domain_ops *ops = info->ops;
|
|
|
|
if (ops == NULL) {
|
|
info->ops = &pci_msi_domain_ops_default;
|
|
} else {
|
|
if (ops->set_desc == NULL)
|
|
ops->set_desc = pci_msi_domain_set_desc;
|
|
if (ops->msi_check == NULL)
|
|
ops->msi_check = pci_msi_domain_check_cap;
|
|
if (ops->handle_error == NULL)
|
|
ops->handle_error = pci_msi_domain_handle_error;
|
|
}
|
|
}
|
|
|
|
static void pci_msi_domain_update_chip_ops(struct msi_domain_info *info)
|
|
{
|
|
struct irq_chip *chip = info->chip;
|
|
|
|
BUG_ON(!chip);
|
|
if (!chip->irq_write_msi_msg)
|
|
chip->irq_write_msi_msg = pci_msi_domain_write_msg;
|
|
if (!chip->irq_mask)
|
|
chip->irq_mask = pci_msi_mask_irq;
|
|
if (!chip->irq_unmask)
|
|
chip->irq_unmask = pci_msi_unmask_irq;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_create_irq_domain - Create a MSI interrupt domain
|
|
* @fwnode: Optional fwnode of the interrupt controller
|
|
* @info: MSI domain info
|
|
* @parent: Parent irq domain
|
|
*
|
|
* Updates the domain and chip ops and creates a MSI interrupt domain.
|
|
*
|
|
* Returns:
|
|
* A domain pointer or NULL in case of failure.
|
|
*/
|
|
struct irq_domain *pci_msi_create_irq_domain(struct fwnode_handle *fwnode,
|
|
struct msi_domain_info *info,
|
|
struct irq_domain *parent)
|
|
{
|
|
struct irq_domain *domain;
|
|
|
|
if (info->flags & MSI_FLAG_USE_DEF_DOM_OPS)
|
|
pci_msi_domain_update_dom_ops(info);
|
|
if (info->flags & MSI_FLAG_USE_DEF_CHIP_OPS)
|
|
pci_msi_domain_update_chip_ops(info);
|
|
|
|
info->flags |= MSI_FLAG_ACTIVATE_EARLY;
|
|
|
|
domain = msi_create_irq_domain(fwnode, info, parent);
|
|
if (!domain)
|
|
return NULL;
|
|
|
|
irq_domain_update_bus_token(domain, DOMAIN_BUS_PCI_MSI);
|
|
return domain;
|
|
}
|
|
EXPORT_SYMBOL_GPL(pci_msi_create_irq_domain);
|
|
|
|
/*
|
|
* Users of the generic MSI infrastructure expect a device to have a single ID,
|
|
* so with DMA aliases we have to pick the least-worst compromise. Devices with
|
|
* DMA phantom functions tend to still emit MSIs from the real function number,
|
|
* so we ignore those and only consider topological aliases where either the
|
|
* alias device or RID appears on a different bus number. We also make the
|
|
* reasonable assumption that bridges are walked in an upstream direction (so
|
|
* the last one seen wins), and the much braver assumption that the most likely
|
|
* case is that of PCI->PCIe so we should always use the alias RID. This echoes
|
|
* the logic from intel_irq_remapping's set_msi_sid(), which presumably works
|
|
* well enough in practice; in the face of the horrible PCIe<->PCI-X conditions
|
|
* for taking ownership all we can really do is close our eyes and hope...
|
|
*/
|
|
static int get_msi_id_cb(struct pci_dev *pdev, u16 alias, void *data)
|
|
{
|
|
u32 *pa = data;
|
|
u8 bus = PCI_BUS_NUM(*pa);
|
|
|
|
if (pdev->bus->number != bus || PCI_BUS_NUM(alias) != bus)
|
|
*pa = alias;
|
|
|
|
return 0;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_domain_get_msi_rid - Get the MSI requester id (RID)
|
|
* @domain: The interrupt domain
|
|
* @pdev: The PCI device.
|
|
*
|
|
* The RID for a device is formed from the alias, with a firmware
|
|
* supplied mapping applied
|
|
*
|
|
* Returns: The RID.
|
|
*/
|
|
u32 pci_msi_domain_get_msi_rid(struct irq_domain *domain, struct pci_dev *pdev)
|
|
{
|
|
struct device_node *of_node;
|
|
u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
|
|
|
|
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
|
|
|
|
of_node = irq_domain_get_of_node(domain);
|
|
rid = of_node ? of_msi_map_rid(&pdev->dev, of_node, rid) :
|
|
iort_msi_map_rid(&pdev->dev, rid);
|
|
|
|
return rid;
|
|
}
|
|
|
|
/**
|
|
* pci_msi_get_device_domain - Get the MSI domain for a given PCI device
|
|
* @pdev: The PCI device
|
|
*
|
|
* Use the firmware data to find a device-specific MSI domain
|
|
* (i.e. not one that is set as a default).
|
|
*
|
|
* Returns: The corresponding MSI domain or NULL if none has been found.
|
|
*/
|
|
struct irq_domain *pci_msi_get_device_domain(struct pci_dev *pdev)
|
|
{
|
|
struct irq_domain *dom;
|
|
u32 rid = PCI_DEVID(pdev->bus->number, pdev->devfn);
|
|
|
|
pci_for_each_dma_alias(pdev, get_msi_id_cb, &rid);
|
|
dom = of_msi_map_get_device_domain(&pdev->dev, rid);
|
|
if (!dom)
|
|
dom = iort_get_device_domain(&pdev->dev, rid);
|
|
return dom;
|
|
}
|
|
#endif /* CONFIG_PCI_MSI_IRQ_DOMAIN */
|