2014-11-24 14:35:11 +00:00
|
|
|
/*
|
2016-12-20 15:11:47 +00:00
|
|
|
* Copyright (C) 2013-2017 ARM Limited, All Rights Reserved.
|
2014-11-24 14:35:11 +00:00
|
|
|
* Author: Marc Zyngier <marc.zyngier@arm.com>
|
|
|
|
*
|
|
|
|
* This program is free software; you can redistribute it and/or modify
|
|
|
|
* it under the terms of the GNU General Public License version 2 as
|
|
|
|
* published by the Free Software Foundation.
|
|
|
|
*
|
|
|
|
* This program is distributed in the hope that it will be useful,
|
|
|
|
* but WITHOUT ANY WARRANTY; without even the implied warranty of
|
|
|
|
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
|
|
|
|
* GNU General Public License for more details.
|
|
|
|
*
|
|
|
|
* You should have received a copy of the GNU General Public License
|
|
|
|
* along with this program. If not, see <http://www.gnu.org/licenses/>.
|
|
|
|
*/
|
|
|
|
|
2016-09-12 18:32:25 +00:00
|
|
|
#include <linux/acpi.h>
|
2017-03-07 12:39:59 +00:00
|
|
|
#include <linux/acpi_iort.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
#include <linux/bitmap.h>
|
|
|
|
#include <linux/cpu.h>
|
|
|
|
#include <linux/delay.h>
|
2016-09-12 16:13:59 +00:00
|
|
|
#include <linux/dma-iommu.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
#include <linux/interrupt.h>
|
2016-09-12 18:32:25 +00:00
|
|
|
#include <linux/irqdomain.h>
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
#include <linux/list.h>
|
|
|
|
#include <linux/list_sort.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
#include <linux/log2.h>
|
|
|
|
#include <linux/mm.h>
|
|
|
|
#include <linux/msi.h>
|
|
|
|
#include <linux/of.h>
|
|
|
|
#include <linux/of_address.h>
|
|
|
|
#include <linux/of_irq.h>
|
|
|
|
#include <linux/of_pci.h>
|
|
|
|
#include <linux/of_platform.h>
|
|
|
|
#include <linux/percpu.h>
|
|
|
|
#include <linux/slab.h>
|
2018-03-01 05:48:18 +00:00
|
|
|
#include <linux/syscore_ops.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2015-07-07 21:11:46 +00:00
|
|
|
#include <linux/irqchip.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
#include <linux/irqchip/arm-gic-v3.h>
|
2016-12-20 09:31:20 +00:00
|
|
|
#include <linux/irqchip/arm-gic-v4.h>
|
2014-11-24 14:35:11 +00:00
|
|
|
|
|
|
|
#include <asm/cputype.h>
|
|
|
|
#include <asm/exception.h>
|
|
|
|
|
2015-09-21 20:58:37 +00:00
|
|
|
#include "irq-gic-common.h"
|
|
|
|
|
2015-09-21 20:58:38 +00:00
|
|
|
#define ITS_FLAGS_CMDQ_NEEDS_FLUSHING (1ULL << 0)
|
|
|
|
#define ITS_FLAGS_WORKAROUND_CAVIUM_22375 (1ULL << 1)
|
2016-05-25 13:29:20 +00:00
|
|
|
#define ITS_FLAGS_WORKAROUND_CAVIUM_23144 (1ULL << 2)
|
2018-03-01 05:48:18 +00:00
|
|
|
#define ITS_FLAGS_SAVE_SUSPEND_STATE (1ULL << 3)
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2014-11-24 14:35:12 +00:00
|
|
|
#define RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING (1 << 0)
|
|
|
|
|
2016-12-19 17:15:24 +00:00
|
|
|
static u32 lpi_id_bits;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We allocate memory for PROPBASE to cover 2 ^ lpi_id_bits LPIs to
|
|
|
|
* deal with (one configuration byte per interrupt). PENDBASE has to
|
|
|
|
* be 64kB aligned (one bit per LPI, plus 8192 bits for SPI/PPI/SGI).
|
|
|
|
*/
|
|
|
|
#define LPI_NRBITS lpi_id_bits
|
|
|
|
#define LPI_PROPBASE_SZ ALIGN(BIT(LPI_NRBITS), SZ_64K)
|
|
|
|
#define LPI_PENDBASE_SZ ALIGN(BIT(LPI_NRBITS) / 8, SZ_64K)
|
|
|
|
|
|
|
|
#define LPI_PROP_DEFAULT_PRIO 0xa0
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
/*
|
|
|
|
* Collection structure - just an ID, and a redistributor address to
|
|
|
|
* ping. We use one per CPU as a bag of interrupts assigned to this
|
|
|
|
* CPU.
|
|
|
|
*/
|
|
|
|
struct its_collection {
|
|
|
|
u64 target_address;
|
|
|
|
u16 col_id;
|
|
|
|
};
|
|
|
|
|
2016-03-10 04:10:49 +00:00
|
|
|
/*
|
2016-06-06 23:17:30 +00:00
|
|
|
* The ITS_BASER structure - contains memory information, cached
|
|
|
|
* value of BASER register configuration and ITS page size.
|
2016-03-10 04:10:49 +00:00
|
|
|
*/
|
|
|
|
struct its_baser {
|
|
|
|
void *base;
|
|
|
|
u64 val;
|
|
|
|
u32 order;
|
2016-06-06 23:17:30 +00:00
|
|
|
u32 psz;
|
2016-03-10 04:10:49 +00:00
|
|
|
};
|
|
|
|
|
2017-10-17 16:55:56 +00:00
|
|
|
struct its_device;
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
/*
|
|
|
|
* The ITS structure - contains most of the infrastructure, with the
|
2015-07-28 13:46:20 +00:00
|
|
|
* top-level MSI domain, the command queue, the collections, and the
|
|
|
|
* list of devices writing to it.
|
2014-11-24 14:35:11 +00:00
|
|
|
*/
|
|
|
|
struct its_node {
|
|
|
|
raw_spinlock_t lock;
|
|
|
|
struct list_head entry;
|
|
|
|
void __iomem *base;
|
2016-09-12 18:32:24 +00:00
|
|
|
phys_addr_t phys_base;
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_block *cmd_base;
|
|
|
|
struct its_cmd_block *cmd_write;
|
2016-03-10 04:10:49 +00:00
|
|
|
struct its_baser tables[GITS_BASER_NR_REGS];
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_collection *collections;
|
2017-10-17 16:55:56 +00:00
|
|
|
struct fwnode_handle *fwnode_handle;
|
|
|
|
u64 (*get_msi_base)(struct its_device *its_dev);
|
2018-03-01 05:48:18 +00:00
|
|
|
u64 cbaser_save;
|
|
|
|
u32 ctlr_save;
|
2014-11-24 14:35:11 +00:00
|
|
|
struct list_head its_device_list;
|
|
|
|
u64 flags;
|
2017-10-08 17:44:42 +00:00
|
|
|
unsigned long list_nr;
|
2014-11-24 14:35:11 +00:00
|
|
|
u32 ite_size;
|
2016-03-10 04:10:49 +00:00
|
|
|
u32 device_ids;
|
2016-05-25 13:29:20 +00:00
|
|
|
int numa_node;
|
2017-10-17 16:55:56 +00:00
|
|
|
unsigned int msi_domain_flags;
|
|
|
|
u32 pre_its_base; /* for Socionext Synquacer */
|
2016-12-19 17:25:54 +00:00
|
|
|
bool is_v4;
|
2017-07-28 20:20:37 +00:00
|
|
|
int vlpi_redist_offset;
|
2014-11-24 14:35:11 +00:00
|
|
|
};
|
|
|
|
|
|
|
|
#define ITS_ITT_ALIGN SZ_256
|
|
|
|
|
irqchip/gic-v3-its: Fix the incorrect parsing of VCPU table size
The VCPU table consists of vPE entries, and its size provides the number
of VPEs supported by GICv4 hardware. Unfortunately the maximum size of
the VPE table is not discoverable like Device table. All VLPI commands
limits the number of bits to 16 to hold VPEID, which is index into VCPU
table. Don't apply DEVID bits for VCPU table instead assume maximum bits
to 16.
ITS log messages on QDF2400 without fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
Virtual CPUs Table too large, reduce ids 32->26
Virtual CPUs too large, reduce ITS pages 8192->256
allocated 2097152 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
ITS log messages on QDF2400 with fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
allocated 65536 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-10-07 20:43:48 +00:00
|
|
|
/* The maximum number of VPEID bits supported by VLPI commands */
|
|
|
|
#define ITS_MAX_VPEID_BITS (16)
|
|
|
|
#define ITS_MAX_VPEID (1 << (ITS_MAX_VPEID_BITS))
|
|
|
|
|
2016-02-17 00:00:36 +00:00
|
|
|
/* Convert page order to size in bytes */
|
|
|
|
#define PAGE_ORDER_TO_SIZE(o) (PAGE_SIZE << (o))
|
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct event_lpi_map {
|
|
|
|
unsigned long *lpi_map;
|
|
|
|
u16 *col_map;
|
|
|
|
irq_hw_number_t lpi_base;
|
|
|
|
int nr_lpis;
|
2016-12-20 09:44:41 +00:00
|
|
|
struct mutex vlpi_lock;
|
|
|
|
struct its_vm *vm;
|
|
|
|
struct its_vlpi_map *vlpi_maps;
|
|
|
|
int nr_vlpis;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
};
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
/*
|
2016-12-20 09:44:41 +00:00
|
|
|
* The ITS view of a device - belongs to an ITS, owns an interrupt
|
|
|
|
* translation table, and a list of interrupts. If it some of its
|
|
|
|
* LPIs are injected into a guest (GICv4), the event_map.vm field
|
|
|
|
* indicates which one.
|
2014-11-24 14:35:11 +00:00
|
|
|
*/
|
|
|
|
struct its_device {
|
|
|
|
struct list_head entry;
|
|
|
|
struct its_node *its;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct event_lpi_map event_map;
|
2014-11-24 14:35:11 +00:00
|
|
|
void *itt;
|
|
|
|
u32 nr_ites;
|
|
|
|
u32 device_id;
|
|
|
|
};
|
|
|
|
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
static struct {
|
|
|
|
raw_spinlock_t lock;
|
|
|
|
struct its_device *dev;
|
|
|
|
struct its_vpe **vpes;
|
|
|
|
int next_victim;
|
|
|
|
} vpe_proxy;
|
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
static LIST_HEAD(its_nodes);
|
2018-07-18 15:42:04 +00:00
|
|
|
static DEFINE_RAW_SPINLOCK(its_lock);
|
2014-11-24 14:35:14 +00:00
|
|
|
static struct rdists *gic_rdists;
|
2016-09-12 18:32:24 +00:00
|
|
|
static struct irq_domain *its_parent;
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2016-12-19 17:25:54 +00:00
|
|
|
static unsigned long its_list_map;
|
2016-12-20 15:17:28 +00:00
|
|
|
static u16 vmovp_seq_num;
|
|
|
|
static DEFINE_RAW_SPINLOCK(vmovp_lock);
|
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
static DEFINE_IDA(its_vpeid_ida);
|
2016-12-19 17:25:54 +00:00
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
#define gic_data_rdist() (raw_cpu_ptr(gic_rdists->rdist))
|
|
|
|
#define gic_data_rdist_rd_base() (gic_data_rdist()->rd_base)
|
2016-12-20 15:09:31 +00:00
|
|
|
#define gic_data_rdist_vlpi_base() (gic_data_rdist_rd_base() + SZ_128K)
|
2014-11-24 14:35:14 +00:00
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
static struct its_collection *dev_event_to_col(struct its_device *its_dev,
|
|
|
|
u32 event)
|
|
|
|
{
|
|
|
|
struct its_node *its = its_dev->its;
|
|
|
|
|
|
|
|
return its->collections + its_dev->event_map.col_map[event];
|
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
static struct its_collection *valid_col(struct its_collection *col)
|
|
|
|
{
|
|
|
|
if (WARN_ON_ONCE(col->target_address & GENMASK_ULL(0, 15)))
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
return col;
|
|
|
|
}
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
static struct its_vpe *valid_vpe(struct its_node *its, struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
if (valid_col(its->collections + vpe->col_idx))
|
|
|
|
return vpe;
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
/*
|
|
|
|
* ITS command descriptors - parameters to be encoded in a command
|
|
|
|
* block.
|
|
|
|
*/
|
|
|
|
struct its_cmd_desc {
|
|
|
|
union {
|
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 event_id;
|
|
|
|
} its_inv_cmd;
|
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 event_id;
|
|
|
|
} its_clear_cmd;
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 event_id;
|
|
|
|
} its_int_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
int valid;
|
|
|
|
} its_mapd_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_collection *col;
|
|
|
|
int valid;
|
|
|
|
} its_mapc_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 phys_id;
|
|
|
|
u32 event_id;
|
2016-12-20 15:52:26 +00:00
|
|
|
} its_mapti_cmd;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
struct its_collection *col;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
u32 event_id;
|
2014-11-24 14:35:11 +00:00
|
|
|
} its_movi_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 event_id;
|
|
|
|
} its_discard_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_collection *col;
|
|
|
|
} its_invall_cmd;
|
2016-12-20 09:44:41 +00:00
|
|
|
|
2016-12-20 14:47:05 +00:00
|
|
|
struct {
|
|
|
|
struct its_vpe *vpe;
|
|
|
|
} its_vinvall_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_vpe *vpe;
|
|
|
|
struct its_collection *col;
|
|
|
|
bool valid;
|
|
|
|
} its_vmapp_cmd;
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
struct {
|
|
|
|
struct its_vpe *vpe;
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 virt_id;
|
|
|
|
u32 event_id;
|
|
|
|
bool db_enabled;
|
|
|
|
} its_vmapti_cmd;
|
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_vpe *vpe;
|
|
|
|
struct its_device *dev;
|
|
|
|
u32 event_id;
|
|
|
|
bool db_enabled;
|
|
|
|
} its_vmovi_cmd;
|
2016-12-20 15:17:28 +00:00
|
|
|
|
|
|
|
struct {
|
|
|
|
struct its_vpe *vpe;
|
|
|
|
struct its_collection *col;
|
|
|
|
u16 seq_num;
|
|
|
|
u16 its_list;
|
|
|
|
} its_vmovp_cmd;
|
2014-11-24 14:35:11 +00:00
|
|
|
};
|
|
|
|
};
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The ITS command block, which is what the ITS actually parses.
|
|
|
|
*/
|
|
|
|
struct its_cmd_block {
|
|
|
|
u64 raw_cmd[4];
|
|
|
|
};
|
|
|
|
|
|
|
|
#define ITS_CMD_QUEUE_SZ SZ_64K
|
|
|
|
#define ITS_CMD_QUEUE_NR_ENTRIES (ITS_CMD_QUEUE_SZ / sizeof(struct its_cmd_block))
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
typedef struct its_collection *(*its_cmd_builder_t)(struct its_node *,
|
|
|
|
struct its_cmd_block *,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *);
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
typedef struct its_vpe *(*its_cmd_vbuilder_t)(struct its_node *,
|
|
|
|
struct its_cmd_block *,
|
2016-12-20 09:44:41 +00:00
|
|
|
struct its_cmd_desc *);
|
|
|
|
|
2016-12-19 17:11:52 +00:00
|
|
|
static void its_mask_encode(u64 *raw_cmd, u64 val, int h, int l)
|
|
|
|
{
|
|
|
|
u64 mask = GENMASK_ULL(h, l);
|
|
|
|
*raw_cmd &= ~mask;
|
|
|
|
*raw_cmd |= (val << l) & mask;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
static void its_encode_cmd(struct its_cmd_block *cmd, u8 cmd_nr)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[0], cmd_nr, 7, 0);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_devid(struct its_cmd_block *cmd, u32 devid)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[0], devid, 63, 32);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_event_id(struct its_cmd_block *cmd, u32 id)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[1], id, 31, 0);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_phys_id(struct its_cmd_block *cmd, u32 phys_id)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[1], phys_id, 63, 32);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_size(struct its_cmd_block *cmd, u8 size)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[1], size, 4, 0);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_itt(struct its_cmd_block *cmd, u64 itt_addr)
|
|
|
|
{
|
2017-10-09 16:46:55 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[2], itt_addr >> 8, 51, 8);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_valid(struct its_cmd_block *cmd, int valid)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[2], !!valid, 63, 63);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_target(struct its_cmd_block *cmd, u64 target_addr)
|
|
|
|
{
|
2017-10-09 16:46:55 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[2], target_addr >> 16, 51, 16);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_collection(struct its_cmd_block *cmd, u16 col)
|
|
|
|
{
|
2016-12-19 17:11:52 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[2], col, 15, 0);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
static void its_encode_vpeid(struct its_cmd_block *cmd, u16 vpeid)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[1], vpeid, 47, 32);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_virt_id(struct its_cmd_block *cmd, u32 virt_id)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[2], virt_id, 31, 0);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_db_phys_id(struct its_cmd_block *cmd, u32 db_phys_id)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[2], db_phys_id, 63, 32);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_db_valid(struct its_cmd_block *cmd, bool db_valid)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[2], db_valid, 0, 0);
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
static void its_encode_seq_num(struct its_cmd_block *cmd, u16 seq_num)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[0], seq_num, 47, 32);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_its_list(struct its_cmd_block *cmd, u16 its_list)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[1], its_list, 15, 0);
|
|
|
|
}
|
|
|
|
|
2016-12-20 14:47:05 +00:00
|
|
|
static void its_encode_vpt_addr(struct its_cmd_block *cmd, u64 vpt_pa)
|
|
|
|
{
|
2017-10-09 16:46:55 +00:00
|
|
|
its_mask_encode(&cmd->raw_cmd[3], vpt_pa >> 16, 51, 16);
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_encode_vpt_size(struct its_cmd_block *cmd, u8 vpt_size)
|
|
|
|
{
|
|
|
|
its_mask_encode(&cmd->raw_cmd[3], vpt_size, 4, 0);
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
static inline void its_fixup_cmd(struct its_cmd_block *cmd)
|
|
|
|
{
|
|
|
|
/* Let's fixup BE commands */
|
|
|
|
cmd->raw_cmd[0] = cpu_to_le64(cmd->raw_cmd[0]);
|
|
|
|
cmd->raw_cmd[1] = cpu_to_le64(cmd->raw_cmd[1]);
|
|
|
|
cmd->raw_cmd[2] = cpu_to_le64(cmd->raw_cmd[2]);
|
|
|
|
cmd->raw_cmd[3] = cpu_to_le64(cmd->raw_cmd[3]);
|
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_mapd_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
unsigned long itt_addr;
|
2014-12-12 10:51:24 +00:00
|
|
|
u8 size = ilog2(desc->its_mapd_cmd.dev->nr_ites);
|
2014-11-24 14:35:11 +00:00
|
|
|
|
|
|
|
itt_addr = virt_to_phys(desc->its_mapd_cmd.dev->itt);
|
|
|
|
itt_addr = ALIGN(itt_addr, ITS_ITT_ALIGN);
|
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_MAPD);
|
|
|
|
its_encode_devid(cmd, desc->its_mapd_cmd.dev->device_id);
|
|
|
|
its_encode_size(cmd, size - 1);
|
|
|
|
its_encode_itt(cmd, itt_addr);
|
|
|
|
its_encode_valid(cmd, desc->its_mapd_cmd.valid);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
return NULL;
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_mapc_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_MAPC);
|
|
|
|
its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
|
|
|
|
its_encode_target(cmd, desc->its_mapc_cmd.col->target_address);
|
|
|
|
its_encode_valid(cmd, desc->its_mapc_cmd.valid);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
|
|
|
return desc->its_mapc_cmd.col;
|
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_mapti_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct its_collection *col;
|
|
|
|
|
2016-12-20 15:52:26 +00:00
|
|
|
col = dev_event_to_col(desc->its_mapti_cmd.dev,
|
|
|
|
desc->its_mapti_cmd.event_id);
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
|
2016-12-20 15:52:26 +00:00
|
|
|
its_encode_cmd(cmd, GITS_CMD_MAPTI);
|
|
|
|
its_encode_devid(cmd, desc->its_mapti_cmd.dev->device_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_mapti_cmd.event_id);
|
|
|
|
its_encode_phys_id(cmd, desc->its_mapti_cmd.phys_id);
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
its_encode_collection(cmd, col->col_id);
|
2014-11-24 14:35:11 +00:00
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_movi_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct its_collection *col;
|
|
|
|
|
|
|
|
col = dev_event_to_col(desc->its_movi_cmd.dev,
|
|
|
|
desc->its_movi_cmd.event_id);
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
its_encode_cmd(cmd, GITS_CMD_MOVI);
|
|
|
|
its_encode_devid(cmd, desc->its_movi_cmd.dev->device_id);
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
its_encode_event_id(cmd, desc->its_movi_cmd.event_id);
|
2014-11-24 14:35:11 +00:00
|
|
|
its_encode_collection(cmd, desc->its_movi_cmd.col->col_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_discard_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct its_collection *col;
|
|
|
|
|
|
|
|
col = dev_event_to_col(desc->its_discard_cmd.dev,
|
|
|
|
desc->its_discard_cmd.event_id);
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
its_encode_cmd(cmd, GITS_CMD_DISCARD);
|
|
|
|
its_encode_devid(cmd, desc->its_discard_cmd.dev->device_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_discard_cmd.event_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_inv_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
struct its_collection *col;
|
|
|
|
|
|
|
|
col = dev_event_to_col(desc->its_inv_cmd.dev,
|
|
|
|
desc->its_inv_cmd.event_id);
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
its_encode_cmd(cmd, GITS_CMD_INV);
|
|
|
|
its_encode_devid(cmd, desc->its_inv_cmd.dev->device_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_inv_cmd.event_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_int_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-19 18:02:13 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
struct its_collection *col;
|
|
|
|
|
|
|
|
col = dev_event_to_col(desc->its_int_cmd.dev,
|
|
|
|
desc->its_int_cmd.event_id);
|
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_INT);
|
|
|
|
its_encode_devid(cmd, desc->its_int_cmd.dev->device_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_int_cmd.event_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2016-12-19 18:02:13 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_clear_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-19 18:02:13 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
struct its_collection *col;
|
|
|
|
|
|
|
|
col = dev_event_to_col(desc->its_clear_cmd.dev,
|
|
|
|
desc->its_clear_cmd.event_id);
|
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_CLEAR);
|
|
|
|
its_encode_devid(cmd, desc->its_clear_cmd.dev->device_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_clear_cmd.event_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
return valid_col(col);
|
2016-12-19 18:02:13 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_collection *its_build_invall_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2014-11-24 14:35:11 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_INVALL);
|
|
|
|
its_encode_collection(cmd, desc->its_mapc_cmd.col->col_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_vpe *its_build_vinvall_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-20 14:47:05 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_VINVALL);
|
|
|
|
its_encode_vpeid(cmd, desc->its_vinvall_cmd.vpe->vpe_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
return valid_vpe(its, desc->its_vinvall_cmd.vpe);
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_vpe *its_build_vmapp_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-20 14:47:05 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
unsigned long vpt_addr;
|
2017-07-28 20:20:37 +00:00
|
|
|
u64 target;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
|
|
|
vpt_addr = virt_to_phys(page_address(desc->its_vmapp_cmd.vpe->vpt_page));
|
2017-07-28 20:20:37 +00:00
|
|
|
target = desc->its_vmapp_cmd.col->target_address + its->vlpi_redist_offset;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_VMAPP);
|
|
|
|
its_encode_vpeid(cmd, desc->its_vmapp_cmd.vpe->vpe_id);
|
|
|
|
its_encode_valid(cmd, desc->its_vmapp_cmd.valid);
|
2017-07-28 20:20:37 +00:00
|
|
|
its_encode_target(cmd, target);
|
2016-12-20 14:47:05 +00:00
|
|
|
its_encode_vpt_addr(cmd, vpt_addr);
|
|
|
|
its_encode_vpt_size(cmd, LPI_NRBITS - 1);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
return valid_vpe(its, desc->its_vmapp_cmd.vpe);
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_vpe *its_build_vmapti_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-20 09:44:41 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
u32 db;
|
|
|
|
|
|
|
|
if (desc->its_vmapti_cmd.db_enabled)
|
|
|
|
db = desc->its_vmapti_cmd.vpe->vpe_db_lpi;
|
|
|
|
else
|
|
|
|
db = 1023;
|
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_VMAPTI);
|
|
|
|
its_encode_devid(cmd, desc->its_vmapti_cmd.dev->device_id);
|
|
|
|
its_encode_vpeid(cmd, desc->its_vmapti_cmd.vpe->vpe_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_vmapti_cmd.event_id);
|
|
|
|
its_encode_db_phys_id(cmd, db);
|
|
|
|
its_encode_virt_id(cmd, desc->its_vmapti_cmd.virt_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
return valid_vpe(its, desc->its_vmapti_cmd.vpe);
|
2016-12-20 09:44:41 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_vpe *its_build_vmovi_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-20 09:44:41 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
|
|
|
u32 db;
|
|
|
|
|
|
|
|
if (desc->its_vmovi_cmd.db_enabled)
|
|
|
|
db = desc->its_vmovi_cmd.vpe->vpe_db_lpi;
|
|
|
|
else
|
|
|
|
db = 1023;
|
|
|
|
|
|
|
|
its_encode_cmd(cmd, GITS_CMD_VMOVI);
|
|
|
|
its_encode_devid(cmd, desc->its_vmovi_cmd.dev->device_id);
|
|
|
|
its_encode_vpeid(cmd, desc->its_vmovi_cmd.vpe->vpe_id);
|
|
|
|
its_encode_event_id(cmd, desc->its_vmovi_cmd.event_id);
|
|
|
|
its_encode_db_phys_id(cmd, db);
|
|
|
|
its_encode_db_valid(cmd, true);
|
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
return valid_vpe(its, desc->its_vmovi_cmd.vpe);
|
2016-12-20 09:44:41 +00:00
|
|
|
}
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static struct its_vpe *its_build_vmovp_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *cmd,
|
2016-12-20 15:17:28 +00:00
|
|
|
struct its_cmd_desc *desc)
|
|
|
|
{
|
2017-07-28 20:20:37 +00:00
|
|
|
u64 target;
|
|
|
|
|
|
|
|
target = desc->its_vmovp_cmd.col->target_address + its->vlpi_redist_offset;
|
2016-12-20 15:17:28 +00:00
|
|
|
its_encode_cmd(cmd, GITS_CMD_VMOVP);
|
|
|
|
its_encode_seq_num(cmd, desc->its_vmovp_cmd.seq_num);
|
|
|
|
its_encode_its_list(cmd, desc->its_vmovp_cmd.its_list);
|
|
|
|
its_encode_vpeid(cmd, desc->its_vmovp_cmd.vpe->vpe_id);
|
2017-07-28 20:20:37 +00:00
|
|
|
its_encode_target(cmd, target);
|
2016-12-20 15:17:28 +00:00
|
|
|
|
|
|
|
its_fixup_cmd(cmd);
|
|
|
|
|
2018-06-22 09:52:53 +00:00
|
|
|
return valid_vpe(its, desc->its_vmovp_cmd.vpe);
|
2016-12-20 15:17:28 +00:00
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
static u64 its_cmd_ptr_to_offset(struct its_node *its,
|
|
|
|
struct its_cmd_block *ptr)
|
|
|
|
{
|
|
|
|
return (ptr - its->cmd_base) * sizeof(*ptr);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_queue_full(struct its_node *its)
|
|
|
|
{
|
|
|
|
int widx;
|
|
|
|
int ridx;
|
|
|
|
|
|
|
|
widx = its->cmd_write - its->cmd_base;
|
|
|
|
ridx = readl_relaxed(its->base + GITS_CREADR) / sizeof(struct its_cmd_block);
|
|
|
|
|
|
|
|
/* This is incredibly unlikely to happen, unless the ITS locks up. */
|
|
|
|
if (((widx + 1) % ITS_CMD_QUEUE_NR_ENTRIES) == ridx)
|
|
|
|
return 1;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct its_cmd_block *its_allocate_entry(struct its_node *its)
|
|
|
|
{
|
|
|
|
struct its_cmd_block *cmd;
|
|
|
|
u32 count = 1000000; /* 1s! */
|
|
|
|
|
|
|
|
while (its_queue_full(its)) {
|
|
|
|
count--;
|
|
|
|
if (!count) {
|
|
|
|
pr_err_ratelimited("ITS queue not draining\n");
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
udelay(1);
|
|
|
|
}
|
|
|
|
|
|
|
|
cmd = its->cmd_write++;
|
|
|
|
|
|
|
|
/* Handle queue wrapping */
|
|
|
|
if (its->cmd_write == (its->cmd_base + ITS_CMD_QUEUE_NR_ENTRIES))
|
|
|
|
its->cmd_write = its->cmd_base;
|
|
|
|
|
2016-12-19 17:16:45 +00:00
|
|
|
/* Clear command */
|
|
|
|
cmd->raw_cmd[0] = 0;
|
|
|
|
cmd->raw_cmd[1] = 0;
|
|
|
|
cmd->raw_cmd[2] = 0;
|
|
|
|
cmd->raw_cmd[3] = 0;
|
|
|
|
|
2014-11-24 14:35:11 +00:00
|
|
|
return cmd;
|
|
|
|
}
|
|
|
|
|
|
|
|
static struct its_cmd_block *its_post_commands(struct its_node *its)
|
|
|
|
{
|
|
|
|
u64 wr = its_cmd_ptr_to_offset(its, its->cmd_write);
|
|
|
|
|
|
|
|
writel_relaxed(wr, its->base + GITS_CWRITER);
|
|
|
|
|
|
|
|
return its->cmd_write;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_flush_cmd(struct its_node *its, struct its_cmd_block *cmd)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* Make sure the commands written to memory are observable by
|
|
|
|
* the ITS.
|
|
|
|
*/
|
|
|
|
if (its->flags & ITS_FLAGS_CMDQ_NEEDS_FLUSHING)
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(cmd, sizeof(*cmd));
|
2014-11-24 14:35:11 +00:00
|
|
|
else
|
|
|
|
dsb(ishst);
|
|
|
|
}
|
|
|
|
|
2017-08-04 16:45:50 +00:00
|
|
|
static int its_wait_for_range_completion(struct its_node *its,
|
|
|
|
struct its_cmd_block *from,
|
|
|
|
struct its_cmd_block *to)
|
2014-11-24 14:35:11 +00:00
|
|
|
{
|
|
|
|
u64 rd_idx, from_idx, to_idx;
|
|
|
|
u32 count = 1000000; /* 1s! */
|
|
|
|
|
|
|
|
from_idx = its_cmd_ptr_to_offset(its, from);
|
|
|
|
to_idx = its_cmd_ptr_to_offset(its, to);
|
|
|
|
|
|
|
|
while (1) {
|
|
|
|
rd_idx = readl_relaxed(its->base + GITS_CREADR);
|
2017-08-19 09:16:02 +00:00
|
|
|
|
|
|
|
/* Direct case */
|
|
|
|
if (from_idx < to_idx && rd_idx >= to_idx)
|
|
|
|
break;
|
|
|
|
|
|
|
|
/* Wrapped case */
|
|
|
|
if (from_idx >= to_idx && rd_idx >= to_idx && rd_idx < from_idx)
|
2014-11-24 14:35:11 +00:00
|
|
|
break;
|
|
|
|
|
|
|
|
count--;
|
|
|
|
if (!count) {
|
2017-08-04 16:45:50 +00:00
|
|
|
pr_err_ratelimited("ITS queue timeout (%llu %llu %llu)\n",
|
|
|
|
from_idx, to_idx, rd_idx);
|
|
|
|
return -1;
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
cpu_relax();
|
|
|
|
udelay(1);
|
|
|
|
}
|
2017-08-04 16:45:50 +00:00
|
|
|
|
|
|
|
return 0;
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:56:32 +00:00
|
|
|
/* Warning, macro hell follows */
|
|
|
|
#define BUILD_SINGLE_CMD_FUNC(name, buildtype, synctype, buildfn) \
|
|
|
|
void name(struct its_node *its, \
|
|
|
|
buildtype builder, \
|
|
|
|
struct its_cmd_desc *desc) \
|
|
|
|
{ \
|
|
|
|
struct its_cmd_block *cmd, *sync_cmd, *next_cmd; \
|
|
|
|
synctype *sync_obj; \
|
|
|
|
unsigned long flags; \
|
|
|
|
\
|
|
|
|
raw_spin_lock_irqsave(&its->lock, flags); \
|
|
|
|
\
|
|
|
|
cmd = its_allocate_entry(its); \
|
|
|
|
if (!cmd) { /* We're soooooo screewed... */ \
|
|
|
|
raw_spin_unlock_irqrestore(&its->lock, flags); \
|
|
|
|
return; \
|
|
|
|
} \
|
2017-07-28 20:16:58 +00:00
|
|
|
sync_obj = builder(its, cmd, desc); \
|
2016-12-19 17:56:32 +00:00
|
|
|
its_flush_cmd(its, cmd); \
|
|
|
|
\
|
|
|
|
if (sync_obj) { \
|
|
|
|
sync_cmd = its_allocate_entry(its); \
|
|
|
|
if (!sync_cmd) \
|
|
|
|
goto post; \
|
|
|
|
\
|
2017-07-28 20:16:58 +00:00
|
|
|
buildfn(its, sync_cmd, sync_obj); \
|
2016-12-19 17:56:32 +00:00
|
|
|
its_flush_cmd(its, sync_cmd); \
|
|
|
|
} \
|
|
|
|
\
|
|
|
|
post: \
|
|
|
|
next_cmd = its_post_commands(its); \
|
|
|
|
raw_spin_unlock_irqrestore(&its->lock, flags); \
|
|
|
|
\
|
2017-08-04 16:45:50 +00:00
|
|
|
if (its_wait_for_range_completion(its, cmd, next_cmd)) \
|
|
|
|
pr_err_ratelimited("ITS cmd %ps failed\n", builder); \
|
2016-12-19 17:56:32 +00:00
|
|
|
}
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static void its_build_sync_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *sync_cmd,
|
2016-12-19 17:56:32 +00:00
|
|
|
struct its_collection *sync_col)
|
|
|
|
{
|
|
|
|
its_encode_cmd(sync_cmd, GITS_CMD_SYNC);
|
|
|
|
its_encode_target(sync_cmd, sync_col->target_address);
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 17:56:32 +00:00
|
|
|
its_fixup_cmd(sync_cmd);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 17:56:32 +00:00
|
|
|
static BUILD_SINGLE_CMD_FUNC(its_send_single_command, its_cmd_builder_t,
|
|
|
|
struct its_collection, its_build_sync_cmd)
|
|
|
|
|
2017-07-28 20:16:58 +00:00
|
|
|
static void its_build_vsync_cmd(struct its_node *its,
|
|
|
|
struct its_cmd_block *sync_cmd,
|
2016-12-20 09:44:41 +00:00
|
|
|
struct its_vpe *sync_vpe)
|
|
|
|
{
|
|
|
|
its_encode_cmd(sync_cmd, GITS_CMD_VSYNC);
|
|
|
|
its_encode_vpeid(sync_cmd, sync_vpe->vpe_id);
|
|
|
|
|
|
|
|
its_fixup_cmd(sync_cmd);
|
|
|
|
}
|
|
|
|
|
|
|
|
static BUILD_SINGLE_CMD_FUNC(its_send_single_vcommand, its_cmd_vbuilder_t,
|
|
|
|
struct its_vpe, its_build_vsync_cmd)
|
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
static void its_send_int(struct its_device *dev, u32 event_id)
|
2014-11-24 14:35:11 +00:00
|
|
|
{
|
2016-12-19 18:02:13 +00:00
|
|
|
struct its_cmd_desc desc;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
desc.its_int_cmd.dev = dev;
|
|
|
|
desc.its_int_cmd.event_id = event_id;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
its_send_single_command(dev->its, its_build_int_cmd, &desc);
|
|
|
|
}
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
static void its_send_clear(struct its_device *dev, u32 event_id)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
desc.its_clear_cmd.dev = dev;
|
|
|
|
desc.its_clear_cmd.event_id = event_id;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
its_send_single_command(dev->its, its_build_clear_cmd, &desc);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_inv(struct its_device *dev, u32 event_id)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_inv_cmd.dev = dev;
|
|
|
|
desc.its_inv_cmd.event_id = event_id;
|
|
|
|
|
|
|
|
its_send_single_command(dev->its, its_build_inv_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_mapd(struct its_device *dev, int valid)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_mapd_cmd.dev = dev;
|
|
|
|
desc.its_mapd_cmd.valid = !!valid;
|
|
|
|
|
|
|
|
its_send_single_command(dev->its, its_build_mapd_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_mapc(struct its_node *its, struct its_collection *col,
|
|
|
|
int valid)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_mapc_cmd.col = col;
|
|
|
|
desc.its_mapc_cmd.valid = !!valid;
|
|
|
|
|
|
|
|
its_send_single_command(its, its_build_mapc_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:52:26 +00:00
|
|
|
static void its_send_mapti(struct its_device *dev, u32 irq_id, u32 id)
|
2014-11-24 14:35:11 +00:00
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
2016-12-20 15:52:26 +00:00
|
|
|
desc.its_mapti_cmd.dev = dev;
|
|
|
|
desc.its_mapti_cmd.phys_id = irq_id;
|
|
|
|
desc.its_mapti_cmd.event_id = id;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
2016-12-20 15:52:26 +00:00
|
|
|
its_send_single_command(dev->its, its_build_mapti_cmd, &desc);
|
2014-11-24 14:35:11 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_movi(struct its_device *dev,
|
|
|
|
struct its_collection *col, u32 id)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_movi_cmd.dev = dev;
|
|
|
|
desc.its_movi_cmd.col = col;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
desc.its_movi_cmd.event_id = id;
|
2014-11-24 14:35:11 +00:00
|
|
|
|
|
|
|
its_send_single_command(dev->its, its_build_movi_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_discard(struct its_device *dev, u32 id)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_discard_cmd.dev = dev;
|
|
|
|
desc.its_discard_cmd.event_id = id;
|
|
|
|
|
|
|
|
its_send_single_command(dev->its, its_build_discard_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_invall(struct its_node *its, struct its_collection *col)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_invall_cmd.col = col;
|
|
|
|
|
|
|
|
its_send_single_command(its, its_build_invall_cmd, &desc);
|
|
|
|
}
|
2014-11-24 14:35:12 +00:00
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
static void its_send_vmapti(struct its_device *dev, u32 id)
|
|
|
|
{
|
|
|
|
struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_vmapti_cmd.vpe = map->vpe;
|
|
|
|
desc.its_vmapti_cmd.dev = dev;
|
|
|
|
desc.its_vmapti_cmd.virt_id = map->vintid;
|
|
|
|
desc.its_vmapti_cmd.event_id = id;
|
|
|
|
desc.its_vmapti_cmd.db_enabled = map->db_enabled;
|
|
|
|
|
|
|
|
its_send_single_vcommand(dev->its, its_build_vmapti_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_send_vmovi(struct its_device *dev, u32 id)
|
|
|
|
{
|
|
|
|
struct its_vlpi_map *map = &dev->event_map.vlpi_maps[id];
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_vmovi_cmd.vpe = map->vpe;
|
|
|
|
desc.its_vmovi_cmd.dev = dev;
|
|
|
|
desc.its_vmovi_cmd.event_id = id;
|
|
|
|
desc.its_vmovi_cmd.db_enabled = map->db_enabled;
|
|
|
|
|
|
|
|
its_send_single_vcommand(dev->its, its_build_vmovi_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
2017-10-08 17:46:39 +00:00
|
|
|
static void its_send_vmapp(struct its_node *its,
|
|
|
|
struct its_vpe *vpe, bool valid)
|
2016-12-20 14:47:05 +00:00
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_vmapp_cmd.vpe = vpe;
|
|
|
|
desc.its_vmapp_cmd.valid = valid;
|
2017-10-08 17:46:39 +00:00
|
|
|
desc.its_vmapp_cmd.col = &its->collections[vpe->col_idx];
|
2016-12-20 14:47:05 +00:00
|
|
|
|
2017-10-08 17:46:39 +00:00
|
|
|
its_send_single_vcommand(its, its_build_vmapp_cmd, &desc);
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
static void its_send_vmovp(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
struct its_node *its;
|
|
|
|
unsigned long flags;
|
|
|
|
int col_id = vpe->col_idx;
|
|
|
|
|
|
|
|
desc.its_vmovp_cmd.vpe = vpe;
|
|
|
|
desc.its_vmovp_cmd.its_list = (u16)its_list_map;
|
|
|
|
|
|
|
|
if (!its_list_map) {
|
|
|
|
its = list_first_entry(&its_nodes, struct its_node, entry);
|
|
|
|
desc.its_vmovp_cmd.seq_num = 0;
|
|
|
|
desc.its_vmovp_cmd.col = &its->collections[col_id];
|
|
|
|
its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Yet another marvel of the architecture. If using the
|
|
|
|
* its_list "feature", we need to make sure that all ITSs
|
|
|
|
* receive all VMOVP commands in the same order. The only way
|
|
|
|
* to guarantee this is to make vmovp a serialization point.
|
|
|
|
*
|
|
|
|
* Wall <-- Head.
|
|
|
|
*/
|
|
|
|
raw_spin_lock_irqsave(&vmovp_lock, flags);
|
|
|
|
|
|
|
|
desc.its_vmovp_cmd.seq_num = vmovp_seq_num++;
|
|
|
|
|
|
|
|
/* Emit VMOVPs */
|
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
if (!its->is_v4)
|
|
|
|
continue;
|
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
if (!vpe->its_vm->vlpi_count[its->list_nr])
|
|
|
|
continue;
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
desc.its_vmovp_cmd.col = &its->collections[col_id];
|
|
|
|
its_send_single_vcommand(its, its_build_vmovp_cmd, &desc);
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&vmovp_lock, flags);
|
|
|
|
}
|
|
|
|
|
2017-10-08 14:16:09 +00:00
|
|
|
static void its_send_vinvall(struct its_node *its, struct its_vpe *vpe)
|
2016-12-20 14:47:05 +00:00
|
|
|
{
|
|
|
|
struct its_cmd_desc desc;
|
|
|
|
|
|
|
|
desc.its_vinvall_cmd.vpe = vpe;
|
2017-10-08 14:16:09 +00:00
|
|
|
its_send_single_vcommand(its, its_build_vinvall_cmd, &desc);
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:12 +00:00
|
|
|
/*
|
|
|
|
* irqchip functions - assumes MSI, mostly.
|
|
|
|
*/
|
|
|
|
|
|
|
|
static inline u32 its_get_event_id(struct irq_data *d)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
return d->hwirq - its_dev->event_map.lpi_base;
|
2014-11-24 14:35:12 +00:00
|
|
|
}
|
|
|
|
|
2016-12-20 09:54:57 +00:00
|
|
|
static void lpi_write_config(struct irq_data *d, u8 clr, u8 set)
|
2014-11-24 14:35:12 +00:00
|
|
|
{
|
2016-12-20 09:54:57 +00:00
|
|
|
irq_hw_number_t hwirq;
|
2016-12-19 19:18:13 +00:00
|
|
|
struct page *prop_page;
|
|
|
|
u8 *cfg;
|
2014-11-24 14:35:12 +00:00
|
|
|
|
2016-12-20 09:54:57 +00:00
|
|
|
if (irqd_is_forwarded_to_vcpu(d)) {
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
2017-10-26 09:44:07 +00:00
|
|
|
struct its_vlpi_map *map;
|
2016-12-20 09:54:57 +00:00
|
|
|
|
|
|
|
prop_page = its_dev->event_map.vm->vprop_page;
|
2017-10-26 09:44:07 +00:00
|
|
|
map = &its_dev->event_map.vlpi_maps[event];
|
|
|
|
hwirq = map->vintid;
|
|
|
|
|
|
|
|
/* Remember the updated property */
|
|
|
|
map->properties &= ~clr;
|
|
|
|
map->properties |= set | LPI_PROP_GROUP1;
|
2016-12-20 09:54:57 +00:00
|
|
|
} else {
|
|
|
|
prop_page = gic_rdists->prop_page;
|
|
|
|
hwirq = d->hwirq;
|
|
|
|
}
|
2016-12-19 19:18:13 +00:00
|
|
|
|
|
|
|
cfg = page_address(prop_page) + hwirq - 8192;
|
|
|
|
*cfg &= ~clr;
|
2016-12-20 09:54:57 +00:00
|
|
|
*cfg |= set | LPI_PROP_GROUP1;
|
2014-11-24 14:35:12 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Make the above write visible to the redistributors.
|
|
|
|
* And yes, we're flushing exactly: One. Single. Byte.
|
|
|
|
* Humpf...
|
|
|
|
*/
|
|
|
|
if (gic_rdists->flags & RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING)
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(cfg, sizeof(*cfg));
|
2014-11-24 14:35:12 +00:00
|
|
|
else
|
|
|
|
dsb(ishst);
|
2016-12-20 09:54:57 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void lpi_update_config(struct irq_data *d, u8 clr, u8 set)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
|
|
|
|
lpi_write_config(d, clr, set);
|
2016-12-19 19:18:13 +00:00
|
|
|
its_send_inv(its_dev, its_get_event_id(d));
|
2014-11-24 14:35:12 +00:00
|
|
|
}
|
|
|
|
|
2016-12-20 09:54:57 +00:00
|
|
|
static void its_vlpi_set_doorbell(struct irq_data *d, bool enable)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
|
|
|
|
if (its_dev->event_map.vlpi_maps[event].db_enabled == enable)
|
|
|
|
return;
|
|
|
|
|
|
|
|
its_dev->event_map.vlpi_maps[event].db_enabled = enable;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* More fun with the architecture:
|
|
|
|
*
|
|
|
|
* Ideally, we'd issue a VMAPTI to set the doorbell to its LPI
|
|
|
|
* value or to 1023, depending on the enable bit. But that
|
|
|
|
* would be issueing a mapping for an /existing/ DevID+EventID
|
|
|
|
* pair, which is UNPREDICTABLE. Instead, let's issue a VMOVI
|
|
|
|
* to the /same/ vPE, using this opportunity to adjust the
|
|
|
|
* doorbell. Mouahahahaha. We loves it, Precious.
|
|
|
|
*/
|
|
|
|
its_send_vmovi(its_dev, event);
|
2014-11-24 14:35:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_mask_irq(struct irq_data *d)
|
|
|
|
{
|
2016-12-20 09:54:57 +00:00
|
|
|
if (irqd_is_forwarded_to_vcpu(d))
|
|
|
|
its_vlpi_set_doorbell(d, false);
|
|
|
|
|
2016-12-19 19:18:13 +00:00
|
|
|
lpi_update_config(d, LPI_PROP_ENABLED, 0);
|
2014-11-24 14:35:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_unmask_irq(struct irq_data *d)
|
|
|
|
{
|
2016-12-20 09:54:57 +00:00
|
|
|
if (irqd_is_forwarded_to_vcpu(d))
|
|
|
|
its_vlpi_set_doorbell(d, true);
|
|
|
|
|
2016-12-19 19:18:13 +00:00
|
|
|
lpi_update_config(d, 0, LPI_PROP_ENABLED);
|
2014-11-24 14:35:12 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int its_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
|
|
|
|
bool force)
|
|
|
|
{
|
2016-05-25 13:29:20 +00:00
|
|
|
unsigned int cpu;
|
|
|
|
const struct cpumask *cpu_mask = cpu_online_mask;
|
2014-11-24 14:35:12 +00:00
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
struct its_collection *target_col;
|
|
|
|
u32 id = its_get_event_id(d);
|
|
|
|
|
2016-12-20 09:54:57 +00:00
|
|
|
/* A forwarded interrupt should use irq_set_vcpu_affinity */
|
|
|
|
if (irqd_is_forwarded_to_vcpu(d))
|
|
|
|
return -EINVAL;
|
|
|
|
|
2016-05-25 13:29:20 +00:00
|
|
|
/* lpi cannot be routed to a redistributor that is on a foreign node */
|
|
|
|
if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
|
|
|
|
if (its_dev->its->numa_node >= 0) {
|
|
|
|
cpu_mask = cpumask_of_node(its_dev->its->numa_node);
|
|
|
|
if (!cpumask_intersects(mask_val, cpu_mask))
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
cpu = cpumask_any_and(mask_val, cpu_mask);
|
|
|
|
|
2014-11-24 14:35:12 +00:00
|
|
|
if (cpu >= nr_cpu_ids)
|
|
|
|
return -EINVAL;
|
|
|
|
|
2017-05-18 08:19:13 +00:00
|
|
|
/* don't set the affinity when the target cpu is same as current one */
|
|
|
|
if (cpu != its_dev->event_map.col_map[id]) {
|
|
|
|
target_col = &its_dev->its->collections[cpu];
|
|
|
|
its_send_movi(its_dev, target_col, id);
|
|
|
|
its_dev->event_map.col_map[id] = cpu;
|
2017-08-18 08:39:18 +00:00
|
|
|
irq_data_update_effective_affinity(d, cpumask_of(cpu));
|
2017-05-18 08:19:13 +00:00
|
|
|
}
|
2014-11-24 14:35:12 +00:00
|
|
|
|
|
|
|
return IRQ_SET_MASK_OK_DONE;
|
|
|
|
}
|
|
|
|
|
2017-10-17 16:55:56 +00:00
|
|
|
static u64 its_irq_get_msi_base(struct its_device *its_dev)
|
|
|
|
{
|
|
|
|
struct its_node *its = its_dev->its;
|
|
|
|
|
|
|
|
return its->phys_base + GITS_TRANSLATER;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:16 +00:00
|
|
|
static void its_irq_compose_msi_msg(struct irq_data *d, struct msi_msg *msg)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
struct its_node *its;
|
|
|
|
u64 addr;
|
|
|
|
|
|
|
|
its = its_dev->its;
|
2017-10-17 16:55:56 +00:00
|
|
|
addr = its->get_msi_base(its_dev);
|
2014-11-24 14:35:16 +00:00
|
|
|
|
2016-11-02 11:54:03 +00:00
|
|
|
msg->address_lo = lower_32_bits(addr);
|
|
|
|
msg->address_hi = upper_32_bits(addr);
|
2014-11-24 14:35:16 +00:00
|
|
|
msg->data = its_get_event_id(d);
|
2016-09-12 16:13:59 +00:00
|
|
|
|
|
|
|
iommu_dma_map_msi_msg(d->irq, msg);
|
2014-11-24 14:35:16 +00:00
|
|
|
}
|
|
|
|
|
2016-12-19 18:02:13 +00:00
|
|
|
static int its_irq_set_irqchip_state(struct irq_data *d,
|
|
|
|
enum irqchip_irq_state which,
|
|
|
|
bool state)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
|
|
|
|
if (which != IRQCHIP_STATE_PENDING)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (state)
|
|
|
|
its_send_int(its_dev, event);
|
|
|
|
else
|
|
|
|
its_send_clear(its_dev, event);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
static void its_map_vm(struct its_node *its, struct its_vm *vm)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/* Not using the ITS list? Everything is always mapped. */
|
|
|
|
if (!its_list_map)
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&vmovp_lock, flags);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* If the VM wasn't mapped yet, iterate over the vpes and get
|
|
|
|
* them mapped now.
|
|
|
|
*/
|
|
|
|
vm->vlpi_count[its->list_nr]++;
|
|
|
|
|
|
|
|
if (vm->vlpi_count[its->list_nr] == 1) {
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < vm->nr_vpes; i++) {
|
|
|
|
struct its_vpe *vpe = vm->vpes[i];
|
2017-10-19 09:11:34 +00:00
|
|
|
struct irq_data *d = irq_get_irq_data(vpe->irq);
|
2017-10-08 17:50:36 +00:00
|
|
|
|
|
|
|
/* Map the VPE to the first possible CPU */
|
|
|
|
vpe->col_idx = cpumask_first(cpu_online_mask);
|
|
|
|
its_send_vmapp(its, vpe, true);
|
|
|
|
its_send_vinvall(its, vpe);
|
2017-10-19 09:11:34 +00:00
|
|
|
irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
|
2017-10-08 17:50:36 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&vmovp_lock, flags);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_unmap_vm(struct its_node *its, struct its_vm *vm)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
/* Not using the ITS list? Everything is always mapped. */
|
|
|
|
if (!its_list_map)
|
|
|
|
return;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&vmovp_lock, flags);
|
|
|
|
|
|
|
|
if (!--vm->vlpi_count[its->list_nr]) {
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < vm->nr_vpes; i++)
|
|
|
|
its_send_vmapp(its, vm->vpes[i], false);
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&vmovp_lock, flags);
|
|
|
|
}
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
static int its_vlpi_map(struct irq_data *d, struct its_cmd_info *info)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
if (!info->map)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
mutex_lock(&its_dev->event_map.vlpi_lock);
|
|
|
|
|
|
|
|
if (!its_dev->event_map.vm) {
|
|
|
|
struct its_vlpi_map *maps;
|
|
|
|
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
maps = kcalloc(its_dev->event_map.nr_lpis, sizeof(*maps),
|
2016-12-20 09:44:41 +00:00
|
|
|
GFP_KERNEL);
|
|
|
|
if (!maps) {
|
|
|
|
ret = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
its_dev->event_map.vm = info->map->vm;
|
|
|
|
its_dev->event_map.vlpi_maps = maps;
|
|
|
|
} else if (its_dev->event_map.vm != info->map->vm) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Get our private copy of the mapping information */
|
|
|
|
its_dev->event_map.vlpi_maps[event] = *info->map;
|
|
|
|
|
|
|
|
if (irqd_is_forwarded_to_vcpu(d)) {
|
|
|
|
/* Already mapped, move it around */
|
|
|
|
its_send_vmovi(its_dev, event);
|
|
|
|
} else {
|
2017-10-08 17:50:36 +00:00
|
|
|
/* Ensure all the VPEs are mapped on this ITS */
|
|
|
|
its_map_vm(its_dev->its, info->map->vm);
|
|
|
|
|
2017-10-26 09:44:07 +00:00
|
|
|
/*
|
|
|
|
* Flag the interrupt as forwarded so that we can
|
|
|
|
* start poking the virtual property table.
|
|
|
|
*/
|
|
|
|
irqd_set_forwarded_to_vcpu(d);
|
|
|
|
|
|
|
|
/* Write out the property to the prop table */
|
|
|
|
lpi_write_config(d, 0xff, info->map->properties);
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
/* Drop the physical mapping */
|
|
|
|
its_send_discard(its_dev, event);
|
|
|
|
|
|
|
|
/* and install the virtual one */
|
|
|
|
its_send_vmapti(its_dev, event);
|
|
|
|
|
|
|
|
/* Increment the number of VLPIs */
|
|
|
|
its_dev->event_map.nr_vlpis++;
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
mutex_unlock(&its_dev->event_map.vlpi_lock);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_vlpi_get(struct irq_data *d, struct its_cmd_info *info)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
mutex_lock(&its_dev->event_map.vlpi_lock);
|
|
|
|
|
|
|
|
if (!its_dev->event_map.vm ||
|
|
|
|
!its_dev->event_map.vlpi_maps[event].vm) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Copy our mapping information to the incoming request */
|
|
|
|
*info->map = its_dev->event_map.vlpi_maps[event];
|
|
|
|
|
|
|
|
out:
|
|
|
|
mutex_unlock(&its_dev->event_map.vlpi_lock);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_vlpi_unmap(struct irq_data *d)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
int ret = 0;
|
|
|
|
|
|
|
|
mutex_lock(&its_dev->event_map.vlpi_lock);
|
|
|
|
|
|
|
|
if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d)) {
|
|
|
|
ret = -EINVAL;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Drop the virtual mapping */
|
|
|
|
its_send_discard(its_dev, event);
|
|
|
|
|
|
|
|
/* and restore the physical one */
|
|
|
|
irqd_clr_forwarded_to_vcpu(d);
|
|
|
|
its_send_mapti(its_dev, d->hwirq, event);
|
|
|
|
lpi_update_config(d, 0xff, (LPI_PROP_DEFAULT_PRIO |
|
|
|
|
LPI_PROP_ENABLED |
|
|
|
|
LPI_PROP_GROUP1));
|
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
/* Potentially unmap the VM from this ITS */
|
|
|
|
its_unmap_vm(its_dev->its, its_dev->event_map.vm);
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
/*
|
|
|
|
* Drop the refcount and make the device available again if
|
|
|
|
* this was the last VLPI.
|
|
|
|
*/
|
|
|
|
if (!--its_dev->event_map.nr_vlpis) {
|
|
|
|
its_dev->event_map.vm = NULL;
|
|
|
|
kfree(its_dev->event_map.vlpi_maps);
|
|
|
|
}
|
|
|
|
|
|
|
|
out:
|
|
|
|
mutex_unlock(&its_dev->event_map.vlpi_lock);
|
|
|
|
return ret;
|
|
|
|
}
|
|
|
|
|
2016-12-20 09:54:57 +00:00
|
|
|
static int its_vlpi_prop_update(struct irq_data *d, struct its_cmd_info *info)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
|
|
|
|
if (!its_dev->event_map.vm || !irqd_is_forwarded_to_vcpu(d))
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (info->cmd_type == PROP_UPDATE_AND_INV_VLPI)
|
|
|
|
lpi_update_config(d, 0xff, info->config);
|
|
|
|
else
|
|
|
|
lpi_write_config(d, 0xff, info->config);
|
|
|
|
its_vlpi_set_doorbell(d, !!(info->config & LPI_PROP_ENABLED));
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-20 09:31:20 +00:00
|
|
|
static int its_irq_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
struct its_cmd_info *info = vcpu_info;
|
|
|
|
|
|
|
|
/* Need a v4 ITS */
|
2016-12-20 09:44:41 +00:00
|
|
|
if (!its_dev->its->is_v4)
|
2016-12-20 09:31:20 +00:00
|
|
|
return -EINVAL;
|
|
|
|
|
2016-12-20 09:44:41 +00:00
|
|
|
/* Unmap request? */
|
|
|
|
if (!info)
|
|
|
|
return its_vlpi_unmap(d);
|
|
|
|
|
2016-12-20 09:31:20 +00:00
|
|
|
switch (info->cmd_type) {
|
|
|
|
case MAP_VLPI:
|
2016-12-20 09:44:41 +00:00
|
|
|
return its_vlpi_map(d, info);
|
2016-12-20 09:31:20 +00:00
|
|
|
|
|
|
|
case GET_VLPI:
|
2016-12-20 09:44:41 +00:00
|
|
|
return its_vlpi_get(d, info);
|
2016-12-20 09:31:20 +00:00
|
|
|
|
|
|
|
case PROP_UPDATE_VLPI:
|
|
|
|
case PROP_UPDATE_AND_INV_VLPI:
|
2016-12-20 09:54:57 +00:00
|
|
|
return its_vlpi_prop_update(d, info);
|
2016-12-20 09:31:20 +00:00
|
|
|
|
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:12 +00:00
|
|
|
static struct irq_chip its_irq_chip = {
|
|
|
|
.name = "ITS",
|
|
|
|
.irq_mask = its_mask_irq,
|
|
|
|
.irq_unmask = its_unmask_irq,
|
2016-02-11 13:38:53 +00:00
|
|
|
.irq_eoi = irq_chip_eoi_parent,
|
2014-11-24 14:35:12 +00:00
|
|
|
.irq_set_affinity = its_set_affinity,
|
2014-11-24 14:35:16 +00:00
|
|
|
.irq_compose_msi_msg = its_irq_compose_msi_msg,
|
2016-12-19 18:02:13 +00:00
|
|
|
.irq_set_irqchip_state = its_irq_set_irqchip_state,
|
2016-12-20 09:31:20 +00:00
|
|
|
.irq_set_vcpu_affinity = its_irq_set_vcpu_affinity,
|
2014-11-24 14:35:16 +00:00
|
|
|
};
|
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
|
2014-11-24 14:35:13 +00:00
|
|
|
/*
|
|
|
|
* How we allocate LPIs:
|
|
|
|
*
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
* lpi_range_list contains ranges of LPIs that are to available to
|
|
|
|
* allocate from. To allocate LPIs, just pick the first range that
|
|
|
|
* fits the required allocation, and reduce it by the required
|
|
|
|
* amount. Once empty, remove the range from the list.
|
|
|
|
*
|
|
|
|
* To free a range of LPIs, add a free range to the list, sort it and
|
|
|
|
* merge the result if the new range happens to be adjacent to an
|
|
|
|
* already free block.
|
2014-11-24 14:35:13 +00:00
|
|
|
*
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
* The consequence of the above is that allocation is cost is low, but
|
|
|
|
* freeing is expensive. We assumes that freeing rarely occurs.
|
|
|
|
*/
|
2018-08-28 04:53:26 +00:00
|
|
|
#define ITS_MAX_LPI_NRBITS 16 /* 64K LPIs */
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
|
|
|
|
static DEFINE_MUTEX(lpi_range_lock);
|
|
|
|
static LIST_HEAD(lpi_range_list);
|
|
|
|
|
|
|
|
struct lpi_range {
|
|
|
|
struct list_head entry;
|
|
|
|
u32 base_id;
|
|
|
|
u32 span;
|
|
|
|
};
|
2014-11-24 14:35:13 +00:00
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
static struct lpi_range *mk_lpi_range(u32 base, u32 span)
|
2014-11-24 14:35:13 +00:00
|
|
|
{
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
struct lpi_range *range;
|
|
|
|
|
|
|
|
range = kzalloc(sizeof(*range), GFP_KERNEL);
|
|
|
|
if (range) {
|
|
|
|
INIT_LIST_HEAD(&range->entry);
|
|
|
|
range->base_id = base;
|
|
|
|
range->span = span;
|
|
|
|
}
|
|
|
|
|
|
|
|
return range;
|
2014-11-24 14:35:13 +00:00
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
static int lpi_range_cmp(void *priv, struct list_head *a, struct list_head *b)
|
2014-11-24 14:35:13 +00:00
|
|
|
{
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
struct lpi_range *ra, *rb;
|
|
|
|
|
|
|
|
ra = container_of(a, struct lpi_range, entry);
|
|
|
|
rb = container_of(b, struct lpi_range, entry);
|
|
|
|
|
|
|
|
return rb->base_id - ra->base_id;
|
2014-11-24 14:35:13 +00:00
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
static void merge_lpi_ranges(void)
|
2014-11-24 14:35:13 +00:00
|
|
|
{
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
struct lpi_range *range, *tmp;
|
2014-11-24 14:35:13 +00:00
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
|
|
|
|
if (!list_is_last(&range->entry, &lpi_range_list) &&
|
|
|
|
(tmp->base_id == (range->base_id + range->span))) {
|
|
|
|
tmp->base_id = range->base_id;
|
|
|
|
tmp->span += range->span;
|
|
|
|
list_del(&range->entry);
|
|
|
|
kfree(range);
|
|
|
|
}
|
2014-11-24 14:35:13 +00:00
|
|
|
}
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
}
|
2014-11-24 14:35:13 +00:00
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
static int alloc_lpi_range(u32 nr_lpis, u32 *base)
|
|
|
|
{
|
|
|
|
struct lpi_range *range, *tmp;
|
|
|
|
int err = -ENOSPC;
|
|
|
|
|
|
|
|
mutex_lock(&lpi_range_lock);
|
|
|
|
|
|
|
|
list_for_each_entry_safe(range, tmp, &lpi_range_list, entry) {
|
|
|
|
if (range->span >= nr_lpis) {
|
|
|
|
*base = range->base_id;
|
|
|
|
range->base_id += nr_lpis;
|
|
|
|
range->span -= nr_lpis;
|
|
|
|
|
|
|
|
if (range->span == 0) {
|
|
|
|
list_del(&range->entry);
|
|
|
|
kfree(range);
|
|
|
|
}
|
|
|
|
|
|
|
|
err = 0;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
mutex_unlock(&lpi_range_lock);
|
|
|
|
|
|
|
|
pr_debug("ITS: alloc %u:%u\n", *base, nr_lpis);
|
|
|
|
return err;
|
2014-11-24 14:35:13 +00:00
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
static int free_lpi_range(u32 base, u32 nr_lpis)
|
2014-11-24 14:35:13 +00:00
|
|
|
{
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
struct lpi_range *new;
|
|
|
|
int err = 0;
|
|
|
|
|
|
|
|
mutex_lock(&lpi_range_lock);
|
|
|
|
|
|
|
|
new = mk_lpi_range(base, nr_lpis);
|
|
|
|
if (!new) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto out;
|
|
|
|
}
|
|
|
|
|
|
|
|
list_add(&new->entry, &lpi_range_list);
|
|
|
|
list_sort(NULL, &lpi_range_list, lpi_range_cmp);
|
|
|
|
merge_lpi_ranges();
|
|
|
|
out:
|
|
|
|
mutex_unlock(&lpi_range_lock);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int __init its_lpi_init(u32 id_bits)
|
|
|
|
{
|
|
|
|
u32 lpis = (1UL << id_bits) - 8192;
|
2018-05-31 08:01:59 +00:00
|
|
|
u32 numlpis;
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
int err;
|
|
|
|
|
2018-05-31 08:01:59 +00:00
|
|
|
numlpis = 1UL << GICD_TYPER_NUM_LPIS(gic_rdists->gicd_typer);
|
|
|
|
|
|
|
|
if (numlpis > 2 && !WARN_ON(numlpis > lpis)) {
|
|
|
|
lpis = numlpis;
|
|
|
|
pr_info("ITS: Using hypervisor restricted LPI range [%u]\n",
|
|
|
|
lpis);
|
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
/*
|
|
|
|
* Initializing the allocator is just the same as freeing the
|
|
|
|
* full range of LPIs.
|
|
|
|
*/
|
|
|
|
err = free_lpi_range(8192, lpis);
|
|
|
|
pr_debug("ITS: Allocator initialized for %u LPIs\n", lpis);
|
|
|
|
return err;
|
|
|
|
}
|
2014-11-24 14:35:13 +00:00
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
static unsigned long *its_lpi_alloc(int nr_irqs, u32 *base, int *nr_ids)
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
{
|
|
|
|
unsigned long *bitmap = NULL;
|
|
|
|
int err = 0;
|
2014-11-24 14:35:13 +00:00
|
|
|
|
|
|
|
do {
|
2018-05-27 16:03:03 +00:00
|
|
|
err = alloc_lpi_range(nr_irqs, base);
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
if (!err)
|
2014-11-24 14:35:13 +00:00
|
|
|
break;
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
nr_irqs /= 2;
|
|
|
|
} while (nr_irqs > 0);
|
2014-11-24 14:35:13 +00:00
|
|
|
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
if (err)
|
2014-11-24 14:35:13 +00:00
|
|
|
goto out;
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
bitmap = kcalloc(BITS_TO_LONGS(nr_irqs), sizeof (long), GFP_ATOMIC);
|
2014-11-24 14:35:13 +00:00
|
|
|
if (!bitmap)
|
|
|
|
goto out;
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
*nr_ids = nr_irqs;
|
2014-11-24 14:35:13 +00:00
|
|
|
|
|
|
|
out:
|
2015-10-02 15:44:05 +00:00
|
|
|
if (!bitmap)
|
|
|
|
*base = *nr_ids = 0;
|
|
|
|
|
2014-11-24 14:35:13 +00:00
|
|
|
return bitmap;
|
|
|
|
}
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
static void its_lpi_free(unsigned long *bitmap, u32 base, u32 nr_ids)
|
2014-11-24 14:35:13 +00:00
|
|
|
{
|
irqchip/gic-v3-its: Refactor LPI allocator
Our current LPI allocator relies on a bitmap, each bit representing
a chunk of 32 LPIs, meaning that each device gets allocated LPIs
in multiple of 32. It served us well so far, but new use cases now
require much more finer grain allocations, down the the individual
LPI.
Given the size of the IntID space (up to 32bit), it isn't practical
to continue using a bitmap, so let's use a different data structure
altogether.
We switch to a list, where each element represent a contiguous range
of LPIs. On allocation, we simply grab the first group big enough to
satisfy the allocation, and substract what we need from it. If the
group becomes empty, we just remove it. On freeing interrupts, we
insert a new group of interrupt in the list, sort it and fuse the
adjacent groups.
This makes freeing interrupt much more expensive than allocating
them (an unusual behaviour), but that's fine as long as we consider
that freeing interrupts is an extremely rare event.
We still allocate interrupts in blocks of 32 for the time being,
but subsequent patches will relax this.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2018-05-27 15:14:15 +00:00
|
|
|
WARN_ON(free_lpi_range(base, nr_ids));
|
2016-12-19 18:49:59 +00:00
|
|
|
kfree(bitmap);
|
2014-11-24 14:35:13 +00:00
|
|
|
}
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2016-12-19 18:15:05 +00:00
|
|
|
static struct page *its_allocate_prop_table(gfp_t gfp_flags)
|
|
|
|
{
|
|
|
|
struct page *prop_page;
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2016-12-19 18:15:05 +00:00
|
|
|
prop_page = alloc_pages(gfp_flags, get_order(LPI_PROPBASE_SZ));
|
|
|
|
if (!prop_page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Priority 0xa0, Group-1, disabled */
|
|
|
|
memset(page_address(prop_page),
|
|
|
|
LPI_PROP_DEFAULT_PRIO | LPI_PROP_GROUP1,
|
|
|
|
LPI_PROPBASE_SZ);
|
|
|
|
|
|
|
|
/* Make sure the GIC will observe the written configuration */
|
|
|
|
gic_flush_dcache_to_poc(page_address(prop_page), LPI_PROPBASE_SZ);
|
|
|
|
|
|
|
|
return prop_page;
|
|
|
|
}
|
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
static void its_free_prop_table(struct page *prop_page)
|
|
|
|
{
|
|
|
|
free_pages((unsigned long)page_address(prop_page),
|
|
|
|
get_order(LPI_PROPBASE_SZ));
|
|
|
|
}
|
2014-11-24 14:35:14 +00:00
|
|
|
|
|
|
|
static int __init its_alloc_lpi_tables(void)
|
|
|
|
{
|
|
|
|
phys_addr_t paddr;
|
|
|
|
|
2018-08-28 04:53:26 +00:00
|
|
|
lpi_id_bits = min_t(u32, GICD_TYPER_ID_BITS(gic_rdists->gicd_typer),
|
|
|
|
ITS_MAX_LPI_NRBITS);
|
2016-12-19 18:15:05 +00:00
|
|
|
gic_rdists->prop_page = its_allocate_prop_table(GFP_NOWAIT);
|
2014-11-24 14:35:14 +00:00
|
|
|
if (!gic_rdists->prop_page) {
|
|
|
|
pr_err("Failed to allocate PROPBASE\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
paddr = page_to_phys(gic_rdists->prop_page);
|
|
|
|
pr_info("GIC: using LPI property table @%pa\n", &paddr);
|
|
|
|
|
irqchip/gic-v3-its: Don't assume GICv3 hardware supports 16bit INTID
The current ITS driver is assuming every ITS hardware implementation
supports minimum of 16bit INTID. But this is not true, as per GICv3
specification, INTID field is IMPLEMENTATION DEFINED in the range of
14-24 bits. We might see an unpredictable system behavior on systems
where hardware support less than 16bits and software tries to use
64K LPI interrupts.
On Qualcomm Datacenter Technologies QDF2400 platform, boot log shows
confusing information about number of LPI chunks as shown below. The
QDF2400 ITS hardware supports 24bit INTID.
This patch allocates the memory resources for PEND/PROP tables based
on discoverable value which is specified in GITS_TYPER.IDbits. Also
it fixes the log message that reflects the correct number of LPI
chunks were allocated.
ITS@0xff7efe0000: allocated 524288 Devices @3c0400000 (indirect, esz 8, psz 64K, shr 1)
ITS@0xff7efe0000: allocated 8192 Interrupt Collections @3c0130000 (flat, esz 8, psz 64K, shr 1)
ITS@0xff7efe0000: allocated 8192 Virtual CPUs @3c0140000 (flat, esz 8, psz 64K, shr 1)
ITS: Allocated 524032 chunks for LPIs
PCI/MSI: ITS@0xff7efe0000 domain created
Platform MSI: ITS@0xff7efe0000 domain created
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-06-22 23:19:14 +00:00
|
|
|
return its_lpi_init(lpi_id_bits);
|
2014-11-24 14:35:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static const char *its_base_type_string[] = {
|
|
|
|
[GITS_BASER_TYPE_DEVICE] = "Devices",
|
|
|
|
[GITS_BASER_TYPE_VCPU] = "Virtual CPUs",
|
2016-12-20 15:50:14 +00:00
|
|
|
[GITS_BASER_TYPE_RESERVED3] = "Reserved (3)",
|
2014-11-24 14:35:14 +00:00
|
|
|
[GITS_BASER_TYPE_COLLECTION] = "Interrupt Collections",
|
|
|
|
[GITS_BASER_TYPE_RESERVED5] = "Reserved (5)",
|
|
|
|
[GITS_BASER_TYPE_RESERVED6] = "Reserved (6)",
|
|
|
|
[GITS_BASER_TYPE_RESERVED7] = "Reserved (7)",
|
|
|
|
};
|
|
|
|
|
2016-06-06 23:17:28 +00:00
|
|
|
static u64 its_read_baser(struct its_node *its, struct its_baser *baser)
|
|
|
|
{
|
|
|
|
u32 idx = baser - its->tables;
|
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
return gits_read_baser(its->base + GITS_BASER + (idx << 3));
|
2016-06-06 23:17:28 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_write_baser(struct its_node *its, struct its_baser *baser,
|
|
|
|
u64 val)
|
|
|
|
{
|
|
|
|
u32 idx = baser - its->tables;
|
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
gits_write_baser(val, its->base + GITS_BASER + (idx << 3));
|
2016-06-06 23:17:28 +00:00
|
|
|
baser->val = its_read_baser(its, baser);
|
|
|
|
}
|
|
|
|
|
2016-06-06 23:17:30 +00:00
|
|
|
static int its_setup_baser(struct its_node *its, struct its_baser *baser,
|
2016-06-06 23:17:32 +00:00
|
|
|
u64 cache, u64 shr, u32 psz, u32 order,
|
|
|
|
bool indirect)
|
2016-06-06 23:17:30 +00:00
|
|
|
{
|
|
|
|
u64 val = its_read_baser(its, baser);
|
|
|
|
u64 esz = GITS_BASER_ENTRY_SIZE(val);
|
|
|
|
u64 type = GITS_BASER_TYPE(val);
|
2017-10-09 16:46:55 +00:00
|
|
|
u64 baser_phys, tmp;
|
2016-06-06 23:17:30 +00:00
|
|
|
u32 alloc_pages;
|
|
|
|
void *base;
|
|
|
|
|
|
|
|
retry_alloc_baser:
|
|
|
|
alloc_pages = (PAGE_ORDER_TO_SIZE(order) / psz);
|
|
|
|
if (alloc_pages > GITS_BASER_PAGES_MAX) {
|
|
|
|
pr_warn("ITS@%pa: %s too large, reduce ITS pages %u->%u\n",
|
|
|
|
&its->phys_base, its_base_type_string[type],
|
|
|
|
alloc_pages, GITS_BASER_PAGES_MAX);
|
|
|
|
alloc_pages = GITS_BASER_PAGES_MAX;
|
|
|
|
order = get_order(GITS_BASER_PAGES_MAX * psz);
|
|
|
|
}
|
|
|
|
|
|
|
|
base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO, order);
|
|
|
|
if (!base)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2017-10-09 16:46:55 +00:00
|
|
|
baser_phys = virt_to_phys(base);
|
|
|
|
|
|
|
|
/* Check if the physical address of the memory is above 48bits */
|
|
|
|
if (IS_ENABLED(CONFIG_ARM64_64K_PAGES) && (baser_phys >> 48)) {
|
|
|
|
|
|
|
|
/* 52bit PA is supported only when PageSize=64K */
|
|
|
|
if (psz != SZ_64K) {
|
|
|
|
pr_err("ITS: no 52bit PA support when psz=%d\n", psz);
|
|
|
|
free_pages((unsigned long)base, order);
|
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Convert 52bit PA to 48bit field */
|
|
|
|
baser_phys = GITS_BASER_PHYS_52_to_48(baser_phys);
|
|
|
|
}
|
|
|
|
|
2016-06-06 23:17:30 +00:00
|
|
|
retry_baser:
|
2017-10-09 16:46:55 +00:00
|
|
|
val = (baser_phys |
|
2016-06-06 23:17:30 +00:00
|
|
|
(type << GITS_BASER_TYPE_SHIFT) |
|
|
|
|
((esz - 1) << GITS_BASER_ENTRY_SIZE_SHIFT) |
|
|
|
|
((alloc_pages - 1) << GITS_BASER_PAGES_SHIFT) |
|
|
|
|
cache |
|
|
|
|
shr |
|
|
|
|
GITS_BASER_VALID);
|
|
|
|
|
2016-06-06 23:17:32 +00:00
|
|
|
val |= indirect ? GITS_BASER_INDIRECT : 0x0;
|
|
|
|
|
2016-06-06 23:17:30 +00:00
|
|
|
switch (psz) {
|
|
|
|
case SZ_4K:
|
|
|
|
val |= GITS_BASER_PAGE_SIZE_4K;
|
|
|
|
break;
|
|
|
|
case SZ_16K:
|
|
|
|
val |= GITS_BASER_PAGE_SIZE_16K;
|
|
|
|
break;
|
|
|
|
case SZ_64K:
|
|
|
|
val |= GITS_BASER_PAGE_SIZE_64K;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
|
|
|
|
its_write_baser(its, baser, val);
|
|
|
|
tmp = baser->val;
|
|
|
|
|
|
|
|
if ((val ^ tmp) & GITS_BASER_SHAREABILITY_MASK) {
|
|
|
|
/*
|
|
|
|
* Shareability didn't stick. Just use
|
|
|
|
* whatever the read reported, which is likely
|
|
|
|
* to be the only thing this redistributor
|
|
|
|
* supports. If that's zero, make it
|
|
|
|
* non-cacheable as well.
|
|
|
|
*/
|
|
|
|
shr = tmp & GITS_BASER_SHAREABILITY_MASK;
|
|
|
|
if (!shr) {
|
|
|
|
cache = GITS_BASER_nC;
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(base, PAGE_ORDER_TO_SIZE(order));
|
2016-06-06 23:17:30 +00:00
|
|
|
}
|
|
|
|
goto retry_baser;
|
|
|
|
}
|
|
|
|
|
|
|
|
if ((val ^ tmp) & GITS_BASER_PAGE_SIZE_MASK) {
|
|
|
|
/*
|
|
|
|
* Page size didn't stick. Let's try a smaller
|
|
|
|
* size and retry. If we reach 4K, then
|
|
|
|
* something is horribly wrong...
|
|
|
|
*/
|
|
|
|
free_pages((unsigned long)base, order);
|
|
|
|
baser->base = NULL;
|
|
|
|
|
|
|
|
switch (psz) {
|
|
|
|
case SZ_16K:
|
|
|
|
psz = SZ_4K;
|
|
|
|
goto retry_alloc_baser;
|
|
|
|
case SZ_64K:
|
|
|
|
psz = SZ_16K;
|
|
|
|
goto retry_alloc_baser;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
if (val != tmp) {
|
2016-11-02 11:54:03 +00:00
|
|
|
pr_err("ITS@%pa: %s doesn't stick: %llx %llx\n",
|
2016-06-06 23:17:30 +00:00
|
|
|
&its->phys_base, its_base_type_string[type],
|
2016-11-02 11:54:03 +00:00
|
|
|
val, tmp);
|
2016-06-06 23:17:30 +00:00
|
|
|
free_pages((unsigned long)base, order);
|
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
baser->order = order;
|
|
|
|
baser->base = base;
|
|
|
|
baser->psz = psz;
|
2016-06-06 23:17:32 +00:00
|
|
|
tmp = indirect ? GITS_LVL1_ENTRY_SIZE : esz;
|
2016-06-06 23:17:30 +00:00
|
|
|
|
2016-06-06 23:17:32 +00:00
|
|
|
pr_info("ITS@%pa: allocated %d %s @%lx (%s, esz %d, psz %dK, shr %d)\n",
|
2016-11-02 11:54:04 +00:00
|
|
|
&its->phys_base, (int)(PAGE_ORDER_TO_SIZE(order) / (int)tmp),
|
2016-06-06 23:17:30 +00:00
|
|
|
its_base_type_string[type],
|
|
|
|
(unsigned long)virt_to_phys(base),
|
2016-06-06 23:17:32 +00:00
|
|
|
indirect ? "indirect" : "flat", (int)esz,
|
2016-06-06 23:17:30 +00:00
|
|
|
psz / SZ_1K, (int)shr >> GITS_BASER_SHAREABILITY_SHIFT);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-19 18:18:34 +00:00
|
|
|
static bool its_parse_indirect_baser(struct its_node *its,
|
|
|
|
struct its_baser *baser,
|
irqchip/gic-v3-its: Fix the incorrect parsing of VCPU table size
The VCPU table consists of vPE entries, and its size provides the number
of VPEs supported by GICv4 hardware. Unfortunately the maximum size of
the VPE table is not discoverable like Device table. All VLPI commands
limits the number of bits to 16 to hold VPEID, which is index into VCPU
table. Don't apply DEVID bits for VCPU table instead assume maximum bits
to 16.
ITS log messages on QDF2400 without fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
Virtual CPUs Table too large, reduce ids 32->26
Virtual CPUs too large, reduce ITS pages 8192->256
allocated 2097152 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
ITS log messages on QDF2400 with fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
allocated 65536 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-10-07 20:43:48 +00:00
|
|
|
u32 psz, u32 *order, u32 ids)
|
2016-06-06 23:17:29 +00:00
|
|
|
{
|
2016-12-19 18:18:34 +00:00
|
|
|
u64 tmp = its_read_baser(its, baser);
|
|
|
|
u64 type = GITS_BASER_TYPE(tmp);
|
|
|
|
u64 esz = GITS_BASER_ENTRY_SIZE(tmp);
|
2017-01-26 03:51:41 +00:00
|
|
|
u64 val = GITS_BASER_InnerShareable | GITS_BASER_RaWaWb;
|
2016-06-06 23:17:29 +00:00
|
|
|
u32 new_order = *order;
|
2016-06-06 23:17:32 +00:00
|
|
|
bool indirect = false;
|
|
|
|
|
|
|
|
/* No need to enable Indirection if memory requirement < (psz*2)bytes */
|
|
|
|
if ((esz << ids) > (psz * 2)) {
|
|
|
|
/*
|
|
|
|
* Find out whether hw supports a single or two-level table by
|
|
|
|
* table by reading bit at offset '62' after writing '1' to it.
|
|
|
|
*/
|
|
|
|
its_write_baser(its, baser, val | GITS_BASER_INDIRECT);
|
|
|
|
indirect = !!(baser->val & GITS_BASER_INDIRECT);
|
|
|
|
|
|
|
|
if (indirect) {
|
|
|
|
/*
|
|
|
|
* The size of the lvl2 table is equal to ITS page size
|
|
|
|
* which is 'psz'. For computing lvl1 table size,
|
|
|
|
* subtract ID bits that sparse lvl2 table from 'ids'
|
|
|
|
* which is reported by ITS hardware times lvl1 table
|
|
|
|
* entry size.
|
|
|
|
*/
|
2016-11-02 11:54:04 +00:00
|
|
|
ids -= ilog2(psz / (int)esz);
|
2016-06-06 23:17:32 +00:00
|
|
|
esz = GITS_LVL1_ENTRY_SIZE;
|
|
|
|
}
|
|
|
|
}
|
2016-06-06 23:17:29 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Allocate as many entries as required to fit the
|
|
|
|
* range of device IDs that the ITS can grok... The ID
|
|
|
|
* space being incredibly sparse, this results in a
|
2016-06-06 23:17:32 +00:00
|
|
|
* massive waste of memory if two-level device table
|
|
|
|
* feature is not supported by hardware.
|
2016-06-06 23:17:29 +00:00
|
|
|
*/
|
|
|
|
new_order = max_t(u32, get_order(esz << ids), new_order);
|
|
|
|
if (new_order >= MAX_ORDER) {
|
|
|
|
new_order = MAX_ORDER - 1;
|
2016-11-02 11:54:04 +00:00
|
|
|
ids = ilog2(PAGE_ORDER_TO_SIZE(new_order) / (int)esz);
|
2016-12-19 18:18:34 +00:00
|
|
|
pr_warn("ITS@%pa: %s Table too large, reduce ids %u->%u\n",
|
|
|
|
&its->phys_base, its_base_type_string[type],
|
|
|
|
its->device_ids, ids);
|
2016-06-06 23:17:29 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
*order = new_order;
|
2016-06-06 23:17:32 +00:00
|
|
|
|
|
|
|
return indirect;
|
2016-06-06 23:17:29 +00:00
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
static void its_free_tables(struct its_node *its)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < GITS_BASER_NR_REGS; i++) {
|
2016-02-02 02:19:44 +00:00
|
|
|
if (its->tables[i].base) {
|
|
|
|
free_pages((unsigned long)its->tables[i].base,
|
|
|
|
its->tables[i].order);
|
|
|
|
its->tables[i].base = NULL;
|
2014-11-24 14:35:14 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-06-06 23:17:31 +00:00
|
|
|
static int its_alloc_tables(struct its_node *its)
|
2014-11-24 14:35:14 +00:00
|
|
|
{
|
|
|
|
u64 shr = GITS_BASER_InnerShareable;
|
2017-01-26 03:51:41 +00:00
|
|
|
u64 cache = GITS_BASER_RaWaWb;
|
2016-06-06 23:17:30 +00:00
|
|
|
u32 psz = SZ_64K;
|
|
|
|
int err, i;
|
2015-09-21 20:58:38 +00:00
|
|
|
|
2017-10-17 16:55:54 +00:00
|
|
|
if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_22375)
|
|
|
|
/* erratum 24313: ignore memory access type */
|
|
|
|
cache = GITS_BASER_nCnB;
|
2016-03-10 04:10:49 +00:00
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
for (i = 0; i < GITS_BASER_NR_REGS; i++) {
|
2016-06-06 23:17:28 +00:00
|
|
|
struct its_baser *baser = its->tables + i;
|
|
|
|
u64 val = its_read_baser(its, baser);
|
2014-11-24 14:35:14 +00:00
|
|
|
u64 type = GITS_BASER_TYPE(val);
|
2016-06-06 23:17:30 +00:00
|
|
|
u32 order = get_order(psz);
|
2016-06-06 23:17:32 +00:00
|
|
|
bool indirect = false;
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2016-12-19 18:18:34 +00:00
|
|
|
switch (type) {
|
|
|
|
case GITS_BASER_TYPE_NONE:
|
2014-11-24 14:35:14 +00:00
|
|
|
continue;
|
|
|
|
|
2016-12-19 18:18:34 +00:00
|
|
|
case GITS_BASER_TYPE_DEVICE:
|
irqchip/gic-v3-its: Fix the incorrect parsing of VCPU table size
The VCPU table consists of vPE entries, and its size provides the number
of VPEs supported by GICv4 hardware. Unfortunately the maximum size of
the VPE table is not discoverable like Device table. All VLPI commands
limits the number of bits to 16 to hold VPEID, which is index into VCPU
table. Don't apply DEVID bits for VCPU table instead assume maximum bits
to 16.
ITS log messages on QDF2400 without fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
Virtual CPUs Table too large, reduce ids 32->26
Virtual CPUs too large, reduce ITS pages 8192->256
allocated 2097152 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
ITS log messages on QDF2400 with fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
allocated 65536 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-10-07 20:43:48 +00:00
|
|
|
indirect = its_parse_indirect_baser(its, baser,
|
|
|
|
psz, &order,
|
|
|
|
its->device_ids);
|
2016-12-19 18:18:34 +00:00
|
|
|
case GITS_BASER_TYPE_VCPU:
|
|
|
|
indirect = its_parse_indirect_baser(its, baser,
|
irqchip/gic-v3-its: Fix the incorrect parsing of VCPU table size
The VCPU table consists of vPE entries, and its size provides the number
of VPEs supported by GICv4 hardware. Unfortunately the maximum size of
the VPE table is not discoverable like Device table. All VLPI commands
limits the number of bits to 16 to hold VPEID, which is index into VCPU
table. Don't apply DEVID bits for VCPU table instead assume maximum bits
to 16.
ITS log messages on QDF2400 without fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
Virtual CPUs Table too large, reduce ids 32->26
Virtual CPUs too large, reduce ITS pages 8192->256
allocated 2097152 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
ITS log messages on QDF2400 with fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
allocated 65536 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-10-07 20:43:48 +00:00
|
|
|
psz, &order,
|
|
|
|
ITS_MAX_VPEID_BITS);
|
2016-12-19 18:18:34 +00:00
|
|
|
break;
|
|
|
|
}
|
2015-03-06 16:37:41 +00:00
|
|
|
|
2016-06-06 23:17:32 +00:00
|
|
|
err = its_setup_baser(its, baser, cache, shr, psz, order, indirect);
|
2016-06-06 23:17:30 +00:00
|
|
|
if (err < 0) {
|
|
|
|
its_free_tables(its);
|
|
|
|
return err;
|
2014-11-24 14:35:14 +00:00
|
|
|
}
|
|
|
|
|
2016-06-06 23:17:30 +00:00
|
|
|
/* Update settings which will be used for next BASERn */
|
|
|
|
psz = baser->psz;
|
|
|
|
cache = baser->val & GITS_BASER_CACHEABILITY_MASK;
|
|
|
|
shr = baser->val & GITS_BASER_SHAREABILITY_MASK;
|
2014-11-24 14:35:14 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_alloc_collections(struct its_node *its)
|
|
|
|
{
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
int i;
|
|
|
|
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
its->collections = kcalloc(nr_cpu_ids, sizeof(*its->collections),
|
2014-11-24 14:35:14 +00:00
|
|
|
GFP_KERNEL);
|
|
|
|
if (!its->collections)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
irqchip/gic-v3-its: Only emit SYNC if targetting a valid collection
It is possible, under obscure circumstances, to convince the ITS driver to
emit a SYNC operation that targets a collection that is not bound to any
redistributor (and the target_address field is zero) because the
corresponding CPU has not been seen yet (the system has been booted with
max_cpus="something small").
If the ITS is using the linear CPU number as the target, this is not a big
deal, as we just end-up issuing a SYNC to CPU0. But if the ITS requires the
physical address of the redistributor (with GITS_TYPER.PTA==1), we end-up
asking the ITS to write to the physical address zero, which is not exactly
a good idea (there has been report of the ITS locking up). This should of
course never happen, but hey, this is SW...
In order to avoid the above disaster, let's track which collections have
been actually initialized, and let's not generate a SYNC if the collection
hasn't been properly bound to a redistributor. Take this opportunity to
spit our a warning, in the hope that someone may report the issue if it
arrises again.
Reported-by: Yang Yingliang <yangyingliang@huawei.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: Alexandre Belloni <alexandre.belloni@bootlin.com>
Cc: Sumit Garg <sumit.garg@linaro.org>
Link: https://lkml.kernel.org/r/20180622095254.5906-6-marc.zyngier@arm.com
2018-06-22 09:52:52 +00:00
|
|
|
for (i = 0; i < nr_cpu_ids; i++)
|
|
|
|
its->collections[i].target_address = ~0ULL;
|
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-19 18:34:38 +00:00
|
|
|
static struct page *its_allocate_pending_table(gfp_t gfp_flags)
|
|
|
|
{
|
|
|
|
struct page *pend_page;
|
|
|
|
/*
|
|
|
|
* The pending pages have to be at least 64kB aligned,
|
|
|
|
* hence the 'max(LPI_PENDBASE_SZ, SZ_64K)' below.
|
|
|
|
*/
|
|
|
|
pend_page = alloc_pages(gfp_flags | __GFP_ZERO,
|
|
|
|
get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
|
|
|
|
if (!pend_page)
|
|
|
|
return NULL;
|
|
|
|
|
|
|
|
/* Make sure the GIC will observe the zero-ed page */
|
|
|
|
gic_flush_dcache_to_poc(page_address(pend_page), LPI_PENDBASE_SZ);
|
|
|
|
|
|
|
|
return pend_page;
|
|
|
|
}
|
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
static void its_free_pending_table(struct page *pt)
|
|
|
|
{
|
|
|
|
free_pages((unsigned long)page_address(pt),
|
|
|
|
get_order(max_t(u32, LPI_PENDBASE_SZ, SZ_64K)));
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:14 +00:00
|
|
|
static void its_cpu_init_lpis(void)
|
|
|
|
{
|
|
|
|
void __iomem *rbase = gic_data_rdist_rd_base();
|
|
|
|
struct page *pend_page;
|
|
|
|
u64 val, tmp;
|
|
|
|
|
|
|
|
/* If we didn't allocate the pending table yet, do it now */
|
|
|
|
pend_page = gic_data_rdist()->pend_page;
|
|
|
|
if (!pend_page) {
|
|
|
|
phys_addr_t paddr;
|
2016-12-19 18:34:38 +00:00
|
|
|
|
|
|
|
pend_page = its_allocate_pending_table(GFP_NOWAIT);
|
2014-11-24 14:35:14 +00:00
|
|
|
if (!pend_page) {
|
|
|
|
pr_err("Failed to allocate PENDBASE for CPU%d\n",
|
|
|
|
smp_processor_id());
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
paddr = page_to_phys(pend_page);
|
|
|
|
pr_info("CPU%d: using LPI pending table @%pa\n",
|
|
|
|
smp_processor_id(), &paddr);
|
|
|
|
gic_data_rdist()->pend_page = pend_page;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set PROPBASE */
|
|
|
|
val = (page_to_phys(gic_rdists->prop_page) |
|
|
|
|
GICR_PROPBASER_InnerShareable |
|
2017-01-26 03:51:41 +00:00
|
|
|
GICR_PROPBASER_RaWaWb |
|
2014-11-24 14:35:14 +00:00
|
|
|
((LPI_NRBITS - 1) & GICR_PROPBASER_IDBITS_MASK));
|
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
gicr_write_propbaser(val, rbase + GICR_PROPBASER);
|
|
|
|
tmp = gicr_read_propbaser(rbase + GICR_PROPBASER);
|
2014-11-24 14:35:14 +00:00
|
|
|
|
|
|
|
if ((tmp ^ val) & GICR_PROPBASER_SHAREABILITY_MASK) {
|
2015-03-27 14:15:05 +00:00
|
|
|
if (!(tmp & GICR_PROPBASER_SHAREABILITY_MASK)) {
|
|
|
|
/*
|
|
|
|
* The HW reports non-shareable, we must
|
|
|
|
* remove the cacheability attributes as
|
|
|
|
* well.
|
|
|
|
*/
|
|
|
|
val &= ~(GICR_PROPBASER_SHAREABILITY_MASK |
|
|
|
|
GICR_PROPBASER_CACHEABILITY_MASK);
|
|
|
|
val |= GICR_PROPBASER_nC;
|
2016-11-02 11:54:06 +00:00
|
|
|
gicr_write_propbaser(val, rbase + GICR_PROPBASER);
|
2015-03-27 14:15:05 +00:00
|
|
|
}
|
2014-11-24 14:35:14 +00:00
|
|
|
pr_info_once("GIC: using cache flushing for LPI property table\n");
|
|
|
|
gic_rdists->flags |= RDIST_FLAGS_PROPBASE_NEEDS_FLUSHING;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* set PENDBASE */
|
|
|
|
val = (page_to_phys(pend_page) |
|
2015-03-27 14:15:04 +00:00
|
|
|
GICR_PENDBASER_InnerShareable |
|
2017-01-26 03:51:41 +00:00
|
|
|
GICR_PENDBASER_RaWaWb);
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
|
|
|
|
tmp = gicr_read_pendbaser(rbase + GICR_PENDBASER);
|
2015-03-27 14:15:05 +00:00
|
|
|
|
|
|
|
if (!(tmp & GICR_PENDBASER_SHAREABILITY_MASK)) {
|
|
|
|
/*
|
|
|
|
* The HW reports non-shareable, we must remove the
|
|
|
|
* cacheability attributes as well.
|
|
|
|
*/
|
|
|
|
val &= ~(GICR_PENDBASER_SHAREABILITY_MASK |
|
|
|
|
GICR_PENDBASER_CACHEABILITY_MASK);
|
|
|
|
val |= GICR_PENDBASER_nC;
|
2016-11-02 11:54:06 +00:00
|
|
|
gicr_write_pendbaser(val, rbase + GICR_PENDBASER);
|
2015-03-27 14:15:05 +00:00
|
|
|
}
|
2014-11-24 14:35:14 +00:00
|
|
|
|
|
|
|
/* Enable LPIs */
|
|
|
|
val = readl_relaxed(rbase + GICR_CTLR);
|
|
|
|
val |= GICR_CTLR_ENABLE_LPIS;
|
|
|
|
writel_relaxed(val, rbase + GICR_CTLR);
|
|
|
|
|
|
|
|
/* Make sure the GIC has seen the above */
|
|
|
|
dsb(sy);
|
|
|
|
}
|
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
static void its_cpu_init_collection(struct its_node *its)
|
2014-11-24 14:35:14 +00:00
|
|
|
{
|
2018-03-01 05:48:20 +00:00
|
|
|
int cpu = smp_processor_id();
|
|
|
|
u64 target;
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
/* avoid cross node collections and its mapping */
|
|
|
|
if (its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144) {
|
|
|
|
struct device_node *cpu_node;
|
2016-05-25 13:29:20 +00:00
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
cpu_node = of_get_cpu_node(cpu, NULL);
|
|
|
|
if (its->numa_node != NUMA_NO_NODE &&
|
|
|
|
its->numa_node != of_node_to_nid(cpu_node))
|
|
|
|
return;
|
|
|
|
}
|
2016-05-25 13:29:20 +00:00
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
/*
|
|
|
|
* We now have to bind each collection to its target
|
|
|
|
* redistributor.
|
|
|
|
*/
|
|
|
|
if (gic_read_typer(its->base + GITS_TYPER) & GITS_TYPER_PTA) {
|
2014-11-24 14:35:14 +00:00
|
|
|
/*
|
2018-03-01 05:48:20 +00:00
|
|
|
* This ITS wants the physical address of the
|
2014-11-24 14:35:14 +00:00
|
|
|
* redistributor.
|
|
|
|
*/
|
2018-03-01 05:48:20 +00:00
|
|
|
target = gic_data_rdist()->phys_base;
|
|
|
|
} else {
|
|
|
|
/* This ITS wants a linear CPU number. */
|
|
|
|
target = gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER);
|
|
|
|
target = GICR_TYPER_CPU_NUMBER(target) << 16;
|
|
|
|
}
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
/* Perform collection mapping */
|
|
|
|
its->collections[cpu].target_address = target;
|
|
|
|
its->collections[cpu].col_id = cpu;
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2018-03-01 05:48:20 +00:00
|
|
|
its_send_mapc(its, &its->collections[cpu], 1);
|
|
|
|
its_send_invall(its, &its->collections[cpu]);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_cpu_init_collections(void)
|
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_lock(&its_lock);
|
2018-03-01 05:48:20 +00:00
|
|
|
|
|
|
|
list_for_each_entry(its, &its_nodes, entry)
|
|
|
|
its_cpu_init_collection(its);
|
2014-11-24 14:35:14 +00:00
|
|
|
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_unlock(&its_lock);
|
2014-11-24 14:35:14 +00:00
|
|
|
}
|
2014-11-24 14:35:15 +00:00
|
|
|
|
|
|
|
static struct its_device *its_find_device(struct its_node *its, u32 dev_id)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = NULL, *tmp;
|
2015-03-06 16:37:43 +00:00
|
|
|
unsigned long flags;
|
2014-11-24 14:35:15 +00:00
|
|
|
|
2015-03-06 16:37:43 +00:00
|
|
|
raw_spin_lock_irqsave(&its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
|
|
|
|
list_for_each_entry(tmp, &its->its_device_list, entry) {
|
|
|
|
if (tmp->device_id == dev_id) {
|
|
|
|
its_dev = tmp;
|
|
|
|
break;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2015-03-06 16:37:43 +00:00
|
|
|
raw_spin_unlock_irqrestore(&its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
|
|
|
|
return its_dev;
|
|
|
|
}
|
|
|
|
|
2016-03-10 04:10:49 +00:00
|
|
|
static struct its_baser *its_get_baser(struct its_node *its, u32 type)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < GITS_BASER_NR_REGS; i++) {
|
|
|
|
if (GITS_BASER_TYPE(its->tables[i].val) == type)
|
|
|
|
return &its->tables[i];
|
|
|
|
}
|
|
|
|
|
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-12-19 18:53:02 +00:00
|
|
|
static bool its_alloc_table_entry(struct its_baser *baser, u32 id)
|
2016-06-06 23:17:32 +00:00
|
|
|
{
|
|
|
|
struct page *page;
|
|
|
|
u32 esz, idx;
|
|
|
|
__le64 *table;
|
|
|
|
|
|
|
|
/* Don't allow device id that exceeds single, flat table limit */
|
|
|
|
esz = GITS_BASER_ENTRY_SIZE(baser->val);
|
|
|
|
if (!(baser->val & GITS_BASER_INDIRECT))
|
2016-12-19 18:53:02 +00:00
|
|
|
return (id < (PAGE_ORDER_TO_SIZE(baser->order) / esz));
|
2016-06-06 23:17:32 +00:00
|
|
|
|
|
|
|
/* Compute 1st level table index & check if that exceeds table limit */
|
2016-12-19 18:53:02 +00:00
|
|
|
idx = id >> ilog2(baser->psz / esz);
|
2016-06-06 23:17:32 +00:00
|
|
|
if (idx >= (PAGE_ORDER_TO_SIZE(baser->order) / GITS_LVL1_ENTRY_SIZE))
|
|
|
|
return false;
|
|
|
|
|
|
|
|
table = baser->base;
|
|
|
|
|
|
|
|
/* Allocate memory for 2nd level table */
|
|
|
|
if (!table[idx]) {
|
|
|
|
page = alloc_pages(GFP_KERNEL | __GFP_ZERO, get_order(baser->psz));
|
|
|
|
if (!page)
|
|
|
|
return false;
|
|
|
|
|
|
|
|
/* Flush Lvl2 table to PoC if hw doesn't support coherency */
|
|
|
|
if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(page_address(page), baser->psz);
|
2016-06-06 23:17:32 +00:00
|
|
|
|
|
|
|
table[idx] = cpu_to_le64(page_to_phys(page) | GITS_BASER_VALID);
|
|
|
|
|
|
|
|
/* Flush Lvl1 entry to PoC if hw doesn't support coherency */
|
|
|
|
if (!(baser->val & GITS_BASER_SHAREABILITY_MASK))
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(table + idx, GITS_LVL1_ENTRY_SIZE);
|
2016-06-06 23:17:32 +00:00
|
|
|
|
|
|
|
/* Ensure updated table contents are visible to ITS hardware */
|
|
|
|
dsb(sy);
|
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2016-12-19 18:53:02 +00:00
|
|
|
static bool its_alloc_device_table(struct its_node *its, u32 dev_id)
|
|
|
|
{
|
|
|
|
struct its_baser *baser;
|
|
|
|
|
|
|
|
baser = its_get_baser(its, GITS_BASER_TYPE_DEVICE);
|
|
|
|
|
|
|
|
/* Don't allow device id that exceeds ITS hardware limit */
|
|
|
|
if (!baser)
|
|
|
|
return (ilog2(dev_id) < its->device_ids);
|
|
|
|
|
|
|
|
return its_alloc_table_entry(baser, dev_id);
|
|
|
|
}
|
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
static bool its_alloc_vpe_table(u32 vpe_id)
|
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure the L2 tables are allocated on *all* v4 ITSs. We
|
|
|
|
* could try and only do it on ITSs corresponding to devices
|
|
|
|
* that have interrupts targeted at this VPE, but the
|
|
|
|
* complexity becomes crazy (and you have tons of memory
|
|
|
|
* anyway, right?).
|
|
|
|
*/
|
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
struct its_baser *baser;
|
|
|
|
|
|
|
|
if (!its->is_v4)
|
|
|
|
continue;
|
2016-06-06 23:17:32 +00:00
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
baser = its_get_baser(its, GITS_BASER_TYPE_VCPU);
|
|
|
|
if (!baser)
|
|
|
|
return false;
|
2016-06-06 23:17:32 +00:00
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
if (!its_alloc_table_entry(baser, vpe_id))
|
|
|
|
return false;
|
2016-06-06 23:17:32 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:15 +00:00
|
|
|
static struct its_device *its_create_device(struct its_node *its, u32 dev_id,
|
2017-08-04 17:37:09 +00:00
|
|
|
int nvecs, bool alloc_lpis)
|
2014-11-24 14:35:15 +00:00
|
|
|
{
|
|
|
|
struct its_device *dev;
|
2017-08-04 17:37:09 +00:00
|
|
|
unsigned long *lpi_map = NULL;
|
2015-03-06 16:37:43 +00:00
|
|
|
unsigned long flags;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
u16 *col_map = NULL;
|
2014-11-24 14:35:15 +00:00
|
|
|
void *itt;
|
|
|
|
int lpi_base;
|
|
|
|
int nr_lpis;
|
2014-12-12 10:51:24 +00:00
|
|
|
int nr_ites;
|
2014-11-24 14:35:15 +00:00
|
|
|
int sz;
|
|
|
|
|
2016-06-06 23:17:32 +00:00
|
|
|
if (!its_alloc_device_table(its, dev_id))
|
2016-03-10 04:10:49 +00:00
|
|
|
return NULL;
|
|
|
|
|
2018-05-27 15:39:55 +00:00
|
|
|
if (WARN_ON(!is_power_of_2(nvecs)))
|
|
|
|
nvecs = roundup_pow_of_two(nvecs);
|
|
|
|
|
2014-11-24 14:35:15 +00:00
|
|
|
dev = kzalloc(sizeof(*dev), GFP_KERNEL);
|
2014-12-12 10:51:24 +00:00
|
|
|
/*
|
2018-05-27 15:39:55 +00:00
|
|
|
* Even if the device wants a single LPI, the ITT must be
|
|
|
|
* sized as a power of two (and you need at least one bit...).
|
2014-12-12 10:51:24 +00:00
|
|
|
*/
|
2018-05-27 15:39:55 +00:00
|
|
|
nr_ites = max(2, nvecs);
|
2014-12-12 10:51:24 +00:00
|
|
|
sz = nr_ites * its->ite_size;
|
2014-11-24 14:35:15 +00:00
|
|
|
sz = max(sz, ITS_ITT_ALIGN) + ITS_ITT_ALIGN - 1;
|
2015-03-06 16:37:46 +00:00
|
|
|
itt = kzalloc(sz, GFP_KERNEL);
|
2017-08-04 17:37:09 +00:00
|
|
|
if (alloc_lpis) {
|
2018-05-27 16:03:03 +00:00
|
|
|
lpi_map = its_lpi_alloc(nvecs, &lpi_base, &nr_lpis);
|
2017-08-04 17:37:09 +00:00
|
|
|
if (lpi_map)
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
col_map = kcalloc(nr_lpis, sizeof(*col_map),
|
2017-08-04 17:37:09 +00:00
|
|
|
GFP_KERNEL);
|
|
|
|
} else {
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
col_map = kcalloc(nr_ites, sizeof(*col_map), GFP_KERNEL);
|
2017-08-04 17:37:09 +00:00
|
|
|
nr_lpis = 0;
|
|
|
|
lpi_base = 0;
|
|
|
|
}
|
2014-11-24 14:35:15 +00:00
|
|
|
|
2017-08-04 17:37:09 +00:00
|
|
|
if (!dev || !itt || !col_map || (!lpi_map && alloc_lpis)) {
|
2014-11-24 14:35:15 +00:00
|
|
|
kfree(dev);
|
|
|
|
kfree(itt);
|
|
|
|
kfree(lpi_map);
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
kfree(col_map);
|
2014-11-24 14:35:15 +00:00
|
|
|
return NULL;
|
|
|
|
}
|
|
|
|
|
2016-11-02 11:54:05 +00:00
|
|
|
gic_flush_dcache_to_poc(itt, sz);
|
2015-09-13 11:14:32 +00:00
|
|
|
|
2014-11-24 14:35:15 +00:00
|
|
|
dev->its = its;
|
|
|
|
dev->itt = itt;
|
2014-12-12 10:51:24 +00:00
|
|
|
dev->nr_ites = nr_ites;
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
dev->event_map.lpi_map = lpi_map;
|
|
|
|
dev->event_map.col_map = col_map;
|
|
|
|
dev->event_map.lpi_base = lpi_base;
|
|
|
|
dev->event_map.nr_lpis = nr_lpis;
|
2016-12-20 09:44:41 +00:00
|
|
|
mutex_init(&dev->event_map.vlpi_lock);
|
2014-11-24 14:35:15 +00:00
|
|
|
dev->device_id = dev_id;
|
|
|
|
INIT_LIST_HEAD(&dev->entry);
|
|
|
|
|
2015-03-06 16:37:43 +00:00
|
|
|
raw_spin_lock_irqsave(&its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
list_add(&dev->entry, &its->its_device_list);
|
2015-03-06 16:37:43 +00:00
|
|
|
raw_spin_unlock_irqrestore(&its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
|
|
|
|
/* Map device to its ITT */
|
|
|
|
its_send_mapd(dev, 1);
|
|
|
|
|
|
|
|
return dev;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_free_device(struct its_device *its_dev)
|
|
|
|
{
|
2015-03-06 16:37:43 +00:00
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&its_dev->its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
list_del(&its_dev->entry);
|
2015-03-06 16:37:43 +00:00
|
|
|
raw_spin_unlock_irqrestore(&its_dev->its->lock, flags);
|
2014-11-24 14:35:15 +00:00
|
|
|
kfree(its_dev->itt);
|
|
|
|
kfree(its_dev);
|
|
|
|
}
|
2014-11-24 14:35:16 +00:00
|
|
|
|
|
|
|
static int its_alloc_device_irq(struct its_device *dev, irq_hw_number_t *hwirq)
|
|
|
|
{
|
|
|
|
int idx;
|
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
idx = find_first_zero_bit(dev->event_map.lpi_map,
|
|
|
|
dev->event_map.nr_lpis);
|
|
|
|
if (idx == dev->event_map.nr_lpis)
|
2014-11-24 14:35:16 +00:00
|
|
|
return -ENOSPC;
|
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
*hwirq = dev->event_map.lpi_base + idx;
|
|
|
|
set_bit(idx, dev->event_map.lpi_map);
|
2014-11-24 14:35:16 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-07-28 13:46:21 +00:00
|
|
|
static int its_msi_prepare(struct irq_domain *domain, struct device *dev,
|
|
|
|
int nvec, msi_alloc_info_t *info)
|
2015-03-06 16:37:42 +00:00
|
|
|
{
|
2014-11-24 14:35:16 +00:00
|
|
|
struct its_node *its;
|
|
|
|
struct its_device *its_dev;
|
2015-07-28 13:46:21 +00:00
|
|
|
struct msi_domain_info *msi_info;
|
|
|
|
u32 dev_id;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We ignore "dev" entierely, and rely on the dev_id that has
|
|
|
|
* been passed via the scratchpad. This limits this domain's
|
|
|
|
* usefulness to upper layers that definitely know that they
|
|
|
|
* are built on top of the ITS.
|
|
|
|
*/
|
|
|
|
dev_id = info->scratchpad[0].ul;
|
|
|
|
|
|
|
|
msi_info = msi_get_domain_info(domain);
|
|
|
|
its = msi_info->data;
|
2015-03-06 16:37:42 +00:00
|
|
|
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
if (!gic_rdists->has_direct_lpi &&
|
|
|
|
vpe_proxy.dev &&
|
|
|
|
vpe_proxy.dev->its == its &&
|
|
|
|
dev_id == vpe_proxy.dev->device_id) {
|
|
|
|
/* Bad luck. Get yourself a better implementation */
|
|
|
|
WARN_ONCE(1, "DevId %x clashes with GICv4 VPE proxy device\n",
|
|
|
|
dev_id);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
2015-07-28 13:46:18 +00:00
|
|
|
its_dev = its_find_device(its, dev_id);
|
2015-03-06 16:37:42 +00:00
|
|
|
if (its_dev) {
|
|
|
|
/*
|
|
|
|
* We already have seen this ID, probably through
|
|
|
|
* another alias (PCI bridge of some sort). No need to
|
|
|
|
* create the device.
|
|
|
|
*/
|
2015-07-28 13:46:18 +00:00
|
|
|
pr_debug("Reusing ITT for devID %x\n", dev_id);
|
2015-03-06 16:37:42 +00:00
|
|
|
goto out;
|
|
|
|
}
|
2014-11-24 14:35:16 +00:00
|
|
|
|
2017-08-04 17:37:09 +00:00
|
|
|
its_dev = its_create_device(its, dev_id, nvec, true);
|
2014-11-24 14:35:16 +00:00
|
|
|
if (!its_dev)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2015-07-28 13:46:18 +00:00
|
|
|
pr_debug("ITT %d entries, %d bits\n", nvec, ilog2(nvec));
|
2015-03-06 16:37:42 +00:00
|
|
|
out:
|
2014-11-24 14:35:16 +00:00
|
|
|
info->scratchpad[0].ptr = its_dev;
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2015-07-28 13:46:21 +00:00
|
|
|
static struct msi_domain_ops its_msi_domain_ops = {
|
|
|
|
.msi_prepare = its_msi_prepare,
|
|
|
|
};
|
|
|
|
|
2014-11-24 14:35:16 +00:00
|
|
|
static int its_irq_gic_domain_alloc(struct irq_domain *domain,
|
|
|
|
unsigned int virq,
|
|
|
|
irq_hw_number_t hwirq)
|
|
|
|
{
|
2015-10-13 11:51:33 +00:00
|
|
|
struct irq_fwspec fwspec;
|
|
|
|
|
|
|
|
if (irq_domain_get_of_node(domain->parent)) {
|
|
|
|
fwspec.fwnode = domain->parent->fwnode;
|
|
|
|
fwspec.param_count = 3;
|
|
|
|
fwspec.param[0] = GIC_IRQ_TYPE_LPI;
|
|
|
|
fwspec.param[1] = hwirq;
|
|
|
|
fwspec.param[2] = IRQ_TYPE_EDGE_RISING;
|
2016-09-12 18:32:25 +00:00
|
|
|
} else if (is_fwnode_irqchip(domain->parent->fwnode)) {
|
|
|
|
fwspec.fwnode = domain->parent->fwnode;
|
|
|
|
fwspec.param_count = 2;
|
|
|
|
fwspec.param[0] = hwirq;
|
|
|
|
fwspec.param[1] = IRQ_TYPE_EDGE_RISING;
|
2015-10-13 11:51:33 +00:00
|
|
|
} else {
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
2014-11-24 14:35:16 +00:00
|
|
|
|
2015-10-13 11:51:33 +00:00
|
|
|
return irq_domain_alloc_irqs_parent(domain, virq, 1, &fwspec);
|
2014-11-24 14:35:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static int its_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
|
|
|
|
unsigned int nr_irqs, void *args)
|
|
|
|
{
|
|
|
|
msi_alloc_info_t *info = args;
|
|
|
|
struct its_device *its_dev = info->scratchpad[0].ptr;
|
|
|
|
irq_hw_number_t hwirq;
|
|
|
|
int err;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_irqs; i++) {
|
|
|
|
err = its_alloc_device_irq(its_dev, &hwirq);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
err = its_irq_gic_domain_alloc(domain, virq + i, hwirq);
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
irq_domain_set_hwirq_and_chip(domain, virq + i,
|
|
|
|
hwirq, &its_irq_chip, its_dev);
|
2017-08-18 08:39:18 +00:00
|
|
|
irqd_set_single_target(irq_desc_get_irq_data(irq_to_desc(virq + i)));
|
2015-07-28 13:46:18 +00:00
|
|
|
pr_debug("ID:%d pID:%d vID:%d\n",
|
|
|
|
(int)(hwirq - its_dev->event_map.lpi_base),
|
|
|
|
(int) hwirq, virq + i);
|
2014-11-24 14:35:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-09-13 21:29:10 +00:00
|
|
|
static int its_irq_domain_activate(struct irq_domain *domain,
|
2017-12-29 15:59:06 +00:00
|
|
|
struct irq_data *d, bool reserve)
|
2014-12-12 10:51:23 +00:00
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
2016-05-25 13:29:20 +00:00
|
|
|
const struct cpumask *cpu_mask = cpu_online_mask;
|
2017-08-18 08:39:18 +00:00
|
|
|
int cpu;
|
2016-05-25 13:29:20 +00:00
|
|
|
|
|
|
|
/* get the cpu_mask of local node */
|
|
|
|
if (its_dev->its->numa_node >= 0)
|
|
|
|
cpu_mask = cpumask_of_node(its_dev->its->numa_node);
|
2014-12-12 10:51:23 +00:00
|
|
|
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
/* Bind the LPI to the first possible CPU */
|
2018-06-22 09:52:51 +00:00
|
|
|
cpu = cpumask_first_and(cpu_mask, cpu_online_mask);
|
|
|
|
if (cpu >= nr_cpu_ids) {
|
|
|
|
if (its_dev->its->flags & ITS_FLAGS_WORKAROUND_CAVIUM_23144)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
cpu = cpumask_first(cpu_online_mask);
|
|
|
|
}
|
|
|
|
|
2017-08-18 08:39:18 +00:00
|
|
|
its_dev->event_map.col_map[event] = cpu;
|
|
|
|
irq_data_update_effective_affinity(d, cpumask_of(cpu));
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
|
2014-12-12 10:51:23 +00:00
|
|
|
/* Map the GIC IRQ and event to the device */
|
2016-12-20 15:52:26 +00:00
|
|
|
its_send_mapti(its_dev, d->hwirq, event);
|
2017-09-13 21:29:10 +00:00
|
|
|
return 0;
|
2014-12-12 10:51:23 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_irq_domain_deactivate(struct irq_domain *domain,
|
|
|
|
struct irq_data *d)
|
|
|
|
{
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
u32 event = its_get_event_id(d);
|
|
|
|
|
|
|
|
/* Stop the delivery of interrupts */
|
|
|
|
its_send_discard(its_dev, event);
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:16 +00:00
|
|
|
static void its_irq_domain_free(struct irq_domain *domain, unsigned int virq,
|
|
|
|
unsigned int nr_irqs)
|
|
|
|
{
|
|
|
|
struct irq_data *d = irq_domain_get_irq_data(domain, virq);
|
|
|
|
struct its_device *its_dev = irq_data_get_irq_chip_data(d);
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_irqs; i++) {
|
|
|
|
struct irq_data *data = irq_domain_get_irq_data(domain,
|
|
|
|
virq + i);
|
2014-12-12 10:51:23 +00:00
|
|
|
u32 event = its_get_event_id(data);
|
2014-11-24 14:35:16 +00:00
|
|
|
|
|
|
|
/* Mark interrupt index as unused */
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
clear_bit(event, its_dev->event_map.lpi_map);
|
2014-11-24 14:35:16 +00:00
|
|
|
|
|
|
|
/* Nuke the entry in the domain */
|
2014-12-12 10:51:22 +00:00
|
|
|
irq_domain_reset_irq_data(data);
|
2014-11-24 14:35:16 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
/* If all interrupts have been freed, start mopping the floor */
|
irqchip/gicv3-its: Fix mapping of LPIs to collections
The GICv3 ITS architecture allows a given [DevID, EventID] pair to be
translated to a [LPI, Collection] pair, where DevID is the device writing
the MSI, EventID is the payload being written, LPI is the actual
interrupt number, and Collection is roughly equivalent to a target CPU.
Each LPI can be mapped to a separate collection, but the ITS driver
insists on maintaining the collection on a device basis, instead of doing
it on a per interrupt basis.
This is obviously flawed, and this patch fixes it by adding a per interrupt
index that indicates which collection number is in use.
Reported-by: Ian Campbell <ian.campbell@citrix.com>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
Cc: <linux-arm-kernel@lists.infradead.org>
Cc: Jason Cooper <jason@lakedaemon.net>
Cc: stable@vger.kernel.org # 4.1, 4.0
Link: http://lkml.kernel.org/r/1437126402-11677-1-git-send-email-marc.zyngier@arm.com
Signed-off-by: Thomas Gleixner <tglx@linutronix.de>
2015-07-17 09:46:42 +00:00
|
|
|
if (bitmap_empty(its_dev->event_map.lpi_map,
|
|
|
|
its_dev->event_map.nr_lpis)) {
|
2018-05-27 16:03:03 +00:00
|
|
|
its_lpi_free(its_dev->event_map.lpi_map,
|
|
|
|
its_dev->event_map.lpi_base,
|
|
|
|
its_dev->event_map.nr_lpis);
|
2016-12-19 18:49:59 +00:00
|
|
|
kfree(its_dev->event_map.col_map);
|
2014-11-24 14:35:16 +00:00
|
|
|
|
|
|
|
/* Unmap device/itt */
|
|
|
|
its_send_mapd(its_dev, 0);
|
|
|
|
its_free_device(its_dev);
|
|
|
|
}
|
|
|
|
|
|
|
|
irq_domain_free_irqs_parent(domain, virq, nr_irqs);
|
|
|
|
}
|
|
|
|
|
|
|
|
static const struct irq_domain_ops its_domain_ops = {
|
|
|
|
.alloc = its_irq_domain_alloc,
|
|
|
|
.free = its_irq_domain_free,
|
2014-12-12 10:51:23 +00:00
|
|
|
.activate = its_irq_domain_activate,
|
|
|
|
.deactivate = its_irq_domain_deactivate,
|
2014-11-24 14:35:16 +00:00
|
|
|
};
|
2014-11-24 14:35:17 +00:00
|
|
|
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
/*
|
|
|
|
* This is insane.
|
|
|
|
*
|
|
|
|
* If a GICv4 doesn't implement Direct LPIs (which is extremely
|
|
|
|
* likely), the only way to perform an invalidate is to use a fake
|
|
|
|
* device to issue an INV command, implying that the LPI has first
|
|
|
|
* been mapped to some event on that device. Since this is not exactly
|
|
|
|
* cheap, we try to keep that mapping around as long as possible, and
|
|
|
|
* only issue an UNMAP if we're short on available slots.
|
|
|
|
*
|
|
|
|
* Broken by design(tm).
|
|
|
|
*/
|
|
|
|
static void its_vpe_db_proxy_unmap_locked(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
/* Already unmapped? */
|
|
|
|
if (vpe->vpe_proxy_event == -1)
|
|
|
|
return;
|
|
|
|
|
|
|
|
its_send_discard(vpe_proxy.dev, vpe->vpe_proxy_event);
|
|
|
|
vpe_proxy.vpes[vpe->vpe_proxy_event] = NULL;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* We don't track empty slots at all, so let's move the
|
|
|
|
* next_victim pointer if we can quickly reuse that slot
|
|
|
|
* instead of nuking an existing entry. Not clear that this is
|
|
|
|
* always a win though, and this might just generate a ripple
|
|
|
|
* effect... Let's just hope VPEs don't migrate too often.
|
|
|
|
*/
|
|
|
|
if (vpe_proxy.vpes[vpe_proxy.next_victim])
|
|
|
|
vpe_proxy.next_victim = vpe->vpe_proxy_event;
|
|
|
|
|
|
|
|
vpe->vpe_proxy_event = -1;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_db_proxy_unmap(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
if (!gic_rdists->has_direct_lpi) {
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
|
|
|
|
its_vpe_db_proxy_unmap_locked(vpe);
|
|
|
|
raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_db_proxy_map_locked(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
/* Already mapped? */
|
|
|
|
if (vpe->vpe_proxy_event != -1)
|
|
|
|
return;
|
|
|
|
|
|
|
|
/* This slot was already allocated. Kick the other VPE out. */
|
|
|
|
if (vpe_proxy.vpes[vpe_proxy.next_victim])
|
|
|
|
its_vpe_db_proxy_unmap_locked(vpe_proxy.vpes[vpe_proxy.next_victim]);
|
|
|
|
|
|
|
|
/* Map the new VPE instead */
|
|
|
|
vpe_proxy.vpes[vpe_proxy.next_victim] = vpe;
|
|
|
|
vpe->vpe_proxy_event = vpe_proxy.next_victim;
|
|
|
|
vpe_proxy.next_victim = (vpe_proxy.next_victim + 1) % vpe_proxy.dev->nr_ites;
|
|
|
|
|
|
|
|
vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = vpe->col_idx;
|
|
|
|
its_send_mapti(vpe_proxy.dev, vpe->vpe_db_lpi, vpe->vpe_proxy_event);
|
|
|
|
}
|
|
|
|
|
2017-08-18 15:14:17 +00:00
|
|
|
static void its_vpe_db_proxy_move(struct its_vpe *vpe, int from, int to)
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
struct its_collection *target_col;
|
|
|
|
|
|
|
|
if (gic_rdists->has_direct_lpi) {
|
|
|
|
void __iomem *rdbase;
|
|
|
|
|
|
|
|
rdbase = per_cpu_ptr(gic_rdists->rdist, from)->rd_base;
|
|
|
|
gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
|
|
|
|
while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
|
|
|
|
cpu_relax();
|
|
|
|
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
|
|
|
|
|
|
|
|
its_vpe_db_proxy_map_locked(vpe);
|
|
|
|
|
|
|
|
target_col = &vpe_proxy.dev->its->collections[to];
|
|
|
|
its_send_movi(vpe_proxy.dev, target_col, vpe->vpe_proxy_event);
|
|
|
|
vpe_proxy.dev->event_map.col_map[vpe->vpe_proxy_event] = to;
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
static int its_vpe_set_affinity(struct irq_data *d,
|
|
|
|
const struct cpumask *mask_val,
|
|
|
|
bool force)
|
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
|
|
|
int cpu = cpumask_first(mask_val);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Changing affinity is mega expensive, so let's be as lazy as
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
* we can and only do it if we really have to. Also, if mapped
|
2017-08-18 15:14:17 +00:00
|
|
|
* into the proxy device, we need to move the doorbell
|
|
|
|
* interrupt to its new location.
|
2016-12-20 15:17:28 +00:00
|
|
|
*/
|
|
|
|
if (vpe->col_idx != cpu) {
|
2017-08-18 15:14:17 +00:00
|
|
|
int from = vpe->col_idx;
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
vpe->col_idx = cpu;
|
|
|
|
its_send_vmovp(vpe);
|
2017-08-18 15:14:17 +00:00
|
|
|
its_vpe_db_proxy_move(vpe, from, cpu);
|
2016-12-20 15:17:28 +00:00
|
|
|
}
|
|
|
|
|
2017-10-19 09:11:34 +00:00
|
|
|
irq_data_update_effective_affinity(d, cpumask_of(cpu));
|
|
|
|
|
2016-12-20 15:17:28 +00:00
|
|
|
return IRQ_SET_MASK_OK_DONE;
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:09:31 +00:00
|
|
|
static void its_vpe_schedule(struct its_vpe *vpe)
|
|
|
|
{
|
2018-02-16 16:57:56 +00:00
|
|
|
void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
|
2016-12-20 15:09:31 +00:00
|
|
|
u64 val;
|
|
|
|
|
|
|
|
/* Schedule the VPE */
|
|
|
|
val = virt_to_phys(page_address(vpe->its_vm->vprop_page)) &
|
|
|
|
GENMASK_ULL(51, 12);
|
|
|
|
val |= (LPI_NRBITS - 1) & GICR_VPROPBASER_IDBITS_MASK;
|
|
|
|
val |= GICR_VPROPBASER_RaWb;
|
|
|
|
val |= GICR_VPROPBASER_InnerShareable;
|
|
|
|
gits_write_vpropbaser(val, vlpi_base + GICR_VPROPBASER);
|
|
|
|
|
|
|
|
val = virt_to_phys(page_address(vpe->vpt_page)) &
|
|
|
|
GENMASK_ULL(51, 16);
|
|
|
|
val |= GICR_VPENDBASER_RaWaWb;
|
|
|
|
val |= GICR_VPENDBASER_NonShareable;
|
|
|
|
/*
|
|
|
|
* There is no good way of finding out if the pending table is
|
|
|
|
* empty as we can race against the doorbell interrupt very
|
|
|
|
* easily. So in the end, vpe->pending_last is only an
|
|
|
|
* indication that the vcpu has something pending, not one
|
|
|
|
* that the pending table is empty. A good implementation
|
|
|
|
* would be able to read its coarse map pretty quickly anyway,
|
|
|
|
* making this a tolerable issue.
|
|
|
|
*/
|
|
|
|
val |= GICR_VPENDBASER_PendingLast;
|
|
|
|
val |= vpe->idai ? GICR_VPENDBASER_IDAI : 0;
|
|
|
|
val |= GICR_VPENDBASER_Valid;
|
|
|
|
gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_deschedule(struct its_vpe *vpe)
|
|
|
|
{
|
2018-02-16 16:57:56 +00:00
|
|
|
void __iomem *vlpi_base = gic_data_rdist_vlpi_base();
|
2016-12-20 15:09:31 +00:00
|
|
|
u32 count = 1000000; /* 1s! */
|
|
|
|
bool clean;
|
|
|
|
u64 val;
|
|
|
|
|
|
|
|
/* We're being scheduled out */
|
|
|
|
val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
|
|
|
|
val &= ~GICR_VPENDBASER_Valid;
|
|
|
|
gits_write_vpendbaser(val, vlpi_base + GICR_VPENDBASER);
|
|
|
|
|
|
|
|
do {
|
|
|
|
val = gits_read_vpendbaser(vlpi_base + GICR_VPENDBASER);
|
|
|
|
clean = !(val & GICR_VPENDBASER_Dirty);
|
|
|
|
if (!clean) {
|
|
|
|
count--;
|
|
|
|
cpu_relax();
|
|
|
|
udelay(1);
|
|
|
|
}
|
|
|
|
} while (!clean && count);
|
|
|
|
|
|
|
|
if (unlikely(!clean && !count)) {
|
|
|
|
pr_err_ratelimited("ITS virtual pending table not cleaning\n");
|
|
|
|
vpe->idai = false;
|
|
|
|
vpe->pending_last = true;
|
|
|
|
} else {
|
|
|
|
vpe->idai = !!(val & GICR_VPENDBASER_IDAI);
|
|
|
|
vpe->pending_last = !!(val & GICR_VPENDBASER_PendingLast);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-08 14:16:09 +00:00
|
|
|
static void its_vpe_invall(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
|
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
if (!its->is_v4)
|
|
|
|
continue;
|
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
if (its_list_map && !vpe->its_vm->vlpi_count[its->list_nr])
|
|
|
|
continue;
|
|
|
|
|
2017-10-09 12:17:43 +00:00
|
|
|
/*
|
|
|
|
* Sending a VINVALL to a single ITS is enough, as all
|
|
|
|
* we need is to reach the redistributors.
|
|
|
|
*/
|
2017-10-08 14:16:09 +00:00
|
|
|
its_send_vinvall(its, vpe);
|
2017-10-09 12:17:43 +00:00
|
|
|
return;
|
2017-10-08 14:16:09 +00:00
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:09:31 +00:00
|
|
|
static int its_vpe_set_vcpu_affinity(struct irq_data *d, void *vcpu_info)
|
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
|
|
|
struct its_cmd_info *info = vcpu_info;
|
|
|
|
|
|
|
|
switch (info->cmd_type) {
|
|
|
|
case SCHEDULE_VPE:
|
|
|
|
its_vpe_schedule(vpe);
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
case DESCHEDULE_VPE:
|
|
|
|
its_vpe_deschedule(vpe);
|
|
|
|
return 0;
|
|
|
|
|
2016-12-20 15:10:50 +00:00
|
|
|
case INVALL_VPE:
|
2017-10-08 14:16:09 +00:00
|
|
|
its_vpe_invall(vpe);
|
2016-12-20 15:10:50 +00:00
|
|
|
return 0;
|
|
|
|
|
2016-12-20 15:09:31 +00:00
|
|
|
default:
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
static void its_vpe_send_cmd(struct its_vpe *vpe,
|
|
|
|
void (*cmd)(struct its_device *, u32))
|
|
|
|
{
|
|
|
|
unsigned long flags;
|
|
|
|
|
|
|
|
raw_spin_lock_irqsave(&vpe_proxy.lock, flags);
|
|
|
|
|
|
|
|
its_vpe_db_proxy_map_locked(vpe);
|
|
|
|
cmd(vpe_proxy.dev, vpe->vpe_proxy_event);
|
|
|
|
|
|
|
|
raw_spin_unlock_irqrestore(&vpe_proxy.lock, flags);
|
|
|
|
}
|
|
|
|
|
2016-12-20 15:20:38 +00:00
|
|
|
static void its_vpe_send_inv(struct irq_data *d)
|
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
|
|
|
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
if (gic_rdists->has_direct_lpi) {
|
|
|
|
void __iomem *rdbase;
|
|
|
|
|
|
|
|
rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
|
|
|
|
gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_INVLPIR);
|
|
|
|
while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
|
|
|
|
cpu_relax();
|
|
|
|
} else {
|
|
|
|
its_vpe_send_cmd(vpe, its_send_inv);
|
|
|
|
}
|
2016-12-20 15:20:38 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_mask_irq(struct irq_data *d)
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* We need to unmask the LPI, which is described by the parent
|
|
|
|
* irq_data. Instead of calling into the parent (which won't
|
|
|
|
* exactly do the right thing, let's simply use the
|
|
|
|
* parent_data pointer. Yes, I'm naughty.
|
|
|
|
*/
|
|
|
|
lpi_write_config(d->parent_data, LPI_PROP_ENABLED, 0);
|
|
|
|
its_vpe_send_inv(d);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_unmask_irq(struct irq_data *d)
|
|
|
|
{
|
|
|
|
/* Same hack as above... */
|
|
|
|
lpi_write_config(d->parent_data, 0, LPI_PROP_ENABLED);
|
|
|
|
its_vpe_send_inv(d);
|
|
|
|
}
|
|
|
|
|
2017-07-31 13:47:24 +00:00
|
|
|
static int its_vpe_set_irqchip_state(struct irq_data *d,
|
|
|
|
enum irqchip_irq_state which,
|
|
|
|
bool state)
|
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
|
|
|
|
|
|
|
if (which != IRQCHIP_STATE_PENDING)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (gic_rdists->has_direct_lpi) {
|
|
|
|
void __iomem *rdbase;
|
|
|
|
|
|
|
|
rdbase = per_cpu_ptr(gic_rdists->rdist, vpe->col_idx)->rd_base;
|
|
|
|
if (state) {
|
|
|
|
gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_SETLPIR);
|
|
|
|
} else {
|
|
|
|
gic_write_lpir(vpe->vpe_db_lpi, rdbase + GICR_CLRLPIR);
|
|
|
|
while (gic_read_lpir(rdbase + GICR_SYNCR) & 1)
|
|
|
|
cpu_relax();
|
|
|
|
}
|
|
|
|
} else {
|
|
|
|
if (state)
|
|
|
|
its_vpe_send_cmd(vpe, its_send_int);
|
|
|
|
else
|
|
|
|
its_vpe_send_cmd(vpe, its_send_clear);
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-20 13:41:55 +00:00
|
|
|
static struct irq_chip its_vpe_irq_chip = {
|
|
|
|
.name = "GICv4-vpe",
|
2016-12-20 15:20:38 +00:00
|
|
|
.irq_mask = its_vpe_mask_irq,
|
|
|
|
.irq_unmask = its_vpe_unmask_irq,
|
|
|
|
.irq_eoi = irq_chip_eoi_parent,
|
2016-12-20 15:17:28 +00:00
|
|
|
.irq_set_affinity = its_vpe_set_affinity,
|
2017-07-31 13:47:24 +00:00
|
|
|
.irq_set_irqchip_state = its_vpe_set_irqchip_state,
|
2016-12-20 15:09:31 +00:00
|
|
|
.irq_set_vcpu_affinity = its_vpe_set_vcpu_affinity,
|
2016-12-20 13:41:55 +00:00
|
|
|
};
|
|
|
|
|
2016-12-20 13:55:54 +00:00
|
|
|
static int its_vpe_id_alloc(void)
|
|
|
|
{
|
irqchip/gic-v3-its: Fix the incorrect parsing of VCPU table size
The VCPU table consists of vPE entries, and its size provides the number
of VPEs supported by GICv4 hardware. Unfortunately the maximum size of
the VPE table is not discoverable like Device table. All VLPI commands
limits the number of bits to 16 to hold VPEID, which is index into VCPU
table. Don't apply DEVID bits for VCPU table instead assume maximum bits
to 16.
ITS log messages on QDF2400 without fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
Virtual CPUs Table too large, reduce ids 32->26
Virtual CPUs too large, reduce ITS pages 8192->256
allocated 2097152 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
ITS log messages on QDF2400 with fix:
allocated 524288 Devices (indirect, esz 8, psz 64K, shr 1)
allocated 8192 Interrupt Collections (flat, esz 8, psz 64K, shr 1)
allocated 65536 Virtual CPUs (flat, esz 8, psz 64K, shr 1)
Signed-off-by: Shanker Donthineni <shankerd@codeaurora.org>
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2017-10-07 20:43:48 +00:00
|
|
|
return ida_simple_get(&its_vpeid_ida, 0, ITS_MAX_VPEID, GFP_KERNEL);
|
2016-12-20 13:55:54 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_id_free(u16 id)
|
|
|
|
{
|
|
|
|
ida_simple_remove(&its_vpeid_ida, id);
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_vpe_init(struct its_vpe *vpe)
|
|
|
|
{
|
|
|
|
struct page *vpt_page;
|
|
|
|
int vpe_id;
|
|
|
|
|
|
|
|
/* Allocate vpe_id */
|
|
|
|
vpe_id = its_vpe_id_alloc();
|
|
|
|
if (vpe_id < 0)
|
|
|
|
return vpe_id;
|
|
|
|
|
|
|
|
/* Allocate VPT */
|
|
|
|
vpt_page = its_allocate_pending_table(GFP_KERNEL);
|
|
|
|
if (!vpt_page) {
|
|
|
|
its_vpe_id_free(vpe_id);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (!its_alloc_vpe_table(vpe_id)) {
|
|
|
|
its_vpe_id_free(vpe_id);
|
|
|
|
its_free_pending_table(vpe->vpt_page);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
vpe->vpe_id = vpe_id;
|
|
|
|
vpe->vpt_page = vpt_page;
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
vpe->vpe_proxy_event = -1;
|
2016-12-20 13:55:54 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_teardown(struct its_vpe *vpe)
|
|
|
|
{
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
its_vpe_db_proxy_unmap(vpe);
|
2016-12-20 13:55:54 +00:00
|
|
|
its_vpe_id_free(vpe->vpe_id);
|
|
|
|
its_free_pending_table(vpe->vpt_page);
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_irq_domain_free(struct irq_domain *domain,
|
|
|
|
unsigned int virq,
|
|
|
|
unsigned int nr_irqs)
|
|
|
|
{
|
|
|
|
struct its_vm *vm = domain->host_data;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
irq_domain_free_irqs_parent(domain, virq, nr_irqs);
|
|
|
|
|
|
|
|
for (i = 0; i < nr_irqs; i++) {
|
|
|
|
struct irq_data *data = irq_domain_get_irq_data(domain,
|
|
|
|
virq + i);
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(data);
|
|
|
|
|
|
|
|
BUG_ON(vm != vpe->its_vm);
|
|
|
|
|
|
|
|
clear_bit(data->hwirq, vm->db_bitmap);
|
|
|
|
its_vpe_teardown(vpe);
|
|
|
|
irq_domain_reset_irq_data(data);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (bitmap_empty(vm->db_bitmap, vm->nr_db_lpis)) {
|
2018-05-27 16:03:03 +00:00
|
|
|
its_lpi_free(vm->db_bitmap, vm->db_lpi_base, vm->nr_db_lpis);
|
2016-12-20 13:55:54 +00:00
|
|
|
its_free_prop_table(vm->vprop_page);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
|
|
|
static int its_vpe_irq_domain_alloc(struct irq_domain *domain, unsigned int virq,
|
|
|
|
unsigned int nr_irqs, void *args)
|
|
|
|
{
|
|
|
|
struct its_vm *vm = args;
|
|
|
|
unsigned long *bitmap;
|
|
|
|
struct page *vprop_page;
|
|
|
|
int base, nr_ids, i, err = 0;
|
|
|
|
|
|
|
|
BUG_ON(!vm);
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
bitmap = its_lpi_alloc(roundup_pow_of_two(nr_irqs), &base, &nr_ids);
|
2016-12-20 13:55:54 +00:00
|
|
|
if (!bitmap)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
|
|
|
if (nr_ids < nr_irqs) {
|
2018-05-27 16:03:03 +00:00
|
|
|
its_lpi_free(bitmap, base, nr_ids);
|
2016-12-20 13:55:54 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
vprop_page = its_allocate_prop_table(GFP_KERNEL);
|
|
|
|
if (!vprop_page) {
|
2018-05-27 16:03:03 +00:00
|
|
|
its_lpi_free(bitmap, base, nr_ids);
|
2016-12-20 13:55:54 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
vm->db_bitmap = bitmap;
|
|
|
|
vm->db_lpi_base = base;
|
|
|
|
vm->nr_db_lpis = nr_ids;
|
|
|
|
vm->vprop_page = vprop_page;
|
|
|
|
|
|
|
|
for (i = 0; i < nr_irqs; i++) {
|
|
|
|
vm->vpes[i]->vpe_db_lpi = base + i;
|
|
|
|
err = its_vpe_init(vm->vpes[i]);
|
|
|
|
if (err)
|
|
|
|
break;
|
|
|
|
err = its_irq_gic_domain_alloc(domain, virq + i,
|
|
|
|
vm->vpes[i]->vpe_db_lpi);
|
|
|
|
if (err)
|
|
|
|
break;
|
|
|
|
irq_domain_set_hwirq_and_chip(domain, virq + i, i,
|
|
|
|
&its_vpe_irq_chip, vm->vpes[i]);
|
|
|
|
set_bit(i, bitmap);
|
|
|
|
}
|
|
|
|
|
|
|
|
if (err) {
|
|
|
|
if (i > 0)
|
|
|
|
its_vpe_irq_domain_free(domain, virq, i - 1);
|
|
|
|
|
2018-05-27 16:03:03 +00:00
|
|
|
its_lpi_free(bitmap, base, nr_ids);
|
2016-12-20 13:55:54 +00:00
|
|
|
its_free_prop_table(vprop_page);
|
|
|
|
}
|
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
2017-09-13 21:29:10 +00:00
|
|
|
static int its_vpe_irq_domain_activate(struct irq_domain *domain,
|
2017-12-29 15:59:06 +00:00
|
|
|
struct irq_data *d, bool reserve)
|
2016-12-20 14:47:05 +00:00
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
2017-10-08 14:16:09 +00:00
|
|
|
struct its_node *its;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
/* If we use the list map, we issue VMAPP on demand... */
|
|
|
|
if (its_list_map)
|
2017-11-07 10:04:38 +00:00
|
|
|
return 0;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
|
|
|
/* Map the VPE to the first possible CPU */
|
|
|
|
vpe->col_idx = cpumask_first(cpu_online_mask);
|
2017-10-08 14:16:09 +00:00
|
|
|
|
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
if (!its->is_v4)
|
|
|
|
continue;
|
|
|
|
|
2017-10-08 17:46:39 +00:00
|
|
|
its_send_vmapp(its, vpe, true);
|
2017-10-08 14:16:09 +00:00
|
|
|
its_send_vinvall(its, vpe);
|
|
|
|
}
|
|
|
|
|
2017-10-19 09:11:34 +00:00
|
|
|
irq_data_update_effective_affinity(d, cpumask_of(vpe->col_idx));
|
|
|
|
|
2017-09-13 21:29:10 +00:00
|
|
|
return 0;
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static void its_vpe_irq_domain_deactivate(struct irq_domain *domain,
|
|
|
|
struct irq_data *d)
|
|
|
|
{
|
|
|
|
struct its_vpe *vpe = irq_data_get_irq_chip_data(d);
|
2017-10-08 17:46:39 +00:00
|
|
|
struct its_node *its;
|
|
|
|
|
2017-10-08 17:50:36 +00:00
|
|
|
/*
|
|
|
|
* If we use the list map, we unmap the VPE once no VLPIs are
|
|
|
|
* associated with the VM.
|
|
|
|
*/
|
|
|
|
if (its_list_map)
|
|
|
|
return;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
2017-10-08 17:46:39 +00:00
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
if (!its->is_v4)
|
|
|
|
continue;
|
2016-12-20 14:47:05 +00:00
|
|
|
|
2017-10-08 17:46:39 +00:00
|
|
|
its_send_vmapp(its, vpe, false);
|
|
|
|
}
|
2016-12-20 14:47:05 +00:00
|
|
|
}
|
|
|
|
|
2016-12-20 13:41:55 +00:00
|
|
|
static const struct irq_domain_ops its_vpe_domain_ops = {
|
2016-12-20 13:55:54 +00:00
|
|
|
.alloc = its_vpe_irq_domain_alloc,
|
|
|
|
.free = its_vpe_irq_domain_free,
|
2016-12-20 14:47:05 +00:00
|
|
|
.activate = its_vpe_irq_domain_activate,
|
|
|
|
.deactivate = its_vpe_irq_domain_deactivate,
|
2016-12-20 13:41:55 +00:00
|
|
|
};
|
|
|
|
|
2015-03-06 16:37:50 +00:00
|
|
|
static int its_force_quiescent(void __iomem *base)
|
|
|
|
{
|
|
|
|
u32 count = 1000000; /* 1s */
|
|
|
|
u32 val;
|
|
|
|
|
|
|
|
val = readl_relaxed(base + GITS_CTLR);
|
2016-08-18 22:41:58 +00:00
|
|
|
/*
|
|
|
|
* GIC architecture specification requires the ITS to be both
|
|
|
|
* disabled and quiescent for writes to GITS_BASER<n> or
|
|
|
|
* GITS_CBASER to not have UNPREDICTABLE results.
|
|
|
|
*/
|
|
|
|
if ((val & GITS_CTLR_QUIESCENT) && !(val & GITS_CTLR_ENABLE))
|
2015-03-06 16:37:50 +00:00
|
|
|
return 0;
|
|
|
|
|
|
|
|
/* Disable the generation of all interrupts to this ITS */
|
2017-06-27 20:24:25 +00:00
|
|
|
val &= ~(GITS_CTLR_ENABLE | GITS_CTLR_ImDe);
|
2015-03-06 16:37:50 +00:00
|
|
|
writel_relaxed(val, base + GITS_CTLR);
|
|
|
|
|
|
|
|
/* Poll GITS_CTLR and wait until ITS becomes quiescent */
|
|
|
|
while (1) {
|
|
|
|
val = readl_relaxed(base + GITS_CTLR);
|
|
|
|
if (val & GITS_CTLR_QUIESCENT)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
count--;
|
|
|
|
if (!count)
|
|
|
|
return -EBUSY;
|
|
|
|
|
|
|
|
cpu_relax();
|
|
|
|
udelay(1);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2017-10-17 16:55:55 +00:00
|
|
|
static bool __maybe_unused its_enable_quirk_cavium_22375(void *data)
|
2015-09-21 20:58:38 +00:00
|
|
|
{
|
|
|
|
struct its_node *its = data;
|
|
|
|
|
2017-10-17 16:55:54 +00:00
|
|
|
/* erratum 22375: only alloc 8MB table size */
|
|
|
|
its->device_ids = 0x14; /* 20 bits, 8MB */
|
2015-09-21 20:58:38 +00:00
|
|
|
its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_22375;
|
2017-10-17 16:55:55 +00:00
|
|
|
|
|
|
|
return true;
|
2015-09-21 20:58:38 +00:00
|
|
|
}
|
|
|
|
|
2017-10-17 16:55:55 +00:00
|
|
|
static bool __maybe_unused its_enable_quirk_cavium_23144(void *data)
|
2016-05-25 13:29:20 +00:00
|
|
|
{
|
|
|
|
struct its_node *its = data;
|
|
|
|
|
|
|
|
its->flags |= ITS_FLAGS_WORKAROUND_CAVIUM_23144;
|
2017-10-17 16:55:55 +00:00
|
|
|
|
|
|
|
return true;
|
2016-05-25 13:29:20 +00:00
|
|
|
}
|
|
|
|
|
2017-10-17 16:55:55 +00:00
|
|
|
static bool __maybe_unused its_enable_quirk_qdf2400_e0065(void *data)
|
2017-03-07 14:20:38 +00:00
|
|
|
{
|
|
|
|
struct its_node *its = data;
|
|
|
|
|
|
|
|
/* On QDF2400, the size of the ITE is 16Bytes */
|
|
|
|
its->ite_size = 16;
|
2017-10-17 16:55:55 +00:00
|
|
|
|
|
|
|
return true;
|
2017-03-07 14:20:38 +00:00
|
|
|
}
|
|
|
|
|
2017-10-17 16:55:56 +00:00
|
|
|
static u64 its_irq_get_msi_base_pre_its(struct its_device *its_dev)
|
|
|
|
{
|
|
|
|
struct its_node *its = its_dev->its;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* The Socionext Synquacer SoC has a so-called 'pre-ITS',
|
|
|
|
* which maps 32-bit writes targeted at a separate window of
|
|
|
|
* size '4 << device_id_bits' onto writes to GITS_TRANSLATER
|
|
|
|
* with device ID taken from bits [device_id_bits + 1:2] of
|
|
|
|
* the window offset.
|
|
|
|
*/
|
|
|
|
return its->pre_its_base + (its_dev->device_id << 2);
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool __maybe_unused its_enable_quirk_socionext_synquacer(void *data)
|
|
|
|
{
|
|
|
|
struct its_node *its = data;
|
|
|
|
u32 pre_its_window[2];
|
|
|
|
u32 ids;
|
|
|
|
|
|
|
|
if (!fwnode_property_read_u32_array(its->fwnode_handle,
|
|
|
|
"socionext,synquacer-pre-its",
|
|
|
|
pre_its_window,
|
|
|
|
ARRAY_SIZE(pre_its_window))) {
|
|
|
|
|
|
|
|
its->pre_its_base = pre_its_window[0];
|
|
|
|
its->get_msi_base = its_irq_get_msi_base_pre_its;
|
|
|
|
|
|
|
|
ids = ilog2(pre_its_window[1]) - 2;
|
|
|
|
if (its->device_ids > ids)
|
|
|
|
its->device_ids = ids;
|
|
|
|
|
|
|
|
/* the pre-ITS breaks isolation, so disable MSI remapping */
|
|
|
|
its->msi_domain_flags &= ~IRQ_DOMAIN_FLAG_MSI_REMAP;
|
|
|
|
return true;
|
|
|
|
}
|
|
|
|
return false;
|
|
|
|
}
|
|
|
|
|
2017-07-28 20:20:37 +00:00
|
|
|
static bool __maybe_unused its_enable_quirk_hip07_161600802(void *data)
|
|
|
|
{
|
|
|
|
struct its_node *its = data;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Hip07 insists on using the wrong address for the VLPI
|
|
|
|
* page. Trick it into doing the right thing...
|
|
|
|
*/
|
|
|
|
its->vlpi_redist_offset = SZ_128K;
|
|
|
|
return true;
|
2017-03-07 14:20:38 +00:00
|
|
|
}
|
|
|
|
|
2015-09-21 20:58:37 +00:00
|
|
|
static const struct gic_quirk its_quirks[] = {
|
2015-09-21 20:58:38 +00:00
|
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_22375
|
|
|
|
{
|
|
|
|
.desc = "ITS: Cavium errata 22375, 24313",
|
|
|
|
.iidr = 0xa100034c, /* ThunderX pass 1.x */
|
|
|
|
.mask = 0xffff0fff,
|
|
|
|
.init = its_enable_quirk_cavium_22375,
|
|
|
|
},
|
2016-05-25 13:29:20 +00:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_CAVIUM_ERRATUM_23144
|
|
|
|
{
|
|
|
|
.desc = "ITS: Cavium erratum 23144",
|
|
|
|
.iidr = 0xa100034c, /* ThunderX pass 1.x */
|
|
|
|
.mask = 0xffff0fff,
|
|
|
|
.init = its_enable_quirk_cavium_23144,
|
|
|
|
},
|
2017-03-07 14:20:38 +00:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_QCOM_QDF2400_ERRATUM_0065
|
|
|
|
{
|
|
|
|
.desc = "ITS: QDF2400 erratum 0065",
|
|
|
|
.iidr = 0x00001070, /* QDF2400 ITS rev 1.x */
|
|
|
|
.mask = 0xffffffff,
|
|
|
|
.init = its_enable_quirk_qdf2400_e0065,
|
|
|
|
},
|
2017-10-17 16:55:56 +00:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_SOCIONEXT_SYNQUACER_PREITS
|
|
|
|
{
|
|
|
|
/*
|
|
|
|
* The Socionext Synquacer SoC incorporates ARM's own GIC-500
|
|
|
|
* implementation, but with a 'pre-ITS' added that requires
|
|
|
|
* special handling in software.
|
|
|
|
*/
|
|
|
|
.desc = "ITS: Socionext Synquacer pre-ITS",
|
|
|
|
.iidr = 0x0001143b,
|
|
|
|
.mask = 0xffffffff,
|
|
|
|
.init = its_enable_quirk_socionext_synquacer,
|
|
|
|
},
|
2017-07-28 20:20:37 +00:00
|
|
|
#endif
|
|
|
|
#ifdef CONFIG_HISILICON_ERRATUM_161600802
|
|
|
|
{
|
|
|
|
.desc = "ITS: Hip07 erratum 161600802",
|
|
|
|
.iidr = 0x00000004,
|
|
|
|
.mask = 0xffffffff,
|
|
|
|
.init = its_enable_quirk_hip07_161600802,
|
|
|
|
},
|
2015-09-21 20:58:38 +00:00
|
|
|
#endif
|
2015-09-21 20:58:37 +00:00
|
|
|
{
|
|
|
|
}
|
|
|
|
};
|
|
|
|
|
|
|
|
static void its_enable_quirks(struct its_node *its)
|
|
|
|
{
|
|
|
|
u32 iidr = readl_relaxed(its->base + GITS_IIDR);
|
|
|
|
|
|
|
|
gic_enable_quirks(iidr, its_quirks, its);
|
|
|
|
}
|
|
|
|
|
2018-03-01 05:48:18 +00:00
|
|
|
static int its_save_disable(void)
|
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
int err = 0;
|
|
|
|
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_lock(&its_lock);
|
2018-03-01 05:48:18 +00:00
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
void __iomem *base;
|
|
|
|
|
|
|
|
if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
base = its->base;
|
|
|
|
its->ctlr_save = readl_relaxed(base + GITS_CTLR);
|
|
|
|
err = its_force_quiescent(base);
|
|
|
|
if (err) {
|
|
|
|
pr_err("ITS@%pa: failed to quiesce: %d\n",
|
|
|
|
&its->phys_base, err);
|
|
|
|
writel_relaxed(its->ctlr_save, base + GITS_CTLR);
|
|
|
|
goto err;
|
|
|
|
}
|
|
|
|
|
|
|
|
its->cbaser_save = gits_read_cbaser(base + GITS_CBASER);
|
|
|
|
}
|
|
|
|
|
|
|
|
err:
|
|
|
|
if (err) {
|
|
|
|
list_for_each_entry_continue_reverse(its, &its_nodes, entry) {
|
|
|
|
void __iomem *base;
|
|
|
|
|
|
|
|
if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
base = its->base;
|
|
|
|
writel_relaxed(its->ctlr_save, base + GITS_CTLR);
|
|
|
|
}
|
|
|
|
}
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_unlock(&its_lock);
|
2018-03-01 05:48:18 +00:00
|
|
|
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void its_restore_enable(void)
|
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
int ret;
|
|
|
|
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_lock(&its_lock);
|
2018-03-01 05:48:18 +00:00
|
|
|
list_for_each_entry(its, &its_nodes, entry) {
|
|
|
|
void __iomem *base;
|
|
|
|
int i;
|
|
|
|
|
|
|
|
if (!(its->flags & ITS_FLAGS_SAVE_SUSPEND_STATE))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
base = its->base;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Make sure that the ITS is disabled. If it fails to quiesce,
|
|
|
|
* don't restore it since writing to CBASER or BASER<n>
|
|
|
|
* registers is undefined according to the GIC v3 ITS
|
|
|
|
* Specification.
|
|
|
|
*/
|
|
|
|
ret = its_force_quiescent(base);
|
|
|
|
if (ret) {
|
|
|
|
pr_err("ITS@%pa: failed to quiesce on resume: %d\n",
|
|
|
|
&its->phys_base, ret);
|
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
gits_write_cbaser(its->cbaser_save, base + GITS_CBASER);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Writing CBASER resets CREADR to 0, so make CWRITER and
|
|
|
|
* cmd_write line up with it.
|
|
|
|
*/
|
|
|
|
its->cmd_write = its->cmd_base;
|
|
|
|
gits_write_cwriter(0, base + GITS_CWRITER);
|
|
|
|
|
|
|
|
/* Restore GITS_BASER from the value cache. */
|
|
|
|
for (i = 0; i < GITS_BASER_NR_REGS; i++) {
|
|
|
|
struct its_baser *baser = &its->tables[i];
|
|
|
|
|
|
|
|
if (!(baser->val & GITS_BASER_VALID))
|
|
|
|
continue;
|
|
|
|
|
|
|
|
its_write_baser(its, baser, baser->val);
|
|
|
|
}
|
|
|
|
writel_relaxed(its->ctlr_save, base + GITS_CTLR);
|
2018-03-01 05:48:20 +00:00
|
|
|
|
|
|
|
/*
|
|
|
|
* Reinit the collection if it's stored in the ITS. This is
|
|
|
|
* indicated by the col_id being less than the HCC field.
|
|
|
|
* CID < HCC as specified in the GIC v3 Documentation.
|
|
|
|
*/
|
|
|
|
if (its->collections[smp_processor_id()].col_id <
|
|
|
|
GITS_TYPER_HCC(gic_read_typer(base + GITS_TYPER)))
|
|
|
|
its_cpu_init_collection(its);
|
2018-03-01 05:48:18 +00:00
|
|
|
}
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_unlock(&its_lock);
|
2018-03-01 05:48:18 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
static struct syscore_ops its_syscore_ops = {
|
|
|
|
.suspend = its_save_disable,
|
|
|
|
.resume = its_restore_enable,
|
|
|
|
};
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
static int its_init_domain(struct fwnode_handle *handle, struct its_node *its)
|
2016-09-12 18:32:23 +00:00
|
|
|
{
|
|
|
|
struct irq_domain *inner_domain;
|
|
|
|
struct msi_domain_info *info;
|
|
|
|
|
|
|
|
info = kzalloc(sizeof(*info), GFP_KERNEL);
|
|
|
|
if (!info)
|
|
|
|
return -ENOMEM;
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
inner_domain = irq_domain_create_tree(handle, &its_domain_ops, its);
|
2016-09-12 18:32:23 +00:00
|
|
|
if (!inner_domain) {
|
|
|
|
kfree(info);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
inner_domain->parent = its_parent;
|
2017-06-22 10:42:50 +00:00
|
|
|
irq_domain_update_bus_token(inner_domain, DOMAIN_BUS_NEXUS);
|
2017-10-17 16:55:56 +00:00
|
|
|
inner_domain->flags |= its->msi_domain_flags;
|
2016-09-12 18:32:23 +00:00
|
|
|
info->ops = &its_msi_domain_ops;
|
|
|
|
info->data = its;
|
|
|
|
inner_domain->host_data = info;
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-20 13:41:55 +00:00
|
|
|
static int its_init_vpe_domain(void)
|
|
|
|
{
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
struct its_node *its;
|
|
|
|
u32 devid;
|
|
|
|
int entries;
|
|
|
|
|
|
|
|
if (gic_rdists->has_direct_lpi) {
|
|
|
|
pr_info("ITS: Using DirectLPI for VPE invalidation\n");
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Any ITS will do, even if not v4 */
|
|
|
|
its = list_first_entry(&its_nodes, struct its_node, entry);
|
|
|
|
|
|
|
|
entries = roundup_pow_of_two(nr_cpu_ids);
|
treewide: kzalloc() -> kcalloc()
The kzalloc() function has a 2-factor argument form, kcalloc(). This
patch replaces cases of:
kzalloc(a * b, gfp)
with:
kcalloc(a * b, gfp)
as well as handling cases of:
kzalloc(a * b * c, gfp)
with:
kzalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kzalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kzalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kzalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kzalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kzalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kzalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kzalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kzalloc
+ kcalloc
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kzalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kzalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kzalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kzalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kzalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kzalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kzalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kzalloc(sizeof(THING) * C2, ...)
|
kzalloc(sizeof(TYPE) * C2, ...)
|
kzalloc(C1 * C2 * C3, ...)
|
kzalloc(C1 * C2, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * E2
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kzalloc
+ kcalloc
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 21:03:40 +00:00
|
|
|
vpe_proxy.vpes = kcalloc(entries, sizeof(*vpe_proxy.vpes),
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
GFP_KERNEL);
|
|
|
|
if (!vpe_proxy.vpes) {
|
|
|
|
pr_err("ITS: Can't allocate GICv4 proxy device array\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
/* Use the last possible DevID */
|
|
|
|
devid = GENMASK(its->device_ids - 1, 0);
|
|
|
|
vpe_proxy.dev = its_create_device(its, devid, entries, false);
|
|
|
|
if (!vpe_proxy.dev) {
|
|
|
|
kfree(vpe_proxy.vpes);
|
|
|
|
pr_err("ITS: Can't allocate GICv4 proxy device\n");
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2017-09-23 18:50:19 +00:00
|
|
|
BUG_ON(entries > vpe_proxy.dev->nr_ites);
|
irqchip/gic-v3-its: Add device proxy for VPE management if !DirectLpi
When we don't have the DirectLPI feature, we must work around the
architecture shortcomings to be able to perform the required
maintenance (interrupt masking, clearing and injection).
For this, we create a fake device whose sole purpose is to
provide a way to issue commands as if we were dealing with LPIs
coming from that device (while they actually originate from
the ITS). This fake device doesn't have LPIs allocated to it,
but instead uses the VPE LPIs.
Of course, this could be a real bottleneck, and a naive
implementation would require 6 commands to issue an invalidation.
Instead, let's allocate at least one event per physical CPU
(rounded up to the next power of 2), and opportunistically
map the VPE doorbell to an event. This doorbell will be mapped
until we roll over and need to reallocate this slot.
This ensures that most of the time, we only need 2 commands
to issue an INV, INT or CLEAR, making the performance a lot
better, given that we always issue a CLEAR on entry, and
an INV on each side of a trapped WFI.
Signed-off-by: Marc Zyngier <marc.zyngier@arm.com>
2016-12-20 15:23:22 +00:00
|
|
|
|
|
|
|
raw_spin_lock_init(&vpe_proxy.lock);
|
|
|
|
vpe_proxy.next_victim = 0;
|
|
|
|
pr_info("ITS: Allocated DevID %x as GICv4 proxy device (%d slots)\n",
|
|
|
|
devid, vpe_proxy.dev->nr_ites);
|
|
|
|
|
2016-12-20 13:41:55 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-12-19 17:25:54 +00:00
|
|
|
static int __init its_compute_its_list_map(struct resource *res,
|
|
|
|
void __iomem *its_base)
|
|
|
|
{
|
|
|
|
int its_number;
|
|
|
|
u32 ctlr;
|
|
|
|
|
|
|
|
/*
|
|
|
|
* This is assumed to be done early enough that we're
|
|
|
|
* guaranteed to be single-threaded, hence no
|
|
|
|
* locking. Should this change, we should address
|
|
|
|
* this.
|
|
|
|
*/
|
2017-10-08 17:48:06 +00:00
|
|
|
its_number = find_first_zero_bit(&its_list_map, GICv4_ITS_LIST_MAX);
|
|
|
|
if (its_number >= GICv4_ITS_LIST_MAX) {
|
2016-12-19 17:25:54 +00:00
|
|
|
pr_err("ITS@%pa: No ITSList entry available!\n",
|
|
|
|
&res->start);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
ctlr = readl_relaxed(its_base + GITS_CTLR);
|
|
|
|
ctlr &= ~GITS_CTLR_ITS_NUMBER;
|
|
|
|
ctlr |= its_number << GITS_CTLR_ITS_NUMBER_SHIFT;
|
|
|
|
writel_relaxed(ctlr, its_base + GITS_CTLR);
|
|
|
|
ctlr = readl_relaxed(its_base + GITS_CTLR);
|
|
|
|
if ((ctlr & GITS_CTLR_ITS_NUMBER) != (its_number << GITS_CTLR_ITS_NUMBER_SHIFT)) {
|
|
|
|
its_number = ctlr & GITS_CTLR_ITS_NUMBER;
|
|
|
|
its_number >>= GITS_CTLR_ITS_NUMBER_SHIFT;
|
|
|
|
}
|
|
|
|
|
|
|
|
if (test_and_set_bit(its_number, &its_list_map)) {
|
|
|
|
pr_err("ITS@%pa: Duplicate ITSList entry %d\n",
|
|
|
|
&res->start, its_number);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
return its_number;
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
static int __init its_probe_one(struct resource *res,
|
|
|
|
struct fwnode_handle *handle, int numa_node)
|
2014-11-24 14:35:17 +00:00
|
|
|
{
|
|
|
|
struct its_node *its;
|
|
|
|
void __iomem *its_base;
|
2016-12-19 17:25:54 +00:00
|
|
|
u32 val, ctlr;
|
|
|
|
u64 baser, tmp, typer;
|
2014-11-24 14:35:17 +00:00
|
|
|
int err;
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
its_base = ioremap(res->start, resource_size(res));
|
2014-11-24 14:35:17 +00:00
|
|
|
if (!its_base) {
|
2016-09-12 18:32:24 +00:00
|
|
|
pr_warn("ITS@%pa: Unable to map ITS registers\n", &res->start);
|
2014-11-24 14:35:17 +00:00
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
|
|
|
val = readl_relaxed(its_base + GITS_PIDR2) & GIC_PIDR2_ARCH_MASK;
|
|
|
|
if (val != 0x30 && val != 0x40) {
|
2016-09-12 18:32:24 +00:00
|
|
|
pr_warn("ITS@%pa: No ITS detected, giving up\n", &res->start);
|
2014-11-24 14:35:17 +00:00
|
|
|
err = -ENODEV;
|
|
|
|
goto out_unmap;
|
|
|
|
}
|
|
|
|
|
2015-03-06 16:37:50 +00:00
|
|
|
err = its_force_quiescent(its_base);
|
|
|
|
if (err) {
|
2016-09-12 18:32:24 +00:00
|
|
|
pr_warn("ITS@%pa: Failed to quiesce, giving up\n", &res->start);
|
2015-03-06 16:37:50 +00:00
|
|
|
goto out_unmap;
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
pr_info("ITS %pR\n", res);
|
2014-11-24 14:35:17 +00:00
|
|
|
|
|
|
|
its = kzalloc(sizeof(*its), GFP_KERNEL);
|
|
|
|
if (!its) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto out_unmap;
|
|
|
|
}
|
|
|
|
|
|
|
|
raw_spin_lock_init(&its->lock);
|
|
|
|
INIT_LIST_HEAD(&its->entry);
|
|
|
|
INIT_LIST_HEAD(&its->its_device_list);
|
2016-12-19 17:25:54 +00:00
|
|
|
typer = gic_read_typer(its_base + GITS_TYPER);
|
2014-11-24 14:35:17 +00:00
|
|
|
its->base = its_base;
|
2016-09-12 18:32:24 +00:00
|
|
|
its->phys_base = res->start;
|
2016-12-19 17:25:54 +00:00
|
|
|
its->ite_size = GITS_TYPER_ITT_ENTRY_SIZE(typer);
|
2017-10-17 16:55:54 +00:00
|
|
|
its->device_ids = GITS_TYPER_DEVBITS(typer);
|
2016-12-19 17:25:54 +00:00
|
|
|
its->is_v4 = !!(typer & GITS_TYPER_VLPIS);
|
|
|
|
if (its->is_v4) {
|
|
|
|
if (!(typer & GITS_TYPER_VMOVP)) {
|
|
|
|
err = its_compute_its_list_map(res, its_base);
|
|
|
|
if (err < 0)
|
|
|
|
goto out_free_its;
|
|
|
|
|
2017-10-08 17:44:42 +00:00
|
|
|
its->list_nr = err;
|
|
|
|
|
2016-12-19 17:25:54 +00:00
|
|
|
pr_info("ITS@%pa: Using ITS number %d\n",
|
|
|
|
&res->start, err);
|
|
|
|
} else {
|
|
|
|
pr_info("ITS@%pa: Single VMOVP capable\n", &res->start);
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
its->numa_node = numa_node;
|
2014-11-24 14:35:17 +00:00
|
|
|
|
2017-02-01 17:38:25 +00:00
|
|
|
its->cmd_base = (void *)__get_free_pages(GFP_KERNEL | __GFP_ZERO,
|
|
|
|
get_order(ITS_CMD_QUEUE_SZ));
|
2014-11-24 14:35:17 +00:00
|
|
|
if (!its->cmd_base) {
|
|
|
|
err = -ENOMEM;
|
|
|
|
goto out_free_its;
|
|
|
|
}
|
|
|
|
its->cmd_write = its->cmd_base;
|
2017-10-17 16:55:56 +00:00
|
|
|
its->fwnode_handle = handle;
|
|
|
|
its->get_msi_base = its_irq_get_msi_base;
|
|
|
|
its->msi_domain_flags = IRQ_DOMAIN_FLAG_MSI_REMAP;
|
2014-11-24 14:35:17 +00:00
|
|
|
|
2015-09-21 20:58:37 +00:00
|
|
|
its_enable_quirks(its);
|
|
|
|
|
2016-06-06 23:17:31 +00:00
|
|
|
err = its_alloc_tables(its);
|
2014-11-24 14:35:17 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free_cmd;
|
|
|
|
|
|
|
|
err = its_alloc_collections(its);
|
|
|
|
if (err)
|
|
|
|
goto out_free_tables;
|
|
|
|
|
|
|
|
baser = (virt_to_phys(its->cmd_base) |
|
2017-01-26 03:51:41 +00:00
|
|
|
GITS_CBASER_RaWaWb |
|
2014-11-24 14:35:17 +00:00
|
|
|
GITS_CBASER_InnerShareable |
|
|
|
|
(ITS_CMD_QUEUE_SZ / SZ_4K - 1) |
|
|
|
|
GITS_CBASER_VALID);
|
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
gits_write_cbaser(baser, its->base + GITS_CBASER);
|
|
|
|
tmp = gits_read_cbaser(its->base + GITS_CBASER);
|
2014-11-24 14:35:17 +00:00
|
|
|
|
2015-03-27 14:15:04 +00:00
|
|
|
if ((tmp ^ baser) & GITS_CBASER_SHAREABILITY_MASK) {
|
2015-03-27 14:15:05 +00:00
|
|
|
if (!(tmp & GITS_CBASER_SHAREABILITY_MASK)) {
|
|
|
|
/*
|
|
|
|
* The HW reports non-shareable, we must
|
|
|
|
* remove the cacheability attributes as
|
|
|
|
* well.
|
|
|
|
*/
|
|
|
|
baser &= ~(GITS_CBASER_SHAREABILITY_MASK |
|
|
|
|
GITS_CBASER_CACHEABILITY_MASK);
|
|
|
|
baser |= GITS_CBASER_nC;
|
2016-11-02 11:54:06 +00:00
|
|
|
gits_write_cbaser(baser, its->base + GITS_CBASER);
|
2015-03-27 14:15:05 +00:00
|
|
|
}
|
2014-11-24 14:35:17 +00:00
|
|
|
pr_info("ITS: using cache flushing for cmd queue\n");
|
|
|
|
its->flags |= ITS_FLAGS_CMDQ_NEEDS_FLUSHING;
|
|
|
|
}
|
|
|
|
|
2016-11-02 11:54:06 +00:00
|
|
|
gits_write_cwriter(0, its->base + GITS_CWRITER);
|
2016-12-19 17:25:54 +00:00
|
|
|
ctlr = readl_relaxed(its->base + GITS_CTLR);
|
2017-06-27 20:24:25 +00:00
|
|
|
ctlr |= GITS_CTLR_ENABLE;
|
|
|
|
if (its->is_v4)
|
|
|
|
ctlr |= GITS_CTLR_ImDe;
|
|
|
|
writel_relaxed(ctlr, its->base + GITS_CTLR);
|
2015-03-27 14:15:05 +00:00
|
|
|
|
2018-03-01 05:48:18 +00:00
|
|
|
if (GITS_TYPER_HCC(typer))
|
|
|
|
its->flags |= ITS_FLAGS_SAVE_SUSPEND_STATE;
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
err = its_init_domain(handle, its);
|
2016-09-12 18:32:23 +00:00
|
|
|
if (err)
|
|
|
|
goto out_free_tables;
|
2014-11-24 14:35:17 +00:00
|
|
|
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_lock(&its_lock);
|
2014-11-24 14:35:17 +00:00
|
|
|
list_add(&its->entry, &its_nodes);
|
2018-07-18 15:42:04 +00:00
|
|
|
raw_spin_unlock(&its_lock);
|
2014-11-24 14:35:17 +00:00
|
|
|
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
out_free_tables:
|
|
|
|
its_free_tables(its);
|
|
|
|
out_free_cmd:
|
2017-02-01 17:38:25 +00:00
|
|
|
free_pages((unsigned long)its->cmd_base, get_order(ITS_CMD_QUEUE_SZ));
|
2014-11-24 14:35:17 +00:00
|
|
|
out_free_its:
|
|
|
|
kfree(its);
|
|
|
|
out_unmap:
|
|
|
|
iounmap(its_base);
|
2016-09-12 18:32:24 +00:00
|
|
|
pr_err("ITS@%pa: failed probing (%d)\n", &res->start, err);
|
2014-11-24 14:35:17 +00:00
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static bool gic_rdists_supports_plpis(void)
|
|
|
|
{
|
2016-10-14 14:13:07 +00:00
|
|
|
return !!(gic_read_typer(gic_data_rdist_rd_base() + GICR_TYPER) & GICR_TYPER_PLPIS);
|
2014-11-24 14:35:17 +00:00
|
|
|
}
|
|
|
|
|
2018-03-22 01:58:49 +00:00
|
|
|
static int redist_disable_lpis(void)
|
|
|
|
{
|
|
|
|
void __iomem *rbase = gic_data_rdist_rd_base();
|
|
|
|
u64 timeout = USEC_PER_SEC;
|
|
|
|
u64 val;
|
|
|
|
|
2018-06-22 09:52:54 +00:00
|
|
|
/*
|
|
|
|
* If coming via a CPU hotplug event, we don't need to disable
|
|
|
|
* LPIs before trying to re-enable them. They are already
|
|
|
|
* configured and all is well in the world. Detect this case
|
|
|
|
* by checking the allocation of the pending table for the
|
|
|
|
* current CPU.
|
|
|
|
*/
|
|
|
|
if (gic_data_rdist()->pend_page)
|
|
|
|
return 0;
|
|
|
|
|
2018-03-22 01:58:49 +00:00
|
|
|
if (!gic_rdists_supports_plpis()) {
|
|
|
|
pr_info("CPU%d: LPIs not supported\n", smp_processor_id());
|
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
val = readl_relaxed(rbase + GICR_CTLR);
|
|
|
|
if (!(val & GICR_CTLR_ENABLE_LPIS))
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
pr_warn("CPU%d: Booted with LPIs enabled, memory probably corrupted\n",
|
|
|
|
smp_processor_id());
|
|
|
|
add_taint(TAINT_CRAP, LOCKDEP_STILL_OK);
|
|
|
|
|
|
|
|
/* Disable LPIs */
|
|
|
|
val &= ~GICR_CTLR_ENABLE_LPIS;
|
|
|
|
writel_relaxed(val, rbase + GICR_CTLR);
|
|
|
|
|
|
|
|
/* Make sure any change to GICR_CTLR is observable by the GIC */
|
|
|
|
dsb(sy);
|
|
|
|
|
|
|
|
/*
|
|
|
|
* Software must observe RWP==0 after clearing GICR_CTLR.EnableLPIs
|
|
|
|
* from 1 to 0 before programming GICR_PEND{PROP}BASER registers.
|
|
|
|
* Error out if we time out waiting for RWP to clear.
|
|
|
|
*/
|
|
|
|
while (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_RWP) {
|
|
|
|
if (!timeout) {
|
|
|
|
pr_err("CPU%d: Timeout while disabling LPIs\n",
|
|
|
|
smp_processor_id());
|
|
|
|
return -ETIMEDOUT;
|
|
|
|
}
|
|
|
|
udelay(1);
|
|
|
|
timeout--;
|
|
|
|
}
|
|
|
|
|
|
|
|
/*
|
|
|
|
* After it has been written to 1, it is IMPLEMENTATION
|
|
|
|
* DEFINED whether GICR_CTLR.EnableLPI becomes RES1 or can be
|
|
|
|
* cleared to 0. Error out if clearing the bit failed.
|
|
|
|
*/
|
|
|
|
if (readl_relaxed(rbase + GICR_CTLR) & GICR_CTLR_ENABLE_LPIS) {
|
|
|
|
pr_err("CPU%d: Failed to disable LPIs\n", smp_processor_id());
|
|
|
|
return -EBUSY;
|
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2014-11-24 14:35:17 +00:00
|
|
|
int its_cpu_init(void)
|
|
|
|
{
|
|
|
|
if (!list_empty(&its_nodes)) {
|
2018-03-22 01:58:49 +00:00
|
|
|
int ret;
|
|
|
|
|
|
|
|
ret = redist_disable_lpis();
|
|
|
|
if (ret)
|
|
|
|
return ret;
|
|
|
|
|
2014-11-24 14:35:17 +00:00
|
|
|
its_cpu_init_lpis();
|
2018-03-01 05:48:20 +00:00
|
|
|
its_cpu_init_collections();
|
2014-11-24 14:35:17 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-06-22 10:35:30 +00:00
|
|
|
static const struct of_device_id its_device_id[] = {
|
2014-11-24 14:35:17 +00:00
|
|
|
{ .compatible = "arm,gic-v3-its", },
|
|
|
|
{},
|
|
|
|
};
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
static int __init its_of_probe(struct device_node *node)
|
2014-11-24 14:35:17 +00:00
|
|
|
{
|
|
|
|
struct device_node *np;
|
2016-09-12 18:32:24 +00:00
|
|
|
struct resource res;
|
2014-11-24 14:35:17 +00:00
|
|
|
|
|
|
|
for (np = of_find_matching_node(node, its_device_id); np;
|
|
|
|
np = of_find_matching_node(np, its_device_id)) {
|
2018-02-01 17:03:29 +00:00
|
|
|
if (!of_device_is_available(np))
|
|
|
|
continue;
|
2016-09-12 18:32:23 +00:00
|
|
|
if (!of_property_read_bool(np, "msi-controller")) {
|
2017-07-18 21:43:10 +00:00
|
|
|
pr_warn("%pOF: no msi-controller property, ITS ignored\n",
|
|
|
|
np);
|
2016-09-12 18:32:23 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
if (of_address_to_resource(np, 0, &res)) {
|
2017-07-18 21:43:10 +00:00
|
|
|
pr_warn("%pOF: no regs?\n", np);
|
2016-09-12 18:32:24 +00:00
|
|
|
continue;
|
|
|
|
}
|
|
|
|
|
|
|
|
its_probe_one(&res, &np->fwnode, of_node_to_nid(np));
|
2014-11-24 14:35:17 +00:00
|
|
|
}
|
2016-09-12 18:32:24 +00:00
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2016-09-12 18:32:25 +00:00
|
|
|
#ifdef CONFIG_ACPI
|
|
|
|
|
|
|
|
#define ACPI_GICV3_ITS_MEM_SIZE (SZ_128K)
|
|
|
|
|
2017-07-12 13:25:09 +00:00
|
|
|
#ifdef CONFIG_ACPI_NUMA
|
2017-06-22 06:10:12 +00:00
|
|
|
struct its_srat_map {
|
|
|
|
/* numa node id */
|
|
|
|
u32 numa_node;
|
|
|
|
/* GIC ITS ID */
|
|
|
|
u32 its_id;
|
|
|
|
};
|
|
|
|
|
2017-07-26 10:15:49 +00:00
|
|
|
static struct its_srat_map *its_srat_maps __initdata;
|
2017-06-22 06:10:12 +00:00
|
|
|
static int its_in_srat __initdata;
|
|
|
|
|
|
|
|
static int __init acpi_get_its_numa_node(u32 its_id)
|
|
|
|
{
|
|
|
|
int i;
|
|
|
|
|
|
|
|
for (i = 0; i < its_in_srat; i++) {
|
|
|
|
if (its_id == its_srat_maps[i].its_id)
|
|
|
|
return its_srat_maps[i].numa_node;
|
|
|
|
}
|
|
|
|
return NUMA_NO_NODE;
|
|
|
|
}
|
|
|
|
|
2017-07-26 10:15:49 +00:00
|
|
|
static int __init gic_acpi_match_srat_its(struct acpi_subtable_header *header,
|
|
|
|
const unsigned long end)
|
|
|
|
{
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
2017-06-22 06:10:12 +00:00
|
|
|
static int __init gic_acpi_parse_srat_its(struct acpi_subtable_header *header,
|
|
|
|
const unsigned long end)
|
|
|
|
{
|
|
|
|
int node;
|
|
|
|
struct acpi_srat_gic_its_affinity *its_affinity;
|
|
|
|
|
|
|
|
its_affinity = (struct acpi_srat_gic_its_affinity *)header;
|
|
|
|
if (!its_affinity)
|
|
|
|
return -EINVAL;
|
|
|
|
|
|
|
|
if (its_affinity->header.length < sizeof(*its_affinity)) {
|
|
|
|
pr_err("SRAT: Invalid header length %d in ITS affinity\n",
|
|
|
|
its_affinity->header.length);
|
|
|
|
return -EINVAL;
|
|
|
|
}
|
|
|
|
|
|
|
|
node = acpi_map_pxm_to_node(its_affinity->proximity_domain);
|
|
|
|
|
|
|
|
if (node == NUMA_NO_NODE || node >= MAX_NUMNODES) {
|
|
|
|
pr_err("SRAT: Invalid NUMA node %d in ITS affinity\n", node);
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
its_srat_maps[its_in_srat].numa_node = node;
|
|
|
|
its_srat_maps[its_in_srat].its_id = its_affinity->its_id;
|
|
|
|
its_in_srat++;
|
|
|
|
pr_info("SRAT: PXM %d -> ITS %d -> Node %d\n",
|
|
|
|
its_affinity->proximity_domain, its_affinity->its_id, node);
|
|
|
|
|
|
|
|
return 0;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init acpi_table_parse_srat_its(void)
|
|
|
|
{
|
2017-07-26 10:15:49 +00:00
|
|
|
int count;
|
|
|
|
|
|
|
|
count = acpi_table_parse_entries(ACPI_SIG_SRAT,
|
|
|
|
sizeof(struct acpi_table_srat),
|
|
|
|
ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
|
|
|
|
gic_acpi_match_srat_its, 0);
|
|
|
|
if (count <= 0)
|
|
|
|
return;
|
|
|
|
|
treewide: kmalloc() -> kmalloc_array()
The kmalloc() function has a 2-factor argument form, kmalloc_array(). This
patch replaces cases of:
kmalloc(a * b, gfp)
with:
kmalloc_array(a * b, gfp)
as well as handling cases of:
kmalloc(a * b * c, gfp)
with:
kmalloc(array3_size(a, b, c), gfp)
as it's slightly less ugly than:
kmalloc_array(array_size(a, b), c, gfp)
This does, however, attempt to ignore constant size factors like:
kmalloc(4 * 1024, gfp)
though any constants defined via macros get caught up in the conversion.
Any factors with a sizeof() of "unsigned char", "char", and "u8" were
dropped, since they're redundant.
The tools/ directory was manually excluded, since it has its own
implementation of kmalloc().
The Coccinelle script used for this was:
// Fix redundant parens around sizeof().
@@
type TYPE;
expression THING, E;
@@
(
kmalloc(
- (sizeof(TYPE)) * E
+ sizeof(TYPE) * E
, ...)
|
kmalloc(
- (sizeof(THING)) * E
+ sizeof(THING) * E
, ...)
)
// Drop single-byte sizes and redundant parens.
@@
expression COUNT;
typedef u8;
typedef __u8;
@@
(
kmalloc(
- sizeof(u8) * (COUNT)
+ COUNT
, ...)
|
kmalloc(
- sizeof(__u8) * (COUNT)
+ COUNT
, ...)
|
kmalloc(
- sizeof(char) * (COUNT)
+ COUNT
, ...)
|
kmalloc(
- sizeof(unsigned char) * (COUNT)
+ COUNT
, ...)
|
kmalloc(
- sizeof(u8) * COUNT
+ COUNT
, ...)
|
kmalloc(
- sizeof(__u8) * COUNT
+ COUNT
, ...)
|
kmalloc(
- sizeof(char) * COUNT
+ COUNT
, ...)
|
kmalloc(
- sizeof(unsigned char) * COUNT
+ COUNT
, ...)
)
// 2-factor product with sizeof(type/expression) and identifier or constant.
@@
type TYPE;
expression THING;
identifier COUNT_ID;
constant COUNT_CONST;
@@
(
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * (COUNT_ID)
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * COUNT_ID
+ COUNT_ID, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * (COUNT_CONST)
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * COUNT_CONST
+ COUNT_CONST, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * (COUNT_ID)
+ COUNT_ID, sizeof(THING)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * COUNT_ID
+ COUNT_ID, sizeof(THING)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * (COUNT_CONST)
+ COUNT_CONST, sizeof(THING)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * COUNT_CONST
+ COUNT_CONST, sizeof(THING)
, ...)
)
// 2-factor product, only identifiers.
@@
identifier SIZE, COUNT;
@@
- kmalloc
+ kmalloc_array
(
- SIZE * COUNT
+ COUNT, SIZE
, ...)
// 3-factor product with 1 sizeof(type) or sizeof(expression), with
// redundant parens removed.
@@
expression THING;
identifier STRIDE, COUNT;
type TYPE;
@@
(
kmalloc(
- sizeof(TYPE) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kmalloc(
- sizeof(TYPE) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kmalloc(
- sizeof(TYPE) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kmalloc(
- sizeof(TYPE) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(TYPE))
, ...)
|
kmalloc(
- sizeof(THING) * (COUNT) * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kmalloc(
- sizeof(THING) * (COUNT) * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kmalloc(
- sizeof(THING) * COUNT * (STRIDE)
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
|
kmalloc(
- sizeof(THING) * COUNT * STRIDE
+ array3_size(COUNT, STRIDE, sizeof(THING))
, ...)
)
// 3-factor product with 2 sizeof(variable), with redundant parens removed.
@@
expression THING1, THING2;
identifier COUNT;
type TYPE1, TYPE2;
@@
(
kmalloc(
- sizeof(TYPE1) * sizeof(TYPE2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kmalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(TYPE2))
, ...)
|
kmalloc(
- sizeof(THING1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kmalloc(
- sizeof(THING1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(THING1), sizeof(THING2))
, ...)
|
kmalloc(
- sizeof(TYPE1) * sizeof(THING2) * COUNT
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
|
kmalloc(
- sizeof(TYPE1) * sizeof(THING2) * (COUNT)
+ array3_size(COUNT, sizeof(TYPE1), sizeof(THING2))
, ...)
)
// 3-factor product, only identifiers, with redundant parens removed.
@@
identifier STRIDE, SIZE, COUNT;
@@
(
kmalloc(
- (COUNT) * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- COUNT * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- COUNT * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- (COUNT) * (STRIDE) * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- COUNT * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- (COUNT) * STRIDE * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- (COUNT) * (STRIDE) * (SIZE)
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
|
kmalloc(
- COUNT * STRIDE * SIZE
+ array3_size(COUNT, STRIDE, SIZE)
, ...)
)
// Any remaining multi-factor products, first at least 3-factor products,
// when they're not all constants...
@@
expression E1, E2, E3;
constant C1, C2, C3;
@@
(
kmalloc(C1 * C2 * C3, ...)
|
kmalloc(
- (E1) * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
|
kmalloc(
- (E1) * (E2) * E3
+ array3_size(E1, E2, E3)
, ...)
|
kmalloc(
- (E1) * (E2) * (E3)
+ array3_size(E1, E2, E3)
, ...)
|
kmalloc(
- E1 * E2 * E3
+ array3_size(E1, E2, E3)
, ...)
)
// And then all remaining 2 factors products when they're not all constants,
// keeping sizeof() as the second factor argument.
@@
expression THING, E1, E2;
type TYPE;
constant C1, C2, C3;
@@
(
kmalloc(sizeof(THING) * C2, ...)
|
kmalloc(sizeof(TYPE) * C2, ...)
|
kmalloc(C1 * C2 * C3, ...)
|
kmalloc(C1 * C2, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * (E2)
+ E2, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(TYPE) * E2
+ E2, sizeof(TYPE)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * (E2)
+ E2, sizeof(THING)
, ...)
|
- kmalloc
+ kmalloc_array
(
- sizeof(THING) * E2
+ E2, sizeof(THING)
, ...)
|
- kmalloc
+ kmalloc_array
(
- (E1) * E2
+ E1, E2
, ...)
|
- kmalloc
+ kmalloc_array
(
- (E1) * (E2)
+ E1, E2
, ...)
|
- kmalloc
+ kmalloc_array
(
- E1 * E2
+ E1, E2
, ...)
)
Signed-off-by: Kees Cook <keescook@chromium.org>
2018-06-12 20:55:00 +00:00
|
|
|
its_srat_maps = kmalloc_array(count, sizeof(struct its_srat_map),
|
|
|
|
GFP_KERNEL);
|
2017-07-26 10:15:49 +00:00
|
|
|
if (!its_srat_maps) {
|
|
|
|
pr_warn("SRAT: Failed to allocate memory for its_srat_maps!\n");
|
|
|
|
return;
|
|
|
|
}
|
|
|
|
|
2017-06-22 06:10:12 +00:00
|
|
|
acpi_table_parse_entries(ACPI_SIG_SRAT,
|
|
|
|
sizeof(struct acpi_table_srat),
|
|
|
|
ACPI_SRAT_TYPE_GIC_ITS_AFFINITY,
|
|
|
|
gic_acpi_parse_srat_its, 0);
|
|
|
|
}
|
2017-07-26 10:15:49 +00:00
|
|
|
|
|
|
|
/* free the its_srat_maps after ITS probing */
|
|
|
|
static void __init acpi_its_srat_maps_free(void)
|
|
|
|
{
|
|
|
|
kfree(its_srat_maps);
|
|
|
|
}
|
2017-06-22 06:10:12 +00:00
|
|
|
#else
|
|
|
|
static void __init acpi_table_parse_srat_its(void) { }
|
|
|
|
static int __init acpi_get_its_numa_node(u32 its_id) { return NUMA_NO_NODE; }
|
2017-07-26 10:15:49 +00:00
|
|
|
static void __init acpi_its_srat_maps_free(void) { }
|
2017-06-22 06:10:12 +00:00
|
|
|
#endif
|
|
|
|
|
2016-09-12 18:32:25 +00:00
|
|
|
static int __init gic_acpi_parse_madt_its(struct acpi_subtable_header *header,
|
|
|
|
const unsigned long end)
|
|
|
|
{
|
|
|
|
struct acpi_madt_generic_translator *its_entry;
|
|
|
|
struct fwnode_handle *dom_handle;
|
|
|
|
struct resource res;
|
|
|
|
int err;
|
|
|
|
|
|
|
|
its_entry = (struct acpi_madt_generic_translator *)header;
|
|
|
|
memset(&res, 0, sizeof(res));
|
|
|
|
res.start = its_entry->base_address;
|
|
|
|
res.end = its_entry->base_address + ACPI_GICV3_ITS_MEM_SIZE - 1;
|
|
|
|
res.flags = IORESOURCE_MEM;
|
|
|
|
|
|
|
|
dom_handle = irq_domain_alloc_fwnode((void *)its_entry->base_address);
|
|
|
|
if (!dom_handle) {
|
|
|
|
pr_err("ITS@%pa: Unable to allocate GICv3 ITS domain token\n",
|
|
|
|
&res.start);
|
|
|
|
return -ENOMEM;
|
|
|
|
}
|
|
|
|
|
2018-02-13 15:20:50 +00:00
|
|
|
err = iort_register_domain_token(its_entry->translation_id, res.start,
|
|
|
|
dom_handle);
|
2016-09-12 18:32:25 +00:00
|
|
|
if (err) {
|
|
|
|
pr_err("ITS@%pa: Unable to register GICv3 ITS domain token (ITS ID %d) to IORT\n",
|
|
|
|
&res.start, its_entry->translation_id);
|
|
|
|
goto dom_err;
|
|
|
|
}
|
|
|
|
|
2017-06-22 06:10:12 +00:00
|
|
|
err = its_probe_one(&res, dom_handle,
|
|
|
|
acpi_get_its_numa_node(its_entry->translation_id));
|
2016-09-12 18:32:25 +00:00
|
|
|
if (!err)
|
|
|
|
return 0;
|
|
|
|
|
|
|
|
iort_deregister_domain_token(its_entry->translation_id);
|
|
|
|
dom_err:
|
|
|
|
irq_domain_free_fwnode(dom_handle);
|
|
|
|
return err;
|
|
|
|
}
|
|
|
|
|
|
|
|
static void __init its_acpi_probe(void)
|
|
|
|
{
|
2017-06-22 06:10:12 +00:00
|
|
|
acpi_table_parse_srat_its();
|
2016-09-12 18:32:25 +00:00
|
|
|
acpi_table_parse_madt(ACPI_MADT_TYPE_GENERIC_TRANSLATOR,
|
|
|
|
gic_acpi_parse_madt_its, 0);
|
2017-07-26 10:15:49 +00:00
|
|
|
acpi_its_srat_maps_free();
|
2016-09-12 18:32:25 +00:00
|
|
|
}
|
|
|
|
#else
|
|
|
|
static void __init its_acpi_probe(void) { }
|
|
|
|
#endif
|
|
|
|
|
2016-09-12 18:32:24 +00:00
|
|
|
int __init its_init(struct fwnode_handle *handle, struct rdists *rdists,
|
|
|
|
struct irq_domain *parent_domain)
|
|
|
|
{
|
|
|
|
struct device_node *of_node;
|
2016-12-20 13:41:55 +00:00
|
|
|
struct its_node *its;
|
|
|
|
bool has_v4 = false;
|
|
|
|
int err;
|
2016-09-12 18:32:24 +00:00
|
|
|
|
|
|
|
its_parent = parent_domain;
|
|
|
|
of_node = to_of_node(handle);
|
|
|
|
if (of_node)
|
|
|
|
its_of_probe(of_node);
|
|
|
|
else
|
2016-09-12 18:32:25 +00:00
|
|
|
its_acpi_probe();
|
2014-11-24 14:35:17 +00:00
|
|
|
|
|
|
|
if (list_empty(&its_nodes)) {
|
|
|
|
pr_warn("ITS: No ITS available, not enabling LPIs\n");
|
|
|
|
return -ENXIO;
|
|
|
|
}
|
|
|
|
|
|
|
|
gic_rdists = rdists;
|
2016-12-20 13:41:55 +00:00
|
|
|
err = its_alloc_lpi_tables();
|
|
|
|
if (err)
|
|
|
|
return err;
|
|
|
|
|
|
|
|
list_for_each_entry(its, &its_nodes, entry)
|
|
|
|
has_v4 |= its->is_v4;
|
|
|
|
|
|
|
|
if (has_v4 & rdists->has_vlpis) {
|
2016-12-20 15:31:54 +00:00
|
|
|
if (its_init_vpe_domain() ||
|
|
|
|
its_init_v4(parent_domain, &its_vpe_domain_ops)) {
|
2016-12-20 13:41:55 +00:00
|
|
|
rdists->has_vlpis = false;
|
|
|
|
pr_err("ITS: Disabling GICv4 support\n");
|
|
|
|
}
|
|
|
|
}
|
|
|
|
|
2018-03-01 05:48:18 +00:00
|
|
|
register_syscore_ops(&its_syscore_ops);
|
|
|
|
|
2016-12-20 13:41:55 +00:00
|
|
|
return 0;
|
2014-11-24 14:35:17 +00:00
|
|
|
}
|