arm64: GICv3 support

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Hi Marc,

Quoted 14 lines · Marc Zyngier

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

[...]

include/linux/irqchip/arm-gic-v3.h line 110

Quoted 11 lines

include/linux/irqchip/arm-gic-v3.h new file +11 -0

+/*
+ * Hypervisor interface registers (SRE only)
+ */
+#define GICH_LR_VIRTUAL_ID_MASK                ((1UL << 32) - 1)
+
+#define GICH_LR_EOI                    (1UL << 41)
+#define GICH_LR_GROUP                  (1UL << 60)
+#define GICH_LR_STATE                  (3UL << 62)
+#define GICH_LR_PENDING_BIT            (1UL << 62)
+#define GICH_LR_ACTIVE_BIT             (1UL << 63)
+

Minor nit:
I think these names can be misleading, since the GICH_LR in GICv3 is the same as in GICv2, and is 32bit.
Since AArch32 splits the LR into two system registers, it may be clearer to name these bits ICH_LR* on arm64, and ICH_LRC* on arm.

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Cc Thomas Gleixner <[email protected]>

Reviewed-by Zi Shen Lim <[email protected]>

Reviewed-by Christoffer Dall <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/Kconfig                 |   1 +
 arch/arm64/kernel/head.S           |  18 +
 arch/arm64/kernel/hyp-stub.S       |   1 +
 drivers/irqchip/Kconfig            |   5 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-gic-v3.c       | 684 +++++++++++++++++++++++++++++++++++++
 include/linux/irqchip/arm-gic-v3.h | 190 +++++++++++
 7 files changed, 900 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3.c
 create mode 100644 include/linux/irqchip/arm-gic-v3.h

drivers/irqchip/irq-gic-v3.c new file +239 -0

new file mode 100644
index 0000000..6570e7b
@@ -0,0 +1,684 @@
+/*
+ * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
+ * Author: Marc Zyngier <[email protected]>
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <linux/cpu.h>
+#include <linux/delay.h>
+#include <linux/interrupt.h>
+#include <linux/of.h>
+#include <linux/of_address.h>
+#include <linux/of_irq.h>
+#include <linux/percpu.h>
+#include <linux/slab.h>
+
+#include <linux/irqchip/arm-gic-v3.h>
+
+#include <asm/cputype.h>
+#include <asm/exception.h>
+#include <asm/smp_plat.h>
+
+#include "irq-gic-common.h"
+#include "irqchip.h"
+
+struct gic_chip_data {
+	void __iomem		*dist_base;
+	void __iomem		**redist_base;
+	void __percpu __iomem	**rdist;
+	struct irq_domain	*domain;
+	u64			redist_stride;
+	u32			redist_regions;
+	unsigned int		irq_nr;
+};
+
+static struct gic_chip_data gic_data __read_mostly;
+
+#define gic_data_rdist()		(this_cpu_ptr(gic_data.rdist))
+#define gic_data_rdist_rd_base()	(*gic_data_rdist())
+#define gic_data_rdist_sgi_base()	(gic_data_rdist_rd_base() + SZ_64K)
+
+#define DEFAULT_PMR_VALUE	0xf0
+
+static inline unsigned int gic_irq(struct irq_data *d)
+{
+	return d->hwirq;
+}
+
+static inline int gic_irq_in_rdist(struct irq_data *d)
+{
+	return gic_irq(d) < 32;
+}
+
+static inline void __iomem *gic_dist_base(struct irq_data *d)
+{
+	if (gic_irq_in_rdist(d))	/* SGI+PPI -> SGI_base for this CPU */
+		return gic_data_rdist_sgi_base();
+
+	if (d->hwirq <= 1023)		/* SPI -> dist_base */
+		return gic_data.dist_base;
+
+	if (d->hwirq >= 8192)
+		BUG();		/* LPI Detected!!! */
+
+	return NULL;
+}
+
+static void gic_do_wait_for_rwp(void __iomem *base)
+{
+	u32 count = 1000000;	/* 1s! */
+
+	while (readl_relaxed(base + GICD_CTLR) & GICD_CTLR_RWP) {
+		count--;
+		if (!count) {
+			pr_err_ratelimited("RWP timeout, gone fishing\n");
+			return;
+		}
+		cpu_relax();
+		udelay(1);
+	};
+}
+
+/* Wait for completion of a distributor change */
+static void gic_dist_wait_for_rwp(void)
+{
+	gic_do_wait_for_rwp(gic_data.dist_base);
+}
+
+/* Wait for completion of a redistributor change */
+static void gic_redist_wait_for_rwp(void)
+{
+	gic_do_wait_for_rwp(gic_data_rdist_rd_base());
+}
+
+/* Low level accessors */
+static u64 gic_read_iar(void)
+{
+	u64 irqstat;
+
+	asm volatile("mrs %0, " __stringify(ICC_IAR1_EL1) : "=r" (irqstat));
+	return irqstat;
+}
+
+static void gic_write_pmr(u64 val)
+{
+	asm volatile("msr " __stringify(ICC_PMR_EL1) ", %0" : : "r" (val));
+}
+
+static void gic_write_ctlr(u64 val)
+{
+	asm volatile("msr " __stringify(ICC_CTLR_EL1) ", %0" : : "r" (val));
+	isb();
+}
+
+static void gic_write_grpen1(u64 val)
+{
+	asm volatile("msr " __stringify(ICC_GRPEN1_EL1) ", %0" : : "r" (val));
+	isb();
+}
+
+static void gic_write_sgi1r(u64 val)
+{
+	asm volatile("msr " __stringify(ICC_SGI1R_EL1) ", %0" : : "r" (val));
+}
+
+static void gic_enable_sre(void)
+{
+	u64 val;
+
+	asm volatile("mrs %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
+	val |= GICC_SRE_EL1_SRE;
+	asm volatile("msr " __stringify(ICC_SRE_EL1) ", %0" : : "r" (val));
+	isb();
+
+	/*
+	 * Need to check that the SRE bit has actually been set. If
+	 * not, it means that SRE is disabled at EL2. We're going to
+	 * die painfully, and there is nothing we can do about it.
+	 *
+	 * Kindly inform the luser.
+	 */
+	asm volatile("mrs %0, " __stringify(ICC_SRE_EL1) : "=r" (val));
+	if (!(val & GICC_SRE_EL1_SRE))
+		pr_err("GIC: unable to set SRE (disabled at EL2), panic ahead\n");
+}
+
+static void gic_enable_redist(void)
+{
+	void __iomem *rbase;
+	u32 count = 1000000;	/* 1s! */
+	u32 val;
+
+	rbase = gic_data_rdist_rd_base();
+
+	/* Wake up this CPU redistributor */
+	val = readl_relaxed(rbase + GICR_WAKER);
+	val &= ~GICR_WAKER_ProcessorSleep;
+	writel_relaxed(val, rbase + GICR_WAKER);
+
+	while (readl_relaxed(rbase + GICR_WAKER) & GICR_WAKER_ChildrenAsleep) {
+		count--;
+		if (!count) {
+			pr_err_ratelimited("redist didn't wake up...\n");
+			return;
+		}
+		cpu_relax();
+		udelay(1);
+	};
+}
+
+/*
+ * Routines to acknowledge, disable and enable interrupts
+ */
+static void gic_poke_irq(struct irq_data *d, u32 offset)
+{
+	u32 mask = 1 << (gic_irq(d) % 32);
+	void (*rwp_wait)(void);
+	void __iomem *base;
+
+	if (gic_irq_in_rdist(d)) {
+		base = gic_data_rdist_sgi_base();
+		rwp_wait = gic_redist_wait_for_rwp;
+	} else {
+		base = gic_data.dist_base;
+		rwp_wait = gic_dist_wait_for_rwp;
+	}
+
+	writel_relaxed(mask, base + offset + (gic_irq(d) / 32) * 4);
+	rwp_wait();
+}
+
+static int gic_peek_irq(struct irq_data *d, u32 offset)
+{
+	u32 mask = 1 << (gic_irq(d) % 32);
+	void __iomem *base;
+
+	if (gic_irq_in_rdist(d))
+		base = gic_data_rdist_sgi_base();
+	else
+		base = gic_data.dist_base;
+
+	return !!(readl_relaxed(base + offset + (gic_irq(d) / 32) * 4) & mask);
+}
+
+static void gic_mask_irq(struct irq_data *d)
+{
+	gic_poke_irq(d, GICD_ICENABLER);
+}
+
+static void gic_unmask_irq(struct irq_data *d)
+{
+	gic_poke_irq(d, GICD_ISENABLER);
+}
+
+static void gic_eoi_irq(struct irq_data *d)
+{
+	gic_write_eoir(gic_irq(d));
+}
+
+static int gic_set_type(struct irq_data *d, unsigned int type)
+{
+	unsigned int irq = gic_irq(d);
+	void (*rwp_wait)(void);
+	void __iomem *base;
+
+	if (gic_irq_in_rdist(d)) {
+		base = gic_data_rdist_sgi_base();
+		rwp_wait = gic_redist_wait_for_rwp;
+	} else {

drivers/irqchip/irq-gic-v3.c new file +16 -0

+
+	gic_configure_irq(irq, type, base, rwp_wait);
+
+	return 0;
+}
+
+static u64 gic_mpidr_to_affinity(u64 mpidr)
+{
+	u64 aff;
+
+	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 0)) & ~GICD_IROUTER_SPI_MODE_ANY;
+
+	return aff;

drivers/irqchip/irq-gic-v3.c new file +14 -0

+			irqnr = irq_find_mapping(gic_data.domain, irqnr);
+			handle_IRQ(irqnr, regs);
+			continue;
+		}
+		if (irqnr < 16) {
+			gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+			handle_IPI(irqnr, regs);
+#else
+			WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+			continue;
+		}
+	} while (irqnr != 0x3ff);

drivers/irqchip/irq-gic-v3.c new file +246 -0

+
+	gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
+
+	/* Enable distributor with ARE, Group1 */
+	writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
+		       base + GICD_CTLR);
+
+	/*
+	 * Set all global interrupts to the boot CPU only. ARE must be
+	 * enabled.
+	 */
+	affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
+	for (i = 32; i < gic_data.irq_nr; i++)
+		writeq_relaxed(affinity, base + GICD_IROUTER + i * 8);
+}
+
+static int gic_populate_rdist(void)
+{
+	u64 mpidr = cpu_logical_map(smp_processor_id());
+	u64 typer;
+	u32 aff;
+	int i;
+
+	/*
+	 * Convert affinity to a 32bit value that can be matched to
+	 * GICR_TYPER bits [63:32].
+	 */
+	aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 24 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8 |
+	       MPIDR_AFFINITY_LEVEL(mpidr, 0));
+
+	for (i = 0; i < gic_data.redist_regions; i++) {
+		void __iomem *ptr = gic_data.redist_base[i];
+		u32 reg;
+
+		reg = readl_relaxed(ptr + GICR_PIDR2) & GIC_PIDR2_ARCH_MASK;
+		if (reg != 0x30 && reg != 0x40) { /* We're in trouble... */
+			pr_warn("No redistributor present @%p\n", ptr);
+			break;
+		}
+
+		do {
+			typer = readq_relaxed(ptr + GICR_TYPER);
+			if ((typer >> 32) == aff) {
+				gic_data_rdist_rd_base() = ptr;
+				pr_info("CPU%d: found redistributor %llx @%p\n",
+					smp_processor_id(),
+					(unsigned long long)mpidr, ptr);
+				return 0;
+			}
+
+			if (gic_data.redist_stride) {
+				ptr += gic_data.redist_stride;
+			} else {
+				ptr += SZ_64K * 2; /* Skip RD_base + SGI_base */
+				if (typer & GICR_TYPER_VLPIS)
+					ptr += SZ_64K * 2; /* Skip VLPI_base + reserved page */
+			}
+		} while (!(typer & GICR_TYPER_LAST));
+	}
+
+	/* We couldn't even deal with ourselves... */
+	WARN(true, "CPU%d: mpidr %llx has no re-distributor!\n",
+	     smp_processor_id(), (unsigned long long)mpidr);
+	return -ENODEV;
+}
+
+static void gic_cpu_init(void)
+{
+	void __iomem *rbase;
+
+	/* Register ourselves with the rest of the world */
+	if (gic_populate_rdist())
+		return;
+
+	gic_enable_redist();
+
+	rbase = gic_data_rdist_sgi_base();
+
+	gic_cpu_config(rbase, gic_redist_wait_for_rwp);
+
+	/* Enable system registers */
+	gic_enable_sre();
+
+	/* Set priority mask register */
+	gic_write_pmr(DEFAULT_PMR_VALUE);
+
+	/* EOI deactivates interrupt too (mode 0) */
+	gic_write_ctlr(GICC_CTLR_EL1_EOImode_drop_dir);
+
+	/* ... and let's hit the road... */
+	gic_write_grpen1(1);
+}
+
+#ifdef CONFIG_SMP
+static int gic_secondary_init(struct notifier_block *nfb,
+			      unsigned long action, void *hcpu)
+{
+	if (action == CPU_STARTING || action == CPU_STARTING_FROZEN)
+		gic_cpu_init();
+	return NOTIFY_OK;
+}
+
+/*
+ * Notifier for enabling the GIC CPU interface. Set an arbitrarily high
+ * priority because the GIC needs to be up before the ARM generic timers.
+ */
+static struct notifier_block gic_cpu_notifier = {
+	.notifier_call = gic_secondary_init,
+	.priority = 100,
+};
+
+static u16 gic_compute_target_list(int *base_cpu, const struct cpumask *mask,
+				   u64 cluster_id)
+{
+	int cpu = *base_cpu;
+	u64 mpidr = cpu_logical_map(cpu);
+	u16 tlist = 0;
+
+	while (cpu < nr_cpu_ids) {
+		/*
+		 * If we ever get a cluster of more than 16 CPUs, just
+		 * scream and skip that CPU.
+		 */
+		if (WARN_ON((mpidr & 0xff) >= 16))
+			goto out;
+
+		tlist |= 1 << (mpidr & 0xf);
+
+		cpu = cpumask_next(cpu, mask);
+		if (cpu == nr_cpu_ids)
+			goto out;
+
+		mpidr = cpu_logical_map(cpu);
+
+		if (cluster_id != (mpidr & ~0xffUL)) {
+			cpu--;
+			goto out;
+		}
+	}
+out:
+	*base_cpu = cpu;
+	return tlist;
+}
+
+static void gic_send_sgi(u64 cluster_id, u16 tlist, unsigned int irq)
+{
+	u64 val;
+
+	val = (MPIDR_AFFINITY_LEVEL(cluster_id, 3) << 48	|
+	       MPIDR_AFFINITY_LEVEL(cluster_id, 2) << 32	|
+	       irq << 24			    		|
+	       MPIDR_AFFINITY_LEVEL(cluster_id, 1) << 16	|
+	       tlist);
+
+	pr_debug("CPU%d: ICC_SGI1R_EL1 %llx\n", smp_processor_id(), val);
+	gic_write_sgi1r(val);
+}
+
+static void gic_raise_softirq(const struct cpumask *mask, unsigned int irq)
+{
+	int cpu;
+
+	if (WARN_ON(irq >= 16))
+		return;
+
+	/*
+	 * Ensure that stores to Normal memory are visible to the
+	 * other CPUs before issuing the IPI.
+	 */
+	smp_wmb();
+
+	for_each_cpu_mask(cpu, *mask) {
+		u64 cluster_id = cpu_logical_map(cpu) & ~0xffUL;
+		u16 tlist;
+
+		tlist = gic_compute_target_list(&cpu, mask, cluster_id);
+		gic_send_sgi(cluster_id, tlist, irq);
+	}
+
+	/* Force the above writes to ICC_SGI1R_EL1 to be executed */
+	isb();
+}
+
+static void gic_smp_init(void)
+{
+	set_smp_cross_call(gic_raise_softirq);
+	register_cpu_notifier(&gic_cpu_notifier);
+}
+
+static int gic_set_affinity(struct irq_data *d, const struct cpumask *mask_val,
+			    bool force)
+{
+	unsigned int cpu = cpumask_any_and(mask_val, cpu_online_mask);
+	void __iomem *reg;
+	int enabled;
+	u64 val;
+
+	if (gic_irq_in_rdist(d))
+		return -EINVAL;
+
+	/* If interrupt was enabled, disable it first */
+	enabled = gic_peek_irq(d, GICD_ISENABLER);
+	if (enabled)
+		gic_mask_irq(d);
+
+	reg = gic_dist_base(d) + GICD_IROUTER + (gic_irq(d) * 8);
+	val = gic_mpidr_to_affinity(cpu_logical_map(cpu));
+
+	writeq_relaxed(val, reg);
+
+	/*
+	 * If the interrupt was enabled, enabled it again. Otherwise,
+	 * just wait for the distributor to have digested our changes.
+	 */
+	if (enabled)
+		gic_unmask_irq(d);
+	else
+		gic_dist_wait_for_rwp();
 
+
 
+	return IRQ_SET_MASK_OK;
+}
+#else
+#define gic_set_affinity	NULL
+#define gic_smp_init()		do { } while(0)
+#endif
+
+static struct irq_chip gic_chip = {
+	.name			= "GICv3",
+	.irq_mask		= gic_mask_irq,
+	.irq_unmask		= gic_unmask_irq,
+	.irq_eoi		= gic_eoi_irq,
+	.irq_set_type		= gic_set_type,
+	.irq_set_affinity	= gic_set_affinity,
+};
+
+static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
+			      irq_hw_number_t hw)
+{
+	/* SGIs are private to the core kernel */
+	if (hw < 16)
+		return -EPERM;
+	/* PPIs */
+	if (hw < 32) {
+		irq_set_percpu_devid(irq);

drivers/irqchip/irq-gic-v3.c new file +129 -0

+	irq_set_chip_data(irq, d->host_data);
+	return 0;
+}
+
+static int gic_irq_domain_xlate(struct irq_domain *d,
+				struct device_node *controller,
+				const u32 *intspec, unsigned int intsize,
+				unsigned long *out_hwirq, unsigned int *out_type)
+{
+	if (d->of_node != controller)
+		return -EINVAL;
+	if (intsize < 3)
+		return -EINVAL;
+
+	switch(intspec[0]) {
+	case 0:			/* SPI */
+		*out_hwirq = intspec[1] + 32;
+		break;
+	case 1:			/* PPI */
+		*out_hwirq = intspec[1] + 16;
+		break;
+	default:
+		return -EINVAL;
+	}
+
+	*out_type = intspec[2] & IRQ_TYPE_SENSE_MASK;
+	return 0;
+}
+
+static const struct irq_domain_ops gic_irq_domain_ops = {
+	.map = gic_irq_domain_map,
+	.xlate = gic_irq_domain_xlate,
+};
+
+static int __init gic_of_init(struct device_node *node, struct device_node *parent)
+{
+	void __iomem *dist_base;
+	void __iomem **redist_base;
+	u64 redist_stride;
+	u32 redist_regions;
+	u32 reg;
+	int gic_irqs;
+	int err;
+	int i;
+
+	dist_base = of_iomap(node, 0);
+	if (!dist_base) {
+		pr_err("%s: unable to map gic dist registers\n",
+			node->full_name);
+		return -ENXIO;
+	}
+
+	reg = readl_relaxed(dist_base + GICD_PIDR2) & GIC_PIDR2_ARCH_MASK;
+	if (reg != 0x30 && reg != 0x40) {
+		pr_err("%s: no distributor detected, giving up\n",
+			node->full_name);
+		err = -ENODEV;
+		goto out_unmap_dist;
+	}
+
+	if (of_property_read_u32(node, "#redistributor-regions", &redist_regions))
+		redist_regions = 1;
+
+	redist_base = kzalloc(sizeof(*redist_base) * redist_regions, GFP_KERNEL);
+	if (!redist_base) {
+		err = -ENOMEM;
+		goto out_unmap_dist;
+	}
+
+	for (i = 0; i < redist_regions; i++) {
+		redist_base[i] = of_iomap(node, 1 + i);
+		if (!redist_base[i]) {
+			pr_err("%s: couldn't map region %d\n",
+			       node->full_name, i);
+			err = -ENODEV;
+			goto out_unmap_rdist;
+		}
+	}
+
+	if (of_property_read_u64(node, "redistributor-stride", &redist_stride))
+		redist_stride = 0;
+
+	gic_data.dist_base = dist_base;
+	gic_data.redist_base = redist_base;
+	gic_data.redist_regions = redist_regions;
+	gic_data.redist_stride = redist_stride;
+
+	/*
+	 * Find out how many interrupts are supported.
+	 * The GIC only supports up to 1020 interrupt sources (SGI+PPI+SPI)
+	 */
+	gic_irqs = readl_relaxed(gic_data.dist_base + GICD_TYPER) & 0x1f;
+	gic_irqs = (gic_irqs + 1) * 32;
+	if (gic_irqs > 1020)
+		gic_irqs = 1020;
+	gic_data.irq_nr = gic_irqs;
+
+	gic_data.domain = irq_domain_add_tree(node, &gic_irq_domain_ops,
+					      &gic_data);
+	gic_data.rdist = alloc_percpu(typeof(*gic_data.rdist));
+
+	if (WARN_ON(!gic_data.domain) || WARN_ON(!gic_data.rdist)) {
+		err = -ENOMEM;
+		goto out_free;
+	}
+
+	set_handle_irq(gic_handle_irq);
+
+	gic_smp_init();
+	gic_dist_init();
+	gic_cpu_init();
+
+	return 0;
+
+out_free:
+	if (gic_data.domain)
+		irq_domain_remove(gic_data.domain);
+	free_percpu(gic_data.rdist);
+out_unmap_rdist:
+	for (i = 0; i < redist_regions; i++)
+		if (redist_base[i])
+			iounmap(redist_base[i]);
+	kfree(redist_base);
+out_unmap_dist:
+	iounmap(dist_base);
+	return err;
+}
+
+IRQCHIP_DECLARE(gic_v3, "arm,gic-v3", gic_of_init);

include/linux/irqchip/arm-gic-v3.h new file +190 -0

new file mode 100644
index 0000000..752cbef
@@ -0,0 +1,190 @@
+/*
+ * Copyright (C) 2013, 2014 ARM Limited, All Rights Reserved.
+ * Author: Marc Zyngier <[email protected]>
+ *
+ *
+ * This program is free software; you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License version 2 as
+ * published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef __LINUX_IRQCHIP_ARM_GIC_V3_H
+#define __LINUX_IRQCHIP_ARM_GIC_V3_H
+
+/*
+ * Distributor registers. We assume we're running non-secure, with ARE
+ * being set. Secure-only and non-ARE registers are not described.
+ */
+#define GICD_CTLR			0x0000
+#define GICD_TYPER			0x0004
+#define GICD_IIDR			0x0008
+#define GICD_STATUSR			0x0010
+#define GICD_SETSPI_NSR			0x0040
+#define GICD_CLRSPI_NSR			0x0048
+#define GICD_SETSPI_SR			0x0050
+#define GICD_CLRSPI_SR			0x0058
+#define GICD_SEIR			0x0068
+#define GICD_ISENABLER			0x0100
+#define GICD_ICENABLER			0x0180
+#define GICD_ISPENDR			0x0200
+#define GICD_ICPENDR			0x0280
+#define GICD_ISACTIVER			0x0300
+#define GICD_ICACTIVER			0x0380
+#define GICD_IPRIORITYR			0x0400
+#define GICD_ICFGR			0x0C00
+#define GICD_IROUTER			0x6000
+#define GICD_PIDR2			0xFFE8
+
+#define GICD_CTLR_RWP			(1U << 31)
+#define GICD_CTLR_ARE_NS		(1U << 4)
+#define GICD_CTLR_ENABLE_G1A		(1U << 1)
+#define GICD_CTLR_ENABLE_G1		(1U << 0)
+
+#define GICD_IROUTER_SPI_MODE_ONE	(0U << 31)
+#define GICD_IROUTER_SPI_MODE_ANY	(1U << 31)
+
+#define GIC_PIDR2_ARCH_MASK		0xf0
+
+/*
+ * Re-Distributor registers, offsets from RD_base
+ */
+#define GICR_CTLR			GICD_CTLR
+#define GICR_IIDR			0x0004
+#define GICR_TYPER			0x0008
+#define GICR_STATUSR			GICD_STATUSR
+#define GICR_WAKER			0x0014
+#define GICR_SETLPIR			0x0040
+#define GICR_CLRLPIR			0x0048
+#define GICR_SEIR			GICD_SEIR
+#define GICR_PROPBASER			0x0070
+#define GICR_PENDBASER			0x0078
+#define GICR_INVLPIR			0x00A0
+#define GICR_INVALLR			0x00B0
+#define GICR_SYNCR			0x00C0
+#define GICR_MOVLPIR			0x0100
+#define GICR_MOVALLR			0x0110
+#define GICR_PIDR2			GICD_PIDR2
+
+#define GICR_WAKER_ProcessorSleep	(1U << 1)
+#define GICR_WAKER_ChildrenAsleep	(1U << 2)
+
+/*
+ * Re-Distributor registers, offsets from SGI_base
+ */
+#define GICR_ISENABLER0			GICD_ISENABLER
+#define GICR_ICENABLER0			GICD_ICENABLER
+#define GICR_ISPENDR0			GICD_ISPENDR
+#define GICR_ICPENDR0			GICD_ICPENDR
+#define GICR_ISACTIVER0			GICD_ISACTIVER
+#define GICR_ICACTIVER0			GICD_ICACTIVER
+#define GICR_IPRIORITYR0		GICD_IPRIORITYR
+#define GICR_ICFGR0			GICD_ICFGR
+
+#define GICR_TYPER_VLPIS		(1U << 1)
+#define GICR_TYPER_LAST			(1U << 4)
+
+/*
+ * CPU interface registers
+ */
+#define GICC_CTLR_EL1_EOImode_drop_dir	(0U << 1)
+#define GICC_CTLR_EL1_EOImode_drop	(1U << 1)
+#define GICC_SRE_EL1_SRE		(1U << 0)
+
+/*
+ * Hypervisor interface registers (SRE only)
+ */
+#define GICH_LR_VIRTUAL_ID_MASK		((1UL << 32) - 1)
+
+#define GICH_LR_EOI			(1UL << 41)
+#define GICH_LR_GROUP			(1UL << 60)
+#define GICH_LR_STATE			(3UL << 62)
+#define GICH_LR_PENDING_BIT		(1UL << 62)
+#define GICH_LR_ACTIVE_BIT		(1UL << 63)
+
+#define GICH_MISR_EOI			(1 << 0)
+#define GICH_MISR_U			(1 << 1)
+
+#define GICH_HCR_EN			(1 << 0)
+#define GICH_HCR_UIE			(1 << 1)
+
+#define GICH_VMCR_CTLR_SHIFT		0
+#define GICH_VMCR_CTLR_MASK		(0x21f << GICH_VMCR_CTLR_SHIFT)
+#define GICH_VMCR_BPR1_SHIFT		18
+#define GICH_VMCR_BPR1_MASK		(7 << GICH_VMCR_BPR1_SHIFT)
+#define GICH_VMCR_BPR0_SHIFT		21
+#define GICH_VMCR_BPR0_MASK		(7 << GICH_VMCR_BPR0_SHIFT)
+#define GICH_VMCR_PMR_SHIFT		24
+#define GICH_VMCR_PMR_MASK		(0xffUL << GICH_VMCR_PMR_SHIFT)
+
+#define ICC_EOIR1_EL1			S3_0_C12_C12_1
+#define ICC_IAR1_EL1			S3_0_C12_C12_0
+#define ICC_SGI1R_EL1			S3_0_C12_C11_5
+#define ICC_PMR_EL1			S3_0_C4_C6_0
+#define ICC_CTLR_EL1			S3_0_C12_C12_4
+#define ICC_SRE_EL1			S3_0_C12_C12_5
+#define ICC_GRPEN1_EL1			S3_0_C12_C12_7
+
+#define ICC_SRE_EL2			S3_4_C12_C9_5
+
+#define ICC_SRE_EL2_ENABLE		(1 << 3)
+
+#define ICH_VSEIR_EL2			S3_4_C12_C9_4
+#define ICH_HCR_EL2			S3_4_C12_C11_0
+#define ICH_VTR_EL2			S3_4_C12_C11_1
+#define ICH_MISR_EL2			S3_4_C12_C11_2
+#define ICH_EISR_EL2			S3_4_C12_C11_3
+#define ICH_ELSR_EL2			S3_4_C12_C11_5
+#define ICH_VMCR_EL2			S3_4_C12_C11_7
+
+#define __LR0_EL2(x)			S3_4_C12_C12_ ## x
+#define __LR8_EL2(x)			S3_4_C12_C13_ ## x
+
+#define ICH_LR0_EL2			__LR0_EL2(0)
+#define ICH_LR1_EL2			__LR0_EL2(1)
+#define ICH_LR2_EL2			__LR0_EL2(2)
+#define ICH_LR3_EL2			__LR0_EL2(3)
+#define ICH_LR4_EL2			__LR0_EL2(4)
+#define ICH_LR5_EL2			__LR0_EL2(5)
+#define ICH_LR6_EL2			__LR0_EL2(6)
+#define ICH_LR7_EL2			__LR0_EL2(7)
+#define ICH_LR8_EL2			__LR8_EL2(0)
+#define ICH_LR9_EL2			__LR8_EL2(1)
+#define ICH_LR10_EL2			__LR8_EL2(2)
+#define ICH_LR11_EL2			__LR8_EL2(3)
+#define ICH_LR12_EL2			__LR8_EL2(4)
+#define ICH_LR13_EL2			__LR8_EL2(5)
+#define ICH_LR14_EL2			__LR8_EL2(6)
+#define ICH_LR15_EL2			__LR8_EL2(7)
+
+#define __AP0Rx_EL2(x)			S3_4_C12_C8_ ## x
+#define ICH_AP0R0_EL2			__AP0Rx_EL2(0)
+#define ICH_AP0R1_EL2			__AP0Rx_EL2(1)
+#define ICH_AP0R2_EL2			__AP0Rx_EL2(2)
+#define ICH_AP0R3_EL2			__AP0Rx_EL2(3)
+
+#define __AP1Rx_EL2(x)			S3_4_C12_C9_ ## x
+#define ICH_AP1R0_EL2			__AP1Rx_EL2(0)
+#define ICH_AP1R1_EL2			__AP1Rx_EL2(1)
+#define ICH_AP1R2_EL2			__AP1Rx_EL2(2)
+#define ICH_AP1R3_EL2			__AP1Rx_EL2(3)
+
+#ifndef __ASSEMBLY__
+
+#include <linux/stringify.h>
+
+static inline void gic_write_eoir(u64 irq)
+{
+	asm volatile("msr " __stringify(ICC_EOIR1_EL1) ", %0" : : "r" (irq));
+	isb();
+}
+
+#endif
+
+#endif

Hi Marc,

Quoted 31 lines · Marc Zyngier

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Cc Thomas Gleixner <[email protected]>

Reviewed-by Zi Shen Lim <[email protected]>

Reviewed-by Christoffer Dall <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/Kconfig                 |   1 +
 arch/arm64/kernel/head.S           |  18 +
 arch/arm64/kernel/hyp-stub.S       |   1 +
 drivers/irqchip/Kconfig            |   5 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-gic-v3.c       | 684 +++++++++++++++++++++++++++++++++++++
 include/linux/irqchip/arm-gic-v3.h | 190 +++++++++++
 7 files changed, 900 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3.c
 create mode 100644 include/linux/irqchip/arm-gic-v3.h

[...]

Quoted 6 lines

drivers/irqchip/irq-gic-v3.c new file +0 -0

new file mode 100644
index 0000000..6570e7b
@@ -0,0 +1,684 @@

Quoted 15 lines

drivers/irqchip/irq-gic-v3.c new file +15 -0

+static int gic_set_type(struct irq_data *d, unsigned int type)
+{
+     unsigned int irq = gic_irq(d);
+     void (*rwp_wait)(void);
+     void __iomem *base;
+
+     if (gic_irq_in_rdist(d)) {
+             base = gic_data_rdist_sgi_base();
+             rwp_wait = gic_redist_wait_for_rwp;
+     } else {
+             base = gic_data.dist_base;
+             rwp_wait = gic_dist_wait_for_rwp;
+     }
+
+     /* Interrupt configuration for SGIs can't be changed */

drivers/irqchip/irq-gic-v3.c line 249

Quoted 5 lines

drivers/irqchip/irq-gic-v3.c new file +5 -0

+     if (irq < 16)
+             return -EINVAL;
+
+     if (type != IRQ_TYPE_LEVEL_HIGH && type != IRQ_TYPE_EDGE_RISING)
+             return -EINVAL;

It would be better if checking the two conditions right after variable declaration.

Quoted 22 lines

drivers/irqchip/irq-gic-v3.c new file +22 -0

+
+     gic_configure_irq(irq, type, base, rwp_wait);
+
+     return 0;
+}
+
+static u64 gic_mpidr_to_affinity(u64 mpidr)
+{
+     u64 aff;
+
+     aff = (MPIDR_AFFINITY_LEVEL(mpidr, 3) << 32 |
+            MPIDR_AFFINITY_LEVEL(mpidr, 2) << 16 |
+            MPIDR_AFFINITY_LEVEL(mpidr, 1) << 8  |
+            MPIDR_AFFINITY_LEVEL(mpidr, 0)) & ~GICD_IROUTER_SPI_MODE_ANY;
+
+     return aff;
+}
+
+static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
+{
+     u64 irqnr;
+

drivers/irqchip/irq-gic-v3.c line 275

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     do {
+             irqnr = gic_read_iar();
+
+             if (likely(irqnr > 15 && irqnr < 1021)) {

I prefer to use gic_data.irq_nr instead of '1021'.
The ranges here should be consistent with the ones in domain.map.

Quoted 19 lines

drivers/irqchip/irq-gic-v3.c new file +19 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;

drivers/irqchip/irq-gic-v3.c line 299

Quoted 5 lines

drivers/irqchip/irq-gic-v3.c new file +5 -0

+     u64 affinity;
+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Quoted 16 lines

drivers/irqchip/irq-gic-v3.c new file +16 -0

+
+     gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
+
+     /* Enable distributor with ARE, Group1 */
+     writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
+                    base + GICD_CTLR);
+
+     /*
+      * Set all global interrupts to the boot CPU only. ARE must be
+      * enabled.
+      */
+     affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
+     for (i = 32; i < gic_data.irq_nr; i++)
+             writeq_relaxed(affinity, base + GICD_IROUTER + i * 8);
+}
+

Quoted 14 lines

drivers/irqchip/irq-gic-v3.c new file +14 -0

+static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
+                           irq_hw_number_t hw)
+{
+     /* SGIs are private to the core kernel */
+     if (hw < 16)
+             return -EPERM;
+     /* PPIs */
+     if (hw < 32) {
+             irq_set_percpu_devid(irq);
+             irq_set_chip_and_handler(irq, &gic_chip,
+                                      handle_percpu_devid_irq);
+             set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
+     }
+     /* SPIs */

drivers/irqchip/irq-gic-v3.c line 555

Quoted 5 lines

drivers/irqchip/irq-gic-v3.c new file +5 -0

+     if (hw >= 32 && hw < gic_data.irq_nr) {
+             irq_set_chip_and_handler(irq, &gic_chip,
+                                      handle_fasteoi_irq);
+             set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+     }

Use 'else' here, since the conditions are non-overlapping.

Hi Abel,

Quoted 35 lines · Abel

Hi Marc,

Quoted 31 lines · Marc Zyngier

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Cc Thomas Gleixner <[email protected]>

Reviewed-by Zi Shen Lim <[email protected]>

Reviewed-by Christoffer Dall <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/Kconfig                 |   1 +
 arch/arm64/kernel/head.S           |  18 +
 arch/arm64/kernel/hyp-stub.S       |   1 +
 drivers/irqchip/Kconfig            |   5 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-gic-v3.c       | 684 +++++++++++++++++++++++++++++++++++++
 include/linux/irqchip/arm-gic-v3.h | 190 +++++++++++
 7 files changed, 900 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3.c
 create mode 100644 include/linux/irqchip/arm-gic-v3.h

Quoted 4 lines

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+static asmlinkage void __exception_irq_entry gic_handle_irq(struct pt_regs *regs)
+{
+     u64 irqnr;
+

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 275

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     do {
+             irqnr = gic_read_iar();
+
+             if (likely(irqnr > 15 && irqnr < 1021)) {

I prefer to use gic_data.irq_nr instead of '1021'.
The ranges here should be consistent with the ones in domain.map.

That's debatable. gic_data.irq_nr is used as a guard for the rest of the kernel, so that we don't map an out-of-range interrupt number. But here, we read the interrupt that has been generated by the actual HW. If such thing happen, we need to know about it, handle it as a spurious interrupt, and EOI it. Otherwise, we're stuck forever (with your proposed change, nothing EOIs the interrupt).

Quoted 20 lines

Quoted 20 lines

drivers/irqchip/irq-gic-v3.c new file +19 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 299

Quoted 5 lines

drivers/irqchip/irq-gic-v3.c new file +5 -0

+     u64 affinity;
+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Hmmm. That's a good point. The spec says the RWP tracks the completion of writes that change (among other things) the state of an interrupt group enable. But the next line will perform a wait_for_rwp anyway after having disabled all SPIs. So no, I don't think we need to wait. We guaranteed to get a consistent state after the call to gic_dist_config.

Quoted 17 lines

Quoted 17 lines

drivers/irqchip/irq-gic-v3.c new file +16 -0

+
+     gic_dist_config(base, gic_data.irq_nr, gic_dist_wait_for_rwp);
+
+     /* Enable distributor with ARE, Group1 */
+     writel_relaxed(GICD_CTLR_ARE_NS | GICD_CTLR_ENABLE_G1A | GICD_CTLR_ENABLE_G1,
+                    base + GICD_CTLR);
+
+     /*
+      * Set all global interrupts to the boot CPU only. ARE must be
+      * enabled.
+      */
+     affinity = gic_mpidr_to_affinity(cpu_logical_map(smp_processor_id()));
+     for (i = 32; i < gic_data.irq_nr; i++)
+             writeq_relaxed(affinity, base + GICD_IROUTER + i * 8);
+}
+

[...]

Quoted 14 lines

Quoted 14 lines

drivers/irqchip/irq-gic-v3.c new file +14 -0

+static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
+                           irq_hw_number_t hw)
+{
+     /* SGIs are private to the core kernel */
+     if (hw < 16)
+             return -EPERM;
+     /* PPIs */
+     if (hw < 32) {
+             irq_set_percpu_devid(irq);
+             irq_set_chip_and_handler(irq, &gic_chip,
+                                      handle_percpu_devid_irq);
+             set_irq_flags(irq, IRQF_VALID | IRQF_NOAUTOEN);
+     }
+     /* SPIs */

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 555

Quoted 5 lines

drivers/irqchip/irq-gic-v3.c new file +5 -0

+     if (hw >= 32 && hw < gic_data.irq_nr) {
+             irq_set_chip_and_handler(irq, &gic_chip,
+                                      handle_fasteoi_irq);
+             set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
+     }

Use 'else' here, since the conditions are non-overlapping.

For the code as it is now, I'd agree. But I'm also looking at the ITS code that will follow as soon as this one is on its way to mainline, and I then have this:

(quoted diff) +13 -0

@@ -553,6 +571,19 @@ static int gic_irq_domain_map(struct irq_domain *d, unsigned int irq,
                                         handle_fasteoi_irq);
                set_irq_flags(irq, IRQF_VALID | IRQF_PROBE);
        }
+       /* Nothing */
+       if (hw >= gic_data.irq_nr && hw < 8192)
+               return -EPERM;
+       /* LPIs */
+       if (hw >= 8192 && hw < GIC_ID_NR) {
+               if (!its_chip)
+                       return -EPERM;
+               irq_set_chip_and_handler(irq, its_chip, handle_fasteoi_irq);
+               set_irq_flags(irq, IRQF_VALID);
+       }
+       /* Off limits */
+       if (hw >= GIC_ID_NR)
+               return -EPERM;
        irq_set_chip_data(irq, d->host_data);
        return 0;
 }

Having a bunch of else clauses looks pretty awful in this context, and I've decided to try and keep it readable instead. This is not on the hot path anyway, so it doesn't really matter if we test an extra variable.

Quoted 39 lines · Marc Zyngier

Hi Abel,

Quoted 36 lines · Abel

Hi Marc,

Quoted 32 lines · Marc Zyngier

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Cc Thomas Gleixner <[email protected]>

Reviewed-by Zi Shen Lim <[email protected]>

Reviewed-by Christoffer Dall <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/Kconfig                 |   1 +
 arch/arm64/kernel/head.S           |  18 +
 arch/arm64/kernel/hyp-stub.S       |   1 +
 drivers/irqchip/Kconfig            |   5 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-gic-v3.c | 684
+++++++++++++++++++++++++++++++++++++
 include/linux/irqchip/arm-gic-v3.h | 190 +++++++++++
 7 files changed, 900 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3.c
 create mode 100644 include/linux/irqchip/arm-gic-v3.h

[...]

Quoted 14 lines

Quoted 5 lines

Quoted 5 lines

+static asmlinkage void __exception_irq_entry
gic_handle_irq(struct pt_regs *regs)

drivers/irqchip/irq-gic-v3.c new file +3 -0

+{
+     u64 irqnr;
+

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 275

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     do {
+             irqnr = gic_read_iar();
+
+             if (likely(irqnr > 15 && irqnr < 1021)) {

I prefer to use gic_data.irq_nr instead of '1021'.
The ranges here should be consistent with the ones in domain.map.

Quoted 6 lines

That's debatable. gic_data.irq_nr is used as a guard for the rest of the kernel, so that we don't map an out-of-range interrupt number. But here, we read the interrupt that has been generated by the actual HW. If such thing happen, we need to know about it, handle it as a spurious interrupt, and EOI it. Otherwise, we're stuck forever (with your proposed change, nothing EOIs the interrupt).

I don't think it's necessary to EOI the spurious interrupts, since EOIing a spurious interrupt won't cause any effect.

As for the interrupts here, the ones entered into this 'if' clause, I think they are expected already been mapped into this irq_domain.

Quoted 30 lines

Quoted 21 lines

Quoted 21 lines

drivers/irqchip/irq-gic-v3.c new file +20 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;
+     u64 affinity;

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 299

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Quoted 5 lines

Hmmm. That's a good point. The spec says the RWP tracks the completion of writes that change (among other things) the state of an interrupt group enable. But the next line will perform a wait_for_rwp anyway after having disabled all SPIs. So no, I don't think we need to wait. We guaranteed to get a consistent state after the call to gic_dist_config.

The spec also says that software must ensure the interrupt is disabled before changing its routing/priority/group information, see chapter 4.5.5 in v18.

Quoted 64 lines · Abel

Quoted 39 lines · Marc Zyngier

Hi Abel,

Quoted 36 lines · Abel

Hi Marc,

Quoted 32 lines · Marc Zyngier

The Generic Interrupt Controller (version 3) offers services that are similar to GICv2, with a number of additional features:
- Affinity routing based on the CPU MPIDR (ARE)
- System register for the CPU interfaces (SRE)
- Support for more that 8 CPUs
- Locality-specific Peripheral Interrupts (LPIs)
- Interrupt Translation Services (ITS)

This patch adds preliminary support for GICv3 with ARE and SRE, non-secure mode only. It relies on higher exception levels to grant ARE and SRE access.

Support for LPI and ITS will be added at a later time.

Cc Thomas Gleixner <[email protected]>

Reviewed-by Zi Shen Lim <[email protected]>

Reviewed-by Christoffer Dall <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 arch/arm64/Kconfig                 |   1 +
 arch/arm64/kernel/head.S           |  18 +
 arch/arm64/kernel/hyp-stub.S       |   1 +
 drivers/irqchip/Kconfig            |   5 +
 drivers/irqchip/Makefile           |   1 +
 drivers/irqchip/irq-gic-v3.c | 684
+++++++++++++++++++++++++++++++++++++
 include/linux/irqchip/arm-gic-v3.h | 190 +++++++++++
 7 files changed, 900 insertions(+)
 create mode 100644 drivers/irqchip/irq-gic-v3.c
 create mode 100644 include/linux/irqchip/arm-gic-v3.h

[...]

Quoted 20 lines

Quoted 5 lines

Quoted 5 lines

+static asmlinkage void __exception_irq_entry
gic_handle_irq(struct pt_regs *regs)

drivers/irqchip/irq-gic-v3.c new file +3 -0

+{
+     u64 irqnr;
+

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 275

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     do {
+             irqnr = gic_read_iar();
+
+             if (likely(irqnr > 15 && irqnr < 1021)) {

I prefer to use gic_data.irq_nr instead of '1021'.
The ranges here should be consistent with the ones in domain.map.

That's debatable. gic_data.irq_nr is used as a guard for the rest of the kernel, so that we don't map an out-of-range interrupt number. But here, we read the interrupt that has been generated by the actual HW. If such thing happen, we need to know about it, handle it as a spurious interrupt, and EOI it. Otherwise, we're stuck forever (with your proposed change, nothing EOIs the interrupt).

Quoted 2 lines

I don't think it's necessary to EOI the spurious interrupts, since EOIing a spurious interrupt won't cause any effect.

My choice of word was rather poor in this context. I wasn't thinking of a spurious interrupt", as defined in the GIC architecture (where ICC_IAR_EL1 returns 0x3ff), but rather of a valid interrupt number (between irq_nr and 1021).

Quoted 2 lines

As for the interrupts here, the ones entered into this 'if' clause, I think they are expected already been mapped into this irq_domain.

Mapped or not is not the problem. Think of a (broken) implementation that would generate random IDs in the irq_nr..1021 range. You'd end up with the deadlock situation I've outlined above.

And actually, irq_find_mapping can fail, and I should handle this.

Quoted 37 lines

Quoted 35 lines

Quoted 21 lines

Quoted 21 lines

drivers/irqchip/irq-gic-v3.c new file +20 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;
+     u64 affinity;

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 299

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Hmmm. That's a good point. The spec says the RWP tracks the completion of writes that change (among other things) the state of an interrupt group enable. But the next line will perform a wait_for_rwp anyway after having disabled all SPIs. So no, I don't think we need to wait. We guaranteed to get a consistent state after the call to gic_dist_config.

Quoted 2 lines

The spec also says that software must ensure the interrupt is disabled before changing its routing/priority/group information, see chapter 4.5.5 in v18.

That's only to ensure consistency when a single interrupt is being reconfigured, and that's what the code already does. In this particular context, waiting for RWP looks completely superfluous.

Quoted 8 lines · Marc Zyngier

Quoted 2 lines · Abel

As for the interrupts here, the ones entered into this 'if' clause, I think they are expected already been mapped into this irq_domain.

Mapped or not is not the problem. Think of a (broken) implementation that would generate random IDs in the irq_nr..1021 range. You'd end up with the deadlock situation I've outlined above.

Quoted 1 line

And actually, irq_find_mapping can fail, and I should handle this.

Yes, makes sense. This case should be properly handled.
And I still have two questions.
a) Does this scenario really exist or necessary? I mean, part of interrupts mapped while others which belongs to the same interrupt controller are not.

b) Why 1021?

Quoted 38 lines

Quoted 38 lines

Quoted 36 lines

Quoted 22 lines

Quoted 22 lines

drivers/irqchip/irq-gic-v3.c new file +20 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;
+     u64 affinity;

Quoted 8 lines

drivers/irqchip/irq-gic-v3.c line 299

Quoted 4 lines

drivers/irqchip/irq-gic-v3.c new file +4 -0

+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Hmmm. That's a good point. The spec says the RWP tracks the completion of writes that change (among other things) the state of an interrupt group enable. But the next line will perform a wait_for_rwp anyway after having disabled all SPIs. So no, I don't think we need to wait. We guaranteed to get a consistent state after the call to gic_dist_config.

Quoted 6 lines

Quoted 2 lines

The spec also says that software must ensure the interrupt is disabled before changing its routing/priority/group information, see chapter 4.5.5 in v18.

That's only to ensure consistency when a single interrupt is being reconfigured, and that's what the code already does. In this particular context, waiting for RWP looks completely superfluous.

It's reasonable to think GIC having been enabled by boot loader or something else in the boot procedure. And here not waiting for RWP may cause hardware implementation defined behavior, that is configuring an enabled interrupt. It's beyond the scope of the GIC architecture, and vendors also don't want to see this happen.

Quoted 13 lines · Abel

Quoted 9 lines · Marc Zyngier

Quoted 2 lines · Abel

As for the interrupts here, the ones entered into this 'if' clause, I think they are expected already been mapped into this irq_domain.

Mapped or not is not the problem. Think of a (broken) implementation that would generate random IDs in the irq_nr..1021 range. You'd end up with the deadlock situation I've outlined above.

And actually, irq_find_mapping can fail, and I should handle this.

Quoted 4 lines

Yes, makes sense. This case should be properly handled.
And I still have two questions.
a) Does this scenario really exist or necessary? I mean, part of interrupts mapped while others which belongs to the same interrupt controller are not.

Not that I know of. But consider this cheap defensive programming. It is actually cheaper to check against a constant boundary than do a load and check against a variable. Safer, faster.

Quoted 2 lines

b) Why 1021?

The maximum SPI ID is 1020.

Quoted 46 lines

Quoted 45 lines

Quoted 39 lines

Quoted 32 lines

Quoted 23 lines

Quoted 23 lines

(quoted diff) +20 -0

+                     irqnr = irq_find_mapping(gic_data.domain, irqnr);
+                     handle_IRQ(irqnr, regs);
+                     continue;
+             }
+             if (irqnr < 16) {
+                     gic_write_eoir(irqnr);
+#ifdef CONFIG_SMP
+                     handle_IPI(irqnr, regs);
+#else
+                     WARN_ONCE(true, "Unexpected SGI received!\n");
+#endif
+                     continue;
+             }
+     } while (irqnr != 0x3ff);
+}
+
+static void __init gic_dist_init(void)
+{
+     unsigned int i;
+     u64 affinity;

Quoted 8 lines

Review

Quoted 4 lines

(quoted diff) +4 -0

+     void __iomem *base = gic_data.dist_base;
+
+     /* Disable the distributor */
+     writel_relaxed(0, base + GICD_CTLR);

I'm not sure whether waiting for write pending here is necessary. What do you think?

Quoted 5 lines

Hmmm. That's a good point. The spec says the RWP tracks the completion of writes that change (among other things) the state of an interrupt group enable. But the next line will perform a wait_for_rwp anyway after having disabled all SPIs. So no, I don't think we need to wait. We guaranteed to get a consistent state after the call to gic_dist_config.

Quoted 2 lines

The spec also says that software must ensure the interrupt is disabled before changing its routing/priority/group information, see chapter 4.5.5 in v18.

That's only to ensure consistency when a single interrupt is being reconfigured, and that's what the code already does. In this particular context, waiting for RWP looks completely superfluous.

Quoted 5 lines

It's reasonable to think GIC having been enabled by boot loader or something else in the boot procedure. And here not waiting for RWP may cause hardware implementation defined behavior, that is configuring an enabled interrupt. It's beyond the scope of the GIC architecture, and vendors also don't want to see this happen.

Well, I guess that since this is a one-off thing, it doesn't really add an extra synchronization. As long as it gives you peace of mind... ;-)

Add the necessary documentation to support GICv3.

Cc Thomas Gleixner <[email protected]>

Cc Mark Rutland <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Acked-by Rob Herring <[email protected]>

Acked-by Christoffer Dall <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 Documentation/devicetree/bindings/arm/gic-v3.txt | 79 ++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/gic-v3.txt

Documentation/devicetree/bindings/arm/gic-v3.txt new file +29 -0

new file mode 100644
index 0000000..33cd05e
@@ -0,0 +1,79 @@
+* ARM Generic Interrupt Controller, version 3
+
+AArch64 SMP cores are often associated with a GICv3, providing Private
+Peripheral Interrupts (PPI), Shared Peripheral Interrupts (SPI),
+Software Generated Interrupts (SGI), and Locality-specific Peripheral
+Interrupts (LPI).
+
+Main node required properties:
+
+- compatible : should at least contain  "arm,gic-v3".
+- interrupt-controller : Identifies the node as an interrupt controller
+- #interrupt-cells : Specifies the number of cells needed to encode an
+  interrupt source. Must be a single cell with a value of at least 3.
+
+  The 1st cell is the interrupt type; 0 for SPI interrupts, 1 for PPI
+  interrupts. Other values are reserved for future use.
+
+  The 2nd cell contains the interrupt number for the interrupt type.
+  SPI interrupts are in the range [0-987]. PPI interrupts are in the
+  range [0-15].
+
+  The 3rd cell is the flags, encoded as follows:
+     bits[3:0] trigger type and level flags.
+             1 = edge triggered
+             4 = level triggered
+
+  Cells 4 and beyond are reserved for future use. When the 1st cell
+  has a value of 0 or 1, cells 4 and beyond act as padding, and may be
+  ignored. It is recommended that padding cells have a value of 0.

Documentation/devicetree/bindings/arm/gic-v3.txt new file +30 -0

+
+- interrupts : Interrupt source of the VGIC maintenance interrupt.
+
+Optional
+
+- redistributor-stride : If using padding pages, specifies the stride
+  of consecutive redistributors. Must be a multiple of 64kB.
+
+- #redistributor-regions: The number of independent contiguous regions
+  occupied by the redistributors. Required if more than one such
+  region is present.
+
+Examples:
+
+     gic: interrupt-controller at 2cf00000 {
+             compatible = "arm,gic-v3";
+             #interrupt-cells = <3>;
+             interrupt-controller;
+             reg = <0x0 0x2f000000 0 0x10000>,       // GICD
+                   <0x0 0x2f100000 0 0x200000>,      // GICR
+                   <0x0 0x2c000000 0 0x2000>,        // GICC
+                   <0x0 0x2c010000 0 0x2000>,        // GICH
+                   <0x0 0x2c020000 0 0x2000>;        // GICV
+             interrupts = <1 9 4>;
+     };
+
+     gic: interrupt-controller at 2c010000 {
+             compatible = "arm,gic-v3";
+             #interrupt-cells = <3>;
+             interrupt-controller;

Quoted 55 lines · Marc Zyngier

Add the necessary documentation to support GICv3.

Cc Thomas Gleixner <[email protected]>

Cc Mark Rutland <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Acked-by Rob Herring <[email protected]>

Acked-by Christoffer Dall <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 Documentation/devicetree/bindings/arm/gic-v3.txt | 79
++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/gic-v3.txt

Documentation/devicetree/bindings/arm/gic-v3.txt new file +34 -0

new file mode 100644
index 0000000..33cd05e
@@ -0,0 +1,79 @@
+* ARM Generic Interrupt Controller, version 3
+
+AArch64 SMP cores are often associated with a GICv3, providing Private
+Peripheral Interrupts (PPI), Shared Peripheral Interrupts (SPI),
+Software Generated Interrupts (SGI), and Locality-specific Peripheral
+Interrupts (LPI).
+
+Main node required properties:
+
+- compatible : should at least contain  "arm,gic-v3".
+- interrupt-controller : Identifies the node as an interrupt controller
+- #interrupt-cells : Specifies the number of cells needed to encode an
+  interrupt source. Must be a single cell with a value of at least 3.
+
+  The 1st cell is the interrupt type; 0 for SPI interrupts, 1 for PPI
+  interrupts. Other values are reserved for future use.
+
+  The 2nd cell contains the interrupt number for the interrupt type.
+  SPI interrupts are in the range [0-987]. PPI interrupts are in the
+  range [0-15].
+
+  The 3rd cell is the flags, encoded as follows:
+	bits[3:0] trigger type and level flags.
+		1 = edge triggered
+		4 = level triggered
+
+  Cells 4 and beyond are reserved for future use. When the 1st cell
+  has a value of 0 or 1, cells 4 and beyond act as padding, and may be
+  ignored. It is recommended that padding cells have a value of 0.
+
+- reg : Specifies base physical address(s) and size of the GIC
+  registers, in the following order:
+  - GIC Distributor interface (GICD)
+  - GIC Redistributors (GICR), one range per redistributor region

Documentation/devicetree/bindings/arm/gic-v3.txt line 39

Quoted 5 lines

Documentation/devicetree/bindings/arm/gic-v3.txt new file +5 -0

+  - GIC CPU interface (GICC)
+  - GIC Hypervisor interface (GICH)
+  - GIC Virtual CPU interface (GICV)
+
+  GICC, GICH and GICV are optional.

I wonder if it should be noted here that a lack of these last three regions denies GICv2 emulation support for KVM guests.

It's probably not worth to introduce a special property for this v2 compat support, but if the lack of these registers is indicating that fact, it should be added here.

Quoted 56 lines · Marc Zyngier

Quoted 55 lines

Add the necessary documentation to support GICv3.

Cc Thomas Gleixner <[email protected]>

Cc Mark Rutland <[email protected]>

Acked-by Catalin Marinas <[email protected]>

Acked-by Rob Herring <[email protected]>

Acked-by Christoffer Dall <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

---
 Documentation/devicetree/bindings/arm/gic-v3.txt | 79
++++++++++++++++++++++++
 1 file changed, 79 insertions(+)
 create mode 100644 Documentation/devicetree/bindings/arm/gic-v3.txt

Documentation/devicetree/bindings/arm/gic-v3.txt new file +34 -0

new file mode 100644
index 0000000..33cd05e
@@ -0,0 +1,79 @@
+* ARM Generic Interrupt Controller, version 3
+
+AArch64 SMP cores are often associated with a GICv3, providing Private
+Peripheral Interrupts (PPI), Shared Peripheral Interrupts (SPI),
+Software Generated Interrupts (SGI), and Locality-specific Peripheral
+Interrupts (LPI).
+
+Main node required properties:
+
+- compatible : should at least contain  "arm,gic-v3".
+- interrupt-controller : Identifies the node as an interrupt controller
+- #interrupt-cells : Specifies the number of cells needed to encode an
+  interrupt source. Must be a single cell with a value of at least 3.
+
+  The 1st cell is the interrupt type; 0 for SPI interrupts, 1 for PPI
+  interrupts. Other values are reserved for future use.
+
+  The 2nd cell contains the interrupt number for the interrupt type.
+  SPI interrupts are in the range [0-987]. PPI interrupts are in the
+  range [0-15].
+
+  The 3rd cell is the flags, encoded as follows:
+   bits[3:0] trigger type and level flags.
+           1 = edge triggered
+           4 = level triggered
+
+  Cells 4 and beyond are reserved for future use. When the 1st cell
+  has a value of 0 or 1, cells 4 and beyond act as padding, and may be
+  ignored. It is recommended that padding cells have a value of 0.
+
+- reg : Specifies base physical address(s) and size of the GIC
+  registers, in the following order:
+  - GIC Distributor interface (GICD)
+  - GIC Redistributors (GICR), one range per redistributor region

Quoted 8 lines

Documentation/devicetree/bindings/arm/gic-v3.txt line 39

Quoted 5 lines

Documentation/devicetree/bindings/arm/gic-v3.txt new file +5 -0

+  - GIC CPU interface (GICC)
+  - GIC Hypervisor interface (GICH)
+  - GIC Virtual CPU interface (GICV)
+
+  GICC, GICH and GICV are optional.

I wonder if it should be noted here that a lack of these last three regions denies GICv2 emulation support for KVM guests.

Only the lack of the last one prevents KVM from emulating a GICv2. But that's a software support issue, and does not impact the hardware description (which is what we're doing in this patch).

KVM can still use software emulation of GICv2 (just like we did back in the days...).

Quoted 3 lines

It's probably not worth to introduce a special property for this v2 compat support, but if the lack of these registers is indicating that fact, it should be added here.

Absence of these ranges just indicates that the OS must be GICv3 aware to boot (i.e can't be GICv2 only).
I don't think that how this
information is used for guests is relevant to the DT documentation.

Thanks,

      M.

In order to split the various register manipulation from the main vgic code, introduce a vgic_ops structure, and start by abstracting the LR manipulation code with a couple of accessors.

Signed-off-by Marc Zyngier <[email protected]>

Move the GICH_ELRSR access to its own functions, and add them to the vgic_ops structure.

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>

Instead of directly messing with with the GICH_VMCR bits for the CPU interface save/restore code, add accessors that encode/decode the entire set of registers exposed by VMCR.

Not the most efficient thing, but given that this code is only used by the save/restore code, performance is far from being critical.

Signed-off-by Marc Zyngier <[email protected]>

So far, irqchip_in_kernel() was implemented by testing the value of vctrl_base, which worked fine with GICv2.

With GICv3, this field is useless, as we're using system registers instead of a emmory mapped interface. To solve this, add a boolean flag indicating if the we're using a vgic or not.

Signed-off-by Marc Zyngier <[email protected]>

Move the GICv2 world switch code into its own file, and add the necessary indirection to the arm64 switch code.

Also introduce a new type field to the vgic_params structure.

Acked-by Catalin Marinas <[email protected]>

Signed-off-by Marc Zyngier <[email protected]>