arm/arm64: gicv3: support two redistributor regions

PATCH Superseded Andrew Jones 2 participants 3 messages Sep 3, 2018 – Sep 4, 2018 KVM arm64

Share & tools

Andrew Jones

author

2018-09-03 16:45 UTC #0 lore raw

PATCH 1/1 arm/arm64: gicv3: support two redistributor regions

We need to support two redistributor regions in order to support more than 123 vcpus. Also bump NR_CPUS to 512, since that's what KVM currently supports.

Signed-off-by Andrew Jones <[email protected]>

---

So far only regression tested on TCG.

 lib/arm/asm/gic-v3.h | 20 +++++++++++++++++---
 lib/arm/asm/setup.h  |  2 +-
 lib/arm/gic-v3.c     | 39 +++++++++++++++++++++++++++++----------
 lib/arm/gic.c        | 16 +++++++++++++---
 4 files changed, 60 insertions(+), 17 deletions(-)

lib/arm/asm/gic-v3.h +17 -3

index 1dceb9541f62..a04e58bd438b 100644
@@ -49,16 +49,30 @@
 #include <asm/smp.h>
 #include <asm/io.h>
 
+#define GICV3_REDIST0_NR_CPUS 123
+
 struct gicv3_data {
 	void *dist_base;
-	void *redist_base[NR_CPUS];
+	void *redist_base[GICV3_REDIST0_NR_CPUS];
+	void *redist2_base[NR_CPUS - GICV3_REDIST0_NR_CPUS];
 	unsigned int irq_nr;
 };
 extern struct gicv3_data gicv3_data;
 
+#define gicv3_redist_base()								\
+({											\
+	int __cpu = smp_processor_id();							\
+	void *__base;									\
+											\
+	if (__cpu < GICV3_REDIST0_NR_CPUS)						\
+		__base = gicv3_data.redist_base[__cpu];					\
+	else										\
+		__base = gicv3_data.redist2_base[__cpu - GICV3_REDIST0_NR_CPUS];	\
+											\
+	__base;										\
+})
 #define gicv3_dist_base()		(gicv3_data.dist_base)
-#define gicv3_redist_base()		(gicv3_data.redist_base[smp_processor_id()])
-#define gicv3_sgi_base()		(gicv3_data.redist_base[smp_processor_id()] + SZ_64K)
+#define gicv3_sgi_base()		(gicv3_redist_base() + SZ_64K)
 
 extern int gicv3_init(void);
 extern void gicv3_enable_defaults(void);

lib/arm/asm/setup.h +1 -1

index b57ea13b9dd2..3215814603e4 100644
@@ -9,7 +9,7 @@
 #include <asm/page.h>
 #include <asm/pgtable-hwdef.h>
 
-#define NR_CPUS			255
+#define NR_CPUS			512
 extern u64 cpus[NR_CPUS];	/* per-cpu IDs (MPIDRs) */
 extern int nr_cpus;

lib/arm/gic-v3.c +29 ~~-10~~

index 9b3fa5730478..e7a48c988cab 100644
@@ -6,20 +6,39 @@
 #include <asm/gic.h>
 #include <asm/io.h>
 
+static void __gicv3_set_redist_base(void *ptr)
+{
+	int cpu = smp_processor_id();
+
+	if (cpu < GICV3_REDIST0_NR_CPUS)
+		gicv3_data.redist_base[cpu] = ptr;
+	else
+		gicv3_data.redist2_base[cpu - GICV3_REDIST0_NR_CPUS] = ptr;
+}
+
 void gicv3_set_redist_base(size_t stride)
 {
 	u32 aff = mpidr_compress(get_mpidr());
-	void *ptr = gicv3_data.redist_base[0];
+	void *redist_base[] =  {
+		gicv3_data.redist_base[0],
+		gicv3_data.redist2_base[0],
+	};
+	void *ptr;
 	u64 typer;
-
-	do {
-		typer = gicv3_read_typer(ptr + GICR_TYPER);
-		if ((typer >> 32) == aff) {
-			gicv3_redist_base() = ptr;
-			return;
-		}
-		ptr += stride; /* skip RD_base, SGI_base, etc. */
-	} while (!(typer & GICR_TYPER_LAST));
+	int i = 0;
+
+	while (redist_base[i]) {
+		ptr = redist_base[i];
+		do {
+			typer = gicv3_read_typer(ptr + GICR_TYPER);
+			if ((typer >> 32) == aff) {
+				__gicv3_set_redist_base(ptr);
+				return;
+			}
+			ptr += stride; /* skip RD_base, SGI_base, etc. */
+		} while (!(typer & GICR_TYPER_LAST));
+		++i;
+	}
 
 	/* should never reach here */
 	assert(0);

lib/arm/gic.c +13 -3

index 59273b1716d6..81973c47d487 100644
@@ -44,7 +44,7 @@ static const struct gic_common_ops gicv3_common_ops = {
  * Documentation/devicetree/bindings/interrupt-controller/arm,gic-v3.txt
  */
 static bool
-gic_get_dt_bases(const char *compatible, void **base1, void **base2)
+gic_get_dt_bases(const char *compatible, void **base1, void **base2, void **base3)
 {
 	struct dt_pbus_reg reg;
 	struct dt_device gic;

@@ -70,19 +70,29 @@ gic_get_dt_bases(const char *compatible, void **base1, void **base2)
 	assert(ret == 0);
 	*base2 = ioremap(reg.addr, reg.size);
 
+	if (base3) {
+		ret = dt_pbus_translate(&gic, 2, &reg);
+		if (ret != -FDT_ERR_NOTFOUND) {
+			assert(ret == 0);
+			*base3 = ioremap(reg.addr, reg.size);
+		} else {
+			*base3 = NULL;
+		}
+	}
+
 	return true;
 }
 
 int gicv2_init(void)
 {
 	return gic_get_dt_bases("arm,cortex-a15-gic",
-			&gicv2_data.dist_base, &gicv2_data.cpu_base);
+			&gicv2_data.dist_base, &gicv2_data.cpu_base, NULL);
 }
 
 int gicv3_init(void)
 {
 	return gic_get_dt_bases("arm,gic-v3", &gicv3_data.dist_base,
-			&gicv3_data.redist_base[0]);
+			&gicv3_data.redist_base[0], &gicv3_data.redist2_base[0]);
 }
 
 int gic_version(void)

Signature · 1 line

2.17.1

Peter Maydell

2018-09-04 00:49 UTC #1 lore raw

Quoted 3 lines · Andrew Jones

We need to support two redistributor regions in order to support more than 123 vcpus. Also bump NR_CPUS to 512, since that's what KVM currently supports.

Wouldn't it be neater to support the arbitrary number of redistributor regions that the h/w spec allows and the dt can describe, rather than just the specific case of 2 regions that QEMU happens to use ?

thanks
-- PMM

Andrew Jones

author

2018-09-04 07:02 UTC #2 lore raw

Quoted 10 lines · Peter Maydell

Quoted 3 lines · Andrew Jones
We need to support two redistributor regions in order to support more than 123 vcpus. Also bump NR_CPUS to 512, since that's what KVM currently supports.
Wouldn't it be neater to support the arbitrary number of redistributor regions that the h/w spec allows and the dt can describe, rather than just the specific case of 2 regions that QEMU happens to use ?

It would be nicer, and I suppose it would be pretty easy to do as well. kvm-unit-tests/arm has always been a balance between being over-engineered for its purposes and being too hacky. In this case, I was probably too hacky. Although KVM only supports up to 512 VCPUS right now and these two regions support up to 635, so it'll be a while before a third region will be needed.

If I find time I'll send a v2, but in the meantime it'd be good to get test results with this version to be sure I didn't miss anything else.

Thanks,
drew