Skip to contentSkip to conversation

perf: Arm SMMU PMU driver

PATCH In review Robin Murphy 3 participants 10 messages Feb 17, 2022 – Mar 7, 2022

Subsystems (7)
lore
Share & tools
lore

Robin Murphy

author reviewer
#0 lore raw

cover perf: Arm SMMU PMU driver

Hi all,

Since our friends at Linaro had some interest in the area, I've dug up and finished off my old SMMUv1/v2 PMU driver, now with a slightly less terrible DT binding to boot. Patch #1 involves a general refactoring of the IRQ setup code in arm-smmu which also now serves to remove the general dependency on explicit IRQ resources for of_platform devices.

ACPI support via IORT (patch #2) is pleasingly trivial, but does have a hard dependency on the previous patch, otherwise SMMU context interrupts may get messed up.

The driver itself in patch #5 is still largely a time capsule of me learning the PMU APIs 5 years ago (as a warm-up for arm-cmn). I've mostly just added IRQ affinity support (which I didn't yet fully understand at the time) and tweaked a few comments. I'm pretty confident all the testing I did back then is still valid, so I've just run some quick sanity checks with patch #6 to verify the new DT binding.

Cheers,
Robin.

Robin Murphy (6):
iommu/arm-smmu: Account for PMU interrupts
acpi/iort: Register SMMUv2 PMU interrupts
iommu/arm-smmu: Add DT PMU support
iommu/smmu: Create child devices for PMUs
perf: Add ARM SMMU PMU driver
arm64: dts: Add SMMU PMUs to Juno

 .../devicetree/bindings/iommu/arm,smmu.yaml   |  19 +-
 arch/arm64/boot/dts/arm/juno-base.dtsi        |  26 +-
 drivers/acpi/arm64/iort.c                     |  18 +-
 drivers/iommu/arm/arm-smmu/arm-smmu.c         | 143 ++--
 drivers/iommu/arm/arm-smmu/arm-smmu.h         |   5 +-
 drivers/perf/Kconfig                          |  24 +-
 drivers/perf/Makefile                         |   1 +
 drivers/perf/arm-smmu-pmu.c                   | 732 ++++++++++++++++++
 8 files changed, 896 insertions(+), 72 deletions(-)
 create mode 100644 drivers/perf/arm-smmu-pmu.c
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
author reviewer
#1 lore raw

PATCH 1/6 iommu/arm-smmu: Account for PMU interrupts

In preparation for SMMUv2 PMU support, rejig our IRQ setup code to account for PMU interrupts as additional resources. We can simplify the whole flow by only storing the context IRQs, since the global IRQs are devres-managed and we never refer to them beyond the initial request.

---
 drivers/iommu/arm/arm-smmu/arm-smmu.c | 93 +++++++++++++--------------
 drivers/iommu/arm/arm-smmu/arm-smmu.h |  5 +-
 2 files changed, 46 insertions(+), 52 deletions(-)
drivers/iommu/arm/arm-smmu/arm-smmu.c +44 -49
index 4bc75c4ce402..e50fcf37af58 100644
@@ -807,7 +807,7 @@ static int arm_smmu_init_domain_context(struct iommu_domain *domain,
 	 * Request context fault interrupt. Do this last to avoid the
 	 * handler seeing a half-initialised domain state.
 	 */
-	irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
+	irq = smmu->irqs[cfg->irptndx];
 
 	if (smmu->impl && smmu->impl->context_fault)
 		context_fault = smmu->impl->context_fault;
@@ -858,7 +858,7 @@ static void arm_smmu_destroy_domain_context(struct iommu_domain *domain)
 	arm_smmu_write_context_bank(smmu, cfg->cbndx);
 
 	if (cfg->irptndx != ARM_SMMU_INVALID_IRPTNDX) {
-		irq = smmu->irqs[smmu->num_global_irqs + cfg->irptndx];
+		irq = smmu->irqs[cfg->irptndx];
 		devm_free_irq(smmu->dev, irq, domain);
 	}
 
@@ -1951,8 +1951,8 @@ static int acpi_smmu_get_data(u32 model, struct arm_smmu_device *smmu)
 	return ret;
 }
 
-static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
-				      struct arm_smmu_device *smmu)
+static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
+				      u32 *global_irqs, u32 *pmu_irqs)
 {
 	struct device *dev = smmu->dev;
 	struct acpi_iort_node *node =
@@ -1968,7 +1968,8 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
 		return ret;
 
 	/* Ignore the configuration access interrupt */
-	smmu->num_global_irqs = 1;
+	*global_irqs = 1;
+	*pmu_irqs = 0;
 
 	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
@@ -1976,25 +1977,24 @@ static int arm_smmu_device_acpi_probe(struct platform_device *pdev,
 	return 0;
 }
 #else
-static inline int arm_smmu_device_acpi_probe(struct platform_device *pdev,
-					     struct arm_smmu_device *smmu)
+static inline int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
+					     u32 *global_irqs, u32 *pmu_irqs)
 {
 	return -ENODEV;
 }
 #endif
 
-static int arm_smmu_device_dt_probe(struct platform_device *pdev,
-				    struct arm_smmu_device *smmu)
+static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
+				    u32 *global_irqs, u32 *pmu_irqs)
 {
 	const struct arm_smmu_match_data *data;
-	struct device *dev = &pdev->dev;
+	struct device *dev = smmu->dev;
 	bool legacy_binding;
 
-	if (of_property_read_u32(dev->of_node, "#global-interrupts",
-				 &smmu->num_global_irqs)) {
-		dev_err(dev, "missing #global-interrupts property\n");
-		return -ENODEV;
-	}
+	if (of_property_read_u32(dev->of_node, "#global-interrupts", global_irqs))
+		return dev_err_probe(dev, -ENODEV,
+				     "missing #global-interrupts property\n");
+	*pmu_irqs = 0;
 
 	data = of_device_get_match_data(dev);
 	smmu->version = data->version;
@@ -2073,6 +2073,7 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	struct arm_smmu_device *smmu;
 	struct device *dev = &pdev->dev;
 	int num_irqs, i, err;
+	u32 global_irqs, pmu_irqs;
 	irqreturn_t (*global_fault)(int irq, void *dev);
 
 	smmu = devm_kzalloc(dev, sizeof(*smmu), GFP_KERNEL);
@@ -2083,10 +2084,9 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	smmu->dev = dev;
 
 	if (dev->of_node)
-		err = arm_smmu_device_dt_probe(pdev, smmu);
+		err = arm_smmu_device_dt_probe(smmu, &global_irqs, &pmu_irqs);
 	else
-		err = arm_smmu_device_acpi_probe(pdev, smmu);
-
+		err = arm_smmu_device_acpi_probe(smmu, &global_irqs, &pmu_irqs);
 	if (err)
 		return err;
 
@@ -2105,31 +2105,25 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	if (IS_ERR(smmu))
 		return PTR_ERR(smmu);
 
-	num_irqs = 0;
-	while ((res = platform_get_resource(pdev, IORESOURCE_IRQ, num_irqs))) {
-		num_irqs++;
-		if (num_irqs > smmu->num_global_irqs)
-			smmu->num_context_irqs++;
-	}
+	num_irqs = platform_irq_count(pdev);
 
-	if (!smmu->num_context_irqs) {
-		dev_err(dev, "found %d interrupts but expected at least %d\n",
-			num_irqs, smmu->num_global_irqs + 1);
-		return -ENODEV;
-	}
+	smmu->num_context_irqs = num_irqs - global_irqs - pmu_irqs;
+	if (smmu->num_context_irqs <= 0)
+		return dev_err_probe(dev, -ENODEV,
+				"found %d interrupts but expected at least %d\n",
+				num_irqs, global_irqs + pmu_irqs + 1);
 
-	smmu->irqs = devm_kcalloc(dev, num_irqs, sizeof(*smmu->irqs),
-				  GFP_KERNEL);
-	if (!smmu->irqs) {
-		dev_err(dev, "failed to allocate %d irqs\n", num_irqs);
-		return -ENOMEM;
-	}
+	smmu->irqs = devm_kcalloc(dev, smmu->num_context_irqs,
+				  sizeof(*smmu->irqs), GFP_KERNEL);
+	if (!smmu->irqs)
+		return dev_err_probe(dev, -ENOMEM, "failed to allocate %d irqs\n",
+				     smmu->num_context_irqs);
 
-	for (i = 0; i < num_irqs; ++i) {
-		int irq = platform_get_irq(pdev, i);
+	for (i = 0; i < smmu->num_context_irqs; i++) {
+		int irq = platform_get_irq(pdev, global_irqs + pmu_irqs + i);
 
 		if (irq < 0)
-			return -ENODEV;
+			return irq;
 		smmu->irqs[i] = irq;
 	}
 
@@ -2165,17 +2159,18 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 	else
 		global_fault = arm_smmu_global_fault;
 
-	for (i = 0; i < smmu->num_global_irqs; ++i) {
-		err = devm_request_irq(smmu->dev, smmu->irqs[i],
-				       global_fault,
-				       IRQF_SHARED,
-				       "arm-smmu global fault",
-				       smmu);
-		if (err) {
-			dev_err(dev, "failed to request global IRQ %d (%u)\n",
-				i, smmu->irqs[i]);
-			return err;
-		}
+	for (i = 0; i < global_irqs; i++) {
+		int irq = platform_get_irq(pdev, i);
+
+		if (irq < 0)
+			return irq;
+
+		err = devm_request_irq(dev, irq, global_fault, IRQF_SHARED,
+				       "arm-smmu global fault", smmu);
+		if (err)
+			return dev_err_probe(dev, err,
+					"failed to request global IRQ %d (%u)\n",
+					i, irq);
 	}
 
 	err = iommu_device_sysfs_add(&smmu->iommu, smmu->dev, NULL,
drivers/iommu/arm/arm-smmu/arm-smmu.h +2 -3
index 432de2f742c3..2b9b42fb6f30 100644
@@ -318,11 +318,10 @@ struct arm_smmu_device {
 	unsigned long			pa_size;
 	unsigned long			pgsize_bitmap;
 
-	u32				num_global_irqs;
-	u32				num_context_irqs;
+	int				num_context_irqs;
+	int				num_clks;
 	unsigned int			*irqs;
 	struct clk_bulk_data		*clks;
-	int				num_clks;
 
 	spinlock_t			global_sync_lock;
 
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
author reviewer
#2 lore raw

PATCH 2/6 acpi/iort: Register SMMUv2 PMU interrupts

In reply to Robin Murphy

In preparation for SMMUv2 PMU support, make sure the PMU IRQs are parsed out of IORT and registered somewhere the SMMU driver can find them. Without making massively invasive changes there aren't many ways to achieve this; inserting them into the SMMU's resource list between the global and context IRQs is easy enough to cope with in the driver, and offers the path of least resistance for the DT binding too.

---
 drivers/acpi/arm64/iort.c             | 18 ++++++++++++++----
 drivers/iommu/arm/arm-smmu/arm-smmu.c |  2 +-
 2 files changed, 15 insertions(+), 5 deletions(-)
drivers/acpi/arm64/iort.c +14 -4
index 3b23fb775ac4..175397913be1 100644
@@ -1277,10 +1277,10 @@ static int __init arm_smmu_count_resources(struct acpi_iort_node *node)
 	 * configuration access interrupt.
 	 *
 	 * MMIO address and global fault interrupt resources are always
-	 * present so add them to the context interrupt count as a static
-	 * value.
+	 * present so add them to the context + PMU interrupt count as a
+	 * static value.
 	 */
-	return smmu->context_interrupt_count + 2;
+	return smmu->pmu_interrupt_count + smmu->context_interrupt_count + 2;
 }
 
 static void __init arm_smmu_init_resources(struct resource *res,
@@ -1288,7 +1288,7 @@ static void __init arm_smmu_init_resources(struct resource *res,
 {
 	struct acpi_iort_smmu *smmu;
 	int i, hw_irq, trigger, num_res = 0;
-	u64 *ctx_irq, *glb_irq;
+	u64 *ctx_irq, *glb_irq, *pmu_irq;
 
 	/* Retrieve SMMU specific data */
 	smmu = (struct acpi_iort_smmu *)node->node_data;
@@ -1306,6 +1306,16 @@ static void __init arm_smmu_init_resources(struct resource *res,
 	acpi_iort_register_irq(hw_irq, "arm-smmu-global", trigger,
 				     &res[num_res++]);
 
+	/* PMU IRQs */
+	pmu_irq = ACPI_ADD_PTR(u64, node, smmu->pmu_interrupt_offset);
+	for (i = 0; i < smmu->pmu_interrupt_count; i++) {
+		hw_irq = IORT_IRQ_MASK(pmu_irq[i]);
+		trigger = IORT_IRQ_TRIGGER_MASK(pmu_irq[i]);
+
+		acpi_iort_register_irq(hw_irq, "arm-smmu-pmu", trigger,
+				       &res[num_res++]);
+	}
+
 	/* Context IRQs */
 	ctx_irq = ACPI_ADD_PTR(u64, node, smmu->context_interrupt_offset);
 	for (i = 0; i < smmu->context_interrupt_count; i++) {
drivers/iommu/arm/arm-smmu/arm-smmu.c +1 -1
index e50fcf37af58..cbfe4cc914f0 100644
@@ -1969,7 +1969,7 @@ static int arm_smmu_device_acpi_probe(struct arm_smmu_device *smmu,
 
 	/* Ignore the configuration access interrupt */
 	*global_irqs = 1;
-	*pmu_irqs = 0;
+	*pmu_irqs = iort_smmu->pmu_interrupt_count;
 
 	if (iort_smmu->flags & ACPI_IORT_SMMU_COHERENT_WALK)
 		smmu->features |= ARM_SMMU_FEAT_COHERENT_WALK;
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
author reviewer
#3 lore raw

PATCH 3/6 iommu/arm-smmu: Add DT PMU support

In reply to Robin Murphy

Since IORT describes PMU interrupts rather inflexibly as an inherent part of the SMMU, the best way to avoid excessive complexity is to make the way we handle DT look as similar as possible. Fortunately the de-facto standard for mentioning PMU interrupts at all under the current binding has been to include them in the global interrupt count, listing them after the actual fault interrupt(s), so we can capitalise on that. It's about 9 years too late to redefine "#global-interrupts" to exclude anything other than context interrupts without breaking compatibility, so we're stuck with a slightly convoluted definition of PMU interrupts as an optional subdivision of the "global" interrupts, but it works.

---

Not sure whether the count-backwards-from-the-middle nature of "number of PMU interrupts" is too ugly and "index of first PMU interrupt" might be any better.

---
 .../devicetree/bindings/iommu/arm,smmu.yaml   | 19 ++++++++++++++++++-
 drivers/iommu/arm/arm-smmu/arm-smmu.c         |  4 ++++
 2 files changed, 22 insertions(+), 1 deletion(-)
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +18 -1
index da5381c8ee11..9d39df42528a 100644
@@ -87,10 +87,18 @@ properties:
 
   '#global-interrupts':
     description: The number of global interrupts exposed by the device.
+      Includes the count of PMU interrupts, if present.
     $ref: /schemas/types.yaml#/definitions/uint32
     minimum: 0
     maximum: 260   # 2 secure, 2 non-secure, and up to 256 perf counters
 
+  '#pmu-interrupts':
+    description: The number of PMU interrupts. Must be equal to the number of
+      implemented counter groups.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 256
+
   '#iommu-cells':
     enum: [ 1, 2 ]
     description: |
@@ -115,6 +123,10 @@ properties:
       context bank. In the case of a single, combined interrupt, it must be
       listed multiple times.
 
+      If a PMU is present, the global interrupt entries consist of any fault
+      interrupts first, followed by #pmu-interrupts entries, one per implemented
+      counter group, specified in order of their indexing by the SMMU.
+
   dma-coherent:
     description: |
       Present if page table walks made by the SMMU are cache coherent with the
@@ -190,9 +202,14 @@ examples:
     smmu1: iommu@ba5e0000 {
             compatible = "arm,smmu-v1";
             reg = <0xba5e0000 0x10000>;
-            #global-interrupts = <2>;
+            #global-interrupts = <6>; /* 2 fault + 4 PMU */
+            #pmu-interrupts = <4>;
             interrupts = <0 32 4>,
                          <0 33 4>,
+                         <0 94 4>, /* This is the first PMU interrupt */
+                         <0 95 4>,
+                         <0 96 4>,
+                         <0 97 4>,
                          <0 34 4>, /* This is the first context interrupt */
                          <0 35 4>,
                          <0 36 4>,
drivers/iommu/arm/arm-smmu/arm-smmu.c +4 -0
index cbfe4cc914f0..8d6c8106fc1d 100644
@@ -1995,6 +1995,10 @@ static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
 		return dev_err_probe(dev, -ENODEV,
 				     "missing #global-interrupts property\n");
 	*pmu_irqs = 0;
+	of_property_read_u32(dev->of_node, "#pmu-interrupts", pmu_irqs);
+	if (*pmu_irqs > *global_irqs)
+		return dev_err_probe(dev, -EINVAL, "invalid #pmu_interrupts property");
+	*global_irqs -= *pmu_irqs;
 
 	data = of_device_get_match_data(dev);
 	smmu->version = data->version;
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
maintainer
#4 lore raw

Re: [PATCH 3/6] iommu/arm-smmu: Add DT PMU support

Send DT patches to the DT list if you want them reviewed with some guarantee that I'll see them. (Except right now as PW stopped getting mail. :( )

Quoted 1 line · Robin Murphy

Since IORT describes PMU interrupts rather inflexibly as an inherent

What's IORT? ;)

Quoted 10 lines

part of the SMMU, the best way to avoid excessive complexity is to make the way we handle DT look as similar as possible. Fortunately the de-facto standard for mentioning PMU interrupts at all under the current binding has been to include them in the global interrupt count, listing them after the actual fault interrupt(s), so we can capitalise on that. It's about 9 years too late to redefine "#global-interrupts" to exclude anything other than context interrupts without breaking compatibility, so we're stuck with a slightly convoluted definition of PMU interrupts as an optional subdivision of the "global" interrupts, but it works.

Quoted 7 lines

---

Not sure whether the count-backwards-from-the-middle nature of "number of PMU interrupts" is too ugly and "index of first PMU interrupt" might be any better.

Can this be solved with 'interrupt-names' instead deprecating #global-interrupts along the way?

Quoted 10 lines
---
 .../devicetree/bindings/iommu/arm,smmu.yaml   | 19 ++++++++++++++++++-
 drivers/iommu/arm/arm-smmu/arm-smmu.c         |  4 ++++
 2 files changed, 22 insertions(+), 1 deletion(-)
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +0 -0
index da5381c8ee11..9d39df42528a 100644

Documentation/devicetree/bindings/iommu/arm,smmu.yaml properties: · line 95

Quoted 9 lines
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +2 -0
@@ -87,10 +87,18 @@ properties:
 
   '#global-interrupts':
     description: The number of global interrupts exposed by the device.
+      Includes the count of PMU interrupts, if present.
     $ref: /schemas/types.yaml#/definitions/uint32
     minimum: 0
     maximum: 260   # 2 secure, 2 non-secure, and up to 256 perf counters
 
+  '#pmu-interrupts':

It would have been great if everything that's a count/size used '#', but that didn't happen. So I'm not that wild about using '#' randomly at all.

This needs a vendor prefix (arm,).

Quoted 59 lines
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +16 -1
+    description: The number of PMU interrupts. Must be equal to the number of
+      implemented counter groups.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 256
+
   '#iommu-cells':
     enum: [ 1, 2 ]
     description: |
@@ -115,6 +123,10 @@ properties:
       context bank. In the case of a single, combined interrupt, it must be
       listed multiple times.
 
+      If a PMU is present, the global interrupt entries consist of any fault
+      interrupts first, followed by #pmu-interrupts entries, one per implemented
+      counter group, specified in order of their indexing by the SMMU.
+
   dma-coherent:
     description: |
       Present if page table walks made by the SMMU are cache coherent with the
@@ -190,9 +202,14 @@ examples:
     smmu1: iommu@ba5e0000 {
             compatible = "arm,smmu-v1";
             reg = <0xba5e0000 0x10000>;
-            #global-interrupts = <2>;
+            #global-interrupts = <6>; /* 2 fault + 4 PMU */
+            #pmu-interrupts = <4>;
             interrupts = <0 32 4>,
                          <0 33 4>,
+                         <0 94 4>, /* This is the first PMU interrupt */
+                         <0 95 4>,
+                         <0 96 4>,
+                         <0 97 4>,
                          <0 34 4>, /* This is the first context interrupt */
                          <0 35 4>,
                          <0 36 4>,
drivers/iommu/arm/arm-smmu/arm-smmu.c +4 -0
index cbfe4cc914f0..8d6c8106fc1d 100644
@@ -1995,6 +1995,10 @@ static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
 		return dev_err_probe(dev, -ENODEV,
 				     "missing #global-interrupts property\n");
 	*pmu_irqs = 0;
+	of_property_read_u32(dev->of_node, "#pmu-interrupts", pmu_irqs);
+	if (*pmu_irqs > *global_irqs)
+		return dev_err_probe(dev, -EINVAL, "invalid #pmu_interrupts property");
+	*global_irqs -= *pmu_irqs;
 
 	data = of_device_get_match_data(dev);
 	smmu->version = data->version;
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

author reviewer
#5 lore raw

Re: [PATCH 3/6] iommu/arm-smmu: Add DT PMU support

Quoted 3 lines · Rob Herring

Send DT patches to the DT list if you want them reviewed with some guarantee that I'll see them. (Except right now as PW stopped getting mail. :( )

Oops, sorry, I think I had some sort of mental block there - I should equally have cc'd the IOMMU list too.

Quoted 4 lines · Robin Murphy
Quoted 1 line

Since IORT describes PMU interrupts rather inflexibly as an inherent

What's IORT? ;)

See the preceding patch :P

But indeed this particular sentence is still its untouched 5-year-old self... today I'd tend to write "ACPI IORT", is that better?

Quoted 12 lines
Quoted 12 lines

part of the SMMU, the best way to avoid excessive complexity is to make the way we handle DT look as similar as possible. Fortunately the de-facto standard for mentioning PMU interrupts at all under the current binding has been to include them in the global interrupt count, listing them after the actual fault interrupt(s), so we can capitalise on that. It's about 9 years too late to redefine "#global-interrupts" to exclude anything other than context interrupts without breaking compatibility, so we're stuck with a slightly convoluted definition of PMU interrupts as an optional subdivision of the "global" interrupts, but it works.

Quoted 8 lines
Quoted 5 lines

---

Not sure whether the count-backwards-from-the-middle nature of "number of PMU interrupts" is too ugly and "index of first PMU interrupt" might be any better.

Can this be solved with 'interrupt-names' instead deprecating #global-interrupts along the way?

My gut feeling is that that's not realistically feasible - we don't have a fixed set of IRQ lines with well-defined names, we have an imp-def set of "things that might raise one or more of global fault conditions", plus some number of context interrupts that work one of two ways, plus possibly some number of PMU counter group interrupts, with the additional fun that for some implementations those may all be the same physical signal (and I've now seen at least one SoC inexplicably wire up a combined interrupt *as well* as the individual outputs, and describe them all!)

My main concern is not breaking backward-compatibility for a new DT with an old OS. From past experience, I'm pretty sure Xen would be up in arms about any breakage in that direction, so "#global-interrupts" is sadly going nowhere for the foreseeable future. However even just thinking about implementing such a thing in the Linux driver puts me off - we'd have to take some horrible backwards approach where we look at all the names first to figure out which IRQs to get, and such an abuse of "interrupt-names" sounds worse than what it's trying to avoid.

The other approach I tried was to have a "pmu" sub-node with its own "interrupts" property, but that turned out to cause problems if it was behind an "interrupt-map", for reasons which didn't entirely make sense. Plus the corresponding DT changes were uglier and more invasive - the nice thing about the method here is that PMU support becomes a simple one-line addition to many existing DTs.

Quoted 18 lines
Quoted 18 lines
---
  .../devicetree/bindings/iommu/arm,smmu.yaml   | 19 ++++++++++++++++++-
  drivers/iommu/arm/arm-smmu/arm-smmu.c         |  4 ++++
  2 files changed, 22 insertions(+), 1 deletion(-)
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +1 -0
index da5381c8ee11..9d39df42528a 100644
@@ -87,10 +87,18 @@ properties:
 
    '#global-interrupts':
      description: The number of global interrupts exposed by the device.
+      Includes the count of PMU interrupts, if present.
      $ref: /schemas/types.yaml#/definitions/uint32
      minimum: 0
      maximum: 260   # 2 secure, 2 non-secure, and up to 256 perf counters
 
Quoted 7 lines

Documentation/devicetree/bindings/iommu/arm,smmu.yaml properties: · line 95

Quoted 1 line
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +1 -0
+  '#pmu-interrupts':

It would have been great if everything that's a count/size used '#', but that didn't happen. So I'm not that wild about using '#' randomly at all.

This needs a vendor prefix (arm,).

Sure - I went for consistency with what's already established, but since I'm similarly none too fond of it, swapping for something like "arm,pmu-irq-start" is fine by me.

Thanks,
Robin.

Quoted 55 lines
Quoted 55 lines
Documentation/devicetree/bindings/iommu/arm,smmu.yaml +16 -1
+    description: The number of PMU interrupts. Must be equal to the number of
+      implemented counter groups.
+    $ref: /schemas/types.yaml#/definitions/uint32
+    minimum: 1
+    maximum: 256
+
    '#iommu-cells':
      enum: [ 1, 2 ]
      description: |
@@ -115,6 +123,10 @@ properties:
        context bank. In the case of a single, combined interrupt, it must be
        listed multiple times.
 
+      If a PMU is present, the global interrupt entries consist of any fault
+      interrupts first, followed by #pmu-interrupts entries, one per implemented
+      counter group, specified in order of their indexing by the SMMU.
+
    dma-coherent:
      description: |
        Present if page table walks made by the SMMU are cache coherent with the
@@ -190,9 +202,14 @@ examples:
      smmu1: iommu@ba5e0000 {
              compatible = "arm,smmu-v1";
              reg = <0xba5e0000 0x10000>;
-            #global-interrupts = <2>;
+            #global-interrupts = <6>; /* 2 fault + 4 PMU */
+            #pmu-interrupts = <4>;
              interrupts = <0 32 4>,
                           <0 33 4>,
+                         <0 94 4>, /* This is the first PMU interrupt */
+                         <0 95 4>,
+                         <0 96 4>,
+                         <0 97 4>,
                           <0 34 4>, /* This is the first context interrupt */
                           <0 35 4>,
                           <0 36 4>,
drivers/iommu/arm/arm-smmu/arm-smmu.c +4 -0
index cbfe4cc914f0..8d6c8106fc1d 100644
@@ -1995,6 +1995,10 @@ static int arm_smmu_device_dt_probe(struct arm_smmu_device *smmu,
  		return dev_err_probe(dev, -ENODEV,
  				     "missing #global-interrupts property\n");
  	*pmu_irqs = 0;
+	of_property_read_u32(dev->of_node, "#pmu-interrupts", pmu_irqs);
+	if (*pmu_irqs > *global_irqs)
+		return dev_err_probe(dev, -EINVAL, "invalid #pmu_interrupts property");
+	*global_irqs -= *pmu_irqs;
 
  	data = of_device_get_match_data(dev);
  	smmu->version = data->version;
Signature · 1 line
2.28.0.dirty
Quoted 4 lines
Quoted 4 lines

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel

author reviewer
#6 lore raw

PATCH 4/6 iommu/smmu: Create child devices for PMUs

In reply to Robin Murphy

Whilst SMMUv3 PMCGs are somewhat free-form, the SMMUv1/v2 PMU has a well-defined and closely-coupled relationship with the main SMMU interface. This makes it logical for the SMMU driver to be in charge of consuming the firmware description and instantiating an abstract PMU device directly, obviating the need for independent detection methods.

---
 drivers/iommu/arm/arm-smmu/arm-smmu.c | 46 +++++++++++++++++++++++++++
 1 file changed, 46 insertions(+)
drivers/iommu/arm/arm-smmu/arm-smmu.c +46 -0
index 8d6c8106fc1d..58daf81adae7 100644
@@ -2070,6 +2070,45 @@ err_reset_platform_ops: __maybe_unused;
 	return err;
 }
 
+static int arm_smmu_pmu_init(struct arm_smmu_device *smmu,
+			     int irq_offset, int num_irqs)
+{
+	struct platform_device *pmu, *smmu_pdev;
+	int i, ret = -ENOMEM;
+
+	pmu = platform_device_alloc("arm-smmu-pmu", PLATFORM_DEVID_AUTO);
+	if (!pmu)
+		return -ENOMEM;
+
+	pmu->dev.parent = smmu->dev;
+	pmu->dev.platform_data = (__force void *)smmu->base + (3 << smmu->pgshift);
+	pmu->num_resources = num_irqs;
+	pmu->resource = kcalloc(num_irqs, sizeof(*pmu->resource), GFP_KERNEL);
+	if (!pmu->resource)
+		goto error;
+
+	smmu_pdev = to_platform_device(smmu->dev);
+	for (i = 0; i < num_irqs; i++) {
+		ret = platform_get_irq(smmu_pdev, irq_offset + i);
+		if (ret < 0)
+			goto error;
+
+		pmu->resource[i].start = ret;
+		pmu->resource[i].end = ret;
+		pmu->resource[i].flags = IORESOURCE_IRQ;
+	}
+
+	ret = platform_device_add(pmu);
+	if (ret)
+		goto error;
+
+	return 0;
+error:
+	pmu->dev.platform_data = NULL;
+	platform_device_put(pmu);
+	return ret;
+}
+
 static int arm_smmu_device_probe(struct platform_device *pdev)
 {
 	struct resource *res;
@@ -2216,6 +2255,13 @@ static int arm_smmu_device_probe(struct platform_device *pdev)
 			goto err_unregister_device;
 	}
 
+	/* Try the PMU last so we don't have to worry about cleaning it up */
+	if (pmu_irqs) {
+		err = arm_smmu_pmu_init(smmu, global_irqs, pmu_irqs);
+		if (err)
+			dev_warn(dev, "Failed to create PMU device: %d", err);
+	}
+
 	return 0;
 
 err_unregister_device:
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
author reviewer
#7 lore raw

PATCH 5/6 perf: Add ARM SMMU PMU driver

In reply to Robin Murphy

Add a basic driver for the SMMUv2 Performance Monitors Extension. This exposes the architecturally-defined events along with a fairly low-level interface, based on the relevant register fields, for filtering.

The relationship between Stream ID Groups and Counter Groups, i.e. which subsets of events may be visible to which sets of counters, is entirely implementation defined and non-discoverable, and attempting to define firmware bindings to describe the mappings would be considerable work for very little benefit. Thus we expect the user to have the relevant implementation knowledge, and explicitly specify the appropriate counter group for each desired event.

---
 drivers/perf/Kconfig        |  24 +-
 drivers/perf/Makefile       |   1 +
 drivers/perf/arm-smmu-pmu.c | 732 ++++++++++++++++++++++++++++++++++++
 3 files changed, 748 insertions(+), 9 deletions(-)
 create mode 100644 drivers/perf/arm-smmu-pmu.c
drivers/perf/Kconfig +15 -9
index e1a0c44bc686..6ee07c392bdb 100644
@@ -48,6 +48,15 @@ config ARM_CMN
 	  Support for PMU events monitoring on the Arm CMN-600 Coherent Mesh
 	  Network interconnect.
 
+config ARM_DSU_PMU
+	tristate "ARM DynamIQ Shared Unit (DSU) PMU"
+	depends on ARM64
+	  help
+	  Provides support for performance monitor unit in ARM DynamIQ Shared
+	  Unit (DSU). The DSU integrates one or more cores with an L3 memory
+	  system, control logic. The PMU allows counting various events related
+	  to DSU.
+
 config ARM_PMU
 	depends on ARM || ARM64
 	bool "ARM PMU framework"
@@ -60,6 +69,12 @@ config ARM_PMU_ACPI
 	depends on ARM_PMU && ACPI
 	def_bool y
 
+config ARM_SMMU_PMU
+	tristate "Arm SMMUv2 PMU support"
+	depends on ARM_SMMU
+	help
+	 Support for ARM SMMUv1/v2 performance monitors.
+
 config ARM_SMMU_V3_PMU
 	 tristate "ARM SMMUv3 Performance Monitors Extension"
 	 depends on (ARM64 && ACPI) || (COMPILE_TEST && 64BIT)
@@ -70,15 +85,6 @@ config ARM_SMMU_V3_PMU
 	   through the SMMU and allow the resulting information to be filtered
 	   based on the Stream ID of the corresponding master.
 
-config ARM_DSU_PMU
-	tristate "ARM DynamIQ Shared Unit (DSU) PMU"
-	depends on ARM64
-	  help
-	  Provides support for performance monitor unit in ARM DynamIQ Shared
-	  Unit (DSU). The DSU integrates one or more cores with an L3 memory
-	  system, control logic. The PMU allows counting various events related
-	  to DSU.
-
 config FSL_IMX8_DDR_PMU
 	tristate "Freescale i.MX8 DDR perf monitor"
 	depends on ARCH_MXC || COMPILE_TEST
drivers/perf/Makefile +1 -0
index 2db5418d5b0a..a0a1483a6825 100644
@@ -5,6 +5,7 @@ obj-$(CONFIG_ARM_CMN) += arm-cmn.o
 obj-$(CONFIG_ARM_DSU_PMU) += arm_dsu_pmu.o
 obj-$(CONFIG_ARM_PMU) += arm_pmu.o arm_pmu_platform.o
 obj-$(CONFIG_ARM_PMU_ACPI) += arm_pmu_acpi.o
+obj-$(CONFIG_ARM_SMMU_PMU) += arm-smmu-pmu.o
 obj-$(CONFIG_ARM_SMMU_V3_PMU) += arm_smmuv3_pmu.o
 obj-$(CONFIG_FSL_IMX8_DDR_PMU) += fsl_imx8_ddr_perf.o
 obj-$(CONFIG_HISI_PMU) += hisilicon/
drivers/perf/arm-smmu-pmu.c new file +732 -0

732 lines not on this page · raw message

Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
author reviewer
#8 lore raw

PATCH 6/6 arm64: dts: Add SMMU PMUs to Juno

In reply to Robin Murphy

MMU-401 implements a single counter group, with correspondingly a single overflow interrupt, which is also muxed into the combined interrupt output; the integrations in Juno rely on the latter.

---
 arch/arm64/boot/dts/arm/juno-base.dtsi | 26 ++++++++++++++++++++------
 1 file changed, 20 insertions(+), 6 deletions(-)
arch/arm64/boot/dts/arm/juno-base.dtsi +20 -6
index 6288e104a089..9e242ea84871 100644
@@ -48,20 +48,25 @@ smmu_pcie: iommu@2b500000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x2b500000 0x0 0x10000>;
 		interrupts = <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 40 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
 		dma-coherent;
 		status = "disabled";
+
 	};
 
 	smmu_etr: iommu@2b600000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x2b600000 0x0 0x10000>;
 		interrupts = <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 42 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
 		dma-coherent;
 		power-domains = <&scpi_devpd 0>;
 	};
@@ -638,9 +643,11 @@ smmu_dma: iommu@7fb00000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x7fb00000 0x0 0x10000>;
 		interrupts = <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 95 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
 		dma-coherent;
 	};
 
@@ -648,27 +655,34 @@ smmu_hdlcd1: iommu@7fb10000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x7fb10000 0x0 0x10000>;
 		interrupts = <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 99 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
+
 	};
 
 	smmu_hdlcd0: iommu@7fb20000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x7fb20000 0x0 0x10000>;
 		interrupts = <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 97 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
 	};
 
 	smmu_usb: iommu@7fb30000 {
 		compatible = "arm,mmu-401", "arm,smmu-v1";
 		reg = <0x0 0x7fb30000 0x0 0x10000>;
 		interrupts = <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
+			     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>,
 			     <GIC_SPI 101 IRQ_TYPE_LEVEL_HIGH>;
 		#iommu-cells = <1>;
-		#global-interrupts = <1>;
+		#global-interrupts = <2>;
+		#pmu-interrupts = <1>;
 		dma-coherent;
 	};
 
Signature · 7 lines
2.28.0.dirty


_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel
maintainer
#9 lore raw
In reply to Robin Murphy
Quoted 11 lines · Robin Murphy

Since our friends at Linaro had some interest in the area, I've dug up and finished off my old SMMUv1/v2 PMU driver, now with a slightly less terrible DT binding to boot. Patch #1 involves a general refactoring of the IRQ setup code in arm-smmu which also now serves to remove the general dependency on explicit IRQ resources for of_platform devices.

ACPI support via IORT (patch #2) is pleasingly trivial, but does have a hard dependency on the previous patch, otherwise SMMU context interrupts may get messed up.

[...]

Applied first patch to will (for-joerg/arm-smmu/updates), thanks!

[1/6] iommu/arm-smmu: Account for PMU interrupts

      https://git.kernel.org/will/c/97dfad194ca8

Cheers,

Signature · 10 lines
Will

https://fixes.arm64.dev
https://next.arm64.dev
https://will.arm64.dev

_______________________________________________
linux-arm-kernel mailing list
[email protected]
http://lists.infradead.org/mailman/listinfo/linux-arm-kernel