Skip to contentSkip to conversation

iommu/arm-smmu: Misc modifications to support SMMUs on Calxeda ECX-2000

PATCH Superseded Andreas Herrmann 4 participants 18 messages Oct 8, 2013 – Oct 9, 2013

Subsystems (4)

v1 of 4 Superseded Latest is v4 →

lore
Share & tools
lore
#0 lore raw

cover iommu/arm-smmu: Misc modifications to support SMMUs on Calxeda ECX-2000

Hi Will,

Here is a reworked patch set
- to introduce device isolation (using bus-notifier).
- to work around secure-only config access to SMMU registers
- (new) to support masking of all streamIDs for an SMMU
- add DT information for SMMUs on Calxeda ECX-2000

I've also separated out the introduction of driver options handling. (Just in case it doesn't please you...)

I still need to add the documentation for the introduced SMMU options to Documentation/devicetree/bindings/iommu/arm,smmu.txt. But first I'd like to know whether addition of the 3 options is acceptable.

Comments welcome.

Thanks,

Andreas

#1 lore raw

PATCH 1/5 iommu/arm-smmu: Introduce driver option handling

Introduce handling of driver options. Options are set based on DT information when probing an SMMU device. The first option introduced is "arm,smmu-isolate-devices". (It will be used in the bus notifier block.)

---
 drivers/iommu/arm-smmu.c |   49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
drivers/iommu/arm-smmu.c +49 -0
index b632bcd..b127f0f 100644
@@ -348,6 +348,7 @@ struct arm_smmu_device {
 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
 #define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
 	u32				features;
+	u32				options;
 	int				version;
 
 	u32				num_context_banks;
@@ -398,6 +399,52 @@ struct arm_smmu_domain {
 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 static LIST_HEAD(arm_smmu_devices);
 
+/* driver options */
+enum arm_smmu_option {
+	ISOLATE_DEVICES = 0,
+	OPTION_MAX,
+};
+
+struct arm_smmu_option_prop {
+	enum arm_smmu_option opt;
+	const char *prop;
+};
+
+static struct arm_smmu_option_prop arm_smmu_options [] = {
+	{ ISOLATE_DEVICES, "arm,smmu-isolate-devices" },
+	{ OPTION_MAX, NULL},
+};
+
+static inline int arm_smmu_has_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	return (smmu->options & (1 << opt));
+}
+
+static inline void arm_smmu_set_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	smmu->options |= (1 << opt);
+}
+
+static inline void arm_smmu_clear_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	smmu->options &= ~(1 << opt);
+}
+
+static void arm_smmu_check_options(struct arm_smmu_device *smmu)
+{
+	int i = 0;
+
+	do {
+		if (of_property_read_bool(smmu->dev->of_node,
+						arm_smmu_options[i].prop))
+			arm_smmu_set_option(smmu, arm_smmu_options[i].opt);
+		i++;
+	} while (arm_smmu_options[i].opt < OPTION_MAX);
+}
+
 static struct arm_smmu_master *find_smmu_master(struct arm_smmu_device *smmu,
 						struct device_node *dev_node)
 {
@@ -1791,6 +1838,8 @@ static int arm_smmu_device_dt_probe(struct platform_device *pdev)
 	}
 	smmu->dev = dev;
 
+	arm_smmu_check_options(smmu);
+
 	res = platform_get_resource(pdev, IORESOURCE_MEM, 0);
 	if (!res) {
 		dev_err(dev, "missing base address/size\n");
Signature · 1 line
1.7.9.5
#2 lore raw

PATCH 1/5 iommu/arm-smmu: Introduce driver option handling

Quoted 15 lines · Andreas Herrmann

Introduce handling of driver options. Options are set based on DT information when probing an SMMU device. The first option introduced is "arm,smmu-isolate-devices". (It will be used in the bus notifier block.)

---
 drivers/iommu/arm-smmu.c |   49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
drivers/iommu/arm-smmu.c +0 -0
index b632bcd..b127f0f 100644

drivers/iommu/arm-smmu.c struct arm_smmu_device { · line 351

Quoted 4 lines
drivers/iommu/arm-smmu.c +1 -0
@@ -348,6 +348,7 @@ struct arm_smmu_device {
 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
 #define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
 	u32				features;
+	u32				options;

This should be enum arm_smmu_option. Also, we should probably be consistent between the options and features (i.e. either use an enum for each of them, or just stick to #defines for both).

Quoted 30 lines
drivers/iommu/arm-smmu.c +23 -0
 	int				version;
 
 	u32				num_context_banks;
@@ -398,6 +399,52 @@ struct arm_smmu_domain {
 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 static LIST_HEAD(arm_smmu_devices);
 
+/* driver options */
+enum arm_smmu_option {
+	ISOLATE_DEVICES = 0,
+	OPTION_MAX,
+};
+
+struct arm_smmu_option_prop {
+	enum arm_smmu_option opt;
+	const char *prop;
+};
+
+static struct arm_smmu_option_prop arm_smmu_options [] = {
+	{ ISOLATE_DEVICES, "arm,smmu-isolate-devices" },
+	{ OPTION_MAX, NULL},
+};
+
+static inline int arm_smmu_has_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	return (smmu->options & (1 << opt));
+}
+
+static inline void arm_smmu_set_option(struct arm_smmu_device *smmu,

drivers/iommu/arm-smmu.c struct arm_smmu_domain { · line 434

Quoted 10 lines
drivers/iommu/arm-smmu.c +10 -0
+				enum arm_smmu_option opt)
+{
+	smmu->options |= (1 << opt);
+}
+
+static inline void arm_smmu_clear_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	smmu->options &= ~(1 << opt);
+}

These three functions are a bit over-engineered! We have things like __set_bit if you really want to use helpers.

Will

#3 lore raw

PATCH 1/5 iommu/arm-smmu: Introduce driver option handling

Quoted 11 lines · Will Deacon
Quoted 10 lines · Andreas Herrmann

Introduce handling of driver options. Options are set based on DT information when probing an SMMU device. The first option introduced is "arm,smmu-isolate-devices". (It will be used in the bus notifier block.)

---
 drivers/iommu/arm-smmu.c |   49 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 49 insertions(+)
Quoted 13 lines

drivers/iommu/arm-smmu.c struct arm_smmu_device { · line 351

Quoted 9 lines
drivers/iommu/arm-smmu.c +1 -0
index b632bcd..b127f0f 100644
@@ -348,6 +348,7 @@ struct arm_smmu_device {
 #define ARM_SMMU_FEAT_TRANS_S2		(1 << 3)
 #define ARM_SMMU_FEAT_TRANS_NESTED	(1 << 4)
 	u32				features;
+	u32				options;

This should be enum arm_smmu_option. Also, we should probably be consistent between the options and features (i.e. either use an enum for each of them, or just stick to #defines for both).

Ok, so I'd prefer to use macros then.

Quoted 35 lines
Quoted 35 lines
drivers/iommu/arm-smmu.c +28 -0
 	int				version;
 
 	u32				num_context_banks;
@@ -398,6 +399,52 @@ struct arm_smmu_domain {
 static DEFINE_SPINLOCK(arm_smmu_devices_lock);
 static LIST_HEAD(arm_smmu_devices);
 
+/* driver options */
+enum arm_smmu_option {
+	ISOLATE_DEVICES = 0,
+	OPTION_MAX,
+};
+
+struct arm_smmu_option_prop {
+	enum arm_smmu_option opt;
+	const char *prop;
+};
+
+static struct arm_smmu_option_prop arm_smmu_options [] = {
+	{ ISOLATE_DEVICES, "arm,smmu-isolate-devices" },
+	{ OPTION_MAX, NULL},
+};
+
+static inline int arm_smmu_has_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	return (smmu->options & (1 << opt));
+}
+
+static inline void arm_smmu_set_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	smmu->options |= (1 << opt);
+}
+
Quoted 8 lines

drivers/iommu/arm-smmu.c struct arm_smmu_domain { · line 434

Quoted 5 lines
drivers/iommu/arm-smmu.c +5 -0
+static inline void arm_smmu_clear_option(struct arm_smmu_device *smmu,
+				enum arm_smmu_option opt)
+{
+	smmu->options &= ~(1 << opt);
+}

These three functions are a bit over-engineered! We have things like __set_bit if you really want to use helpers.

Right, will adapt it.

Andreas

#4 lore raw

PATCH 2/5 iommu/arm-smmu: Introduce bus notifier block

In reply to Andreas Herrmann

At the moment just handle BUS_NOTIFY_BIND_DRIVER to conditionally isolate all master devices for an SMMU.

Depending on DT information each device is put into its own protection domain (if possible). For configuration with one or just a few masters per SMMU that is easy to achieve.

In case of many devices per SMMU (e.g. MMU-500 with it's distributed translation support) isolation of each device might not be possible -- depending on number of available SMR groups and/or context banks.

Default is that device isolation is contolled per SMMU with SMMU node property "arm,smmu-isolate-devices" in a DT. If this property is set for an SMMU node, device isolation is performed.

W/o device isolation the driver detects SMMUs but no translation is configured (transactions just bypass translation process).

Note that for device isolation dma_base and size are fixed as 0 and SZ_128M at the moment.

---
 drivers/iommu/arm-smmu.c |   53 ++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 53 insertions(+)
drivers/iommu/arm-smmu.c +53 -0
index b127f0f..9a73618 100644
@@ -46,6 +46,7 @@
 #include <linux/amba/bus.h>
 
 #include <asm/pgalloc.h>
+#include <asm/dma-iommu.h>
 
 /* Maximum number of stream IDs assigned to a single device */
 #define MAX_MASTER_STREAMIDS		8
@@ -1996,6 +1997,56 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 	return 0;
 }
 
+static int arm_smmu_device_notifier(struct notifier_block *nb,
+				unsigned long action, void *data)
+{
+	struct device *dev = data;
+	struct dma_iommu_mapping *mapping;
+	struct arm_smmu_device *smmu;
+	void __iomem *gr0_base;
+	u32 cr0;
+	int ret;
+
+	switch (action) {
+	case BUS_NOTIFY_BIND_DRIVER:
+
+		arm_smmu_add_device(dev);
+		smmu = dev->archdata.iommu;
+		if (!smmu || !(arm_smmu_has_option(smmu, ISOLATE_DEVICES)))
+			break;
+
+		mapping = arm_iommu_create_mapping(&platform_bus_type,
+						0, SZ_128M, 0);
+		if (IS_ERR(mapping)) {
+			ret = PTR_ERR(mapping);
+			dev_info(dev, "arm_iommu_create_mapping failed\n");
+			break;
+		}
+
+		ret = arm_iommu_attach_device(dev, mapping);
+		if (ret < 0) {
+			dev_info(dev, "arm_iommu_attach_device failed\n");
+			arm_iommu_release_mapping(mapping);
+		}
+
+		gr0_base = ARM_SMMU_GR0_NS(smmu);
+		cr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
+		cr0 |= sCR0_USFCFG;
+		writel(cr0, gr0_base + ARM_SMMU_GR0_sCR0);
+
+		break;
+
+	default:
+		break;
+	}
+
+	return 0;
+}
+
+static struct notifier_block device_nb = {
+	.notifier_call = arm_smmu_device_notifier,
+};
+
 #ifdef CONFIG_OF
 static struct of_device_id arm_smmu_of_match[] = {
 	{ .compatible = "arm,smmu-v1", },
@@ -2032,6 +2083,8 @@ static int __init arm_smmu_init(void)
 	if (!iommu_present(&amba_bustype))
 		bus_set_iommu(&amba_bustype, &arm_smmu_ops);
 
+	bus_register_notifier(&platform_bus_type, &device_nb);
+
 	return 0;
 }
 
Signature · 1 line
1.7.9.5
#5 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

In reply to Andreas Herrmann

Ie. use a mask based on smr_mask_bits to map all stream IDs of an SMMU to one context.

This behaviour is controlled per SMMU node with DT property "arm,smmu-mask-stream-ids" and is only allowed if just a single master is attached to an SMMU. If the option is specified, all stream-ids that are provided in DT for the single master have no relevance.

This is useful/convenient if a master has more than 8 stream-ids or if not all stream-ids are known for a master device.

---
 drivers/iommu/arm-smmu.c |   43 +++++++++++++++++++++++++++++++------------
 1 file changed, 31 insertions(+), 12 deletions(-)
drivers/iommu/arm-smmu.c +31 -12
index 9a73618..e2dbcbb 100644
@@ -359,6 +359,7 @@ struct arm_smmu_device {
 
 	u32				num_mapping_groups;
 	DECLARE_BITMAP(smr_map, ARM_SMMU_MAX_SMRS);
+	u32				smr_mask_bits;
 
 	unsigned long			input_size;
 	unsigned long			s1_output_size;
@@ -370,6 +371,7 @@ struct arm_smmu_device {
 
 	struct list_head		list;
 	struct rb_root			masters;
+	u32				num_masters;
 };
 
 struct arm_smmu_cfg {
@@ -403,6 +405,7 @@ static LIST_HEAD(arm_smmu_devices);
 /* driver options */
 enum arm_smmu_option {
 	ISOLATE_DEVICES = 0,
+	MASK_STREAM_IDS,
 	OPTION_MAX,
 };
 
@@ -413,6 +416,7 @@ struct arm_smmu_option_prop {
 
 static struct arm_smmu_option_prop arm_smmu_options [] = {
 	{ ISOLATE_DEVICES, "arm,smmu-isolate-devices" },
+	{ MASK_STREAM_IDS, "arm,smmu-mask-stream-ids" },
 	{ OPTION_MAX, NULL},
 };
 
@@ -488,6 +492,7 @@ static int insert_smmu_master(struct arm_smmu_device *smmu,
 
 	rb_link_node(&master->node, parent, new);
 	rb_insert_color(&master->node, &smmu->masters);
+	smmu->num_masters++;
 	return 0;
 }
 
@@ -520,8 +525,19 @@ static int register_smmu_master(struct arm_smmu_device *smmu,
 	master->of_node		= masterspec->np;
 	master->num_streamids	= masterspec->args_count;
 
-	for (i = 0; i < master->num_streamids; ++i)
-		master->streamids[i] = masterspec->args[i];
+	if (arm_smmu_has_option(smmu, MASK_STREAM_IDS)) {
+		if (smmu->num_masters) {
+			dev_err(dev, "option %s not supported with multiple masters\n",
+				arm_smmu_options[MASK_STREAM_IDS].prop);
+			return -EINVAL;
+		}
+		/* set fixed streamid (0) that will be used for masking */
+		master->num_streamids = 1;
+		master->streamids[0] = 0;
+	} else {
+		for (i = 0; i < master->num_streamids; ++i)
+			master->streamids[i] = masterspec->args[i];
+	}
 
 	return insert_smmu_master(smmu, master);
 }
@@ -1063,17 +1079,20 @@ static int arm_smmu_master_configure_smrs(struct arm_smmu_device *smmu,
 			goto err_free_smrs;
 		}
 
-		smrs[i] = (struct arm_smmu_smr) {
-			.idx	= idx,
-			.mask	= 0, /* We don't currently share SMRs */
-			.id	= master->streamids[i],
-		};
+		smrs[i].idx = idx;
+		smrs[i].id = master->streamids[i];
+
+		if (arm_smmu_has_option(smmu, MASK_STREAM_IDS))
+			smrs[i].mask = smmu->smr_mask_bits;
+		else
+			smrs[i].mask = 0;
 	}
 
 	/* It worked! Now, poke the actual hardware */
 	for (i = 0; i < master->num_streamids; ++i) {
 		u32 reg = SMR_VALID | smrs[i].id << SMR_ID_SHIFT |
 			  smrs[i].mask << SMR_MASK_SHIFT;
+		dev_dbg(smmu->dev, "SMR%d: 0x%x\n", smrs[i].idx, reg);
 		writel_relaxed(reg, gr0_base + ARM_SMMU_GR0_SMR(smrs[i].idx));
 	}
 
@@ -1726,7 +1745,7 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 	}
 
 	if (id & ID0_SMS) {
-		u32 smr, sid, mask;
+		u32 smr, sid;
 
 		smmu->features |= ARM_SMMU_FEAT_STREAM_MATCH;
 		smmu->num_mapping_groups = (id >> ID0_NUMSMRG_SHIFT) &
@@ -1742,18 +1761,18 @@ static int arm_smmu_device_cfg_probe(struct arm_smmu_device *smmu)
 		writel_relaxed(smr, gr0_base + ARM_SMMU_GR0_SMR(0));
 		smr = readl_relaxed(gr0_base + ARM_SMMU_GR0_SMR(0));
 
-		mask = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
+		smmu->smr_mask_bits = (smr >> SMR_MASK_SHIFT) & SMR_MASK_MASK;
 		sid = (smr >> SMR_ID_SHIFT) & SMR_ID_MASK;
-		if ((mask & sid) != sid) {
+		if ((smmu->smr_mask_bits & sid) != sid) {
 			dev_err(smmu->dev,
 				"SMR mask bits (0x%x) insufficient for ID field (0x%x)\n",
-				mask, sid);
+				smmu->smr_mask_bits, sid);
 			return -ENODEV;
 		}
 
 		dev_notice(smmu->dev,
 			   "\tstream matching with %u register groups, mask 0x%x",
-			   smmu->num_mapping_groups, mask);
+			   smmu->num_mapping_groups, smmu->smr_mask_bits);
 	}
 
 	/* ID1 */
Signature · 1 line
1.7.9.5
#6 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 10 lines · Andreas Herrmann

Ie. use a mask based on smr_mask_bits to map all stream IDs of an SMMU to one context.

This behaviour is controlled per SMMU node with DT property "arm,smmu-mask-stream-ids" and is only allowed if just a single master is attached to an SMMU. If the option is specified, all stream-ids that are provided in DT for the single master have no relevance.

This is useful/convenient if a master has more than 8 stream-ids or if not all stream-ids are known for a master device.

Hmmm, this really scares me. What's the use-case? I worry about people inadvertently putting devices into an iommu domain because they happen to be included in a mask. Your solution seems to be targetting a single master with lots of IDs -- is this a PCI RC?

Will

#7 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 9 lines · Will Deacon
Quoted 8 lines · Andreas Herrmann

Ie. use a mask based on smr_mask_bits to map all stream IDs of an SMMU to one context.

This behaviour is controlled per SMMU node with DT property "arm,smmu-mask-stream-ids" and is only allowed if just a single master is attached to an SMMU. If the option is specified, all stream-ids that are provided in DT for the single master have no relevance.

Quoted 7 lines
Quoted 2 lines

This is useful/convenient if a master has more than 8 stream-ids or if not all stream-ids are known for a master device.

Hmmm, this really scares me. What's the use-case? I worry about people inadvertently putting devices into an iommu domain because they happen to be included in a mask. Your solution seems to be targetting a single master with lots of IDs -- is this a PCI RC?

No, it's the sata device. It has 0xf as smr_mask_bits.

But it's just one device and it (usually) should be mapped to one context -- no matter how many StreamIDs are really used.

(PCI RC has also 0xf but there it's clear there can be multiple devices and the masking should not be done.)

Andreas

#8 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 18 lines · Andreas Herrmann
Quoted 9 lines · Will Deacon
Quoted 8 lines · Andreas Herrmann

Ie. use a mask based on smr_mask_bits to map all stream IDs of an SMMU to one context.

This behaviour is controlled per SMMU node with DT property "arm,smmu-mask-stream-ids" and is only allowed if just a single master is attached to an SMMU. If the option is specified, all stream-ids that are provided in DT for the single master have no relevance.

Quoted 7 lines
Quoted 2 lines

This is useful/convenient if a master has more than 8 stream-ids or if not all stream-ids are known for a master device.

Hmmm, this really scares me. What's the use-case? I worry about people inadvertently putting devices into an iommu domain because they happen to be included in a mask. Your solution seems to be targetting a single master with lots of IDs -- is this a PCI RC?

Quoted 4 lines

No, it's the sata device. It has 0xf as smr_mask_bits.

But it's just one device and it (usually) should be mapped to one context -- no matter how many StreamIDs are really used.

To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

Andreas

PS: I think (2) needs to be addressed sooner or later. We should use
only as many SMR groups as really required -- ie. use masking of StreamIds if possible. If more than one StreamID is given for a master it might be possible to calculate a mask for a (power-of-two) number of adjacent StreamIds and then use only one SMR group to map these IDs to a context. (But I think that should only be done if multiple masters are attached to an SMMU.)

#9 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 10 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

Quoted 8 lines

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

Quoted 7 lines

PS: I think (2) needs to be addressed sooner or later. We should use
only as many SMR groups as really required -- ie. use masking of StreamIds if possible. If more than one StreamID is given for a master it might be possible to calculate a mask for a (power-of-two) number of adjacent StreamIds and then use only one SMR group to map these IDs to a context. (But I think that should only be done if multiple masters are attached to an SMMU.)

I spent a few weeks looking at doing minimal SMR grouping whilst writing the driver and ended up convincing myself that it's an NP-complete problem (I tried a reduction involving Hamiltonian Cycles). Of course, if you have some ideas here, we can try to implement something for a constrained instance of the problem.

For example, a simple solution is to xor all the IDs together and check no other IDs fall under the resulting mask. However, this again relies on the DT telling us complete topological information as well as the IDs being organised in a particular way.

Will

#10 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 27 lines

-----Original Message-----
From: iommu-bounces at lists.linux-foundation.org [mailto:iommu- bounces at lists.linux-foundation.org] On Behalf Of Will Deacon
Sent: Tuesday, October 08, 2013 10:29 PM
To: Andreas Herrmann
Cc: iommu at lists.linux-foundation.org; Rob Herring; linux-arm- kernel at lists.infradead.org
Subject: Re: [PATCH 3/5] iommu/arm-smmu: Introduce stream ID masking

Quoted 15 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Quoted 9 lines
Quoted 3 lines

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

[Sethi Varun-B16395] I don't quiet follow the point about topological information. How do you determine from the topology if the stream ids should share the iommu domain? Another question, how would iommu group creation work for dma masters with multiple stream ids?

-Varun

#11 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

In reply to Will Deacon
Quoted 16 lines · Will Deacon
Quoted 15 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Quoted 8 lines
Quoted 3 lines

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

Couldn't this information be implied from the DT when you have no streamID and only a single mmu-master?

Rob

Quoted 21 lines
Quoted 7 lines

PS: I think (2) needs to be addressed sooner or later. We should use
only as many SMR groups as really required -- ie. use masking of StreamIds if possible. If more than one StreamID is given for a master it might be possible to calculate a mask for a (power-of-two) number of adjacent StreamIds and then use only one SMR group to map these IDs to a context. (But I think that should only be done if multiple masters are attached to an SMMU.)

I spent a few weeks looking at doing minimal SMR grouping whilst writing the driver and ended up convincing myself that it's an NP-complete problem (I tried a reduction involving Hamiltonian Cycles). Of course, if you have some ideas here, we can try to implement something for a constrained instance of the problem.

For example, a simple solution is to xor all the IDs together and check no other IDs fall under the resulting mask. However, this again relies on the DT telling us complete topological information as well as the IDs being organised in a particular way.

Will

#12 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 21 lines · Rob Herring
Quoted 20 lines · Will Deacon
Quoted 18 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

Quoted 7 lines
Quoted 4 lines

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

Couldn't this information be implied from the DT when you have no streamID and only a single mmu-master?

It's still fairly fragile though, since you (a) don't know that the stream IDs of the master are matchable by the SMRs (if we had the stream IDs, we can check this) and (b) you can still end up pulling devices into your address space that would otherwise happily operate using passthrough.

Will

#13 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

Quoted 27 lines · Will Deacon
Quoted 26 lines · Rob Herring
Quoted 24 lines · Will Deacon
Quoted 10 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

Quoted 8 lines

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

Quoted 7 lines
Quoted 2 lines

Couldn't this information be implied from the DT when you have no streamID and only a single mmu-master?

It's still fairly fragile though, since you (a) don't know that the stream IDs of the master are matchable by the SMRs (if we had the stream IDs, we can check this) and (b) you can still end up pulling devices into your address space that would otherwise happily operate using passthrough.

FYI, for SATA I'd have to specify below IDs (which requires some other minor adaptions). I really would like to avoid specification of all these StreamIDs and rather use stream matching with an appropriate mask.

I can't end up pulling other devices into that mapping. It's just this single master device at this SMMU.

And I also think that this "mask-all-stream-ids" option belongs to the device tree as an hint to the driver that it is sane to use smr_mask_bits as a mask for stream matching for this particular SMMU.

Andreas

---
ecx-2000, iommu/arm-smmu, of: Add all 10 StreamIDs for sata-smmu

Unfortunately this requires adaptions of MAX_MASTER_STREAMIDS and MAX_PHANDLE_ARGS.

It still won't work with the current arm-smmu driver as it strictly uses one SMR group for each StreamID. (But the corresponding SMMU has just 4 SMR groups).

---
 arch/arm/boot/dts/ecx-2000.dts    |    2 +-
 arch/arm/boot/dts/ecx-common.dtsi |    2 +-
 drivers/iommu/arm-smmu.c          |    2 +-
 include/linux/of.h                |    2 +-
 4 files changed, 4 insertions(+), 4 deletions(-)
arch/arm/boot/dts/ecx-2000.dts +1 -1
index 270f052..3904c2b 100644
@@ -135,7 +135,7 @@
 		smmu_sata: smmu at 920180000 {
 			compatible = "arm,mmu-400";
 			reg = <0x00000009 0x20180000 0x10000>;
-			mmu-masters = <&sata>;
+			mmu-masters = <&sata 0 1 2 3 4 5 6 7 8 9>;
 			#global-interrupts = <1>;
 			interrupts = <0 114 4 0 114 4>;
 			arm,smmu-secure-config-access;
arch/arm/boot/dts/ecx-common.dtsi +1 -1
index 50e401e..961dc5b 100644
@@ -35,7 +35,7 @@
 					     &combophy0 3>;
 			calxeda,sgpio-gpio =<&gpioh 5 1 &gpioh 6 1 &gpioh 7 1>;
 			calxeda,led-order = <4 0 1 2 3>;
-			#stream-id-cells = <0>;
+			#stream-id-cells = <10>;
 		};
 
 		sdhci at ffe0e000 {
drivers/iommu/arm-smmu.c +1 -1
index 4e8ceab..cd370f7 100644
@@ -55,7 +55,7 @@
 #define ARM_SMMU_OPT_SECURE_CONFIG_ACCESS	(1 << 2)
 
 /* Maximum number of stream IDs assigned to a single device */
-#define MAX_MASTER_STREAMIDS		8
+#define MAX_MASTER_STREAMIDS		10
 
 /* Maximum number of context banks per SMMU */
 #define ARM_SMMU_MAX_CBS		128
include/linux/of.h +1 -1
index f95aee3..47f4857 100644
@@ -67,7 +67,7 @@ struct device_node {
 #endif
 };
 
-#define MAX_PHANDLE_ARGS 8
+#define MAX_PHANDLE_ARGS 10
 struct of_phandle_args {
 	struct device_node *np;
 	int args_count;
Signature · 1 line
1.7.9.5
#14 lore raw

PATCH 3/5 iommu/arm-smmu: Introduce stream ID masking

In reply to Will Deacon
Quoted 16 lines · Will Deacon
Quoted 15 lines · Andreas Herrmann

On Tue, Oct 08, 2013 at 05:20:08PM +0200, Andreas Herrmann wrote:
To be more specific: For SATA I'd need to specify 10 StreamIds. This would

(1) exceed MAX_MASTER_STREAMIDS (currently it's 8)

(Can easily be fixed by adapting a macro.)

(2) exceed number of available SMR groups to map the IDs to a context.

This can be solved by caclulating an appropriate mask for the mapping (but with a non-power-of-two number of StreamIds that's already non-trivial -- for the trivial case I have some code to do this).

Quoted 8 lines
Quoted 3 lines

Both problems are avoided by introducing this patch -- use smr_mask_bits to map all StreamIDs to the same context and be done with it. (for the "single-master-SMMU" case)

The problem is, this information *really* doesn't belong in the device tree, but I think computing the general case dynamically is incredibly difficult too (and requires *complete* topological information in the device-tree, so you don't accidentally pull in other devices).

So, if this kind of driver option does not belong to the device tree, what's the preferred solution to tell the driver to use smr_mask_bits as a mask for stream matching for a certain SMMU (that has only one master)?

Quoted 14 lines
Quoted 7 lines

PS: I think (2) needs to be addressed sooner or later. We should use
only as many SMR groups as really required -- ie. use masking of StreamIds if possible. If more than one StreamID is given for a master it might be possible to calculate a mask for a (power-of-two) number of adjacent StreamIds and then use only one SMR group to map these IDs to a context. (But I think that should only be done if multiple masters are attached to an SMMU.)

I spent a few weeks looking at doing minimal SMR grouping whilst writing the driver and ended up convincing myself that it's an NP-complete problem (I tried a reduction involving Hamiltonian Cycles). Of course, if you have some ideas here, we can try to implement something for a constrained instance of the problem.

Quoted 4 lines

For example, a simple solution is to xor all the IDs together and check no other IDs fall under the resulting mask. However, this again relies on the DT telling us complete topological information as well as the IDs being organised in a particular way.

Yes, it is an ugly problem. But assuming that the StreamIDs in many cases have a particular organsiation it should be possible to calculate a mask with reasonable effort. (I've attached a patch to do this for certain cases).

I think it could be extended, such that in a case where StreamIDs 0, ... 9 are specified for a master the driver could set up

SMR0 with mask 0x7, id 0 (mapping StreamIDs 0, ..., 7) SMR1 for StreamID 8
SMR2 for StreamID 9

Thus 3 SMR groups would be used instead of 10 (as it is done with the current implementation).

Andreas

#15 lore raw

PATCH 4/5 iommu/arm-smmu: Support buggy implementations where all config accesses are secure

In reply to Andreas Herrmann

In such a case we have to use secure aliases of some non-secure registers.

This handling is switched on by DT property
"arm,smmu-secure-config-access" for an SMMU node.

---
 drivers/iommu/arm-smmu.c |   30 +++++++++++++++++++++---------
 1 file changed, 21 insertions(+), 9 deletions(-)
drivers/iommu/arm-smmu.c +21 -9
index e2dbcbb..de74273 100644
@@ -61,6 +61,15 @@
 #define ARM_SMMU_GR0(smmu)		((smmu)->base)
 #define ARM_SMMU_GR1(smmu)		((smmu)->base + (smmu)->pagesize)
 
+/*
+ * SMMU global address space with conditional offset to access secure aliases of
+ * non-secure registers (e.g. nsCR0: 0x400, nsGFSR: 0x448, nsGFSYNR0: 0x450)
+ */
+#define ARM_SMMU_GR0_NS(smmu)						\
+	((smmu)->base +							\
+		(arm_smmu_has_option(smmu, SECURE_CONFIG_ACCESS) \
+			? 0x400 : 0))
+
 /* Page table bits */
 #define ARM_SMMU_PTE_PAGE		(((pteval_t)3) << 0)
 #define ARM_SMMU_PTE_CONT		(((pteval_t)1) << 52)
@@ -406,6 +415,7 @@ static LIST_HEAD(arm_smmu_devices);
 enum arm_smmu_option {
 	ISOLATE_DEVICES = 0,
 	MASK_STREAM_IDS,
+	SECURE_CONFIG_ACCESS,
 	OPTION_MAX,
 };
 
@@ -417,6 +427,7 @@ struct arm_smmu_option_prop {
 static struct arm_smmu_option_prop arm_smmu_options [] = {
 	{ ISOLATE_DEVICES, "arm,smmu-isolate-devices" },
 	{ MASK_STREAM_IDS, "arm,smmu-mask-stream-ids" },
+	{ SECURE_CONFIG_ACCESS, "arm,smmu-secure-config-access" },
 	{ OPTION_MAX, NULL},
 };
 
@@ -675,16 +686,16 @@ static irqreturn_t arm_smmu_global_fault(int irq, void *dev)
 {
 	u32 gfsr, gfsynr0, gfsynr1, gfsynr2;
 	struct arm_smmu_device *smmu = dev;
-	void __iomem *gr0_base = ARM_SMMU_GR0(smmu);
+	void __iomem *gr0_base = ARM_SMMU_GR0_NS(smmu);
 
 	gfsr = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
-	if (!gfsr)
-		return IRQ_NONE;
-
 	gfsynr0 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR0);
 	gfsynr1 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR1);
 	gfsynr2 = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSYNR2);
 
+	if (!gfsr)
+		return IRQ_NONE;
+
 	dev_err_ratelimited(smmu->dev,
 		"Unexpected global fault, this could be serious\n");
 	dev_err_ratelimited(smmu->dev,
@@ -1634,8 +1645,8 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 	u32 reg;
 
 	/* clear global FSR */
-	reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sGFSR);
-	writel(reg, gr0_base + ARM_SMMU_GR0_sGFSR);
+	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
+	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sGFSR);
 
 	/* Mark all SMRn as invalid and all S2CRn as bypass */
 	for (i = 0; i < smmu->num_mapping_groups; ++i) {
@@ -1655,7 +1666,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 	writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLH);
 	writel_relaxed(0, gr0_base + ARM_SMMU_GR0_TLBIALLNSNH);
 
-	reg = readl_relaxed(gr0_base + ARM_SMMU_GR0_sCR0);
+	reg = readl_relaxed(ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
 
 	/* Enable fault reporting */
 	reg |= (sCR0_GFRE | sCR0_GFIE | sCR0_GCFGFRE | sCR0_GCFGFIE);
@@ -1674,7 +1685,7 @@ static void arm_smmu_device_reset(struct arm_smmu_device *smmu)
 
 	/* Push the button */
 	arm_smmu_tlb_sync(smmu);
-	writel(reg, gr0_base + ARM_SMMU_GR0_sCR0);
+	writel(reg, ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
 }
 
 static int arm_smmu_id_size_to_bits(int size)
@@ -2012,7 +2023,8 @@ static int arm_smmu_device_remove(struct platform_device *pdev)
 		free_irq(smmu->irqs[i], smmu);
 
 	/* Turn the thing off */
-	writel(sCR0_CLIENTPD, ARM_SMMU_GR0(smmu) + ARM_SMMU_GR0_sCR0);
+	writel(sCR0_CLIENTPD,
+		ARM_SMMU_GR0_NS(smmu) + ARM_SMMU_GR0_sCR0);
 	return 0;
 }
 
Signature · 1 line
1.7.9.5
#16 lore raw

PATCH 5/5 ARM: dts: Add nodes for SMMUs on Calxeda ECX-2000

In reply to Andreas Herrmann
---
 arch/arm/boot/dts/ecx-2000.dts    |   45 +++++++++++++++++++++++++++++++++++--
 arch/arm/boot/dts/ecx-common.dtsi |    9 +++++---
 2 files changed, 49 insertions(+), 5 deletions(-)
arch/arm/boot/dts/ecx-2000.dts +43 -2
index 139b40c..4444905 100644
@@ -76,10 +76,11 @@
 	};
 
 	soc {
-		ranges = <0x00000000 0x00000000 0x00000000 0xffffffff>;
+		ranges = <0x0 0x0 0x0 0xffffffff>;
 
 		timer {
-			compatible = "arm,cortex-a15-timer", "arm,armv7-timer"; 			interrupts = <1 13 0xf08>,
+			compatible = "arm,cortex-a15-timer", "arm,armv7-timer";
+			interrupts = <1 13 0xf08>,
 				<1 14 0xf08>,
 				<1 11 0xf08>,
 				<1 10 0xf08>;
@@ -103,6 +104,46 @@
 			interrupts = <0 76 4  0 75 4  0 74 4  0 73 4>;
 		};
 	};
+
+	soc at 920000000 {
+		ranges = <0x9 0x20000000 0x9 0x20000000 0x290000>;
+		#address-cells = <2>;
+		#size-cells = <1>;
+		compatible = "simple-bus";
+		interrupt-parent = <&intc>;
+
+		smmu_mac0: smmu at 920000000 {
+			compatible = "arm,mmu-400";
+			reg = <0x9 0x20000000 0x10000>;
+			#global-interrupts = <1>;
+			interrupts = <0 106 4 0 106 4>;
+			mmu-masters = <&mac0 0x0 0x1>;
+			arm,smmu-secure-config-access;
+			arm,smmu-isolate-devices;
+		};
+
+		smmu_mac1: smmu at 920080000 {
+			compatible = "arm,mmu-400";
+			reg = <0x9 0x20080000 0x10000>;
+			#global-interrupts = <1>;
+			interrupts = <0 108 4 0 108 4>;
+			mmu-masters = <&mac1 0x0 0x1>;
+			arm,smmu-secure-config-access;
+			arm,smmu-isolate-devices;
+		};
+
+		smmu_sata: smmu at 920180000 {
+			compatible = "arm,mmu-400";
+			reg = <0x00000009 0x20180000 0x10000>;
+			mmu-masters = <&sata>;
+			#global-interrupts = <1>;
+			interrupts = <0 114 4 0 114 4>;
+			arm,smmu-secure-config-access;
+			arm,smmu-isolate-devices;
+			arm,smmu-mask-stream-ids;
+		};
+	};
+
 };
 
 /include/ "ecx-common.dtsi"
arch/arm/boot/dts/ecx-common.dtsi +6 -3
index e8559b7..50e401e 100644
@@ -25,7 +25,7 @@
 		compatible = "simple-bus";
 		interrupt-parent = <&intc>;
 
-		sata at ffe08000 {
+		sata: sata at ffe08000 {
 			compatible = "calxeda,hb-ahci";
 			reg = <0xffe08000 0x10000>;
 			interrupts = <0 83 4>;
@@ -35,6 +35,7 @@
 					     &combophy0 3>;
 			calxeda,sgpio-gpio =<&gpioh 5 1 &gpioh 6 1 &gpioh 7 1>;
 			calxeda,led-order = <4 0 1 2 3>;
+			#stream-id-cells = <0>;
 		};
 
 		sdhci at ffe0e000 {
@@ -208,18 +209,20 @@
 			clock-names = "apb_pclk";
 		};
 
-		ethernet at fff50000 {
+		mac0: ethernet at fff50000 {
 			compatible = "calxeda,hb-xgmac";
 			reg = <0xfff50000 0x1000>;
 			interrupts = <0 77 4  0 78 4  0 79 4>;
 			dma-coherent;
+			#stream-id-cells = <2>;
 		};
 
-		ethernet at fff51000 {
+		mac1: ethernet at fff51000 {
 			compatible = "calxeda,hb-xgmac";
 			reg = <0xfff51000 0x1000>;
 			interrupts = <0 80 4  0 81 4  0 82 4>;
 			dma-coherent;
+			#stream-id-cells = <2>;
 		};
 
 		combophy0: combo-phy at fff58000 {
Signature · 1 line
1.7.9.5
#17 lore raw

PATCH 1/1 documentation/iommu: Update description of ARM System MMU binding

In reply to Andreas Herrmann

This patch adds descriptions fore new properties of device tree binding for the ARM SMMU architecture. These properties control arm-smmu driver options.

Cc Rob Herring <[email protected]>
Cc Grant Likely <[email protected]>
Cc Will Deacon <[email protected]>
---
 .../devicetree/bindings/iommu/arm,smmu.txt         |   23 ++++++++++++++++++++
 1 file changed, 23 insertions(+)
Documentation/devicetree/bindings/iommu/arm,smmu.txt +23 -0
index e34c6cd..6415a88 100644
@@ -48,6 +48,28 @@ conditions.
                   from the mmu-masters towards memory) node for this
                   SMMU.
 
+- arm,smmu-isolate-devices : Enable device isolation for all masters
+                             of this SMMU. Ie. each master will be
+                             attached to its own iommu domain.
+
+- arm,smmu-mask-stream-ids : Enable mapping of all StreamIDs to one
+                             context for this SMMU. It is only allowed
+                             if there is one single master. It saves
+                             the need to know and specify all
+                             StreamIDs in cases where just one master
+                             is attached to an SMMU. If StreamIDs are
+                             specified in the mmu-masters property
+                             those are ignored. (The referred device
+                             node must still have a "#stream-id-cells"
+                             property. But it can be set to 0.)
+
+- arm,smmu-secure-config-access : Enable proper handling of buggy
+                                  implementations that always use
+                                  secure access to SMMU configuration
+                                  registers. In this case non-secure
+                                  aliases of secure registers have to
+                                  be used during SMMU configuration.
+
 Example:
 
         smmu {
@@ -67,4 +89,5 @@ Example:
                  */
                 mmu-masters = <&dma0 0xd01d 0xd01e>,
                               <&dma1 0xd11c>;
+                arm,smmu-isolate-devices;
         };
Signature · 1 line
1.7.9.5