IOMMU driver improvements for modern Exynos SysMMUs

Newer SysMMU v7+ instances can lack BLOCK mode: CAPA1 bit 15 reports that the CTRL_BLOCK function is not implemented, and MMU_STATUS never reports a blocked state. The SysMMUs on Exynos 8835 are such instances.

The driver currently assumes blocking always works, with two consequences on such hardware:

The enable path writes CTRL_BLOCK first, which there acts as a plain enable and starts translation before the page table base is programmed.

Worse, both TLB invalidation paths gate the invalidation writes on sysmmu_block() succeeding, which it never does - so every unmap silently skips the invalidation. A stale TLB entry is a valid entry pointing at a freed page, so nothing ever faults: the device reads back garbage and its writebacks corrupt whatever the kernel has since reused those pages for.

This was tracked down on Exynos 8835 with the MFC, where the first decoder session of a boot worked and later sessions produced garbage along with random kernel memory corruption. Patches 1-3 add detection of the capability bit and adapt the enable sequence and the invalidation paths, matching the vendor driver's handling of these parts. Patch 4 is an independent debugging
improvement: decode the v7 fault transaction info word (AxID/AxLEN), which identifies the issuing port when a master containing several DMA engines faults.

Tested on the Samsung Galaxy Tab S9 FE (Exynos 8835/Exynos 1380).

---
Markuss Broks (4):
iommu/exynos: detect SysMMUs without BLOCK mode iommu/exynos: fix the enable sequence for no-block SysMMUs iommu/exynos: fix TLB invalidation for no-block SysMMUs iommu/exynos: decode the v7 fault transaction info

 drivers/iommu/exynos-iommu.c | 37 +++++++++++++++++++++++++++++++++----
 1 file changed, 33 insertions(+), 4 deletions(-)

---
base-commit: 415606a7be939835db9b0d6b711887586646346d
change-id: 20260820-exynos-iommu-fixes-e0e4d8f0fc06

Best regards,

Markuss Broks <[email protected]>

Newer SysMMU instances may not implement BLOCK mode: writing CTRL_BLOCK does not stop translation, and MMU_STATUS never reports a blocked state. The hardware advertises this in CAPA1 bit 15, which the vendor driver reads as MMU_CAPA1_NO_BLOCK_MODE; the SysMMUs on Exynos8835 are such instances.

---
 drivers/iommu/exynos-iommu.c | 3 +++
 1 file changed, 3 insertions(+)

index 874d05f4b396..0319da9fd831 100644
@@ -143,6 +143,7 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
 #define CTRL_VM_FAULT_MODE_STALL	BIT(3)
 #define CAPA0_CAPA1_EXIST		BIT(11)
 #define CAPA1_VCR_ENABLED		BIT(14)
+#define CAPA1_NO_BLOCK_MODE		BIT(15)
 
 /* common registers */
 #define REG_MMU_CTRL		0x000

@@ -306,6 +307,7 @@ struct sysmmu_drvdata {
 
 	/* v7 fields */
 	bool has_vcr;			/* virtual machine control register */
+	bool no_block;			/* BLOCK mode not implemented */
 };
 
 #define SYSMMU_REG(data, reg) ((data)->sfrbase + (data)->variant->reg)

@@ -511,6 +513,7 @@ static void __sysmmu_get_vcr(struct sysmmu_drvdata *data)
 	u32 capa1 = readl(data->sfrbase + REG_V7_CAPA1);
 
 	data->has_vcr = capa1 & CAPA1_VCR_ENABLED;
+	data->no_block = capa1 & CAPA1_NO_BLOCK_MODE;
 }
 
 static void __sysmmu_get_version(struct sysmmu_drvdata *data)

2.55.0

__sysmmu_enable() writes CTRL_BLOCK before programming CFG, the page table base and the VM registers. CTRL_BLOCK has the enable bit set, so on hardware without BLOCK mode this write immediately starts translation with whatever FLPT base the registers hold: reset values on the first enable, a stale page table on re-enable. A master that is already emitting traffic at that point gets its transactions translated through that garbage.

Keep the MMU disabled while it is being programmed on such hardware; the final CTRL_ENABLE write then brings it up with a consistent configuration in one step, which is also the same as the vendor driver sequence.

---
 drivers/iommu/exynos-iommu.c | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

index 0319da9fd831..a3a59d8a4cf1 100644
@@ -655,7 +655,12 @@ static void __sysmmu_enable(struct sysmmu_drvdata *data)
 	__sysmmu_enable_clocks(data);
 
 	spin_lock_irqsave(&data->lock, flags);
-	writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
+	/*
+	 * On no-block hardware CTRL_BLOCK acts as a plain enable; keep the
+	 * MMU disabled until it is fully programmed.
+	 */
+	if (!data->no_block)
+		writel(CTRL_BLOCK, data->sfrbase + REG_MMU_CTRL);
 	__sysmmu_init_config(data);
 	__sysmmu_set_ptbase(data, data->pgtable);
 	__sysmmu_enable_vid(data);

2.55.0

sysmmu_block() polls MMU_STATUS for the blocked state, which no-block hardware never reports, so on such SysMMUs it always fails, and both sysmmu_tlb_invalidate_entry() and sysmmu_tlb_invalidate_flpdcache() skip the invalidation entirely when blocking fails.

On no-block hardware the invalidation registers are written with the MMU running; the vendor driver never blocks around invalidation. Do the same and write the invalidation directly. In the FLPD cache path no-block implies a v7+ SysMMU, hence the unconditional flush-all.

---
 drivers/iommu/exynos-iommu.c | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

index a3a59d8a4cf1..29b27c7e400a 100644
@@ -685,7 +685,14 @@ static void sysmmu_tlb_invalidate_flpdcache(struct sysmmu_drvdata *data,
 	spin_lock_irqsave(&data->lock, flags);
 	if (data->active && data->version >= MAKE_MMU_VER(3, 3)) {
 		clk_enable(data->clk_master);
-		if (sysmmu_block(data)) {
+		/*
+		 * No-block hardware accepts invalidation writes while the
+		 * MMU is running; it is also v7+, so flush-all covers the
+		 * FLPD cache.
+		 */
+		if (data->no_block) {
+			__sysmmu_tlb_invalidate(data);
+		} else if (sysmmu_block(data)) {
 			if (data->version >= MAKE_MMU_VER(5, 0))
 				__sysmmu_tlb_invalidate(data);
 			else

@@ -721,7 +728,9 @@ static void sysmmu_tlb_invalidate_entry(struct sysmmu_drvdata *data,
 		if (MMU_MAJ_VER(data->version) == 2)
 			num_inv = min_t(unsigned int, size / SPAGE_SIZE, 64);
 
-		if (sysmmu_block(data)) {
+		if (data->no_block) {
+			__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
+		} else if (sysmmu_block(data)) {
 			__sysmmu_tlb_invalidate_entry(data, iova, num_inv);
 			sysmmu_unblock(data);
 		}

2.55.0

The v7+ fault registers carry a transaction-info word alongside the faulting address, but it is only used to derive the read/write direction and then thrown away. Keep it and print it: AxID identifies which port inside the master issued the faulting transaction, which is the only way to tell apart the several DMA engines a single block can contain, and the remaining bits of the raw word are implementation defined and worth having in a fault report.

Define the known fields of the word instead of open-coding the masks; this also names the direction bit the driver was already testing as a bare BIT(20).

Gate the print on the variant having a fault_info register, so older SysMMUs are unaffected.

---
 drivers/iommu/exynos-iommu.c | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

index 29b27c7e400a..0bca3662fdc2 100644
@@ -8,6 +8,7 @@
 #define DEBUG
 #endif
 
+#include <linux/bitfield.h>
 #include <linux/clk.h>
 #include <linux/dma-mapping.h>
 #include <linux/err.h>

@@ -144,6 +145,9 @@ static u32 lv2ent_offset(sysmmu_iova_t iova)
 #define CAPA0_CAPA1_EXIST		BIT(11)
 #define CAPA1_VCR_ENABLED		BIT(14)
 #define CAPA1_NO_BLOCK_MODE		BIT(15)
+#define FAULT_INFO_AXID			GENMASK(15, 0)
+#define FAULT_INFO_AXLEN		GENMASK(19, 16)
+#define FAULT_INFO_WRITE		BIT(20)
 
 /* common registers */
 #define REG_MMU_CTRL		0x000

@@ -194,6 +198,7 @@ struct sysmmu_fault {
 	sysmmu_iova_t addr;	/* IOVA address that caused fault */
 	const char *name;	/* human readable fault name */
 	unsigned int type;	/* fault type for report_iommu_fault() */
+	u32 info;		/* raw transaction info (v7+ only) */
 };
 
 struct sysmmu_v1_fault_info {

@@ -360,7 +365,8 @@ static int exynos_sysmmu_v7_get_fault_info(struct sysmmu_drvdata *data,
 
 	fault->addr = readl(SYSMMU_REG(data, fault_va));
 	fault->name = sysmmu_v7_fault_names[itype % 4];
-	fault->type = (info & BIT(20)) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
+	fault->type = (info & FAULT_INFO_WRITE) ? IOMMU_FAULT_WRITE : IOMMU_FAULT_READ;
+	fault->info = info;
 
 	return 0;
 }

@@ -558,6 +564,12 @@ static void show_fault_information(struct sysmmu_drvdata *data,
 		dev_name(data->master),
 		fault->type == IOMMU_FAULT_READ ? "READ" : "WRITE",
 		fault->name, fault->addr);
+	/* AxID identifies the issuing port inside the master */
+	if (data->variant->fault_info)
+		dev_err(data->sysmmu, "transaction info %#010x: AxID %#lx, AxLEN %lu\n",
+			fault->info,
+			FIELD_GET(FAULT_INFO_AXID, fault->info),
+			FIELD_GET(FAULT_INFO_AXLEN, fault->info));
 	dev_dbg(data->sysmmu, "Page table base: %pa\n", &data->pgtable);
 	ent = section_entry(phys_to_virt(data->pgtable), fault->addr);
 	dev_dbg(data->sysmmu, "\tLv1 entry: %#x\n", *ent);

2.55.0

For the whole patchset:

Best regards

Marek Szyprowski, PhD
Samsung R&D Institute Poland

Applied, thanks.