Skip to contentSkip to conversation

iommu/vt-d: Reset DMAR_UNITS_SUPPORTED

PATCH Superseded Lu Baolu 3 participants 19 messages Jun 25, 2022 – Jul 6, 2022

Subsystems (2)

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore

Lu Baolu

author maintainer
#0 lore raw

cover iommu/vt-d: Reset DMAR_UNITS_SUPPORTED

Hi folks,

This is a follow-up series of changes proposed by this patch:

https://lore.kernel.org/linux-iommu/[email protected]/

It removes several static arrays of size DMAR_UNITS_SUPPORTED and sets the DMAR_UNITS_SUPPORTED to 1024.

Please help review and suggest.

Best regards,
baolu

Lu Baolu (6):
iommu/vt-d: Remove unused domain_get_iommu() iommu/vt-d: Use IDA interface to manage iommu sequence id iommu/vt-d: Refactor iommu information of each domain iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag iommu/vt-d: Remove global g_iommus array iommu/vt-d: Make DMAR_UNITS_SUPPORTED default 1024

 include/linux/dmar.h        |   6 +-
 drivers/iommu/intel/iommu.h |  29 ++++--
 drivers/iommu/intel/dmar.c  |  33 ++-----
 drivers/iommu/intel/iommu.c | 188 ++++++++++++++----------------------
 drivers/iommu/intel/pasid.c |   2 +-
 drivers/iommu/intel/svm.c   |   2 +-
 6 files changed, 103 insertions(+), 157 deletions(-)
Signature · 1 line
2.25.1
author maintainer
#1 lore raw

PATCH 1/6 iommu/vt-d: Remove unused domain_get_iommu()

It is not used anywhere. Remove it to avoid dead code.

---
 drivers/iommu/intel/iommu.h |  1 -
 drivers/iommu/intel/iommu.c | 18 ------------------
 2 files changed, 19 deletions(-)
drivers/iommu/intel/iommu.h +0 -1
index df64d3d9c49a..56e0d8cd2102 100644
@@ -725,7 +725,6 @@ extern int dmar_ir_support(void);
 
 void *alloc_pgtable_page(int node);
 void free_pgtable_page(void *vaddr);
-struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
 void iommu_flush_write_buffer(struct intel_iommu *iommu);
 int intel_iommu_enable_pasid(struct intel_iommu *iommu, struct device *dev);
 struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn);
drivers/iommu/intel/iommu.c +0 -18
index da6cfea0f0d6..781e060352e6 100644
@@ -445,24 +445,6 @@ int iommu_calculate_agaw(struct intel_iommu *iommu)
 	return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
 }
 
-/* This functionin only returns single iommu in a domain */
-struct intel_iommu *domain_get_iommu(struct dmar_domain *domain)
-{
-	int iommu_id;
-
-	/* si_domain and vm domain should not get here. */
-	if (WARN_ON(!iommu_is_dma_domain(&domain->domain)))
-		return NULL;
-
-	for_each_domain_iommu(iommu_id, domain)
-		break;
-
-	if (iommu_id < 0 || iommu_id >= g_num_of_iommus)
-		return NULL;
-
-	return g_iommus[iommu_id];
-}
-
 static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
 {
 	return sm_supported(iommu) ?
Signature · 1 line
2.25.1
author maintainer
#3 lore raw

PATCH 2/6 iommu/vt-d: Use IDA interface to manage iommu sequence id

In reply to Lu Baolu

Switch dmar unit sequence id allocation and release from bitmap to IDA interface.

---
 drivers/iommu/intel/dmar.c | 33 +++++++--------------------------
 1 file changed, 7 insertions(+), 26 deletions(-)
drivers/iommu/intel/dmar.c +7 -26
index 2a5e0f91e647..bf43889b9d2a 100644
@@ -60,7 +60,7 @@ LIST_HEAD(dmar_drhd_units);
 
 struct acpi_table_header * __initdata dmar_tbl;
 static int dmar_dev_scope_status = 1;
-static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
+static DEFINE_IDA(dmar_seq_ids);
 
 static int alloc_iommu(struct dmar_drhd_unit *drhd);
 static void free_iommu(struct intel_iommu *iommu);
@@ -1023,28 +1023,6 @@ static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 	return err;
 }
 
-static int dmar_alloc_seq_id(struct intel_iommu *iommu)
-{
-	iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
-					    DMAR_UNITS_SUPPORTED);
-	if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
-		iommu->seq_id = -1;
-	} else {
-		set_bit(iommu->seq_id, dmar_seq_ids);
-		sprintf(iommu->name, "dmar%d", iommu->seq_id);
-	}
-
-	return iommu->seq_id;
-}
-
-static void dmar_free_seq_id(struct intel_iommu *iommu)
-{
-	if (iommu->seq_id >= 0) {
-		clear_bit(iommu->seq_id, dmar_seq_ids);
-		iommu->seq_id = -1;
-	}
-}
-
 static int alloc_iommu(struct dmar_drhd_unit *drhd)
 {
 	struct intel_iommu *iommu;
@@ -1062,11 +1040,14 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
 	if (!iommu)
 		return -ENOMEM;
 
-	if (dmar_alloc_seq_id(iommu) < 0) {
+	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
+					DMAR_UNITS_SUPPORTED, GFP_KERNEL);
+	if (iommu->seq_id < 0) {
 		pr_err("Failed to allocate seq_id\n");
 		err = -ENOSPC;
 		goto error;
 	}
+	sprintf(iommu->name, "dmar%d", iommu->seq_id);
 
 	err = map_iommu(iommu, drhd->reg_base_addr);
 	if (err) {
@@ -1150,7 +1131,7 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
 err_unmap:
 	unmap_iommu(iommu);
 error_free_seq_id:
-	dmar_free_seq_id(iommu);
+	ida_free(&dmar_seq_ids, iommu->seq_id);
 error:
 	kfree(iommu);
 	return err;
@@ -1183,7 +1164,7 @@ static void free_iommu(struct intel_iommu *iommu)
 	if (iommu->reg)
 		unmap_iommu(iommu);
 
-	dmar_free_seq_id(iommu);
+	ida_free(&dmar_seq_ids, iommu->seq_id);
 	kfree(iommu);
 }
 
Signature · 1 line
2.25.1
#4 lore raw

RE: [PATCH v1 2/6] iommu/vt-d: Use IDA interface to manage iommu sequence id

drivers/iommu/intel/dmar.c static int alloc_iommu(struct dmar_drhd_unit · line 1045

Quoted 11 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

drivers/iommu/intel/dmar.c +2 -1
@@ -1062,11 +1040,14 @@ static int alloc_iommu(struct dmar_drhd_unit
 *drhd)
 	if (!iommu)
 		return -ENOMEM;
 
-	if (dmar_alloc_seq_id(iommu) < 0) {
+	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
+					DMAR_UNITS_SUPPORTED,

should be "DMAR_UNITS_SUPPORTED - 1"

drivers/iommu/intel/dmar.c static int alloc_iommu(struct dmar_drhd_unit *drhd) · line 1049

Quoted 6 lines
drivers/iommu/intel/dmar.c +1 -0
 GFP_KERNEL);
+	if (iommu->seq_id < 0) {
 		pr_err("Failed to allocate seq_id\n");
 		err = -ENOSPC;
 		goto error;
 	}

ida_alloc_range() returns error code already. No need to change it.

author maintainer
#5 lore raw

Re: [PATCH v1 2/6] iommu/vt-d: Use IDA interface to manage iommu sequence id

Quoted 8 lines · Kevin
Quoted 8 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

(quoted diff) +0 -0
@@ -1062,11 +1040,14 @@ static int alloc_iommu(struct dmar_drhd_unit
 *drhd)
  	if (!iommu)
  		return -ENOMEM;
 
Quoted 5 lines

Review static int alloc_iommu(struct dmar_drhd_unit · line 1045

Quoted 3 lines
(quoted diff) +2 -1
-	if (dmar_alloc_seq_id(iommu) < 0) {
+	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
+					DMAR_UNITS_SUPPORTED,

should be "DMAR_UNITS_SUPPORTED - 1"

Yes, according to "@max: Highest ID to allocate.". Updated.

Quoted 9 lines

Review static int alloc_iommu(struct dmar_drhd_unit · line 1051

Quoted 6 lines
(quoted diff) +1 -0
 GFP_KERNEL);
+	if (iommu->seq_id < 0) {
  		pr_err("Failed to allocate seq_id\n");
  		err = -ENOSPC;
  		goto error;
  	}

ida_alloc_range() returns error code already. No need to change it.

Agreed. Updated as below:

drivers/iommu/intel/dmar.c +2 -2
index bf43889b9d2a..6327b34f5aa7 100644
@@ -1041,10 +1041,10 @@ static int alloc_iommu(struct dmar_drhd_unit *drhd)
                return -ENOMEM;
 
        iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
-                                       DMAR_UNITS_SUPPORTED, GFP_KERNEL);
+                                       DMAR_UNITS_SUPPORTED - 1, GFP_KERNEL);
        if (iommu->seq_id < 0) {
                pr_err("Failed to allocate seq_id\n");
-               err = -ENOSPC;
+               err = iommu->seq_id;
                goto error;
        }
        sprintf(iommu->name, "dmar%d", iommu->seq_id);

Best regards,
baolu

author maintainer
#6 lore raw

PATCH 3/6 iommu/vt-d: Refactor iommu information of each domain

In reply to Lu Baolu

When a DMA domain is attached to a device, it needs to allocate a domain ID from its IOMMU. Currently, the domain ID information is stored in two static arrays embedded in the domain structure. This can lead to memory waste when the driver is running on a small platform.

This optimizes these static arrays by replacing them with an xarray and consuming memory on demand.

---
 drivers/iommu/intel/iommu.h |  27 +++++---
 drivers/iommu/intel/iommu.c | 123 ++++++++++++++++++++----------------
 drivers/iommu/intel/pasid.c |   2 +-
 drivers/iommu/intel/svm.c   |   2 +-
 4 files changed, 90 insertions(+), 64 deletions(-)
drivers/iommu/intel/iommu.h +18 -9
index 56e0d8cd2102..56c3d1a9e155 100644
@@ -21,6 +21,7 @@
 #include <linux/dmar.h>
 #include <linux/ioasid.h>
 #include <linux/bitfield.h>
+#include <linux/xarray.h>
 
 #include <asm/cacheflush.h>
 #include <asm/iommu.h>
@@ -524,17 +525,15 @@ struct context_entry {
  */
 #define DOMAIN_FLAG_USE_FIRST_LEVEL		BIT(1)
 
+struct iommu_domain_info {
+	struct intel_iommu *iommu;
+	unsigned int refcnt;
+	u16 did;
+};
+
 struct dmar_domain {
 	int	nid;			/* node id */
-
-	unsigned int iommu_refcnt[DMAR_UNITS_SUPPORTED];
-					/* Refcount of devices per iommu */
-
-
-	u16		iommu_did[DMAR_UNITS_SUPPORTED];
-					/* Domain ids per IOMMU. Use u16 since
-					 * domain ids are 16 bit wide according
-					 * to VT-d spec, section 9.3 */
+	struct xarray iommu_array;	/* Attached IOMMU array */
 
 	u8 has_iotlb_device: 1;
 	u8 iommu_coherency: 1;		/* indicate coherency of iommu access */
@@ -640,6 +639,16 @@ static inline struct dmar_domain *to_dmar_domain(struct iommu_domain *dom)
 	return container_of(dom, struct dmar_domain, domain);
 }
 
+/* Retrieve the domain ID which has allocated to the domain */
+static inline u16
+domain_id_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
+{
+	struct iommu_domain_info *info =
+			xa_load(&domain->iommu_array, iommu->seq_id);
+
+	return info->did;
+}
+
 /*
  * 0: readable
  * 1: writable
drivers/iommu/intel/iommu.c +70 -53

237 lines not on this page · raw message

drivers/iommu/intel/pasid.c +1 -1
index c30de2339de5..006026dc18d4 100644
@@ -626,7 +626,7 @@ int intel_pasid_setup_second_level(struct intel_iommu *iommu,
 	}
 
 	pgd_val = virt_to_phys(pgd);
-	did = domain->iommu_did[iommu->seq_id];
+	did = domain_id_iommu(domain, iommu);
 
 	spin_lock(&iommu->lock);
 	pte = intel_pasid_get_entry(dev, pasid);
drivers/iommu/intel/svm.c +1 -1
index 64072e628bbd..8bcfb93dda56 100644
@@ -541,7 +541,7 @@ static void intel_svm_drain_prq(struct device *dev, u32 pasid)
 	domain = info->domain;
 	pdev = to_pci_dev(dev);
 	sid = PCI_DEVID(info->bus, info->devfn);
-	did = domain->iommu_did[iommu->seq_id];
+	did = domain_id_iommu(domain, iommu);
 	qdep = pci_ats_queue_depth(pdev);
 
 	/*
Signature · 1 line
2.25.1
#7 lore raw

RE: [PATCH v1 3/6] iommu/vt-d: Refactor iommu information of each domain

Quoted 21 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

drivers/iommu/intel/iommu.h +6 -7
+struct iommu_domain_info {
+	struct intel_iommu *iommu;
+	unsigned int refcnt;
+	u16 did;
+};
+
 struct dmar_domain {
 	int	nid;			/* node id */
-
-	unsigned int iommu_refcnt[DMAR_UNITS_SUPPORTED];
-					/* Refcount of devices per iommu */
-
-
-	u16		iommu_did[DMAR_UNITS_SUPPORTED];
-					/* Domain ids per IOMMU. Use u16

since

(quoted diff) +0 -1
-					 * domain ids are 16 bit wide

according

drivers/iommu/intel/iommu.h struct context_entry { · line 537

Quoted 1 line
drivers/iommu/intel/iommu.h +0 -1
-					 * to VT-d spec, section 9.3 */

It'd make sense to keep the comments when moving above fields.

drivers/iommu/intel/iommu.c static struct dmar_domain *alloc_domain(unsigned int type) · line 1766

Quoted 6 lines
drivers/iommu/intel/iommu.c +6 -0
+	spin_lock(&iommu->lock);
+	curr = xa_load(&domain->iommu_array, iommu->seq_id);
+	if (curr) {
+		curr->refcnt++;
+		kfree(info);
+		goto success;

Not a fan of adding a label for success. Just putting two lines (unlock+ return) here is clearer.

drivers/iommu/intel/iommu.c static struct dmar_domain *alloc_domain(unsigned int type) · line 1782

Quoted 2 lines
drivers/iommu/intel/iommu.c +2 -0
+	ret = xa_err(xa_store(&domain->iommu_array, iommu->seq_id,
+			      info, GFP_ATOMIC));

check xa_err in separate line.

Quoted 9 lines

static void domain_detach_iommu(struct dmar_domain *domain,

drivers/iommu/intel/iommu.c +1 -2
 				struct intel_iommu *iommu)
 {
-	int num;
+	struct iommu_domain_info *info;
 
 	spin_lock(&iommu->lock);
-	domain->iommu_refcnt[iommu->seq_id] -= 1;

drivers/iommu/intel/iommu.c static struct dmar_domain *alloc_domain(unsigned int type) · line 1810

Quoted 10 lines
drivers/iommu/intel/iommu.c +5 -4
-	if (domain->iommu_refcnt[iommu->seq_id] == 0) {
-		num = domain->iommu_did[iommu->seq_id];
-		clear_bit(num, iommu->domain_ids);
+	info = xa_load(&domain->iommu_array, iommu->seq_id);
+	if (--info->refcnt == 0) {
+		clear_bit(info->did, iommu->domain_ids);
+		xa_erase(&domain->iommu_array, iommu->seq_id);
 		domain_update_iommu_cap(domain);
-		domain->iommu_did[iommu->seq_id] = 0;
+		kfree(info);

domain->nid may still point to the node of the removed iommu.

author maintainer
#8 lore raw

Re: [PATCH v1 3/6] iommu/vt-d: Refactor iommu information of each domain

Quoted 16 lines · Kevin
Quoted 16 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

(quoted diff) +6 -5
+struct iommu_domain_info {
+	struct intel_iommu *iommu;
+	unsigned int refcnt;
+	u16 did;
+};
+
  struct dmar_domain {
  	int	nid;			/* node id */
-
-	unsigned int iommu_refcnt[DMAR_UNITS_SUPPORTED];
-					/* Refcount of devices per iommu */
-
-
Quoted 8 lines

Review

Quoted 6 lines
(quoted diff) +0 -2
-	u16		iommu_did[DMAR_UNITS_SUPPORTED];
-					/* Domain ids per IOMMU. Use u16

since

(quoted diff) +0 -1
-					 * domain ids are 16 bit wide

according

(quoted diff) +0 -1
-					 * to VT-d spec, section 9.3 */

It'd make sense to keep the comments when moving above fields.

Sure. Updated.

drivers/iommu/intel/iommu.h +4 -2
index 56c3d1a9e155..fae45bbb0c7f 100644
@@ -527,8 +527,10 @@ struct context_entry {
 
 struct iommu_domain_info {
        struct intel_iommu *iommu;
-       unsigned int refcnt;
-       u16 did;
+       unsigned int refcnt;            /* Refcount of devices per iommu */
+       u16 did;                        /* Domain ids per IOMMU. Use u16 since
+                                        * domain ids are 16 bit wide according
+                                        * to VT-d spec, section 9.3 */
 };
 
 
Quoted 10 lines

Review

Quoted 7 lines
(quoted diff) +6 -0
+	spin_lock(&iommu->lock);
+	curr = xa_load(&domain->iommu_array, iommu->seq_id);
+	if (curr) {
+		curr->refcnt++;
+		kfree(info);
+		goto success;

Not a fan of adding a label for success. Just putting two lines (unlock+ return) here is clearer.

Updated.

Quoted 5 lines

Review

Quoted 3 lines
(quoted diff) +2 -0
+	ret = xa_err(xa_store(&domain->iommu_array, iommu->seq_id,
+			      info, GFP_ATOMIC));

check xa_err in separate line.

Sure. Updated as below:

@@ -1778,13 +1780,14 @@ static int domain_attach_iommu(struct dmar_domain *domain,

        info->did       = num;
        info->iommu     = iommu;
        domain->nid     = iommu->node;

- ret = xa_err(xa_store(&domain->iommu_array, iommu->seq_id,
- info, GFP_ATOMIC));
- if (ret)
+ curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
+ NULL, info, GFP_ATOMIC);
+ if (curr) {

+               ret = xa_err(curr) ? : -EBUSY;
                goto err_clear;
+       }
        domain_update_iommu_cap(domain);
Quoted 8 lines
Quoted 8 lines

static void domain_detach_iommu(struct dmar_domain *domain,
struct intel_iommu *iommu)
{

(quoted diff) +1 -1
-	int num;
+	struct iommu_domain_info *info;
Quoted 14 lines

Review

Quoted 12 lines
  	spin_lock(&iommu->lock);
(quoted diff) +5 -5
-	domain->iommu_refcnt[iommu->seq_id] -= 1;
-	if (domain->iommu_refcnt[iommu->seq_id] == 0) {
-		num = domain->iommu_did[iommu->seq_id];
-		clear_bit(num, iommu->domain_ids);
+	info = xa_load(&domain->iommu_array, iommu->seq_id);
+	if (--info->refcnt == 0) {
+		clear_bit(info->did, iommu->domain_ids);
+		xa_erase(&domain->iommu_array, iommu->seq_id);
  		domain_update_iommu_cap(domain);
-		domain->iommu_did[iommu->seq_id] = 0;
+		kfree(info);

domain->nid may still point to the node of the removed iommu.

Good catch! I should assign it to NUMA_NO_NODE, so that it could be updated in the next domain_update_iommu_cap(). Updated as below:

@@ -1806,6 +1809,7 @@ static void domain_detach_iommu(struct dmar_domain *domain,

        if (--info->refcnt == 0) {
                clear_bit(info->did, iommu->domain_ids);
                xa_erase(&domain->iommu_array, iommu->seq_id);
+               domain->nid = NUMA_NO_NODE;
                domain_update_iommu_cap(domain);
                kfree(info);
        }

Best regards,
baolu

author maintainer
#9 lore raw

PATCH 4/6 iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag

In reply to Lu Baolu

In the IOMMU hot-add path, there's a need to check whether an IOMMU has been probed. Instead of checking the IOMMU pointer in the global list, it's better to allocate a flag bit in iommu->flags for this purpose.

---
 drivers/iommu/intel/iommu.h | 1 +
 drivers/iommu/intel/iommu.c | 5 ++++-
 2 files changed, 5 insertions(+), 1 deletion(-)
drivers/iommu/intel/iommu.h +1 -0
index 56c3d1a9e155..105a1e7c60d9 100644
@@ -479,6 +479,7 @@ enum {
 #define VTD_FLAG_TRANS_PRE_ENABLED	(1 << 0)
 #define VTD_FLAG_IRQ_REMAP_PRE_ENABLED	(1 << 1)
 #define VTD_FLAG_SVM_CAPABLE		(1 << 2)
+#define VTD_FLAG_IOMMU_PROBED		(1 << 3)
 
 extern int intel_iommu_sm;
 
drivers/iommu/intel/iommu.c +4 -1
index 78b26fef685e..f6d7055cffd7 100644
@@ -1695,6 +1695,7 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
 	}
 
 	g_iommus[iommu->seq_id] = NULL;
+	iommu->flags &= ~VTD_FLAG_IOMMU_PROBED;
 
 	/* free context mapping */
 	free_context_table(iommu);
@@ -2951,6 +2952,7 @@ static int __init init_dmars(void)
 		}
 
 		g_iommus[iommu->seq_id] = iommu;
+		iommu->flags |= VTD_FLAG_IOMMU_PROBED;
 
 		intel_iommu_init_qi(iommu);
 
@@ -3460,7 +3462,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
 	int sp, ret;
 	struct intel_iommu *iommu = dmaru->iommu;
 
-	if (g_iommus[iommu->seq_id])
+	if (iommu->flags & VTD_FLAG_IOMMU_PROBED)
 		return 0;
 
 	ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
@@ -3487,6 +3489,7 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
 		iommu_disable_translation(iommu);
 
 	g_iommus[iommu->seq_id] = iommu;
+	iommu->flags |= VTD_FLAG_IOMMU_PROBED;
 	ret = iommu_init_domains(iommu);
 	if (ret == 0)
 		ret = iommu_alloc_root_entry(iommu);
Signature · 1 line
2.25.1
#10 lore raw

RE: [PATCH v1 4/6] iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag

Quoted 7 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

In the IOMMU hot-add path, there's a need to check whether an IOMMU has been probed. Instead of checking the IOMMU pointer in the global list, it's better to allocate a flag bit in iommu->flags for this purpose.

Sorry I didn't get the point of original check. This is the hotplug path hence the caller of this function should already figure out it's a new iommu before reaching this point?

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

author maintainer Reviewed-by Yijing Wang
#11 lore raw

Re: [PATCH v1 4/6] iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag

Quoted 12 lines · Kevin
Quoted 7 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

In the IOMMU hot-add path, there's a need to check whether an IOMMU has been probed. Instead of checking the IOMMU pointer in the global list, it's better to allocate a flag bit in iommu->flags for this purpose.

Sorry I didn't get the point of original check. This is the hotplug path hence the caller of this function should already figure out it's a new iommu before reaching this point?

Either did I. It was added by below commit without any comments about this check.

commit ffebeb46dd34736c90ffbca1ccb0bef8f4827c44

Author: Jiang Liu <[email protected]>
Date:   Sun Nov 9 22:48:02 2014 +0800

iommu/vt-d: Enhance intel-iommu driver to support DMAR unit hotplug

Implement required callback functions for intel-iommu driver to support DMAR unit hotplug.

I went through the whole hot-add process and found this check seemed to be duplicate.

Hot-add process starts from dmar_device_hotplug(), it uses a rwlock to synchronize the hot-add paths.

2386 down_write(&dmar_global_lock);
2387 if (insert)
2388 ret = dmar_hotplug_insert(tmp);
2389 else
2390 ret = dmar_hotplug_remove(tmp);
2391 up_write(&dmar_global_lock);

dmar_device_hotplug()
->dmar_hotplug_insert()
-->dmar_parse_one_drhd() /* the added intel_iommu is allocated here*/
-->dmar_hp_add_drhd() /* the intel_iommu is about to bring up */
--->intel_iommu_add()

The duplicate check here:

        if (g_iommus[iommu->seq_id])
                return 0;

All the iommu units are allocated and then initialized in the same synchronized path. There is no need to check a duplicate initialization.

I would like to remove this check if no objection.

Best regards,
baolu

#12 lore raw

RE: [PATCH v1 4/6] iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag

Quoted 54 lines

From: Baolu Lu <[email protected]>
Sent: Friday, July 1, 2022 11:13 AM

Quoted 12 lines · Kevin
Quoted 7 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

In the IOMMU hot-add path, there's a need to check whether an IOMMU has been probed. Instead of checking the IOMMU pointer in the global list, it's better to allocate a flag bit in iommu->flags for this purpose.

Sorry I didn't get the point of original check. This is the hotplug path hence the caller of this function should already figure out it's a new iommu before reaching this point?

Either did I. It was added by below commit without any comments about this check.

commit ffebeb46dd34736c90ffbca1ccb0bef8f4827c44

Author: Jiang Liu <[email protected]>
Date:   Sun Nov 9 22:48:02 2014 +0800

iommu/vt-d: Enhance intel-iommu driver to support DMAR unit hotplug

Implement required callback functions for intel-iommu driver to support DMAR unit hotplug.

I went through the whole hot-add process and found this check seemed to be duplicate.

Hot-add process starts from dmar_device_hotplug(), it uses a rwlock to synchronize the hot-add paths.

2386 down_write(&dmar_global_lock);
2387 if (insert)
2388 ret = dmar_hotplug_insert(tmp);
2389 else
2390 ret = dmar_hotplug_remove(tmp);
2391 up_write(&dmar_global_lock);

dmar_device_hotplug()
->dmar_hotplug_insert()
-->dmar_parse_one_drhd() /* the added intel_iommu is allocated here*/
-->dmar_hp_add_drhd() /* the intel_iommu is about to bring up */
--->intel_iommu_add()

The duplicate check here:

Quoted 8 lines
         if (g_iommus[iommu->seq_id])
                 return 0;

All the iommu units are allocated and then initialized in the same synchronized path. There is no need to check a duplicate initialization.

I would like to remove this check if no objection.

This matches my impression.

author maintainer
#13 lore raw

PATCH 5/6 iommu/vt-d: Remove global g_iommus array

In reply to Lu Baolu

The g_iommus is not used anywhere. Remove it to avoid dead code.

---
 drivers/iommu/intel/iommu.c | 42 -------------------------------------
 1 file changed, 42 deletions(-)
drivers/iommu/intel/iommu.c +0 -42
index f6d7055cffd7..9a284394b2c5 100644
@@ -126,9 +126,6 @@ static inline unsigned long virt_to_dma_pfn(void *p)
 	return page_to_dma_pfn(virt_to_page(p));
 }
 
-/* global iommu list, set NULL for ignored DMAR units */
-static struct intel_iommu **g_iommus;
-
 static void __init check_tylersburg_isoch(void);
 static int rwbf_quirk;
 
@@ -287,9 +284,6 @@ static LIST_HEAD(dmar_satc_units);
 #define for_each_rmrr_units(rmrr) \
 	list_for_each_entry(rmrr, &dmar_rmrr_units, list)
 
-/* bitmap for indexing intel_iommus */
-static int g_num_of_iommus;
-
 static void dmar_remove_one_dev_info(struct device *dev);
 
 int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
@@ -1694,7 +1688,6 @@ static void free_dmar_iommu(struct intel_iommu *iommu)
 		iommu->domain_ids = NULL;
 	}
 
-	g_iommus[iommu->seq_id] = NULL;
 	iommu->flags &= ~VTD_FLAG_IOMMU_PROBED;
 
 	/* free context mapping */
@@ -2899,36 +2892,6 @@ static int __init init_dmars(void)
 	struct intel_iommu *iommu;
 	int ret;
 
-	/*
-	 * for each drhd
-	 *    allocate root
-	 *    initialize and program root entry to not present
-	 * endfor
-	 */
-	for_each_drhd_unit(drhd) {
-		/*
-		 * lock not needed as this is only incremented in the single
-		 * threaded kernel __init code path all other access are read
-		 * only
-		 */
-		if (g_num_of_iommus < DMAR_UNITS_SUPPORTED) {
-			g_num_of_iommus++;
-			continue;
-		}
-		pr_err_once("Exceeded %d IOMMUs\n", DMAR_UNITS_SUPPORTED);
-	}
-
-	/* Preallocate enough resources for IOMMU hot-addition */
-	if (g_num_of_iommus < DMAR_UNITS_SUPPORTED)
-		g_num_of_iommus = DMAR_UNITS_SUPPORTED;
-
-	g_iommus = kcalloc(g_num_of_iommus, sizeof(struct intel_iommu *),
-			GFP_KERNEL);
-	if (!g_iommus) {
-		ret = -ENOMEM;
-		goto error;
-	}
-
 	ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
 	if (ret)
 		goto free_iommu;
@@ -2951,7 +2914,6 @@ static int __init init_dmars(void)
 						   intel_pasid_max_id);
 		}
 
-		g_iommus[iommu->seq_id] = iommu;
 		iommu->flags |= VTD_FLAG_IOMMU_PROBED;
 
 		intel_iommu_init_qi(iommu);
@@ -3079,9 +3041,6 @@ static int __init init_dmars(void)
 		free_dmar_iommu(iommu);
 	}
 
-	kfree(g_iommus);
-
-error:
 	return ret;
 }
 
@@ -3488,7 +3447,6 @@ static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
 	if (iommu->gcmd & DMA_GCMD_TE)
 		iommu_disable_translation(iommu);
 
-	g_iommus[iommu->seq_id] = iommu;
 	iommu->flags |= VTD_FLAG_IOMMU_PROBED;
 	ret = iommu_init_domains(iommu);
 	if (ret == 0)
Signature · 1 line
2.25.1
author maintainer
#15 lore raw

PATCH 6/6 iommu/vt-d: Make DMAR_UNITS_SUPPORTED default 1024

In reply to Lu Baolu

If the available hardware exceeds DMAR_UNITS_SUPPORTED (previously set to MAX_IO_APICS, or 128), it causes these messages: "DMAR: Failed to allocate seq_id", "DMAR: Parse DMAR table failure.", and "x2apic: IRQ remapping doesn't support X2APIC mode x2apic disabled"; and the system fails to boot properly.

To support up to 64 sockets with 10 DMAR units each (640), make the value of DMAR_UNITS_SUPPORTED default 1024.

---
 include/linux/dmar.h | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)
include/linux/dmar.h +1 -5
index cbd714a198a0..d81a51978d01 100644
@@ -18,11 +18,7 @@
 
 struct acpi_dmar_header;
 
-#ifdef	CONFIG_X86
-# define	DMAR_UNITS_SUPPORTED	MAX_IO_APICS
-#else
-# define	DMAR_UNITS_SUPPORTED	64
-#endif
+#define DMAR_UNITS_SUPPORTED	1024
 
 /* DMAR Flags */
 #define DMAR_INTR_REMAP		0x1
Signature · 1 line
2.25.1
Reviewed-by on v1 6/6
#16 lore raw

RE: [PATCH v1 6/6] iommu/vt-d: Make DMAR_UNITS_SUPPORTED default 1024

Quoted 9 lines

From: Lu Baolu <[email protected]>
Sent: Saturday, June 25, 2022 8:52 PM

If the available hardware exceeds DMAR_UNITS_SUPPORTED (previously set to MAX_IO_APICS, or 128), it causes these messages: "DMAR: Failed to allocate seq_id", "DMAR: Parse DMAR table failure.", and "x2apic: IRQ remapping doesn't support X2APIC mode x2apic disabled"; and the system fails to boot properly.

Quoted 5 lines

To support up to 64 sockets with 10 DMAR units each (640), make the value of DMAR_UNITS_SUPPORTED default 1024.

author maintainer
#17 lore raw
In reply to Lu Baolu
Quoted 10 lines · Lu Baolu

Hi folks,

This is a follow-up series of changes proposed by this patch:

https://lore.kernel.org/linux-iommu/[email protected]/

It removes several static arrays of size DMAR_UNITS_SUPPORTED and sets the DMAR_UNITS_SUPPORTED to 1024.

Please help review and suggest.

This series is also available on github:

https://github.com/LuBaolu/intel-iommu/commits/vtd-next-for-v5.20

Best regards,
baolu

Quoted 12 lines

Best regards,
baolu

Lu Baolu (6):
iommu/vt-d: Remove unused domain_get_iommu() iommu/vt-d: Use IDA interface to manage iommu sequence id iommu/vt-d: Refactor iommu information of each domain iommu/vt-d: Add VTD_FLAG_IOMMU_PROBED flag iommu/vt-d: Remove global g_iommus array iommu/vt-d: Make DMAR_UNITS_SUPPORTED default 1024

Quoted 8 lines
  include/linux/dmar.h        |   6 +-
  drivers/iommu/intel/iommu.h |  29 ++++--
  drivers/iommu/intel/dmar.c  |  33 ++-----
  drivers/iommu/intel/iommu.c | 188 ++++++++++++++----------------------
  drivers/iommu/intel/pasid.c |   2 +-
  drivers/iommu/intel/svm.c   |   2 +-
  6 files changed, 103 insertions(+), 157 deletions(-)

_______________________________________________
iommu mailing list
[email protected]
https://lists.linuxfoundation.org/mailman/listinfo/iommu

Reviewed-by on the v1 series
#18 lore raw
In reply to Lu Baolu
Quoted 9 lines · Lu Baolu

Hi folks,

This is a follow-up series of changes proposed by this patch:

https://lore.kernel.org/linux-iommu/[email protected]/

It removes several static arrays of size DMAR_UNITS_SUPPORTED and sets the DMAR_UNITS_SUPPORTED to 1024.

After Kevin Tian's comments, for the whole series:

--> Steve

Signature · 1 line
Steve Wahl, Hewlett Packard Enterprise