iommu/vt-d: Add scalable mode support

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

The Intel vt-d spec rev3.0 introduces a new translation mode called scalable mode, which enables PASID-granular translations for first level, second level, nested and pass-through modes. At the same time, the previous Extended Context (ECS) mode is deprecated (no production ever implements ECS).

This patch adds enumeration for Scalable Mode and removes the deprecated ECS enumeration. It provides a boot time option to disable scalable mode even hardware claims to support it.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

Quoted 15 lines

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

The Intel vt-d spec rev3.0 introduces a new translation mode called scalable mode, which enables PASID-granular translations for first level, second level, nested and pass-through modes. At the same time, the previous Extended Context (ECS) mode is deprecated (no production ever implements ECS).

This patch adds enumeration for Scalable Mode and removes the deprecated ECS enumeration. It provides a boot time option to disable scalable mode even hardware claims to support it.

Quoted 7 lines

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

Reviewed-by Kevin Tian <[email protected]>

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

In scalable mode, pasid structure is a two level table with a pasid directory table and a pasid table. Any pasid entry can be identified by a pasid value in below way.

   1
   9                       6 5      0
    .-----------------------.-------.
    |              PASID    |       |
    '-----------------------'-------'    .-------------.
             |                    |      |             |
             |                    |      |             |
             |                    |      |             |
             |     .-----------.  |      .-------------.
             |     |           |  |----->| PASID Entry |
             |     |           |  |      '-------------'
             |     |           |  |Plus  |             |
             |     .-----------.  |      |             |
             |---->| DIR Entry |-------->|             |
             |     '-----------'         '-------------'
.---------.  |Plus |           |
| Context |  |     |           |
|  Entry  |------->|           |
'---------'        '-----------'

This changes the pasid table APIs to support scalable mode PASID directory and PASID table. It also adds a helper to get the PASID table entry according to the pasid value.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

---
 drivers/iommu/intel-iommu.c |  2 +-
 drivers/iommu/intel-pasid.c | 72 ++++++++++++++++++++++++++++++++----

(quoted diff) +0 -1

-
 drivers/iommu/intel-pasid.h | 10 +++++-
 drivers/iommu/intel-svm.c   |  6 +---
 4 files changed, 74 insertions(+), 16 deletions(-)

drivers/iommu/intel-iommu.c +0 -0

index 5845edf4dcf9..b0da4f765274 100644

drivers/iommu/intel-iommu.c +0 -0

 		ret = intel_pasid_alloc_table(dev);
 		if (ret) {

drivers/iommu/intel-iommu.c +0 -0

 			__dmar_remove_one_dev_info(info);

drivers/iommu/intel-pasid.c +0 -0

index fe95c9bd4d33..d6e90cd5b062 100644

drivers/iommu/intel-pasid.c +2 -1

 	/* DMA alias device already has a pasid table, use it: */

@@ -143,8 +142,9 @@ int intel_pasid_alloc_table(struct device *dev)
 		return -ENOMEM;
 	INIT_LIST_HEAD(&pasid_table->dev);
 
-	size = sizeof(struct pasid_entry);
+	size = sizeof(struct pasid_dir_entry);
 	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);
+	count >>= PASID_PDE_SHIFT;
 	order = get_order(size * count);
 	pages = alloc_pages_node(info->iommu->node,

drivers/iommu/intel-pasid.c +0 -0

 				 GFP_ATOMIC | __GFP_ZERO,

drivers/iommu/intel-pasid.c +46 -4

+}
+
+/* Get PASID table from a PASID directory entry. */
+static inline struct pasid_entry *
+get_pasid_table_from_pde(struct pasid_dir_entry *pde)
+{
+	if (!pasid_pde_is_present(pde))
+		return NULL;
+
+	return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK);
+}
+
 void intel_pasid_free_table(struct device *dev)
 {
 	struct device_domain_info *info;
 	struct pasid_table *pasid_table;
+	struct pasid_dir_entry *dir;
+	struct pasid_entry *table;
+	int i, max_pde;
 
 	info = dev->archdata.iommu;
-	if (!info || !dev_is_pci(dev) ||
-	    !info->pasid_supported || !info->pasid_table)
+	if (!info || !dev_is_pci(dev) || !info->pasid_table)
 		return;
 
 	pasid_table = info->pasid_table;

@@ -178,6 +197,14 @@ void intel_pasid_free_table(struct device *dev)
 	if (!list_empty(&pasid_table->dev))
 		return;
 
+	/* Free scalable mode PASID directory tables: */
+	dir = pasid_table->table;
+	max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT;
+	for (i = 0; i < max_pde; i++) {
+		table = get_pasid_table_from_pde(&dir[i]);
+		free_pgtable_page(table);
+	}
+
 	free_pages((unsigned long)pasid_table->table, pasid_table->order);
 	kfree(pasid_table);
 }

@@ -206,17 +233,37 @@ int intel_pasid_get_dev_max_id(struct device
 *dev)
 
 struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
 {
+	struct device_domain_info *info;
 	struct pasid_table *pasid_table;
+	struct pasid_dir_entry *dir;
 	struct pasid_entry *entries;
+	int dir_index, index;
 
 	pasid_table = intel_pasid_get_table(dev);
 	if (WARN_ON(!pasid_table || pasid < 0 ||
 		    pasid >= intel_pasid_get_dev_max_id(dev)))
 		return NULL;
 
-	entries = pasid_table->table;
+	dir = pasid_table->table;
+	info = dev->archdata.iommu;
+	dir_index = pasid >> PASID_PDE_SHIFT;
+	index = pasid & PASID_PTE_MASK;
+
+	spin_lock(&pasid_lock);
+	entries = get_pasid_table_from_pde(&dir[dir_index]);
+	if (!entries) {
+		entries = alloc_pgtable_page(info->iommu->node);
+		if (!entries) {
+			spin_unlock(&pasid_lock);
+			return NULL;
+		}
+
+		WRITE_ONCE(dir[dir_index].val,
+			   (u64)virt_to_phys(entries) | PASID_PTE_PRESENT);
+	}
+	spin_unlock(&pasid_lock);
 
-	return &entries[pasid];
+	return &entries[index];
 }
 

/*

drivers/iommu/intel-pasid.c +0 -0

@@ -224,7 +271,14 @@ struct pasid_entry *intel_pasid_get_entry(struct
 device *dev, int pasid)
  */
 static inline void pasid_clear_entry(struct pasid_entry *pe)

drivers/iommu/intel-pasid.c +0 -0

 }
 

void intel_pasid_clear_entry(struct device *dev, int pasid)

drivers/iommu/intel-pasid.h +9 -1

index 1c05ed6fc5a5..12f480c2bb8b 100644
@@ -12,11 +12,19 @@
 
 #define PASID_MIN			0x1
 #define PASID_MAX			0x100000
+#define PASID_PTE_MASK			0x3F
+#define PASID_PTE_PRESENT		1
+#define PDE_PFN_MASK			PAGE_MASK
+#define PASID_PDE_SHIFT			6
 
-struct pasid_entry {
+struct pasid_dir_entry {
 	u64 val;
 };
 
+struct pasid_entry {
+	u64 val[8];
+};
+
 /* The representative of a PASID table */
 struct pasid_table {
 	void			*table;		/* pasid table pointer */

drivers/iommu/intel-svm.c +0 -2

index 4a03e5090952..6c0bd9ee9602 100644
@@ -65,8 +65,6 @@ int intel_svm_init(struct intel_iommu *iommu)
 
 	order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max);
 	if (ecap_dis(iommu->ecap)) {
-		/* Just making it explicit... */
-		BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct
 pasid_state_entry));
 		pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
 		if (pages)

drivers/iommu/intel-svm.c +1 -3

 			iommu->pasid_state_table = page_address(pages);

@@ -406,9 +404,7 @@ int intel_svm_bind_mm(struct device *dev, int
 *pasid, int flags, struct svm_dev_
 			pasid_entry_val |= PASID_ENTRY_FLPM_5LP;
 
 		entry = intel_pasid_get_entry(dev, svm->pasid);
-		entry->val = pasid_entry_val;
-
-		wmb();
+		WRITE_ONCE(entry->val[0], pasid_entry_val);
 
 		/*

* Flush PASID cache when a PASID table entry becomes
--
2.17.1

Quoted 55 lines

From: Lu Baolu [mailto:baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

In scalable mode, pasid structure is a two level table with a pasid directory table and a pasid table. Any pasid entry can be identified by a pasid value in below way.

    1
    9                       6 5      0
     .-----------------------.-------.
     |              PASID    |       |
     '-----------------------'-------'    .-------------.
              |                    |      |             |
              |                    |      |             |
              |                    |      |             |
              |     .-----------.  |      .-------------.
              |     |           |  |----->| PASID Entry |
              |     |           |  |      '-------------'
              |     |           |  |Plus  |             |
              |     .-----------.  |      |             |
              |---->| DIR Entry |-------->|             |
              |     '-----------'         '-------------'
.---------.  |Plus |           |
| Context |  |     |           |
|  Entry  |------->|           |
'---------'        '-----------'

This changes the pasid table APIs to support scalable mode PASID directory and PASID table. It also adds a helper to get the PASID table entry according to the pasid value.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <jacob.jun.pan-VuQAYsv1563Yd54FQh9/[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]>

Reviewed-by Ashok Raj <[email protected]>

---
  drivers/iommu/intel-iommu.c |  2 +-
  drivers/iommu/intel-pasid.c | 72 ++++++++++++++++++++++++++++++++----

(quoted diff) +0 -1

-
  drivers/iommu/intel-pasid.h | 10 +++++-
  drivers/iommu/intel-svm.c   |  6 +---
  4 files changed, 74 insertions(+), 16 deletions(-)

drivers/iommu/intel-iommu.c +0 -0

index 5845edf4dcf9..b0da4f765274 100644
@@ -2507,7 +2507,7 @@ static struct dmar_domain
 *dmar_insert_one_dev_info(struct intel_iommu *iommu,
  	if (dev)
  		dev->archdata.iommu = info;
 

drivers/iommu/intel-iommu.c static struct dmar_domain *dmar_insert_one_dev_info(struct intel_iommu *iommu, · line 2510

Quoted 2 lines

drivers/iommu/intel-iommu.c +1 -1

-	if (dev && dev_is_pci(dev) && info->pasid_supported) {
+	if (dev && dev_is_pci(dev) && sm_supported(iommu)) {

worthy of a comment here that PASID table now is mandatory in scalable mode, instead of optional for 1st level usage before.

Quoted 11 lines

drivers/iommu/intel-iommu.c +0 -0

  		ret = intel_pasid_alloc_table(dev);
  		if (ret) {

  			__dmar_remove_one_dev_info(info);

drivers/iommu/intel-pasid.c +0 -0

index fe95c9bd4d33..d6e90cd5b062 100644
@@ -127,8 +127,7 @@ int intel_pasid_alloc_table(struct device *dev)
  	int ret, order;
 

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 131

Quoted 5 lines

drivers/iommu/intel-pasid.c +1 -2

  	info = dev->archdata.iommu;
-	if (WARN_ON(!info || !dev_is_pci(dev) ||
-		    !info->pasid_supported || info->pasid_table))
+	if (WARN_ON(!info || !dev_is_pci(dev) || info->pasid_table))
  		return -EINVAL;

following same logic should you check sm_supported here?

Quoted 17 lines

drivers/iommu/intel-pasid.c +2 -1

  	/* DMA alias device already has a pasid table, use it: */

@@ -143,8 +142,9 @@ int intel_pasid_alloc_table(struct device *dev)
  		return -ENOMEM;
  	INIT_LIST_HEAD(&pasid_table->dev);
 
-	size = sizeof(struct pasid_entry);
+	size = sizeof(struct pasid_dir_entry);
  	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);
+	count >>= PASID_PDE_SHIFT;
  	order = get_order(size * count);
  	pages = alloc_pages_node(info->iommu->node,

GFP_ATOMIC | __GFP_ZERO,

drivers/iommu/intel-pasid.c +0 -0

@@ -154,7 +154,7 @@ int intel_pasid_alloc_table(struct device *dev)
 

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 157

Quoted 4 lines

drivers/iommu/intel-pasid.c +1 -1

  	pasid_table->table = page_address(pages);
  	pasid_table->order = order;
-	pasid_table->max_pasid = count;
+	pasid_table->max_pasid = count << PASID_PDE_SHIFT;

are you sure of that count is PDE_SHIFT aligned? otherwise >> then << would lose some bits. If sure, then better add some check.

Quoted 8 lines

drivers/iommu/intel-pasid.c +0 -0

  attach_out:
  	device_attach_pasid_table(info, pasid_table);

@@ -162,14 +162,33 @@ int intel_pasid_alloc_table(struct device *dev)
  	return 0;
  }
 

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 169

Quoted 5 lines

drivers/iommu/intel-pasid.c +5 -0

+/* Get PRESENT bit of a PASID directory entry. */
+static inline bool
+pasid_pde_is_present(struct pasid_dir_entry *pde)
+{
+	return READ_ONCE(pde->val) & PASID_PTE_PRESENT;

curious why adding READ_ONCE specifically for PASID structure, but not used for any other existing vtd structures? Is it to address some specific requirement on PASID structure as defined in spec?

Quoted 92 lines

drivers/iommu/intel-pasid.c +46 -4

+}
+
+/* Get PASID table from a PASID directory entry. */
+static inline struct pasid_entry *
+get_pasid_table_from_pde(struct pasid_dir_entry *pde)
+{
+	if (!pasid_pde_is_present(pde))
+		return NULL;
+
+	return phys_to_virt(READ_ONCE(pde->val) & PDE_PFN_MASK);
+}
+
  void intel_pasid_free_table(struct device *dev)
  {
  	struct device_domain_info *info;
  	struct pasid_table *pasid_table;
+	struct pasid_dir_entry *dir;
+	struct pasid_entry *table;
+	int i, max_pde;
 
  	info = dev->archdata.iommu;
-	if (!info || !dev_is_pci(dev) ||
-	    !info->pasid_supported || !info->pasid_table)
+	if (!info || !dev_is_pci(dev) || !info->pasid_table)
  		return;
 
  	pasid_table = info->pasid_table;

@@ -178,6 +197,14 @@ void intel_pasid_free_table(struct device *dev)
  	if (!list_empty(&pasid_table->dev))
  		return;
 
+	/* Free scalable mode PASID directory tables: */
+	dir = pasid_table->table;
+	max_pde = pasid_table->max_pasid >> PASID_PDE_SHIFT;
+	for (i = 0; i < max_pde; i++) {
+		table = get_pasid_table_from_pde(&dir[i]);
+		free_pgtable_page(table);
+	}
+
  	free_pages((unsigned long)pasid_table->table, pasid_table->order);
  	kfree(pasid_table);
  }

@@ -206,17 +233,37 @@ int intel_pasid_get_dev_max_id(struct device
 *dev)
 
  struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid)
  {
+	struct device_domain_info *info;
  	struct pasid_table *pasid_table;
+	struct pasid_dir_entry *dir;
  	struct pasid_entry *entries;
+	int dir_index, index;
 
  	pasid_table = intel_pasid_get_table(dev);
  	if (WARN_ON(!pasid_table || pasid < 0 ||
  		    pasid >= intel_pasid_get_dev_max_id(dev)))
  		return NULL;
 
-	entries = pasid_table->table;
+	dir = pasid_table->table;
+	info = dev->archdata.iommu;
+	dir_index = pasid >> PASID_PDE_SHIFT;
+	index = pasid & PASID_PTE_MASK;
+
+	spin_lock(&pasid_lock);
+	entries = get_pasid_table_from_pde(&dir[dir_index]);
+	if (!entries) {
+		entries = alloc_pgtable_page(info->iommu->node);
+		if (!entries) {
+			spin_unlock(&pasid_lock);
+			return NULL;
+		}
+
+		WRITE_ONCE(dir[dir_index].val,
+			   (u64)virt_to_phys(entries) | PASID_PTE_PRESENT);
+	}
+	spin_unlock(&pasid_lock);
 
-	return &entries[pasid];
+	return &entries[index];
  }
 

/*

drivers/iommu/intel-pasid.c +2 -1

@@ -224,7 +271,14 @@ struct pasid_entry *intel_pasid_get_entry(struct
 device *dev, int pasid)
   */
  static inline void pasid_clear_entry(struct pasid_entry *pe)
  {
-	WRITE_ONCE(pe->val, 0);
+	WRITE_ONCE(pe->val[0], 0);
+	WRITE_ONCE(pe->val[1], 0);

drivers/iommu/intel-pasid.c struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid) · line 281

Quoted 6 lines

drivers/iommu/intel-pasid.c +6 -0

+	WRITE_ONCE(pe->val[2], 0);
+	WRITE_ONCE(pe->val[3], 0);
+	WRITE_ONCE(pe->val[4], 0);
+	WRITE_ONCE(pe->val[5], 0);
+	WRITE_ONCE(pe->val[6], 0);
+	WRITE_ONCE(pe->val[7], 0);

memset?

Quoted 53 lines

drivers/iommu/intel-pasid.c +0 -0

  }
 

void intel_pasid_clear_entry(struct device *dev, int pasid)

drivers/iommu/intel-pasid.h +9 -1

index 1c05ed6fc5a5..12f480c2bb8b 100644
@@ -12,11 +12,19 @@
 
  #define PASID_MIN			0x1
  #define PASID_MAX			0x100000
+#define PASID_PTE_MASK			0x3F
+#define PASID_PTE_PRESENT		1
+#define PDE_PFN_MASK			PAGE_MASK
+#define PASID_PDE_SHIFT			6
 
-struct pasid_entry {
+struct pasid_dir_entry {
  	u64 val;
  };
 
+struct pasid_entry {
+	u64 val[8];
+};
+
  /* The representative of a PASID table */
  struct pasid_table {
  	void			*table;		/* pasid table pointer */

drivers/iommu/intel-svm.c +0 -2

index 4a03e5090952..6c0bd9ee9602 100644
@@ -65,8 +65,6 @@ int intel_svm_init(struct intel_iommu *iommu)
 
  	order = get_order(sizeof(struct pasid_entry) * iommu->pasid_max);
  	if (ecap_dis(iommu->ecap)) {
-		/* Just making it explicit... */
-		BUILD_BUG_ON(sizeof(struct pasid_entry) != sizeof(struct
 pasid_state_entry));
  		pages = alloc_pages(GFP_KERNEL | __GFP_ZERO, order);
  		if (pages)

  			iommu->pasid_state_table = page_address(pages);

drivers/iommu/intel-svm.c +1 -3

@@ -406,9 +404,7 @@ int intel_svm_bind_mm(struct device *dev, int
 *pasid, int flags, struct svm_dev_
  			pasid_entry_val |= PASID_ENTRY_FLPM_5LP;
 
  		entry = intel_pasid_get_entry(dev, svm->pasid);
-		entry->val = pasid_entry_val;
-
-		wmb();
+		WRITE_ONCE(entry->val[0], pasid_entry_val);
 

Quoted 4 lines

drivers/iommu/intel-svm.c +0 -0

  		/*

* Flush PASID cache when a PASID table entry becomes
--
2.17.1

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, September 6, 2018 10:46 AM

Quoted 14 lines

Quoted 14 lines

drivers/iommu/intel-pasid.c +2 -1

@@ -143,8 +142,9 @@ int intel_pasid_alloc_table(struct device *dev)
  		return -ENOMEM;
  	INIT_LIST_HEAD(&pasid_table->dev);
 
-	size = sizeof(struct pasid_entry);
+	size = sizeof(struct pasid_dir_entry);
  	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);
+	count >>= PASID_PDE_SHIFT;
  	order = get_order(size * count);
  	pages = alloc_pages_node(info->iommu->node,

GFP_ATOMIC | __GFP_ZERO,

(quoted diff) +0 -0

@@ -154,7 +154,7 @@ int intel_pasid_alloc_table(struct device *dev)
 

Quoted 7 lines

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 157

Quoted 4 lines

drivers/iommu/intel-pasid.c +1 -1

  	pasid_table->table = page_address(pages);
  	pasid_table->order = order;
-	pasid_table->max_pasid = count;
+	pasid_table->max_pasid = count << PASID_PDE_SHIFT;

are you sure of that count is PDE_SHIFT aligned? otherwise >> then << would lose some bits. If sure, then better add some check.

I am making the max_pasid PDE_SHIFT aligned as the result of shift operations.

Quoted 2 lines

Review int intel_pasid_alloc_table(struct device *dev) · line 159

Quoted 2 lines

(quoted diff) +0 -0

  	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);

Quoted 8 lines

Quoted 8 lines

drivers/iommu/intel-pasid.c +0 -0

  attach_out:

  	device_attach_pasid_table(info, pasid_table);

drivers/iommu/intel-pasid.c +0 -0

@@ -162,14 +162,33 @@ int intel_pasid_alloc_table(struct device *dev)
  	return 0;
  }
 

Quoted 9 lines

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 169

Quoted 5 lines

drivers/iommu/intel-pasid.c +5 -0

+/* Get PRESENT bit of a PASID directory entry. */
+static inline bool
+pasid_pde_is_present(struct pasid_dir_entry *pde)
+{
+	return READ_ONCE(pde->val) & PASID_PTE_PRESENT;

curious why adding READ_ONCE specifically for PASID structure, but not used for any other existing vtd structures? Is it to address some specific requirement on PASID structure as defined in spec?

READ/WRITE_ONCE are used in pasid entry read/write to prevent the compiler from merging, refetching or reordering successive instances of read/write.

Quoted 3 lines

From: Lu Baolu [mailto:baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]]
Sent: Thursday, September 6, 2018 10:46 AM

[...]

Quoted 22 lines

Quoted 14 lines

Quoted 14 lines

drivers/iommu/intel-pasid.c +2 -1

@@ -143,8 +142,9 @@ int intel_pasid_alloc_table(struct device *dev)
   		return -ENOMEM;
   	INIT_LIST_HEAD(&pasid_table->dev);
 
-	size = sizeof(struct pasid_entry);
+	size = sizeof(struct pasid_dir_entry);
   	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);
+	count >>= PASID_PDE_SHIFT;
   	order = get_order(size * count);
   	pages = alloc_pages_node(info->iommu->node,

GFP_ATOMIC | __GFP_ZERO,

(quoted diff) +0 -0

@@ -154,7 +154,7 @@ int intel_pasid_alloc_table(struct device *dev)
 

Quoted 7 lines

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 157

Quoted 4 lines

drivers/iommu/intel-pasid.c +1 -1

   	pasid_table->table = page_address(pages);
   	pasid_table->order = order;
-	pasid_table->max_pasid = count;
+	pasid_table->max_pasid = count << PASID_PDE_SHIFT;

are you sure of that count is PDE_SHIFT aligned? otherwise >> then << would lose some bits. If sure, then better add some check.

Quoted 3 lines

I am making the max_pasid PDE_SHIFT aligned as the result of shift operations.

earlier:

Quoted 2 lines

Quoted 2 lines

Review int intel_pasid_alloc_table(struct device *dev) · line 159

Quoted 2 lines

(quoted diff) +0 -0

   	count = min_t(int, pci_max_pasids(to_pci_dev(dev)),
 intel_pasid_max_id);

so you decided to truncate count to be PDE_SHIFT aligned. Is PASID value user configurable? if not, then it's fine.

Quoted 19 lines

Quoted 18 lines

Quoted 6 lines

drivers/iommu/intel-pasid.c +0 -0

   attach_out:

   	device_attach_pasid_table(info, pasid_table);

drivers/iommu/intel-pasid.c int intel_pasid_alloc_table(struct device *dev) · line 169

Quoted 8 lines

drivers/iommu/intel-pasid.c +5 -0

@@ -162,14 +162,33 @@ int intel_pasid_alloc_table(struct device *dev)
   	return 0;
   }
 
+/* Get PRESENT bit of a PASID directory entry. */
+static inline bool
+pasid_pde_is_present(struct pasid_dir_entry *pde)
+{
+	return READ_ONCE(pde->val) & PASID_PTE_PRESENT;

curious why adding READ_ONCE specifically for PASID structure, but not used for any other existing vtd structures? Is it to address some specific requirement on PASID structure as defined in spec?

Quoted 4 lines

READ/WRITE_ONCE are used in pasid entry read/write to prevent the compiler from merging, refetching or reordering successive instances of read/write.

that's fine. I'm just curious why this is the first user of such macros in intel-iommu driver. Even before with ecs we have PASID table too.

Quoted 8 lines

Quoted 8 lines

drivers/iommu/intel-pasid.c +2 -1

@@ -224,7 +271,14 @@ struct pasid_entry
 *intel_pasid_get_entry(struct device *dev, int pasid)
   */
  static inline void pasid_clear_entry(struct pasid_entry *pe)
  {
-	WRITE_ONCE(pe->val, 0);
+	WRITE_ONCE(pe->val[0], 0);
+	WRITE_ONCE(pe->val[1], 0);

Quoted 8 lines

drivers/iommu/intel-pasid.c struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid) · line 281

Quoted 6 lines

drivers/iommu/intel-pasid.c +6 -0

+	WRITE_ONCE(pe->val[2], 0);
+	WRITE_ONCE(pe->val[3], 0);
+	WRITE_ONCE(pe->val[4], 0);
+	WRITE_ONCE(pe->val[5], 0);
+	WRITE_ONCE(pe->val[6], 0);
+	WRITE_ONCE(pe->val[7], 0);  

memset?

The order is important here. Otherwise, the PRESENT bit of this pasid entry might still set while other fields contains invalid values.

Quoted 17 lines · On Thu, 6 Sep 2018 10:46:03 +0800 Lu Baolu <[email protected]> wrote:

Quoted 16 lines

Quoted 4 lines

drivers/iommu/intel-pasid.c +0 -0

@@ -224,7 +271,14 @@ struct pasid_entry
 *intel_pasid_get_entry(struct device *dev, int pasid)
    */
   static inline void pasid_clear_entry(struct pasid_entry *pe)

drivers/iommu/intel-pasid.c struct pasid_entry · line 282

Quoted 10 lines

drivers/iommu/intel-pasid.c +8 -1

   {
-	WRITE_ONCE(pe->val, 0);
+	WRITE_ONCE(pe->val[0], 0);
+	WRITE_ONCE(pe->val[1], 0);
+	WRITE_ONCE(pe->val[2], 0);
+	WRITE_ONCE(pe->val[3], 0);
+	WRITE_ONCE(pe->val[4], 0);
+	WRITE_ONCE(pe->val[5], 0);
+	WRITE_ONCE(pe->val[6], 0);
+	WRITE_ONCE(pe->val[7], 0);

memset?

Quoted 2 lines

The order is important here. Otherwise, the PRESENT bit of this pasid entry might still set while other fields contains invalid values.

WRITE_ONCE/READ_ONCE will switch to __builtin_memcpy() in if the size exceeds word size, ie. 64bit in this case. I don;t think compiler will reorder built-in function. Beside, we only need to clear present and FDP bit, right?

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

So that they could also be used in other source files.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

---
 drivers/iommu/intel-iommu.c | 43 -------------------------------------
 include/linux/intel-iommu.h | 43
+++++++++++++++++++++++++++++++++++++
 2 files changed, 43 insertions(+), 43 deletions(-)

drivers/iommu/intel-iommu.c +0 -43

index b0da4f765274..93cde957adc7 100644
@@ -315,49 +315,6 @@ static inline void context_clear_entry(struct
 context_entry *context)
 	context->hi = 0;
 }
 
-/*
- * 0: readable
- * 1: writable
- * 2-6: reserved
- * 7: super page
- * 8-10: available
- * 11: snoop behavior
- * 12-63: Host physcial address
- */
-struct dma_pte {
-	u64 val;
-};
-
-static inline void dma_clear_pte(struct dma_pte *pte)
-{
-	pte->val = 0;
-}
-
-static inline u64 dma_pte_addr(struct dma_pte *pte)
-{
-#ifdef CONFIG_64BIT
-	return pte->val & VTD_PAGE_MASK;
-#else
-	/* Must have a full atomic 64-bit read */
-	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
-#endif
-}
-
-static inline bool dma_pte_present(struct dma_pte *pte)
-{
-	return (pte->val & 3) != 0;
-}
-
-static inline bool dma_pte_superpage(struct dma_pte *pte)
-{
-	return (pte->val & DMA_PTE_LARGE_PAGE);
-}
-
-static inline int first_pte_in_page(struct dma_pte *pte)
-{
-	return !((unsigned long)pte & ~VTD_PAGE_MASK);
-}
-
 /*
  * This domain is a statically identity mapping domain.

* 1. This domain creats a static 1:1 mapping to all usable memory.

include/linux/intel-iommu.h +43 -1

index 2173ae35f1dc..41791903a5e3 100644
@@ -501,6 +501,49 @@ static inline void __iommu_flush_cache(
 		clflush_cache_range(addr, size);
 }
 
+/*
+ * 0: readable
+ * 1: writable
+ * 2-6: reserved
+ * 7: super page
+ * 8-10: available
+ * 11: snoop behavior
+ * 12-63: Host physcial address
+ */
+struct dma_pte {
+	u64 val;
+};
+
+static inline void dma_clear_pte(struct dma_pte *pte)
+{
+	pte->val = 0;
+}
+
+static inline u64 dma_pte_addr(struct dma_pte *pte)
+{
+#ifdef CONFIG_64BIT
+	return pte->val & VTD_PAGE_MASK;
+#else
+	/* Must have a full atomic 64-bit read */
+	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
+#endif
+}
+
+static inline bool dma_pte_present(struct dma_pte *pte)
+{
+	return (pte->val & 3) != 0;
+}
+
+static inline bool dma_pte_superpage(struct dma_pte *pte)
+{
+	return (pte->val & DMA_PTE_LARGE_PAGE);
+}
+
+static inline int first_pte_in_page(struct dma_pte *pte)
+{
+	return !((unsigned long)pte & ~VTD_PAGE_MASK);
+}
+
 extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct
 pci_dev *dev);
 extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
 
--

2.17.1

Quoted 5 lines

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

So that they could also be used in other source files.

Quoted 6 lines

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

Reviewed-by Kevin Tian <[email protected]>

Quoted 119 lines

---
  drivers/iommu/intel-iommu.c | 43 -------------------------------------
  include/linux/intel-iommu.h | 43
+++++++++++++++++++++++++++++++++++++
  2 files changed, 43 insertions(+), 43 deletions(-)

drivers/iommu/intel-iommu.c +0 -43

index b0da4f765274..93cde957adc7 100644
@@ -315,49 +315,6 @@ static inline void context_clear_entry(struct
 context_entry *context)
  	context->hi = 0;
  }
 
-/*
- * 0: readable
- * 1: writable
- * 2-6: reserved
- * 7: super page
- * 8-10: available
- * 11: snoop behavior
- * 12-63: Host physcial address
- */
-struct dma_pte {
-	u64 val;
-};
-
-static inline void dma_clear_pte(struct dma_pte *pte)
-{
-	pte->val = 0;
-}
-
-static inline u64 dma_pte_addr(struct dma_pte *pte)
-{
-#ifdef CONFIG_64BIT
-	return pte->val & VTD_PAGE_MASK;
-#else
-	/* Must have a full atomic 64-bit read */
-	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
-#endif
-}
-
-static inline bool dma_pte_present(struct dma_pte *pte)
-{
-	return (pte->val & 3) != 0;
-}
-
-static inline bool dma_pte_superpage(struct dma_pte *pte)
-{
-	return (pte->val & DMA_PTE_LARGE_PAGE);
-}
-
-static inline int first_pte_in_page(struct dma_pte *pte)
-{
-	return !((unsigned long)pte & ~VTD_PAGE_MASK);
-}
-
  /*
   * This domain is a statically identity mapping domain.

* 1. This domain creats a static 1:1 mapping to all usable memory.

include/linux/intel-iommu.h +43 -1

index 2173ae35f1dc..41791903a5e3 100644
@@ -501,6 +501,49 @@ static inline void __iommu_flush_cache(
  		clflush_cache_range(addr, size);
  }
 
+/*
+ * 0: readable
+ * 1: writable
+ * 2-6: reserved
+ * 7: super page
+ * 8-10: available
+ * 11: snoop behavior
+ * 12-63: Host physcial address
+ */
+struct dma_pte {
+	u64 val;
+};
+
+static inline void dma_clear_pte(struct dma_pte *pte)
+{
+	pte->val = 0;
+}
+
+static inline u64 dma_pte_addr(struct dma_pte *pte)
+{
+#ifdef CONFIG_64BIT
+	return pte->val & VTD_PAGE_MASK;
+#else
+	/* Must have a full atomic 64-bit read */
+	return  __cmpxchg64(&pte->val, 0ULL, 0ULL) & VTD_PAGE_MASK;
+#endif
+}
+
+static inline bool dma_pte_present(struct dma_pte *pte)
+{
+	return (pte->val & 3) != 0;
+}
+
+static inline bool dma_pte_superpage(struct dma_pte *pte)
+{
+	return (pte->val & DMA_PTE_LARGE_PAGE);
+}
+
+static inline int first_pte_in_page(struct dma_pte *pte)
+{
+	return !((unsigned long)pte & ~VTD_PAGE_MASK);
+}
+
  extern struct dmar_drhd_unit * dmar_find_matched_drhd_unit(struct
 pci_dev *dev);
  extern int dmar_find_matched_atsr_unit(struct pci_dev *dev);
 
--

2.17.1

From: Lu Baolu [mailto:baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

Intel vt-d spec rev3.0 requires software to use 256-bit descriptors in invalidation queue. As the spec reads in section 6.5.2:

Remapping hardware supporting Scalable Mode Translations (ECAP_REG.SMTS=1) allow software to additionally program the width of the descriptors (128-bits or 256-bits) that will be written into the Queue. Software should setup the Invalidation Queue for 256-bit descriptors before progra- mming remapping hardware for scalable-mode translation as 128-bit descriptors are treated as invalid descriptors (see Table 21 in Section 6.5.2.10) in scalable-mode.

This patch adds 256-bit invalidation descriptor support if the hardware presents scalable mode capability.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <jacob.jun.pan-VuQAYsv1563Yd54FQh9/[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]>

---
 drivers/iommu/dmar.c                | 83 +++++++++++++++++++----------
 drivers/iommu/intel-svm.c           | 76 ++++++++++++++++----------
 drivers/iommu/intel_irq_remapping.c |  6 ++-
 include/linux/intel-iommu.h         |  7 ++-
 4 files changed, 113 insertions(+), 59 deletions(-)

drivers/iommu/dmar.c +0 -0

index d9c748b6f9e4..b1429fa2cf29 100644

drivers/iommu/dmar.c +2 -0

+			memcpy(desc, qi->desc + (wait_index << shift),
+			       1 << shift);
 			writel(DMA_FSTS_IQE, iommu->reg +
 DMAR_FSTS_REG);

drivers/iommu/dmar.c +2 -2

 			return -EINVAL;
 		}

@@ -1191,10 +1194,10 @@ static int qi_check_fault(struct intel_iommu
 *iommu, int index)
 	 */
 	if (fault & DMA_FSTS_ITE) {
 		head = readl(iommu->reg + DMAR_IQH_REG);
-		head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) %
 QI_LENGTH;
+		head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
 		head |= 1;
 		tail = readl(iommu->reg + DMAR_IQT_REG);
-		tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) %
 QI_LENGTH;
+		tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;

drivers/iommu/dmar.c +12 -8

 		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

@@ -1222,15 +1225,14 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
 {
 	int rc;
 	struct q_inval *qi = iommu->qi;
-	struct qi_desc *hw, wait_desc;
+	int offset, shift, length;
+	struct qi_desc wait_desc;
 	int wait_index, index;
 	unsigned long flags;
 
 	if (!qi)
 		return 0;
 
-	hw = qi->desc;
-
 restart:
 	rc = 0;

@@ -1243,16 +1245,21 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
 
 	index = qi->free_head;
 	wait_index = (index + 1) % QI_LENGTH;
+	shift = DMAR_IQ_SHIFT + !!ecap_smts(iommu->ecap);
+	length = 1 << shift;
 
 	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 
-	hw[index] = *desc;
-
-	wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
+	offset = index << shift;
+	memcpy(qi->desc + offset, desc, length);
+	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
 			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
-	wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
+	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
+	wait_desc.qw2 = 0;
+	wait_desc.qw3 = 0;
 
-	hw[wait_index] = wait_desc;
+	offset = wait_index << shift;
+	memcpy(qi->desc + offset, &wait_desc, length);
 
 	qi->free_head = (qi->free_head + 2) % QI_LENGTH;

drivers/iommu/dmar.c +1 -1

 	qi->free_cnt -= 2;

@@ -1261,7 +1268,7 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
 	 * update the HW tail register indicating the presence of
 	 * new descriptors.
 	 */
-	writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg +
 DMAR_IQT_REG);
+	writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
 

drivers/iommu/dmar.c +8 -4

 	while (qi->desc_status[wait_index] != QI_DONE) {
 		/*

@@ -1298,8 +1305,10 @@ void qi_global_iec(struct intel_iommu *iommu)
 {
 	struct qi_desc desc;
 
-	desc.low = QI_IEC_TYPE;
-	desc.high = 0;
+	desc.qw0 = QI_IEC_TYPE;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
 	/* should never fail */
 	qi_submit_sync(&desc, iommu);

@@ -1310,9 +1319,11 @@ void qi_flush_context(struct intel_iommu
 *iommu, u16 did, u16 sid, u8 fm,
 {
 	struct qi_desc desc;
 
-	desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
+	desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
 			| QI_CC_GRAN(type) | QI_CC_TYPE;
-	desc.high = 0;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
 	qi_submit_sync(&desc, iommu);

}

drivers/iommu/dmar.c +4 -2

@@ -1331,10 +1342,12 @@ void qi_flush_iotlb(struct intel_iommu
 *iommu, u16 did, u64 addr,
 	if (cap_read_drain(iommu->cap))
 		dr = 1;
 
-	desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) |
 QI_IOTLB_DW(dw)
+	desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) |
 QI_IOTLB_DW(dw)
 		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
-	desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
+	desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
 		| QI_IOTLB_AM(size_order);
+	desc.qw2 = 0;
+	desc.qw3 = 0;

drivers/iommu/dmar.c +14 -4

 	qi_submit_sync(&desc, iommu);
 }

@@ -1347,15 +1360,17 @@ void qi_flush_dev_iotlb(struct intel_iommu
 *iommu, u16 sid, u16 pfsid,
 	if (mask) {
 		WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT +
 mask)) - 1));
 		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
-		desc.high = QI_DEV_IOTLB_ADDR(addr) |
 QI_DEV_IOTLB_SIZE;
+		desc.qw1 = QI_DEV_IOTLB_ADDR(addr) |
 QI_DEV_IOTLB_SIZE;
 	} else
-		desc.high = QI_DEV_IOTLB_ADDR(addr);
+		desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
 
 	if (qdep >= QI_DEV_IOTLB_MAX_INVS)
 		qdep = 0;
 
-	desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
+	desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
 		   QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
 	qi_submit_sync(&desc, iommu);
 }

@@ -1403,16 +1418,24 @@ static void __dmar_enable_qi(struct
 intel_iommu *iommu)
 	u32 sts;
 	unsigned long flags;
 	struct q_inval *qi = iommu->qi;
+	u64 val = virt_to_phys(qi->desc);
 
 	qi->free_head = qi->free_tail = 0;
 	qi->free_cnt = QI_LENGTH;
 
+	/*
+	 * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
+	 * is present.
+	 */
+	if (ecap_smts(iommu->ecap))
+		val |= (1 << 11) | 1;
+
 	raw_spin_lock_irqsave(&iommu->register_lock, flags);
 
 	/* write zero to the tail reg */
 	writel(0, iommu->reg + DMAR_IQT_REG);
 
-	dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi-
 >desc));
+	dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
 

drivers/iommu/dmar.c +6 -2

 	iommu->gcmd |= DMA_GCMD_QIE;
 	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

@@ -1448,8 +1471,12 @@ int dmar_enable_qi(struct intel_iommu
 *iommu)
 
 	qi = iommu->qi;
 
-
-	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC |
 __GFP_ZERO, 0);
+	/*
+	 * Need two pages to accommodate 256 descriptors of 256 bits each
+	 * if the remapping hardware supports scalable mode translation.
+	 */
+	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC |
 __GFP_ZERO,
+				     !!ecap_smts(iommu->ecap));

drivers/iommu/dmar.c +0 -0

 	if (!desc_page) {
 		kfree(qi);
 		iommu->qi = NULL;

drivers/iommu/intel-svm.c +0 -0

index 6c0bd9ee9602..a06ed098e928 100644
@@ -161,27 +161,40 @@ static void intel_flush_svm_range_dev (struct

intel_svm *svm, struct intel_svm_d
* because that's the only option the hardware gives us.
Despite
* the fact that they are actually only accessible through one.
*/

drivers/iommu/intel-svm.c +0 -1

 		if (gl)
-			desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-				QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |

QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +3 -0

+			desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+					QI_EIOTLB_DID(sdev->did) |
+

QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |

drivers/iommu/intel-svm.c +1 -1

+					QI_EIOTLB_TYPE;
 		else
-			desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID)

| QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +3 -1

-		desc.high = 0;
+			desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+					QI_EIOTLB_DID(sdev->did) |
+

QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |

drivers/iommu/intel-svm.c +2 -1

+					QI_EIOTLB_TYPE;
+		desc.qw1 = 0;
 	} else {
 		int mask = ilog2(__roundup_pow_of_two(pages));
 
-		desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-			QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |

QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +10 -3

-		desc.high = QI_EIOTLB_ADDR(address) | QI_EIOTLB_GL(gl) |
-			QI_EIOTLB_IH(ih) | QI_EIOTLB_AM(mask);
+		desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+				QI_EIOTLB_DID(sdev->did) |
+				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
+				QI_EIOTLB_TYPE;
+		desc.qw1 = QI_EIOTLB_ADDR(address) |
+				QI_EIOTLB_GL(gl) |
+				QI_EIOTLB_IH(ih) |
+				QI_EIOTLB_AM(mask);
 	}
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 	qi_submit_sync(&desc, svm->iommu);
 
 	if (sdev->dev_iotlb) {
-		desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) |

QI_DEV_EIOTLB_SID(sdev->sid) |

drivers/iommu/intel-svm.c +0 -1

-			QI_DEV_EIOTLB_QDEP(sdev->qdep) |

QI_DEIOTLB_TYPE;

drivers/iommu/intel-svm.c +4 -1

+		desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
+				QI_DEV_EIOTLB_SID(sdev->sid) |
+				QI_DEV_EIOTLB_QDEP(sdev->qdep) |
+				QI_DEIOTLB_TYPE;
 		if (pages == -1) {
-			desc.high = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |

QI_DEV_EIOTLB_SIZE;

drivers/iommu/intel-svm.c +2 -0

+			desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |
+					QI_DEV_EIOTLB_SIZE;
 		} else if (pages > 1) {
 			/* The least significant zero bit indicates the size. So,
 			 * for example, an "address" value of 0x12345f000

will

drivers/iommu/intel-svm.c +0 -0

@@ -189,10 +202,13 @@ static void intel_flush_svm_range_dev (struct

intel_svm *svm, struct intel_svm_d
unsigned long last = address + ((unsigned
long)(pages - 1) << VTD_PAGE_SHIFT);
unsigned long mask =
__rounddown_pow_of_two(address ^ last);

drivers/iommu/intel-svm.c +0 -1

-			desc.high = QI_DEV_EIOTLB_ADDR((address &

~mask) | (mask - 1)) | QI_DEV_EIOTLB_SIZE;

drivers/iommu/intel-svm.c +1 -0

+			desc.qw1 = QI_DEV_EIOTLB_ADDR((address &

~mask) |

drivers/iommu/intel-svm.c +9 -3

+					(mask - 1)) | QI_DEV_EIOTLB_SIZE;
 		} else {
-			desc.high = QI_DEV_EIOTLB_ADDR(address);
+			desc.qw1 = QI_DEV_EIOTLB_ADDR(address);
 		}
+		desc.qw2 = 0;
+		desc.qw3 = 0;
 		qi_submit_sync(&desc, svm->iommu);
 	}
 }

@@ -237,8 +253,11 @@ static void intel_flush_pasid_dev(struct intel_svm
 *svm, struct intel_svm_dev *s
 {
 	struct qi_desc desc;
 
-	desc.high = 0;
-	desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL
 | QI_PC_PASID(pasid);
+	desc.qw0 = QI_PC_TYPE | QI_PC_DID(sdev->did) |
+			QI_PC_PASID_SEL | QI_PC_PASID(pasid);
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 

drivers/iommu/intel-svm.c +1 -1

 	qi_submit_sync(&desc, svm->iommu);
 }

@@ -668,24 +687,27 @@ static irqreturn_t prq_event_thread(int irq, void
 *d)
 	no_pasid:
 		if (req->lpig) {
 			/* Page Group Response */
-			resp.low = QI_PGRP_PASID(req->pasid) |
+			resp.qw0 = QI_PGRP_PASID(req->pasid) |
 				QI_PGRP_DID((req->bus << 8) | req->devfn)

|

drivers/iommu/intel-svm.c +0 -2

 				QI_PGRP_PASID_P(req->pasid_present) |
 				QI_PGRP_RESP_TYPE;
-			resp.high = QI_PGRP_IDX(req->prg_index) |
-				QI_PGRP_PRIV(req->private) |

QI_PGRP_RESP_CODE(result);

drivers/iommu/intel-svm.c +3 -4

-
-			qi_submit_sync(&resp, iommu);
+			resp.qw1 = QI_PGRP_IDX(req->prg_index) |
+				QI_PGRP_PRIV(req->private) |
+				QI_PGRP_RESP_CODE(result);
 		} else if (req->srr) {
 			/* Page Stream Response */
-			resp.low = QI_PSTRM_IDX(req->prg_index) |
-				QI_PSTRM_PRIV(req->private) |

QI_PSTRM_BUS(req->bus) |

drivers/iommu/intel-svm.c +0 -1

-				QI_PSTRM_PASID(req->pasid) |

QI_PSTRM_RESP_TYPE;

drivers/iommu/intel-svm.c +0 -1

-			resp.high = QI_PSTRM_ADDR(address) |

QI_PSTRM_DEVFN(req->devfn) |

drivers/iommu/intel-svm.c +10 -2

+			resp.qw0 = QI_PSTRM_IDX(req->prg_index) |
+				QI_PSTRM_PRIV(req->private) |
+				QI_PSTRM_BUS(req->bus) |
+				QI_PSTRM_PASID(req->pasid) |
+				QI_PSTRM_RESP_TYPE;
+			resp.qw1 = QI_PSTRM_ADDR(address) |
+				QI_PSTRM_DEVFN(req->devfn) |
 				QI_PSTRM_RESP_CODE(result);
-
-			qi_submit_sync(&resp, iommu);
 		}
+		resp.qw2 = 0;
+		resp.qw3 = 0;
+		qi_submit_sync(&resp, iommu);
 
 		head = (head + sizeof(*req)) & PRQ_RING_MASK;
 	}

drivers/iommu/intel_irq_remapping.c +4 -2

index 967450bd421a..916391f33ca6 100644
@@ -145,9 +145,11 @@ static int qi_flush_iec(struct intel_iommu *iommu,
 int index, int mask)
 {
 	struct qi_desc desc;
 
-	desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
+	desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
 		   | QI_IEC_SELECTIVE;
-	desc.high = 0;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
 	return qi_submit_sync(&desc, iommu);

}

include/linux/intel-iommu.h +5 -3

index 41791903a5e3..72aff482b293 100644
@@ -340,12 +340,15 @@ enum {
 #define QI_GRAN_PSI_PASID		3
 
 struct qi_desc {
-	u64 low, high;
+	u64 qw0;
+	u64 qw1;
+	u64 qw2;
+	u64 qw3;
 };
 
 struct q_inval {
 	raw_spinlock_t  q_lock;
-	struct qi_desc  *desc;          /* invalidation queue */
+	void		*desc;          /* invalidation queue */
 	int             *desc_status;   /* desc status */
 	int             free_head;      /* first free entry */
 	int             free_tail;      /* last free entry */
--

2.17.1

Quoted 32 lines

From: Lu Baolu [mailto:baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

Intel vt-d spec rev3.0 requires software to use 256-bit descriptors in invalidation queue. As the spec reads in section 6.5.2:

Remapping hardware supporting Scalable Mode Translations (ECAP_REG.SMTS=1) allow software to additionally program the width of the descriptors (128-bits or 256-bits) that will be written into the Queue. Software should setup the Invalidation Queue for 256-bit descriptors before progra- mming remapping hardware for scalable-mode translation as 128-bit descriptors are treated as invalid descriptors (see Table 21 in Section 6.5.2.10) in scalable-mode.

This patch adds 256-bit invalidation descriptor support if the hardware presents scalable mode capability.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <jacob.jun.pan-VuQAYsv1563Yd54FQh9/[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]>

---
  drivers/iommu/dmar.c                | 83 +++++++++++++++++++----------
  drivers/iommu/intel-svm.c           | 76 ++++++++++++++++----------
  drivers/iommu/intel_irq_remapping.c |  6 ++-
  include/linux/intel-iommu.h         |  7 ++-
  4 files changed, 113 insertions(+), 59 deletions(-)

drivers/iommu/dmar.c static int qi_check_fault(struct intel_iommu · line 1164

Quoted 10 lines

drivers/iommu/dmar.c +1 -0

index d9c748b6f9e4..b1429fa2cf29 100644
@@ -1160,6 +1160,7 @@ static int qi_check_fault(struct intel_iommu
 *iommu, int index)
  	int head, tail;
  	struct q_inval *qi = iommu->qi;
  	int wait_index = (index + 1) % QI_LENGTH;
+	int shift = DMAR_IQ_SHIFT + !!ecap_smts(iommu->ecap);

could add a new macro: qi_shift()

drivers/iommu/dmar.c static int qi_check_fault(struct intel_iommu · line 1178

Quoted 11 lines

drivers/iommu/dmar.c +1 -1

  	if (qi->desc_status[wait_index] == QI_ABORT)
  		return -EAGAIN;

@@ -1173,13 +1174,15 @@ static int qi_check_fault(struct intel_iommu
 *iommu, int index)
  	 */
  	if (fault & DMA_FSTS_IQE) {
  		head = readl(iommu->reg + DMAR_IQH_REG);
-		if ((head >> DMAR_IQ_SHIFT) == index) {
+		if ((head >> shift) == index) {

could be another macro: qi_index(head)

drivers/iommu/dmar.c static int qi_check_fault(struct intel_iommu *iommu, int index) · line 1183

Quoted 11 lines

drivers/iommu/dmar.c +4 -4

+			struct qi_desc *desc = qi->desc + head;
+
  			pr_err("VT-d detected invalid descriptor: "
  				"low=%llx, high=%llx\n",
-				(unsigned long long)qi->desc[index].low,
-				(unsigned long long)qi->desc[index].high);
-			memcpy(&qi->desc[index], &qi->desc[wait_index],
-					sizeof(struct qi_desc));
+				(unsigned long long)desc->qw0,
+				(unsigned long long)desc->qw1);

what about qw2 and qw3 in 256-bit case?

Quoted 387 lines

drivers/iommu/dmar.c +2 -0

+			memcpy(desc, qi->desc + (wait_index << shift),
+			       1 << shift);
  			writel(DMA_FSTS_IQE, iommu->reg +
 DMAR_FSTS_REG);

  			return -EINVAL;
  		}

drivers/iommu/dmar.c +2 -2

@@ -1191,10 +1194,10 @@ static int qi_check_fault(struct intel_iommu
 *iommu, int index)
  	 */
  	if (fault & DMA_FSTS_ITE) {
  		head = readl(iommu->reg + DMAR_IQH_REG);
-		head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) %
 QI_LENGTH;
+		head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
  		head |= 1;
  		tail = readl(iommu->reg + DMAR_IQT_REG);
-		tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) %
 QI_LENGTH;
+		tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;

  		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);

drivers/iommu/dmar.c +12 -8

@@ -1222,15 +1225,14 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
  {
  	int rc;
  	struct q_inval *qi = iommu->qi;
-	struct qi_desc *hw, wait_desc;
+	int offset, shift, length;
+	struct qi_desc wait_desc;
  	int wait_index, index;
  	unsigned long flags;
 
  	if (!qi)
  		return 0;
 
-	hw = qi->desc;
-
  restart:
  	rc = 0;

@@ -1243,16 +1245,21 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
 
  	index = qi->free_head;
  	wait_index = (index + 1) % QI_LENGTH;
+	shift = DMAR_IQ_SHIFT + !!ecap_smts(iommu->ecap);
+	length = 1 << shift;
 
  	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
 
-	hw[index] = *desc;
-
-	wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
+	offset = index << shift;
+	memcpy(qi->desc + offset, desc, length);
+	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
  			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
-	wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
+	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
+	wait_desc.qw2 = 0;
+	wait_desc.qw3 = 0;
 
-	hw[wait_index] = wait_desc;
+	offset = wait_index << shift;
+	memcpy(qi->desc + offset, &wait_desc, length);
 
  	qi->free_head = (qi->free_head + 2) % QI_LENGTH;

  	qi->free_cnt -= 2;

drivers/iommu/dmar.c +1 -1

@@ -1261,7 +1268,7 @@ int qi_submit_sync(struct qi_desc *desc, struct
 intel_iommu *iommu)
  	 * update the HW tail register indicating the presence of
  	 * new descriptors.
  	 */
-	writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg +
 DMAR_IQT_REG);
+	writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
 

  	while (qi->desc_status[wait_index] != QI_DONE) {
  		/*

drivers/iommu/dmar.c +8 -4

@@ -1298,8 +1305,10 @@ void qi_global_iec(struct intel_iommu *iommu)
  {
  	struct qi_desc desc;
 
-	desc.low = QI_IEC_TYPE;
-	desc.high = 0;
+	desc.qw0 = QI_IEC_TYPE;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
  	/* should never fail */
  	qi_submit_sync(&desc, iommu);

@@ -1310,9 +1319,11 @@ void qi_flush_context(struct intel_iommu
 *iommu, u16 did, u16 sid, u8 fm,
  {
  	struct qi_desc desc;
 
-	desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
+	desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
  			| QI_CC_GRAN(type) | QI_CC_TYPE;
-	desc.high = 0;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
  	qi_submit_sync(&desc, iommu);

  }

drivers/iommu/dmar.c +4 -2

@@ -1331,10 +1342,12 @@ void qi_flush_iotlb(struct intel_iommu
 *iommu, u16 did, u64 addr,
  	if (cap_read_drain(iommu->cap))
  		dr = 1;
 
-	desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) |
 QI_IOTLB_DW(dw)
+	desc.qw0 = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) |
 QI_IOTLB_DW(dw)
  		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
-	desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
+	desc.qw1 = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
  		| QI_IOTLB_AM(size_order);
+	desc.qw2 = 0;
+	desc.qw3 = 0;

  	qi_submit_sync(&desc, iommu);
  }

drivers/iommu/dmar.c +14 -4

@@ -1347,15 +1360,17 @@ void qi_flush_dev_iotlb(struct intel_iommu
 *iommu, u16 sid, u16 pfsid,
  	if (mask) {
  		WARN_ON_ONCE(addr & ((1ULL << (VTD_PAGE_SHIFT +
 mask)) - 1));
  		addr |= (1ULL << (VTD_PAGE_SHIFT + mask - 1)) - 1;
-		desc.high = QI_DEV_IOTLB_ADDR(addr) |
 QI_DEV_IOTLB_SIZE;
+		desc.qw1 = QI_DEV_IOTLB_ADDR(addr) |
 QI_DEV_IOTLB_SIZE;
  	} else
-		desc.high = QI_DEV_IOTLB_ADDR(addr);
+		desc.qw1 = QI_DEV_IOTLB_ADDR(addr);
 
  	if (qdep >= QI_DEV_IOTLB_MAX_INVS)
  		qdep = 0;
 
-	desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
+	desc.qw0 = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
  		   QI_DIOTLB_TYPE | QI_DEV_IOTLB_PFSID(pfsid);
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
  	qi_submit_sync(&desc, iommu);
  }

@@ -1403,16 +1418,24 @@ static void __dmar_enable_qi(struct
 intel_iommu *iommu)
  	u32 sts;
  	unsigned long flags;
  	struct q_inval *qi = iommu->qi;
+	u64 val = virt_to_phys(qi->desc);
 
  	qi->free_head = qi->free_tail = 0;
  	qi->free_cnt = QI_LENGTH;
 
+	/*
+	 * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
+	 * is present.
+	 */
+	if (ecap_smts(iommu->ecap))
+		val |= (1 << 11) | 1;
+
  	raw_spin_lock_irqsave(&iommu->register_lock, flags);
 
  	/* write zero to the tail reg */
  	writel(0, iommu->reg + DMAR_IQT_REG);
 
-	dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi-
 > desc));
+	dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
 

  	iommu->gcmd |= DMA_GCMD_QIE;
  	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);

drivers/iommu/dmar.c +6 -2

@@ -1448,8 +1471,12 @@ int dmar_enable_qi(struct intel_iommu
 *iommu)
 
  	qi = iommu->qi;
 
-
-	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC |
 __GFP_ZERO, 0);
+	/*
+	 * Need two pages to accommodate 256 descriptors of 256 bits each
+	 * if the remapping hardware supports scalable mode translation.
+	 */
+	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC |
 __GFP_ZERO,
+				     !!ecap_smts(iommu->ecap));

  	if (!desc_page) {
  		kfree(qi);
  		iommu->qi = NULL;

drivers/iommu/intel-svm.c +0 -0

index 6c0bd9ee9602..a06ed098e928 100644
@@ -161,27 +161,40 @@ static void intel_flush_svm_range_dev (struct

intel_svm *svm, struct intel_svm_d
* because that's the only option the hardware gives us.
Despite
* the fact that they are actually only accessible through one.
*/
if (gl)

drivers/iommu/intel-svm.c +0 -1

-			desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-				QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |

QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +3 -0

+			desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+					QI_EIOTLB_DID(sdev->did) |
+

QI_EIOTLB_GRAN(QI_GRAN_ALL_ALL) |

drivers/iommu/intel-svm.c +1 -1

+					QI_EIOTLB_TYPE;
  		else
-			desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-				QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID)

| QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +3 -1

-		desc.high = 0;
+			desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+					QI_EIOTLB_DID(sdev->did) |
+

QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |

drivers/iommu/intel-svm.c +2 -1

+					QI_EIOTLB_TYPE;
+		desc.qw1 = 0;
  	} else {
  		int mask = ilog2(__roundup_pow_of_two(pages));
 
-		desc.low = QI_EIOTLB_PASID(svm->pasid) |

QI_EIOTLB_DID(sdev->did) |

drivers/iommu/intel-svm.c +0 -1

-			QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |

QI_EIOTLB_TYPE;

drivers/iommu/intel-svm.c +10 -3

-		desc.high = QI_EIOTLB_ADDR(address) | QI_EIOTLB_GL(gl) |
-			QI_EIOTLB_IH(ih) | QI_EIOTLB_AM(mask);
+		desc.qw0 = QI_EIOTLB_PASID(svm->pasid) |
+				QI_EIOTLB_DID(sdev->did) |
+				QI_EIOTLB_GRAN(QI_GRAN_PSI_PASID) |
+				QI_EIOTLB_TYPE;
+		desc.qw1 = QI_EIOTLB_ADDR(address) |
+				QI_EIOTLB_GL(gl) |
+				QI_EIOTLB_IH(ih) |
+				QI_EIOTLB_AM(mask);
  	}
+	desc.qw2 = 0;
+	desc.qw3 = 0;
  	qi_submit_sync(&desc, svm->iommu);
 
  	if (sdev->dev_iotlb) {
-		desc.low = QI_DEV_EIOTLB_PASID(svm->pasid) |

QI_DEV_EIOTLB_SID(sdev->sid) |

drivers/iommu/intel-svm.c +0 -1

-			QI_DEV_EIOTLB_QDEP(sdev->qdep) |

QI_DEIOTLB_TYPE;

drivers/iommu/intel-svm.c +4 -1

+		desc.qw0 = QI_DEV_EIOTLB_PASID(svm->pasid) |
+				QI_DEV_EIOTLB_SID(sdev->sid) |
+				QI_DEV_EIOTLB_QDEP(sdev->qdep) |
+				QI_DEIOTLB_TYPE;
  		if (pages == -1) {
-			desc.high = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |

QI_DEV_EIOTLB_SIZE;

drivers/iommu/intel-svm.c +2 -0

+			desc.qw1 = QI_DEV_EIOTLB_ADDR(-1ULL >> 1) |
+					QI_DEV_EIOTLB_SIZE;
  		} else if (pages > 1) {
  			/* The least significant zero bit indicates the size. So,
  			 * for example, an "address" value of 0x12345f000

will

drivers/iommu/intel-svm.c +0 -0

@@ -189,10 +202,13 @@ static void intel_flush_svm_range_dev (struct

intel_svm *svm, struct intel_svm_d
unsigned long last = address + ((unsigned
long)(pages - 1) << VTD_PAGE_SHIFT);
unsigned long mask =
__rounddown_pow_of_two(address ^ last);

drivers/iommu/intel-svm.c +0 -1

-			desc.high = QI_DEV_EIOTLB_ADDR((address &

~mask) | (mask - 1)) | QI_DEV_EIOTLB_SIZE;

drivers/iommu/intel-svm.c +1 -0

+			desc.qw1 = QI_DEV_EIOTLB_ADDR((address &

~mask) |

drivers/iommu/intel-svm.c +9 -3

+					(mask - 1)) | QI_DEV_EIOTLB_SIZE;
  		} else {
-			desc.high = QI_DEV_EIOTLB_ADDR(address);
+			desc.qw1 = QI_DEV_EIOTLB_ADDR(address);
  		}
+		desc.qw2 = 0;
+		desc.qw3 = 0;
  		qi_submit_sync(&desc, svm->iommu);
  	}
  }

@@ -237,8 +253,11 @@ static void intel_flush_pasid_dev(struct intel_svm
 *svm, struct intel_svm_dev *s
  {
  	struct qi_desc desc;
 
-	desc.high = 0;
-	desc.low = QI_PC_TYPE | QI_PC_DID(sdev->did) | QI_PC_PASID_SEL
 | QI_PC_PASID(pasid);
+	desc.qw0 = QI_PC_TYPE | QI_PC_DID(sdev->did) |
+			QI_PC_PASID_SEL | QI_PC_PASID(pasid);
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 

  	qi_submit_sync(&desc, svm->iommu);
  }

drivers/iommu/intel-svm.c +1 -1

@@ -668,24 +687,27 @@ static irqreturn_t prq_event_thread(int irq, void
 *d)
  	no_pasid:
  		if (req->lpig) {
  			/* Page Group Response */
-			resp.low = QI_PGRP_PASID(req->pasid) |
+			resp.qw0 = QI_PGRP_PASID(req->pasid) |
  				QI_PGRP_DID((req->bus << 8) | req->devfn)

|
QI_PGRP_PASID_P(req->pasid_present) | QI_PGRP_RESP_TYPE;

drivers/iommu/intel-svm.c +0 -2

-			resp.high = QI_PGRP_IDX(req->prg_index) |
-				QI_PGRP_PRIV(req->private) |

QI_PGRP_RESP_CODE(result);

drivers/iommu/intel-svm.c +3 -4

-
-			qi_submit_sync(&resp, iommu);
+			resp.qw1 = QI_PGRP_IDX(req->prg_index) |
+				QI_PGRP_PRIV(req->private) |
+				QI_PGRP_RESP_CODE(result);
  		} else if (req->srr) {
  			/* Page Stream Response */
-			resp.low = QI_PSTRM_IDX(req->prg_index) |
-				QI_PSTRM_PRIV(req->private) |

QI_PSTRM_BUS(req->bus) |

drivers/iommu/intel-svm.c +0 -1

-				QI_PSTRM_PASID(req->pasid) |

QI_PSTRM_RESP_TYPE;

drivers/iommu/intel-svm.c +0 -1

-			resp.high = QI_PSTRM_ADDR(address) |

QI_PSTRM_DEVFN(req->devfn) |

drivers/iommu/intel-svm.c +10 -2

+			resp.qw0 = QI_PSTRM_IDX(req->prg_index) |
+				QI_PSTRM_PRIV(req->private) |
+				QI_PSTRM_BUS(req->bus) |
+				QI_PSTRM_PASID(req->pasid) |
+				QI_PSTRM_RESP_TYPE;
+			resp.qw1 = QI_PSTRM_ADDR(address) |
+				QI_PSTRM_DEVFN(req->devfn) |
  				QI_PSTRM_RESP_CODE(result);
-
-			qi_submit_sync(&resp, iommu);
  		}
+		resp.qw2 = 0;
+		resp.qw3 = 0;
+		qi_submit_sync(&resp, iommu);
 
  		head = (head + sizeof(*req)) & PRQ_RING_MASK;
  	}

drivers/iommu/intel_irq_remapping.c +4 -2

index 967450bd421a..916391f33ca6 100644
@@ -145,9 +145,11 @@ static int qi_flush_iec(struct intel_iommu *iommu,
 int index, int mask)
  {
  	struct qi_desc desc;
 
-	desc.low = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
+	desc.qw0 = QI_IEC_IIDEX(index) | QI_IEC_TYPE | QI_IEC_IM(mask)
  		   | QI_IEC_SELECTIVE;
-	desc.high = 0;
+	desc.qw1 = 0;
+	desc.qw2 = 0;
+	desc.qw3 = 0;
 
  	return qi_submit_sync(&desc, iommu);

  }

include/linux/intel-iommu.h +4 -1

index 41791903a5e3..72aff482b293 100644
@@ -340,12 +340,15 @@ enum {
  #define QI_GRAN_PSI_PASID		3
 
  struct qi_desc {
-	u64 low, high;
+	u64 qw0;
+	u64 qw1;
+	u64 qw2;
+	u64 qw3;
  };
 

Quoted 9 lines

include/linux/intel-iommu.h +1 -2

  struct q_inval {
  	raw_spinlock_t  q_lock;
-	struct qi_desc  *desc;          /* invalidation queue */
+	void		*desc;          /* invalidation queue */
  	int             *desc_status;   /* desc status */
  	int             free_head;      /* first free entry */
  	int             free_tail;      /* last free entry */
--

2.17.1

From: Lu Baolu [mailto:[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

This adds the interfaces to setup or tear down the structures for second level page table translations. This includes types of second level only translation and pass through.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <[email protected]>

Reviewed-by Ashok Raj <[email protected]>

---
 drivers/iommu/intel-iommu.c |   2 +-
 drivers/iommu/intel-pasid.c | 246
++++++++++++++++++++++++++++++++++++
 drivers/iommu/intel-pasid.h |   7 +
 include/linux/intel-iommu.h |   3 +
 4 files changed, 257 insertions(+), 1 deletion(-)

drivers/iommu/intel-iommu.c +1 -1

index 562da10bf93e..de6b909bb47a 100644
@@ -1232,7 +1232,7 @@ static void iommu_set_root_entry(struct
 intel_iommu *iommu)
 	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
 }
 
-static void iommu_flush_write_buffer(struct intel_iommu *iommu)
+void iommu_flush_write_buffer(struct intel_iommu *iommu)
 {
 	u32 val;

drivers/iommu/intel-iommu.c +0 -0

 	unsigned long flag;

drivers/iommu/intel-pasid.c +16 -0

index d6e90cd5b062..edcea1d8b9fc 100644
@@ -9,6 +9,7 @@
 
 #define pr_fmt(fmt)	"DMAR: " fmt
 
+#include <linux/bitops.h>
 #include <linux/dmar.h>
 #include <linux/intel-iommu.h>
 #include <linux/iommu.h>

@@ -291,3 +292,248 @@ void intel_pasid_clear_entry(struct device *dev,
 int pasid)
 
 	pasid_clear_entry(pe);
 }
+
+static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits)
+{
+	u64 old;
+
+	old = READ_ONCE(*ptr);
+	WRITE_ONCE(*ptr, (old & ~mask) | bits);
+}
+
+/*
+ * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode
+ * PASID entry.
+ */
+static inline void
+pasid_set_domain_id(struct pasid_entry *pe, u64 value)

drivers/iommu/intel-pasid.c +40 -0

+{
+	pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value);
+}
+
+/*
+ * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID
+ * entry.
+ */
+static inline void
+pasid_set_address_width(struct pasid_entry *pe, u64 value)
+{
+	pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2);
+}
+
+/*
+ * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8)
+ * of a scalable mode PASID entry.
+ */
+static inline void
+pasid_set_translation_type(struct pasid_entry *pe, u64 value)
+{
+	pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6);
+}
+
+/*
+ * Enable fault processing by clearing the FPD(Fault Processing
+ * Disable) field (Bit 1) of a scalable mode PASID entry.
+ */
+static inline void pasid_set_fault_enable(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[0], 1 << 1, 0);
+}
+
+/*
+ * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a
+ * scalable mode PASID entry.
+ */
+static inline void pasid_set_sre(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[2], 1 << 0, 1);

drivers/iommu/intel-pasid.c +4 -0

+{
+	struct qi_desc desc;
+
+	desc.qw0 = QI_PC_DID(did) | QI_PC_PASID_SEL |

QI_PC_PASID(pasid);

drivers/iommu/intel-pasid.c +5 -0

+{
+	struct qi_desc desc;
+
+	desc.qw0 = QI_EIOTLB_PASID(pasid) | QI_EIOTLB_DID(did) |
+			QI_EIOTLB_GRAN(QI_GRAN_NONG_PASID) |

QI_EIOTLB_TYPE;

drivers/iommu/intel-pasid.c +13 -0

+{
+	struct device_domain_info *info;
+	u16 sid, qdep, pfsid;
+
+	info = dev->archdata.iommu;
+	if (!info || !info->ats_enabled)
+		return;
+
+	sid = info->bus << 8 | info->devfn;
+	qdep = info->ats_qdep;
+	pfsid = info->pfsid;
+
+	qi_flush_dev_iotlb(iommu, sid, pfsid, qdep, 0, 64 -

VTD_PAGE_SHIFT);

drivers/iommu/intel-pasid.c +14 -0

+
+	intel_pasid_clear_entry(dev, pasid);
+
+	if (!ecap_coherent(iommu->ecap)) {
+		pte = intel_pasid_get_entry(dev, pasid);
+		clflush_cache_range(pte, sizeof(*pte));
+	}
+
+	pasid_based_pasid_cache_invalidation(iommu, did, pasid);
+	pasid_based_iotlb_cache_invalidation(iommu, did, pasid);
+
+	/* Device IOTLB doesn't need to be flushed in caching mode. */
+	if (!cap_caching_mode(iommu->cap))
+		pasid_based_dev_iotlb_cache_invalidation(iommu, dev,

pasid);

drivers/iommu/intel-pasid.c +6 -0

+				   struct dmar_domain *domain,
+				   struct device *dev, int pasid,
+				   bool pass_through)
+{
+	struct pasid_entry *pte;
+	struct dma_pte *pgd;

drivers/iommu/intel-pasid.c +5 -0

+	 * than default. Unnecessary for PT mode.
+	 */
+	pgd = domain->pgd;
+	if (!pass_through) {
+		for (agaw = domain->agaw; agaw != iommu->agaw; agaw--)

{

drivers/iommu/intel-pasid.c +14 -0

+			pgd = phys_to_virt(dma_pte_addr(pgd));
+			if (!dma_pte_present(pgd)) {
+				dev_err(dev, "Invalid domain page table\n");
+				return -EINVAL;
+			}
+		}
+	}
+	pgd_val = pass_through ? 0 : virt_to_phys(pgd);
+	did = pass_through ? FLPT_DEFAULT_DID :
+			domain->iommu_did[iommu->seq_id];
+
+	pte = intel_pasid_get_entry(dev, pasid);
+	if (!pte) {
+		dev_err(dev, "Failed to get pasid entry of PASID %d\n",

pasid);

drivers/iommu/intel-pasid.c +46 -0

+		return -ENODEV;
+	}
+
+	pasid_clear_entry(pte);
+	pasid_set_domain_id(pte, did);
+
+	if (!pass_through)
+		pasid_set_address_root(pte, pgd_val);
+
+	pasid_set_address_width(pte, iommu->agaw);
+	pasid_set_translation_type(pte, pass_through ? 4 : 2);
+	pasid_set_fault_enable(pte);
+	pasid_set_page_snoop(pte, !!ecap_smpwc(iommu->ecap));
+
+	/*
+	 * Since it is a second level only translation setup, we should
+	 * set SRE bit as well (addresses are expected to be GPAs).
+	 */
+	pasid_set_sre(pte);
+	pasid_set_present(pte);
+
+	if (!ecap_coherent(iommu->ecap))
+		clflush_cache_range(pte, sizeof(*pte));
+
+	if (cap_caching_mode(iommu->cap)) {
+		pasid_based_pasid_cache_invalidation(iommu, did, pasid);
+		pasid_based_iotlb_cache_invalidation(iommu, did, pasid);
+	} else {
+		iommu_flush_write_buffer(iommu);
+	}
+
+	return 0;
+}
+
+/*
+ * Tear down the scalable mode pasid table entry for second only or
+ * passthrough translation type.
+ */
+void intel_pasid_tear_down_second_level(struct intel_iommu *iommu,
+					struct dmar_domain *domain,
+					struct device *dev, int pasid)
+{
+	u16 did = domain->iommu_did[iommu->seq_id];
+
+	tear_down_one_pasid_entry(iommu, dev, did, pasid);
+}

drivers/iommu/intel-pasid.h +7 -0

index 03c1612d173c..85b158a1826a 100644
@@ -49,5 +49,12 @@ struct pasid_table *intel_pasid_get_table(struct
 device *dev);
 int intel_pasid_get_dev_max_id(struct device *dev);
 struct pasid_entry *intel_pasid_get_entry(struct device *dev, int pasid);
 void intel_pasid_clear_entry(struct device *dev, int pasid);
+int intel_pasid_setup_second_level(struct intel_iommu *iommu,
+				   struct dmar_domain *domain,
+				   struct device *dev, int pasid,
+				   bool pass_through);
+void intel_pasid_tear_down_second_level(struct intel_iommu *iommu,
+					struct dmar_domain *domain,
+					struct device *dev, int pasid);
 

#endif /* __INTEL_PASID_H */

include/linux/intel-iommu.h +3 -1

index 72aff482b293..d77d23dfd221 100644
@@ -115,6 +115,8 @@
  * Extended Capability Register
  */
 
+#define ecap_smpwc(e)		(((e) >> 48) & 0x1)
+#define ecap_slts(e)		(((e) >> 46) & 0x1)
 #define ecap_smts(e)		(((e) >> 43) & 0x1)
 #define ecap_dit(e)		((e >> 41) & 0x1)
 #define ecap_pasid(e)		((e >> 40) & 0x1)

@@ -571,6 +573,7 @@ void free_pgtable_page(void *vaddr);
 struct intel_iommu *domain_get_iommu(struct dmar_domain *domain);
 int for_each_device_domain(int (*fn)(struct device_domain_info *info,
 				     void *data), void *data);
+void iommu_flush_write_buffer(struct intel_iommu *iommu);
 
 #ifdef CONFIG_INTEL_IOMMU_SVM
 int intel_svm_init(struct intel_iommu *iommu);
--

2.17.1

Quoted 73 lines

From: Lu Baolu [mailto:baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]]
Sent: Thursday, August 30, 2018 9:35 AM

This adds the interfaces to setup or tear down the structures for second level page table translations. This includes types of second level only translation and pass through.

Cc Ashok Raj <[email protected]>

Cc Jacob Pan <jacob.jun.pan-VuQAYsv1563Yd54FQh9/[email protected]>

Cc Kevin Tian <[email protected]>

Cc Liu Yi L <[email protected]>

Signed-off-by Sanjay Kumar <[email protected]>

Signed-off-by Lu Baolu <baolu.lu-VuQAYsv1563Yd54FQh9/[email protected]>

Reviewed-by Ashok Raj <[email protected]>

---
  drivers/iommu/intel-iommu.c |   2 +-
  drivers/iommu/intel-pasid.c | 246
++++++++++++++++++++++++++++++++++++
  drivers/iommu/intel-pasid.h |   7 +
  include/linux/intel-iommu.h |   3 +
  4 files changed, 257 insertions(+), 1 deletion(-)

drivers/iommu/intel-iommu.c +1 -1

index 562da10bf93e..de6b909bb47a 100644
@@ -1232,7 +1232,7 @@ static void iommu_set_root_entry(struct
 intel_iommu *iommu)
  	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
  }
 
-static void iommu_flush_write_buffer(struct intel_iommu *iommu)
+void iommu_flush_write_buffer(struct intel_iommu *iommu)
  {
  	u32 val;

unsigned long flag;

drivers/iommu/intel-pasid.c +21 -0

index d6e90cd5b062..edcea1d8b9fc 100644
@@ -9,6 +9,7 @@
 
  #define pr_fmt(fmt)	"DMAR: " fmt
 
+#include <linux/bitops.h>
  #include <linux/dmar.h>
  #include <linux/intel-iommu.h>
  #include <linux/iommu.h>

@@ -291,3 +292,248 @@ void intel_pasid_clear_entry(struct device *dev,
 int pasid)
 
  	pasid_clear_entry(pe);
  }
+
+static inline void pasid_set_bits(u64 *ptr, u64 mask, u64 bits)
+{
+	u64 old;
+
+	old = READ_ONCE(*ptr);
+	WRITE_ONCE(*ptr, (old & ~mask) | bits);
+}
+
+/*
+ * Setup the DID(Domain Identifier) field (Bit 64~79) of scalable mode
+ * PASID entry.
+ */
+static inline void
+pasid_set_domain_id(struct pasid_entry *pe, u64 value)
+{
+	pasid_set_bits(&pe->val[1], GENMASK_ULL(15, 0), value);
+}
+
+/*

drivers/iommu/intel-pasid.c

Quoted 5 lines

+ * Setup the SLPTPTR(Second Level Page Table Pointer) field (Bit 12~63)
+ * of a scalable mode PASID entry.
+ */

drivers/iommu/intel-pasid.c +2 -0

+static inline void
+pasid_set_address_root(struct pasid_entry *pe, u64 value)

is address_root too general? especially when the entry could contain both 1st level and 2nd level pointers.

Quoted 40 lines

drivers/iommu/intel-pasid.c +40 -0

+{
+	pasid_set_bits(&pe->val[0], VTD_PAGE_MASK, value);
+}
+
+/*
+ * Setup the AW(Address Width) field (Bit 2~4) of a scalable mode PASID
+ * entry.
+ */
+static inline void
+pasid_set_address_width(struct pasid_entry *pe, u64 value)
+{
+	pasid_set_bits(&pe->val[0], GENMASK_ULL(4, 2), value << 2);
+}
+
+/*
+ * Setup the PGTT(PASID Granular Translation Type) field (Bit 6~8)
+ * of a scalable mode PASID entry.
+ */
+static inline void
+pasid_set_translation_type(struct pasid_entry *pe, u64 value)
+{
+	pasid_set_bits(&pe->val[0], GENMASK_ULL(8, 6), value << 6);
+}
+
+/*
+ * Enable fault processing by clearing the FPD(Fault Processing
+ * Disable) field (Bit 1) of a scalable mode PASID entry.
+ */
+static inline void pasid_set_fault_enable(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[0], 1 << 1, 0);
+}
+
+/*
+ * Setup the SRE(Supervisor Request Enable) field (Bit 128) of a
+ * scalable mode PASID entry.
+ */
+static inline void pasid_set_sre(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[2], 1 << 0, 1);

drivers/iommu/intel-pasid.c void intel_pasid_clear_entry(struct device *dev, int pasid) · line 369

Quoted 10 lines

drivers/iommu/intel-pasid.c +10 -0

+}
+
+/*
+ * Setup the P(Present) field (Bit 0) of a scalable mode PASID
+ * entry.
+ */
+static inline void pasid_set_present(struct pasid_entry *pe)
+{
+	pasid_set_bits(&pe->val[0], 1 << 0, 1);
+}

it's a long list and there could be more in the future. What about defining some macro to simplify LOC, e.g.

#define PASID_SET(name, i, m, b) \
static inline void pasid_set_name(struct pasid_entry *pe) \