iommu/vt-d: Fixes for v6.0-rc3
Subsystems (2)
Lu Baolu
maintainercover iommu/vt-d: Fixes for v6.0-rc3
Hi Joerg,
Below fixes are queued for v6.0-rc3. They aim to fix:
- Boot kdump kernels with VT-d scalable mode on
- Calculate the right page table levels
- Fix a recursive lock issue
- Fix a lockdep splat issue
Please consider it for the iommu/fix branch.
Best regards,
Lu Baolu
Lu Baolu (4):
iommu/vt-d: Fix kdump kernels boot failure with scalable mode iommu/vt-d: Correctly calculate sagaw value of IOMMU iommu/vt-d: Fix recursive lock issue in iommu_flush_dev_iotlb() iommu/vt-d: Fix lockdep splat due to klist iteration in atomic context
drivers/iommu/intel/iommu.h | 9 +- drivers/iommu/intel/iommu.c | 214 +++++++++++++++++++----------------- 2 files changed, 117 insertions(+), 106 deletions(-)
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 1/4 iommu/vt-d: Fix kdump kernels boot failure with scalable mode
The translation table copying code for kdump kernels is currently based on the extended root/context entry formats of ECS mode defined in older VT-d v2.5, and doesn't handle the scalable mode formats. This causes the kexec capture kernel boot failure with DMAR faults if the IOMMU was enabled in scalable mode by the previous kernel.
The ECS mode has already been deprecated by the VT-d spec since v3.0 and Intel IOMMU driver doesn't support this mode as there's no real hardware implementation. Hence this converts ECS checking in copying table code into scalable mode.
The existing copying code consumes a bit in the context entry as a mark of copied entry. It needs to work for the old format as well as for the extended context entries. As it's hard to find such a common bit for both legacy and scalable mode context entries. This replaces it with a per- IOMMU bitmap.
--- drivers/iommu/intel/iommu.h | 9 ++-- drivers/iommu/intel/iommu.c | 100 ++++++++++++++++-------------------- 2 files changed, 50 insertions(+), 59 deletions(-)
drivers/iommu/intel/iommu.h +6 -3
@@ -197,7 +197,6 @@#define ecap_dis (e) (((e) >> 27) & 0x1)#define ecap_nest (e) (((e) >> 26) & 0x1)#define ecap_mts (e) (((e) >> 25) & 0x1) -#define ecap_ecs (e) (((e) >> 24) & 0x1)#define ecap_iotlb_offset (e) ((((e) >> 8) & 0x3ff) * 16)#define ecap_max_iotlb_offset (e) (ecap_iotlb_offset(e) + 16)#define ecap_coherent (e) ((e) & 0x1)
@@ -265,7 +264,6 @@#define DMA_GSTS_CFIS (((u32)1) << 23)/* DMA_RTADDR_REG */ -#define DMA_RTADDR_RTT (((u64)1) << 11)#define DMA_RTADDR_SMT (((u64)1) << 10)/* CCMD_REG */
@@ -579,6 +577,7 @@ struct intel_iommu {#ifdef CONFIG_INTEL_IOMMU unsigned long *domain_ids;/* bitmap of domains */ +unsigned long *copied_tables;/* bitmap of copied tables */ spinlock_t lock;/* protect context, domain ids */ struct root_entry *root_entry;/* virtual address */
@@ -701,6 +700,11 @@ static inline int nr_pte_to_next_page(struct dma_pte *pte) (struct dma_pte *)ALIGN ((unsigned long )pte,VTD_PAGE_SIZE ) - pte; } +static inline bool context_present (struct context_entry *context) +{ +return (context->lo &1 ); +} +extern struct dmar_drhd_unit *dmar_find_matched_drhd_unit (struct pci_dev *dev);extern int dmar_enable_qi (struct intel_iommu *iommu);
@@ -784,7 +788,6 @@ static inline void intel_iommu_debugfs_init(void) {}#endif /* CONFIG_INTEL_IOMMU_DEBUGFS */ extern const struct attribute_group *intel_iommu_groups[]; -bool context_present (struct context_entry *context);struct context_entry *iommu_context_addr (struct intel_iommu *iommu,u8 bus,u8 devfn,int alloc);
drivers/iommu/intel/iommu.c +44 -56
166 lines not on this page · raw message
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 2/4 iommu/vt-d: Correctly calculate sagaw value of IOMMU
In reply to Lu BaoluThe Intel IOMMU driver possibly selects between the first-level and the second-level translation tables for DMA address translation. However, the levels of page-table walks for the 4KB base page size are calculated from the SAGAW field of the capability register, which is only valid for the second-level page table. This causes the IOMMU driver to stop working if the hardware (or the emulated IOMMU) advertises only first-level translation capability and reports the SAGAW field as 0.
This solves the above problem by considering both the first level and the second level when calculating the supported page table levels.
--- drivers/iommu/intel/iommu.c | 28 +++++++++++++++++++++++++--- 1 file changed, 25 insertions(+), 3 deletions(-)
drivers/iommu/intel/iommu.c +25 -3
@@ -390,14 +390,36 @@ static inline int domain_pfn_supported(struct dmar_domain *domain,return !(addr_width <BITS_PER_LONG && pfn >> addr_width); } +/* +* Calculate the Supported Adjusted Guest Address Widths of an IOMMU. +* Refer to 11.4.2 of the VT-d spec for the encoding of each bit of +* the returned SAGAW. +*/ +static unsigned long __iommu_calculate_sagaw (struct intel_iommu *iommu) +{ +unsigned long fl_sagaw, sl_sagaw; + + fl_sagaw =BIT (2 ) | (cap_fl1gp_support (iommu->cap) ?BIT (3 ) :0 ); + sl_sagaw =cap_sagaw (iommu->cap); + +/* Second level only. */ +if (!sm_supported (iommu) || !ecap_flts (iommu->ecap)) +return sl_sagaw; + +/* First level only. */ +if (!ecap_slts (iommu->ecap)) +return fl_sagaw; + +return fl_sagaw & sl_sagaw; +} +static int __iommu_calculate_agaw (struct intel_iommu *iommu,int max_gaw) {unsigned long sagaw;int agaw; - sagaw =cap_sagaw (iommu->cap); -for (agaw =width_to_agaw (max_gaw); - agaw >=0 ; agaw--) { + sagaw =__iommu_calculate_sagaw (iommu); +for (agaw =width_to_agaw (max_gaw); agaw >=0 ; agaw--) {if (test_bit (agaw, &sagaw))break ; }
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 3/4 iommu/vt-d: Fix recursive lock issue in iommu_flush_dev_iotlb()
In reply to Lu BaoluThe per domain spinlock is acquired in iommu_flush_dev_iotlb(), which is possbile to be called in the interrupt context. For example, the drm-intel's CI system got completely blocked with below error:
WARNING: inconsistent lock state
6.0.0-rc1-CI_DRM_11990-g6590d43d39b9+ #1 Not tainted
--------------------------------
inconsistent {SOFTIRQ-ON-W} -> {IN-SOFTIRQ-W} usage.
swapper/6/0 [HC0[0]:SC1[1]:HE1:SE0] takes:
ffff88810440d678 (&domain->lock){+.?.}-{2:2}, at: iommu_flush_dev_iotlb.part.61+0x23/0x80 {SOFTIRQ-ON-W} state was registered at:
lock_acquire+0xd3/0x310
_raw_spin_lock+0x2a/0x40
domain_update_iommu_cap+0x20b/0x2c0
intel_iommu_attach_device+0x5bd/0x860
__iommu_attach_device+0x18/0xe0
bus_iommu_probe+0x1f3/0x2d0
bus_set_iommu+0x82/0xd0
intel_iommu_init+0xe45/0x102a
pci_iommu_init+0x9/0x31
do_one_initcall+0x53/0x2f0
kernel_init_freeable+0x18f/0x1e1
kernel_init+0x11/0x120
ret_from_fork+0x1f/0x30
irq event stamp: 162354
hardirqs last enabled at (162354): [<ffffffff81b59274>] _raw_spin_unlock_irqrestore+0x54/0x70
hardirqs last disabled at (162353): [<ffffffff81b5901b>] _raw_spin_lock_irqsave+0x4b/0x50
softirqs last enabled at (162338): [<ffffffff81e00323>] __do_softirq+0x323/0x48e
softirqs last disabled at (162349): [<ffffffff810c1588>] irq_exit_rcu+0xb8/0xe0
other info that might help us debug this:
Possible unsafe locking scenario:
CPU0
----
lock(&domain->lock);
<Interrupt>
lock(&domain->lock);
*** DEADLOCK ***
1 lock held by swapper/6/0:This coverts the spin_lock/unlock() into the irq save/restore varieties to fix the recursive locking issues.
--- drivers/iommu/intel/iommu.c | 39 ++++++++++++++++++++++--------------- 1 file changed, 23 insertions(+), 16 deletions(-)
drivers/iommu/intel/iommu.c +23 -16
@@ -515,8 +515,9 @@ static int domain_update_device_node(struct dmar_domain *domain) {struct device_domain_info *info;int nid =NUMA_NO_NODE ; +unsigned long flags; -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_for_each_entry (info, &domain->devices, link) {/* * There could possibly be multiple device numa nodes as devices
@@ -528,7 +529,7 @@ static int domain_update_device_node(struct dmar_domain *domain)if (nid !=NUMA_NO_NODE )break ; } -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags);return nid; }
@@ -1362,19 +1363,20 @@ iommu_support_dev_iotlb(struct dmar_domain *domain, struct intel_iommu *iommu, u8 bus, u8 devfn) {struct device_domain_info *info; +unsigned long flags;if (!iommu->qi)return NULL ; -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_for_each_entry (info, &domain->devices, link) {if (info->iommu == iommu && info->bus == bus && info->devfn == devfn) { -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags);return info->ats_supported ? info :NULL ; } } -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags);return NULL ; }
@@ -1383,8 +1385,9 @@ static void domain_update_iotlb(struct dmar_domain *domain) {struct device_domain_info *info;bool has_iotlb_device = false; +unsigned long flags; -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_for_each_entry (info, &domain->devices, link) {if (info->ats_enabled) { has_iotlb_device = true;
@@ -1392,7 +1395,7 @@ static void domain_update_iotlb(struct dmar_domain *domain) } } domain->has_iotlb_device = has_iotlb_device; -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags); }static void iommu_enable_dev_iotlb (struct device_domain_info *info)
@@ -1484,14 +1487,15 @@ static void iommu_flush_dev_iotlb(struct dmar_domain *domain, u64 addr, unsigned mask) {struct device_domain_info *info; +unsigned long flags;if (!domain->has_iotlb_device)return ; -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_for_each_entry (info, &domain->devices, link)__iommu_flush_dev_iotlb (info, addr, mask); -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags); } static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
@@ -2453,6 +2457,7 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev) {struct device_domain_info *info =dev_iommu_priv_get (dev);struct intel_iommu *iommu; +unsigned long flags;u8 bus, devfn;int ret;
@@ -2464,9 +2469,9 @@ static int domain_add_dev_info(struct dmar_domain *domain, struct device *dev)if (ret)return ret; info->domain = domain; -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_add (&info->link, &domain->devices); -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags);/* PASID table is mandatory for a PCI device in scalable mode. */ if (sm_supported (iommu) && !dev_is_real_dma_subdevice (dev)) {
@@ -4090,6 +4095,7 @@ static void dmar_remove_one_dev_info(struct device *dev)struct device_domain_info *info =dev_iommu_priv_get (dev);struct dmar_domain *domain = info->domain;struct intel_iommu *iommu = info->iommu; +unsigned long flags;if (!dev_is_real_dma_subdevice (info->dev)) {if (dev_is_pci (info->dev) &&sm_supported (iommu))
@@ -4101,9 +4107,9 @@ static void dmar_remove_one_dev_info(struct device *dev)intel_pasid_free_table (info->dev); } -spin_lock (&domain->lock); +spin_lock_irqsave (&domain->lock, flags);list_del (&info->link); -spin_unlock (&domain->lock); +spin_unlock_irqrestore (&domain->lock, flags);domain_detach_iommu (domain, iommu); info->domain =NULL ;
@@ -4422,19 +4428,20 @@ static void domain_set_force_snooping(struct dmar_domain *domain)static bool intel_iommu_enforce_cache_coherency (struct iommu_domain *domain) {struct dmar_domain *dmar_domain =to_dmar_domain (domain); +unsigned long flags;if (dmar_domain->force_snooping)return true; -spin_lock (&dmar_domain->lock); +spin_lock_irqsave (&dmar_domain->lock, flags);if (!domain_support_force_snooping (dmar_domain)) { -spin_unlock (&dmar_domain->lock); +spin_unlock_irqrestore (&dmar_domain->lock, flags);return false; }domain_set_force_snooping (dmar_domain); dmar_domain->force_snooping = true; -spin_unlock (&dmar_domain->lock); +spin_unlock_irqrestore (&dmar_domain->lock, flags);return true; }
Signature · 1 line
2.25.1
Lu Baolu
maintainerPATCH 4/4 iommu/vt-d: Fix lockdep splat due to klist iteration in atomic context
In reply to Lu BaoluWith CONFIG_INTEL_IOMMU_DEBUGFS enabled, below lockdep splat are seen when an I/O fault occurs on a machine with an Intel IOMMU in it.
DMAR: DRHD: handling fault status reg 3
DMAR: [DMA Write NO_PASID] Request device [00:1a.0] fault addr 0x0
[fault reason 0x05] PTE Write access is not set
DMAR: Dump dmar0 table entries for IOVA 0x0
DMAR: root entry: 0x0000000127f42001
DMAR: context entry: hi 0x0000000000001502, low 0x000000012d8ab001
================================
WARNING: inconsistent lock state
5.20.0-0.rc0.20220812git7ebfc85e2cd7.10.fc38.x86_64 #1 Not tainted
--------------------------------
inconsistent {HARDIRQ-ON-W} -> {IN-HARDIRQ-W} usage.
rngd/1006 [HC1[1]:SC0[0]:HE0:SE1] takes:
ff177021416f2d78 (&k->k_lock){?.+.}-{2:2}, at: klist_next+0x1b/0x160
{HARDIRQ-ON-W} state was registered at:
lock_acquire+0xce/0x2d0
_raw_spin_lock+0x33/0x80
klist_add_tail+0x46/0x80
bus_add_device+0xee/0x150
device_add+0x39d/0x9a0
add_memory_block+0x108/0x1d0
memory_dev_init+0xe1/0x117
driver_init+0x43/0x4d
kernel_init_freeable+0x1c2/0x2cc
kernel_init+0x16/0x140
ret_from_fork+0x1f/0x30
irq event stamp: 7812
hardirqs last enabled at (7811): [<ffffffff85000e86>] asm_sysvec_apic_timer_interrupt+0x16/0x20
hardirqs last disabled at (7812): [<ffffffff84f16894>] irqentry_enter+0x54/0x60
softirqs last enabled at (7794): [<ffffffff840ff669>] __irq_exit_rcu+0xf9/0x170
softirqs last disabled at (7787): [<ffffffff840ff669>] __irq_exit_rcu+0xf9/0x170The klist iterator functions using spin_*lock_irq*() but the klist insertion functions using spin_*lock(), combined with the Intel DMAR IOMMU driver iterating over klists from atomic (hardirq) context, where pci_get_domain_bus_and_slot() calls into bus_find_device() which iterates over klists.
As currently there's no plan to fix the klist to make it safe to use in atomic context, this fixes the lockdep splat by avoid calling pci_get_domain_bus_and_slot() in the hardirq context.
--- drivers/iommu/intel/iommu.c | 47 +++++++++++++++---------------------- 1 file changed, 19 insertions(+), 28 deletions(-)
drivers/iommu/intel/iommu.c +19 -28
@@ -813,32 +813,11 @@ static void free_context_table(struct intel_iommu *iommu) }#ifdef CONFIG_DMAR_DEBUG -static void pgtable_walk (struct intel_iommu *iommu,unsigned long pfn,u8 bus,u8 devfn) +static void pgtable_walk (struct intel_iommu *iommu,unsigned long pfn, +u8 bus,u8 devfn,struct dma_pte *parent,int level) { -struct device_domain_info *info; -struct dma_pte *parent, *pte; -struct dmar_domain *domain; -struct pci_dev *pdev; -int offset, level; - - pdev =pci_get_domain_bus_and_slot (iommu->segment, bus, devfn); -if (!pdev) -return ; - - info =dev_iommu_priv_get (&pdev->dev); -if (!info || !info->domain) { -pr_info ("device [%02x:%02x.%d] not probed\n" , - bus,PCI_SLOT (devfn),PCI_FUNC (devfn)); -return ; - } - - domain = info->domain; - level =agaw_to_level (domain->agaw); - parent = domain->pgd; -if (!parent) { -pr_info ("no page table setup\n" ); -return ; - } +struct dma_pte *pte; +int offset;while (1 ) { offset =pfn_level_offset (pfn, level);
@@ -865,9 +844,10 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,struct pasid_entry *entries, *pte;struct context_entry *ctx_entry;struct root_entry *rt_entry; +int i, dir_index, index, level;u8 devfn = source_id &0xff ;u8 bus = source_id >>8 ; -int i, dir_index, index; +struct dma_pte *pgtable;pr_info ("Dump %s table entries for IOVA 0x%llx\n" , iommu->name, addr);
@@ -895,8 +875,11 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id, ctx_entry->hi, ctx_entry->lo);/* legacy mode does not require PASID entries */ -if (!sm_supported (iommu)) +if (!sm_supported (iommu)) { + level =agaw_to_level (ctx_entry->hi &7 ); + pgtable =phys_to_virt (ctx_entry->lo &VTD_PAGE_MASK ); goto pgtable_walk; + }/* get the pointer to pasid directory entry */ dir =phys_to_virt (ctx_entry->lo &VTD_PAGE_MASK );
@@ -923,8 +906,16 @@ void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,for (i =0 ; i <ARRAY_SIZE (pte->val); i++)pr_info ("pasid table entry[%d]: 0x%016llx\n" , i, pte->val[i]); +if (pasid_pte_get_pgtt (pte) ==PASID_ENTRY_PGTT_FL_ONLY ) { + level = pte->val[2 ] &BIT_ULL (2 ) ?5 :4 ; + pgtable =phys_to_virt (pte->val[2 ] &VTD_PAGE_MASK ); + }else { + level =agaw_to_level ((pte->val[0 ] >>2 ) &0x7 ); + pgtable =phys_to_virt (pte->val[0 ] &VTD_PAGE_MASK ); + } + pgtable_walk: - pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn); + pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level); }#endif
Signature · 1 line
2.25.1
Quoted 4 lines · Lu Baolu
- Boot kdump kernels with VT-d scalable mode on
- Calculate the right page table levels
- Fix a recursive lock issue
- Fix a lockdep splat issue
Applied, thanks Baolu.