Skip to contentSkip to conversation

iommu/vt-d: Revert ATS timing change to fix boot failure

PATCH In review Lu Baolu 4 participants 6 messages Apr 16, 2025 – Apr 17, 2025

Subsystems (2)
lore
Share & tools
lore

Lu Baolu

author maintainer
#0 lore raw

PATCH 1/1 iommu/vt-d: Revert ATS timing change to fix boot failure

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to probe path") changed the PCI ATS enablement logic to run earlier, specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing the kernel to panic with the following message and call trace:

 Kernel panic - not syncing: DMAR hardware is malfunctioning
 CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
 Call Trace:
  <TASK>
  dump_stack_lvl+0x6f/0xb0
  dump_stack+0x10/0x16
  panic+0x10a/0x2b7
  iommu_enable_translation.cold+0xc/0xc
  intel_iommu_init+0xe39/0xec0
  ? trace_hardirqs_on+0x1e/0xd0
  ? __pfx_pci_iommu_init+0x10/0x10
  pci_iommu_init+0xd/0x40
  do_one_initcall+0x5b/0x390
  kernel_init_freeable+0x26d/0x2b0
  ? __pfx_kernel_init+0x10/0x10
  kernel_init+0x15/0x120
  ret_from_fork+0x35/0x60
  ? __pfx_kernel_init+0x10/0x10
  ret_from_fork_asm+0x1a/0x30
 RIP: 1f0f:0x0
 Code: Unable to access opcode bytes at 0xffffffffffffffd6.
 RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
      1f0f2e6600000000
 RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
      2e66000000000084
 RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
      00841f0f2e660000
 RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
      000000841f0f2e66
 R10: 0000000000841f0f R11: 2e66000000000084 R12:
      000000841f0f2e66
 R13: 0000000000841f0f R14: 2e66000000000084 R15:
      1f0f2e6600000000
  </TASK>
 ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---

Fix this by reverting the timing change for ATS enablement introduced by the offending commit and restoring the previous behavior.

Fixes 5518f239aff1 ("iommu/vt-d: Move scalable mode ATS enablement to probe path")
---
 drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++------------
 1 file changed, 19 insertions(+), 12 deletions(-)
drivers/iommu/intel/iommu.c +19 -12
index b29da2d96d0b..e60b699e7b8c 100644
@@ -3785,6 +3785,22 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 
 	intel_iommu_debugfs_create_dev(info);
 
+	return &iommu->iommu;
+free_table:
+	intel_pasid_free_table(dev);
+clear_rbtree:
+	device_rbtree_remove(info);
+free:
+	kfree(info);
+
+	return ERR_PTR(ret);
+}
+
+static void intel_iommu_probe_finalize(struct device *dev)
+{
+	struct device_domain_info *info = dev_iommu_priv_get(dev);
+	struct intel_iommu *iommu = info->iommu;
+
 	/*
 	 * The PCIe spec, in its wisdom, declares that the behaviour of the
 	 * device is undefined if you enable PASID support after ATS support.
@@ -3792,22 +3808,12 @@ static struct iommu_device *intel_iommu_probe_device(struct device *dev)
 	 * we can't yet know if we're ever going to use it.
 	 */
 	if (info->pasid_supported &&
-	    !pci_enable_pasid(pdev, info->pasid_supported & ~1))
+	    !pci_enable_pasid(to_pci_dev(dev), info->pasid_supported & ~1))
 		info->pasid_enabled = 1;
 
-	if (sm_supported(iommu))
+	if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev))
 		iommu_enable_pci_ats(info);
 	iommu_enable_pci_pri(info);
-
-	return &iommu->iommu;
-free_table:
-	intel_pasid_free_table(dev);
-clear_rbtree:
-	device_rbtree_remove(info);
-free:
-	kfree(info);
-
-	return ERR_PTR(ret);
 }
 
 static void intel_iommu_release_device(struct device *dev)
@@ -4391,6 +4397,7 @@ const struct iommu_ops intel_iommu_ops = {
 	.domain_alloc_sva	= intel_svm_domain_alloc,
 	.domain_alloc_nested	= intel_iommu_domain_alloc_nested,
 	.probe_device		= intel_iommu_probe_device,
+	.probe_finalize		= intel_iommu_probe_finalize,
 	.release_device		= intel_iommu_release_device,
 	.get_resv_regions	= intel_iommu_get_resv_regions,
 	.device_group		= intel_iommu_device_group,
Signature · 1 line
2.43.0
Tested-by on v1 1/1
#1 lore raw
Quoted 46 lines · Lu Baolu

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to probe path") changed the PCI ATS enablement logic to run earlier, specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing the kernel to panic with the following message and call trace:

  Kernel panic - not syncing: DMAR hardware is malfunctioning
  CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
  Call Trace:
   <TASK>
   dump_stack_lvl+0x6f/0xb0
   dump_stack+0x10/0x16
   panic+0x10a/0x2b7
   iommu_enable_translation.cold+0xc/0xc
   intel_iommu_init+0xe39/0xec0
   ? trace_hardirqs_on+0x1e/0xd0
   ? __pfx_pci_iommu_init+0x10/0x10
   pci_iommu_init+0xd/0x40
   do_one_initcall+0x5b/0x390
   kernel_init_freeable+0x26d/0x2b0
   ? __pfx_kernel_init+0x10/0x10
   kernel_init+0x15/0x120
   ret_from_fork+0x35/0x60
   ? __pfx_kernel_init+0x10/0x10
   ret_from_fork_asm+0x1a/0x30
  RIP: 1f0f:0x0
  Code: Unable to access opcode bytes at 0xffffffffffffffd6.
  RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
       1f0f2e6600000000
  RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
       2e66000000000084
  RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
       00841f0f2e660000
  RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
       000000841f0f2e66
  R10: 0000000000841f0f R11: 2e66000000000084 R12:
       000000841f0f2e66
  R13: 0000000000841f0f R14: 2e66000000000084 R15:
       1f0f2e6600000000
   </TASK>
  ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---

Fix this by reverting the timing change for ATS enablement introduced by the offending commit and restoring the previous behavior.

Quoted 8 lines
Fixes 5518f239aff1 ("iommu/vt-d: Move scalable mode ATS enablement to probe path")
---
  drivers/iommu/intel/iommu.c | 31 +++++++++++++++++++------------
  1 file changed, 19 insertions(+), 12 deletions(-)
#2 lore raw
In reply to Lu Baolu
Quoted 47 lines

From: Lu Baolu <[email protected]>
Sent: Wednesday, April 16, 2025 3:36 PM

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to
probe path") changed the PCI ATS enablement logic to run earlier, specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing the kernel to panic with the following message and call trace:

 Kernel panic - not syncing: DMAR hardware is malfunctioning
 CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
 Call Trace:
  <TASK>
  dump_stack_lvl+0x6f/0xb0
  dump_stack+0x10/0x16
  panic+0x10a/0x2b7
  iommu_enable_translation.cold+0xc/0xc
  intel_iommu_init+0xe39/0xec0
  ? trace_hardirqs_on+0x1e/0xd0
  ? __pfx_pci_iommu_init+0x10/0x10
  pci_iommu_init+0xd/0x40
  do_one_initcall+0x5b/0x390
  kernel_init_freeable+0x26d/0x2b0
  ? __pfx_kernel_init+0x10/0x10
  kernel_init+0x15/0x120
  ret_from_fork+0x35/0x60
  ? __pfx_kernel_init+0x10/0x10
  ret_from_fork_asm+0x1a/0x30
 RIP: 1f0f:0x0
 Code: Unable to access opcode bytes at 0xffffffffffffffd6.
 RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
      1f0f2e6600000000
 RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
      2e66000000000084
 RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
      00841f0f2e660000
 RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
      000000841f0f2e66
 R10: 0000000000841f0f R11: 2e66000000000084 R12:
      000000841f0f2e66
 R13: 0000000000841f0f R14: 2e66000000000084 R15:
      1f0f2e6600000000
  </TASK>
 ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
Quoted 3 lines

Fix this by reverting the timing change for ATS enablement introduced by the offending commit and restoring the previous behavior.

it's unclear how this timing is related to the dumped stack. Is there more detail how they are related?

author maintainer
#3 lore raw
Quoted 47 lines · Kevin
Quoted 47 lines

From: Lu Baolu <[email protected]>
Sent: Wednesday, April 16, 2025 3:36 PM

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS enablement to
probe path") changed the PCI ATS enablement logic to run earlier, specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing the kernel to panic with the following message and call trace:

  Kernel panic - not syncing: DMAR hardware is malfunctioning
  CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
  Call Trace:
   <TASK>
   dump_stack_lvl+0x6f/0xb0
   dump_stack+0x10/0x16
   panic+0x10a/0x2b7
   iommu_enable_translation.cold+0xc/0xc
   intel_iommu_init+0xe39/0xec0
   ? trace_hardirqs_on+0x1e/0xd0
   ? __pfx_pci_iommu_init+0x10/0x10
   pci_iommu_init+0xd/0x40
   do_one_initcall+0x5b/0x390
   kernel_init_freeable+0x26d/0x2b0
   ? __pfx_kernel_init+0x10/0x10
   kernel_init+0x15/0x120
   ret_from_fork+0x35/0x60
   ? __pfx_kernel_init+0x10/0x10
   ret_from_fork_asm+0x1a/0x30
  RIP: 1f0f:0x0
  Code: Unable to access opcode bytes at 0xffffffffffffffd6.
  RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
       1f0f2e6600000000
  RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
       2e66000000000084
  RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
       00841f0f2e660000
  RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
       000000841f0f2e66
  R10: 0000000000841f0f R11: 2e66000000000084 R12:
       000000841f0f2e66
  R13: 0000000000841f0f R14: 2e66000000000084 R15:
       1f0f2e6600000000
   </TASK>
  ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---
Quoted 7 lines
Quoted 3 lines

Fix this by reverting the timing change for ATS enablement introduced by the offending commit and restoring the previous behavior.

it's unclear how this timing is related to the dumped stack. Is there more detail how they are related?

I'm not sure, but I'm trying to find a machine and get more information. Anyway, let's revert the change and remove the boot regression first.

Thanks,
baolu

Reviewed-by
#4 lore raw
Quoted 56 lines

From: Baolu Lu <[email protected]>
Sent: Thursday, April 17, 2025 10:46 AM

Quoted 4 lines · Kevin
Quoted 4 lines

From: Lu Baolu <[email protected]>
Sent: Wednesday, April 16, 2025 3:36 PM

Commit <5518f239aff1> ("iommu/vt-d: Move scalable mode ATS

enablement

Quoted 47 lines
Quoted 46 lines

to
probe path") changed the PCI ATS enablement logic to run earlier, specifically before the default domain attachment.

On some client platforms, this change resulted in boot failures, causing the kernel to panic with the following message and call trace:

  Kernel panic - not syncing: DMAR hardware is malfunctioning
  CPU: 0 UID: 0 PID: 1 Comm: swapper/0 Not tainted 6.14.0-rc3+ #175
  Call Trace:
   <TASK>
   dump_stack_lvl+0x6f/0xb0
   dump_stack+0x10/0x16
   panic+0x10a/0x2b7
   iommu_enable_translation.cold+0xc/0xc
   intel_iommu_init+0xe39/0xec0
   ? trace_hardirqs_on+0x1e/0xd0
   ? __pfx_pci_iommu_init+0x10/0x10
   pci_iommu_init+0xd/0x40
   do_one_initcall+0x5b/0x390
   kernel_init_freeable+0x26d/0x2b0
   ? __pfx_kernel_init+0x10/0x10
   kernel_init+0x15/0x120
   ret_from_fork+0x35/0x60
   ? __pfx_kernel_init+0x10/0x10
   ret_from_fork_asm+0x1a/0x30
  RIP: 1f0f:0x0
  Code: Unable to access opcode bytes at 0xffffffffffffffd6.
  RSP: 0000:0000000000000000 EFLAGS: 841f0f2e66 ORIG_RAX:
       1f0f2e6600000000
  RAX: 0000000000000000 RBX: 1f0f2e6600000000 RCX:
       2e66000000000084
  RDX: 0000000000841f0f RSI: 000000841f0f2e66 RDI:
       00841f0f2e660000
  RBP: 00841f0f2e660000 R08: 00841f0f2e660000 R09:
       000000841f0f2e66
  R10: 0000000000841f0f R11: 2e66000000000084 R12:
       000000841f0f2e66
  R13: 0000000000841f0f R14: 2e66000000000084 R15:
       1f0f2e6600000000
   </TASK>
  ---[ end Kernel panic - not syncing: DMAR hardware is malfunctioning ]---

Fix this by reverting the timing change for ATS enablement introduced by the offending commit and restoring the previous behavior.

Quoted 7 lines
Quoted 3 lines

it's unclear how this timing is related to the dumped stack. Is there more detail how they are related?

I'm not sure, but I'm trying to find a machine and get more information. Anyway, let's revert the change and remove the boot regression first.

I'm fine with this fix for regression but let's do investigate more.