Improved fault handling for AMD-IOMMU (AMD-Vi)
cover Improved fault handling for AMD-IOMMU (AMD-Vi)
Hi,
this series of patches implements improved fault handling in the AMD-IOMMU (AMD-Vi) driver. The debug output is more verbose for some errors. With these patches the driver also tries to recover from IOMMU command handling errors instead of just do a panic.
Joerg
Diffstat:
arch/x86/include/asm/amd_iommu_types.h | 6 ++ arch/x86/kernel/amd_iommu.c | 109 ++++++++++++++++++++++++++------ arch/x86/kernel/amd_iommu_init.c | 20 +++++-- 3 files changed, 111 insertions(+), 24 deletions(-)
Shortlog:
Joerg Roedel (8):
x86/amd-iommu: Dump fault entry on DTE error x86/amd-iommu: Dump illegal command on ILLEGAL_COMMAND_ERROR x86/amd-iommu: Introduce function for iommu-local domain flush x86/amd-iommu: Add function to flush all DTEs on one IOMMU x86/amd-iommu: Add reset function for command buffers x86/amd-iommu: Reset command buffer on ILLEGAL_COMMAND_ERROR x86/amd-iommu: Panic if IOMMU command buffer reset fails x86/amd-iommu: Reset command buffer if wait loop fails
PATCH 1/8 x86/amd-iommu: Dump fault entry on DTE error
This patch adds code to dump the content of the device table entry which caused an ILLEGAL_DEV_TABLE_ENTRY error from the IOMMU hardware.
--- arch/x86/kernel/amd_iommu.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu.c +10 -0
@@ -138,6 +138,15 @@ static int iommu_has_npcache(struct amd_iommu *iommu)* ****************************************************************************/ +static void dump_dte_entry (u16 devid) +{ +int i; + +for (i =0 ; i <8 ; ++i) +pr_err ("AMD-Vi: DTE[%d]: %08x\n" , i, + amd_iommu_dev_table[devid].data[i]); +} +static void iommu_print_event (void *__evt) {u32 *event = __evt;
@@ -155,6 +164,7 @@ static void iommu_print_event(void *__evt)"address=0x%016llx flags=0x%04x]\n" ,PCI_BUS (devid),PCI_SLOT (devid),PCI_FUNC (devid), address, flags); +dump_dte_entry (devid);break ;case EVENT_TYPE_IO_FAULT : printk("IO_PAGE_FAULT device=%02x:%02x.%x "
Signature · 1 line
1.6.3.3
PATCH 2/8 x86/amd-iommu: Dump illegal command on ILLEGAL_COMMAND_ERROR
In reply to Joerg RoedelThis patch adds code to dump the command which caused an ILLEGAL_COMMAND_ERROR raised by the IOMMU hardware.
--- arch/x86/kernel/amd_iommu.c | 10 ++++++++++ 1 files changed, 10 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu.c +10 -0
@@ -147,6 +147,15 @@ static void dump_dte_entry(u16 devid) amd_iommu_dev_table[devid].data[i]); } +static void dump_command (unsigned long phys_addr) +{ +struct iommu_cmd *cmd =phys_to_virt (phys_addr); +int i; + +for (i =0 ; i <4 ; ++i) +pr_err ("AMD-Vi: CMD[%d]: %08x\n" , i, cmd->data[i]); +} +static void iommu_print_event (void *__evt) {u32 *event = __evt;
@@ -186,6 +195,7 @@ static void iommu_print_event(void *__evt)break ;case EVENT_TYPE_ILL_CMD :printk ("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n" , address); +dump_command (address);break ;case EVENT_TYPE_CMD_HARD_ERR : printk("COMMAND_HARDWARE_ERROR address=0x%016llx "
Signature · 1 line
1.6.3.3
PATCH 3/8 x86/amd-iommu: Introduce function for iommu-local domain flush
In reply to Joerg RoedelThis patch introduces a function to flush all domain tlbs for on one given IOMMU. This is required later to reset the command buffer on one IOMMU.
--- arch/x86/kernel/amd_iommu.c | 49 +++++++++++++++++++++++++++++-------------- 1 files changed, 33 insertions(+), 16 deletions(-)
arch/x86/kernel/amd_iommu.c +33 -16
@@ -465,38 +465,55 @@ static void iommu_flush_tlb_pde(struct amd_iommu *iommu, u16 domid) }/* -* This function is used to flush the IO/TLB for a given protection domain -* on every IOMMU in the system +* This function flushes one domain on one IOMMU */ -static void iommu_flush_domain (u16 domid) +static void flush_domain_on_iommu (struct amd_iommu *iommu,u16 domid) { -unsigned long flags; -struct amd_iommu *iommu;struct iommu_cmd cmd; - -INC_STATS_COUNTER (domain_flush_all); +unsigned long flags;__iommu_build_inv_iommu_pages (&cmd,CMD_INV_IOMMU_ALL_PAGES_ADDRESS , domid,1 ,1 ); -for_each_iommu (iommu) { -spin_lock_irqsave (&iommu->lock, flags); -__iommu_queue_command (iommu, &cmd); -__iommu_completion_wait (iommu); -__iommu_wait_for_completion (iommu); -spin_unlock_irqrestore (&iommu->lock, flags); - } +spin_lock_irqsave (&iommu->lock, flags); +__iommu_queue_command (iommu, &cmd); +__iommu_completion_wait (iommu); +__iommu_wait_for_completion (iommu); +spin_unlock_irqrestore (&iommu->lock, flags); } -void amd_iommu_flush_all_domains (void ) +static void flush_all_domains_on_iommu (struct amd_iommu *iommu) {int i;for (i =1 ; i <MAX_DOMAIN_ID ; ++i) {if (!test_bit (i, amd_iommu_pd_alloc_bitmap))continue ; -iommu_flush_domain (i); +flush_domain_on_iommu (iommu, i); } + +} + +/* +* This function is used to flush the IO/TLB for a given protection domain +* on every IOMMU in the system +*/ +static void iommu_flush_domain (u16 domid) +{ +struct amd_iommu *iommu; + +INC_STATS_COUNTER (domain_flush_all); + +for_each_iommu (iommu) +flush_domain_on_iommu (iommu, domid); +} + +void amd_iommu_flush_all_domains (void ) +{ +struct amd_iommu *iommu; + +for_each_iommu (iommu) +flush_all_domains_on_iommu (iommu); }void amd_iommu_flush_all_devices (void )
Signature · 1 line
1.6.3.3
PATCH 4/8 x86/amd-iommu: Add function to flush all DTEs on one IOMMU
In reply to Joerg RoedelThis function flushes all DTE entries on one IOMMU for all devices behind this IOMMU. This is required for command buffer resetting later.
--- arch/x86/kernel/amd_iommu.c | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu.c +13 -0
@@ -516,6 +516,19 @@ void amd_iommu_flush_all_domains(void)flush_all_domains_on_iommu (iommu); } +static void flush_all_devices_for_iommu (struct amd_iommu *iommu) +{ +int i; + +for (i =0 ; i <= amd_iommu_last_bdf; ++i) { +if (iommu != amd_iommu_rlookup_table[i]) +continue ; + +iommu_queue_inv_dev_entry (iommu, i); +iommu_completion_wait (iommu); + } +} +void amd_iommu_flush_all_devices (void ) {struct amd_iommu *iommu;
Signature · 1 line
1.6.3.3
PATCH 5/8 x86/amd-iommu: Add reset function for command buffers
In reply to Joerg RoedelThis patch factors parts of the command buffer initialization code into a seperate function which can be used to reset the command buffer later.
--- arch/x86/include/asm/amd_iommu_types.h | 3 +++ arch/x86/kernel/amd_iommu_init.c | 20 +++++++++++++++----- 2 files changed, 18 insertions(+), 5 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +3 -0
@@ -457,4 +457,7 @@ static inline void amd_iommu_stats_init(void) { }#endif /* CONFIG_AMD_IOMMU_STATS */ +/* some function prototypes */ +extern void amd_iommu_reset_cmd_buffer (struct amd_iommu *iommu); +#endif /* _ASM_X86_AMD_IOMMU_TYPES_H */
arch/x86/kernel/amd_iommu_init.c +15 -5
@@ -435,6 +435,20 @@ static u8 * __init alloc_command_buffer(struct amd_iommu *iommu) }/* +* This function resets the command buffer if the IOMMU stopped fetching +* commands from it. +*/ +void amd_iommu_reset_cmd_buffer (struct amd_iommu *iommu) +{ +iommu_feature_disable (iommu,CONTROL_CMDBUF_EN ); + +writel (0x00 , iommu->mmio_base +MMIO_CMD_HEAD_OFFSET ); +writel (0x00 , iommu->mmio_base +MMIO_CMD_TAIL_OFFSET ); + +iommu_feature_enable (iommu,CONTROL_CMDBUF_EN ); +} + +/* * This function writes the command buffer address to the hardware and * enables it. */
@@ -450,11 +464,7 @@ static void iommu_enable_command_buffer(struct amd_iommu *iommu)memcpy_toio (iommu->mmio_base +MMIO_CMD_BUF_OFFSET , &entry,sizeof (entry)); -/* set head and tail to zero manually */ -writel (0x00 , iommu->mmio_base +MMIO_CMD_HEAD_OFFSET ); -writel (0x00 , iommu->mmio_base +MMIO_CMD_TAIL_OFFSET ); - -iommu_feature_enable (iommu,CONTROL_CMDBUF_EN ); +amd_iommu_reset_cmd_buffer (iommu); }static void __init free_command_buffer (struct amd_iommu *iommu)
Signature · 1 line
1.6.3.3
PATCH 6/8 x86/amd-iommu: Reset command buffer on ILLEGAL_COMMAND_ERROR
In reply to Joerg RoedelOn an ILLEGAL_COMMAND_ERROR the IOMMU stops executing further commands. This patch changes the code to handle this case better by resetting the command buffer in the IOMMU.
--- arch/x86/kernel/amd_iommu.c | 15 +++++++++++++-- 1 files changed, 13 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +13 -2
@@ -61,6 +61,7 @@ static u64* alloc_pte(struct protection_domain *dom,static void dma_ops_reserve_addresses (struct dma_ops_domain *dom,unsigned long start_page,unsigned int pages); +static void reset_iommu_command_buffer (struct amd_iommu *iommu);#ifndef BUS_NOTIFY_UNBOUND_DRIVER #define BUS_NOTIFY_UNBOUND_DRIVER 0x0005
@@ -156,7 +157,7 @@ static void dump_command(unsigned long phys_addr)pr_err ("AMD-Vi: CMD[%d]: %08x\n" , i, cmd->data[i]); } -static void iommu_print_event (void *__evt) +static void iommu_print_event (struct amd_iommu *iommu,void *__evt) {u32 *event = __evt;int type = (event[1 ] >>EVENT_TYPE_SHIFT ) &EVENT_TYPE_MASK ;
@@ -195,6 +196,7 @@ static void iommu_print_event(void *__evt)break ;case EVENT_TYPE_ILL_CMD :printk ("ILLEGAL_COMMAND_ERROR address=0x%016llx]\n" , address); +reset_iommu_command_buffer (iommu);dump_command (address);break ;case EVENT_TYPE_CMD_HARD_ERR :
@@ -229,7 +231,7 @@ static void iommu_poll_events(struct amd_iommu *iommu) tail =readl (iommu->mmio_base +MMIO_EVT_TAIL_OFFSET );while (head != tail) { -iommu_print_event (iommu->evt_buf + head); +iommu_print_event (iommu, iommu->evt_buf + head); head = (head +EVENT_ENTRY_SIZE ) % iommu->evt_buf_size; }
@@ -547,6 +549,15 @@ void amd_iommu_flush_all_devices(void) } } +static void reset_iommu_command_buffer (struct amd_iommu *iommu) +{ +pr_err ("AMD-Vi: Resetting IOMMU command buffer\n" ); + +amd_iommu_reset_cmd_buffer (iommu); +flush_all_devices_for_iommu (iommu); +flush_all_domains_on_iommu (iommu); +} +/**************************************************************************** * * The functions below are used the create the page table mappings for
Signature · 1 line
1.6.3.3
PATCH 7/8 x86/amd-iommu: Panic if IOMMU command buffer reset fails
In reply to Joerg RoedelTo prevent the driver from doing recursive command buffer resets, just panic when that recursion happens.
--- arch/x86/include/asm/amd_iommu_types.h | 3 +++ arch/x86/kernel/amd_iommu.c | 7 +++++++ 2 files changed, 10 insertions(+), 0 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +3 -0
@@ -337,6 +337,9 @@ struct amd_iommu {/* if one, we need to send a completion wait command */ bool need_sync; +/* becomes true if a command buffer reset is running */ +bool reset_in_progress; +/* default dma_ops domain for that IOMMU */ struct dma_ops_domain *default_dom; };
arch/x86/kernel/amd_iommu.c +7 -0
@@ -553,9 +553,16 @@ static void reset_iommu_command_buffer(struct amd_iommu *iommu) {pr_err ("AMD-Vi: Resetting IOMMU command buffer\n" ); +if (iommu->reset_in_progress) +panic ("AMD-Vi: ILLEGAL_COMMAND_ERROR while resetting command buffer\n" ); + + iommu->reset_in_progress = true; +amd_iommu_reset_cmd_buffer (iommu);flush_all_devices_for_iommu (iommu);flush_all_domains_on_iommu (iommu); + + iommu->reset_in_progress = false; }/****************************************************************************
Signature · 1 line
1.6.3.3
PATCH 8/8 x86/amd-iommu: Reset command buffer if wait loop fails
In reply to Joerg RoedelInstead of a panic on an comletion wait loop failure, try to recover from that event from resetting the command buffer.
--- arch/x86/kernel/amd_iommu.c | 7 +++++-- 1 files changed, 5 insertions(+), 2 deletions(-)
arch/x86/kernel/amd_iommu.c +5 -2
@@ -318,8 +318,11 @@ static void __iommu_wait_for_completion(struct amd_iommu *iommu) status &= ~MMIO_STATUS_COM_WAIT_INT_MASK ;writel (status, iommu->mmio_base +MMIO_STATUS_OFFSET ); -if (unlikely (i ==EXIT_LOOP_COUNT )) -panic ("AMD IOMMU: Completion wait loop failed\n" ); +if (unlikely (i ==EXIT_LOOP_COUNT )) { +spin_unlock (&iommu->lock); +reset_iommu_command_buffer (iommu); +spin_lock (&iommu->lock); + } }/*
Signature · 1 line
1.6.3.3