suspend/resume support for amd iommu
cover suspend/resume support for amd iommu
This patchset implements support for suspend/resume in the AMD IOMMU driver for Linux. Please review.
diffstat:
arch/x86/include/asm/amd_iommu.h | 2 + arch/x86/include/asm/amd_iommu_types.h | 8 ++ arch/x86/kernel/amd_iommu.c | 37 +++++++- arch/x86/kernel/amd_iommu_init.c | 144 ++++++++++++++++---------------- 4 files changed, 114 insertions(+), 77 deletions(-)
shortlog:
Joerg Roedel (9):
amd-iommu: introduce for_each_iommu* macros
amd-iommu: consolidate hardware initialization to one function
amd-iommu: drop pointless iommu-loop in msi setup code
amd-iommu: remove support for msi-x
amd-iommu: add function to disable all iommus
amd-iommu: add function to flush tlb for all domains
amd-iommu: add function to flush tlb for all devices
amd_iommu: un __init functions required for suspend/resume
amd-iommu: implement suspend/resume
PATCH 1/9 amd-iommu: introduce for_each_iommu* macros
Impact: simplify iterating over all iommus in the system
This patch introduces the for_each_iommu and for_each_iommu_safe macros to simplify the developers life when having to iterate over all AMD IOMMUs in the system.
--- arch/x86/include/asm/amd_iommu_types.h | 8 ++++++++ arch/x86/kernel/amd_iommu.c | 8 ++++---- arch/x86/kernel/amd_iommu_init.c | 8 ++++---- 3 files changed, 16 insertions(+), 8 deletions(-)
arch/x86/include/asm/amd_iommu_types.h +8 -0
@@ -196,6 +196,14 @@domain for an IOMMU */ /* +* Make iterating over all IOMMUs easier +*/ +#define for_each_iommu (iommu) \ + list_for_each_entry((iommu), &amd_iommu_list, list) +#define for_each_iommu_safe (iommu, next) \ + list_for_each_entry_safe((iommu), (next), &amd_iommu_list, list) + +/* * This structure contains generic data for IOMMU protection domains * independent of their use. */
arch/x86/kernel/amd_iommu.c +4 -4
@@ -213,7 +213,7 @@ irqreturn_t amd_iommu_int_handler(int irq, void *data) {struct amd_iommu *iommu; -list_for_each_entry (iommu, &amd_iommu_list, list) +for_each_iommu (iommu)iommu_poll_events (iommu);return IRQ_HANDLED ;
@@ -440,7 +440,7 @@ static void iommu_flush_domain(u16 domid)__iommu_build_inv_iommu_pages (&cmd,CMD_INV_IOMMU_ALL_PAGES_ADDRESS , domid,1 ,1 ); -list_for_each_entry (iommu, &amd_iommu_list, list) { +for_each_iommu (iommu) {spin_lock_irqsave (&iommu->lock, flags);__iommu_queue_command (iommu, &cmd);__iommu_completion_wait (iommu);
@@ -1672,7 +1672,7 @@ int __init amd_iommu_init_dma_ops(void)* found in the system. Devices not assigned to any other * protection domain will be assigned to the default one. */ -list_for_each_entry (iommu, &amd_iommu_list, list) { +for_each_iommu (iommu) { iommu->default_dom =dma_ops_domain_alloc (iommu, order);if (iommu->default_dom ==NULL )return -ENOMEM ;
@@ -1710,7 +1710,7 @@ int __init amd_iommu_init_dma_ops(void) free_domains: -list_for_each_entry (iommu, &amd_iommu_list, list) { +for_each_iommu (iommu) {if (iommu->default_dom)dma_ops_domain_free (iommu->default_dom); }
arch/x86/kernel/amd_iommu_init.c +4 -4
@@ -679,7 +679,7 @@ static void __init free_iommu_all(void) {struct amd_iommu *iommu, *next; -list_for_each_entry_safe (iommu, next, &amd_iommu_list, list) { +for_each_iommu_safe (iommu, next) {list_del (&iommu->list);free_iommu_one (iommu);kfree (iommu);
@@ -779,7 +779,7 @@ static int __init iommu_setup_msix(struct amd_iommu *iommu)struct msix_entry entries[32 ];/* only 32 supported by AMD IOMMU */ int nvec =0 , i; -list_for_each_entry (curr, &amd_iommu_list, list) { +for_each_iommu (curr) {if (curr->dev == iommu->dev) { entries[nvec].entry = curr->evt_msi_num; entries[nvec].vector =0 ;
@@ -818,7 +818,7 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)int r;struct amd_iommu *curr; -list_for_each_entry (curr, &amd_iommu_list, list) { +for_each_iommu (curr) {if (curr->dev == iommu->dev) curr->int_enabled = true; }
@@ -971,7 +971,7 @@ static void __init enable_iommus(void) {struct amd_iommu *iommu; -list_for_each_entry (iommu, &amd_iommu_list, list) { +for_each_iommu (iommu) {iommu_set_exclusion_range (iommu);iommu_init_msi (iommu);iommu_enable_event_logging (iommu);
Signature · 1 line
1.6.3.1
PATCH 2/9 amd-iommu: consolidate hardware initialization to one function
In reply to Joerg RoedelImpact: cleanup and consolidate hardware initialization
This patch restructures the AMD IOMMU initialization code to initialize all hardware registers with one single function call.
This is helpful for suspend/resume support.
--- arch/x86/kernel/amd_iommu_init.c | 50 ++++++++++++++++++++++++------------- 1 files changed, 32 insertions(+), 18 deletions(-)
arch/x86/kernel/amd_iommu_init.c +32 -18
@@ -252,13 +252,6 @@ static void __init iommu_enable(struct amd_iommu *iommu)iommu_feature_enable (iommu,CONTROL_IOMMU_EN ); } -/* Function to enable IOMMU event logging and event interrupts */ -static void __init iommu_enable_event_logging (struct amd_iommu *iommu) -{ -iommu_feature_enable (iommu,CONTROL_EVT_LOG_EN ); -iommu_feature_enable (iommu,CONTROL_EVT_INT_EN ); -} -/* * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in * the system has one.
@@ -413,25 +406,36 @@ static u8 * __init alloc_command_buffer(struct amd_iommu *iommu) {u8 *cmd_buf = (u8 *)__get_free_pages (GFP_KERNEL | __GFP_ZERO,get_order (CMD_BUFFER_SIZE )); -u64 entry;if (cmd_buf ==NULL )return NULL ; iommu->cmd_buf_size =CMD_BUFFER_SIZE ; - entry = (u64 )virt_to_phys (cmd_buf); +return cmd_buf; +} + +/* +* This function writes the command buffer address to the hardware and +* enables it. +*/ +static void iommu_enable_command_buffer (struct amd_iommu *iommu) +{ +u64 entry; + +BUG_ON (iommu->cmd_buf ==NULL ); + + entry = (u64 )virt_to_phys (iommu->cmd_buf); entry |=MMIO_CMD_SIZE_512 ; +memcpy_toio (iommu->mmio_base +MMIO_CMD_BUF_OFFSET , - &entry,sizeof (entry)); + &entry,sizeof (entry));/* set head and tail to zero manually */ writel (0x00 , iommu->mmio_base +MMIO_CMD_HEAD_OFFSET );writel (0x00 , iommu->mmio_base +MMIO_CMD_TAIL_OFFSET );iommu_feature_enable (iommu,CONTROL_CMDBUF_EN ); - -return cmd_buf; }static void __init free_command_buffer (struct amd_iommu *iommu)
@@ -443,20 +447,27 @@ static void __init free_command_buffer(struct amd_iommu *iommu)/* allocates the memory where the IOMMU will log its events to */ static u8 *__init alloc_event_buffer (struct amd_iommu *iommu) { -u64 entry; iommu->evt_buf = (u8 *)__get_free_pages (GFP_KERNEL | __GFP_ZERO,get_order (EVT_BUFFER_SIZE ));if (iommu->evt_buf ==NULL )return NULL ; +return iommu->evt_buf; +} + +static void iommu_enable_event_buffer (struct amd_iommu *iommu) +{ +u64 entry; + +BUG_ON (iommu->evt_buf ==NULL ); + entry = (u64 )virt_to_phys (iommu->evt_buf) |EVT_LEN_MASK ; +memcpy_toio (iommu->mmio_base +MMIO_EVT_BUF_OFFSET , &entry,sizeof (entry)); - iommu->evt_buf_size =EVT_BUFFER_SIZE ; - -return iommu->evt_buf; +iommu_feature_enable (iommu,CONTROL_EVT_LOG_EN ); }static void __init free_event_buffer (struct amd_iommu *iommu)
@@ -710,7 +721,6 @@ static int __init init_iommu_one(struct amd_iommu *iommu, struct ivhd_header *h)if (!iommu->mmio_base)return -ENOMEM ; -iommu_set_device_table (iommu); iommu->cmd_buf =alloc_command_buffer (iommu);if (!iommu->cmd_buf)return -ENOMEM ;
@@ -837,6 +847,8 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)return 1 ; } +iommu_feature_enable (iommu,CONTROL_EVT_INT_EN ); +return 0 ; }
@@ -972,9 +984,11 @@ static void __init enable_iommus(void)struct amd_iommu *iommu;for_each_iommu (iommu) { +iommu_set_device_table (iommu); +iommu_enable_command_buffer (iommu); +iommu_enable_event_buffer (iommu);iommu_set_exclusion_range (iommu);iommu_init_msi (iommu); -iommu_enable_event_logging (iommu);iommu_enable (iommu); } }
Signature · 1 line
1.6.3.1
PATCH 3/9 amd-iommu: drop pointless iommu-loop in msi setup code
In reply to Joerg RoedelImpact: cleanup
It is not necessary to loop again over all IOMMUs in this code. So drop the loop.
--- arch/x86/kernel/amd_iommu_init.c | 8 +------- 1 files changed, 1 insertions(+), 7 deletions(-)
arch/x86/kernel/amd_iommu_init.c +1 -7
@@ -826,13 +826,6 @@ out_free:static int __init iommu_setup_msi (struct amd_iommu *iommu) {int r; -struct amd_iommu *curr; - -for_each_iommu (curr) { -if (curr->dev == iommu->dev) - curr->int_enabled = true; - } -if (pci_enable_msi (iommu->dev))return 1 ;
@@ -847,6 +840,7 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)return 1 ; } + iommu->int_enabled = true;iommu_feature_enable (iommu,CONTROL_EVT_INT_EN );return 0 ;
Signature · 1 line
1.6.3.1
PATCH 4/9 amd-iommu: remove support for msi-x
In reply to Joerg RoedelImpact: drop support for msi-x in amd iommu code
Current hardware uses msi instead of msi-x so this code it not necessary and can not be tested. The best thing is to drop this code.
--- arch/x86/kernel/amd_iommu_init.c | 44 +------------------------------------- 1 files changed, 1 insertions(+), 43 deletions(-)
arch/x86/kernel/amd_iommu_init.c +1 -43
@@ -783,46 +783,6 @@ static int __init init_iommu_all(struct acpi_table_header *table)* ****************************************************************************/ -static int __init iommu_setup_msix (struct amd_iommu *iommu) -{ -struct amd_iommu *curr; -struct msix_entry entries[32 ];/* only 32 supported by AMD IOMMU */ -int nvec =0 , i; - -for_each_iommu (curr) { -if (curr->dev == iommu->dev) { - entries[nvec].entry = curr->evt_msi_num; - entries[nvec].vector =0 ; - curr->int_enabled = true; - nvec++; - } - } - -if (pci_enable_msix (iommu->dev, entries, nvec)) { -pci_disable_msix (iommu->dev); -return 1 ; - } - -for (i =0 ; i < nvec; ++i) { -int r =request_irq (entries->vector, amd_iommu_int_handler, -IRQF_SAMPLE_RANDOM , -"AMD IOMMU" , -NULL ); -if (r) - goto out_free; - } - -return 0 ; - -out_free: -for (i -=1 ; i >=0 ; --i) -free_irq (entries->vector,NULL ); - -pci_disable_msix (iommu->dev); - -return 1 ; -} -static int __init iommu_setup_msi (struct amd_iommu *iommu) {int r;
@@ -851,9 +811,7 @@ static int __init iommu_init_msi(struct amd_iommu *iommu)if (iommu->int_enabled)return 0 ; -if (pci_find_capability (iommu->dev,PCI_CAP_ID_MSIX )) -return iommu_setup_msix (iommu); -else if (pci_find_capability (iommu->dev,PCI_CAP_ID_MSI )) +if (pci_find_capability (iommu->dev,PCI_CAP_ID_MSI ))return iommu_setup_msi (iommu);return 1 ;
Signature · 1 line
1.6.3.1
PATCH 5/9 amd-iommu: add function to disable all iommus
In reply to Joerg Roedel[ impact: add disable_iommus function needed for suspend/resume ]
--- arch/x86/kernel/amd_iommu_init.c | 13 +++++++++++++ 1 files changed, 13 insertions(+), 0 deletions(-)
arch/x86/kernel/amd_iommu_init.c +13 -0
@@ -252,6 +252,11 @@ static void __init iommu_enable(struct amd_iommu *iommu)iommu_feature_enable (iommu,CONTROL_IOMMU_EN ); } +static void iommu_disable (struct amd_iommu *iommu) +{ +iommu_feature_disable (iommu,CONTROL_IOMMU_EN ); +} +/* * mapping and unmapping functions for the IOMMU MMIO space. Each AMD IOMMU in * the system has one.
@@ -945,6 +950,14 @@ static void __init enable_iommus(void) } } +static void disable_iommus (void ) +{ +struct amd_iommu *iommu; + +for_each_iommu (iommu) +iommu_disable (iommu); +} +/* * Suspend/Resume support * disable suspend until real resume implemented
Signature · 1 line
1.6.3.1
PATCH 6/9 amd-iommu: add function to flush tlb for all domains
In reply to Joerg Roedel[ impact: be able to flush io/tlb for all allocated domains ]
--- arch/x86/include/asm/amd_iommu.h | 1 + arch/x86/kernel/amd_iommu.c | 11 +++++++++++ 2 files changed, 12 insertions(+), 0 deletions(-)
arch/x86/include/asm/amd_iommu.h +1 -0
@@ -27,6 +27,7 @@ extern int amd_iommu_init(void);extern int amd_iommu_init_dma_ops (void );extern void amd_iommu_detect (void );extern irqreturn_t amd_iommu_int_handler (int irq,void *data); +extern void amd_iommu_flush_all_domains (void );#else static inline int amd_iommu_init (void ) {return -ENODEV ; }static inline void amd_iommu_detect (void ) { }
arch/x86/kernel/amd_iommu.c +11 -0
@@ -449,6 +449,17 @@ static void iommu_flush_domain(u16 domid) } } +void amd_iommu_flush_all_domains (void ) +{ +int i; + +for (i =1 ; i <MAX_DOMAIN_ID ; ++i) { +if (!test_bit (i, amd_iommu_pd_alloc_bitmap)) +continue ; +iommu_flush_domain (i); + } +} +/**************************************************************************** * * The functions below are used the create the page table mappings for
Signature · 1 line
1.6.3.1
PATCH 7/9 amd-iommu: add function to flush tlb for all devices
In reply to Joerg Roedel[ impact: be able to flush cache for all devtable entries ]
--- arch/x86/include/asm/amd_iommu.h | 1 + arch/x86/kernel/amd_iommu.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 0 deletions(-)
arch/x86/include/asm/amd_iommu.h +1 -0
@@ -28,6 +28,7 @@ extern int amd_iommu_init_dma_ops(void);extern void amd_iommu_detect (void );extern irqreturn_t amd_iommu_int_handler (int irq,void *data);extern void amd_iommu_flush_all_domains (void ); +extern void amd_iommu_flush_all_devices (void );#else static inline int amd_iommu_init (void ) {return -ENODEV ; }static inline void amd_iommu_detect (void ) { }
arch/x86/kernel/amd_iommu.c +18 -0
@@ -460,6 +460,24 @@ void amd_iommu_flush_all_domains(void) } } +void amd_iommu_flush_all_devices (void ) +{ +struct amd_iommu *iommu; +int i; + +for (i =0 ; i <= amd_iommu_last_bdf; ++i) { +if (amd_iommu_pd_table[i] ==NULL ) +continue ; + + iommu = amd_iommu_rlookup_table[i]; +if (!iommu) +continue ; + +iommu_queue_inv_dev_entry (iommu, i); +iommu_completion_wait (iommu); + } +} +/**************************************************************************** * * The functions below are used the create the page table mappings for
Signature · 1 line
1.6.3.1
PATCH 8/9 amd_iommu: un __init functions required for suspend/resume
In reply to Joerg Roedel[ impact: keep functions in kernel .text required in resume path ]
--- arch/x86/kernel/amd_iommu_init.c | 10 +++++----- 1 files changed, 5 insertions(+), 5 deletions(-)
arch/x86/kernel/amd_iommu_init.c +5 -5
@@ -193,7 +193,7 @@ static inline unsigned long tbl_size(int entry_size)* This function set the exclusion range in the IOMMU. DMA accesses to the * exclusion range are passed through untranslated */ -static void __init iommu_set_exclusion_range (struct amd_iommu *iommu) +static void iommu_set_exclusion_range (struct amd_iommu *iommu) {u64 start = iommu->exclusion_start &PAGE_MASK ;u64 limit = (start + iommu->exclusion_length) &PAGE_MASK ;
@@ -225,7 +225,7 @@ static void __init iommu_set_device_table(struct amd_iommu *iommu) }/* Generic functions to enable/disable certain features of the IOMMU. */ -static void __init iommu_feature_enable (struct amd_iommu *iommu,u8 bit) +static void iommu_feature_enable (struct amd_iommu *iommu,u8 bit) {u32 ctrl;
@@ -244,7 +244,7 @@ static void __init iommu_feature_disable(struct amd_iommu *iommu, u8 bit) }/* Function to enable the hardware */ -static void __init iommu_enable (struct amd_iommu *iommu) +static void iommu_enable (struct amd_iommu *iommu) {printk (KERN_INFO "AMD IOMMU: Enabling IOMMU at %s cap 0x%hx\n" ,dev_name (&iommu->dev->dev), iommu->cap_ptr);
@@ -811,7 +811,7 @@ static int __init iommu_setup_msi(struct amd_iommu *iommu)return 0 ; } -static int __init iommu_init_msi (struct amd_iommu *iommu) +static int iommu_init_msi (struct amd_iommu *iommu) {if (iommu->int_enabled)return 0 ;
@@ -936,7 +936,7 @@ static void init_device_table(void)* This function finally enables all IOMMUs found in the system after * they have been initialized */ -static void __init enable_iommus (void ) +static void enable_iommus (void ) {struct amd_iommu *iommu;
Signature · 1 line
1.6.3.1
PATCH 9/9 amd-iommu: implement suspend/resume
In reply to Joerg Roedel[ impact: put everything together and enable suspend/resume ]
--- arch/x86/kernel/amd_iommu_init.c | 21 ++++++++++++++++++++- 1 files changed, 20 insertions(+), 1 deletions(-)
arch/x86/kernel/amd_iommu_init.c +20 -1
@@ -965,12 +965,31 @@ static void disable_iommus(void)static int amd_iommu_resume (struct sys_device *dev) { +/* +* Disable IOMMUs before reprogramming the hardware registers. +* IOMMU is still enabled from the resume kernel. +*/ +disable_iommus (); + +/* re-load the hardware */ +enable_iommus (); + +/* +* we have to flush after the IOMMUs are enabled because a +* disabled IOMMU will never execute the commands we send +*/ +amd_iommu_flush_all_domains (); +amd_iommu_flush_all_devices (); +return 0 ; }static int amd_iommu_suspend (struct sys_device *dev,pm_message_t state) { -return -EINVAL ; +/* disable IOMMUs to go out of the way for BIOS */ +disable_iommus (); + +return 0 ; }static struct sysdev_class amd_iommu_sysdev_class = {
Signature · 1 line
1.6.3.1