iommu/amd: lock splitting & GFP_KERNEL allocation
v3 cover iommu/amd: lock splitting & GFP_KERNEL allocation
Hi,
this is the rebase on top of iommu/x86/amd of my last series. It takes Scotts comments into consideration from my v2.
It contains lock splitting and GFP_KERNEL allocation of remap-table. Mostly cleanup.
The patches were boot tested on an AMD EPYC 7601.
Sebastian
PATCH v3 1/10 iommu/amd: take into account that alloc_dev_data() may return NULL
find_dev_data() does not check whether the return value alloc_dev_data() is NULL. This was okay once because the pointer was returned once as-is. Since commit df3f7a6e8e85 ("iommu/amd: Use is_attach_deferred call-back") the pointer may be used within find_dev_data() so a NULL check is required.
--- drivers/iommu/amd_iommu.c | 2 ++ 1 file changed, 2 insertions(+)
drivers/iommu/amd_iommu.c +2 -0
@@ -310,6 +310,8 @@ static struct iommu_dev_data *find_dev_data(u16 devid)if (dev_data ==NULL ) { dev_data =alloc_dev_data (devid); +if (!dev_data) +return NULL ;if (translation_pre_enabled (iommu)) dev_data->defer_attach = true;
Signature · 1 line
2.16.2
PATCH v3 2/10 iommu/amd: turn dev_data_list into a lock less list
In reply to Sebastian Andrzej Siewioralloc_dev_data() adds new items to dev_data_list and search_dev_data() is searching for items in this list. Both protect the access to the list with a spinlock.
There is no need to navigate forth and back within the list and there is also no deleting of a specific item. This qualifies the list to become a lock less list and as part of this, the spinlock can be removed. With this change the ordering of those items within the list is changed:
before the change new items were added to the end of the list, now they are added to the front. I don't think it matters but wanted to mention it.
--- drivers/iommu/amd_iommu.c | 28 ++++++++++------------------ drivers/iommu/amd_iommu_types.h | 2 +- 2 files changed, 11 insertions(+), 19 deletions(-)
drivers/iommu/amd_iommu.c +10 -18
@@ -83,8 +83,7 @@static DEFINE_RWLOCK (amd_iommu_devtable_lock);/* List of all available dev_data structures */ -static LIST_HEAD (dev_data_list); -static DEFINE_SPINLOCK (dev_data_list_lock); +static LLIST_HEAD (dev_data_list);LIST_HEAD (ioapic_map);LIST_HEAD (hpet_map);
@@ -203,40 +202,33 @@ static struct dma_ops_domain* to_dma_ops_domain(struct protection_domain *domainstatic struct iommu_dev_data *alloc_dev_data (u16 devid) {struct iommu_dev_data *dev_data; -unsigned long flags; dev_data =kzalloc (sizeof (*dev_data),GFP_KERNEL );if (!dev_data)return NULL ; dev_data->devid = devid; - -spin_lock_irqsave (&dev_data_list_lock, flags); -list_add_tail (&dev_data->dev_data_list, &dev_data_list); -spin_unlock_irqrestore (&dev_data_list_lock, flags); -ratelimit_default_init (&dev_data->rs); +llist_add (&dev_data->dev_data_list, &dev_data_list);return dev_data; }static struct iommu_dev_data *search_dev_data (u16 devid) {struct iommu_dev_data *dev_data; -unsigned long flags; +struct llist_node *node; -spin_lock_irqsave (&dev_data_list_lock, flags); -list_for_each_entry (dev_data, &dev_data_list, dev_data_list) { +if (llist_empty (&dev_data_list)) +return NULL ; + + node = dev_data_list.first; +llist_for_each_entry (dev_data, node, dev_data_list) {if (dev_data->devid == devid) - goto out_unlock; +return dev_data; } - dev_data =NULL ; - -out_unlock: -spin_unlock_irqrestore (&dev_data_list_lock, flags); - -return dev_data; +return NULL ; }static int __last_alias (struct pci_dev *pdev,u16 alias,void *data)
drivers/iommu/amd_iommu_types.h +1 -1
@@ -627,7 +627,7 @@ struct devid_map {*/ struct iommu_dev_data {struct list_head list;/* For domain->dev_list */ -struct list_head dev_data_list;/* For global dev_data_list */ +struct llist_node dev_data_list;/* For global dev_data_list */ struct protection_domain *domain;/* Domain the device is bound to */ u16 devid;/* PCI Device ID */ u16 alias;/* Alias Device ID */
Signature · 1 line
2.16.2
PATCH v3 3/10 iommu/amd: split domain id out of amd_iommu_devtable_lock
In reply to Sebastian Andrzej Siewiordomain_id_alloc() and domain_id_free() is used for id management. Those two function share a bitmap (amd_iommu_pd_alloc_bitmap) and set/clear bits based on id allocation. There is no need to share this with amd_iommu_devtable_lock, it can use its own lock for this operation.
--- drivers/iommu/amd_iommu.c | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-)
drivers/iommu/amd_iommu.c +5 -7
@@ -81,6 +81,7 @@#define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38))static DEFINE_RWLOCK (amd_iommu_devtable_lock); +static DEFINE_SPINLOCK (pd_bitmap_lock);/* List of all available dev_data structures */ static LLIST_HEAD (dev_data_list);
@@ -1600,29 +1601,26 @@ static void del_domain_from_list(struct protection_domain *domain)static u16 domain_id_alloc (void ) { -unsigned long flags;int id; -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock (&pd_bitmap_lock); id =find_first_zero_bit (amd_iommu_pd_alloc_bitmap,MAX_DOMAIN_ID );BUG_ON (id ==0 );if (id >0 && id <MAX_DOMAIN_ID )__set_bit (id, amd_iommu_pd_alloc_bitmap);else id =0 ; -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock (&pd_bitmap_lock);return id; }static void domain_id_free (int id) { -unsigned long flags; - -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock (&pd_bitmap_lock);if (id >0 && id <MAX_DOMAIN_ID )__clear_bit (id, amd_iommu_pd_alloc_bitmap); -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock (&pd_bitmap_lock); }#define DEFINE_FREE_PT_FN (LVL ,FN ) \
Signature · 1 line
2.16.2
PATCH v3 4/10 iommu/amd: split irq_lookup_table out of the amd_iommu_devtable_lock
In reply to Sebastian Andrzej SiewiorThe function get_irq_table() reads/writes irq_lookup_table while holding the amd_iommu_devtable_lock. It also modifies amd_iommu_dev_table[].data[2].set_dte_entry() is using amd_iommu_dev_table[].data[0|1] (under the domain->lock) so it should be okay. The access to the iommu is serialized with its own (iommu's) lock.
So split out get_irq_table() out of amd_iommu_devtable_lock's lock.
--- drivers/iommu/amd_iommu.c | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-)
drivers/iommu/amd_iommu.c +3 -2
@@ -82,6 +82,7 @@static DEFINE_RWLOCK (amd_iommu_devtable_lock);static DEFINE_SPINLOCK (pd_bitmap_lock); +static DEFINE_SPINLOCK (iommu_table_lock);/* List of all available dev_data structures */ static LLIST_HEAD (dev_data_list);
@@ -3623,7 +3624,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic)unsigned long flags;u16 alias; -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock_irqsave (&iommu_table_lock, flags); iommu = amd_iommu_rlookup_table[devid];if (!iommu)
@@ -3688,7 +3689,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic)iommu_completion_wait (iommu); out_unlock: -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock_irqrestore (&iommu_table_lock, flags);return table; }
Signature · 1 line
2.16.2
PATCH v3 5/10 iommu/amd: remove the special case from alloc_irq_table()
In reply to Sebastian Andrzej Siewioralloc_irq_table() has a special ioapic argument. If set then it will pre-allocate / reserve the first 32 indexes. The argument is only once true and it would make alloc_irq_table() a little simpler if we would extract the special bits to the caller.
The caller of irq_remapping_alloc() is holding irq_domain_mutex so the initialization of iommu->irte_ops->set_allocated() should not race against other user.
--- drivers/iommu/amd_iommu.c | 34 ++++++++++++++++++++-------------- 1 file changed, 20 insertions(+), 14 deletions(-)
drivers/iommu/amd_iommu.c +20 -14
@@ -3617,7 +3617,7 @@ static struct irq_remap_table *get_irq_table(u16 devid)return table; } -static struct irq_remap_table *alloc_irq_table (u16 devid,bool ioapic) +static struct irq_remap_table *alloc_irq_table (u16 devid) {struct irq_remap_table *table =NULL ;struct amd_iommu *iommu;
@@ -3651,10 +3651,6 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic)/* Initialize table spin-lock */ raw_spin_lock_init (&table->lock); -if (ioapic) -/* Keep the first 32 indexes free for IOAPIC interrupts */ - table->min_index =32 ; - table->table =kmem_cache_alloc (amd_iommu_irq_cache,GFP_ATOMIC );if (!table->table) {kfree (table);
@@ -3669,12 +3665,6 @@ static struct irq_remap_table *alloc_irq_table(u16 devid, bool ioapic)memset (table->table,0 , (MAX_IRQS_PER_TABLE * (sizeof (u64) *2 ))); -if (ioapic) { -int i; - -for (i =0 ; i <32 ; ++i) - iommu->irte_ops->set_allocated (table, i); - } irq_lookup_table[devid] = table;set_dte_irq_entry (devid, table);
@@ -3704,7 +3694,7 @@ static int alloc_irq_index(u16 devid, int count, bool align)if (!iommu)return -ENODEV ; - table =alloc_irq_table (devid, false); + table =alloc_irq_table (devid);if (!table)return -ENODEV ;
@@ -4130,10 +4120,26 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,return ret;if (info->type ==X86_IRQ_ALLOC_TYPE_IOAPIC ) { -if (alloc_irq_table (devid, true)) +struct irq_remap_table *table; +struct amd_iommu *iommu; + + table =alloc_irq_table (devid); +if (table) { +if (!table->min_index) { +/* +* Keep the first 32 indexes free for IOAPIC +* interrupts. +*/ + table->min_index =32 ; + iommu = amd_iommu_rlookup_table[devid]; +for (i =0 ; i <32 ; ++i) + iommu->irte_ops->set_allocated (table, i); + } +WARN_ON (table->min_index !=32 ); index = info->ioapic_pin; -else + }else { ret = -ENOMEM ; + } }else {bool align = (info->type ==X86_IRQ_ALLOC_TYPE_MSI );
Signature · 1 line
2.16.2
PATCH v3 6/10 iommu/amd: use `table' instead `irt' as variable name in amd_iommu_update_ga()
In reply to Sebastian Andrzej SiewiorThe variable of type struct irq_remap_table is always named table' except in amd_iommu_update_ga() where it is called irt'. Make it consistent and name it also `table'.
--- drivers/iommu/amd_iommu.c | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-)
drivers/iommu/amd_iommu.c +5 -5
@@ -4405,7 +4405,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data) {unsigned long flags;struct amd_iommu *iommu; -struct irq_remap_table *irt; +struct irq_remap_table *table;struct amd_ir_data *ir_data = (struct amd_ir_data *)data;int devid = ir_data->irq_2_irte.devid;struct irte_ga *entry = (struct irte_ga *) ir_data->entry;
@@ -4419,11 +4419,11 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data)if (!iommu)return -ENODEV ; - irt =get_irq_table (devid); -if (!irt) + table =get_irq_table (devid); +if (!table)return -ENODEV ; -raw_spin_lock_irqsave (&irt->lock, flags); +raw_spin_lock_irqsave (&table->lock, flags);if (ref->lo.fields_vapic.guest_mode) {if (cpu >=0 )
@@ -4432,7 +4432,7 @@ int amd_iommu_update_ga(int cpu, bool is_run, void *data)barrier (); } -raw_spin_unlock_irqrestore (&irt->lock, flags); +raw_spin_unlock_irqrestore (&table->lock, flags);iommu_flush_irt (iommu, devid);iommu_completion_wait (iommu);
Signature · 1 line
2.16.2
PATCH v3 7/10 iommu/amd: factor out setting the remap table for a devid
In reply to Sebastian Andrzej SiewiorSetting the IRQ remap table for a specific devid (or its alias devid) includes three steps. Those three steps are always repeated each time this is done.
Introduce a new helper function, move those steps there and use that function instead. The compiler can still decide if it is worth to inline.
--- drivers/iommu/amd_iommu.c | 23 ++++++++++++----------- 1 file changed, 12 insertions(+), 11 deletions(-)
drivers/iommu/amd_iommu.c +12 -11
@@ -3617,6 +3617,14 @@ static struct irq_remap_table *get_irq_table(u16 devid)return table; } +static void set_remap_table_entry (struct amd_iommu *iommu,u16 devid, +struct irq_remap_table *table) +{ + irq_lookup_table[devid] = table; +set_dte_irq_entry (devid, table); +iommu_flush_dte (iommu, devid); +} +static struct irq_remap_table *alloc_irq_table (u16 devid) {struct irq_remap_table *table =NULL ;
@@ -3637,9 +3645,7 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) alias = amd_iommu_alias_table[devid]; table = irq_lookup_table[alias];if (table) { - irq_lookup_table[devid] = table; -set_dte_irq_entry (devid, table); -iommu_flush_dte (iommu, devid); +set_remap_table_entry (iommu, devid, table); goto out; }
@@ -3666,14 +3672,9 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) (MAX_IRQS_PER_TABLE * (sizeof(u64) * 2))); - irq_lookup_table[devid] = table; -set_dte_irq_entry (devid, table); -iommu_flush_dte (iommu, devid); -if (devid != alias) { - irq_lookup_table[alias] = table; -set_dte_irq_entry (alias, table); -iommu_flush_dte (iommu, alias); - } +set_remap_table_entry (iommu, devid, table); +if (devid != alias) +set_remap_table_entry (iommu, alias, table); out:iommu_completion_wait (iommu);
Signature · 1 line
2.16.2
PATCH v3 8/10 iommu/amd: drop the lock while allocating new irq remap table
In reply to Sebastian Andrzej SiewiorThe irq_remap_table is allocated while the iommu_table_lock is held with interrupts disabled.
Quoted 1 line
From looking at the call sites, all callers are in the early device
initialisation (apic_bsp_setup(), pci_enable_device(), pci_enable_msi()) so make sense to drop the lock which also enables interrupts and try to allocate that memory with GFP_KERNEL instead GFP_ATOMIC.
Since during the allocation the iommu_table_lock is dropped, we need to recheck if table exists after the lock has been reacquired. I *think* that it is impossible that the "devid" entry appears in irq_lookup_table while the lock is dropped since the same device can only be probed once. However I check for both cases, just to be sure.
--- drivers/iommu/amd_iommu.c | 65 +++++++++++++++++++++++++++++++++-------------- 1 file changed, 46 insertions(+), 19 deletions(-)
drivers/iommu/amd_iommu.c +46 -19
@@ -3617,6 +3617,30 @@ static struct irq_remap_table *get_irq_table(u16 devid)return table; } +static struct irq_remap_table *__alloc_irq_table (void ) +{ +struct irq_remap_table *table; + + table =kzalloc (sizeof (*table),GFP_KERNEL ); +if (!table) +return NULL ; + + table->table =kmem_cache_alloc (amd_iommu_irq_cache,GFP_KERNEL ); +if (!table->table) { +kfree (table); +return NULL ; + } +raw_spin_lock_init (&table->lock); + +if (!AMD_IOMMU_GUEST_IR_GA (amd_iommu_guest_ir)) +memset (table->table,0 , +MAX_IRQS_PER_TABLE *sizeof (u32)); +else +memset (table->table,0 , + (MAX_IRQS_PER_TABLE * (sizeof (u64) *2 ))); +return table; +} +static void set_remap_table_entry (struct amd_iommu *iommu,u16 devid,struct irq_remap_table *table) {
@@ -3628,6 +3652,7 @@ static void set_remap_table_entry(struct amd_iommu *iommu, u16 devid,static struct irq_remap_table *alloc_irq_table (u16 devid) {struct irq_remap_table *table =NULL ; +struct irq_remap_table *new_table =NULL ;struct amd_iommu *iommu;unsigned long flags;u16 alias;
@@ -3646,42 +3671,44 @@ static struct irq_remap_table *alloc_irq_table(u16 devid) table = irq_lookup_table[alias];if (table) {set_remap_table_entry (iommu, devid, table); - goto out; + goto out_wait; } +spin_unlock_irqrestore (&iommu_table_lock, flags);/* Nothing there yet, allocate new irq remapping table */ - table =kzalloc (sizeof (*table),GFP_ATOMIC ); -if (!table) + new_table =__alloc_irq_table (); +if (!new_table) +return NULL ; + +spin_lock_irqsave (&iommu_table_lock, flags); + + table = irq_lookup_table[devid]; +if (table) goto out_unlock; -/* Initialize table spin-lock */ -raw_spin_lock_init (&table->lock); - - table->table =kmem_cache_alloc (amd_iommu_irq_cache,GFP_ATOMIC ); -if (!table->table) { -kfree (table); - table =NULL ; - goto out_unlock; + table = irq_lookup_table[alias]; +if (table) { +set_remap_table_entry (iommu, devid, table); + goto out_wait; } -if (!AMD_IOMMU_GUEST_IR_GA (amd_iommu_guest_ir)) -memset (table->table,0 , -MAX_IRQS_PER_TABLE *sizeof (u32)); -else -memset (table->table,0 , - (MAX_IRQS_PER_TABLE * (sizeof (u64) *2 ))); - + table = new_table; + new_table =NULL ;set_remap_table_entry (iommu, devid, table);if (devid != alias)set_remap_table_entry (iommu, alias, table); -out: +out_wait:iommu_completion_wait (iommu); out_unlock:spin_unlock_irqrestore (&iommu_table_lock, flags); +if (new_table) { +kmem_cache_free (amd_iommu_irq_cache, new_table->table); +kfree (new_table); + }return table; }
Signature · 1 line
2.16.2
PATCH v3 9/10 iommu/amd: make amd_iommu_devtable_lock a spin_lock
In reply to Sebastian Andrzej SiewiorBefore commit 0bb6e243d7fb ("iommu/amd: Support IOMMU_DOMAIN_DMA type allocation") amd_iommu_devtable_lock had a read_lock() user but now there are none. In fact, after the mentioned commit we had only write_lock() user of the lock. Since there is no reason to keep it as writer lock, change its type to a spin_lock.
I *think* that we might even be able to remove the lock because all its current user seem to have their own protection.
--- drivers/iommu/amd_iommu.c | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-)
drivers/iommu/amd_iommu.c +7 -7
@@ -80,7 +80,7 @@*/ #define AMD_IOMMU_PGSIZES ((~0xFFFUL) & ~(2ULL << 38)) -static DEFINE_RWLOCK (amd_iommu_devtable_lock); +static DEFINE_SPINLOCK (amd_iommu_devtable_lock);static DEFINE_SPINLOCK (pd_bitmap_lock);static DEFINE_SPINLOCK (iommu_table_lock);
@@ -2097,9 +2097,9 @@ static int attach_device(struct device *dev, } skip_ats_check: -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock_irqsave (&amd_iommu_devtable_lock, flags); ret =__attach_device (dev_data, domain); -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock_irqrestore (&amd_iommu_devtable_lock, flags);/* * We might boot into a crash-kernel here. The crashed kernel
@@ -2149,9 +2149,9 @@ static void detach_device(struct device *dev) domain = dev_data->domain;/* lock device table */ -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock_irqsave (&amd_iommu_devtable_lock, flags);__detach_device (dev_data); -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock_irqrestore (&amd_iommu_devtable_lock, flags);if (!dev_is_pci (dev))return ;
@@ -2814,7 +2814,7 @@ static void cleanup_domain(struct protection_domain *domain)struct iommu_dev_data *entry;unsigned long flags; -write_lock_irqsave (&amd_iommu_devtable_lock, flags); +spin_lock_irqsave (&amd_iommu_devtable_lock, flags);while (!list_empty (&domain->dev_list)) { entry = list_first_entry(&domain->dev_list,
@@ -2822,7 +2822,7 @@ static void cleanup_domain(struct protection_domain *domain)__detach_device (entry); } -write_unlock_irqrestore (&amd_iommu_devtable_lock, flags); +spin_unlock_irqrestore (&amd_iommu_devtable_lock, flags); }static void protection_domain_free (struct protection_domain *domain)
Signature · 1 line
2.16.2
PATCH v3 10/10 iommu/amd: return proper error code in irq_remapping_alloc()
In reply to Sebastian Andrzej SiewiorIn the unlikely case when alloc_irq_table() is not able to return a remap table then "ret" will be assigned with an error code. Later, the code checks index' and if it is negative (which it is because it is initialized with -1') and then then function properly aborts but returns -1' instead -ENOMEM' what was intended.
In order to correct this, I assign -ENOMEM to index.
--- drivers/iommu/amd_iommu.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
drivers/iommu/amd_iommu.c +2 -2
@@ -4124,7 +4124,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,struct amd_ir_data *data =NULL ;struct irq_cfg *cfg;int i, ret, devid; -int index =-1 ; +int index;if (!info)return -EINVAL ;
@@ -4166,7 +4166,7 @@ static int irq_remapping_alloc(struct irq_domain *domain, unsigned int virq,WARN_ON (table->min_index !=32 ); index = info->ioapic_pin; }else { - ret = -ENOMEM ; + index = -ENOMEM ; } }else {bool align = (info->type ==X86_IRQ_ALLOC_TYPE_MSI );
Signature · 1 line
2.16.2
Quoted 9 lines · Sebastian Andrzej Siewior
Hi,
this is the rebase on top of iommu/x86/amd of my last series. It takes Scotts comments into consideration from my v2.
It contains lock splitting and
GFP_KERNELallocation of remap-table. Mostly cleanup.The patches were boot tested on an AMD EPYC 7601.
Applied, thanks.