KVM/arm updates for 5.4
Subsystems (5)
Marc Zyngier
maintainercover KVM/arm updates for 5.4
Paolo, Radim,
Here is the KVM/arm updates for 5.4: a new ITS translation cache improving performance for interrupt injection of of assigned devices, a couple of fixes to allow up to 512 vcpus, and a number of fixes and other cleanups.
Please pull,
M.
The following changes since commit d45331b00ddb179e291766617259261c112db872:
Linux 5.3-rc4 (2019-08-11 13:26:41 -0700)
are available in the Git repository at:
git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.git tags/kvmarm-5.4
for you to fetch changes up to 92f35b751c71d14250a401246f2c792e3aa5b386:
KVM: arm/arm64: vgic: Allow more than 256 vcpus for KVM_IRQ_LINE (2019-09-09 12:29:09 +0100)
----------------------------------------------------------------
KVM/arm updates for 5.4
- New ITS translation cache
- Allow up to 512 CPUs to be supported with GICv3 (for real this time)
- Now call kvm_arch_vcpu_blocking early in the blocking sequence
- Tidy-up device mappings in S2 when DIC is available
- Clean icache invalidation on VMID rollover
- General cleanup
----------------------------------------------------------------
Alexandru Elisei (1):
KVM: arm/arm64: vgic: Make function comments match function declarations
Eric Auger (1):
KVM: arm/arm64: vgic: Use a single IO device per redistributor
James Morse (1):
arm64: KVM: Device mappings should be execute-never
Marc Zyngier (13):
KVM: arm/arm64: vgic: Add LPI translation cache definition
KVM: arm/arm64: vgic: Add __vgic_put_lpi_locked primitive
KVM: arm/arm64: vgic-its: Add MSI-LPI translation cache invalidation
KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on specific commands
KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on disabling LPIs
KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on ITS disable
KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on vgic teardown
KVM: arm/arm64: vgic-its: Cache successful MSI->LPI translation
KVM: arm/arm64: vgic-its: Check the LPI translation cache on MSI injection
KVM: arm/arm64: vgic-irqfd: Implement kvm_arch_set_irq_inatomic
KVM: Call kvm_arch_vcpu_blocking early into the blocking sequence
KVM: arm/arm64: vgic: Remove spurious semicolons
KVM: arm/arm64: vgic: Allow more than 256 vcpus for KVM_IRQ_LINE
Mark Rutland (1):
arm64/kvm: Remove VMID rollover I-cache maintenance
Documentation/virt/kvm/api.txt | 12 +- arch/arm/include/uapi/asm/kvm.h | 4 +- arch/arm64/include/asm/pgtable-prot.h | 2 +- arch/arm64/include/uapi/asm/kvm.h | 4 +- arch/arm64/kvm/hyp/tlb.c | 14 ++- include/kvm/arm_vgic.h | 4 +- include/uapi/linux/kvm.h | 1 + virt/kvm/arm/arm.c | 2 + virt/kvm/arm/vgic/vgic-init.c | 8 +- virt/kvm/arm/vgic/vgic-irqfd.c | 36 +++++- virt/kvm/arm/vgic/vgic-its.c | 207 ++++++++++++++++++++++++++++++++++ virt/kvm/arm/vgic/vgic-mmio-v3.c | 85 +++++--------- virt/kvm/arm/vgic/vgic-v2.c | 7 +- virt/kvm/arm/vgic/vgic-v3.c | 7 +- virt/kvm/arm/vgic/vgic.c | 26 +++-- virt/kvm/arm/vgic/vgic.h | 5 + virt/kvm/kvm_main.c | 7 +- 17 files changed, 339 insertions(+), 92 deletions(-)
Marc Zyngier
maintainerPATCH 1/17 KVM: arm/arm64: vgic: Add LPI translation cache definition
Add the basic data structure that expresses an MSI to LPI translation as well as the allocation/release hooks.
The size of the cache is arbitrarily defined as 16*nr_vcpus.
--- include/kvm/arm_vgic.h | 3 +++ virt/kvm/arm/vgic/vgic-init.c | 5 ++++ virt/kvm/arm/vgic/vgic-its.c | 49 +++++++++++++++++++++++++++++++++++ virt/kvm/arm/vgic/vgic.h | 2 ++ 4 files changed, 59 insertions(+)
include/kvm/arm_vgic.h +3 -0
@@ -249,6 +249,9 @@ struct vgic_dist {struct list_head lpi_list_head;int lpi_list_count; +/* LPI translation cache */ +struct list_head lpi_translation_cache; +/* used by vgic-debug */ struct vgic_state_iter *iter;
virt/kvm/arm/vgic/vgic-init.c +5 -0
@@ -53,6 +53,7 @@ void kvm_vgic_early_init(struct kvm *kvm)struct vgic_dist *dist = &kvm->arch.vgic;INIT_LIST_HEAD (&dist->lpi_list_head); +INIT_LIST_HEAD (&dist->lpi_translation_cache);raw_spin_lock_init (&dist->lpi_list_lock); }
@@ -294,6 +295,7 @@ int vgic_init(struct kvm *kvm) }if (vgic_has_its (kvm)) { +vgic_lpi_translation_cache_init (kvm); ret =vgic_v4_init (kvm);if (ret) goto out;
@@ -335,6 +337,9 @@ static void kvm_vgic_dist_destroy(struct kvm *kvm)INIT_LIST_HEAD (&dist->rd_regions); } +if (vgic_has_its (kvm)) +vgic_lpi_translation_cache_destroy (kvm); +if (vgic_supports_direct_msis (kvm))vgic_v4_teardown (kvm); }
virt/kvm/arm/vgic/vgic-its.c +49 -0
67 lines not on this page · raw message
virt/kvm/arm/vgic/vgic.h +2 -0
@@ -307,6 +307,8 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr);int vgic_its_resolve_lpi (struct kvm *kvm,struct vgic_its *its,u32 devid,u32 eventid,struct vgic_irq **irq);struct vgic_its *vgic_msi_to_its (struct kvm *kvm,struct kvm_msi *msi); +void vgic_lpi_translation_cache_init (struct kvm *kvm); +void vgic_lpi_translation_cache_destroy (struct kvm *kvm);bool vgic_supports_direct_msis (struct kvm *kvm);int vgic_v4_init (struct kvm *kvm);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 2/17 KVM: arm/arm64: vgic: Add __vgic_put_lpi_locked primitive
In reply to Marc ZyngierOur LPI translation cache needs to be able to drop the refcount on an LPI whilst already holding the lpi_list_lock.
Let's add a new primitive for this.
--- virt/kvm/arm/vgic/vgic.c | 26 +++++++++++++++++--------- virt/kvm/arm/vgic/vgic.h | 1 + 2 files changed, 18 insertions(+), 9 deletions(-)
virt/kvm/arm/vgic/vgic.c +17 -9
@@ -119,6 +119,22 @@ static void vgic_irq_release(struct kref *ref) { } +/* +* Drop the refcount on the LPI. Must be called with lpi_list_lock held. +*/ +void __vgic_put_lpi_locked (struct kvm *kvm,struct vgic_irq *irq) +{ +struct vgic_dist *dist = &kvm->arch.vgic; + +if (!kref_put (&irq->refcount, vgic_irq_release)) +return ; + +list_del (&irq->lpi_list); + dist->lpi_list_count--; + +kfree (irq); +} +void vgic_put_irq (struct kvm *kvm,struct vgic_irq *irq) {struct vgic_dist *dist = &kvm->arch.vgic;
@@ -128,16 +144,8 @@ void vgic_put_irq(struct kvm *kvm, struct vgic_irq *irq)return ;raw_spin_lock_irqsave (&dist->lpi_list_lock, flags); -if (!kref_put (&irq->refcount, vgic_irq_release)) { -raw_spin_unlock_irqrestore (&dist->lpi_list_lock, flags); -return ; - }; - -list_del (&irq->lpi_list); - dist->lpi_list_count--; +__vgic_put_lpi_locked (kvm, irq);raw_spin_unlock_irqrestore (&dist->lpi_list_lock, flags); - -kfree (irq); }void vgic_flush_pending_lpis (struct kvm_vcpu *vcpu)
virt/kvm/arm/vgic/vgic.h +1 -0
@@ -161,6 +161,7 @@ vgic_get_mmio_region(struct kvm_vcpu *vcpu, struct vgic_io_device *iodev, gpa_t addr, int len);struct vgic_irq *vgic_get_irq (struct kvm *kvm,struct kvm_vcpu *vcpu,u32 intid); +void __vgic_put_lpi_locked (struct kvm *kvm,struct vgic_irq *irq);void vgic_put_irq (struct kvm *kvm,struct vgic_irq *irq);bool vgic_get_phys_line_level (struct vgic_irq *irq);void vgic_irq_set_phys_pending (struct vgic_irq *irq,bool pending);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 3/17 KVM: arm/arm64: vgic-its: Add MSI-LPI translation cache invalidation
In reply to Marc ZyngierThere's a number of cases where we need to invalidate the caching of translations, so let's add basic support for that.
--- virt/kvm/arm/vgic/vgic-its.c | 23 +++++++++++++++++++++++ virt/kvm/arm/vgic/vgic.h | 1 + 2 files changed, 24 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +23 -0
@@ -535,6 +535,29 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,return 0 ; } +void vgic_its_invalidate_cache (struct kvm *kvm) +{ +struct vgic_dist *dist = &kvm->arch.vgic; +struct vgic_translation_cache_entry *cte; +unsigned long flags; + +raw_spin_lock_irqsave (&dist->lpi_list_lock, flags); + +list_for_each_entry (cte, &dist->lpi_translation_cache, entry) { +/* +* If we hit a NULL entry, there is nothing after this +* point. +*/ +if (!cte->irq) +break ; + +__vgic_put_lpi_locked (kvm, cte->irq); + cte->irq =NULL ; + } + +raw_spin_unlock_irqrestore (&dist->lpi_list_lock, flags); +} +int vgic_its_resolve_lpi (struct kvm *kvm,struct vgic_its *its,u32 devid,u32 eventid,struct vgic_irq **irq) {
virt/kvm/arm/vgic/vgic.h +1 -0
@@ -310,6 +310,7 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,struct vgic_its *vgic_msi_to_its (struct kvm *kvm,struct kvm_msi *msi);void vgic_lpi_translation_cache_init (struct kvm *kvm);void vgic_lpi_translation_cache_destroy (struct kvm *kvm); +void vgic_its_invalidate_cache (struct kvm *kvm);bool vgic_supports_direct_msis (struct kvm *kvm);int vgic_v4_init (struct kvm *kvm);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 4/17 KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on specific commands
In reply to Marc ZyngierThe LPI translation cache needs to be discarded when an ITS command may affect the translation of an LPI (DISCARD, MAPC and MAPD with V=0) or the routing of an LPI to a redistributor with disabled LPIs (MOVI, MOVALL).
We decide to perform a full invalidation of the cache, irrespective of the LPI that is affected. Commands are supposed to be rare enough that it doesn't matter.
--- virt/kvm/arm/vgic/vgic-its.c | 9 +++++++++ 1 file changed, 9 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +9 -0
@@ -722,6 +722,8 @@ static int vgic_its_cmd_handle_discard(struct kvm *kvm, struct vgic_its *its,* don't bother here since we clear the ITTE anyway and the * pending state is a property of the ITTE struct. */ +vgic_its_invalidate_cache (kvm); +its_free_ite (kvm, ite);return 0 ; }
@@ -757,6 +759,8 @@ static int vgic_its_cmd_handle_movi(struct kvm *kvm, struct vgic_its *its, ite->collection = collection; vcpu =kvm_get_vcpu (kvm, collection->target_addr); +vgic_its_invalidate_cache (kvm); +return update_affinity (ite->irq, vcpu); }
@@ -985,6 +989,8 @@ static void vgic_its_free_device(struct kvm *kvm, struct its_device *device)list_for_each_entry_safe (ite, temp, &device->itt_head, ite_list)its_free_ite (kvm, ite); +vgic_its_invalidate_cache (kvm); +list_del (&device->dev_list);kfree (device); }
@@ -1090,6 +1096,7 @@ static int vgic_its_cmd_handle_mapc(struct kvm *kvm, struct vgic_its *its,if (!valid) {vgic_its_free_collection (its, coll_id); +vgic_its_invalidate_cache (kvm); }else { collection =find_collection (its, coll_id);
@@ -1238,6 +1245,8 @@ static int vgic_its_cmd_handle_movall(struct kvm *kvm, struct vgic_its *its,vgic_put_irq (kvm, irq); } +vgic_its_invalidate_cache (kvm); +kfree (intids);return 0 ; }
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 5/17 KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on disabling LPIs
In reply to Marc ZyngierIf a vcpu disables LPIs at its redistributor level, we need to make sure we won't pend more interrupts. For this, we need to invalidate the LPI translation cache.
--- virt/kvm/arm/vgic/vgic-mmio-v3.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
virt/kvm/arm/vgic/vgic-mmio-v3.c +3 -1
@@ -192,8 +192,10 @@ static void vgic_mmio_write_v3r_ctlr(struct kvm_vcpu *vcpu, vgic_cpu->lpis_enabled = val &GICR_CTLR_ENABLE_LPIS ; -if (was_enabled && !vgic_cpu->lpis_enabled) +if (was_enabled && !vgic_cpu->lpis_enabled) {vgic_flush_pending_lpis (vcpu); +vgic_its_invalidate_cache (vcpu->kvm); + }if (!was_enabled && vgic_cpu->lpis_enabled)vgic_enable_lpis (vcpu);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 6/17 KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on ITS disable
In reply to Marc ZyngierIf an ITS gets disabled, we need to make sure that further interrupts won't hit in the cache. For that, we invalidate the translation cache when the ITS is disabled.
--- virt/kvm/arm/vgic/vgic-its.c | 2 ++ 1 file changed, 2 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +2 -0
@@ -1597,6 +1597,8 @@ static void vgic_mmio_write_its_ctlr(struct kvm *kvm, struct vgic_its *its, goto out; its->enabled = !!(val &GITS_CTLR_ENABLE ); +if (!its->enabled) +vgic_its_invalidate_cache (kvm);/* * Try to process any pending commands. This function bails out early
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 7/17 KVM: arm/arm64: vgic-its: Invalidate MSI-LPI translation cache on vgic teardown
In reply to Marc ZyngierIn order to avoid leaking vgic_irq structures on teardown, we need to drop all references to LPIs before deallocating the cache itself.
This is done by invalidating the cache on vgic teardown.
--- virt/kvm/arm/vgic/vgic-its.c | 2 ++ 1 file changed, 2 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +2 -0
@@ -1731,6 +1731,8 @@ void vgic_lpi_translation_cache_destroy(struct kvm *kvm)struct vgic_dist *dist = &kvm->arch.vgic;struct vgic_translation_cache_entry *cte, *tmp; +vgic_its_invalidate_cache (kvm); +list_for_each_entry_safe (cte, tmp, &dist->lpi_translation_cache, entry) {list_del (&cte->entry);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 8/17 KVM: arm/arm64: vgic-its: Cache successful MSI->LPI translation
In reply to Marc ZyngierOn a successful translation, preserve the parameters in the LPI translation cache. Each translation is reusing the last slot in the list, naturally evicting the least recently used entry.
--- virt/kvm/arm/vgic/vgic-its.c | 86 ++++++++++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +86 -0
@@ -535,6 +535,90 @@ static unsigned long vgic_mmio_read_its_idregs(struct kvm *kvm,return 0 ; } +static struct vgic_irq *__vgic_its_check_cache (struct vgic_dist *dist, +phys_addr_t db, +u32 devid,u32 eventid) +{ +struct vgic_translation_cache_entry *cte; + +list_for_each_entry (cte, &dist->lpi_translation_cache, entry) { +/* +* If we hit a NULL entry, there is nothing after this +* point. +*/ +if (!cte->irq) +break ; + +if (cte->db != db || cte->devid != devid || + cte->eventid != eventid) +continue ; + +/* +* Move this entry to the head, as it is the most +* recently used. +*/ +if (!list_is_first (&cte->entry, &dist->lpi_translation_cache)) +list_move (&cte->entry, &dist->lpi_translation_cache); + +return cte->irq; + } + +return NULL ; +} + +static void vgic_its_cache_translation (struct kvm *kvm,struct vgic_its *its, +u32 devid,u32 eventid, +struct vgic_irq *irq) +{ +struct vgic_dist *dist = &kvm->arch.vgic; +struct vgic_translation_cache_entry *cte; +unsigned long flags; +phys_addr_t db; + +/* Do not cache a directly injected interrupt */ +if (irq->hw) +return ; + +raw_spin_lock_irqsave (&dist->lpi_list_lock, flags); + +if (unlikely (list_empty (&dist->lpi_translation_cache))) + goto out; + +/* +* We could have raced with another CPU caching the same +* translation behind our back, so let's check it is not in +* already +*/ + db = its->vgic_its_base +GITS_TRANSLATER ; +if (__vgic_its_check_cache (dist, db, devid, eventid)) + goto out; + +/* Always reuse the last entry (LRU policy) */ + cte =list_last_entry (&dist->lpi_translation_cache, +typeof (*cte), entry); + +/* +* Caching the translation implies having an extra reference +* to the interrupt, so drop the potential reference on what +* was in the cache, and increment it on the new interrupt. +*/ +if (cte->irq) +__vgic_put_lpi_locked (kvm, cte->irq); + +vgic_get_irq_kref (irq); + + cte->db = db; + cte->devid = devid; + cte->eventid = eventid; + cte->irq = irq; + +/* Move the new translation to the head of the list */ +list_move (&cte->entry, &dist->lpi_translation_cache); + +out: +raw_spin_unlock_irqrestore (&dist->lpi_list_lock, flags); +} +void vgic_its_invalidate_cache (struct kvm *kvm) {struct vgic_dist *dist = &kvm->arch.vgic;
@@ -578,6 +662,8 @@ int vgic_its_resolve_lpi(struct kvm *kvm, struct vgic_its *its,if (!vcpu->arch.vgic_cpu.lpis_enabled)return -EBUSY ; +vgic_its_cache_translation (kvm, its, devid, eventid, ite->irq); + *irq = ite->irq;return 0 ; }
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 9/17 KVM: arm/arm64: vgic-its: Check the LPI translation cache on MSI injection
In reply to Marc ZyngierWhen performing an MSI injection, let's first check if the translation is already in the cache. If so, let's inject it quickly without going through the whole translation process.
--- virt/kvm/arm/vgic/vgic-its.c | 36 ++++++++++++++++++++++++++++++++++++ virt/kvm/arm/vgic/vgic.h | 1 + 2 files changed, 37 insertions(+)
virt/kvm/arm/vgic/vgic-its.c +36 -0
@@ -566,6 +566,20 @@ static struct vgic_irq *__vgic_its_check_cache(struct vgic_dist *dist,return NULL ; } +static struct vgic_irq *vgic_its_check_cache (struct kvm *kvm,phys_addr_t db, +u32 devid,u32 eventid) +{ +struct vgic_dist *dist = &kvm->arch.vgic; +struct vgic_irq *irq; +unsigned long flags; + +raw_spin_lock_irqsave (&dist->lpi_list_lock, flags); + irq =__vgic_its_check_cache (dist, db, devid, eventid); +raw_spin_unlock_irqrestore (&dist->lpi_list_lock, flags); + +return irq; +} +static void vgic_its_cache_translation (struct kvm *kvm,struct vgic_its *its,u32 devid,u32 eventid,struct vgic_irq *irq)
@@ -725,6 +739,25 @@ static int vgic_its_trigger_msi(struct kvm *kvm, struct vgic_its *its,return 0 ; } +int vgic_its_inject_cached_translation (struct kvm *kvm,struct kvm_msi *msi) +{ +struct vgic_irq *irq; +unsigned long flags; +phys_addr_t db; + + db = (u64 )msi->address_hi <<32 | msi->address_lo; + irq =vgic_its_check_cache (kvm, db, msi->devid, msi->data); + +if (!irq) +return -1 ; + +raw_spin_lock_irqsave (&irq->irq_lock, flags); + irq->pending_latch = true; +vgic_queue_irq_unlock (kvm, irq, flags); + +return 0 ; +} +/* * Queries the KVM IO bus framework to get the ITS pointer from the given * doorbell address.
@@ -736,6 +769,9 @@ int vgic_its_inject_msi(struct kvm *kvm, struct kvm_msi *msi)struct vgic_its *its;int ret; +if (!vgic_its_inject_cached_translation (kvm, msi)) +return 1 ; + its =vgic_msi_to_its (kvm, msi);if (IS_ERR (its))return PTR_ERR (its);
virt/kvm/arm/vgic/vgic.h +1 -0
@@ -308,6 +308,7 @@ int vgic_copy_lpi_list(struct kvm *kvm, struct kvm_vcpu *vcpu, u32 **intid_ptr);int vgic_its_resolve_lpi (struct kvm *kvm,struct vgic_its *its,u32 devid,u32 eventid,struct vgic_irq **irq);struct vgic_its *vgic_msi_to_its (struct kvm *kvm,struct kvm_msi *msi); +int vgic_its_inject_cached_translation (struct kvm *kvm,struct kvm_msi *msi);void vgic_lpi_translation_cache_init (struct kvm *kvm);void vgic_lpi_translation_cache_destroy (struct kvm *kvm);void vgic_its_invalidate_cache (struct kvm *kvm);
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 10/17 KVM: arm/arm64: vgic-irqfd: Implement kvm_arch_set_irq_inatomic
In reply to Marc ZyngierNow that we have a cache of MSI->LPI translations, it is pretty easy to implement kvm_arch_set_irq_inatomic (this cache can be parsed without sleeping).
Hopefully, this will improve some LPI-heavy workloads.
--- virt/kvm/arm/vgic/vgic-irqfd.c | 36 ++++++++++++++++++++++++++++------ 1 file changed, 30 insertions(+), 6 deletions(-)
virt/kvm/arm/vgic/vgic-irqfd.c +30 -6
@@ -66,6 +66,15 @@ int kvm_set_routing_entry(struct kvm *kvm,return r; } +static void kvm_populate_msi (struct kvm_kernel_irq_routing_entry *e, +struct kvm_msi *msi) +{ + msi->address_lo = e->msi.address_lo; + msi->address_hi = e->msi.address_hi; + msi->data = e->msi.data; + msi->flags = e->msi.flags; + msi->devid = e->msi.devid; +}/** * kvm_set_msi: inject the MSI corresponding to the * MSI routing entry
@@ -79,21 +88,36 @@ int kvm_set_msi(struct kvm_kernel_irq_routing_entry *e, {struct kvm_msi msi; - msi.address_lo = e->msi.address_lo; - msi.address_hi = e->msi.address_hi; - msi.data = e->msi.data; - msi.flags = e->msi.flags; - msi.devid = e->msi.devid; -if (!vgic_has_its (kvm))return -ENODEV ;if (!level)return -1 ; +kvm_populate_msi (e, &msi);return vgic_its_inject_msi (kvm, &msi); } +/** +* kvm_arch_set_irq_inatomic: fast-path for irqfd injection +* +* Currently only direct MSI injection is supported. +*/ +int kvm_arch_set_irq_inatomic (struct kvm_kernel_irq_routing_entry *e, +struct kvm *kvm,int irq_source_id,int level, +bool line_status) +{ +if (e->type ==KVM_IRQ_ROUTING_MSI &&vgic_has_its (kvm) && level) { +struct kvm_msi msi; + +kvm_populate_msi (e, &msi); +if (!vgic_its_inject_cached_translation (kvm, &msi)) +return 0 ; + } + +return -EWOULDBLOCK ; +} +int kvm_vgic_setup_default_irq_routing (struct kvm *kvm) {struct kvm_irq_routing_entry *entries;
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 11/17 arm64/kvm: Remove VMID rollover I-cache maintenance
In reply to Marc ZyngierFrom: Mark Rutland <[email protected]>
For VPIPT I-caches, we need I-cache maintenance on VMID rollover to avoid an ABA problem. Consider a single vCPU VM, with a pinned stage-2, running with an idmap VA->IPA and idmap IPA->PA. If we don't do maintenance on rollover:
// VMID A
Writes insn X to PA 0xF
Invalidates PA 0xF (for VMID A)
I$ contains [{A,F}->X]
[VMID ROLLOVER]
// VMID B
Writes insn Y to PA 0xF
Invalidates PA 0xF (for VMID B)
I$ contains [{A,F}->X, {B,F}->Y]
[VMID ROLLOVER]
// VMID A
I$ contains [{A,F}->X, {B,F}->Y]
Unexpectedly hits stale I$ line {A,F}->X.However, for PIPT and VIPT I-caches, the VMID doesn't affect lookup or constrain maintenance. Given the VMID doesn't affect PIPT and VIPT I-caches, and given VMID rollover is independent of changes to stage-2 mappings, I-cache maintenance cannot be necessary on VMID rollover for PIPT or VIPT I-caches.
This patch removes the maintenance on rollover for VIPT and PIPT I-caches. At the same time, the unnecessary colons are removed from the asm statement to make it more legible.
--- arch/arm64/kvm/hyp/tlb.c | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-)
arch/arm64/kvm/hyp/tlb.c +13 -1
@@ -193,6 +193,18 @@ void __hyp_text __kvm_flush_vm_context(void) {dsb (ishst);__tlbi (alle1is); - asmvolatile ("ic ialluis" : : ); + +/* +* VIPT and PIPT caches are not affected by VMID, so no maintenance +* is necessary across a VMID rollover. +* +* VPIPT caches constrain lookup and maintenance to the active VMID, +* so we need to invalidate lines with a stale VMID to avoid an ABA +* race after multiple rollovers. +* +*/ +if (icache_is_vpipt ()) + asmvolatile ("ic ialluis" ); +dsb (ish); }
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 12/17 KVM: arm/arm64: vgic: Make function comments match function declarations
In reply to Marc ZyngierFrom: Alexandru Elisei <[email protected]>
Since commit 503a62862e8f ("KVM: arm/arm64: vgic: Rely on the GIC driver to parse the firmware tables"), the vgic_v{2,3}_probe functions stopped using a DT node. Commit 909777324588 ("KVM: arm/arm64: vgic-new: vgic_init:
implement kvm_vgic_hyp_init") changed the functions again, and now they require exactly one argument, a struct gic_kvm_info populated by the GIC driver. Unfortunately the comments regressed and state that a DT node is used instead. Change the function comments to reflect the current prototypes.
--- virt/kvm/arm/vgic/vgic-v2.c | 7 ++++--- virt/kvm/arm/vgic/vgic-v3.c | 7 ++++--- 2 files changed, 8 insertions(+), 6 deletions(-)
virt/kvm/arm/vgic/vgic-v2.c +4 -3
@@ -354,10 +354,11 @@ int vgic_v2_map_resources(struct kvm *kvm)DEFINE_STATIC_KEY_FALSE (vgic_v2_cpuif_trap);/** -* vgic_v2_probe - probe for a GICv2 compatible interrupt controller in DT -* @node: pointer to the DT node +* vgic_v2_probe - probe for a VGICv2 compatible interrupt controller +* @info: pointer to the GIC description * -* Returns 0 if a GICv2 has been found, returns an error code otherwise +* Returns 0 if the VGICv2 has been probed successfully, returns an error code +* otherwise */ int vgic_v2_probe (const struct gic_kvm_info *info) {
virt/kvm/arm/vgic/vgic-v3.c +4 -3
@@ -570,10 +570,11 @@ static int __init early_gicv4_enable(char *buf)early_param ("kvm-arm.vgic_v4_enable" , early_gicv4_enable);/** -* vgic_v3_probe - probe for a GICv3 compatible interrupt controller in DT -* @node: pointer to the DT node +* vgic_v3_probe - probe for a VGICv3 compatible interrupt controller +* @info: pointer to the GIC description * -* Returns 0 if a GICv3 has been found, returns an error code otherwise +* Returns 0 if the VGICv3 has been probed successfully, returns an error code +* otherwise */ int vgic_v3_probe (const struct gic_kvm_info *info) {
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 13/17 KVM: Call kvm_arch_vcpu_blocking early into the blocking sequence
In reply to Marc ZyngierWhen a vpcu is about to block by calling kvm_vcpu_block, we call back into the arch code to allow any form of synchronization that may be required at this point (SVN stops the AVIC, ARM synchronises the VMCR and enables GICv4 doorbells). But this synchronization comes in quite late, as we've potentially waited for halt_poll_ns to expire.
Instead, let's move kvm_arch_vcpu_blocking() to the beginning of kvm_vcpu_block(), which on ARM has several benefits:
- VMCR gets synchronised early, meaning that any interrupt delivered during the polling window will be evaluated with the correct guest PMR
- GICv4 doorbells are enabled, which means that any guest interrupt directly injected during that window will be immediately recognised
Tang Nianyao ran some tests on a GICv4 machine to evaluate such change, and reported up to a 10% improvement for netperf:
<quote>
netperf result:
D06 as server, intel 8180 server as client with change:
package 512 bytes - 5500 Mbits/s package 64 bytes - 760 Mbits/s without change:
package 512 bytes - 5000 Mbits/s package 64 bytes - 710 Mbits/s
</quote>
--- virt/kvm/kvm_main.c | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-)
virt/kvm/kvm_main.c +3 -4
@@ -2321,6 +2321,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)bool waited = false;u64 block_ns; +kvm_arch_vcpu_blocking (vcpu); + start = cur =ktime_get ();if (vcpu->halt_poll_ns && !kvm_arch_no_poll (vcpu)) {ktime_t stop =ktime_add_ns (ktime_get (), vcpu->halt_poll_ns);
@@ -2341,8 +2343,6 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu) }while (single_task_running () &&ktime_before (cur, stop)); } -kvm_arch_vcpu_blocking (vcpu); -for (;;) {prepare_to_swait_exclusive (&vcpu->wq, &wait,TASK_INTERRUPTIBLE );
@@ -2355,9 +2355,8 @@ void kvm_vcpu_block(struct kvm_vcpu *vcpu)finish_swait (&vcpu->wq, &wait); cur =ktime_get (); - -kvm_arch_vcpu_unblocking (vcpu); out: +kvm_arch_vcpu_unblocking (vcpu); block_ns =ktime_to_ns (cur) -ktime_to_ns (start);if (!vcpu_valid_wakeup (vcpu))
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 14/17 KVM: arm/arm64: vgic: Remove spurious semicolons
In reply to Marc ZyngierDetected by Coccinelle (and Will Deacon) usingscripts/coccinelle/misc/semicolon.cocci.
--- virt/kvm/arm/vgic/vgic-init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
virt/kvm/arm/vgic/vgic-init.c +1 -1
@@ -510,7 +510,7 @@ int kvm_vgic_hyp_init(void)break ;default : ret = -ENODEV ; - }; + }if (ret)return ret;
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 15/17 KVM: arm/arm64: vgic: Use a single IO device per redistributor
In reply to Marc ZyngierFrom: Eric Auger <[email protected]>
At the moment we use 2 IO devices per GICv3 redistributor: one one for the RD_base frame and one for the SGI_base frame.
Instead we can use a single IO device per redistributor (the 2 frames are contiguous). This saves slots on the KVM_MMIO_BUS which is currently limited to NR_IOBUS_DEVS (1000).
This change allows to instantiate up to 512 redistributors and may speed the guest boot with a large number of VCPUs.
--- include/kvm/arm_vgic.h | 1 - virt/kvm/arm/vgic/vgic-init.c | 1 - virt/kvm/arm/vgic/vgic-mmio-v3.c | 81 ++++++++++---------------------- 3 files changed, 24 insertions(+), 59 deletions(-)
include/kvm/arm_vgic.h +0 -1
@@ -314,7 +314,6 @@ struct vgic_cpu {* parts of the redistributor. */ struct vgic_io_device rd_iodev; -struct vgic_io_device sgi_iodev;struct vgic_redist_region *rdreg;/* Contains the attributes and gpa of the LPI pending tables. */
virt/kvm/arm/vgic/vgic-init.c +0 -1
@@ -193,7 +193,6 @@ int kvm_vgic_vcpu_init(struct kvm_vcpu *vcpu)int i; vgic_cpu->rd_iodev.base_addr =VGIC_ADDR_UNDEF ; - vgic_cpu->sgi_iodev.base_addr =VGIC_ADDR_UNDEF ;INIT_LIST_HEAD (&vgic_cpu->ap_list_head);raw_spin_lock_init (&vgic_cpu->ap_list_lock);
virt/kvm/arm/vgic/vgic-mmio-v3.c +24 -57
165 lines not on this page · raw message
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 16/17 arm64: KVM: Device mappings should be execute-never
In reply to Marc ZyngierFrom: James Morse <[email protected]>
Since commit 2f6ea23f63cca ("arm64: KVM: Avoid marking pages as XN in Stage-2 if CTR_EL0.DIC is set"), KVM has stopped marking normal memory as execute-never at stage2 when the system supports D->I Coherency at the PoU. This avoids KVM taking a trap when the page is first executed, in order to clean it to PoU.
The patch that added this change also wrapped PAGE_S2_DEVICE mappings up in this too. The upshot is, if your CPU caches support DIC ... you can execute devices.
Revert the PAGE_S2_DEVICE change so PTE_S2_XN is always used directly.
--- arch/arm64/include/asm/pgtable-prot.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
arch/arm64/include/asm/pgtable-prot.h +1 -1
@@ -77,7 +77,7 @@ })#define PAGE_S2 __pgprot(_PROT_DEFAULT | PAGE_S2_MEMATTR(NORMAL) | PTE_S2_RDONLY | PAGE_S2_XN) -#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT | PAGE_S2_MEMATTR(DEVICE_nGnRE) | PTE_S2_RDONLY | PAGE_S2_XN) +#define PAGE_S2_DEVICE __pgprot(_PROT_DEFAULT | PAGE_S2_MEMATTR(DEVICE_nGnRE) | PTE_S2_RDONLY | PTE_S2_XN)#define PAGE_NONE __pgprot(((_PAGE_DEFAULT) & ~PTE_VALID) | PTE_PROT_NONE | PTE_RDONLY | PTE_NG | PTE_PXN | PTE_UXN)#define PAGE_SHARED __pgprot(_PAGE_DEFAULT | PTE_USER | PTE_NG | PTE_PXN | PTE_UXN | PTE_WRITE)
Signature · 1 line
2.20.1
Marc Zyngier
maintainerPATCH 17/17 KVM: arm/arm64: vgic: Allow more than 256 vcpus for KVM_IRQ_LINE
In reply to Marc ZyngierWhile parts of the VGIC support a large number of vcpus (we bravely allow up to 512), other parts are more limited.
One of these limits is visible in the KVM_IRQ_LINE ioctl, which only allows 256 vcpus to be signalled when using the CPU or PPI types. Unfortunately, we've cornered ourselves badly by allocating all the bits in the irq field.
Since the irq_type subfield (8 bit wide) is currently only taking the values 0, 1 and 2 (and we have been careful not to allow anything else), let's reduce this field to only 4 bits, and allocate the remaining 4 bits to a vcpu2_index, which acts as a multiplier:
vcpu_id = 256 * vcpu2_index + vcpu_index
With that, and a new capability (KVM_CAP_ARM_IRQ_LINE_LAYOUT_2) allowing this to be discovered, it becomes possible to inject PPIs to up to 4096 vcpus. But please just don't.
Whilst we're there, add a clarification about the use of KVM_IRQ_LINE on arm, which is not completely conditionned by KVM_CAP_IRQCHIP.
--- Documentation/virt/kvm/api.txt | 12 ++++++++++-- arch/arm/include/uapi/asm/kvm.h | 4 +++- arch/arm64/include/uapi/asm/kvm.h | 4 +++- include/uapi/linux/kvm.h | 1 + virt/kvm/arm/arm.c | 2 ++ 5 files changed, 19 insertions(+), 4 deletions(-)
Documentation/virt/kvm/api.txt +10 -2
@@ -753,8 +753,8 @@ in-kernel irqchip (GIC), and for in-kernel irqchip can tell the GIC to use PPIs designated for specific cpus. The irq field is interpreted like this: - bits: | 31 ... 24 | 23 ... 16 | 15 ... 0 | - field: | irq_type | vcpu_index | irq_id | + bits: | 31 ... 28 | 27 ... 24 | 23 ... 16 | 15 ... 0 | + field: | vcpu2_index | irq_type | vcpu_index | irq_id | The irq_type field has the following values: - irq_type[0]: out-of-kernel GIC: irq_id 0 is IRQ, irq_id 1 is FIQ
@@ -766,6 +766,14 @@ The irq_type field has the following values: In both cases, level is used to assert/deassert the line. +When KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 is supported, the target vcpu is +identified as (256 * vcpu2_index + vcpu_index). Otherwise, vcpu2_index +must be zero. + +Note that on arm/arm64, the KVM_CAP_IRQCHIP capability only conditions +injection of interrupts for the in-kernel irqchip. KVM_IRQ_LINE can always +be used for a userspace interrupt controller. + struct kvm_irq_level { union { __u32 irq; /* GSI */
arch/arm/include/uapi/asm/kvm.h +3 -1
@@ -266,8 +266,10 @@ struct kvm_vcpu_events {#define KVM_DEV_ARM_ITS_CTRL_RESET 4/* KVM_IRQ_LINE irq field index values */ +#define KVM_ARM_IRQ_VCPU2_SHIFT 28 +#define KVM_ARM_IRQ_VCPU2_MASK 0xf#define KVM_ARM_IRQ_TYPE_SHIFT 24 -#define KVM_ARM_IRQ_TYPE_MASK 0xff +#define KVM_ARM_IRQ_TYPE_MASK 0xf#define KVM_ARM_IRQ_VCPU_SHIFT 16#define KVM_ARM_IRQ_VCPU_MASK 0xff#define KVM_ARM_IRQ_NUM_SHIFT 0
arch/arm64/include/uapi/asm/kvm.h +3 -1
@@ -325,8 +325,10 @@ struct kvm_vcpu_events {#define KVM_ARM_VCPU_TIMER_IRQ_PTIMER 1/* KVM_IRQ_LINE irq field index values */ +#define KVM_ARM_IRQ_VCPU2_SHIFT 28 +#define KVM_ARM_IRQ_VCPU2_MASK 0xf#define KVM_ARM_IRQ_TYPE_SHIFT 24 -#define KVM_ARM_IRQ_TYPE_MASK 0xff +#define KVM_ARM_IRQ_TYPE_MASK 0xf#define KVM_ARM_IRQ_VCPU_SHIFT 16#define KVM_ARM_IRQ_VCPU_MASK 0xff#define KVM_ARM_IRQ_NUM_SHIFT 0
include/uapi/linux/kvm.h +1 -0
@@ -996,6 +996,7 @@ struct kvm_ppc_resize_hpt {#define KVM_CAP_ARM_PTRAUTH_ADDRESS 171#define KVM_CAP_ARM_PTRAUTH_GENERIC 172#define KVM_CAP_PMU_EVENT_FILTER 173 +#define KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 174#ifdef KVM_CAP_IRQ_ROUTING
virt/kvm/arm/arm.c +2 -0
@@ -196,6 +196,7 @@ int kvm_vm_ioctl_check_extension(struct kvm *kvm, long ext)case KVM_CAP_MP_STATE :case KVM_CAP_IMMEDIATE_EXIT :case KVM_CAP_VCPU_EVENTS : +case KVM_CAP_ARM_IRQ_LINE_LAYOUT_2 : r =1 ;break ;case KVM_CAP_ARM_SET_DEVICE_ADDR :
@@ -888,6 +889,7 @@ int kvm_vm_ioctl_irq_line(struct kvm *kvm, struct kvm_irq_level *irq_level, irq_type = (irq >>KVM_ARM_IRQ_TYPE_SHIFT ) &KVM_ARM_IRQ_TYPE_MASK ; vcpu_idx = (irq >>KVM_ARM_IRQ_VCPU_SHIFT ) &KVM_ARM_IRQ_VCPU_MASK ; + vcpu_idx += ((irq >>KVM_ARM_IRQ_VCPU2_SHIFT ) &KVM_ARM_IRQ_VCPU2_MASK ) * (KVM_ARM_IRQ_VCPU_MASK +1 ); irq_num = (irq >>KVM_ARM_IRQ_NUM_SHIFT ) &KVM_ARM_IRQ_NUM_MASK ;trace_kvm_irq_line (irq_type, vcpu_idx, irq_num, irq_level->level);
Signature · 1 line
2.20.1
Paolo Bonzini
maintainerQuoted 1 line · Marc Zyngier
git://git.kernel.org/pub/scm/linux/kernel/git/kvmarm/kvmarm.gittags/kvmarm-5.4
Pulled, thanks.
Paolo