xen: arm: improve VMID allocation.
PATCH 1/1 xen: arm: improve VMID allocation.
The VMID field is 8 bits. Rather than allowing only up to 256 VMs per host reboot before things start "acting strange" instead maintain a simple bitmap of used VMIDs and allocate them statically to guests upon creation.
This limits us to 256 concurrent VMs which is a reasonable improvement. Eventually we will want a proper scheme to allocate VMIDs on context switch.
The existing code reserves VMID==0, IIRC I thought this was "hypervisor context". Re-reading the ARM ARM I'm not spotting the bit which made me think this. However I haven't changed this behaviour here.
---
NB: compile test only.
--- xen/arch/arm/p2m.c | 57 +++++++++++++++++++++++++++++++++++++++++---- xen/arch/arm/setup.c | 2 ++ xen/include/asm-arm/p2m.h | 3 +++ 3 files changed, 58 insertions(+), 4 deletions(-)
xen/arch/arm/p2m.c +53 -4
@@ -3,6 +3,7 @@#include <xen/lib.h> #include <xen/errno.h> #include <xen/domain_page.h> +#include <xen/bitops.h> #include <asm/flushtlb.h> #include <asm/gic.h>
@@ -306,6 +307,46 @@ int p2m_alloc_table(struct domain *d)return 0 ; } +#define MAX_VMID 256 + +/* VTTBR_EL2 VMID field is 8 bits. Using a bitmap here limits us to +* 256 concurrent domains. */ +DECLARE_BITMAP (vmid_mask,MAX_VMID ); + +void p2m_vmid_allocator_init (void ) +{ +/* VMID 0 is reserved */ +set_bit (0 , vmid_mask); +} + +/* p2m_alloc|free_vmid must both be called with the p2m->lock */ +static int p2m_alloc_vmid (struct domain *d) +{ +struct p2m_domain *p2m = &d->arch.p2m; + +int nr =find_first_zero_bit (vmid_mask,MAX_VMID ); + +ASSERT (nr >0 );/* VMID is reserved */ + +if ( nr ==MAX_VMID ) + { +printk ("p2m.c: dom%d: VMID pool exhausted\n" , d->domain_id); +return -EBUSY ; + } + +set_bit (nr, vmid_mask); + + p2m->vmid = nr; + +return 0 ; +} + +static void p2m_free_vmid (struct domain *d) +{ +struct p2m_domain *p2m = &d->arch.p2m; +clear_bit (p2m->vmid, vmid_mask); +} +void p2m_teardown (struct domain *d) {struct p2m_domain *p2m = &d->arch.p2m;
@@ -318,25 +359,33 @@ void p2m_teardown(struct domain *d) p2m->first_level =NULL ; +p2m_free_vmid (d); +spin_unlock (&p2m->lock); }int p2m_init (struct domain *d) {struct p2m_domain *p2m = &d->arch.p2m; +int rc =0 ;spin_lock_init (&p2m->lock);INIT_PAGE_LIST_HEAD (&p2m->pages); -/* XXX allocate properly */ -/* Zero is reserved */ - p2m->vmid = d->domain_id +1 ; +spin_lock (&p2m->lock); + + rc =p2m_alloc_vmid (d); +if ( rc !=0 ) + goto err; d->arch.vttbr =0 ; p2m->first_level =NULL ; -return 0 ; +err: +spin_unlock (&p2m->lock); + +return rc; }unsigned long gmfn_to_mfn (struct domain *d,unsigned long gpfn)
xen/arch/arm/setup.c +2 -0
@@ -544,6 +544,8 @@ void __init start_xen(unsigned long boot_phys_offset,setup_virt_paging (); +p2m_vmid_allocator_init (); +softirq_init ();tasklet_subsys_init ();
xen/include/asm-arm/p2m.h +3 -0
@@ -20,6 +20,9 @@ struct p2m_domain {uint8_t vmid; }; +/* Initialise vmid allocator */ +void p2m_vmid_allocator_init (void ); +/* Init the datastructures for later use by the p2m code */ int p2m_init (struct domain *d);
Signature · 1 line
1.7.10.4
Quoted 10 lines · Ian Campbell
The VMID field is 8 bits. Rather than allowing only up to 256 VMs per host reboot before things start "acting strange" instead maintain a simple bitmap of used VMIDs and allocate them statically to guests upon creation.
This limits us to 256 concurrent VMs which is a reasonable improvement. Eventually we will want a proper scheme to allocate VMIDs on context switch.
The existing code reserves VMID==0, IIRC I thought this was "hypervisor context". Re-reading the ARM ARM I'm not spotting the bit which made me think this. However I haven't changed this behaviour here.
The ARM ARM only says the VMID==0 is used during reset, so I think we can assume that ID is reserved.
Quoted 24 lines
Signed-off-by Ian Campbell <[email protected]>---
NB: compile test only.
--- xen/arch/arm/p2m.c | 57 +++++++++++++++++++++++++++++++++++++++++---- xen/arch/arm/setup.c | 2 ++ xen/include/asm-arm/p2m.h | 3 +++ 3 files changed, 58 insertions(+), 4 deletions(-)xen/arch/arm/p2m.c +1
-0@@ -3,6 +3,7 @@#include <xen/lib.h> #include <xen/errno.h> #include <xen/domain_page.h> +#include <xen/bitops.h> #include <asm/flushtlb.h> #include <asm/gic.h>
xen/arch/arm/p2m.c int p2m_alloc_table(struct domain *d) · line 314
Quoted 8 lines
xen/arch/arm/p2m.c +5
-0@@ -306,6 +307,46 @@ int p2m_alloc_table(struct domain *d)return 0 ; } +#define MAX_VMID 256 + +/* VTTBR_EL2 VMID field is 8 bits. Using a bitmap here limits us to +* 256 concurrent domains. */ +DECLARE_BITMAP (vmid_mask,MAX_VMID );
static DECLARE_...
xen/arch/arm/p2m.c int p2m_alloc_table(struct domain *d) · line 322
Quoted 8 lines
xen/arch/arm/p2m.c +8
-0+ +void p2m_vmid_allocator_init (void ) +{ +/* VMID 0 is reserved */ +set_bit (0 , vmid_mask); +} + +/* p2m_alloc|free_vmid must both be called with the p2m->lock */
Taking p2m->lock is not enough, this function can be called concurrently if the 2 domains are created at the same time.
In this case, it's possible to have the same vmid for 2 domains. How about a global lock?
xen/arch/arm/p2m.c int p2m_alloc_table(struct domain *d) · line 333
Quoted 11 lines
xen/arch/arm/p2m.c +11
-0+static int p2m_alloc_vmid (struct domain *d) +{ +struct p2m_domain *p2m = &d->arch.p2m; + +int nr =find_first_zero_bit (vmid_mask,MAX_VMID ); + +ASSERT (nr >0 );/* VMID is reserved */ + +if ( nr ==MAX_VMID ) + { +printk ("p2m.c: dom%d: VMID pool exhausted\n" , d->domain_id);
printk(XENLOG_ERR...?
Quoted 20 lines
xen/arch/arm/p2m.c +16
-0+return -EBUSY ; + } + +set_bit (nr, vmid_mask); + + p2m->vmid = nr; + +return 0 ; +} + +static void p2m_free_vmid (struct domain *d) +{ +struct p2m_domain *p2m = &d->arch.p2m; +clear_bit (p2m->vmid, vmid_mask); +} +void p2m_teardown (struct domain *d) {struct p2m_domain *p2m = &d->arch.p2m;
xen/arch/arm/p2m.c void p2m_teardown(struct domain *d) · line 363
Quoted 5 lines
xen/arch/arm/p2m.c +2
-0@@ -318,25 +359,33 @@ void p2m_teardown(struct domain *d) p2m->first_level =NULL ; +p2m_free_vmid (d); +
p2m_teardown is called when the domain is destroyed. If the vmid was not correctly set (for instance because VMID pool exhausted), we will clear the wrong bit (here 0).
At the next domain creation, we will assert because nr = 0.
Quoted 60 lines
xen/arch/arm/p2m.c +10
-4spin_unlock (&p2m->lock); }int p2m_init (struct domain *d) {struct p2m_domain *p2m = &d->arch.p2m; +int rc =0 ;spin_lock_init (&p2m->lock);INIT_PAGE_LIST_HEAD (&p2m->pages); -/* XXX allocate properly */ -/* Zero is reserved */ - p2m->vmid = d->domain_id +1 ; +spin_lock (&p2m->lock); + + rc =p2m_alloc_vmid (d); +if ( rc !=0 ) + goto err; d->arch.vttbr =0 ; p2m->first_level =NULL ; -return 0 ; +err: +spin_unlock (&p2m->lock); + +return rc; }unsigned long gmfn_to_mfn (struct domain *d,unsigned long gpfn)xen/arch/arm/setup.c +2
-0@@ -544,6 +544,8 @@ void __init start_xen(unsigned long boot_phys_offset,setup_virt_paging (); +p2m_vmid_allocator_init (); +softirq_init ();tasklet_subsys_init ();xen/include/asm-arm/p2m.h +3
-0@@ -20,6 +20,9 @@ struct p2m_domain {uint8_t vmid; }; +/* Initialise vmid allocator */ +void p2m_vmid_allocator_init (void ); +/* Init the datastructures for later use by the p2m code */ int p2m_init (struct domain *d);
Signature · 1 line
Julien Grall
Quoted 5 lines · Julien Grall
Quoted 5 lines
xen/arch/arm/p2m.c +5
-0+ +void p2m_vmid_allocator_init (void ) +{ +/* VMID 0 is reserved */ +set_bit (0 , vmid_mask);
Quoted 8 lines
xen/arch/arm/p2m.c int p2m_alloc_table(struct domain *d) · line 322
Quoted 3 lines
xen/arch/arm/p2m.c +3
-0+} + +/* p2m_alloc|free_vmid must both be called with the p2m->lock */ Taking
p2m->lockis not enough, this function can be called concurrently if the 2 domains are created at the same time.
In this case, it's possible to have the same vmid for 2 domains. How about a global lock?
Brain fart, I forgot this lock was per p2m.
The domain lock is probably OK to use here, in fact we may already hold it. I'll fix this.
Quoted 11 lines
xen/arch/arm/p2m.c void p2m_teardown(struct domain *d) · line 363
Quoted 6 lines
xen/arch/arm/p2m.c +2
-0@@ -318,25 +359,33 @@ void p2m_teardown(struct domain *d) p2m->first_level =NULL ; +p2m_free_vmid (d); +
p2m_teardownis called when the domain is destroyed. If the vmid was not correctly set (for instance because VMID pool exhausted), we will clear the wrong bit (here 0).
At the next domain creation, we will assert because nr = 0.
Oops! Will fix.
Ian.