Implement 3-level event channel in Xen
PATCH v2 1/1 Implement 3-level event channel in Xen
Changes from V1:
* move all evtchn related macros / struct definitions to event.h
* only allow 3-level evtchn for Dom0 and driver domains
* add evtchn_l3 flag in libxl
PATCH v2 1/15 Dynamically allocate d->evtchn
As we move to N level evtchn we need bigger d->evtchn, as a result this will bloat struct domain. So move this array out of struct domain and allocate a dedicated page for it.
--- xen/common/event_channel.c | 17 +++++++++++++++-- xen/include/xen/sched.h | 2 +- 2 files changed, 16 insertions(+), 3 deletions(-)
xen/common/event_channel.c +15 -2
@@ -1172,15 +1172,26 @@ void notify_via_xen_event_channel(struct domain *ld, int lport)int evtchn_init (struct domain *d) { +BUILD_BUG_ON (sizeof (struct evtchn *) *NR_EVTCHN_BUCKETS >PAGE_SIZE ); + d->evtchn =alloc_xenheap_page (); + +if ( d->evtchn ==NULL ) +return -ENOMEM ; +clear_page (d->evtchn); +spin_lock_init (&d->event_lock); -if (get_free_port (d) !=0 ) +if (get_free_port (d) !=0 ) { +free_xenheap_page (d->evtchn);return -EINVAL ; + }evtchn_from_port (d,0 )->state =ECS_RESERVED ;#if MAX_VIRT_CPUS >BITS_PER_LONG d->poll_mask = xmalloc_array(unsigned long, BITS_TO_LONGS(MAX_VIRT_CPUS)); -if ( !d->poll_mask ) +if ( !d->poll_mask ) { +free_xenheap_page (d->evtchn);return -ENOMEM ; + }bitmap_zero (d->poll_mask,MAX_VIRT_CPUS );#endif
@@ -1214,6 +1225,8 @@ void evtchn_destroy(struct domain *d)spin_unlock (&d->event_lock);clear_global_virq_handlers (d); + +free_xenheap_page (d->evtchn); }
xen/include/xen/sched.h +1 -1
@@ -260,7 +260,7 @@ struct domainspinlock_t rangesets_lock;/* Event channel information. */ -struct evtchn *evtchn[NR_EVTCHN_BUCKETS ]; +struct evtchn **evtchn;spinlock_t event_lock;struct grant_table *grant_table;
Signature · 1 line
1.7.10.4
PATCH v2 2/15 Move event channel macros / struct definition to proper place
In reply to Wei LiuAfter remove reference to NR_EVTCHN_BUCKETS in struct domain, we can move those macros / struct definitions to event.h.
--- xen/include/xen/event.h | 46 ++++++++++++++++++++++++++++++++++++++++++++++ xen/include/xen/sched.h | 45 --------------------------------------------- 2 files changed, 46 insertions(+), 45 deletions(-)
xen/include/xen/event.h +46 -0
@@ -15,6 +15,52 @@#include <asm/bitops.h> #include <asm/event.h> +#ifndef CONFIG_COMPAT +#define BITS_PER_EVTCHN_WORD (d) BITS_PER_LONG +#else +#define BITS_PER_EVTCHN_WORD (d) (has_32bit_shinfo(d) ? 32 : BITS_PER_LONG) +#endif +#define MAX_EVTCHNS (d) (BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d)) + +#define EVTCHNS_PER_BUCKET 128 +#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET) + +struct evtchn +{ +#define ECS_FREE 0/* Channel is available for use. */ +#define ECS_RESERVED 1/* Channel is reserved. */ +#define ECS_UNBOUND 2/* Channel is waiting to bind to a remote domain. */ +#define ECS_INTERDOMAIN 3/* Channel is bound to another domain. */ +#define ECS_PIRQ 4/* Channel is bound to a physical IRQ line. */ +#define ECS_VIRQ 5/* Channel is bound to a virtual IRQ line. */ +#define ECS_IPI 6/* Channel is bound to a virtual IPI line. */ +u8 state;/* ECS_* */ +u8 xen_consumer;/* Consumer in Xen, if any? (0 = send to guest) */ +u16 notify_vcpu_id;/* VCPU for local delivery notification */ +union { +struct { +domid_t remote_domid; + } unbound;/* state == ECS_UNBOUND */ +struct { +u16 remote_port; +struct domain *remote_dom; + } interdomain;/* state == ECS_INTERDOMAIN */ +struct { +u16 irq; +u16 next_port; +u16 prev_port; + } pirq;/* state == ECS_PIRQ */ +u16 virq;/* state == ECS_VIRQ */ + } u; +#ifdef FLASK_ENABLE +void *ssid; +#endif +}; + +int evtchn_init (struct domain *d);/* from domain_create */ +void evtchn_destroy (struct domain *d);/* from domain_kill */ +void evtchn_destroy_final (struct domain *d);/* from complete_domain_destroy */ +/* * send_guest_vcpu_virq: Notify guest via a per-VCPU VIRQ. * @v: VCPU to which virtual IRQ should be sent
xen/include/xen/sched.h +0 -45
51 lines not on this page · raw message
Signature · 1 line
1.7.10.4
PATCH v2 3/15 Add evtchn_level in struct domain
In reply to Wei LiuThis field is manipulated by hypervisor only, so if anything goes wrong it is a bug.
The default event channel is 2, which has two level lookup structure: a selector in struct vcpu and a shared bitmap in shared info.
The up coming 3-level event channel utilizes three level lookup structure: a top level selector and second level selector for every vcpu, and shared bitmap.
When constructing a domain, it starts with 2-level event channel, which is guaranteed to be supported by the hypervisor. If a domain wants to use N (N>=3) level event channel, it must explicitly issue a hypercall to setup N-level event channel.
--- xen/common/event_channel.c | 1 + xen/include/xen/event.h | 16 +++++++++++++++- xen/include/xen/sched.h | 1 + 3 files changed, 17 insertions(+), 1 deletion(-)
xen/common/event_channel.c +1 -0
@@ -1180,6 +1180,7 @@ int evtchn_init(struct domain *d)clear_page (d->evtchn);spin_lock_init (&d->event_lock); + d->evtchn_level =EVTCHN_DEFAULT_LEVEL ;if (get_free_port (d) !=0 ) {free_xenheap_page (d->evtchn);return -EINVAL ;
xen/include/xen/event.h +15 -1
@@ -20,7 +20,21 @@#else #define BITS_PER_EVTCHN_WORD (d) (has_32bit_shinfo(d) ? 32 : BITS_PER_LONG)#endif -#define MAX_EVTCHNS (d) (BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d)) +#define EVTCHN_2_LEVEL 2 +#define EVTCHN_3_LEVEL 3 +#define EVTCHN_DEFAULT_LEVEL EVTCHN_2_LEVEL +#define MAX_EVTCHNS_L2 (d) (BITS_PER_EVTCHN_WORD(d) * BITS_PER_EVTCHN_WORD(d)) +#define MAX_EVTCHNS_L3 (d) (MAX_EVTCHNS_L2(d) * BITS_PER_EVTCHN_WORD(d)) +#define MAX_EVTCHNS (d) ({ int __v = 0; \ + switch ( d->evtchn_level ) { \ + case EVTCHN_2_LEVEL: \ + __v = MAX_EVTCHNS_L2(d); break; \ + case EVTCHN_3_LEVEL: \ + __v = MAX_EVTCHNS_L3(d); break; \ + default: \ + BUG(); \ + }; \ + __v;})#define EVTCHNS_PER_BUCKET 128#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET)
xen/include/xen/sched.h +1 -0
@@ -217,6 +217,7 @@ struct domain/* Event channel information. */ struct evtchn **evtchn;spinlock_t event_lock; +unsigned int evtchn_level;struct grant_table *grant_table;
Signature · 1 line
1.7.10.4
PATCH v2 4/15 Bump EVTCHNS_PER_BUCKET to 512
In reply to Wei LiuFor 64 bit build and 3-level event channel and the original value of EVTCHNS_PER_BUCKET (128), the space needed to accommodate d->evtchn would be 4 pages (PAGE_SIZE = 4096). Given that not every domain needs 3-level event channel, this leads to waste of memory. Also we've restricted d->evtchn to one page, if we move to 3-level event channel, Xen cannot build.
Having EVTCHN_PER_BUCKETS to be 512 can occupy exact one page.
--- xen/include/xen/event.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
xen/include/xen/event.h +1 -1
@@ -36,7 +36,7 @@ }; \ __v;}) -#define EVTCHNS_PER_BUCKET 128 +#define EVTCHNS_PER_BUCKET 512#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET)struct evtchn
Signature · 1 line
1.7.10.4
PATCH v2 5/15 Add evtchn_is_{pending, masked} and evtchn_clear_pending
In reply to Wei LiuSome code paths access the arrays in shared info directly. This only works with 2-level event channel.
Add functions to abstract away implementation details.
--- xen/arch/x86/irq.c | 7 +++---- xen/common/event_channel.c | 22 +++++++++++++++++++--- xen/common/keyhandler.c | 6 ++---- xen/common/schedule.c | 2 +- xen/include/xen/event.h | 6 ++++++ 5 files changed, 31 insertions(+), 12 deletions(-)
xen/arch/x86/irq.c +3 -4
@@ -1452,7 +1452,7 @@ int pirq_guest_unmask(struct domain *d) { pirq = pirqs[i]->pirq;if ( pirqs[i]->masked && - !test_bit (pirqs[i]->evtchn, &shared_info (d, evtchn_mask)) ) + !evtchn_is_masked (d, pirqs[i]->evtchn) )pirq_guest_eoi (pirqs[i]); } }while ( ++pirq < d->nr_pirqs && n ==ARRAY_SIZE (pirqs) );
@@ -2093,13 +2093,12 @@ static void dump_irqs(unsigned char key) info =pirq_info (d, pirq);printk ("%u:%3d(%c%c%c%c)" , d->domain_id, pirq, - (test_bit (info->evtchn, - &shared_info (d, evtchn_pending)) ? + (evtchn_is_pending (d, info->evtchn) ?'P' :'-' ), (test_bit (info->evtchn /BITS_PER_EVTCHN_WORD (d), &vcpu_info (d->vcpu[0 ], evtchn_pending_sel)) ?'S' :'-' ), - (test_bit (info->evtchn, &shared_info (d, evtchn_mask)) ? + (evtchn_is_masked (d, info->evtchn) ?'M' :'-' ), (info->masked ?'M' :'-' ));if ( i != action->nr_guests )
xen/common/event_channel.c +19 -3
@@ -95,6 +95,7 @@ static uint8_t get_xen_consumer(xen_event_channel_notification_t fn)#define xen_notification_fn (e) (xen_consumers[(e)->xen_consumer-1])static void evtchn_set_pending (struct vcpu *v,int port); +static void evtchn_clear_pending (struct domain *d,int port);static int virq_is_global (uint32_t virq) {
@@ -156,6 +157,16 @@ static int get_free_port(struct domain *d)return port; } +int evtchn_is_pending (struct domain *d,int port) +{ +return test_bit (port, &shared_info (d, evtchn_pending)); +} + +int evtchn_is_masked (struct domain *d,int port) +{ +return test_bit (port, &shared_info (d, evtchn_mask)); +} +static long evtchn_alloc_unbound (evtchn_alloc_unbound_t *alloc) {
@@ -529,7 +540,7 @@ static long __evtchn_close(struct domain *d1, int port1) }/* Clear pending event to avoid unexpected behavior on re-bind. */ -clear_bit (port1, &shared_info (d1, evtchn_pending)); +evtchn_clear_pending (d1, port1);/* Reset binding to vcpu0 when the channel is freed. */ chn1->state =ECS_FREE ;
@@ -653,6 +664,11 @@ static void evtchn_set_pending(struct vcpu *v, int port) } } +static void evtchn_clear_pending (struct domain *d,int port) +{ +clear_bit (port, &shared_info (d, evtchn_pending)); +} +int guest_enabled_event (struct vcpu *v,uint32_t virq) {return ((v !=NULL ) && (v->virq_to_evtchn[virq] !=0 ));
@@ -1283,8 +1299,8 @@ static void domain_dump_evtchn_info(struct domain *d)printk (" %4u [%d/%d]: s=%d n=%d x=%d" , port, - !!test_bit (port, &shared_info (d, evtchn_pending)), - !!test_bit (port, &shared_info (d, evtchn_mask)), + !!evtchn_is_pending (d, port), + !!evtchn_is_masked (d, port), chn->state, chn->notify_vcpu_id, chn->xen_consumer); switch ( chn->state )
xen/common/keyhandler.c +2 -4
@@ -301,10 +301,8 @@ static void dump_domains(unsigned char key)printk ("Notifying guest %d:%d (virq %d, port %d, stat %d/%d/%d)\n" , d->domain_id, v->vcpu_id,VIRQ_DEBUG , v->virq_to_evtchn[VIRQ_DEBUG ], -test_bit (v->virq_to_evtchn[VIRQ_DEBUG ], - &shared_info (d, evtchn_pending)), -test_bit (v->virq_to_evtchn[VIRQ_DEBUG ], - &shared_info (d, evtchn_mask)), +evtchn_is_pending (d, v->virq_to_evtchn[VIRQ_DEBUG ]), +evtchn_is_masked (d, v->virq_to_evtchn[VIRQ_DEBUG ]),test_bit (v->virq_to_evtchn[VIRQ_DEBUG ] /BITS_PER_EVTCHN_WORD (d), &vcpu_info (v, evtchn_pending_sel)));
xen/common/schedule.c +1 -1
@@ -693,7 +693,7 @@ static long do_poll(struct sched_poll *sched_poll) goto out; rc =0 ; -if (test_bit (port, &shared_info (d, evtchn_pending)) ) +if (evtchn_is_pending (d, port) ) goto out; }
xen/include/xen/event.h +6 -0
@@ -114,6 +114,12 @@ int evtchn_unmask(unsigned int port);/* Move all PIRQs after a vCPU was moved to another pCPU. */ void evtchn_move_pirqs (struct vcpu *v); +/* Tell a given event-channel port is pending or not */ +int evtchn_is_pending (struct domain *d,int port); + +/* Tell a given event-channel port is masked or not */ +int evtchn_is_masked (struct domain *d,int port); +/* Allocate/free a Xen-attached event channel port. */ typedef void (*xen_event_channel_notification_t )(struct vcpu *v,unsigned int port);
Signature · 1 line
1.7.10.4
PATCH v2 6/15 Introduce some macros for event channels
In reply to Wei LiuFor N-level event channels, the shared bitmaps in the hypervisor are by design not guaranteed to be contigious.
These macros are used to calculate page number / offset within a page of a given event channel.
--- xen/include/asm-arm/types.h | 7 +++++-- xen/include/asm-x86/config.h | 4 +++- xen/include/xen/event.h | 13 +++++++++++++ 3 files changed, 21 insertions(+), 3 deletions(-)
xen/include/asm-arm/types.h +5 -2
@@ -41,10 +41,13 @@ typedef char bool_t;#define test_and_clear_bool (b) xchg(&(b), 0)#endif /* __ASSEMBLY__ */ +#define BYTE_BITORDER 3 +#define BITS_PER_BYTE (1 << BYTE_BITORDER) -#define BITS_PER_LONG 32 -#define BYTES_PER_LONG 4 +#define BITS_PER_LONG (1 << LONG_BITORDER)#define LONG_BYTEORDER 2 +#define LONG_BITORDER (LONG_BYTEORDER + BYTE_BITORDER) +#define BYTES_PER_LONG (1 << LONG_BYTEORDER)#endif /* __ARM_TYPES_H__ */ /*
xen/include/asm-x86/config.h +3 -1
@@ -8,11 +8,13 @@#define __X86_CONFIG_H__#define LONG_BYTEORDER 3 +#define BYTE_BITORDER 3 +#define LONG_BITORDER (BYTE_BITORDER + LONG_BYTEORDER)#define CONFIG_PAGING_LEVELS 4#define BYTES_PER_LONG (1 << LONG_BYTEORDER)#define BITS_PER_LONG (BYTES_PER_LONG << 3) -#define BITS_PER_BYTE 8 +#define BITS_PER_BYTE (1 << BYTE_BITORDER)#define CONFIG_X86 1#define CONFIG_X86_HT 1
xen/include/xen/event.h +13 -0
@@ -39,6 +39,19 @@#define EVTCHNS_PER_BUCKET 512#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET) +/* N.B. EVTCHNS_PER_PAGE is always powers of 2, use shifts to optimize */ +#define EVTCHNS_SHIFT (PAGE_SHIFT+BYTE_BITORDER) +#define EVTCHNS_PER_PAGE (_AC(1,L) << EVTCHNS_SHIFT) +#define EVTCHN_MASK (~(EVTCHNS_PER_PAGE-1)) +#define EVTCHN_PAGE_NO (chn) ((chn) >> EVTCHNS_SHIFT) +#define EVTCHN_OFFSET_IN_PAGE (chn) ((chn) & ~EVTCHN_MASK) + +#ifndef CONFIG_COMPAT +#define EVTCHN_WORD_BITORDER (d) LONG_BITORDER +#else +#define EVTCHN_WORD_BITORDER (d) (has_32bit_shinfo(d) ? 5 : LONG_BITORDER) +#endif +struct evtchn {#define ECS_FREE 0/* Channel is available for use. */
Signature · 1 line
1.7.10.4
PATCH v2 7/15 Update Xen public header
In reply to Wei Liu--- xen/include/public/xen.h | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-)
xen/include/public/xen.h +8 -1
@@ -554,9 +554,16 @@ DEFINE_XEN_GUEST_HANDLE(multicall_entry_t);/* * Event channel endpoints per domain: +* 2-level: * 1024 if a long is 32 bits; 4096 if a long is 64 bits. +* 3-level: +* 32k if a long is 32 bits; 256k if a long is 64 bits. */ -#define NR_EVENT_CHANNELS (sizeof(unsigned long) * sizeof(unsigned long) * 64) +#define NR_EVENT_CHANNELS_L2 (sizeof(unsigned long) * sizeof(unsigned long) * 64) +#define NR_EVENT_CHANNELS_L3 (NR_EVENT_CHANNELS_L2 * 64) +#if !defined(__XEN__) && !defined(__XEN_TOOLS__) +#define NR_EVENT_CHANNELS NR_EVENT_CHANNELS_L2/* for compatibility */ +#endif struct vcpu_time_info {/*
Signature · 1 line
1.7.10.4
PATCH v2 8/15 Define N-level event channel registration interface
In reply to Wei Liu--- xen/include/public/event_channel.h | 33 +++++++++++++++++++++++++++++++++ 1 file changed, 33 insertions(+)
xen/include/public/event_channel.h +33 -0
@@ -71,6 +71,7 @@#define EVTCHNOP_bind_vcpu 8#define EVTCHNOP_unmask 9#define EVTCHNOP_reset 10 +#define EVTCHNOP_register_nlevel 11/* ` } */ typedef uint32_t evtchn_port_t ;
@@ -258,6 +259,38 @@ struct evtchn_reset {typedef struct evtchn_reset evtchn_reset_t ;/* +* EVTCHNOP_register_nlevel: Register N-level event channel +* NOTES: +* 1. Currently only 3-level is supported. +* 2. Should fall back to 2-level if this call fails. +*/ +/* 64 bit guests need 8 pages for evtchn_pending and evtchn_mask for +* 256k event channels while 32 bit ones only need 1 page for 32k +* event channels. */ +#define EVTCHN_MAX_L3_PAGES 8 +struct evtchn_register_3level { +/* IN parameters. */ +uint32_t nr_pages;/* for evtchn_{pending,mask} */ +uint32_t nr_vcpus;/* for l2sel_{mfns,offsets} */ +XEN_GUEST_HANDLE (xen_pfn_t) evtchn_pending; +XEN_GUEST_HANDLE (xen_pfn_t) evtchn_mask; +XEN_GUEST_HANDLE (xen_pfn_t) l2sel_mfns; +XEN_GUEST_HANDLE (xen_pfn_t) l2sel_offsets; +}; +typedef struct evtchn_register_3level evtchn_register_3level_t ; +DEFINE_XEN_GUEST_HANDLE (evtchn_register_3level_t); + +struct evtchn_register_nlevel { +/* IN parameters. */ +uint32_t level; +union { +evtchn_register_3level_t l3; + } u; +}; +typedef struct evtchn_register_nlevel evtchn_register_nlevel_t ; +DEFINE_XEN_GUEST_HANDLE (evtchn_register_nlevel_t); + +/* * ` enum neg_errnoval * ` HYPERVISOR_event_channel_op_compat(struct evtchn_op *op) * `
Signature · 1 line
1.7.10.4
PATCH v2 9/15 Add control structures for 3-level event channel
In reply to Wei LiuThe references to shared bitmap pending / mask are embedded in struct domain. And pointer to the second level selector is embedded in struct vcpu.
--- xen/include/xen/sched.h | 6 ++++++ 1 file changed, 6 insertions(+)
xen/include/xen/sched.h +6 -0
@@ -24,6 +24,7 @@#include <public/sysctl.h> #include <public/vcpu.h> #include <public/mem_event.h> +#include <public/event_channel.h> #ifdef CONFIG_COMPAT #include <compat/vcpu.h>
@@ -57,6 +58,9 @@ struct vcpustruct domain *domain; +/* For 3-level event channels */ +unsigned long *evtchn_pending_sel_l2; +struct vcpu *next_in_list;s_time_t periodic_period;
@@ -218,6 +222,8 @@ struct domainstruct evtchn **evtchn;spinlock_t event_lock;unsigned int evtchn_level; +unsigned long *evtchn_pending[EVTCHN_MAX_L3_PAGES ]; +unsigned long *evtchn_mask[EVTCHN_MAX_L3_PAGES ];struct grant_table *grant_table;
Signature · 1 line
1.7.10.4
PATCH v2 10/15 Make NR_EVTCHN_BUCKETS 3-level ready
In reply to Wei Liu--- xen/include/xen/event.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
xen/include/xen/event.h +1 -1
@@ -37,7 +37,7 @@ __v;})#define EVTCHNS_PER_BUCKET 512 -#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS / EVTCHNS_PER_BUCKET) +#define NR_EVTCHN_BUCKETS (NR_EVENT_CHANNELS_L3 / EVTCHNS_PER_BUCKET)/* N.B. EVTCHNS_PER_PAGE is always powers of 2, use shifts to optimize */ #define EVTCHNS_SHIFT (PAGE_SHIFT+BYTE_BITORDER)
Signature · 1 line
1.7.10.4
PATCH v2 11/15 Genneralized event channel operations
In reply to Wei LiuUse pointer in struct domain to reference evtchn_pending and evtchn_mask bitmaps.
When building a domain, the default operation set is 2-level operation set.
--- xen/arch/arm/domain.c | 1 + xen/arch/x86/domain.c | 1 + xen/common/event_channel.c | 65 ++++++++++++++++++++++++++++++++++++-------- xen/include/xen/event.h | 3 ++ 4 files changed, 59 insertions(+), 11 deletions(-)
xen/arch/arm/domain.c +1 -0
@@ -417,6 +417,7 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags) goto fail;clear_page (d->shared_info); +evtchn_set_default_bitmap (d);share_xen_page_with_guest (virt_to_page (d->shared_info), d, XENSHARE_writable);
xen/arch/x86/domain.c +1 -0
@@ -580,6 +580,7 @@ int arch_domain_create(struct domain *d, unsigned int domcr_flags) goto fail;clear_page (d->shared_info); +evtchn_set_default_bitmap (d);share_xen_page_with_guest (virt_to_page (d->shared_info), d, XENSHARE_writable);
xen/common/event_channel.c +54 -11
130 lines not on this page · raw message
xen/include/xen/event.h +3 -0
@@ -148,6 +148,9 @@ int guest_enabled_event(struct vcpu *v, uint32_t virq);/* Notify remote end of a Xen-attached event channel.*/ void notify_via_xen_event_channel (struct domain *ld,int lport); +/* This is called after domain's shared info page is setup */ +void evtchn_set_default_bitmap (struct domain *d); +/* Internal event channel object accessors */ #define bucket_from_port (d,p) \ ((d)->evtchn[(p)/EVTCHNS_PER_BUCKET])
Signature · 1 line
1.7.10.4
PATCH v2 12/15 Infrastructure for manipulating 3-level event channel pages
In reply to Wei LiuNOTE: the registration call is always failed because other part of the code is not yet completed.
--- xen/common/event_channel.c | 280 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 280 insertions(+)
xen/common/event_channel.c +280 -0
@@ -26,6 +26,7 @@#include <xen/compat.h> #include <xen/guest_access.h> #include <xen/keyhandler.h> +#include <xen/paging.h> #include <asm/current.h> #include <public/xen.h>
@@ -1024,6 +1025,260 @@ out: } +static long __map_l3_arrays (struct domain *d,xen_pfn_t *pending, +xen_pfn_t *mask,int nr_pages) +{ +int rc; +void *mapping; +struct page_info *pginfo; +unsigned long gfn; +int pending_count =0 , mask_count =0 ; + +#define __MAP (src, dst, cnt) \ + for ( (cnt) = 0; (cnt) < nr_pages; (cnt)++ ) \ + { \ + rc = -EINVAL; \ + gfn = (src)[(cnt)]; \ + pginfo = get_page_from_gfn(d, gfn, NULL, P2M_ALLOC); \ + if ( !pginfo ) \ + goto err; \ + if ( !get_page_type(pginfo, PGT_writable_page) ) \ + { \ + put_page(pginfo); \ + goto err; \ + } \ + mapping = __map_domain_page_global(pginfo); \ + if ( !mapping ) \ + { \ + put_page_and_type(pginfo); \ + rc = -ENOMEM; \ + goto err; \ + } \ + (dst)[(cnt)] = mapping; \ + } + + __MAP(pending, d->evtchn_pending, pending_count) + __MAP(mask, d->evtchn_mask, mask_count) +#undef __MAP + + rc =0 ; + + err: +return rc; +} + +static void __unmap_l3_arrays (struct domain *d) +{ +int i; +unsigned long mfn; + +for ( i =0 ; i <EVTCHN_MAX_L3_PAGES ; i++ ) + { +if ( d->evtchn_pending[i] !=0 ) + { + mfn =domain_page_map_to_mfn (d->evtchn_pending[i]); +unmap_domain_page_global (d->evtchn_pending[i]); +put_page_and_type (mfn_to_page (mfn)); + d->evtchn_pending[i] =0 ; + } +if ( d->evtchn_mask[i] !=0 ) + { + mfn =domain_page_map_to_mfn (d->evtchn_mask[i]); +unmap_domain_page_global (d->evtchn_mask[i]); +put_page_and_type (mfn_to_page (mfn)); + d->evtchn_mask[i] =0 ; + } + } +} + +static long __map_l2_selector (struct vcpu *v,unsigned long gfn, +unsigned long off) +{ +void *mapping; +int rc; +struct page_info *page; +struct domain *d = v->domain; + + rc = -EINVAL ;/* common errno for following operations */ + +/* Sanity check: L2 selector has maximum size of sizeof(unsigned +* long) * 8, this size is equal to the size of shared bitmap +* array of 2-level event channel. */ +if ( off +sizeof (unsigned long ) *8 >=PAGE_SIZE ) + goto out; + + page =get_page_from_gfn (d, gfn,NULL ,P2M_ALLOC ); +if ( !page ) + goto out; + +if ( !get_page_type (page, PGT_writable_page) ) + { +put_page (page); + goto out; + } + +/* Use global mapping here, because these selectors will also be +* accessed by other domains when setting pending for inter-domain +* event channels. +*/ + mapping =__map_domain_page_global (page); + +if ( mapping ==NULL ) + { +put_page_and_type (page); + rc = -ENOMEM ; + goto out; + } + + v->evtchn_pending_sel_l2 = mapping + off; + rc =0 ; + + out: +return rc; +} + +static void __unmap_l2_selector (struct vcpu *v) +{ +unsigned long mfn; + +if ( v->evtchn_pending_sel_l2 ) + { + mfn =domain_page_map_to_mfn (v->evtchn_pending_sel_l2); +unmap_domain_page_global (v->evtchn_pending_sel_l2); +put_page_and_type (mfn_to_page (mfn)); + v->evtchn_pending_sel_l2 =NULL ; + } +} + +static void __evtchn_unmap_all_3level (struct domain *d) +{ +struct vcpu *v; + for_each_vcpu ( d, v ) +__unmap_l2_selector (v); +__unmap_l3_arrays (d); +} + +static void __evtchn_setup_bitmap_l3 (struct domain *d) +{ +struct vcpu *v; + +/* Easy way to setup 3-level bitmap, just move existing selector +* to next level then copy pending array and mask array */ + for_each_vcpu ( d, v ) + { +memcpy (&v->evtchn_pending_sel_l2[0 ], + &vcpu_info (v, evtchn_pending_sel), +sizeof (vcpu_info (v, evtchn_pending_sel))); +memset (&vcpu_info (v, evtchn_pending_sel),0 , +sizeof (vcpu_info (v, evtchn_pending_sel))); +set_bit (0 , &vcpu_info (v, evtchn_pending_sel)); + } + +memcpy (d->evtchn_pending[0 ], &shared_info (d, evtchn_pending), +sizeof (shared_info (d, evtchn_pending))); +memcpy (d->evtchn_mask[0 ], &shared_info (d, evtchn_mask), +sizeof (shared_info (d, evtchn_mask))); +} + +static long evtchn_register_3level (evtchn_register_3level_t *arg) +{ +struct domain *d = current->domain; +struct vcpu *v; +int rc =0 ; +xen_pfn_t evtchn_pending[EVTCHN_MAX_L3_PAGES ]; +xen_pfn_t evtchn_mask[EVTCHN_MAX_L3_PAGES ]; +xen_pfn_t l2sel_mfn =0 ; +xen_pfn_t l2sel_offset =0 ; + +if ( d->evtchn_level ==EVTCHN_3_LEVEL ) + { + rc = -EINVAL ; + goto out; + } + +if ( arg->nr_vcpus > d->max_vcpus || + arg->nr_pages >EVTCHN_MAX_L3_PAGES ) + { + rc = -EINVAL ; + goto out; + } + +memset (evtchn_pending,0 ,sizeof (xen_pfn_t) *EVTCHN_MAX_L3_PAGES ); +memset (evtchn_mask,0 ,sizeof (xen_pfn_t) *EVTCHN_MAX_L3_PAGES ); + + rc = -EFAULT ;/* common error code for following operations */ +if (copy_from_guest (evtchn_pending, arg->evtchn_pending, arg->nr_pages) ) + goto out; +if (copy_from_guest (evtchn_mask, arg->evtchn_mask, arg->nr_pages) ) + goto out; + + rc =__map_l3_arrays (d, evtchn_pending, evtchn_mask, arg->nr_pages); +if ( rc ) + goto out; + + for_each_vcpu ( d, v ) + { +int vcpu_id = v->vcpu_id; + + rc = -EFAULT ;/* common error code for following operations */ +if (unlikely (copy_from_guest_offset (&l2sel_mfn, arg->l2sel_mfns, + vcpu_id,1 )) ) + { +__evtchn_unmap_all_3level (d); + goto out; + } +if (unlikely (copy_from_guest_offset (&l2sel_offset, arg->l2sel_offsets, + vcpu_id,1 )) ) + { +__evtchn_unmap_all_3level (d); + goto out; + } + +if ( (rc =__map_l2_selector (v, l2sel_mfn, l2sel_offset)) ) + { +__evtchn_unmap_all_3level (d); + goto out; + } + } + +__evtchn_setup_bitmap_l3 (d); + + d->evtchn_level =EVTCHN_3_LEVEL ; + + rc =0 ; + + out: +return rc; +} + +/* +* NOTE to N-level event channel users: +* N-level channels are likely to consume lots large global mapping +* area in Xen. For example, 3-level event channel consumes 16 + +* nr_vcpus pages global mapping area. So *ONLY* enable N-level event +* channel for Dom0 or driver domains. +*/ +static long evtchn_register_nlevel (struct evtchn_register_nlevel *reg) +{ +struct domain *d = current->domain; +int rc; + +spin_lock (&d->event_lock); + +switch ( reg->level ) + { +case EVTCHN_3_LEVEL : + rc =evtchn_register_3level (®->u.l3); +break ; +default : + rc = -EINVAL ; + } + +spin_unlock (&d->event_lock); + +return rc; +} +long do_event_channel_op (int cmd,XEN_GUEST_HANDLE_PARAM (void ) arg) {long rc;
@@ -1132,6 +1387,18 @@ long do_event_channel_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) arg)break ; } +case EVTCHNOP_register_nlevel: { +struct evtchn_register_nlevel reg; +if (copy_from_guest (®, arg,1 ) !=0 ) +return -EFAULT ; + rc =evtchn_register_nlevel (®); + +/* XXX always fails this call because it is not yet completed */ + rc = -EINVAL ; + +break ; + } +default : rc = -ENOSYS ;break ;
@@ -1258,6 +1525,17 @@ int evtchn_init(struct domain *d)return 0 ; } +static void evtchn_unmap_nlevel (struct domain *d) +{ +switch ( d->evtchn_level ) + { +case EVTCHN_3_LEVEL : +__evtchn_unmap_all_3level (d); +break ; +default : +break ; + } +}void evtchn_destroy (struct domain *d) {
@@ -1286,6 +1564,8 @@ void evtchn_destroy(struct domain *d)clear_global_virq_handlers (d); +evtchn_unmap_nlevel (d); +free_xenheap_page (d->evtchn); }
Signature · 1 line
1.7.10.4
PATCH v2 13/15 Implement 3-level event channel routines
In reply to Wei Liu--- xen/common/event_channel.c | 110 ++++++++++++++++++++++++++++++++++++-------- 1 file changed, 90 insertions(+), 20 deletions(-)
xen/common/event_channel.c +90 -20
@@ -627,10 +627,33 @@ out:return ret; } +static void __check_vcpu_polling (struct vcpu *v,int port) +{ +int vcpuid; +struct domain *d = v->domain; + +/* Check if some VCPU might be polling for this event. */ +if (likely (bitmap_empty (d->poll_mask, d->max_vcpus)) ) +return ; + +/* Wake any interested (or potentially interested) pollers. */ +for ( vcpuid =find_first_bit (d->poll_mask, d->max_vcpus); + vcpuid < d->max_vcpus; + vcpuid =find_next_bit (d->poll_mask, d->max_vcpus, vcpuid+1 ) ) + { + v = d->vcpu[vcpuid]; +if ( ((v->poll_evtchn <=0 ) || (v->poll_evtchn == port)) && +test_and_clear_bit (vcpuid, d->poll_mask) ) + { + v->poll_evtchn =0 ; +vcpu_unblock (v); + } + } +} +static void evtchn_set_pending_l2 (struct vcpu *v,int port) {struct domain *d = v->domain; -int vcpuid;/* * The following bit operations must happen in strict order.
@@ -649,23 +672,35 @@ static void evtchn_set_pending_l2(struct vcpu *v, int port)vcpu_mark_events_pending (v); } -/* Check if some VCPU might be polling for this event. */ -if (likely (bitmap_empty (d->poll_mask, d->max_vcpus)) ) -return ; +__check_vcpu_polling (v, port); +} -/* Wake any interested (or potentially interested) pollers. */ -for ( vcpuid =find_first_bit (d->poll_mask, d->max_vcpus); - vcpuid < d->max_vcpus; - vcpuid =find_next_bit (d->poll_mask, d->max_vcpus, vcpuid+1 ) ) +static void evtchn_set_pending_l3 (struct vcpu *v,int port) +{ +struct domain *d = v->domain; +unsigned int page_no =EVTCHN_PAGE_NO (port); +unsigned int offset =EVTCHN_OFFSET_IN_PAGE (port); +unsigned int l1bit = port >> (EVTCHN_WORD_BITORDER (d) <<1 ); +unsigned int l2bit = port >>EVTCHN_WORD_BITORDER (d); + +/* +* The following bit operations must happen in strict order. +* NB. On x86, the atomic bit operations also act as memory barriers. +* There is therefore sufficiently strict ordering for this architecture -- +* others may require explicit memory barriers. +*/ + +if (test_and_set_bit (offset, d->evtchn_pending[page_no]) ) +return ; + +if ( !test_bit (offset, d->evtchn_mask[page_no]) && + !test_and_set_bit (l2bit, v->evtchn_pending_sel_l2) && + !test_and_set_bit (l1bit, &vcpu_info (v, evtchn_pending_sel)) ) { - v = d->vcpu[vcpuid]; -if ( ((v->poll_evtchn <=0 ) || (v->poll_evtchn == port)) && -test_and_clear_bit (vcpuid, d->poll_mask) ) - { - v->poll_evtchn =0 ; -vcpu_unblock (v); - } +vcpu_mark_events_pending (v); } + +__check_vcpu_polling (v, port); }static void evtchn_set_pending (struct vcpu *v,int port)
@@ -677,6 +712,9 @@ static void evtchn_set_pending(struct vcpu *v, int port)case EVTCHN_2_LEVEL :evtchn_set_pending_l2 (v, port);break ; +case 3 : +evtchn_set_pending_l3 (v, port); +break ;default :BUG (); }
@@ -981,6 +1019,37 @@ static int evtchn_unmask_l2(unsigned int port)return 0 ; } +static int evtchn_unmask_l3 (unsigned int port) +{ +struct domain *d = current->domain; +struct vcpu *v; +unsigned int page_no =EVTCHN_PAGE_NO (port); +unsigned int offset =EVTCHN_OFFSET_IN_PAGE (port); +unsigned int l1bit = port >> (EVTCHN_WORD_BITORDER (d) <<1 ); +unsigned int l2bit = port >>EVTCHN_WORD_BITORDER (d); + +ASSERT (spin_is_locked (&d->event_lock)); + +if (unlikely (!port_is_valid (d, port)) ) +return -EINVAL ; + + v = d->vcpu[evtchn_from_port (d, port)->notify_vcpu_id]; + +/* +* These operations must happen in strict order. Based on +* include/xen/event.h:evtchn_set_pending(). +*/ +if (test_and_clear_bit (offset, d->evtchn_mask[page_no]) && +test_bit (offset, d->evtchn_pending[page_no]) && + !test_and_set_bit (l2bit, v->evtchn_pending_sel_l2) && + !test_and_set_bit (l1bit, &vcpu_info (v, evtchn_pending_sel)) ) + { +vcpu_mark_events_pending (v); + } + +return 0 ; +} +int evtchn_unmask (unsigned int port) {struct domain *d = current->domain;
@@ -991,6 +1060,9 @@ int evtchn_unmask(unsigned int port)case EVTCHN_2_LEVEL : rc =evtchn_unmask_l2 (port);break ; +case 3 : + rc =evtchn_unmask_l3 (port); +break ;default :BUG (); }
@@ -1392,10 +1464,6 @@ long do_event_channel_op(int cmd, XEN_GUEST_HANDLE_PARAM(void) arg)if (copy_from_guest (®, arg,1 ) !=0 )return -EFAULT ; rc =evtchn_register_nlevel (®); - -/* XXX always fails this call because it is not yet completed */ - rc = -EINVAL ; -break ; }
@@ -1604,8 +1672,10 @@ static void domain_dump_evtchn_info(struct domain *d)bitmap_scnlistprintf (keyhandler_scratch,sizeof (keyhandler_scratch), d->poll_mask, d->max_vcpus);printk ("Event channel information for domain %d:\n" +"Using %d-level event channel\n" "Polling vCPUs: {%s}\n" -" port [p/m]\n" , d->domain_id, keyhandler_scratch); +" port [p/m]\n" , + d->domain_id, d->evtchn_level, keyhandler_scratch);spin_lock (&d->event_lock);
Signature · 1 line
1.7.10.4
PATCH v2 14/15 Only allow 3-level event channel on Dom0 and driver domain
In reply to Wei LiuFor non-Dom0 domains, add a flag to indicate whether it can use 3-level event channel, admins can specify this flag when creating a driver domain.
--- xen/common/domain.c | 3 +++ xen/common/domctl.c | 5 ++++- xen/common/event_channel.c | 3 +++ xen/include/public/domctl.h | 3 +++ xen/include/xen/sched.h | 5 +++++ 5 files changed, 18 insertions(+), 1 deletion(-)
xen/common/domain.c +3 -0
@@ -250,6 +250,9 @@ struct domain *domain_create(if ( domcr_flags & DOMCRF_dummy )return d; +if ( domcr_flags & DOMCRF_evtchn_l3 ) + d->evtchn_l3 =1 ; +if ( !is_idle_domain (d) ) {if ( (err =xsm_domain_create (XSM_HOOK , d, ssidref)) !=0 )
xen/common/domctl.c +4 -1
@@ -369,7 +369,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl)if ( supervisor_mode_kernel || (op->u.createdomain.flags & ~(XEN_DOMCTL_CDF_hvm_guest | XEN_DOMCTL_CDF_hap | - XEN_DOMCTL_CDF_s3_integrity | XEN_DOMCTL_CDF_oos_off)) ) + XEN_DOMCTL_CDF_s3_integrity | XEN_DOMCTL_CDF_oos_off | + XEN_DOMCTL_CDF_evtchn_l3)) )break ; dom = op->domain;
@@ -405,6 +406,8 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) domcr_flags |= DOMCRF_s3_integrity;if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_oos_off ) domcr_flags |= DOMCRF_oos_off; +if ( op->u.createdomain.flags & XEN_DOMCTL_CDF_evtchn_l3 ) + domcr_flags |= DOMCRF_evtchn_l3; d =domain_create (dom, domcr_flags, op->u.createdomain.ssidref);if (IS_ERR (d) )
xen/common/event_channel.c +3 -0
@@ -1262,6 +1262,9 @@ static long evtchn_register_3level(evtchn_register_3level_t *arg)xen_pfn_t l2sel_mfn =0 ;xen_pfn_t l2sel_offset =0 ; +if ( d->evtchn_l3 ==0 && d->domain_id !=0 ) +return -EPERM ; +if ( d->evtchn_level ==EVTCHN_3_LEVEL ) { rc = -EINVAL ;
xen/include/public/domctl.h +3 -0
@@ -59,6 +59,9 @@ struct xen_domctl_createdomain {/* Disable out-of-sync shadow page tables? */ #define _XEN_DOMCTL_CDF_oos_off 3#define XEN_DOMCTL_CDF_oos_off (1U<<_XEN_DOMCTL_CDF_oos_off) +/* Can this domain use 3-level event channel? */ +#define _XEN_DOMCTL_CDF_evtchn_l3 4 +#define XEN_DOMCTL_CDF_evtchn_l3 (1U<<_XEN_DOMCTL_CDF_evtchn_l3)uint32_t flags; };typedef struct xen_domctl_createdomain xen_domctl_createdomain_t ;
xen/include/xen/sched.h +5 -0
@@ -224,6 +224,8 @@ struct domainunsigned int evtchn_level;unsigned long *evtchn_pending[EVTCHN_MAX_L3_PAGES ];unsigned long *evtchn_mask[EVTCHN_MAX_L3_PAGES ]; +/* Can the domain use 3-level event channel? */ +bool_t evtchn_l3;struct grant_table *grant_table;
@@ -411,6 +413,9 @@ struct domain *domain_create(/* DOMCRF_oos_off: dont use out-of-sync optimization for shadow page tables */ #define _DOMCRF_oos_off 4#define DOMCRF_oos_off (1U<<_DOMCRF_oos_off) +/* DOMCRF_evtchn_l3: this domain can use 3-level event channel (driver domain) */ +#define _DOMCRF_evtchn_l3 5 +#define DOMCRF_evtchn_l3 (1U<<_DOMCRF_evtchn_l3)/* * rcu_lock_domain_by_id() is more efficient than get_domain_by_id().
Signature · 1 line
1.7.10.4
Re: [PATCH V2 14/15] Only allow 3-level event channel on Dom0 and driver domain
xen/include/xen/sched.h struct domain *domain_create( · line 419
Quoted 9 lines
Quoted 1 line
Quoted 1 line · Wei Liu
xen/include/xen/sched.h +3
-0@@ -411,6 +413,9 @@ struct domain *domain_create(/* DOMCRF_oos_off: dont use out-of-sync optimization for shadow page tables */ #define _DOMCRF_oos_off 4#define DOMCRF_oos_off (1U<<_DOMCRF_oos_off) +/* DOMCRF_evtchn_l3: this domain can use 3-level event channel (driver domain) */ +#define _DOMCRF_evtchn_l3 5 +#define DOMCRF_evtchn_l3 (1U<<_DOMCRF_evtchn_l3)
Please don't use "l3" in this name; instead, make it generic to say "extended event channels" in some way.
Also, the patch description should explain why this restriction is being put in place.
Jan
PATCH v2 15/15 libxl: add evtchn_l3 flag
In reply to Wei LiuAdmins can add "evtchn_l3 = 1" in domain config file to enable 3-level event channel for a domain.
--- tools/libxl/libxl_create.c | 3 +++ tools/libxl/libxl_types.idl | 1 + tools/libxl/xl_cmdimpl.c | 2 ++ tools/libxl/xl_sxp.c | 1 + 4 files changed, 7 insertions(+)
tools/libxl/libxl_create.c +3 -0
@@ -35,6 +35,8 @@ int libxl__domain_create_info_setdefault(libxl__gc *gc,libxl_defbool_setdefault (&c_info->oos, true); } +libxl_defbool_setdefault (&c_info->evtchn_l3, false); +libxl_defbool_setdefault (&c_info->run_hotplug_scripts, true);return 0 ;
@@ -375,6 +377,7 @@ int libxl__domain_make(libxl__gc *gc, libxl_domain_create_info *info, flags |=libxl_defbool_val (info->hap) ? XEN_DOMCTL_CDF_hap :0 ; flags |=libxl_defbool_val (info->oos) ?0 : XEN_DOMCTL_CDF_oos_off; } + flags |=libxl_defbool_val (info->evtchn_l3) ? XEN_DOMCTL_CDF_evtchn_l3 :0 ; *domid =-1 ;/* Ultimately, handle is an array of 16 uint8_t, same as uuid */
tools/libxl/libxl_types.idl +1 -0
@@ -236,6 +236,7 @@ libxl_domain_create_info = Struct("domain_create_info",[ ("type", libxl_domain_type), ("hap", libxl_defbool), ("oos", libxl_defbool), + ("evtchn_l3", libxl_defbool), ("ssidref", uint32), ("name", string), ("uuid", libxl_uuid),
tools/libxl/xl_cmdimpl.c +2 -0
@@ -651,6 +651,8 @@ static void parse_config_data(const char *config_source,xlu_cfg_get_defbool (config,"oos" , &c_info->oos,0 ); +xlu_cfg_get_defbool (config,"evtchn_l3" , &c_info->evtchn_l3,0 ); +if (!xlu_cfg_get_string (config,"pool" , &buf,0 )) { c_info->poolid =-1 ;cpupool_qualifier_to_cpupoolid (buf, &c_info->poolid,NULL );
tools/libxl/xl_sxp.c +1 -0
@@ -44,6 +44,7 @@ void printf_info_sexp(int domid, libxl_domain_config *d_config)printf ("\t(hvm %d)\n" , c_info->type ==LIBXL_DOMAIN_TYPE_HVM );printf ("\t(hap %s)\n" ,libxl_defbool_to_string (c_info->hap));printf ("\t(oos %s)\n" ,libxl_defbool_to_string (c_info->oos)); +printf ("\t(evtchn_l3 %s)\n" ,libxl_defbool_to_string (c_info->evtchn_l3));printf ("\t(ssidref %d)\n" , c_info->ssidref);printf ("\t(name %s)\n" , c_info->name);
Signature · 1 line
1.7.10.4