RFC Switch mem event ABI from domctl to memops
PATCH 1/1 RFC Switch mem event ABI from domctl to memops
Swith mem event per page operations from domctl to XENMEM_* interface. Improve scalabilty.
Looking for further feedback on the proposed ABI change.
This code has been tested to work generally well, save for a few blue screens. It might be that removing the global domctl lock uncovers hidden races.
tools/libxc/xc_mem_access.c | 12 +- tools/libxc/xc_mem_event.c | 23 +++- tools/libxc/xc_mem_paging.c | 44 ++++---- tools/libxc/xc_memshr.c | 182 ++++++++++++++++------------------ tools/libxc/xenctrl.h | 6 +- tools/memshr/interface.c | 4 +- tools/tests/mem-sharing/memshrtool.c | 4 +- xen/arch/x86/domctl.c | 1 - xen/arch/x86/mm/mem_access.c | 7 +- xen/arch/x86/mm/mem_event.c | 68 ++++++++++-- xen/arch/x86/mm/mem_paging.c | 13 +- xen/arch/x86/mm/mem_sharing.c | 101 +++++++++++------- xen/arch/x86/x86_64/compat/mm.c | 23 ++++ xen/arch/x86/x86_64/mm.c | 23 ++++ xen/include/asm-x86/mem_access.h | 3 +- xen/include/asm-x86/mem_event.h | 2 + xen/include/asm-x86/mem_paging.h | 3 +- xen/include/asm-x86/mem_sharing.h | 3 + xen/include/public/domctl.h | 90 +++------------- xen/include/public/memory.h | 87 ++++++++++++++++ tools/libxc/xc_memshr.c | 11 ++ tools/libxc/xenctrl.h | 1 + tools/tests/mem-sharing/memshrtool.c | 11 ++ xen/arch/x86/mm/mem_sharing.c | 13 +- xen/arch/x86/x86_64/compat/mm.c | 3 + xen/arch/x86/x86_64/mm.c | 2 + xen/include/asm-x86/mem_sharing.h | 3 +- xen/include/public/memory.h | 1 + 28 files changed, 460 insertions(+), 284 deletions(-)
PATCH 1/1 Use memops for mem paging, sharing, and access, instead of domctls
tools/libxc/xc_mem_access.c | 12 +- tools/libxc/xc_mem_event.c | 23 +++- tools/libxc/xc_mem_paging.c | 44 ++++---- tools/libxc/xc_memshr.c | 182 ++++++++++++++++------------------ tools/libxc/xenctrl.h | 6 +- tools/memshr/interface.c | 4 +- tools/tests/mem-sharing/memshrtool.c | 4 +- xen/arch/x86/domctl.c | 1 - xen/arch/x86/mm/mem_access.c | 7 +- xen/arch/x86/mm/mem_event.c | 68 ++++++++++-- xen/arch/x86/mm/mem_paging.c | 13 +- xen/arch/x86/mm/mem_sharing.c | 101 +++++++++++------- xen/arch/x86/x86_64/compat/mm.c | 23 ++++ xen/arch/x86/x86_64/mm.c | 23 ++++ xen/include/asm-x86/mem_access.h | 3 +- xen/include/asm-x86/mem_event.h | 2 + xen/include/asm-x86/mem_paging.h | 3 +- xen/include/asm-x86/mem_sharing.h | 3 + xen/include/public/domctl.h | 90 +++------------- xen/include/public/memory.h | 87 ++++++++++++++++ 20 files changed, 423 insertions(+), 276 deletions(-)
Per page operations in the paging, sharing, and access tracking subsystems are all implemented with domctls (e.g. a domctl to evict one page, or to share one page).
Under heavy load, the domctl path reveals a lack of scalability. The domctl lock serializes dom0's vcpus in the hypervisor. When performing thousands of per-page operations on dozens of domains, these vcpus will spin in the hypervisor. Beyond the aggressive locking, an added inefficiency of blocking vcpus in the domctl lock is that dom0 is prevented from re-scheduling any of its other work-starved processes.
We retain the domctl interface for setting up and tearing down paging/sharing/mem access for a domain. But we migrate all the per page operations to use the memory_op hypercalls (e.g XENMEM_*).
tools/libxc/xc_mem_access.c +6 -6
@@ -30,7 +30,7 @@ int xc_mem_access_enable(xc_interface *xreturn xc_mem_event_control (xch, domain_id,XEN_DOMCTL_MEM_EVENT_OP_ACCESS_ENABLE ,XEN_DOMCTL_MEM_EVENT_OP_ACCESS , - shared_page, ring_page,INVALID_MFN ); + shared_page, ring_page); }int xc_mem_access_disable (xc_interface *xch,domid_t domain_id)
@@ -38,15 +38,15 @@ int xc_mem_access_disable(xc_interface *return xc_mem_event_control (xch, domain_id,XEN_DOMCTL_MEM_EVENT_OP_ACCESS_DISABLE ,XEN_DOMCTL_MEM_EVENT_OP_ACCESS , -NULL ,NULL ,INVALID_MFN ); +NULL ,NULL ); }int xc_mem_access_resume (xc_interface *xch,domid_t domain_id,unsigned long gfn) { -return xc_mem_event_control (xch, domain_id, -XEN_DOMCTL_MEM_EVENT_OP_ACCESS_RESUME , -XEN_DOMCTL_MEM_EVENT_OP_ACCESS , -NULL ,NULL , gfn); +return xc_mem_event_memop (xch, domain_id, + XENMEM_access_op_resume, + XENMEM_access_op, + gfn,NULL ); }/*
tools/libxc/xc_mem_event.c +18 -5
@@ -24,8 +24,7 @@#include "xc_private.h" int xc_mem_event_control (xc_interface *xch,domid_t domain_id,unsigned int op, -unsigned int mode,void *page, -void *ring_page,unsigned long gfn) +unsigned int mode,void *page,void *ring_page) {DECLARE_DOMCTL ;
@@ -34,11 +33,25 @@ int xc_mem_event_control(xc_interface *x domctl.u.mem_event_op.op = op; domctl.u.mem_event_op.mode = mode; - domctl.u.mem_event_op.u.shared_addr = (unsigned long )page; + domctl.u.mem_event_op.shared_addr = (unsigned long )page; domctl.u.mem_event_op.ring_addr = (unsigned long )ring_page; - - domctl.u.mem_event_op.gfn = gfn;return do_domctl (xch, &domctl); } +int xc_mem_event_memop (xc_interface *xch,domid_t domain_id, +unsigned int op,unsigned int mode, +uint64_t gfn,void *buffer) +{ +xen_mem_event_op_t meo; + +memset (&meo,0 ,sizeof (meo)); + + meo.op = op; + meo.domain = domain_id; + meo.gfn = gfn; + meo.buffer = (unsigned long ) buffer; + +return do_memory_op (xch, mode, &meo,sizeof (meo)); +} +
tools/libxc/xc_mem_paging.c +22 -22
80 lines not on this page · raw message
tools/libxc/xc_memshr.c +84 -98
265 lines not on this page · raw message
tools/libxc/xenctrl.h +4 -2
@@ -1879,8 +1879,10 @@ int xc_tmem_restore_extra(xc_interface ** mem_event operations */ int xc_mem_event_control (xc_interface *xch,domid_t domain_id,unsigned int op, -unsigned int mode,void *shared_page, -void *ring_page,unsigned long gfn); +unsigned int mode,void *shared_page,void *ring_page); +int xc_mem_event_memop (xc_interface *xch,domid_t domain_id, +unsigned int op,unsigned int mode, +uint64_t gfn,void *buffer);int xc_mem_paging_enable (xc_interface *xch,domid_t domain_id,void *shared_page,void *ring_page);
tools/memshr/interface.c +2 -2
@@ -186,12 +186,12 @@ int memshr_vbd_issue_ro_request(char *buremove the relevant ones from the map */ switch (ret) { -case XEN_DOMCTL_MEM_SHARING_S_HANDLE_INVALID : +case XENMEM_SHARING_OP_S_HANDLE_INVALID : ret =blockshr_shrhnd_remove (memshr.blks, source_st,NULL );if (ret)DPRINTF ("Could not rm invl s_hnd: %u %" PRId64" %" PRId64"\n" , source_st.domain, source_st.frame, source_st.handle);break ; -case XEN_DOMCTL_MEM_SHARING_C_HANDLE_INVALID : +case XENMEM_SHARING_OP_C_HANDLE_INVALID : ret =blockshr_shrhnd_remove (memshr.blks, client_st,NULL );if (ret)DPRINTF ("Could not rm invl c_hnd: %u %" PRId64" %" PRId64"\n" , client_st.domain, client_st.frame, client_st.handle);
tools/tests/mem-sharing/memshrtool.c +2 -2
@@ -34,9 +34,9 @@ static int usage(const char* prog)int rc = f; \if ( rc <0 ) { \ printf("error executing %s: %s\n", #f, \ - ((errno *-1 ) ==XEN_DOMCTL_MEM_SHARING_C_HANDLE_INVALID ) ? \ + ((errno *-1 ) ==XENMEM_SHARING_OP_S_HANDLE_INVALID ) ? \"problem with client handle" :\ - ((errno *-1 ) ==XEN_DOMCTL_MEM_SHARING_S_HANDLE_INVALID ) ? \ + ((errno *-1 ) ==XENMEM_SHARING_OP_C_HANDLE_INVALID ) ? \"problem with source handle" :strerror (errno)); \return rc; \ } \
xen/arch/x86/domctl.c +0 -1
@@ -1463,7 +1463,6 @@ long arch_do_domctl(if ( !ret ) ret =mem_sharing_domctl (d, &domctl->u.mem_sharing_op);rcu_unlock_domain (d); -copy_to_guest (u_domctl, domctl,1 ); } }break ;
xen/arch/x86/mm/mem_access.c +3 -4
@@ -25,14 +25,13 @@#include <asm/mem_event.h> -int mem_access_domctl (struct domain *d,xen_domctl_mem_event_op_t *mec, -XEN_GUEST_HANDLE (void ) u_domctl) +int mem_access_memop (struct domain *d,xen_mem_event_op_t *meo) {int rc; -switch ( mec->op ) +switch ( meo->op ) { -case XEN_DOMCTL_MEM_EVENT_OP_ACCESS_RESUME : +case XENMEM_access_op_resume: {p2m_mem_access_resume (d); rc =0 ;
xen/arch/x86/mm/mem_event.c +57 -11
101 lines not on this page · raw message
xen/arch/x86/mm/mem_paging.c +6 -7
@@ -25,33 +25,32 @@#include <asm/mem_event.h> -int mem_paging_domctl (struct domain *d,xen_domctl_mem_event_op_t *mec, -XEN_GUEST_HANDLE (void ) u_domctl) +int mem_paging_memop (struct domain *d,xen_mem_event_op_t *mec) {switch ( mec->op ) { -case XEN_DOMCTL_MEM_EVENT_OP_PAGING_NOMINATE : +case XENMEM_paging_op_nominate: {unsigned long gfn = mec->gfn;return p2m_mem_paging_nominate (d, gfn); }break ; -case XEN_DOMCTL_MEM_EVENT_OP_PAGING_EVICT : +case XENMEM_paging_op_evict: {unsigned long gfn = mec->gfn;return p2m_mem_paging_evict (d, gfn); }break ; -case XEN_DOMCTL_MEM_EVENT_OP_PAGING_PREP : +case XENMEM_paging_op_prep: {unsigned long gfn = mec->gfn; -return p2m_mem_paging_prep (d, gfn, mec->u.buffer); +return p2m_mem_paging_prep (d, gfn, mec->buffer); }break ; -case XEN_DOMCTL_MEM_EVENT_OP_PAGING_RESUME : +case XENMEM_paging_op_resume: {p2m_mem_paging_resume (d);return 0 ;
xen/arch/x86/mm/mem_sharing.c +60 -41
260 lines not on this page · raw message
xen/arch/x86/x86_64/compat/mm.c +23 -0
@@ -2,6 +2,7 @@#include <xen/multicall.h> #include <compat/memory.h> #include <compat/xen.h> +#include <asm/mem_event.h> int compat_set_gdt (XEN_GUEST_HANDLE (uint) frame_list,unsigned int entries) {
@@ -211,6 +212,28 @@ int compat_arch_memory_op(int op, XEN_GUcase XENMEM_get_sharing_shared_pages:return mem_sharing_get_nr_shared_mfns (); +case XENMEM_paging_op: +case XENMEM_access_op: + { +xen_mem_event_op_t meo; +if (copy_from_guest (&meo, arg,1 ) ) +return -EFAULT ; + rc =do_mem_event_op (op, meo.domain, (void *) &meo); +if ( !rc &©_to_guest (arg, &meo,1 ) ) +return -EFAULT ; +break ; + } +case XENMEM_sharing_op: + { +xen_mem_sharing_op_t mso; +if (copy_from_guest (&mso, arg,1 ) ) +return -EFAULT ; + rc =do_mem_event_op (op, mso.domain, (void *) &mso); +if ( !rc &©_to_guest (arg, &mso,1 ) ) +return -EFAULT ; +break ; + } +default : rc = -ENOSYS ;break ;
xen/arch/x86/x86_64/mm.c +23 -0
@@ -34,6 +34,7 @@#include <asm/msr.h> #include <asm/setup.h> #include <asm/numa.h> +#include <asm/mem_event.h> #include <asm/mem_sharing.h> #include <public/memory.h>
@@ -1100,6 +1101,28 @@ long subarch_memory_op(int op, XEN_GUESTcase XENMEM_get_sharing_shared_pages:return mem_sharing_get_nr_shared_mfns (); +case XENMEM_paging_op: +case XENMEM_access_op: + { +xen_mem_event_op_t meo; +if (copy_from_guest (&meo, arg,1 ) ) +return -EFAULT ; + rc =do_mem_event_op (op, meo.domain, (void *) &meo); +if ( !rc &©_to_guest (arg, &meo,1 ) ) +return -EFAULT ; +break ; + } +case XENMEM_sharing_op: + { +xen_mem_sharing_op_t mso; +if (copy_from_guest (&mso, arg,1 ) ) +return -EFAULT ; + rc =do_mem_event_op (op, mso.domain, (void *) &mso); +if ( !rc &©_to_guest (arg, &mso,1 ) ) +return -EFAULT ; +break ; + } +default : rc = -ENOSYS ;break ;
xen/include/asm-x86/mem_access.h +1 -2
@@ -21,8 +21,7 @@*/ -int mem_access_domctl (struct domain *d,xen_domctl_mem_event_op_t *mec, -XEN_GUEST_HANDLE (void ) u_domctl); +int mem_access_memop (struct domain *d,xen_mem_event_op_t *meo);/*
xen/include/asm-x86/mem_event.h +2 -0
@@ -39,6 +39,8 @@ void mem_event_put_request(struct domainint mem_event_get_response (struct domain *d,struct mem_event_domain *med,mem_event_response_t *rsp); +struct domain *get_mem_event_op_target (uint32_t domain,int *rc); +int do_mem_event_op (int op,uint32_t domain,void *arg);int mem_event_domctl (struct domain *d,xen_domctl_mem_event_op_t *mec,XEN_GUEST_HANDLE (void ) u_domctl);
xen/include/asm-x86/mem_paging.h +1 -2
@@ -21,8 +21,7 @@*/ -int mem_paging_domctl (struct domain *d,xen_domctl_mem_event_op_t *mec, -XEN_GUEST_HANDLE (void ) u_domctl); +int mem_paging_memop (struct domain *d,xen_mem_event_op_t *meo);/*
xen/include/asm-x86/mem_sharing.h +3 -0
@@ -23,6 +23,7 @@#define __MEM_SHARING_H__#include <public/domctl.h> +#include <public/memory.h> /* Auditing of memory sharing code? */ #define MEM_SHARING_AUDIT 0
@@ -56,6 +57,8 @@ int mem_sharing_unshare_page(struct domaunsigned long gfn, uint16_t flags);int mem_sharing_sharing_resume (struct domain *d); +int mem_sharing_memop (struct domain *d, +xen_mem_sharing_op_t *mec);int mem_sharing_domctl (struct domain *d,xen_domctl_mem_sharing_op_t *mec);void mem_sharing_init (void );
xen/include/public/domctl.h +19 -71
129 lines not on this page · raw message
xen/include/public/memory.h +87 -0
99 lines not on this page · raw message
PATCH 1/1 x86/mm: New sharing audit memop
In reply to Andres Lagar-Cavillatools/libxc/xc_memshr.c | 11 +++++++++++ tools/libxc/xenctrl.h | 1 + tools/tests/mem-sharing/memshrtool.c | 11 +++++++++++ xen/arch/x86/mm/mem_sharing.c | 13 ++++++------- xen/arch/x86/x86_64/compat/mm.c | 3 +++ xen/arch/x86/x86_64/mm.c | 2 ++ xen/include/asm-x86/mem_sharing.h | 3 ++- xen/include/public/memory.h | 1 + 8 files changed, 37 insertions(+), 8 deletions(-)
Remove costly mem_sharting audits from the inline path, and instead make them callable as a memop.
Have the audit function return the number of errors detected.
Update memshrtool to be able to trigger audits.
Set sharing audits as enabled by default.
tools/libxc/xc_memshr.c +11 -0
@@ -211,6 +211,17 @@ int xc_memshr_debug_gref(xc_interface *xreturn xc_memshr_memop (xch, domid, &mso); } +int xc_memshr_audit (xc_interface *xch) +{ +xen_mem_sharing_op_t mso; + +memset (&mso,0 ,sizeof (mso)); + + mso.op = XENMEM_sharing_op_audit; + +return do_memory_op (xch, XENMEM_sharing_op, &mso,sizeof (mso)); +} +long xc_sharing_freed_pages (xc_interface *xch) {return do_memory_op (xch, XENMEM_get_sharing_freed_pages,NULL ,0 );
tools/libxc/xenctrl.h +1 -0
@@ -1947,6 +1947,7 @@ int xc_memshr_debug_mfn(xc_interface *xcint xc_memshr_debug_gref (xc_interface *xch,domid_t domid,grant_ref_t gref); +int xc_memshr_audit (xc_interface *xch);int xc_flask_load (xc_interface *xc_handle,char *buf,uint32_t size);int xc_flask_context_to_sid (xc_interface *xc_handle,char *buf,uint32_t size,uint32_t *sid);
tools/tests/mem-sharing/memshrtool.c +11 -0
@@ -27,6 +27,7 @@ static int usage(const char* prog)printf (" add-to-physmap <domid> <gfn> <source> <source-gfn> <source-handle>\n" );printf (" - Populate a page in a domain with a shared page.\n" );printf (" debug-gfn <domid> <gfn> - Debug a particular domain and gfn.\n" ); +printf (" audit - Audit the sharing subsytem in Xen.\n" );return 1 ; }
@@ -160,6 +161,16 @@ int main(int argc, const char** argv) gfn =strtol (argv[3 ],NULL ,0 );R (xc_memshr_debug_gfn (xch, domid, gfn)); } +else if ( !strcasecmp (cmd,"audit" ) ) + { +int rc =xc_memshr_audit (xch); +if ( rc <0 ) + { +printf ("error executing xc_memshr_audit: %s\n" ,strerror (errno)); +return rc; + } +printf ("Audit returned %d errors.\n" , rc); + }return 0 ; }
xen/arch/x86/mm/mem_sharing.c +6 -7
@@ -47,8 +47,6 @@ DEFINE_PER_CPU(pg_lock_data_t, __pld);#if MEM_SHARING_AUDIT -static void mem_sharing_audit (void ); - #define MEM_SHARING_DEBUG(_f, _a...) \ debugtrace_printk("mem_sharing_debug: %s(): " _f, __func__, ##_a)
@@ -81,7 +79,10 @@ static inline void audit_del_list(struct#else -#define mem_sharing_audit () ((void)0) +int mem_sharing_audit (void ) +{ +return -ENOSYS ; +}#define audit_add_list (p) ((void)0)static inline void audit_del_list (struct page_info *page)
@@ -207,7 +208,7 @@ static struct page_info* mem_sharing_loo }#if MEM_SHARING_AUDIT -static void mem_sharing_audit (void ) +int mem_sharing_audit (void ) {int errors =0 ;unsigned long count_expected;
@@ -333,6 +334,7 @@ static void mem_sharing_audit(void) errors++; } +return errors; }#endif
@@ -903,7 +905,6 @@ int mem_sharing_unshare_page(struct domagfn_info_t *gfn_info =NULL ;struct list_head *le; -mem_sharing_audit (); mfn =get_gfn (d, gfn, &p2mt);/* Has someone already unshared it? */
@@ -1169,8 +1170,6 @@ int mem_sharing_memop(struct domain *d,break ; } -mem_sharing_audit (); -return rc; }
xen/arch/x86/x86_64/compat/mm.c +3 -0
@@ -3,6 +3,7 @@#include <compat/memory.h> #include <compat/xen.h> #include <asm/mem_event.h> +#include <asm/mem_sharing.h> int compat_set_gdt (XEN_GUEST_HANDLE (uint) frame_list,unsigned int entries) {
@@ -228,6 +229,8 @@ int compat_arch_memory_op(int op, XEN_GUxen_mem_sharing_op_t mso;if (copy_from_guest (&mso, arg,1 ) )return -EFAULT ; +if ( mso.op == XENMEM_sharing_op_audit ) +return mem_sharing_audit (); rc =do_mem_event_op (op, mso.domain, (void *) &mso);if ( !rc &©_to_guest (arg, &mso,1 ) )return -EFAULT ;
xen/arch/x86/x86_64/mm.c +2 -0
@@ -1117,6 +1117,8 @@ long subarch_memory_op(int op, XEN_GUESTxen_mem_sharing_op_t mso;if (copy_from_guest (&mso, arg,1 ) )return -EFAULT ; +if ( mso.op == XENMEM_sharing_op_audit ) +return mem_sharing_audit (); rc =do_mem_event_op (op, mso.domain, (void *) &mso);if ( !rc &©_to_guest (arg, &mso,1 ) )return -EFAULT ;
xen/include/asm-x86/mem_sharing.h +2 -1
@@ -26,7 +26,7 @@#include <public/memory.h> /* Auditing of memory sharing code? */ -#define MEM_SHARING_AUDIT 0 +#define MEM_SHARING_AUDIT 1typedef uint64_t shr_handle_t ;
@@ -61,6 +61,7 @@ int mem_sharing_memop(struct domain *d, xen_mem_sharing_op_t *mec);int mem_sharing_domctl (struct domain *d,xen_domctl_mem_sharing_op_t *mec); +int mem_sharing_audit (void );void mem_sharing_init (void );#else
xen/include/public/memory.h +1 -0
@@ -333,6 +333,7 @@ DEFINE_XEN_GUEST_HANDLE(xen_mem_event_op#define XENMEM_sharing_op_debug_mfn 5#define XENMEM_sharing_op_debug_gref 6#define XENMEM_sharing_op_add_physmap 7 +#define XENMEM_sharing_op_audit 8#define XENMEM_SHARING_OP_S_HANDLE_INVALID (-10)#define XENMEM_SHARING_OP_C_HANDLE_INVALID (-9)