xenguest optimisations
Subsystems (3)
v5 cover xenguest optimisations
From: Frediano Ziglio <[email protected]>
Reduce some number of parts passed to writev. Avoid possible allocation sending data with writev. Reduce number of allocations sending memory state.
Implement and use new Xen and Linux kernel ABI to copy foreign memory. This new ABI allows to replace theexpensive map/copy/unmap sequence with a single call.
Changes since v1:
- add commit to cache up to 4 pages in hypercall;
- add other 2 commits reducing chunks passed to write/writev.
Changes since v2:
- update patches commit prefixes;
- add other 2 optisations.
Changes since v3:
- address some comments;
- add patches for foreign copy optimisation.
Changes since v4:
- added Reviewed-by;
- improved commit messages;
- other minor fixes, see individual commits.
Edwin Török (3):
libs/guest: allocate various migration arrays just once libs/call: cache up to 4 pages in hypercall bounce buffers
PoC: libs/guest: use foreign copy during migration
Frediano Ziglio (12):
libs/guest: Reduce number of parts in write_split_record libs/guest: Reduce number of I/O vectors in write_batch libs/guest: Reduce number of I/O vectors in write_batch libs/guest: Use a single write_exact in write_headers libs/guest: avoids using 2 indexes
libs/guest: fill directly iov structure
libs/ctrl: Allows writev_exact to change iov array libs/guest: add xg_foreignmemory_copy_{from,to}
xen: implement new foreign copy hypercall
privcmd: Add definition for new Linux privcmd to access new Xen
hypercall
libs/guest: use new hypercall if available libs/guest: finalize PoC
tools/include/xen-sys/Linux/privcmd.h | 10 ++ tools/libs/call/buffer.c | 28 ++-- tools/libs/call/core.c | 3 +- tools/libs/call/private.h | 8 +- tools/libs/ctrl/xc_private.c | 26 +-- tools/libs/ctrl/xc_private.h | 2 +- tools/libs/guest/xg_sr_common.c | 95 ++++++++++- tools/libs/guest/xg_sr_common.h | 21 +++ tools/libs/guest/xg_sr_restore.c | 100 ++++++------ tools/libs/guest/xg_sr_save.c | 224 +++++++++++--------------- xen/common/memory.c | 134 +++++++++++++++ xen/include/public/memory.h | 40 ++++- 13 files changed, 474 insertions(+), 219 deletions(-)
Signature · 1 line
2.43.0
PATCH v5 1/16 libs/guest: Reduce number of parts in write_split_record
From: Frediano Ziglio <[email protected]>
Small optimization.
There's no much sense to split the header in 2 pieces, it will just take more time and space to reassemble them in the final buffer.
This also avoids truncating combined_length to 32 bit in case of 64 bit machines potentially avoiding following record_length check (it could still be truncated writing it in xc_sr_rhdr structure but the following check will catch it).
The function become more coherent with following read_record function.
--
Changes since v2:
- change prefix in subject.
Changes since v3:
- clarify commit message.
Changes since v4:
- added Reviewed-by;
- improved commit message.
--- tools/libs/guest/xg_sr_common.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-)
tools/libs/guest/xg_sr_common.c +3 -3
@@ -59,11 +59,11 @@ int write_split_record(struct xc_sr_context *ctx, struct xc_sr_record *rec,static const char zeroes[REC_ALIGN ] = {};xc_interface *xch = ctx->xch; - typeof(rec->length) combined_length = rec->length + sz; +size_t combined_length = rec->length + sz;size_t record_length =ROUNDUP (combined_length,REC_ALIGN ); +struct xc_sr_rhdr rhdr = { rec->type, combined_length };struct iovec parts[] = { - { &rec->type,sizeof (rec->type) }, - { &combined_length,sizeof (combined_length) }, + { &rhdr,sizeof (rhdr) }, { rec->data, rec->length }, { buf, sz }, { (void *)zeroes, record_length - combined_length },
Signature · 1 line
2.43.0
PATCH v5 2/16 libs/guest: Reduce number of I/O vectors in write_batch
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Small optimization.
Reduce number of pieces passed to writev.
--
Changes since v2:
- change prefix in subject.
Changes since v4:
- added Reviewed-by.
--- tools/libs/guest/xg_sr_save.c | 34 +++++++++++++++------------------- 1 file changed, 15 insertions(+), 19 deletions(-)
tools/libs/guest/xg_sr_save.c +15 -19
@@ -97,9 +97,11 @@ static int write_batch(struct xc_sr_context *ctx)void *page, *orig_page;uint64_t *rec_pfns =NULL ;struct iovec *iov =NULL ;int iovcnt =0 ; -struct xc_sr_rec_page_data_header hdr = {0 }; -struct xc_sr_record rec = { - .type =REC_TYPE_PAGE_DATA , +struct { +struct xc_sr_rhdr rec; +struct xc_sr_rec_page_data_header page_data; + } hdrs = { + { .type =REC_TYPE_PAGE_DATA }, };assert (nr_pfns !=0 );
@@ -115,7 +117,7 @@ static int write_batch(struct xc_sr_context *ctx)/* Pointers to locally allocated pages. Need freeing. */ local_pages =calloc (nr_pfns,sizeof (*local_pages));/* iovec[] for writev(). */ - iov =malloc ((nr_pfns +4 ) *sizeof (*iov)); + iov =malloc ((nr_pfns +2 ) *sizeof (*iov));if ( !mfns || !types || !errors || !guest_data || !local_pages || !iov ) {
@@ -216,28 +218,22 @@ static int write_batch(struct xc_sr_context *ctx) goto err; } - hdr.count = nr_pfns; + hdrs.rec.length =sizeof (hdrs.page_data); + hdrs.rec.length += nr_pfns *sizeof (*rec_pfns); + hdrs.rec.length += nr_pages *PAGE_SIZE ; - rec.length =sizeof (hdr); - rec.length += nr_pfns *sizeof (*rec_pfns); - rec.length += nr_pages *PAGE_SIZE ; + hdrs.page_data.count = nr_pfns;for ( i =0 ; i < nr_pfns; ++i ) rec_pfns[i] = ((uint64_t )(types[i]) <<32 ) | ctx->save.batch_pfns[i]; - iov[0 ].iov_base = &rec.type; - iov[0 ].iov_len =sizeof (rec.type); + iov[0 ].iov_base = &hdrs; + iov[0 ].iov_len =sizeof (hdrs); - iov[1 ].iov_base = &rec.length; - iov[1 ].iov_len =sizeof (rec.length); + iov[1 ].iov_base = rec_pfns; + iov[1 ].iov_len = nr_pfns *sizeof (*rec_pfns); - iov[2 ].iov_base = &hdr; - iov[2 ].iov_len =sizeof (hdr); - - iov[3 ].iov_base = rec_pfns; - iov[3 ].iov_len = nr_pfns *sizeof (*rec_pfns); - - iovcnt =4 ; + iovcnt =2 ;if ( nr_pages ) {
Signature · 1 line
2.43.0
PATCH v5 3/16 libs/guest: Reduce number of I/O vectors in write_batch
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Each page was sent using a different iovec item. This potentially exceed Linux maximum (1024).
Coalesce adjacent IO vector elements to attempt to reduce the number of overall IO vectors for each operation.
Also some implementation (MiniOS) emulate writev with multiple write calls.
--
Changes since v2:
- change prefix in subject.
Changes since v4:
- added Reviewed-by;
- improved commit message;
- minor style fix.
--- tools/libs/guest/xg_sr_save.c | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-)
tools/libs/guest/xg_sr_save.c +10 -2
@@ -239,13 +239,21 @@ static int write_batch(struct xc_sr_context *ctx) {for ( i =0 ; i < nr_pfns; ++i ) { -if ( guest_data[i] ) +if ( !guest_data[i] ) +continue ; + +if ( iov[iovcnt -1 ].iov_base + iov[iovcnt -1 ].iov_len != + guest_data[i] ) { iov[iovcnt].iov_base = guest_data[i]; iov[iovcnt].iov_len =PAGE_SIZE ; iovcnt++; - --nr_pages; } +else + { + iov[iovcnt -1 ].iov_len +=PAGE_SIZE ; + } + --nr_pages; } }
Signature · 1 line
2.43.0
PATCH v5 4/16 libs/guest: Use a single write_exact in write_headers
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Reduce number of syscalls by coalescing the image and the domain headers into a single I/O vector array.
--
Changes since v2:
- change prefix in subject.
Changes since v4:
- added Reviewed-by;
- improved commit message.
--- tools/libs/guest/xg_sr_save.c | 37 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 19 deletions(-)
tools/libs/guest/xg_sr_save.c +18 -19
@@ -10,17 +10,22 @@ static int write_headers(struct xc_sr_context *ctx, uint16_t guest_type) {xc_interface *xch = ctx->xch;int32_t xen_version =xc_version (xch, XENVER_version,NULL ); -struct xc_sr_ihdr ihdr = { - .marker =IHDR_MARKER , - .id =htonl (IHDR_ID ), - .version =htonl (3 ), - .options =htons (IHDR_OPT_LITTLE_ENDIAN ), - }; -struct xc_sr_dhdr dhdr = { - .type = guest_type, - .page_shift =XC_PAGE_SHIFT , - .xen_major = (xen_version >>16 ) &0xffff , - .xen_minor = (xen_version) &0xffff , +struct { +struct xc_sr_ihdr ihdr; +struct xc_sr_dhdr dhdr; + } hdrs = { + { + .marker =IHDR_MARKER , + .id =htonl (IHDR_ID ), + .version =htonl (3 ), + .options =htons (IHDR_OPT_LITTLE_ENDIAN ), + }, + { + .type = guest_type, + .page_shift =XC_PAGE_SHIFT , + .xen_major = (xen_version >>16 ) &0xffff , + .xen_minor = (xen_version) &0xffff , + }, };if ( xen_version <0 )
@@ -29,15 +34,9 @@ static int write_headers(struct xc_sr_context *ctx, uint16_t guest_type)return -1 ; } -if (write_exact (ctx->fd, &ihdr,sizeof (ihdr)) ) - { -PERROR ("Unable to write Image Header to stream" ); -return -1 ; - } - -if (write_exact (ctx->fd, &dhdr,sizeof (dhdr)) ) +if (write_exact (ctx->fd, &hdrs,sizeof (hdrs)) ) { -PERROR ("Unable to write Domain Header to stream" ); +PERROR ("Unable to write Headers to stream" );return -1 ; }
Signature · 1 line
2.43.0
PATCH v5 5/16 libs/guest: allocate various migration arrays just once
In reply to Frediano ZiglioFrom: Edwin Török <[email protected]>
Allocate these array just once at the start of migration, using the maximum batch size, and free them at the end.
--
Changes since v2:
- change prefix in subject.
Changes since v3:
- fix comment style
Changes since v4:
- change order of fields in structure.
--- tools/libs/guest/xg_sr_common.h | 13 +++++++ tools/libs/guest/xg_sr_save.c | 66 +++++++++++++-------------------- 2 files changed, 39 insertions(+), 40 deletions(-)
tools/libs/guest/xg_sr_common.h +13 -0
@@ -209,6 +209,18 @@ static inline int update_blob(struct xc_sr_blob *blob,return 0 ; } +struct xc_sr_context_save_buffers +{ +xen_pfn_t batch_pfns[MAX_BATCH_SIZE ]; +xen_pfn_t mfns[MAX_BATCH_SIZE ]; +xen_pfn_t types[MAX_BATCH_SIZE ]; +void *guest_data[MAX_BATCH_SIZE ]; +void *local_pages[MAX_BATCH_SIZE ]; +struct iovec iov[MAX_BATCH_SIZE +2 ];/* Headers + data. */ +uint64_t rec_pfns[MAX_BATCH_SIZE ]; +int errors[MAX_BATCH_SIZE ]; +}; +struct xc_sr_context {xc_interface *xch;
@@ -244,6 +256,7 @@ struct xc_sr_contextunsigned long *deferred_pages;unsigned long nr_deferred_pages;xc_hypercall_buffer_t dirty_bitmap_hbuf; +struct xc_sr_context_save_buffers *buffers; } save; struct /* Restore data. */
tools/libs/guest/xg_sr_save.c +26 -40
124 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v5 6/16 libs/call: cache up to 4 pages in hypercall bounce buffers
In reply to Frediano ZiglioFrom: Edwin Török <[email protected]>
During migration there are a lot of mmap/munmap calls, because xc_get_pfn_type_batch exceeds the default hypercall bounce buffer cache size, and needs to allocate every time it is called.
munmap is slow, especially in a PV Dom0 (takes an emulation fault), so is best avoided.
Eventually it'd be good if the memory pool from xmalloc_tlsf.c was reused here, but for now make it handle the commonly encountered sizes (so far up to 4 pages).
--
Changes since v2:
- change prefix in subject.
Changes since v4:
- fix off-by-one bug.
--- tools/libs/call/buffer.c | 28 +++++++++++++++++----------- tools/libs/call/core.c | 3 ++- tools/libs/call/private.h | 8 +++++--- 3 files changed, 24 insertions(+), 15 deletions(-)
tools/libs/call/buffer.c +17 -11
@@ -56,13 +56,13 @@ static void *cache_alloc(xencall_handle *xcall, size_t nr_pages)if ( xcall->buffer_current_allocations > xcall->buffer_maximum_allocations ) xcall->buffer_maximum_allocations = xcall->buffer_current_allocations; -if ( nr_pages >1 ) +if ( nr_pages >ARRAY_SIZE (xcall->buffer_cache) ) { xcall->buffer_cache_toobig++; } -else if ( xcall->buffer_cache_nr >0 ) +else if ( xcall->buffer_cache_nr[nr_pages-1 ] >0 ) { - p = xcall->buffer_cache[--xcall->buffer_cache_nr]; + p = xcall->buffer_cache[nr_pages-1 ][--xcall->buffer_cache_nr[nr_pages-1 ]]; xcall->buffer_cache_hits++; }else
@@ -84,10 +84,10 @@ static int cache_free(xencall_handle *xcall, void *p, size_t nr_pages) xcall->buffer_total_releases++; xcall->buffer_current_allocations--; -if ( nr_pages ==1 && - xcall->buffer_cache_nr <BUFFER_CACHE_SIZE ) +if ( nr_pages && nr_pages <=ARRAY_SIZE (xcall->buffer_cache) && + xcall->buffer_cache_nr[nr_pages-1 ] <BUFFER_CACHE_SIZE ) { - xcall->buffer_cache[xcall->buffer_cache_nr++] = p; + xcall->buffer_cache[nr_pages-1 ][xcall->buffer_cache_nr[nr_pages-1 ]++] = p; rc =1 ; }
@@ -108,17 +108,23 @@ void buffer_release_cache(xencall_handle *xcall)DBGPRINTF ("current allocations:%d maximum allocations:%d" , xcall->buffer_current_allocations, xcall->buffer_maximum_allocations); -DBGPRINTF ("cache current size:%d" , - xcall->buffer_cache_nr); +for (unsigned i =0 ; i <ARRAY_SIZE (xcall->buffer_cache_nr); ++i ) + { +DBGPRINTF ("cache current size[%u pages]:%d" , i+1 , + xcall->buffer_cache_nr[i]); + }DBGPRINTF ("cache hits:%d misses:%d toobig:%d" , xcall->buffer_cache_hits, xcall->buffer_cache_misses, xcall->buffer_cache_toobig); -while ( xcall->buffer_cache_nr >0 ) +for (unsigned i =0 ; i <ARRAY_SIZE (xcall->buffer_cache_nr); ++i ) { - p = xcall->buffer_cache[--xcall->buffer_cache_nr]; -osdep_free_pages (xcall, p,1 ); +while ( xcall->buffer_cache_nr[i] >0 ) + { + p = xcall->buffer_cache[i][--xcall->buffer_cache_nr[i]]; +osdep_free_pages (xcall, p, i +1 ); + } }cache_unlock (xcall);
tools/libs/call/core.c +2 -1
@@ -14,6 +14,7 @@*/ #include <stdlib.h> +#include <string.h> #include "private.h"
@@ -44,7 +45,7 @@ xencall_handle *xencall_open(xentoollog_logger *logger, unsigned open_flags)xentoolcore__register_active_handle (&xcall->tc_ah); xcall->flags = open_flags; - xcall->buffer_cache_nr =0 ; +memset (xcall->buffer_cache_nr,0 ,sizeof (xcall->buffer_cache_nr)); xcall->buffer_total_allocations =0 ; xcall->buffer_total_releases =0 ;
tools/libs/call/private.h +5 -3
@@ -31,13 +31,15 @@ struct xencall_handle {Xentoolcore__Active_Handle tc_ah;/* -* A simple cache of unused, single page, hypercall buffers +* A simple cache of unused, small, hypercall buffers +* buffer_cache[i]'s size is (i+1) pages * * Protected by a global lock. */ #define BUFFER_CACHE_SIZE 4 -int buffer_cache_nr; -void *buffer_cache[BUFFER_CACHE_SIZE ]; +#define BUFFER_CACHE_NRPAGES 4 +int buffer_cache_nr[BUFFER_CACHE_NRPAGES ]; +void *buffer_cache[BUFFER_CACHE_NRPAGES ][BUFFER_CACHE_SIZE ];/* * Hypercall buffer statistics. All protected by the global
Signature · 1 line
2.43.0
PATCH v5 7/16 libs/guest: avoids using 2 indexes
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Simplify code, after the first scan of the various arrays we don't need to keep original types and PFNs but only the ones having data.
--
Changes since v4:
- added Reviewed-by.
--- tools/libs/guest/xg_sr_restore.c | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-)
tools/libs/guest/xg_sr_restore.c +13 -13
@@ -260,9 +260,7 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,int *map_errs =malloc (count *sizeof (*map_errs));int rc;void *mapping =NULL , *guest_page =NULL ; -unsigned int i,/* i indexes the pfns from the record. */ - j,/* j indexes the subset of pfns we decide to map. */ - nr_pages =0 ; +unsigned nr_pages;if ( !mfns || !map_errs ) {
@@ -279,12 +277,18 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, goto err; } -for ( i =0 ; i < count; ++i ) + nr_pages =0 ; +for (unsigned i =0 ; i < count; ++i ) { ctx->restore.ops.set_page_type (ctx, pfns[i], types[i]); -if (page_type_has_stream_data (types[i]) ) - mfns[nr_pages++] = ctx->restore.ops.pfn_to_gfn (ctx, pfns[i]); +if ( !page_type_has_stream_data (types[i]) ) +continue ; + + mfns[nr_pages] = ctx->restore.ops.pfn_to_gfn (ctx, pfns[i]); + pfns[nr_pages] = pfns[i]; + types[nr_pages] = types[i]; + nr_pages++; }/* Nothing to do? */
@@ -302,16 +306,13 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count, goto err; } -for ( i =0 , j =0 ; i < count; ++i ) +for (unsigned i =0 ; i < nr_pages; ++i ) { -if ( !page_type_has_stream_data (types[i]) ) -continue ; - -if ( map_errs[j] ) +if ( map_errs[i] ) { rc =-1 ;ERROR ("Mapping pfn %#" PRIpfn" (mfn %#" PRIpfn", type %#" PRIx32") failed with %d" , - pfns[i], mfns[j], types[i], map_errs[j]); + pfns[i], mfns[i], types[i], map_errs[i]); goto err; }
@@ -337,7 +338,6 @@ static int process_page_data(struct xc_sr_context *ctx, unsigned int count,memcpy (guest_page, page_data,PAGE_SIZE ); } - ++j; guest_page +=PAGE_SIZE ; page_data +=PAGE_SIZE ; }
Signature · 1 line
2.43.0
PATCH v5 8/16 libs/guest: fill directly iov structure
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Instead of storing page pointers into an array and lately adding to iov vector add the pages directly to iov to avoid "guest_data" array.
--- tools/libs/guest/xg_sr_common.h | 1 - tools/libs/guest/xg_sr_save.c | 64 ++++++++++++--------------------- 2 files changed, 23 insertions(+), 42 deletions(-)
tools/libs/guest/xg_sr_common.h +0 -1
@@ -214,7 +214,6 @@ struct xc_sr_context_save_buffersxen_pfn_t batch_pfns[MAX_BATCH_SIZE ];xen_pfn_t mfns[MAX_BATCH_SIZE ];xen_pfn_t types[MAX_BATCH_SIZE ]; -void *guest_data[MAX_BATCH_SIZE ];void *local_pages[MAX_BATCH_SIZE ];struct iovec iov[MAX_BATCH_SIZE +2 ];/* Headers + data. */ uint64_t rec_pfns[MAX_BATCH_SIZE ];
tools/libs/guest/xg_sr_save.c +23 -41
106 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v5 9/16 libs/ctrl: Allows writev_exact to change iov array
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Avoid having to allocate and copy the array if a partial write happens.
The implementation in tools/libs/store/xs.c already use this signature and method.
--
Changes since v2:
- change prefix in subject.
--- tools/libs/ctrl/xc_private.c | 26 +++++--------------------- tools/libs/ctrl/xc_private.h | 2 +- 2 files changed, 6 insertions(+), 22 deletions(-)
tools/libs/ctrl/xc_private.c +5 -21
@@ -635,7 +635,7 @@ int write_exact(int fd, const void *data, size_t size)/* * MiniOS's libc doesn't know about writev(). Implement it as multiple write()s. */ -int writev_exact (int fd,const struct iovec *iov,int iovcnt) +int writev_exact (int fd,struct iovec *iov,int iovcnt) {int rc, i;
@@ -649,9 +649,8 @@ int writev_exact(int fd, const struct iovec *iov, int iovcnt) return 0; }#else -int writev_exact (int fd,const struct iovec *iov,int iovcnt) +int writev_exact (int fd,struct iovec *iov,int iovcnt) { -struct iovec *local_iov =NULL ;int rc =0 , iov_idx =0 , saved_errno =0 ;ssize_t len;
@@ -686,23 +685,9 @@ int writev_exact(int fd, const struct iovec *iov, int iovcnt) len -= iov[iov_idx++].iov_len; else { -/* Partial write of iov[iov_idx]. Copy iov so we can adjust -* element iov_idx and resubmit the rest. */ -if ( !local_iov ) - { - local_iov =malloc (iovcnt *sizeof (*iov)); -if ( !local_iov ) - { - saved_errno =ENOMEM ; - rc =-1 ; - goto out; - } - - iov =memcpy (local_iov, iov, iovcnt *sizeof (*iov)); - } - - local_iov[iov_idx].iov_base += len; - local_iov[iov_idx].iov_len -= len; +/* Partial write of iov[iov_idx]. */ + iov[iov_idx].iov_base += len; + iov[iov_idx].iov_len -= len;break ; } }
@@ -711,7 +696,6 @@ int writev_exact(int fd, const struct iovec *iov, int iovcnt) saved_errno =0 ; out: -free (local_iov); errno = saved_errno;return rc; }
tools/libs/ctrl/xc_private.h +1 -1
@@ -383,7 +383,7 @@ int xc_flush_mmu_updates(xc_interface *xch, struct xc_mmu *mmu);/* Return 0 on success; -1 on error setting errno. */ int read_exact (int fd,void *data,size_t size);/* EOF => -1, errno=0 */ int write_exact (int fd,const void *data,size_t size); -int writev_exact (int fd,const struct iovec *iov,int iovcnt); +int writev_exact (int fd,struct iovec *iov,int iovcnt);int xc_ffs8 (uint8_t x);int xc_ffs16 (uint16_t x);
Signature · 1 line
2.43.0
PATCH v5 10/16 libs/guest: add xg_foreignmemory_copy_{from,to}
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
This change prepare code to use a new "foreign copy" hypercall. The new hypercall will copy memory from/to a foreign domain. The new hypercall can be emulated with a sequence of:
- map foreign memory;
- copy memory;
- unmap foreign memory.
--- tools/libs/guest/xg_sr_common.c | 60 +++++++++++++++++++++++++++++++++ tools/libs/guest/xg_sr_common.h | 8 +++++ 2 files changed, 68 insertions(+)
tools/libs/guest/xg_sr_common.c +60 -0
@@ -156,6 +156,66 @@ static void __attribute__((unused)) build_assertions(void)BUILD_BUG_ON (sizeof (struct xc_sr_rec_hvm_params ) !=8 ); } +enum { + foreigncopy_from, + foreigncopy_to +}; + +static int xg_foreignmemory_copy (xc_interface *xch,domid_t domid, +int dir,size_t nr_pages,void *buffer, +const xen_pfn_t foreign_pfns[nr_pages]) +{ +if ( nr_pages ==0 ) +return 0 ; + +if ( !buffer || !foreign_pfns ) + { + errno =EINVAL ; +return -1 ; + } + +int err[nr_pages]; +const int prot = (dir == foreigncopy_from) ?PROT_READ :PROT_READ |PROT_WRITE ; + +void *p =xenforeignmemory_map (xch->fmem, domid, prot, nr_pages, foreign_pfns, err); +if ( !p ) + { + errno =EINVAL ; +return -1 ; + } + +for (size_t n =0 ; n < nr_pages; ++n ) +if ( err[n] ) + { +xenforeignmemory_unmap (xch->fmem, p, nr_pages); + errno = -err[n]; +return -1 ; + } + +if ( dir == foreigncopy_from ) +memcpy (buffer, p, nr_pages *XC_PAGE_SIZE ); +else +memcpy (p, buffer, nr_pages *XC_PAGE_SIZE ); + +return xenforeignmemory_unmap (xch->fmem, p, nr_pages); +} + +int xg_foreignmemory_copy_from (xc_interface *xch,domid_t dom, +size_t nr_pages,void *dest, +const xen_pfn_t source[nr_pages]) +{ +return xg_foreignmemory_copy (xch, dom, foreigncopy_from, + nr_pages, dest, source); +} + +int xg_foreignmemory_copy_to (xc_interface *xch,domid_t dom, +size_t nr_pages,const xen_pfn_t dest[nr_pages], +const void *source) +{ +return xg_foreignmemory_copy (xch, dom, foreigncopy_to, + nr_pages, (void *) source, dest); +} +/* * Local variables: * mode: C
tools/libs/guest/xg_sr_common.h +8 -0
@@ -553,6 +553,14 @@ static inline bool page_type_has_stream_data(uint32_t type) } } +int xg_foreignmemory_copy_from (xc_interface *xch,domid_t dom, +size_t nr_pages,void *dest, +const xen_pfn_t source[nr_pages]); + +int xg_foreignmemory_copy_to (xc_interface *xch,domid_t dom, +size_t nr_pages,const xen_pfn_t dest[nr_pages], +const void *source); +#endif /* * Local variables:
Signature · 1 line
2.43.0
PATCH v5 11/16 PoC: libs/guest: use foreign copy during migration
In reply to Frediano ZiglioFrom: Edwin Török <[email protected]>
ministat confirms the improvement:
```
x baseline
+ foreigncopy
N Min Max Median Avg Stddev
x 20 1.1306997 1.1447931 1.1356569 1.1365742 0.003242175
+ 20 0.4311504 0.44180303 0.43616705 0.43600089 0.0031094689
Difference at 95.0% confidence
-0.700573 +/- 0.00203311
-61.639% +/- 0.133355%
(Student's t, pooled s = 0.00317652)
```The tests pass too, which means that it has correctly migrated all guest memory.
Frediano: This PoC was adapted to be included in a final series.
--- tools/libs/guest/xg_sr_common.h | 1 + tools/libs/guest/xg_sr_restore.c | 43 +++-------------- tools/libs/guest/xg_sr_save.c | 82 +++++++++----------------------- 3 files changed, 31 insertions(+), 95 deletions(-)
tools/libs/guest/xg_sr_common.h +1 -0
@@ -256,6 +256,7 @@ struct xc_sr_contextunsigned long nr_deferred_pages;xc_hypercall_buffer_t dirty_bitmap_hbuf;struct xc_sr_context_save_buffers *buffers; +void *dest_buf; } save; struct /* Restore data. */
tools/libs/guest/xg_sr_restore.c +7 -36
71 lines not on this page · raw message
tools/libs/guest/xg_sr_save.c +23 -59
123 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v5 12/16 xen: implement new foreign copy hypercall
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Add a sub hypercall to __HYPERVISOR_memory_op to allow to read/write memory from/to a foreign domain.
--
Changes since v4:
- Fix typo in comment;
- removed from check.
--- xen/common/memory.c | 134 ++++++++++++++++++++++++++++++++++++ xen/include/public/memory.h | 40 ++++++++++- 2 files changed, 173 insertions(+), 1 deletion(-)
xen/common/memory.c +134 -0
@@ -1545,6 +1545,133 @@ static int acquire_resource(return rc; } +/* +* The "noinline" qualifier avoids the compiler to create a large function +* consuming quite a lot of stack. +*/ +static int noinline mem_foreigncopy( +XEN_GUEST_HANDLE_PARAM (xen_foreigncopy_t) arg) +{ +struct domain *d, *const currd = current->domain; +xen_foreigncopy_t copy; +int rc, direction; + +if (copy_from_guest (©, arg,1 ) ) +return -EFAULT ; + +if ( copy.flags & ~1U ) +return -EINVAL ; + + direction = copy.flags & XENMEM_foreigncopy_direction; + +if ( copy.nr_frames ==0 ) +return 0 ; + + rc =rcu_lock_remote_domain_by_id (copy.domid, &d); +if ( rc ) +return rc; + +/* +* Check we are allowed to map and access these foreign pages. +*/ + rc =xsm_map_gmfn_foreign (XSM_TARGET , currd, d); +if ( rc ) + goto out; + +do { +/* +* Arbitrary size. Not too much stack space, and a reasonable stride +* for continuation checks. +*/ +xen_pfn_t gfn_list[32 ]; +unsigned int todo =MIN (ARRAY_SIZE (gfn_list), copy.nr_frames); + + rc = -EFAULT ; +if (copy_from_guest (gfn_list, copy.frame_list, todo) ) + goto out; + +for (unsigned i =0 ; i < todo; i++ ) + { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 +p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif + + foreign_page =get_page_from_gfn (d, gfn_list[i], &p2mt,P2M_ALLOC ); + +if (unlikely (p2m_to_mask (p2mt) & valid_mask) && foreign_page ) + { +put_page (foreign_page); + foreign_page =NULL ; + } +if (unlikely (!foreign_page) ) + { +gdprintk (XENLOG_WARNING , +"Error accessing foreign gfn %" PRI_gfn"\n" , + gfn_list[i]); + rc = -EINVAL ; + copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } + +/* A page is dirtied when it's being copied to. */ +if ( direction == XENMEM_foreigncopy_to ) +paging_mark_dirty (d,page_to_mfn (foreign_page)); + + foreign =map_domain_page (page_to_mfn (foreign_page)); +if ( direction == XENMEM_foreigncopy_from ) + rc =copy_to_guest (copy.buffer, foreign,PAGE_SIZE ); +else + rc =copy_from_guest (foreign, copy.buffer,PAGE_SIZE ); +unmap_domain_page (foreign); +put_page (foreign_page); + +if (unlikely (rc) ) + { +gdprintk (XENLOG_WARNING , +"Error %d copying gfn %" PRI_gfn"\n" , + -rc, gfn_list[i]); + copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } + +guest_handle_add_offset (copy.buffer,PAGE_SIZE ); + } + + copy.nr_frames -= todo; +guest_handle_add_offset (copy.frame_list, todo); + +if ( copy.nr_frames &&hypercall_preempt_check () ) + { + rc =hypercall_create_continuation ( + __HYPERVISOR_memory_op,"lh" , XENMEM_foreigncopy, arg); + goto out; + } + }while ( copy.nr_frames ); + + rc =0 ; + + out: +rcu_unlock_domain (d); + +/* +* Update in all cases, it allows the caller to know how many +* frames were successfully copied and the continuation to +* continue correctly. +*/ +if (__copy_to_guest (arg, ©,1 ) ) + rc = -EFAULT ; + +return rc; +} +long do_memory_op (unsigned long cmd,XEN_GUEST_HANDLE_PARAM (void ) arg) {struct domain *d, *curr_d = current->domain;
@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ; +case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ; + + rc =mem_foreigncopy (guest_handle_cast (arg, xen_foreigncopy_t)); +break ; +default : rc =arch_memory_op (cmd, arg);break ;
xen/include/public/memory.h +39 -1
@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ +domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* +* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames; + +/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer; +}; +typedef struct xen_foreigncopy xen_foreigncopy_t ; +DEFINE_XEN_GUEST_HANDLE (xen_foreigncopy_t); + +/* Next available subop number is 30 */ #endif /* __XEN_PUBLIC_MEMORY_H__ */
Signature · 1 line
2.43.0
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 21 lines · Frediano Ziglio
xen/common/memory.c +15
-0@@ -1545,6 +1545,133 @@ static int acquire_resource(return rc; } +/* +* The "noinline" qualifier avoids the compiler to create a large function +* consuming quite a lot of stack. +*/ +static int noinline mem_foreigncopy( +XEN_GUEST_HANDLE_PARAM (xen_foreigncopy_t) arg) +{ +struct domain *d, *const currd = current->domain; +xen_foreigncopy_t copy; +int rc, direction; + +if (copy_from_guest (©, arg,1 ) ) +return -EFAULT ; + +if ( copy.flags & ~1U )
xen/common/memory.c static int acquire_resource( · line 1572
Quoted 10 lines
xen/common/memory.c +10
-0+return -EINVAL ; + + direction = copy.flags & XENMEM_foreigncopy_direction; + +if ( copy.nr_frames ==0 ) +return 0 ; + + rc =rcu_lock_remote_domain_by_id (copy.domid, &d); +if ( rc ) +return rc;
Extending my v4 remark: How about a "fully foreign" copy? I.e. one between two pages in two foreign domains? Getting me back also to the un-answered v4 question of mine as to MMUEXT_COPY_PAGE.
Further, as to the order of checks: I'm not going to insist on re-ordering, yet I'd like to point out that copying 0 pages to/from a bad domid might better yield an error.
xen/common/memory.c static int acquire_resource( · line 1579
Quoted 6 lines
xen/common/memory.c +6
-0+/* +* Check we are allowed to map and access these foreign pages. +*/ + rc =xsm_map_gmfn_foreign (XSM_TARGET , currd, d); +if ( rc ) + goto out;
As per
K: xsm_.*
K: \b(xsm|XSM)\b
in ./MAINTAINERS please Cc the XSM/Flask maintainer for such changes. I for one question the re-use of an existing predicate here.
xen/common/memory.c static int acquire_resource( · line 1593
Quoted 13 lines
xen/common/memory.c +13
-0+ do { + /* + * Arbitrary size. Not too much stack space, and a reasonable stride + * for continuation checks. + */ + xen_pfn_t gfn_list[32]; + unsigned int todo = MIN(ARRAY_SIZE(gfn_list), copy.nr_frames); + + rc = -EFAULT; + if ( copy_from_guest(gfn_list, copy.frame_list, todo) ) + goto out; + + for ( unsigned i = 0; i < todo; i++ )
Nit: "unsigned int" please (like you have it above).
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 10 lines
xen/common/memory.c +10
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 +p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif
What about, for example, p2m_ram_ro? Or p2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.
xen/common/memory.c static int acquire_resource( · line 1607
Quoted 3 lines
xen/common/memory.c +3
-0+ foreign_page =get_page_from_gfn (d, gfn_list[i], &p2mt,P2M_ALLOC ); + +if (unlikely (p2m_to_mask (p2mt) & valid_mask) && foreign_page )
Was this meant to be
if ( unlikely(!(p2m_to_mask(p2mt) & valid_mask)) && foreign_page )
?
Quoted 10 lines
xen/common/memory.c +10
-0+ { +put_page (foreign_page); + foreign_page =NULL ; + } +if (unlikely (!foreign_page) ) + { +gdprintk (XENLOG_WARNING , +"Error accessing foreign gfn %" PRI_gfn"\n" , + gfn_list[i]); + rc = -EINVAL ;
xen/common/memory.c static int acquire_resource( · line 1627
Quoted 10 lines
xen/common/memory.c +10
-0+ copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } + +/* A page is dirtied when it's being copied to. */ +if ( direction == XENMEM_foreigncopy_to ) +paging_mark_dirty (d,page_to_mfn (foreign_page)); + + foreign =map_domain_page (page_to_mfn (foreign_page));
Latch page_to_mfn(foreign_page) into a local variable to avoid the redundant translation (which the compiler may not be able to fold)?
Quoted 33 lines
xen/common/memory.c +33
-0+if ( direction == XENMEM_foreigncopy_from ) + rc =copy_to_guest (copy.buffer, foreign,PAGE_SIZE ); +else + rc =copy_from_guest (foreign, copy.buffer,PAGE_SIZE ); +unmap_domain_page (foreign); +put_page (foreign_page); + +if (unlikely (rc) ) + { +gdprintk (XENLOG_WARNING , +"Error %d copying gfn %" PRI_gfn"\n" , + -rc, gfn_list[i]); + copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } + +guest_handle_add_offset (copy.buffer,PAGE_SIZE ); + } + + copy.nr_frames -= todo; +guest_handle_add_offset (copy.frame_list, todo); + +if ( copy.nr_frames &&hypercall_preempt_check () ) + { + rc =hypercall_create_continuation ( + __HYPERVISOR_memory_op,"lh" , XENMEM_foreigncopy, arg); + goto out; + } + }while ( copy.nr_frames ); + + rc =0 ; +
xen/common/memory.c static int acquire_resource( · line 1670
Quoted 10 lines
xen/common/memory.c +10
-0+ out: +rcu_unlock_domain (d); + +/* +* Update in all cases, it allows the caller to know how many +* frames were successfully copied and the continuation to +* continue correctly. +*/ +if (__copy_to_guest (arg, ©,1 ) ) + rc = -EFAULT ;
It's not quite clear whether replacing an earlier non-zero value held in rc is a good thing here. The only time when it's strictly necessary is when you schedule a continuation. I'd suggest to add "&& rc >= 0".
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 7 lines
xen/common/memory.c +3
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ; +case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;
Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */
There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 4 lines
xen/include/public/memory.h +4
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 10 lines
xen/include/public/memory.h +10
-0+#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* +* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;
This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 776
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;
With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
Jan
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 29 lines · Jan Beulich
Quoted 27 lines · Frediano Ziglio
xen/common/memory.c +21
-0@@ -1545,6 +1545,133 @@ static int acquire_resource(return rc; } +/* +* The "noinline" qualifier avoids the compiler to create a large function +* consuming quite a lot of stack. +*/ +static int noinline mem_foreigncopy( +XEN_GUEST_HANDLE_PARAM (xen_foreigncopy_t) arg) +{ +struct domain *d, *const currd = current->domain; +xen_foreigncopy_t copy; +int rc, direction; + +if (copy_from_guest (©, arg,1 ) ) +return -EFAULT ; + +if ( copy.flags & ~1U ) +return -EINVAL ; + + direction = copy.flags & XENMEM_foreigncopy_direction; + +if ( copy.nr_frames ==0 ) +return 0 ;
Quoted 9 lines
xen/common/memory.c static int acquire_resource( · line 1572
Quoted 4 lines
xen/common/memory.c +4
-0+ + rc =rcu_lock_remote_domain_by_id (copy.domid, &d); +if ( rc ) +return rc;Extending my v4 remark: How about a "fully foreign" copy? I.e. one between two pages in two foreign domains? Getting me back also to the un-answered v4 question of mine as to
MMUEXT_COPY_PAGE.
I really thought I replied to this but I cannot find it. MMUEXT_COPY_PAGE was the first attempt to do this but mmuext is designed for PV and extending was made the code confusing.
Quoted 4 lines
Further, as to the order of checks: I'm not going to insist on re-ordering, yet I'd like to point out that copying 0 pages to/from a bad domid might better yield an error.
Not strong about this, changed to return -EINVAL. Reordering after this change won't make much sense, -EINVAL is returned both for wrong flags or no frames.
Quoted 7 lines
xen/common/memory.c static int acquire_resource( · line 1579
Quoted 6 lines
xen/common/memory.c +6
-0+/* +* Check we are allowed to map and access these foreign pages. +*/ + rc =xsm_map_gmfn_foreign (XSM_TARGET , currd, d); +if ( rc ) + goto out;
Quoted 8 lines
As per
K: xsm_.* K: \b(xsm|XSM)\bin ./MAINTAINERS please Cc the XSM/Flask maintainer for such changes. I for one question the re-use of an existing predicate here.
Added for the next series.
Quoted 7 lines
Quoted 7 lines
xen/common/memory.c +7
-0+ do { + /* + * Arbitrary size. Not too much stack space, and a reasonable stride + * for continuation checks. + */ + xen_pfn_t gfn_list[32]; + unsigned int todo = MIN(ARRAY_SIZE(gfn_list), copy.nr_frames);
Quoted 9 lines
xen/common/memory.c static int acquire_resource( · line 1593
Quoted 6 lines
xen/common/memory.c +6
-0+ + rc = -EFAULT ; +if (copy_from_guest (gfn_list, copy.frame_list, todo) ) + goto out; + +for (unsigned i =0 ; i < todo; i++ )Nit: "unsigned int" please (like you have it above).
Changed. Note that "long" is also not a type but a modifier. Would it be good to change both the above to "unsigned" instead for consistency with "long" ?
Quoted 6 lines
Quoted 6 lines
xen/common/memory.c +6
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; +const unsigned long valid_mask = +#ifdef CONFIG_X86
Quoted 9 lines
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 4 lines
xen/common/memory.c +4
-0+p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif What about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.
The base of this was taken from migration code where there is such a check. I suppose that adding p2m_ram_ro (where available) won't hurt. p2m_ram_shared I'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.
Quoted 10 lines
xen/common/memory.c static int acquire_resource( · line 1607
Quoted 3 lines
xen/common/memory.c +3
-0+ foreign_page =get_page_from_gfn (d, gfn_list[i], &p2mt,P2M_ALLOC ); + +if (unlikely (p2m_to_mask (p2mt) & valid_mask) && foreign_page )Was this meant to be
if ( unlikely(!(p2m_to_mask(p2mt) & valid_mask)) && foreign_page )?
Yes, fixed.
Quoted 15 lines
Quoted 15 lines
xen/common/memory.c +15
-0+ { +put_page (foreign_page); + foreign_page =NULL ; + } +if (unlikely (!foreign_page) ) + { +gdprintk (XENLOG_WARNING , +"Error accessing foreign gfn %" PRI_gfn"\n" , + gfn_list[i]); + rc = -EINVAL ; + copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } +
Quoted 9 lines
xen/common/memory.c static int acquire_resource( · line 1627
Quoted 5 lines
xen/common/memory.c +5
-0+/* A page is dirtied when it's being copied to. */ +if ( direction == XENMEM_foreigncopy_to ) +paging_mark_dirty (d,page_to_mfn (foreign_page)); + + foreign =map_domain_page (page_to_mfn (foreign_page));Latch
page_to_mfn(foreign_page)into a local variable to avoid the redundant translation (which the compiler may not be able to fold)?
Sure.
Quoted 39 lines
Quoted 39 lines
xen/common/memory.c +39
-0+if ( direction == XENMEM_foreigncopy_from ) + rc =copy_to_guest (copy.buffer, foreign,PAGE_SIZE ); +else + rc =copy_from_guest (foreign, copy.buffer,PAGE_SIZE ); +unmap_domain_page (foreign); +put_page (foreign_page); + +if (unlikely (rc) ) + { +gdprintk (XENLOG_WARNING , +"Error %d copying gfn %" PRI_gfn"\n" , + -rc, gfn_list[i]); + copy.nr_frames -= i; +guest_handle_add_offset (copy.frame_list, i); + goto out; + } + +guest_handle_add_offset (copy.buffer,PAGE_SIZE ); + } + + copy.nr_frames -= todo; +guest_handle_add_offset (copy.frame_list, todo); + +if ( copy.nr_frames &&hypercall_preempt_check () ) + { + rc =hypercall_create_continuation ( + __HYPERVISOR_memory_op,"lh" , XENMEM_foreigncopy, arg); + goto out; + } + }while ( copy.nr_frames ); + + rc =0 ; + + out: +rcu_unlock_domain (d); + +/* +* Update in all cases, it allows the caller to know how many +* frames were successfully copied and the continuation to
Quoted 9 lines
xen/common/memory.c static int acquire_resource( · line 1670
Quoted 4 lines
xen/common/memory.c +4
-0+* continue correctly. +*/ +if (__copy_to_guest (arg, ©,1 ) ) + rc = -EFAULT ;It's not quite clear whether replacing an earlier non-zero value held in rc is a good thing here. The only time when it's strictly necessary is when you schedule a continuation. I'd suggest to add "&& rc >= 0".
Added. Looking at the possible client usage it could think that the error happened copying the first page but otherwise the copy should not fail as the opposite direction was fine.
Quoted 4 lines
Quoted 4 lines
xen/common/memory.c +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ;
Quoted 8 lines
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
xen/common/memory.c +3
-0+case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limiting nr_frames at (UINT_MAX >> MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "start_entent" won't allow this.
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +*
Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#define XENMEM_foreigncopy 29
struct xen_foreigncopy {
/* IN - The domain whose memory is to be copied. */
domid_t domid;
/* IN - Flags. */
#define XENMEM_foreigncopy_from 0
#define XENMEM_foreigncopy_to 1
#define XENMEM_foreigncopy_direction 1
uint16_t flags;
/*
* IN/OUT
*
* As an IN parameter number of frames of the domain to be copied.
* On output on error updated number of frames left.
*/
uint32_t nr_frames;
/*
* IN/OUT
*
* Frames to be copied.
* On output on error updated to point to first frame unhandled.
*/
XEN_GUEST_HANDLE(xen_pfn_t) frame_list;
/*
* IN/OUT
*
* Userspace buffer to read/write from.
*/
XEN_GUEST_HANDLE(uint8) buffer;
};typedef struct xen_foreigncopy xen_foreigncopy_t;DEFINE_XEN_GUEST_HANDLE(xen_foreigncopy_t);
Quoted 14 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 776
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;
Quoted 3 lines
With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
Quoted 1 line
Jan
Frediano
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 38 lines · Frediano Ziglio
Quoted 28 lines · Jan Beulich
Quoted 27 lines · Frediano Ziglio
xen/common/memory.c +21
-0@@ -1545,6 +1545,133 @@ static int acquire_resource(return rc; } +/* +* The "noinline" qualifier avoids the compiler to create a large function +* consuming quite a lot of stack. +*/ +static int noinline mem_foreigncopy( +XEN_GUEST_HANDLE_PARAM (xen_foreigncopy_t) arg) +{ +struct domain *d, *const currd = current->domain; +xen_foreigncopy_t copy; +int rc, direction; + +if (copy_from_guest (©, arg,1 ) ) +return -EFAULT ; + +if ( copy.flags & ~1U ) +return -EINVAL ; + + direction = copy.flags & XENMEM_foreigncopy_direction; + +if ( copy.nr_frames ==0 ) +return 0 ;Quoted 8 lines
xen/common/memory.c static int acquire_resource( · line 1572
Quoted 4 lines
xen/common/memory.c +4
-0+ + rc =rcu_lock_remote_domain_by_id (copy.domid, &d); +if ( rc ) +return rc;Extending my v4 remark: How about a "fully foreign" copy? I.e. one between two pages in two foreign domains? Getting me back also to the un-answered v4 question of mine as to
MMUEXT_COPY_PAGE.
Quoted 3 lines
I really thought I replied to this but I cannot find it.
MMUEXT_COPY_PAGEwas the first attempt to do this but mmuext is designed for PV and extending was made the code confusing.
We discussed this on the x86 call, and yes - the major op being PV-only is getting in the way here.
Quoted 7 lines
Quoted 3 lines
Further, as to the order of checks: I'm not going to insist on re-ordering, yet I'd like to point out that copying 0 pages to/from a bad domid might better yield an error.
Not strong about this, changed to return
-EINVAL. Reordering after this change won't make much sense,-EINVALis returned both for wrong flags or no frames.
Please don't - copying 0 frames with all other arguments correct is not an error.
Quoted 16 lines
Quoted 7 lines
Quoted 7 lines
xen/common/memory.c +7
-0+ do { + /* + * Arbitrary size. Not too much stack space, and a reasonable stride + * for continuation checks. + */ + xen_pfn_t gfn_list[32]; + unsigned int todo = MIN(ARRAY_SIZE(gfn_list), copy.nr_frames);Quoted 8 lines
xen/common/memory.c static int acquire_resource( · line 1593
Quoted 6 lines
xen/common/memory.c +6
-0+ + rc = -EFAULT ; +if (copy_from_guest (gfn_list, copy.frame_list, todo) ) + goto out; + +for (unsigned i =0 ; i < todo; i++ )Nit: "unsigned int" please (like you have it above).
Quoted 3 lines
Changed. Note that "long" is also not a type but a modifier. Would it be good to change both the above to "unsigned" instead for consistency with "long" ?
No, the way we spell types has historical reasons and isn't tied to the exact lexical meaning of the keywords.
Quoted 15 lines
Quoted 6 lines
Quoted 6 lines
xen/common/memory.c +6
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; +const unsigned long valid_mask = +#ifdef CONFIG_X86Quoted 8 lines
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 4 lines
xen/common/memory.c +4
-0+p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif What about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.
Quoted 2 lines
The base of this was taken from migration code where there is such a check. I suppose that adding
p2m_ram_ro(where available) won't hurt.
Just to mention, to avoid another round trip just because of this: p2m_ram_ro has different meaning on x86 vs Arm/RISC-V.
Quoted 6 lines
p2m_ram_sharedI'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.
It could copy the contents of the grant mapped page by other means. Why not allow it in this new sub-op as well then?
Talking of security: When the page you copy to is owned by a PV guest, I think you further need to obtain a PGT_writable type ref. (Of course it then likely is easier to always do this, not just for PV.)
Quoted 21 lines
Quoted 4 lines
Quoted 4 lines
xen/common/memory.c +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ;Quoted 8 lines
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
xen/common/memory.c +3
-0+case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limitingnr_framesat (UINT_MAX>>MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/
Quoted 3 lines
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "
start_entent" won't allow this.
You'll want to mention this in the description and/or a code comment. It wants to become clear that the inconsistency in behavior (with other sub- ops) is deliberate rather than accidental.
Quoted 54 lines
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#defineXENMEM_foreigncopy29struct xen_foreigncopy { /* IN - The domain whose memory is to be copied. */ domid_t domid; /* IN - Flags. */ #define XENMEM_foreigncopy_from 0 #define XENMEM_foreigncopy_to 1 #define XENMEM_foreigncopy_direction 1 uint16_t flags;
Quoted 7 lines
/*
* IN/OUT
*
* As an IN parameter number of frames of the domain to be copied.
* On output on error updated number of frames left.
*/
uint32_t nr_frames;This is updated not only on error, but also when encoding continuations.
Quoted 14 lines
Quoted 14 lines
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;
Quoted 6 lines
Quoted 2 lines
With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
How do you know? Why shouldn't e.g. XTF be permitted to test this in all possible modes? And even if all arguments end up in favor of "no compat support", this then wants spelling out to make clear this wasn't an oversight, but rather a conscious decision.
Jan
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 40 lines · Jan Beulich
Quoted 38 lines · Frediano Ziglio
Quoted 36 lines · Jan Beulich
Quoted 21 lines · Frediano Ziglio
xen/common/memory.c +15
-0@@ -1545,6 +1545,133 @@ static int acquire_resource(return rc; } +/* +* The "noinline" qualifier avoids the compiler to create a large function +* consuming quite a lot of stack. +*/ +static int noinline mem_foreigncopy( +XEN_GUEST_HANDLE_PARAM (xen_foreigncopy_t) arg) +{ +struct domain *d, *const currd = current->domain; +xen_foreigncopy_t copy; +int rc, direction; + +if (copy_from_guest (©, arg,1 ) ) +return -EFAULT ; + +if ( copy.flags & ~1U )xen/common/memory.c static int acquire_resource( · line 1572
Quoted 10 lines
xen/common/memory.c +10
-0+return -EINVAL ; + + direction = copy.flags & XENMEM_foreigncopy_direction; + +if ( copy.nr_frames ==0 ) +return 0 ; + + rc =rcu_lock_remote_domain_by_id (copy.domid, &d); +if ( rc ) +return rc;Extending my v4 remark: How about a "fully foreign" copy? I.e. one between two pages in two foreign domains? Getting me back also to the un-answered v4 question of mine as to
MMUEXT_COPY_PAGE.
Quoted 7 lines
Quoted 3 lines
I really thought I replied to this but I cannot find it.
MMUEXT_COPY_PAGEwas the first attempt to do this but mmuext is designed for PV and extending was made the code confusing.We discussed this on the x86 call, and yes - the major op being PV-only is getting in the way here.
Added a comment in the commit message:
Extending MMUEXT_COPY_PAGE seems better on first sight but considering that MMUEXT is meant for PV only and trying to change that sub-op this solution is better.
Quoted 11 lines
Quoted 7 lines
Quoted 3 lines
Further, as to the order of checks: I'm not going to insist on re-ordering, yet I'd like to point out that copying 0 pages to/from a bad domid might better yield an error.
Not strong about this, changed to return
-EINVAL. Reordering after this change won't make much sense,-EINVALis returned both for wrong flags or no frames.Please don't - copying 0 frames with all other arguments correct is not an error.
This comment seems to contradict the not insisting on re-ordering. Changed the order so to check the domid before nr_frames == 0 :
rc = rcu_lock_remote_domain_by_id(copy.domid, &d);
if ( rc )
return rc;
if ( copy.nr_frames == 0 )
{
rcu_unlock_domain(d);
return 0;
}Quoted 38 lines
Quoted 16 lines
Quoted 7 lines
Quoted 7 lines
xen/common/memory.c +7
-0+ do { + /* + * Arbitrary size. Not too much stack space, and a reasonable stride + * for continuation checks. + */ + xen_pfn_t gfn_list[32]; + unsigned int todo = MIN(ARRAY_SIZE(gfn_list), copy.nr_frames);Quoted 8 lines
xen/common/memory.c static int acquire_resource( · line 1593
Quoted 6 lines
xen/common/memory.c +6
-0+ + rc = -EFAULT ; +if (copy_from_guest (gfn_list, copy.frame_list, todo) ) + goto out; + +for (unsigned i =0 ; i < todo; i++ )Nit: "unsigned int" please (like you have it above).
Quoted 3 lines
Changed. Note that "long" is also not a type but a modifier. Would it be good to change both the above to "unsigned" instead for consistency with "long" ?
No, the way we spell types has historical reasons and isn't tied to the exact lexical meaning of the keywords.
Quoted 15 lines
Quoted 14 lines
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 10 lines
xen/common/memory.c +10
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 +p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif What about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.
Quoted 6 lines
Quoted 2 lines
The base of this was taken from migration code where there is such a check. I suppose that adding
p2m_ram_ro(where available) won't hurt.Just to mention, to avoid another round trip just because of this:
p2m_ram_rohas different meaning on x86 vs Arm/RISC-V.
That's confusing... should not this be fixed somehow? It won't save much from a round-trip. Should I allow it or not ?
Quoted 10 lines
Quoted 6 lines
p2m_ram_sharedI'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.It could copy the contents of the grant mapped page by other means. Why not allow it in this new sub-op as well then?
I'm more afraid of writing the content of the grant pages than copying it.
Quoted 4 lines
Talking of security: When the page you copy to is owned by a PV guest, I think you further need to obtain a
PGT_writabletype ref. (Of course it then likely is easier to always do this, not just for PV.)
Wondering how save/restore (or migration) works in this case.
Quoted 21 lines
Quoted 21 lines
Quoted 4 lines
Quoted 4 lines
xen/common/memory.c +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ;Quoted 8 lines
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
xen/common/memory.c +3
-0+case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limitingnr_framesat (UINT_MAX>>MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/
Quoted 8 lines
Quoted 3 lines
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "
start_entent" won't allow this.You'll want to mention this in the description and/or a code comment. It wants to become clear that the inconsistency in behavior (with other sub- ops) is deliberate rather than accidental.
Added a comment in the code:
if ( copy.nr_frames && hypercall_preempt_check() )
{
/*
* Instead of using "start_extent" we update the structure back,
* we update it back in anyway to tell caller were the copy
* stopped.
*/
rc = hypercall_create_continuation(
__HYPERVISOR_memory_op, "lh", XENMEM_foreigncopy, arg);
goto out;
}Quoted 62 lines
Quoted 54 lines
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#defineXENMEM_foreigncopy29struct xen_foreigncopy { /* IN - The domain whose memory is to be copied. */ domid_t domid; /* IN - Flags. */ #define XENMEM_foreigncopy_from 0 #define XENMEM_foreigncopy_to 1 #define XENMEM_foreigncopy_direction 1 uint16_t flags;Quoted 7 lines
/* * IN/OUT * * As an IN parameter number of frames of the domain to be copied. * On output on error updated number of frames left. */ uint32_t nr_frames;
Quoted 2 lines
This is updated not only on error, but also when encoding continuations.
Yes, but this seems more an implementation detail to me. I don't think the caller cares about how the continuation is implemented.
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 776
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
Quoted 9 lines
Quoted 3 lines
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
How do you know? Why shouldn't e.g. XTF be permitted to test this in all possible modes? And even if all arguments end up in favor of "no compat support", this then wants spelling out to make clear this wasn't an oversight, but rather a conscious decision.
XTF can do something like
#if COMPAT_GUEST
/* Compat guests are not supported, return success. */
return 0;
# endif(or can be done in the Makefiles I suppose).
Added a comment in memory.h:
/*
* Copy memory from/to a given domain.
* As this call requires target access and guest with target access won't be
* compat guests supported for compat guests this is not implemented.
*/
#define XENMEM_foreigncopy 29
struct xen_foreigncopy {Quoted 1 line
Jan
Frediano
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 19 lines · Frediano Ziglio
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Frediano Ziglio
Quoted 15 lines · Jan Beulich
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 10 lines · Frediano Ziglio
xen/common/memory.c +10
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 +p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif What about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.
Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
The base of this was taken from migration code where there is such a check. I suppose that adding
p2m_ram_ro(where available) won't hurt.Just to mention, to avoid another round trip just because of this:
p2m_ram_rohas different meaning on x86 vs Arm/RISC-V.That's confusing... should not this be fixed somehow? It won't save much from a round-trip. Should I allow it or not ?
Ask the Arm maintainers. I raised this issue more than once, without any real success.
Quoted 11 lines
Quoted 9 lines
Quoted 6 lines
p2m_ram_sharedI'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.It could copy the contents of the grant mapped page by other means. Why not allow it in this new sub-op as well then?
I'm more afraid of writing the content of the grant pages than copying it.
But the same is true for writing to the granted page: The domain could do so by other means.
Quoted 5 lines
Quoted 3 lines
Talking of security: When the page you copy to is owned by a PV guest, I think you further need to obtain a
PGT_writabletype ref. (Of course it then likely is easier to always do this, not just for PV.)Wondering how save/restore (or migration) works in this case.
Save is not relevant here. Restore happens before the guest gains control. Which is entirely different from hypercall handling.
Quoted 32 lines
Quoted 21 lines
Quoted 21 lines
Quoted 4 lines
Quoted 4 lines
xen/common/memory.c +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ;Quoted 8 lines
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
xen/common/memory.c +3
-0+case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limitingnr_framesat (UINT_MAX>>MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/Quoted 8 lines
Quoted 3 lines
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "
start_entent" won't allow this.You'll want to mention this in the description and/or a code comment. It wants to become clear that the inconsistency in behavior (with other sub- ops) is deliberate rather than accidental.
Added a comment in the code:
Quoted 11 lines
if ( copy.nr_frames && hypercall_preempt_check() )
{
/*
* Instead of using "start_extent" we update the structure back,
* we update it back in anyway to tell caller were the copy
* stopped.
*/
rc = hypercall_create_continuation(
__HYPERVISOR_memory_op, "lh", XENMEM_foreigncopy, arg);
goto out;
}Please can this go into the hunk that I commented on?
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#defineXENMEM_foreigncopy29struct xen_foreigncopy { /* IN - The domain whose memory is to be copied. */ domid_t domid; /* IN - Flags. */ #define XENMEM_foreigncopy_from 0 #define XENMEM_foreigncopy_to 1 #define XENMEM_foreigncopy_direction 1 uint16_t flags; /* * IN/OUT * * As an IN parameter number of frames of the domain to be copied. * On output on error updated number of frames left. */ uint32_t nr_frames;
Quoted 5 lines
Quoted 2 lines
This is updated not only on error, but also when encoding continuations.
Yes, but this seems more an implementation detail to me. I don't think the caller cares about how the continuation is implemented.
You just can't know what a caller may or may not care about. You want to be precise.
Quoted 36 lines
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 776
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
Quoted 9 lines
Quoted 3 lines
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
How do you know? Why shouldn't e.g. XTF be permitted to test this in all possible modes? And even if all arguments end up in favor of "no compat support", this then wants spelling out to make clear this wasn't an oversight, but rather a conscious decision.
XTF can do something like
#if COMPAT_GUEST /* Compat guests are not supported, return success. */ return 0; # endif(or can be done in the Makefiles I suppose).
Quoted 7 lines
Added a comment in
memory.h:/*
* Copy memory from/to a given domain.
* As this call requires target access and guest with target access won't be
* compat guests supported for compat guests this is not implemented. */
Well, okay. Right now what I can say is that with this it's then going to be rather unlikely that I'd ack the overall change. You make assumptions on what people may or may not do. There are still benefits to 32-bit environments in certain situations, even more so that the x32 mode of x86-64 didn't really take off.
Jan
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 218 lines · Jan Beulich
Quoted 19 lines · Frediano Ziglio
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Frediano Ziglio
Quoted 15 lines · Jan Beulich
xen/common/memory.c static int acquire_resource( · line 1603
Quoted 10 lines · Frediano Ziglio
xen/common/memory.c +10
-0+ { +struct page_info *foreign_page; +void *foreign; +p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 +p2m_to_mask (p2m_ram_rw) |p2m_to_mask (p2m_ram_logdirty); +#else +p2m_to_mask (p2m_ram_rw); +#endif What about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
The base of this was taken from migration code where there is such a check. I suppose that adding
p2m_ram_ro(where available) won't hurt.Just to mention, to avoid another round trip just because of this:
p2m_ram_rohas different meaning on x86 vs Arm/RISC-V.That's confusing... should not this be fixed somehow? It won't save much from a round-trip. Should I allow it or not ?
Ask the Arm maintainers. I raised this issue more than once, without any real success.
Quoted 11 lines
Quoted 9 lines
Quoted 6 lines
p2m_ram_sharedI'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.It could copy the contents of the grant mapped page by other means. Why not allow it in this new sub-op as well then?
I'm more afraid of writing the content of the grant pages than copying it.
But the same is true for writing to the granted page: The domain could do so by other means.
Quoted 5 lines
Quoted 3 lines
Talking of security: When the page you copy to is owned by a PV guest, I think you further need to obtain a
PGT_writabletype ref. (Of course it then likely is easier to always do this, not just for PV.)Wondering how save/restore (or migration) works in this case.
Save is not relevant here. Restore happens before the guest gains control. Which is entirely different from hypercall handling.
Quoted 32 lines
Quoted 21 lines
Quoted 21 lines
Quoted 4 lines
Quoted 4 lines
xen/common/memory.c +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent);break ;Quoted 8 lines
xen/common/memory.c long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
xen/common/memory.c +3
-0+case XENMEM_foreigncopy: +if (unlikely (start_extent) ) +return -EINVAL ;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limitingnr_framesat (UINT_MAX>>MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/Quoted 8 lines
Quoted 3 lines
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "
start_entent" won't allow this.You'll want to mention this in the description and/or a code comment. It wants to become clear that the inconsistency in behavior (with other sub- ops) is deliberate rather than accidental.
Added a comment in the code:
Quoted 11 lines
if ( copy.nr_frames && hypercall_preempt_check() ) { /* * Instead of using "start_extent" we update the structure back, * we update it back in anyway to tell caller were the copy * stopped. */ rc = hypercall_create_continuation( __HYPERVISOR_memory_op, "lh", XENMEM_foreigncopy, arg); goto out; }Please can this go into the hunk that I commented on?
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#defineXENMEM_foreigncopy29struct xen_foreigncopy { /* IN - The domain whose memory is to be copied. */ domid_t domid; /* IN - Flags. */ #define XENMEM_foreigncopy_from 0 #define XENMEM_foreigncopy_to 1 #define XENMEM_foreigncopy_direction 1 uint16_t flags; /* * IN/OUT * * As an IN parameter number of frames of the domain to be copied. * On output on error updated number of frames left. */ uint32_t nr_frames;Quoted 5 lines
Quoted 2 lines
This is updated not only on error, but also when encoding continuations.
Yes, but this seems more an implementation detail to me. I don't think the caller cares about how the continuation is implemented.
You just can't know what a caller may or may not care about. You want to be precise.
Quoted 36 lines
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 776
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
Quoted 9 lines
Quoted 3 lines
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
How do you know? Why shouldn't e.g. XTF be permitted to test this in all possible modes? And even if all arguments end up in favor of "no compat support", this then wants spelling out to make clear this wasn't an oversight, but rather a conscious decision.
XTF can do something like
#if COMPAT_GUEST /* Compat guests are not supported, return success. */ return 0; # endif(or can be done in the Makefiles I suppose).
Quoted 7 lines
Added a comment in
memory.h:/*
* Copy memory from/to a given domain.
* As this call requires target access and guest with target access won't be
* compat guests supported for compat guests this is not implemented. */
Quoted 6 lines
Well, okay. Right now what I can say is that with this it's then going to be rather unlikely that I'd ack the overall change. You make assumptions on what people may or may not do. There are still benefits to 32-bit environments in certain situations, even more so that the x32 mode of x86-64 didn't really take off.
Won't it save some round trips if you just told this a couple of emails earlier? If you already knew that compat is worth doing why asking to put a comment on it?
Similar to the discussion about Arm, if you know the issue, can you explain it in more details? The description for the constant is the same in all architecture, so, what is the difference?
Quoted 1 line
Jan
Frediano
Re: [PATCH v5 12/16] xen: implement new foreign copy hypercall
Quoted 225 lines · Frediano Ziglio
Quoted 218 lines · Jan Beulich
Quoted 19 lines · Frediano Ziglio
Quoted 18 lines · Jan Beulich
Quoted 17 lines · Frediano Ziglio
Quoted 15 lines · Jan Beulich
Review
Quoted 10 lines · Frediano Ziglio
(quoted diff) +10
-0+ { + struct page_info *foreign_page; + void *foreign; + p2m_type_t p2mt; + const unsigned long valid_mask = +#ifdef CONFIG_X86 + p2m_to_mask(p2m_ram_rw) | p2m_to_mask(p2m_ram_logdirty); +#else + p2m_to_mask(p2m_ram_rw); +#endifWhat about, for example,
p2m_ram_ro? Orp2m_ram_shared? Or p2m_grant_map_*? Etc. Any artificial constraining wants justifying in the description and/or mentioning in the public header.Quoted 8 lines
Quoted 5 lines
Quoted 2 lines
The base of this was taken from migration code where there is such a check. I suppose that adding
p2m_ram_ro(where available) won't hurt.Just to mention, to avoid another round trip just because of this:
p2m_ram_rohas different meaning on x86 vs Arm/RISC-V.That's confusing... should not this be fixed somehow? It won't save much from a round-trip. Should I allow it or not ?
Ask the Arm maintainers. I raised this issue more than once, without any real success.
Quoted 11 lines
Quoted 9 lines
Quoted 6 lines
p2m_ram_sharedI'm not sure but seems fine too.
For p2m_grant_map_* it feels a bit a security issue to me. It would allow a guest to give access to pages of other domains. It's true that the current domain would have to have write access to this domain anyway but extend these permissions sounds something it should not be able to do.It could copy the contents of the grant mapped page by other means. Why not allow it in this new sub-op as well then?
I'm more afraid of writing the content of the grant pages than copying it.
But the same is true for writing to the granted page: The domain could do so by other means.
Quoted 5 lines
Quoted 3 lines
Talking of security: When the page you copy to is owned by a PV guest, I think you further need to obtain a
PGT_writabletype ref. (Of course it then likely is easier to always do this, not just for PV.)Wondering how save/restore (or migration) works in this case.
Save is not relevant here. Restore happens before the guest gains control. Which is entirely different from hypercall handling.
Quoted 32 lines
Quoted 21 lines
Quoted 21 lines
Quoted 4 lines
Quoted 4 lines
(quoted diff) +0
-0@@ -2012,6 +2139,13 @@ long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) start_extent); break;Quoted 8 lines
Review long do_memory_op(unsigned long cmd, XEN_GUEST_HANDLE_PARAM(void) arg) · line 2144
Quoted 3 lines
(quoted diff) +3
-0+ case XENMEM_foreigncopy: + if ( unlikely(start_extent) ) + return -EINVAL;Please address review comments (verbally or by code changes) before submitting a new version. Here I had asked "Why make this different from other continuable sub-ops?"
There's already a comment in the same file for similar reason
/*
* Limitingnr_framesat (UINT_MAX>>MEMOP_EXTENT_SHIFT) isn't ideal. If
* it ever becomes a practical problem, we can switch to mutating
* xmar.{frame,nr_frames,frame_list} in guest memory.
*/Quoted 8 lines
Quoted 3 lines
so to avoid the doubt and possible future change I mutate the structure. Also I use the mutation to give more information to the caller, using "
start_entent" won't allow this.You'll want to mention this in the description and/or a code comment. It wants to become clear that the inconsistency in behavior (with other sub- ops) is deliberate rather than accidental.
Added a comment in the code:
Quoted 11 lines
if ( copy.nr_frames && hypercall_preempt_check() ) { /* * Instead of using "start_extent" we update the structure back, * we update it back in anyway to tell caller were the copy * stopped. */ rc = hypercall_create_continuation( __HYPERVISOR_memory_op, "lh", XENMEM_foreigncopy, arg); goto out; }Please can this go into the hunk that I commented on?
Quoted 62 lines
Quoted 62 lines
Quoted 61 lines
Quoted 28 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 748
Quoted 13 lines
xen/include/public/memory.h +6
-1@@ -740,7 +740,45 @@ struct xen_vnuma_topology_info {typedef struct xen_vnuma_topology_info xen_vnuma_topology_info_t ;DEFINE_XEN_GUEST_HANDLE (xen_vnuma_topology_info_t); -/* Next available subop number is 29 */ +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied. */ There's still "resource" here, when this really is about RAM (memory) only, not any other kind of resource.
Quoted 11 lines
xen/include/public/memory.h +11
-0+domid_t domid; + +/* IN - Flags. */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* Quoted 9 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 762
Quoted 3 lines
xen/include/public/memory.h +3
-0+* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames;This isn't just an input, as you update the field (and the handles below). This property of fields wants reflecting here, so callers know that they (a) can't re-use the struct on a subsequent call without re-initializing the fields which may have changed, and (b) can't put the struct in r/o memory.
Update comments:
/*
* Copy memory from/to a given domain.
*/
#defineXENMEM_foreigncopy29struct xen_foreigncopy { /* IN - The domain whose memory is to be copied. */ domid_t domid; /* IN - Flags. */ #define XENMEM_foreigncopy_from 0 #define XENMEM_foreigncopy_to 1 #define XENMEM_foreigncopy_direction 1 uint16_t flags; /* * IN/OUT * * As an IN parameter number of frames of the domain to be copied. * On output on error updated number of frames left. */ uint32_t nr_frames;Quoted 5 lines
Quoted 2 lines
This is updated not only on error, but also when encoding continuations.
Yes, but this seems more an implementation detail to me. I don't think the caller cares about how the continuation is implemented.
You just can't know what a caller may or may not care about. You want to be precise.
Quoted 36 lines
Quoted 17 lines
Quoted 17 lines
Quoted 16 lines
xen/include/public/memory.h struct xen_vnuma_topology_info { · line 775
Quoted 13 lines
xen/include/public/memory.h +13
-0+/* +* IN +* +* Frames to be copied. +*/ +XEN_GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +XEN_GUEST_HANDLE (uint8) buffer;With these two handles, there continues to be a need to (explicitly) deal with the compat case as well.
Quoted 9 lines
Quoted 3 lines
I don't agree with this. Domains having access to other domains are limited (like stub domains for Qemu) and won't be 32 bits today so why allow 32 bits guests if not ever used?
How do you know? Why shouldn't e.g. XTF be permitted to test this in all possible modes? And even if all arguments end up in favor of "no compat support", this then wants spelling out to make clear this wasn't an oversight, but rather a conscious decision.
XTF can do something like
#if COMPAT_GUEST /* Compat guests are not supported, return success. */ return 0; # endif(or can be done in the Makefiles I suppose).
Quoted 7 lines
Added a comment in
memory.h:/*
* Copy memory from/to a given domain.
* As this call requires target access and guest with target access won't be
* compat guests supported for compat guests this is not implemented. */Quoted 5 lines
Well, okay. Right now what I can say is that with this it's then going to be rather unlikely that I'd ack the overall change. You make assumptions on what people may or may not do. There are still benefits to 32-bit environments in certain situations, even more so that the x32 mode of x86-64 didn't really take off.
Quoted 3 lines
Won't it save some round trips if you just told this a couple of emails earlier? If you already knew that compat is worth doing why asking to put a comment on it?
I'm sorry if this ended up confusing, but I specifically had made that request conditional: "if all arguments end up in favor of". To me this was pretty much rhetorical, but apparently it wasn't to you.
Quoted 3 lines
Similar to the discussion about Arm, if you know the issue, can you explain it in more details? The description for the constant is the same in all architecture, so, what is the difference?
While Arm also says "writes are silently dropped" in the comment, I don't think this matches the present use of the type: Afaict the type simply means "r/o", without there being code to actually handle (skip) respective guest insns. Part of my previous suggestion was to, at the very least, get comment and implementation in line (i.e. even if p2m_ram_ro then would end up having different properties on x86 and Arm).
Jan
PATCH v5 13/16 privcmd: Add definition for new Linux privcmd to access new Xen hypercall
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Userspace should use new ioctl to access new hypercall.
--
Changes since v4:
- update comment.
--- tools/include/xen-sys/Linux/privcmd.h | 10 ++++++++++ 1 file changed, 10 insertions(+)
tools/include/xen-sys/Linux/privcmd.h +10 -0
@@ -100,6 +100,14 @@ typedef struct privcmd_pcidev_get_gsi {__u32 gsi; } privcmd_pcidev_get_gsi_t; +typedef struct privcmd_foreigncopy { +domid_t dom;/* Foreign domain. */ +__u16 dir;/* Direction, 0 from, 1 to. */ +__u32 num;/* Number of pages to copy. */ +const xen_pfn_t __user *pfns;/* Array of pfns. */ +void __user *buffer;/* Buffer to copy to/from. */ +}privcmd_foreigncopy_t ; +/* * @cmd: IOCTL_PRIVCMD_HYPERCALL * @arg: &privcmd_hypercall_t
@@ -121,6 +129,8 @@ typedef struct privcmd_pcidev_get_gsi { _IOC(_IOC_NONE, 'P', 7, sizeof(privcmd_mmap_resource_t))#define IOCTL_PRIVCMD_PCIDEV_GET_GSI \ _IOC(_IOC_NONE, 'P', 10, sizeof(privcmd_pcidev_get_gsi_t)) +#define IOCTL_PRIVCMD_FOREIGNCOPY \ + _IOWR('P', 11, privcmd_foreigncopy_t)#define IOCTL_PRIVCMD_UNIMPLEMENTED \ _IOC(_IOC_NONE, 'P', 0xFF, 0)
Signature · 1 line
2.43.0
PATCH v5 14/16 libs/guest: use new hypercall if available
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Use new hypercall if available, otherwise fall back to map+copy+unmap sequence.
--
Changes since v4:
- use int8_t instead of char for signed type.
--- tools/libs/guest/xg_sr_common.c | 47 ++++++++++++++++++++++++++------- 1 file changed, 38 insertions(+), 9 deletions(-)
tools/libs/guest/xg_sr_common.c +38 -9
@@ -156,11 +156,6 @@ static void __attribute__((unused)) build_assertions(void)BUILD_BUG_ON (sizeof (struct xc_sr_rec_hvm_params ) !=8 ); } -enum { - foreigncopy_from, - foreigncopy_to -}; -static int xg_foreignmemory_copy (xc_interface *xch,domid_t domid,int dir,size_t nr_pages,void *buffer,const xen_pfn_t foreign_pfns[nr_pages])
@@ -174,8 +169,42 @@ static int xg_foreignmemory_copy(xc_interface *xch, domid_t domid,return -1 ; } +/* +* If foreign copy is supported, -1 not initialized, 0 not supported, +* 1 supported. +*/ +static int8_t foreign_copy_supported =-1 ; + +if ( foreign_copy_supported ) + { +int rc; +privcmd_foreigncopy_t copy = { + .dom = domid, + .dir = dir, + .num = nr_pages, + .buffer = buffer, + }; +DECLARE_HYPERCALL_BOUNCE_IN (foreign_pfns, nr_pages *sizeof (xen_pfn_t)); + +if (xc_hypercall_bounce_pre (xch, foreign_pfns) ) +return -1 ; + + copy.pfns = foreign_pfns; + + rc =ioctl (xencall_fd (xch->xcall),IOCTL_PRIVCMD_FOREIGNCOPY , ©); +if ( foreign_copy_supported <0 ) + foreign_copy_supported = + (!rc || (errno !=ENOTTY && errno !=ENOSYS )); + +xc_hypercall_bounce_post (xch, foreign_pfns); + +if ( foreign_copy_supported ) +return rc; + } + +/* Fallback, emulate. */ int err[nr_pages]; -const int prot = (dir == foreigncopy_from) ?PROT_READ :PROT_READ |PROT_WRITE ; +const int prot = (dir == XENMEM_foreigncopy_from) ?PROT_READ :PROT_READ |PROT_WRITE ;void *p =xenforeignmemory_map (xch->fmem, domid, prot, nr_pages, foreign_pfns, err);if ( !p )
@@ -192,7 +221,7 @@ static int xg_foreignmemory_copy(xc_interface *xch, domid_t domid,return -1 ; } -if ( dir == foreigncopy_from ) +if ( dir == XENMEM_foreigncopy_from )memcpy (buffer, p, nr_pages *XC_PAGE_SIZE );else memcpy (p, buffer, nr_pages *XC_PAGE_SIZE );
@@ -204,7 +233,7 @@ int xg_foreignmemory_copy_from(xc_interface *xch, domid_t dom, size_t nr_pages, void *dest, const xen_pfn_t source[nr_pages]) { - return xg_foreignmemory_copy(xch, dom, foreigncopy_from, + return xg_foreignmemory_copy(xch, dom, XENMEM_foreigncopy_from,nr_pages ,dest ,source ); }
@@ -212,7 +241,7 @@ int xg_foreignmemory_copy_to(xc_interface *xch, domid_t dom, size_t nr_pages, const xen_pfn_t dest[nr_pages], const void *source) { - return xg_foreignmemory_copy(xch, dom, foreigncopy_to, + return xg_foreignmemory_copy(xch, dom, XENMEM_foreigncopy_to, nr_pages, (void *) source, dest); }
Signature · 1 line
2.43.0
PATCH v5 15/16 libs/guest: finalize PoC
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
Remove now unused map_errs array.
Test and restore verification code.
Report correctly errors from writev_exact.
Allocate verification buffer using hypercall buffer to avoid errors using hypercall.
--- tools/libs/guest/xg_sr_common.h | 4 +- tools/libs/guest/xg_sr_restore.c | 45 +++++++++++++++-- tools/libs/guest/xg_sr_save.c | 83 +++++++++++++++++++++----------- 3 files changed, 98 insertions(+), 34 deletions(-)
tools/libs/guest/xg_sr_common.h +2 -2
@@ -217,7 +217,6 @@ struct xc_sr_context_save_buffersvoid *local_pages[MAX_BATCH_SIZE ];struct iovec iov[MAX_BATCH_SIZE +2 ];/* Headers + data. */ uint64_t rec_pfns[MAX_BATCH_SIZE ]; -int errors[MAX_BATCH_SIZE ]; };struct xc_sr_context
@@ -255,8 +254,8 @@ struct xc_sr_contextunsigned long *deferred_pages;unsigned long nr_deferred_pages;xc_hypercall_buffer_t dirty_bitmap_hbuf; +xc_hypercall_buffer_t dest_buf;struct xc_sr_context_save_buffers *buffers; -void *dest_buf; } save; struct /* Restore data. */
@@ -267,6 +266,7 @@ struct xc_sr_contextint send_back_fd;unsigned long p2m_size;xc_hypercall_buffer_t dirty_bitmap_hbuf; +xc_hypercall_buffer_t verify_buf;/* From Image Header. */ uint32_t format_version;
tools/libs/guest/xg_sr_restore.c +40 -5
87 lines not on this page · raw message
tools/libs/guest/xg_sr_save.c +56 -27
141 lines not on this page · raw message
Signature · 1 line
2.43.0
PATCH v5 16/16 xen/privcmd: Add new ABI to allow copying foreign memory
In reply to Frediano ZiglioFrom: Frediano Ziglio <[email protected]>
This new ABI allows to copy foreign domain memory to/from a buffer. This avoids having to map/copy/unmap foreign memory which is expensive.
This operation is done particularly when migrating VMs.
--
Changes since v4:
- fix wrong assign;
- use set_xen_guest_handle to set handle;
- wrap slow hypercall with xen_preemptible_hcall_{begin,end};
- use _IOWR for ioctl code to be more specific;
- use __copy_to_user if buffer already checked.
--- arch/x86/include/asm/xen/interface.h | 1 + drivers/xen/privcmd.c | 49 ++++++++++++++++++++++++++++ include/uapi/xen/privcmd.h | 10 ++++++ include/xen/interface/memory.h | 37 ++++++++++++++++++++ 4 files changed, 97 insertions(+)
arch/x86/include/asm/xen/interface.h +1 -0
@@ -91,6 +91,7 @@ DEFINE_GUEST_HANDLE(int);DEFINE_GUEST_HANDLE (void);DEFINE_GUEST_HANDLE (uint64_t);DEFINE_GUEST_HANDLE (uint32_t); +DEFINE_GUEST_HANDLE (uint8_t);DEFINE_GUEST_HANDLE (xen_pfn_t);DEFINE_GUEST_HANDLE (xen_ulong_t);#endif
drivers/xen/privcmd.c +49 -0
61 lines not on this page · raw message
include/uapi/xen/privcmd.h +10 -0
@@ -131,6 +131,14 @@ struct privcmd_pcidev_get_gsi {__u32 gsi; }; +struct privcmd_foreigncopy { +domid_t dom;/* foreign domain */ +__u16 dir;/* direction, 0 from, 1 to */ +__u32 num;/* number of pages to copy */ +const xen_pfn_t __user *pfns;/* array of pfns */ +void __user *buffer;/* buffer to copy to/from */ +}; +/* * @cmd: IOCTL_PRIVCMD_HYPERCALL * @arg: &privcmd_hypercall_t
@@ -164,5 +172,7 @@ struct privcmd_pcidev_get_gsi { _IOW('P', 9, struct privcmd_ioeventfd)#define IOCTL_PRIVCMD_PCIDEV_GET_GSI \ _IOC(_IOC_NONE, 'P', 10, sizeof(struct privcmd_pcidev_get_gsi)) +#define IOCTL_PRIVCMD_FOREIGNCOPY \ + _IOWR('P', 11, struct privcmd_foreigncopy)#endif /* __LINUX_PUBLIC_PRIVCMD_H__ */
include/xen/interface/memory.h +37 -0
@@ -325,4 +325,41 @@ struct xen_mem_acquire_resource { };DEFINE_GUEST_HANDLE_STRUCT (xen_mem_acquire_resource); +/* +* Copy memory from/to a given domain. +*/ +#define XENMEM_foreigncopy 29 +struct xen_foreigncopy { +/* IN - The domain whose resource is to be copied */ +domid_t domid; + +/* IN - Flags */ +#define XENMEM_foreigncopy_from 0 +#define XENMEM_foreigncopy_to 1 +#define XENMEM_foreigncopy_direction 1 +uint16_t flags; + +/* +* IN +* +* As an IN parameter number of frames of the domain to be copied. +*/ +uint32_t nr_frames; + +/* +* IN +* +* Frames to be copied. +*/ +GUEST_HANDLE (xen_pfn_t) frame_list; + +/* +* IN/OUT +* +* Userspace buffer to read/write from. +*/ +GUEST_HANDLE (uint8_t ) buffer; +}; +DEFINE_GUEST_HANDLE_STRUCT (xen_foreigncopy); +#endif /* __XEN_PUBLIC_MEMORY_H__ */
Signature · 1 line
2.54.0