domain context infrastructure
cover domain context infrastructure
Paul Durrant (5):
xen/common: introduce a new framework for save/restore of 'domain'
context
xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext tools/misc: add xen-ctx to present domain context common/domain: add a domain context record for shared_info... tools/libxc: make use of domain context SHARED_INFO record...
.gitignore | 1 + tools/flask/policy/modules/xen.if | 4 +- tools/libxc/include/xenctrl.h | 11 ++ tools/libxc/xc_domain.c | 52 ++++++ tools/libxc/xc_sr_common.h | 7 +- tools/libxc/xc_sr_common_x86.c | 58 ++++++ tools/libxc/xc_sr_common_x86.h | 4 + tools/libxc/xc_sr_common_x86_pv.c | 52 ++++++ tools/libxc/xc_sr_common_x86_pv.h | 3 + tools/libxc/xc_sr_restore_x86_pv.c | 40 ++--- tools/libxc/xc_sr_save_x86_pv.c | 26 +-- tools/libxc/xg_save_restore.h | 1 + tools/misc/Makefile | 4 + tools/misc/xen-ctx.c | 152 ++++++++++++++++ xen/common/Makefile | 1 + xen/common/domain.c | 55 ++++++ xen/common/domctl.c | 115 ++++++++++++ xen/common/save.c | 262 ++++++++++++++++++++++++++++ xen/include/public/domctl.h | 41 ++++- xen/include/public/save.h | 82 +++++++++ xen/include/public/xen.h | 3 + xen/include/xen/save.h | 115 ++++++++++++ xen/xsm/flask/hooks.c | 6 + xen/xsm/flask/policy/access_vectors | 4 + 24 files changed, 1047 insertions(+), 52 deletions(-) create mode 100644 tools/misc/xen-ctx.c create mode 100644 xen/common/save.c create mode 100644 xen/include/public/save.h create mode 100644 xen/include/xen/save.h
---
Cc: Andrew Cooper <[email protected]>
Cc: Daniel De Graaf <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>
Cc: Wei Liu <[email protected]>
Signature · 1 line
2.20.1
PATCH 1/5 xen/common: introduce a new framework for save/restore of 'domain' context
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
This patch adds the new public header and the low level implementation, entered via the domain_save() or domain_load() functions. Subsequent patches will introduce other parts of the framwork, and code that will make use of it within the current version of the libxc migration stream.
---
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>
Cc: Wei Liu <[email protected]>
--- xen/common/Makefile | 1 + xen/common/save.c | 262 ++++++++++++++++++++++++++++++++++++++ xen/include/public/save.h | 74 +++++++++++ xen/include/xen/save.h | 115 +++++++++++++++++ 4 files changed, 452 insertions(+) create mode 100644 xen/common/save.c create mode 100644 xen/include/public/save.h create mode 100644 xen/include/xen/save.h
xen/common/Makefile +1 -0
@@ -37,6 +37,7 @@ obj-y += radix-tree.o obj-y += rbtree.o obj-y += rcupdate.o obj-y += rwlock.o +obj-y += save.o obj-y += shutdown.o obj-y += softirq.o obj-y += sort.o
xen/common/save.c new file +262 -0
262 lines not on this page · raw message
xen/include/public/save.h new file +74 -0
74 lines not on this page · raw message
xen/include/xen/save.h new file +115 -0
115 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Hi Paul,
Quoted 5 lines · Paul Durrant
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
Quoted 8 lines
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
This patch adds the new public header and the low level implementation, entered via the
domain_save()ordomain_load()functions. Subsequent patches will introduce other parts of the framwork, and code that will
Typo: s/framwork/framework/
Quoted 53 lines
make use of it within the current version of the libxc migration stream.
Signed-off-by Paul Durrant <[email protected]>---
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>
Cc: Wei Liu <[email protected]>--- xen/common/Makefile | 1 + xen/common/save.c | 262 ++++++++++++++++++++++++++++++++++++++ xen/include/public/save.h | 74 +++++++++++ xen/include/xen/save.h | 115 +++++++++++++++++ 4 files changed, 452 insertions(+)create mode 100644
xen/common/save.c
create mode 100644xen/include/public/save.h
create mode 100644xen/include/xen/save.hxen/common/Makefile +1
-0@@ -37,6 +37,7 @@ obj-y += radix-tree.o obj-y += rbtree.o obj-y += rcupdate.o obj-y += rwlock.o +obj-y += save.o obj-y += shutdown.o obj-y += softirq.o obj-y += sort.oxen/common/save.c new file +14
-0@@ -0,0 +1,262 @@ +/* +* save.c: Save and restore PV guest state common to all domain types. +* +* Copyright Amazon.com Inc. or its affiliates. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +*
xen/common/save.c line 24
Quoted 10 lines
xen/common/save.c new file +10
-0+* You should have received a copy of the GNU General Public License along with +* this program; If not, see <http://www.gnu.org/licenses/>. +*/ + +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;
As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 18 lines
xen/common/save.c new file +18
-0+void *priv; +}; + +static struct { +const char *name; +bool per_vcpu; +domain_save_handler save; +domain_load_handler load; +} handlers[DOMAIN_SAVE_CODE_MAX +1 ]; + +void __init domain_register_save_type (unsigned int tc,const char *name, +bool per_vcpu, +domain_save_handler save, +domain_load_handler load) +{ +BUG_ON (tc >ARRAY_SIZE (handlers)); + +ASSERT (!handlers[tc].save);
xen/common/save.c line 52
Quoted 10 lines
xen/common/save.c new file +10
-0+ASSERT (!handlers[tc].load); + + handlers[tc].name = name; + handlers[tc].per_vcpu = per_vcpu; + handlers[tc].save = save; + handlers[tc].load = load; +} + +int domain_save_entry(struct domain_context *c, unsigned int tc, + const char *name, const struct vcpu *v, void *src,
I realize that 'src' is not const because how you define copy, however I would rather prefer if we rework the interface to avoid to keep the const in the definition. This may mean to have to define two callback rather than one.
xen/common/save.c line 53
Quoted 1 line
xen/common/save.c new file +1
-0+ size_t src_len)
On 64-bit architecture, size_t would be 64-bit. But the record is only storing 32-bit. Would it make sense to add some sanity check in the code?
xen/common/save.c line 61
Quoted 8 lines
xen/common/save.c new file +8
-0+{ +int rc; + +if ( c->log && tc !=DOMAIN_SAVE_CODE (HEADER ) && + tc !=DOMAIN_SAVE_CODE (END ) ) +gdprintk (XENLOG_INFO ,"%pv save: %s (%lu)\n" , v, name, src_len); + +if ( !IS_ALIGNED (src_len,8 ) )
Why not adding an implicit padding? This would avoid to chase error during saving because the len wasn't a multiple of 8.
xen/common/save.c line 65
Quoted 4 lines
xen/common/save.c new file +4
-0+return -EINVAL ; + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);
Does the descriptor really need to be filled by domain_save()? Can't it be done here, so we can avoid the two BUG_ON()s?
Quoted 29 lines
xen/common/save.c new file +29
-0+ c->desc.length = src_len; + + rc = c->copy (c->priv, &c->desc,sizeof (c->desc)); +if ( rc ) +return rc; + +return c->copy (c->priv, src, src_len); +} + +int domain_save (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask,bool dry_run) +{ +struct domain_context c = { + .copy = copy, + .priv = priv, + .log = !dry_run, + }; +struct domain_save_header h = { + .magic =DOMAIN_SAVE_MAGIC , + .version =DOMAIN_SAVE_VERSION , + }; +struct domain_save_header e; +unsigned int i; +int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ;
xen/common/save.c line 104
Quoted 10 lines
xen/common/save.c new file +10
-0+ +domain_pause (d); + + c.desc.typecode =DOMAIN_SAVE_CODE (HEADER ); + + rc =DOMAIN_SAVE_ENTRY (HEADER , &c, d->vcpu[0 ], &h,sizeof (h)); +if ( rc ) + goto out; + +for ( i =0 ; i <ARRAY_SIZE (handlers); i++ )
AFAIU, with this solution, if there are dependency between records, the dependencies will have to a lower "index". I think we may have some dependency with guest transparent migration such as we need to restore the event ABI before restoring the event channels.
Should we rely on the index for the dependency?
In any case, I think we want to document it.
xen/common/save.c line 108
Quoted 4 lines
xen/common/save.c new file +4
-0+ { +domain_save_handler save = handlers[i].save; + +if ( (mask && !test_bit (i, &mask)) || !save )
The type of mask suggests it is not possible to have more than 32 different types of record if we wanted to be arch agnostic. Even if we focus on 64-bit arch, this is only 64 records.
This is not very future proof, but might be ok if this is not exposed outside of the hypervisor (I haven't looked at the rest of the series yet). However, we at least need a BUILD_BUG_ON() in place. So please make sure DOMAIN_SAVE_CODE_MAX is always less than 64.
Also what if there is a bit set in the mask and the record is not existing? Shouldn't we return an error?
Quoted 39 lines
xen/common/save.c new file +39
-0+continue ; + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode = i; + +if ( handlers[i].per_vcpu ) + { +struct vcpu *v; + + for_each_vcpu ( d, v ) + { + c.desc.instance = v->vcpu_id; + + rc =save (v, &c, dry_run); +if ( rc ) + goto out; + } + } +else + { + rc =save (d->vcpu[0 ], &c, dry_run); +if ( rc ) + goto out; + } + } + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode =DOMAIN_SAVE_CODE (END ); + + rc =DOMAIN_SAVE_ENTRY (END , &c, d->vcpu[0 ], &e,0 ); + + out: +domain_unpause (d); + +return rc; +} + +int domain_load_entry(struct domain_context *c, unsigned int tc, + const char *name, const struct vcpu *v, void *dst,
xen/common/save.c line 157
Quoted 10 lines
xen/common/save.c new file +10
-0+ size_t dst_len, bool exact) +{ + int rc; + +if ( c->log && tc !=DOMAIN_SAVE_CODE (HEADER ) && + tc !=DOMAIN_SAVE_CODE (END ) ) +gdprintk (XENLOG_INFO ,"%pv load: %s (%lu)\n" , v, name, dst_len); + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);
Is it really warrant to crash the host? What would happen if the values were wrong?
xen/common/save.c line 160
Quoted 3 lines
xen/common/save.c new file +3
-0+ + if ( (exact ? + (dst_len != c->desc.length) : (dst_len < c->desc.length)) ||
Using ternary in if is really confusing. How about:
dst_len < c->desc.length || (exact && dst_len != c->desc.length) ||
I understand that there would be two check for the exact case but I think it is better than a ternary.
However what is the purpose of the param 'exact'? If you set it to false how do you know the size you read?
Quoted 21 lines
xen/common/save.c new file +21
-0+ !IS_ALIGNED(c->desc.length, 8) ) +return -EINVAL ; + + rc = c->copy (c->priv, dst, c->desc.length); +if ( rc ) +return rc; + +if ( !exact ) + { + dst += c->desc.length; +memset (dst,0 , dst_len - c->desc.length); + } + +return 0 ; +} + +int domain_load (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask) +{ +struct domain_context c = { + .copy = copy,
xen/common/save.c line 191
Quoted 10 lines
xen/common/save.c new file +10
-0+ .priv = priv, + .log = true, + }; +struct domain_save_header h, e; +int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ;
What does protect you against the doing dying right after this check?
Quoted 26 lines
xen/common/save.c new file +26
-0+ + rc = c.copy (c.priv, &c.desc,sizeof (c.desc)); +if ( rc ) +return rc; + +if ( c.desc.typecode !=DOMAIN_SAVE_CODE (HEADER ) || + c.desc.instance !=0 ) +return -EINVAL ; + + rc =DOMAIN_LOAD_ENTRY (HEADER , &c, d->vcpu[0 ], &h,sizeof (h), true); +if ( rc ) +return rc; + +if ( h.magic !=DOMAIN_SAVE_MAGIC || h.version !=DOMAIN_SAVE_VERSION ) +return -EINVAL ; + +domain_pause (d); + +for (;;) + { +unsigned int i; +domain_load_handler load; +struct vcpu *v; + + rc = c.copy (c.priv, &c.desc,sizeof (c.desc)); +if ( rc )
xen/common/save.c line 227
Quoted 10 lines
xen/common/save.c new file +10
-0+break ; + +if ( c.desc.typecode ==DOMAIN_SAVE_CODE (END ) ) { + rc =DOMAIN_LOAD_ENTRY (END , &c, d->vcpu[0 ], &e,0 , true); +break ; + } + + rc = -EINVAL ; +if ( c.desc.typecode >=ARRAY_SIZE (handlers) || + c.desc.instance >= d->max_vcpus )
Nothing in the documention suggests that c.desc.instance should be 0 when the record is for the domain.
Quoted 99 lines
xen/common/save.c new file +35
-0+break ; + + i = c.desc.typecode; + load = handlers[i].load; + v = d->vcpu[c.desc.instance]; + +if ( mask && !test_bit (i, &mask) ) + { +/* Sink the data */ + rc = c.copy (c.priv,NULL , c.desc.length); +if ( rc ) +break ; + +continue ; + } + + rc = load ?load (v, &c) : -EOPNOTSUPP ; +if ( rc ) +break ; + } + +domain_unpause (d); + +return rc; +} + +/* +* Local variables: +* mode: C +* c-file-style: "BSD" +* c-basic-offset: 4 +* tab-width: 4 +* indent-tabs-mode: nil +* End: +*/ xen/include/public/save.h new file +58
-0@@ -0,0 +1,74 @@ +/* +* save.h +* +* Structure definitions for common PV/HVM domain state that is held by +* Xen and must be saved along with the domain's memory. +* +* Copyright Amazon.com Inc. or its affiliates. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +*/ + +#ifndef __XEN_PUBLIC_SAVE_H__ +#define __XEN_PUBLIC_SAVE_H__ + +#include "xen.h" + +/* Each entry is preceded by a descriptor */ +struct domain_save_descriptor { +uint16_t typecode; +uint16_t instance; +/* +* Entry length not including this descriptor. Entries must be padded +* to a multiple of 8 bytes to make sure descriptors remain correctly +* aligned. +*/ +uint32_t length; +}; + +/* +* Each entry has a type associated with it. DECLARE_DOMAIN_SAVE_TYPE +* binds these things together. +*/ +#define DECLARE_DOMAIN_SAVE_TYPE (_x, _code, _type) \ + struct __DOMAIN_SAVE_TYPE_##_x { _type t; char c[_code]; }; + +#define DOMAIN_SAVE_TYPE (_x) \ + typeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->t) +#define DOMAIN_SAVE_CODE (_x) \ + (sizeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->c)) +#define DOMAIN_SAVE_MASK (_x) (1u << DOMAIN_SAVE_CODE(_x)) + +/* Terminating entry */
xen/include/public/save.h line 68
Quoted 10 lines
xen/include/public/save.h new file +10
-0+struct domain_save_end {}; +DECLARE_DOMAIN_SAVE_TYPE(END, 0, struct domain_save_end); + +#define DOMAIN_SAVE_MAGIC 0x53415645 +#define DOMAIN_SAVE_VERSION 0x00000001 + +/* Initial entry */ +struct domain_save_header { +uint32_t magic;/* Must be DOMAIN_SAVE_MAGIC */ +uint32_t version;/* Save format version */
Would it make sense to have the version of Xen in the stream?
Quoted 118 lines
xen/include/public/save.h new file +6
-0+}; +DECLARE_DOMAIN_SAVE_TYPE(HEADER, 1, struct domain_save_header); + +#define DOMAIN_SAVE_CODE_MAX 1 + +#endif /* __XEN_PUBLIC_SAVE_H__ */ xen/include/xen/save.h new file +105
-0@@ -0,0 +1,115 @@ +/* +* save.h: support routines for save/restore +* +* Copyright Amazon.com Inc. or its affiliates. +* +* This program is free software; you can redistribute it and/or modify it +* under the terms and conditions of the GNU General Public License, +* version 2, as published by the Free Software Foundation. +* +* This program is distributed in the hope it will be useful, but WITHOUT +* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for +* more details. +* +* You should have received a copy of the GNU General Public License along with +* this program; If not, see <http://www.gnu.org/licenses/>. +*/ + +#ifndef __XEN_SAVE_H__ +#define __XEN_SAVE_H__ + +#include <xen/sched.h> +#include <xen/types.h> +#include <xen/init.h> + +#include <public/xen.h> +#include <public/save.h> + +struct domain_context ; + +int domain_save_entry (struct domain_context *c,unsigned int tc, +const char *name,const struct vcpu *v,void *src, +size_t src_len); + +#define DOMAIN_SAVE_ENTRY (_x, _c, _v, _src, _len) \ + domain_save_entry((_c), DOMAIN_SAVE_CODE(_x), #_x, (_v), (_src), \ + (_len)); + +int domain_load_entry (struct domain_context *c,unsigned int tc, +const char *name,const struct vcpu *v,void *dest, +size_t dest_len,bool exact); + +#define DOMAIN_LOAD_ENTRY (_x, _c, _v, _src, _len, _exact) \ + domain_load_entry((_c), DOMAIN_SAVE_CODE(_x), #_x, (_v), (_src), \ + (_len), (_exact)); + +/* +* The 'dry_run' flag indicates that the caller of domain_save() (see +* below) is not trying to actually acquire the data, only the size +* of the data. The save handler can therefore limit work to only that +* which is necessary to call DOMAIN_SAVE_ENTRY() with an accurate value +* for '_len'. +*/ +typedef int (*domain_save_handler )(const struct vcpu *v, +struct domain_context *h, +bool dry_run); +typedef int (*domain_load_handler )(struct vcpu *v, +struct domain_context *h); + +void domain_register_save_type (unsigned int tc,const char *name, +bool per_vcpu, +domain_save_handler save, +domain_load_handler load); + +#define DOMAIN_REGISTER_SAVE_RESTORE (_x, _per_vcpu, _save, _load) \ +static int __init __domain_register_##_x##_save_restore(void) \ +{ \ + domain_register_save_type( \ + DOMAIN_SAVE_CODE(_x), \ + #_x, \ + (_per_vcpu), \ + &(_save), \ + &(_load)); \ + \ + return 0; \ +} \ +__initcall(__domain_register_##_x##_save_restore); + +/* Copy callback function */ +typedef int (*domain_copy_entry )(void *priv,void *data,size_t len); + +/* +* Entry points: +* +* int domain_save(struct domain *d, domain_copy_entry copy, void *priv, +* unsigned long mask, bool dry_run); +* int domain_load(struct domain *d, domain_copy_entry copy, void *priv, +* unsigned long mask); +* +* copy: This is a callback function provided by the caller that will be +* used to write to (in the save case) or read from (in the load +* case) the context buffer. +* priv: This is a pointer that will be passed to the copy function to +* allow it to identify the context buffer and the current state +* of the save or load operation. +* mask: This is a mask to determine which save record types should be +* copied to or from the buffer. +* If it is zero then all save record types will be copied. +* If it is non-zero then only record types with codes +* corresponding to set bits will be copied. I.e. to copy save +* record 'type', set the bit in position DOMAIN_SAVE_CODE(type). +* DOMAIN_SAVE_CODE(HEADER) and DOMAIN_SAVE_CODE(END) records must +* always be present and thus will be copied regardless of whether +* the bits in those positions are set or not. +* dry_run: See the comment concerning (*domain_save) above.
xen/include/xen/save.h line 115
Quoted 10 lines
xen/include/xen/save.h new file +10
-0+* +* NOTE: A convenience macro, DOMAIN_SAVE_MASK(type), is defined to support +* setting bits in the mask field. +*/ +int domain_save (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask,bool dry_run); +int domain_load (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask); + +#endif /* __XEN_SAVE_H__ */
Cheers,
Signature · 1 line
Julien Grall
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 9 lines · Julien Grall
Quoted 2 lines · Paul Durrant
+ if ( (exact ?
+ (dst_len!=c->desc.length) : (dst_len<c->desc.length)) ||Using ternary in if is really confusing. How about:
dst_len<c->desc.length|| (exact &&dst_len!=c->desc.length) ||I understand that there would be two check for the exact case but I think it is better than a ternary.
I'm of the opposite opinion, and hence with Paul. While the alternative you suggest is still reasonable because of the special case here, I find it confusing / more difficult to read / follow
if ( (a && b) || (!a && c) )
(and I've seen quite a few instances of such over time) instead of
if ( a ? b : c )
Jan
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Hi Jan,
Quoted 15 lines · Jan Beulich
Quoted 9 lines · Julien Grall
Quoted 2 lines · Paul Durrant
+ if ( (exact ?
+ (dst_len!=c->desc.length) : (dst_len<c->desc.length)) ||Using ternary in if is really confusing. How about:
dst_len<c->desc.length|| (exact &&dst_len!=c->desc.length) ||I understand that there would be two check for the exact case but I think it is better than a ternary.
I'm of the opposite opinion, and hence with Paul. While the alternative you suggest is still reasonable because of the special case here, I find it confusing / more difficult to read / follow
Quoted 5 lines
if ( (a && b) || (!a && c) )(and I've seen quite a few instances of such over time) instead of
if ( a ? b : c )
If the ternary was the only condition and in a single line then it would be okay. However, the if is split over 3 lines...
The more stuff you put in an if, then more chance you are going to misread/make a mistake (you likely know what I am referring about here ;)).
So if you prefer the ternary, then we should at least write 2 ifs.
Cheers,
Signature · 1 line
Julien Grall
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 22 lines · Julien Grall
Hi Jan,
Quoted 19 lines · Jan Beulich
Quoted 9 lines · Julien Grall
Quoted 2 lines · Paul Durrant
+ if ( (exact ?
+ (dst_len!=c->desc.length) : (dst_len<c->desc.length)) ||Using ternary in if is really confusing. How about:
dst_len<c->desc.length|| (exact &&dst_len!=c->desc.length) ||I understand that there would be two check for the exact case but I think it is better than a ternary.
I'm of the opposite opinion, and hence with Paul. While the alternative you suggest is still reasonable because of the special case here, I find it confusing / more difficult to read / follow
if ( (a && b) || (!a && c) )
(and I've seen quite a few instances of such over time) instead of
Quoted 7 lines
Quoted 1 line
if ( a ? b : c )
If the ternary was the only condition and in a single line then it would be okay. However, the if is split over 3 lines...
The more stuff you put in an if, then more chance you are going to misread/make a mistake (you likely know what I am referring about here ;)).
So if you prefer the ternary, then we should at least write 2 ifs.
Since it's || that would be fine (albeit not preferred) by me. If it was &&, would be be suggesting two nested if()-s (which generally in reviews we ask to be avoided)? I see nothing wrong with e.g.
if ( (a ? b : c) &&
(x ? y : z) )Nevertheless I agree that very large conditionals often are more difficult to read.
Jan
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
In reply to Julien GrallQuoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
xen/common/save.c line 24
Quoted 7 lines
xen/common/save.c new file +7
-0+ +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
I don't follow. What is diverging? I explain in the commit comment that this is a parallel framework. Do I need to justify why it is not a carbon copy of the HVM one?
Quoted 25 lines
Quoted 25 lines
xen/common/save.c new file +25
-0+void *priv; +}; + +static struct { +const char *name; +bool per_vcpu; +domain_save_handler save; +domain_load_handler load; +} handlers[DOMAIN_SAVE_CODE_MAX +1 ]; + +void __init domain_register_save_type (unsigned int tc,const char *name, +bool per_vcpu, +domain_save_handler save, +domain_load_handler load) +{ +BUG_ON (tc >ARRAY_SIZE (handlers)); + +ASSERT (!handlers[tc].save); +ASSERT (!handlers[tc].load); + + handlers[tc].name = name; + handlers[tc].per_vcpu = per_vcpu; + handlers[tc].save = save; + handlers[tc].load = load; +}
Quoted 8 lines
xen/common/save.c line 52
Quoted 3 lines
xen/common/save.c new file +3
-0+ +int domain_save_entry(struct domain_context *c, unsigned int tc, + const char *name, const struct vcpu *v, void *src,I realize that 'src' is not const because how you define copy, however I would rather prefer if we rework the interface to avoid to keep the const in the definition. This may mean to have to define two callback rather than one.
That seems a bit ugly for the sake of a const but I guess I could create a union with a copy_in and copy_out. I'll see how that looks.
Quoted 6 lines
Quoted 2 lines
+
size_tsrc_len)On 64-bit architecture,
size_twould be 64-bit. But the record is only storing 32-bit. Would it make sense to add some sanity check in the code?
True. Given this is new I think I'll just bump the length to 64-bits.
Quoted 12 lines
xen/common/save.c line 61
Quoted 8 lines
xen/common/save.c new file +8
-0+{ +int rc; + +if ( c->log && tc !=DOMAIN_SAVE_CODE (HEADER ) && + tc !=DOMAIN_SAVE_CODE (END ) ) +gdprintk (XENLOG_INFO ,"%pv save: %s (%lu)\n" , v, name, src_len); + +if ( !IS_ALIGNED (src_len,8 ) )Why not adding an implicit padding? This would avoid to chase error during saving because the len wasn't a multiple of 8.
I don't think implicit padding is worth it. This error should only happen if a badly defined save record type is added to the code so perhaps I ought to add an ASSERT here as well as deal with the error.
Quoted 8 lines
xen/common/save.c line 65
Quoted 4 lines
xen/common/save.c new file +4
-0+return -EINVAL ; + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);Does the descriptor really need to be filled by
domain_save()? Can't it be done here, so we can avoid the two BUG_ON()s?
Yes it can but this serves as a sanity check that the save code is actually doing what it should. Hence why these are BUG_ON()s and not error exits.
Quoted 36 lines
Quoted 36 lines
xen/common/save.c new file +36
-0+ c->desc.length = src_len; + + rc = c->copy (c->priv, &c->desc,sizeof (c->desc)); +if ( rc ) +return rc; + +return c->copy (c->priv, src, src_len); +} + +int domain_save (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask,bool dry_run) +{ +struct domain_context c = { + .copy = copy, + .priv = priv, + .log = !dry_run, + }; +struct domain_save_header h = { + .magic =DOMAIN_SAVE_MAGIC , + .version =DOMAIN_SAVE_VERSION , + }; +struct domain_save_header e; +unsigned int i; +int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ; + +domain_pause (d); + + c.desc.typecode =DOMAIN_SAVE_CODE (HEADER ); + + rc =DOMAIN_SAVE_ENTRY (HEADER , &c, d->vcpu[0 ], &h,sizeof (h)); +if ( rc )
Quoted 9 lines
xen/common/save.c line 104
Quoted 3 lines
xen/common/save.c new file +3
-0+ goto out; + +for ( i =0 ; i <ARRAY_SIZE (handlers); i++ )AFAIU, with this solution, if there are dependency between records, the dependencies will have to a lower "index". I think we may have some dependency with guest transparent migration such as we need to restore the event ABI before restoring the event channels.
Is that just a downstream implementation detail though? I would hope that there are no ordering dependencies between records.
Quoted 2 lines
Should we rely on the index for the dependency?
If we do need ordering dependencies then I guess it would need to be explicit.
Quoted 2 lines
In any case, I think we want to document it.
I can certainly document that save handlers are invoked in code order.
Quoted 9 lines
xen/common/save.c line 108
Quoted 4 lines
xen/common/save.c new file +4
-0+ { +domain_save_handler save = handlers[i].save; + +if ( (mask && !test_bit (i, &mask)) || !save )The type of mask suggests it is not possible to have more than 32 different types of record if we wanted to be arch agnostic. Even if we focus on 64-bit arch, this is only 64 records.
Quoted 8 lines
This is not very future proof, but might be ok if this is not exposed outside of the hypervisor (I haven't looked at the rest of the series yet). However, we at least need a
BUILD_BUG_ON()in place. So please make sureDOMAIN_SAVE_CODE_MAXis always less than 64.Also what if there is a bit set in the mask and the record is not existing? Shouldn't we return an error?
TBH I think 32 will be enough... I would not expect this context space to grow very much, if at all, once transparent migration is working. I think I'll just drop the mask for now; it's not actually clear to me we'll make use of it... just seemed like a nice-to-have.
Quoted 44 lines
Quoted 44 lines
xen/common/save.c new file +44
-0+continue ; + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode = i; + +if ( handlers[i].per_vcpu ) + { +struct vcpu *v; + + for_each_vcpu ( d, v ) + { + c.desc.instance = v->vcpu_id; + + rc =save (v, &c, dry_run); +if ( rc ) + goto out; + } + } +else + { + rc =save (d->vcpu[0 ], &c, dry_run); +if ( rc ) + goto out; + } + } + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode =DOMAIN_SAVE_CODE (END ); + + rc =DOMAIN_SAVE_ENTRY (END , &c, d->vcpu[0 ], &e,0 ); + + out: +domain_unpause (d); + +return rc; +} + +int domain_load_entry (struct domain_context *c,unsigned int tc, +const char *name,const struct vcpu *v,void *dst, +size_t dst_len,bool exact) +{ +int rc; + + if ( c->log && tc != DOMAIN_SAVE_CODE(HEADER) &&
Quoted 9 lines
xen/common/save.c line 157
Quoted 5 lines
xen/common/save.c new file +5
-0+ tc != DOMAIN_SAVE_CODE(END) ) +gdprintk (XENLOG_INFO ,"%pv load: %s (%lu)\n" , v, name, dst_len); + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);Is it really warrant to crash the host? What would happen if the values were wrong?
It would mean the code is fairly seriously buggy in that the load handler is trying to load a record other than the type it registered for, or for a vcpu other than the one it was passed.
Quoted 10 lines
xen/common/save.c line 160
Quoted 3 lines
xen/common/save.c new file +3
-0+ + if ( (exact ? + (dst_len != c->desc.length) : (dst_len < c->desc.length)) ||Using ternary in if is really confusing. How about:
dst_len<c->desc.length|| (exact &&dst_len!=c->desc.length) ||I understand that there would be two check for the exact case but I think it is better than a ternary.
I'm going to re-work this I think.
Quoted 3 lines
However what is the purpose of the param 'exact'? If you set it to false how do you know the size you read?
The point of the exact parameter is that whether the caller can only consume a record of the correct length, or whether it can handle an undersize one which gets zero-padded. (It's the same as the zeroextend option in HVM records).
Quoted 26 lines
Quoted 26 lines
xen/common/save.c new file +25
-0+ !IS_ALIGNED(c->desc.length, 8) ) +return -EINVAL ; + + rc = c->copy (c->priv, dst, c->desc.length); +if ( rc ) +return rc; + +if ( !exact ) + { + dst += c->desc.length; +memset (dst,0 , dst_len - c->desc.length); + } + +return 0 ; +} + +int domain_load (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask) +{ +struct domain_context c = { + .copy = copy, + .priv = priv, + .log = true, + }; +struct domain_save_header h, e;
Quoted 9 lines
xen/common/save.c line 94
Quoted 6 lines
xen/common/save.c new file +6
-0+int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ;What does protect you against the doing dying right after this check?
Nothing. It's just to avoid doing pointless work if we can.
Quoted 31 lines
Quoted 31 lines
xen/common/save.c new file +31
-0+ + rc = c.copy (c.priv, &c.desc,sizeof (c.desc)); +if ( rc ) +return rc; + +if ( c.desc.typecode !=DOMAIN_SAVE_CODE (HEADER ) || + c.desc.instance !=0 ) +return -EINVAL ; + + rc =DOMAIN_LOAD_ENTRY (HEADER , &c, d->vcpu[0 ], &h,sizeof (h), true); +if ( rc ) +return rc; + +if ( h.magic !=DOMAIN_SAVE_MAGIC || h.version !=DOMAIN_SAVE_VERSION ) +return -EINVAL ; + +domain_pause (d); + +for (;;) + { +unsigned int i; +domain_load_handler load; +struct vcpu *v; + + rc = c.copy (c.priv, &c.desc,sizeof (c.desc)); +if ( rc ) +break ; + +if ( c.desc.typecode ==DOMAIN_SAVE_CODE (END ) ) { + rc =DOMAIN_LOAD_ENTRY (END , &c, d->vcpu[0 ], &e,0 , true); +break ;
Quoted 9 lines
xen/common/save.c line 227
Quoted 5 lines
xen/common/save.c new file +5
-0+ } + + rc = -EINVAL ; +if ( c.desc.typecode >=ARRAY_SIZE (handlers) || + c.desc.instance >= d->max_vcpus )Nothing in the documention suggests that c.desc.instance should be 0 when the record is for the domain.
Ok. I'll put a comment somewhere.
Quoted 103 lines
Quoted 103 lines
xen/common/save.c new file +35
-0+break ; + + i = c.desc.typecode; + load = handlers[i].load; + v = d->vcpu[c.desc.instance]; + +if ( mask && !test_bit (i, &mask) ) + { +/* Sink the data */ + rc = c.copy (c.priv,NULL , c.desc.length); +if ( rc ) +break ; + +continue ; + } + + rc = load ?load (v, &c) : -EOPNOTSUPP ; +if ( rc ) +break ; + } + +domain_unpause (d); + +return rc; +} + +/* +* Local variables: +* mode: C +* c-file-style: "BSD" +* c-basic-offset: 4 +* tab-width: 4 +* indent-tabs-mode: nil +* End: +*/ xen/include/public/save.h new file +62
-0@@ -0,0 +1,74 @@ +/* +* save.h +* +* Structure definitions for common PV/HVM domain state that is held by +* Xen and must be saved along with the domain's memory. +* +* Copyright Amazon.com Inc. or its affiliates. +* +* Permission is hereby granted, free of charge, to any person obtaining a copy +* of this software and associated documentation files (the "Software"), to +* deal in the Software without restriction, including without limitation the +* rights to use, copy, modify, merge, publish, distribute, sublicense, and/or +* sell copies of the Software, and to permit persons to whom the Software is +* furnished to do so, subject to the following conditions: +* +* The above copyright notice and this permission notice shall be included in +* all copies or substantial portions of the Software. +* +* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +* IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +* FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +* AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +* LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING +* FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER +* DEALINGS IN THE SOFTWARE. +*/ + +#ifndef __XEN_PUBLIC_SAVE_H__ +#define __XEN_PUBLIC_SAVE_H__ + +#include "xen.h" + +/* Each entry is preceded by a descriptor */ +struct domain_save_descriptor { +uint16_t typecode; +uint16_t instance; +/* +* Entry length not including this descriptor. Entries must be padded +* to a multiple of 8 bytes to make sure descriptors remain correctly +* aligned. +*/ +uint32_t length; +}; + +/* +* Each entry has a type associated with it. DECLARE_DOMAIN_SAVE_TYPE +* binds these things together. +*/ +#define DECLARE_DOMAIN_SAVE_TYPE (_x, _code, _type) \ + struct __DOMAIN_SAVE_TYPE_##_x { _type t; char c[_code]; }; + +#define DOMAIN_SAVE_TYPE (_x) \ + typeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->t) +#define DOMAIN_SAVE_CODE (_x) \ + (sizeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->c)) +#define DOMAIN_SAVE_MASK (_x) (1u << DOMAIN_SAVE_CODE(_x)) + +/* Terminating entry */ +struct domain_save_end {}; +DECLARE_DOMAIN_SAVE_TYPE(END, 0, struct domain_save_end); + +#define DOMAIN_SAVE_MAGIC 0x53415645
Quoted 9 lines
xen/include/public/save.h line 68
Quoted 6 lines
xen/include/public/save.h new file +6
-0+#define DOMAIN_SAVE_VERSION 0x00000001 + +/* Initial entry */ +struct domain_save_header { +uint32_t magic;/* Must be DOMAIN_SAVE_MAGIC */ +uint32_t version;/* Save format version */ Would it make sense to have the version of Xen in the stream?
Why? How would it help?
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Hi Paul,
Quoted 14 lines · Paul Durrant
Quoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
xen/common/save.c line 24
Quoted 7 lines
xen/common/save.c new file +7
-0+ +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 1 line
I don't follow. What is diverging? I explain in the commit comment that this is a parallel framework. Do I need to justify why it is not a carbon copy of the HVM one?
Well, they are both restoring/saving guest state. The only difference is the existing one is focusing on HVM state.
So it would make sense long term to have only one hypercall and tell what you want to save. In fact, some of the improvement here would definitely make the HVM one nicer to use (at least in the context of LU).
From the commit message, it is not clear to me why a new framework and why the infrastructure is at the same time different but not.
Quoted 35 lines
Quoted 26 lines
Quoted 26 lines
xen/common/save.c new file +25
-0+void *priv; +}; + +static struct { +const char *name; +bool per_vcpu; +domain_save_handler save; +domain_load_handler load; +} handlers[DOMAIN_SAVE_CODE_MAX +1 ]; + +void __init domain_register_save_type (unsigned int tc,const char *name, +bool per_vcpu, +domain_save_handler save, +domain_load_handler load) +{ +BUG_ON (tc >ARRAY_SIZE (handlers)); + +ASSERT (!handlers[tc].save); +ASSERT (!handlers[tc].load); + + handlers[tc].name = name; + handlers[tc].per_vcpu = per_vcpu; + handlers[tc].save = save; + handlers[tc].load = load; +}Quoted 8 lines
xen/common/save.c line 52
Quoted 3 lines
xen/common/save.c new file +3
-0+ +int domain_save_entry(struct domain_context *c, unsigned int tc, + const char *name, const struct vcpu *v, void *src,I realize that 'src' is not const because how you define copy, however I would rather prefer if we rework the interface to avoid to keep the const in the definition. This may mean to have to define two callback rather than one.
Quoted 1 line
That seems a bit ugly for the sake of a const but I guess I could create a union with a
copy_inandcopy_out. I'll see how that looks.
I would really like to map the Live-Update as read-only (it is not meant to be modified by the new Xen) and therefore adding const here would enable us to catch most of the mistake at build rather than during runtime.
Quoted 5 lines
Quoted 3 lines
Why not adding an implicit padding? This would avoid to chase error during saving because the len wasn't a multiple of 8.
I don't think implicit padding is worth it. This error should only happen if a badly defined save record type is added to the code so perhaps I ought to add an ASSERT here as well as deal with the error.
I wish I would be able to say every error can be caught during review. Unfortunately this is not true.
If you define the size dynamically, then a misalignment may not be noticed until you go to prod. The implicit padding would at least allow you to be more resilient.
It may not be that bad as you would not be able to save the guest. Anyway, this would be nice a feature to have but not a must.
Quoted 11 lines
Quoted 9 lines
xen/common/save.c line 65
Quoted 5 lines
xen/common/save.c new file +4
-0+return -EINVAL ; + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);Does the descriptor really need to be filled by
domain_save()? Can't it be done here, so we can avoid the two BUG_ON()s?Yes it can but this serves as a sanity check that the save code is actually doing what it should. Hence why these are BUG_ON()s and not error exits.
This does not really answer to the question why the c->desc.instance and c->desc.typecode are not set here. What is the advantage to do it earlier on and still passing the information in parameters? Why not just setting them here?
Quoted 47 lines
Quoted 37 lines
Quoted 37 lines
xen/common/save.c new file +36
-0+ c->desc.length = src_len; + + rc = c->copy (c->priv, &c->desc,sizeof (c->desc)); +if ( rc ) +return rc; + +return c->copy (c->priv, src, src_len); +} + +int domain_save (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask,bool dry_run) +{ +struct domain_context c = { + .copy = copy, + .priv = priv, + .log = !dry_run, + }; +struct domain_save_header h = { + .magic =DOMAIN_SAVE_MAGIC , + .version =DOMAIN_SAVE_VERSION , + }; +struct domain_save_header e; +unsigned int i; +int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ; + +domain_pause (d); + + c.desc.typecode =DOMAIN_SAVE_CODE (HEADER ); + + rc =DOMAIN_SAVE_ENTRY (HEADER , &c, d->vcpu[0 ], &h,sizeof (h)); +if ( rc )Quoted 9 lines
xen/common/save.c line 104
Quoted 3 lines
xen/common/save.c new file +3
-0+ goto out; + +for ( i =0 ; i <ARRAY_SIZE (handlers); i++ )AFAIU, with this solution, if there are dependency between records, the dependencies will have to a lower "index". I think we may have some dependency with guest transparent migration such as we need to restore the event ABI before restoring the event channels.
Quoted 1 line
Is that just a downstream implementation detail though? I would hope that there are no ordering dependencies between records.
Until now, I never managed to avoid ordering between records on the Live Update side.
In my previous e-mail, I suggested there are dependency between restoring the event ABI (such as control page...) and restoring the event channels. How do you suggest to solve it without imposing an order between records?
Quoted 14 lines
Quoted 14 lines
xen/common/save.c line 108
Quoted 4 lines
xen/common/save.c new file +4
-0+ { +domain_save_handler save = handlers[i].save; + +if ( (mask && !test_bit (i, &mask)) || !save )The type of mask suggests it is not possible to have more than 32 different types of record if we wanted to be arch agnostic. Even if we focus on 64-bit arch, this is only 64 records.
This is not very future proof, but might be ok if this is not exposed outside of the hypervisor (I haven't looked at the rest of the series yet). However, we at least need a
BUILD_BUG_ON()in place. So please make sureDOMAIN_SAVE_CODE_MAXis always less than 64.
Quoted 5 lines
Quoted 3 lines
Also what if there is a bit set in the mask and the record is not existing? Shouldn't we return an error?
TBH I think 32 will be enough... I would not expect this context space to grow very much, if at all, once transparent migration is working. I think I'll just drop the mask for now; it's not actually clear to me we'll make use of it... just seemed like a nice-to-have.
So far for Live-Update we have 20 records (not counting the Guest Tranparent one). We might be able to do without some, but we also didn't restore all the states yet.
Quoted 55 lines
Quoted 45 lines
Quoted 45 lines
xen/common/save.c new file +44
-0+continue ; + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode = i; + +if ( handlers[i].per_vcpu ) + { +struct vcpu *v; + + for_each_vcpu ( d, v ) + { + c.desc.instance = v->vcpu_id; + + rc =save (v, &c, dry_run); +if ( rc ) + goto out; + } + } +else + { + rc =save (d->vcpu[0 ], &c, dry_run); +if ( rc ) + goto out; + } + } + +memset (&c.desc,0 ,sizeof (c.desc)); + c.desc.typecode =DOMAIN_SAVE_CODE (END ); + + rc =DOMAIN_SAVE_ENTRY (END , &c, d->vcpu[0 ], &e,0 ); + + out: +domain_unpause (d); + +return rc; +} + +int domain_load_entry (struct domain_context *c,unsigned int tc, +const char *name,const struct vcpu *v,void *dst, +size_t dst_len,bool exact) +{ +int rc; + + if ( c->log && tc != DOMAIN_SAVE_CODE(HEADER) &&Quoted 9 lines
xen/common/save.c line 157
Quoted 5 lines
xen/common/save.c new file +5
-0+ tc != DOMAIN_SAVE_CODE(END) ) +gdprintk (XENLOG_INFO ,"%pv load: %s (%lu)\n" , v, name, dst_len); + +BUG_ON (tc != c->desc.typecode); +BUG_ON (v->vcpu_id != c->desc.instance);Is it really warrant to crash the host? What would happen if the values were wrong?
Quoted 1 line
It would mean the code is fairly seriously buggy in that the load handler is trying to load a record other than the type it registered for, or for a vcpu other than the one it was passed.
I understand that, but is warrant to crash the host? Couldn't you just return an error and continue to run?
Quoted 56 lines
Quoted 11 lines
xen/common/save.c line 160
Quoted 4 lines
xen/common/save.c new file +3
-0+ + if ( (exact ? + (dst_len != c->desc.length) : (dst_len < c->desc.length)) ||Using ternary in if is really confusing. How about:
dst_len<c->desc.length|| (exact &&dst_len!=c->desc.length) ||I understand that there would be two check for the exact case but I think it is better than a ternary.
I'm going to re-work this I think.
Quoted 3 lines
However what is the purpose of the param 'exact'? If you set it to false how do you know the size you read?
The point of the exact parameter is that whether the caller can only consume a record of the correct length, or whether it can handle an undersize one which gets zero-padded. (It's the same as the zeroextend option in HVM records).
Quoted 26 lines
Quoted 26 lines
xen/common/save.c new file +25
-0+ !IS_ALIGNED(c->desc.length, 8) ) +return -EINVAL ; + + rc = c->copy (c->priv, dst, c->desc.length); +if ( rc ) +return rc; + +if ( !exact ) + { + dst += c->desc.length; +memset (dst,0 , dst_len - c->desc.length); + } + +return 0 ; +} + +int domain_load (struct domain *d,domain_copy_entry copy,void *priv, +unsigned long mask) +{ +struct domain_context c = { + .copy = copy, + .priv = priv, + .log = true, + }; +struct domain_save_header h, e;Quoted 9 lines
xen/common/save.c line 94
Quoted 6 lines
xen/common/save.c new file +6
-0+int rc; + +ASSERT (d != current->domain); + +if ( d->is_dying ) +return -EINVAL ;What does protect you against the doing dying right after this check?
Quoted 1 line
Nothing. It's just to avoid doing pointless work if we can.
I wasn't thinking about pointless work but whether the rest of the code is relying on a sound domain. Is it going to be fine?
[...]
Quoted 40 lines
Quoted 30 lines
Quoted 30 lines
xen/include/public/save.h new file +30
-0+/* Each entry is preceded by a descriptor */ +struct domain_save_descriptor { +uint16_t typecode; +uint16_t instance; +/* +* Entry length not including this descriptor. Entries must be padded +* to a multiple of 8 bytes to make sure descriptors remain correctly +* aligned. +*/ +uint32_t length; +}; + +/* +* Each entry has a type associated with it. DECLARE_DOMAIN_SAVE_TYPE +* binds these things together. +*/ +#define DECLARE_DOMAIN_SAVE_TYPE (_x, _code, _type) \ + struct __DOMAIN_SAVE_TYPE_##_x { _type t; char c[_code]; }; + +#define DOMAIN_SAVE_TYPE (_x) \ + typeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->t) +#define DOMAIN_SAVE_CODE (_x) \ + (sizeof (((struct __DOMAIN_SAVE_TYPE_##_x *)(0))->c)) +#define DOMAIN_SAVE_MASK (_x) (1u << DOMAIN_SAVE_CODE(_x)) + +/* Terminating entry */ +struct domain_save_end {}; +DECLARE_DOMAIN_SAVE_TYPE(END, 0, struct domain_save_end); + +#define DOMAIN_SAVE_MAGIC 0x53415645Quoted 9 lines
xen/include/public/save.h line 68
Quoted 6 lines
xen/include/public/save.h new file +6
-0+#define DOMAIN_SAVE_VERSION 0x00000001 + +/* Initial entry */ +struct domain_save_header { +uint32_t magic;/* Must be DOMAIN_SAVE_MAGIC */ +uint32_t version;/* Save format version */ Would it make sense to have the version of Xen in the stream?
Quoted 1 line
Why? How would it help?
Let's imagine in 4.14 we introduced a bug in the saving part. This is solved in 4.15 but somehow the version is not bumped. How would you differentiate the streams saved by Xen 4.14 so you can still migrate?
If you record the version of Xen in the record automatically, then you at least have a way to differentiate the two versions.
Cheers,
Signature · 1 line
Julien Grall
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 27 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 03 April 2020 18:24
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 14 lines · Paul Durrant
Quoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
xen/common/save.c line 24
Quoted 7 lines
xen/common/save.c new file +7
-0+ +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 1 line
I don't follow. What is diverging? I explain in the commit comment that this is a parallel
Quoted 9 lines
framework. Do I need to justify why it is not a carbon copy of the HVM one?
Well, they are both restoring/saving guest state. The only difference is the existing one is focusing on HVM state.
So it would make sense long term to have only one hypercall and tell what you want to save. In fact, some of the improvement here would definitely make the HVM one nicer to use (at least in the context of LU).
I guess we could move the HVM save records over to the new framework, but it works for the moment so I don't want to bring it into scope now.
Quoted 3 lines
From the commit message, it is not clear to me why a new framework and
why the infrastructure is at the same time different but not.
An alternative would be to move the HVM save code into common code and then try to adapt it. I think that would result in more code churn and ultimately be harder to review. The extra infrastructure introduced here is fairly minimal and, for the moment, only targeting PV state. As I said above there's nothing stopping the HVM records being ported over later once any initial issues have been shaken out.
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Hi Paul,
Quoted 32 lines · Paul Durrant
Quoted 32 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 03 April 2020 18:24
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 14 lines · Paul Durrant
Quoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
xen/common/save.c line 24
Quoted 7 lines
xen/common/save.c new file +7
-0+ +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 1 line
I don't follow. What is diverging? I explain in the commit comment that this is a parallel
framework. Do I need to justify why it is not a carbon copy of the HVM one?
Well, they are both restoring/saving guest state. The only difference is the existing one is focusing on HVM state.
Quoted 6 lines
Quoted 4 lines
So it would make sense long term to have only one hypercall and tell what you want to save. In fact, some of the improvement here would definitely make the HVM one nicer to use (at least in the context of LU).
I guess we could move the HVM save records over to the new framework, but it works for the moment so I don't want to bring it into scope now.
And I agree, hence why I say "long term" :).
Quoted 6 lines
Quoted 4 lines
From the commit message, it is not clear to me why a new framework and
why the infrastructure is at the same time different but not.An alternative would be to move the HVM save code into common code and then try to adapt it. I think that would result in more code churn and ultimately be harder to review. The extra infrastructure introduced here is fairly minimal and, for the moment, only targeting PV state. As I said above there's nothing stopping the HVM records being ported over later once any initial issues have been shaken out.
Code churn is always going to necessary one day or another if we want to consolidate the two.
Having two frameworks is not without risks. There are a few unknown to be answered:
* Is there any dependency between the two? If yes, what is the ordering?
* The name of the hypercall does not say anything about "PV". So a
contributor could think it would be fine to save/restore new HVM state in the new generic hypercall. Is it going to be an issue? If so, how do we prevent it?
* Are we going to deprecate the existing framework?
I am not suggesting we should not go with two frameworks, but the reasons and implications are not clear to me. Hence, why I think the commit message should be expanded with some rationale.
Cheers,
Signature · 1 line
Julien Grall
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 65 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 06 April 2020 10:08
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 32 lines · Paul Durrant
Quoted 32 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 03 April 2020 18:24
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 14 lines · Paul Durrant
Quoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
xen/common/save.c line 24
Quoted 7 lines
xen/common/save.c new file +7
-0+ +#include <xen/save.h> + +struct domain_context { +bool log; +struct domain_save_descriptor desc; +domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 1 line
I don't follow. What is diverging? I explain in the commit comment that this is a parallel
framework. Do I need to justify why it is not a carbon copy of the HVM one?
Well, they are both restoring/saving guest state. The only difference is the existing one is focusing on HVM state.
Quoted 6 lines
Quoted 4 lines
So it would make sense long term to have only one hypercall and tell what you want to save. In fact, some of the improvement here would definitely make the HVM one nicer to use (at least in the context of LU).
I guess we could move the HVM save records over to the new framework, but it works for the moment so
I don't want to bring it into scope now.
And I agree, hence why I say "long term" :).
Quoted 6 lines
Quoted 4 lines
From the commit message, it is not clear to me why a new framework and
why the infrastructure is at the same time different but not.An alternative would be to move the HVM save code into common code and then try to adapt it. I think
that would result in more code churn and ultimately be harder to review. The extra infrastructure introduced here is fairly minimal and, for the moment, only targeting PV state. As I said above there's nothing stopping the HVM records being ported over later once any initial issues have been shaken out.
Quoted 6 lines
Code churn is always going to necessary one day or another if we want to consolidate the two.
Having two frameworks is not without risks. There are a few unknown to be answered:
* Is there any dependency between the two? If yes, what is the ordering?
There isn't any dependency at the moment so need I say anything? If an ordering dependency is introduced by a future patch then surely that would be time to call it out.
Quoted 4 lines
* The name of the hypercall does not say anything about "PV". So a
contributor could think it would be fine to save/restore new HVM state in the new generic hypercall. Is it going to be an issue? If so, how do we prevent it?
The commit message says:
"Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc."
Do you think this should also appear in a comment? Do we really care? Nothing fundamentally prevents the mechanism being used for HVM state, but that may introduce an ordering dependency.
Quoted 2 lines
* Are we going to deprecate the existing framework?
There is no intention as yet.
Quoted 4 lines
I am not suggesting we should not go with two frameworks, but the reasons and implications are not clear to me. Hence, why I think the commit message should be expanded with some rationale.
Ok, I can add a paragraph to try to explain a little more.
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Hi Paul,
Quoted 65 lines · Paul Durrant
Quoted 65 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 06 April 2020 10:08
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 32 lines · Paul Durrant
Quoted 32 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 03 April 2020 18:24
To: [email protected]; [email protected]
Cc: 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Jan Beulich' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextHi Paul,
Quoted 14 lines · Paul Durrant
Quoted 1 line
-----Original Message-----
[snip]
Quoted 11 lines
Review
Quoted 7 lines
(quoted diff) +7
-0+ +#include <xen/save.h> + +struct domain_context { + bool log; + struct domain_save_descriptor desc; + domain_copy_entry copy;As your new framework is technically an extension of existing one, it would be good to explain why we diverge in the definitions.
Quoted 1 line
I don't follow. What is diverging? I explain in the commit comment that this is a parallel
framework. Do I need to justify why it is not a carbon copy of the HVM one?
Well, they are both restoring/saving guest state. The only difference is the existing one is focusing on HVM state.
Quoted 6 lines
Quoted 4 lines
So it would make sense long term to have only one hypercall and tell what you want to save. In fact, some of the improvement here would definitely make the HVM one nicer to use (at least in the context of LU).
I guess we could move the HVM save records over to the new framework, but it works for the moment so
I don't want to bring it into scope now.
And I agree, hence why I say "long term" :).
Quoted 6 lines
Quoted 4 lines
From the commit message, it is not clear to me why a new framework and
why the infrastructure is at the same time different but not.An alternative would be to move the HVM save code into common code and then try to adapt it. I think
that would result in more code churn and ultimately be harder to review. The extra infrastructure introduced here is fairly minimal and, for the moment, only targeting PV state. As I said above there's nothing stopping the HVM records being ported over later once any initial issues have been shaken out.
Quoted 8 lines
Quoted 6 lines
Code churn is always going to necessary one day or another if we want to consolidate the two.
Having two frameworks is not without risks. There are a few unknown to be answered:
* Is there any dependency between the two? If yes, what is the ordering?There isn't any dependency at the moment so need I say anything? If an ordering dependency is introduced by a future patch then surely that would be time to call it out.
If we don't spell out a dependency now, then the risk is we have to modify the toolstack at the same time as spelling out the dependency.
I am not sure whether this matters thought. This would only affect the save part as the restore part should be just reading records one by one.
Quoted 6 lines
Quoted 5 lines
* The name of the hypercall does not say anything about "PV". So a
contributor could think it would be fine to save/restore new HVM state in the new generic hypercall. Is it going to be an issue? If so, how do we prevent it?
Quoted 6 lines
The commit message says:
"Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc."
This does not seem to cover all the cases. For instance, where would you save IOREQ servers information?
Quoted 2 lines
Do you think this should also appear in a comment? Do we really care? Nothing fundamentally prevents the mechanism being used for HVM state, but that may introduce an ordering dependency.
I don't particularly like the idea to let the contributor decide where "HVM context" or as part of the "Domain context".
This is going to result to unwanted dependency and possibly bikeshedding on where they should be saved.
My preference would be to mark the existing framework as deprecated and force all the new states to be saved as part of the new "Domain Context".
Quoted 4 lines
Quoted 3 lines
* Are we going to deprecate the existing framework?
Quoted 8 lines
There is no intention as yet.
Quoted 4 lines
I am not suggesting we should not go with two frameworks, but the reasons and implications are not clear to me. Hence, why I think the commit message should be expanded with some rationale.
Ok, I can add a paragraph to try to explain a little more.
That would be appreciated. Thank you!
Cheers,
Signature · 1 line
Julien Grall
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 1 line
-----Original Message-----
[snip]
Quoted 7 lines
Quoted 7 lines
Quoted 4 lines
* The name of the hypercall does not say anything about "PV". So a
contributor could think it would be fine to save/restore new HVM state in the new generic hypercall. Is it going to be an issue? If so, how do we prevent it?The commit message says:
Quoted 8 lines
Quoted 4 lines
"Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc."
This does not seem to cover all the cases. For instance, where would you save IOREQ servers information?
Ok, I agree that is ambiguous. I'd still call it PV state but of course it does only relate to HVM guests.
Quoted 4 lines
Quoted 2 lines
Do you think this should also appear in a comment? Do we really care? Nothing fundamentally prevents
the mechanism being used for HVM state, but that may introduce an ordering dependency.
Quoted 9 lines
I don't particularly like the idea to let the contributor decide where "HVM context" or as part of the "Domain context".
This is going to result to unwanted dependency and possibly bikeshedding on where they should be saved.
My preference would be to mark the existing framework as deprecated and force all the new states to be saved as part of the new "Domain Context".
I'm ok with that. Any others have any opinion to the contrary?
Quoted 6 lines
Quoted 6 lines
Quoted 3 lines
* Are we going to deprecate the existing framework?
There is no intention as yet.
Quoted 9 lines
Quoted 6 lines
Quoted 4 lines
I am not suggesting we should not go with two frameworks, but the reasons and implications are not clear to me. Hence, why I think the commit message should be expanded with some rationale.
Ok, I can add a paragraph to try to explain a little more.
That would be appreciated. Thank you!
I'll mention the deprecation of the HVM context interface there too.
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 6 lines · Paul Durrant
Quoted 6 lines
Quoted 1 line
Do you think this should also appear in a comment? Do we really care? Nothing fundamentally prevents
the mechanism being used for HVM state, but that may introduce an ordering dependency.
I don't particularly like the idea to let the contributor decide where "HVM context" or as part of the "Domain context".
Quoted 7 lines
Quoted 5 lines
This is going to result to unwanted dependency and possibly bikeshedding on where they should be saved.
My preference would be to mark the existing framework as deprecated and force all the new states to be saved as part of the new "Domain Context".
I'm ok with that. Any others have any opinion to the contrary?
Well, not in principle, but I won't have a firm opinion until I've actually looked at (relevant parts of) the patches.
Jan
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
In reply to Paul DurrantQuoted 4 lines · Paul Durrant
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
Without looking at the patch details yet, I'm having some difficulty understanding how this is going to work in a safe/correct manner. I suppose for LU the system is in a frozen enough state that snapshotting and copying state like this is okay, but ...
Quoted 9 lines
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
This patch adds the new public header and the low level implementation, entered via the
domain_save()ordomain_load()functions. Subsequent patches will introduce other parts of the framwork, and code that will make use of it within the current version of the libxc migration stream.
... here you suggest (and patch 5 appears to match this) that this is going to be used even in "normal" migration streams. All of the items named are communication vehicles, and hence there are always two sides that can influence the state. For event channels, the other side (which isn't paused) or the hardware (for passed through devices) might signal them, or it (just the former obviously) could close their end, resulting in a state change also for the domain being migrated. If this happens after the snapshot was taken, the state change is lost.
Otoh I'm sure the case was considered, so perhaps I'm simply missing some crucial aspect (which then could do with spelling out in the description of the cover letter).
Jan
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 25 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 01 April 2020 15:51
To: Paul Durrant <[email protected]>
Cc: [email protected]; Andrew Cooper <[email protected]>; George Dunlap <[email protected]>; Ian Jackson <[email protected]>; Julien Grall <[email protected]>; Stefano Stabellini <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextQuoted 4 lines · Paul Durrant
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
Without looking at the patch details yet, I'm having some difficulty understanding how this is going to work in a safe/correct manner. I suppose for LU the system is in a frozen enough state that snapshotting and copying state like this is okay, but ...
Quoted 5 lines
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
Quoted 7 lines
Quoted 4 lines
This patch adds the new public header and the low level implementation, entered via the
domain_save()ordomain_load()functions. Subsequent patches will introduce other parts of the framwork, and code that will make use of it within the current version of the libxc migration stream.... here you suggest (and patch 5 appears to match this) that this is going to be used even in "normal" migration streams.
Well, 'transparent' (or non-cooperative) migration will only work in some cases but it definitely does work.
Quoted 8 lines
All of the
items named are communication vehicles, and hence there are always two sides that can influence the state. For event channels, the other side (which isn't paused) or the hardware (for passed through devices) might signal them, or it (just the former obviously) could close their end, resulting in a state change also for the domain being migrated. If this happens after the snapshot was taken, the state change is lost.
Indeed, which is why we *do* rely on co-operation from the other end of the event channels in the migration case. In the initial case it is likely we'll veto transparent migration unless all event channels are connected to either dom0 or Xen.
Quoted 5 lines
Otoh I'm sure the case was considered, so perhaps I'm simply missing some crucial aspect (which then could do with spelling out in the description of the cover letter).
Does that need to be explained for a series that is just infrastructure? The overall design doc is now committed in the repo (although may need some expansion in future) so I could point at that.
I don't think I'm giving anything away when I say that EC2's downstream code simply (ab)uses HVM save records for transferring the extra state; all I'm trying to do here is create something cleaner onto which I can re-base and upstream the EC2 code.
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 44 lines · Paul Durrant
Quoted 25 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 01 April 2020 15:51
To: Paul Durrant <[email protected]>
Cc: [email protected]; Andrew Cooper <[email protected]>; George Dunlap <[email protected]>; Ian Jackson <[email protected]>; Julien Grall <[email protected]>; Stefano Stabellini <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextQuoted 4 lines · Paul Durrant
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
Without looking at the patch details yet, I'm having some difficulty understanding how this is going to work in a safe/correct manner. I suppose for LU the system is in a frozen enough state that snapshotting and copying state like this is okay, but ...
Quoted 5 lines
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
Quoted 7 lines
Quoted 4 lines
This patch adds the new public header and the low level implementation, entered via the
domain_save()ordomain_load()functions. Subsequent patches will introduce other parts of the framwork, and code that will make use of it within the current version of the libxc migration stream.... here you suggest (and patch 5 appears to match this) that this is going to be used even in "normal" migration streams.
Well, 'transparent' (or non-cooperative) migration will only work in some cases but it definitely does work.
Quoted 8 lines
All of the
items named are communication vehicles, and hence there are always two sides that can influence the state. For event channels, the other side (which isn't paused) or the hardware (for passed through devices) might signal them, or it (just the former obviously) could close their end, resulting in a state change also for the domain being migrated. If this happens after the snapshot was taken, the state change is lost.
Quoted 4 lines
Indeed, which is why we *do* rely on co-operation from the other end of the event channels in the migration case. In the initial case it is likely we'll veto transparent migration unless all event channels are connected to either dom0 or Xen.
Co-operation for "normal" migration, iirc, consists of tearing down and re-establishing everything. There's simply no risk of losing e.g. events this way.
Quoted 7 lines
Quoted 4 lines
Otoh I'm sure the case was considered, so perhaps I'm simply missing some crucial aspect (which then could do with spelling out in the description of the cover letter).
Does that need to be explained for a series that is just infrastructure?
I think so, yes - this infrastructure is pointless to introduce if it doesn't allow fulfilling all requirements. Pointing at the design doc (in the cover letter) may be enough if all aspects are covered by what's there. I wouldn't have assumed using this infrastructure also for co-operative migration to also be mentioned there.
Considering the situation with event channels (all closed), doing what you do for the shared info page is probably going to be fine; large parts of it are in a known state (or need re-filling on the destination) anyway. What other plans do you have for non-LU migration wrt this new infrastructure? If the shared info page is all that's going to get migrated with its help, I'd wonder whether the infrastructure wasn't better conditional upon a LU config option, and the shared info migration was left as it is now.
Quoted 6 lines
The overall design doc is now committed in the repo (although may need some expansion in future) so I could point at that. I don't think I'm giving anything away when I say that EC2's downstream code simply (ab)uses HVM save records for transferring the extra state; all I'm trying to do here is create something cleaner onto which I can re-base and upstream the EC2 code.
That was my guess, indeed.
Jan
RE: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 55 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 02 April 2020 12:08
To: [email protected]
Cc: [email protected]; 'Andrew Cooper' <[email protected]>; 'George Dunlap' <[email protected]>; 'Ian Jackson' <[email protected]>; 'Julien Grall' <[email protected]>; 'Stefano Stabellini' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextQuoted 30 lines · Paul Durrant
Quoted 30 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 01 April 2020 15:51
To: Paul Durrant <[email protected]>
Cc: [email protected]; Andrew Cooper <[email protected]>; George Dunlap <[email protected]>; Ian Jackson <[email protected]>; Julien Grall <[email protected]>; Stefano Stabellini <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' contextQuoted 4 lines · Paul Durrant
Domain context is state held in the hypervisor that does not come under the category of 'HVM state' but is instead 'PV state' that is common between PV guests and enlightened HVM guests (i.e. those that have PV drivers) such as event channel state, grant entry state, etc.
Without looking at the patch details yet, I'm having some difficulty understanding how this is going to work in a safe/correct manner. I suppose for LU the system is in a frozen enough state that snapshotting and copying state like this is okay, but ...
Quoted 9 lines
To allow enlightened HVM guests to be migrated without their co-operation it will be necessary to transfer such state along with the domain's memory image, architectural state, etc. This framework is introduced for that purpose.
This patch adds the new public header and the low level implementation, entered via the
domain_save()ordomain_load()functions. Subsequent patches will introduce other parts of the framwork, and code that will make use of it within the current version of the libxc migration stream.Quoted 4 lines
Quoted 2 lines
... here you suggest (and patch 5 appears to match this) that this is going to be used even in "normal" migration streams.
Well, 'transparent' (or non-cooperative) migration will only work in some cases but it definitely
does work.
Quoted 10 lines
Quoted 9 lines
All of the
items named are communication vehicles, and hence there are always two sides that can influence the state. For event channels, the other side (which isn't paused) or the hardware (for passed through devices) might signal them, or it (just the former obviously) could close their end, resulting in a state change also for the domain being migrated. If this happens after the snapshot was taken, the state change is lost.
Quoted 8 lines
Quoted 4 lines
Indeed, which is why we *do* rely on co-operation from the other end of the event channels in the migration case. In the initial case it is likely we'll veto transparent migration unless all event channels are connected to either dom0 or Xen.
Co-operation for "normal" migration, iirc, consists of tearing down and re-establishing everything. There's simply no risk of losing e.g. events this way.
No, indeed, everything basically has to be re-established from scratch for normal migration. Transparent migration, as it is implemented at the moment, does rely on injecting potentially spurious events on resume. I think the alternative would be to have the PV backends send an event when they re-connect to a shared ring rather than having Xen do it.
Quoted 6 lines
Quoted 6 lines
Quoted 5 lines
Otoh I'm sure the case was considered, so perhaps I'm simply missing some crucial aspect (which then could do with spelling out in the description of the cover letter).
Quoted 8 lines
Quoted 2 lines
Does that need to be explained for a series that is just infrastructure?
I think so, yes - this infrastructure is pointless to introduce if it doesn't allow fulfilling all requirements. Pointing at the design doc (in the cover letter) may be enough if all aspects are covered by what's there. I wouldn't have assumed using this infrastructure also for co-operative migration to also be mentioned there.
No, I can mention the plan to use it to replace existing libxc migration records in the cover letter even though it is stated in the comment for patch #5.
Quoted 6 lines
Considering the situation with event channels (all closed), doing what you do for the shared info page is probably going to be fine; large parts of it are in a known state (or need re-filling on the destination) anyway. What other plans do you have for non-LU migration wrt this new infrastructure?
Well, as discussed above, event channels are one, then there's the grant table. The downstream code as a record for the wallclock but I think the RTC covers that now that added the code to preserve the offset. We also have records for vcpu timers and runstate, but I'm not convinced we actually need those.
Quoted 5 lines
If the shared info page is
all that's going to get migrated with its help, I'd wonder whether the infrastructure wasn't better conditional upon a LU config option, and the shared info migration was left as it is now.
The shared info is also migrated for HVM guests so it would have meant moving the mapping code around anyway, thus it seems sensible to use the new domain context for that for PV guests in their normal migration stream anyway.
Paul
Re: [PATCH 1/5] xen/common: introduce a new framework for save/restore of 'domain' context
Quoted 11 lines · Paul Durrant
Quoted 9 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 02 April 2020 12:08Considering the situation with event channels (all closed), doing what you do for the shared info page is probably going to be fine; large parts of it are in a known state (or need re-filling on the destination) anyway. What other plans do you have for non-LU migration wrt this new infrastructure?
Well, as discussed above, event channels are one, then there's the grant table. The downstream code as a record for the wallclock but I think the RTC covers that now that added the code to preserve the offset. We also have records for vcpu timers and runstate, but I'm not convinced we actually need those.
Timers may need preserving, but runstate may be avoidable. Depends on whether the accumulation in time[4] is fine to start from zero again after (transparent) migration.
Quoted 6 lines
Quoted 4 lines
If the shared info page is
all that's going to get migrated with its help, I'd wonder whether the infrastructure wasn't better conditional upon a LU config option, and the shared info migration was left as it is now.The shared info is also migrated for HVM guests so it would have meant moving the mapping code around anyway, thus it seems sensible to use the new domain context for that for PV guests in their normal migration stream anyway.
Hmm, okay - I'll see to get to look at the actual code then.
Jan
PATCH 2/5 xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext
In reply to Paul DurrantThese domctls provide a mechanism to get and set domain context from the toolstack.
---
Cc: Daniel De Graaf <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>
--- tools/flask/policy/modules/xen.if | 4 +- tools/libxc/include/xenctrl.h | 11 +++ tools/libxc/xc_domain.c | 52 +++++++++++++ xen/common/domctl.c | 115 ++++++++++++++++++++++++++++ xen/include/public/domctl.h | 41 +++++++++- xen/xsm/flask/hooks.c | 6 ++ xen/xsm/flask/policy/access_vectors | 4 + 7 files changed, 230 insertions(+), 3 deletions(-)
tools/flask/policy/modules/xen.if +2 -2
@@ -53,7 +53,7 @@ define(`create_domain_common', ` allow $1 $2:domain2 { set_cpu_policy settsc setscheduler setclaim set_vnumainfo get_vnumainfo cacheflush psr_cmt_op psr_alloc soft_reset - resource_map get_cpu_policy }; + resource_map get_cpu_policy setcontext }; allow $1 $2:security check_context; allow $1 $2:shadow enable; allow $1 $2:mmu { map_read map_write adjust memorymap physmap pinpage mmuext_op updatemp };
@@ -97,7 +97,7 @@ define(`migrate_domain_out', ` allow $1 $2:hvm { gethvmc getparam }; allow $1 $2:mmu { stat pageinfo map_read }; allow $1 $2:domain { getaddrsize getvcpucontext pause destroy }; - allow $1 $2:domain2 gettsc; + allow $1 $2:domain2 { gettsc getcontext }; allow $1 $2:shadow { enable disable logdirty }; ')
tools/libxc/include/xenctrl.h +11 -0
@@ -867,6 +867,17 @@ int xc_domain_hvm_setcontext(xc_interface *xch,uint8_t *hvm_ctxt, uint32_t size); +int xc_domain_getcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size); +int xc_domain_setcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size); +/** * This function will return guest IO ABI protocol *
tools/libxc/xc_domain.c +52 -0
58 lines not on this page · raw message
xen/common/domctl.c +115 -0
133 lines not on this page · raw message
xen/include/public/domctl.h +40 -1
65 lines not on this page · raw message
xen/xsm/flask/hooks.c +6 -0
@@ -744,6 +744,12 @@ static int flask_domctl(struct domain *d, int cmd)case XEN_DOMCTL_get_cpu_policy:return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__GET_CPU_POLICY ); +case XEN_DOMCTL_setdomaincontext: +return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__SETCONTEXT ); + +case XEN_DOMCTL_getdomaincontext: +return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__GETCONTEXT ); +default :return avc_unknown_permission ("domctl" , cmd); }
xen/xsm/flask/policy/access_vectors +4 -0
@@ -245,6 +245,10 @@ class domain2 resource_map # XEN_DOMCTL_get_cpu_policy get_cpu_policy +# XEN_DOMCTL_setdomaincontext + setcontext +# XEN_DOMCTL_getdomaincontext + getcontext } # Similar to class domain, but primarily contains domctls related to HVM domains
Signature · 1 line
2.20.1
Re: [PATCH 2/5] xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext
Hi Paul,
Quoted 50 lines · Paul Durrant
These domctls provide a mechanism to get and set domain context from the toolstack.
Signed-off-by Paul Durrant <[email protected]>---
Cc: Daniel De Graaf <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>--- tools/flask/policy/modules/xen.if | 4 +- tools/libxc/include/xenctrl.h | 11 +++ tools/libxc/xc_domain.c | 52 +++++++++++++ xen/common/domctl.c | 115 ++++++++++++++++++++++++++++ xen/include/public/domctl.h | 41 +++++++++- xen/xsm/flask/hooks.c | 6 ++ xen/xsm/flask/policy/access_vectors | 4 + 7 files changed, 230 insertions(+), 3 deletions(-)tools/flask/policy/modules/xen.if +2
-2@@ -53,7 +53,7 @@ define(`create_domain_common', ` allow $1 $2:domain2 { set_cpu_policy settsc setscheduler setclaim set_vnumainfo get_vnumainfo cacheflush psr_cmt_op psr_alloc soft_reset - resource_map get_cpu_policy }; + resource_map get_cpu_policy setcontext }; allow $1 $2:security check_context; allow $1 $2:shadow enable; allow $1 $2:mmu { map_read map_write adjust memorymap physmap pinpage mmuext_op updatemp };@@ -97,7 +97,7 @@ define(`migrate_domain_out', ` allow $1 $2:hvm { gethvmc getparam }; allow $1 $2:mmu { stat pageinfo map_read }; allow $1 $2:domain { getaddrsize getvcpucontext pause destroy }; - allow $1 $2:domain2 gettsc; + allow $1 $2:domain2 { gettsc getcontext }; allow $1 $2:shadow { enable disable logdirty }; ')tools/libxc/include/xenctrl.h +0
-0
tools/libxc/include/xenctrl.h int xc_domain_hvm_setcontext(xc_interface *xch, · line 873
Quoted 7 lines
tools/libxc/include/xenctrl.h +4
-0@@ -867,6 +867,17 @@ int xc_domain_hvm_setcontext(xc_interface *xch,uint8_t *hvm_ctxt, uint32_t size); +int xc_domain_getcontext(xc_interface *xch, + uint32_t domid, + uint32_t mask, + uint8_t *ctxt_buf,
Why do you use uint8_t rather than void here?
Quoted 32 lines
tools/libxc/include/xenctrl.h +7
-0+ uint32_t size); +int xc_domain_setcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size); +/** * This function will return guest IO ABI protocol * tools/libxc/xc_domain.c +14
-0@@ -537,6 +537,58 @@ int xc_domain_hvm_setcontext(xc_interface *xch,return ret; } +int xc_domain_getcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size) +{ +int ret; +DECLARE_DOMCTL ; +DECLARE_HYPERCALL_BOUNCE (ctxt_buf, size,XC_HYPERCALL_BUFFER_BOUNCE_OUT ); + +if (xc_hypercall_bounce_pre (xch, ctxt_buf) ) +return -1 ; + + domctl.cmd = XEN_DOMCTL_getdomaincontext;
tools/libxc/xc_domain.c int xc_domain_hvm_setcontext(xc_interface *xch, · line 563
Quoted 10 lines
tools/libxc/xc_domain.c +10
-0+ domctl.domain = domid; + domctl.u.domaincontext.mask = mask; + domctl.u.domaincontext.size = size; +set_xen_guest_handle (domctl.u.domaincontext.buffer, ctxt_buf); + + ret =do_domctl (xch, &domctl); + +xc_hypercall_bounce_post (xch, ctxt_buf); + +return !ret ? domctl.u.domaincontext.size :-1 ;
Looking at the domctl interface, size would be a 32-bit unsigned value. However the return is a 32-bit signed value. This is a call for trouble if the size is too big.
tools/libxc/xc_domain.c int xc_domain_hvm_setcontext(xc_interface *xch, · line 569
Quoted 6 lines
tools/libxc/xc_domain.c +6
-0+} + +int xc_domain_setcontext(xc_interface *xch, + uint32_t domid, + uint32_t mask, + uint8_t *ctxt_buf,
The context is not meant to be modified. So this should really be const.
Quoted 43 lines
tools/libxc/xc_domain.c +22
-0+ uint32_t size) +{ +int ret; +DECLARE_DOMCTL ; +DECLARE_HYPERCALL_BOUNCE (ctxt_buf, size,XC_HYPERCALL_BUFFER_BOUNCE_IN ); + +if (xc_hypercall_bounce_pre (xch, ctxt_buf) ) +return -1 ; + + domctl.cmd = XEN_DOMCTL_setdomaincontext; + domctl.domain = domid; + domctl.u.domaincontext.mask = mask; + domctl.u.domaincontext.size = size; +set_xen_guest_handle (domctl.u.domaincontext.buffer, ctxt_buf); + + ret =do_domctl (xch, &domctl); + +xc_hypercall_bounce_post (xch, ctxt_buf); + +return ret; +} + int xc_vcpu_getcontext(xc_interface *xch, uint32_t domid, uint32_t vcpu,xen/common/domctl.c +3
-0@@ -25,6 +25,7 @@#include <xen/hypercall.h> #include <xen/vm_event.h> #include <xen/monitor.h> +#include <xen/save.h> #include <asm/current.h> #include <asm/irq.h> #include <asm/page.h> @@ -358,6 +359,111 @@ static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo,return ERR_PTR (ret); } +struct domctl_context +{
xen/common/domctl.c static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo, · line 373
Quoted 10 lines
xen/common/domctl.c +10
-0+void *buffer; +size_t len; +size_t cur; +}; + +static int accumulate_size (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + + c->len += len;
What does prevent overflow?
Quoted 28 lines
xen/common/domctl.c +28
-0+ +return 0 ; +} + +static int save_data (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + +if ( c->len - c->cur < len ) +return -ENOSPC ; + +memcpy (c->buffer + c->cur, data, len); + c->cur += len; + +return 0 ; +} + +static int getdomaincontext (struct domain *d, +struct xen_domctl_domaincontext *dc) +{ +struct domctl_context c = { }; +int rc; + +if ( d == current->domain ) +return -EPERM ; + +if (guest_handle_is_null (dc->buffer) )/* query for buffer size */ +
xen/common/domctl.c static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo, · line 411
Quoted 10 lines
xen/common/domctl.c +10
-0+ { +if ( dc->size ) +return -EINVAL ; + +/* dry run to acquire buffer size */ + rc =domain_save (d, accumulate_size, &c, dc->mask, true); +if ( rc ) +return rc; + + dc->size = c.len;
len is a size_t (so 64-bit on 64-bit arch) value, but size is 32-bit. So we return an error if the stream is going to be bigger than 4G?
Quoted 115 lines
xen/common/domctl.c +64
-0+return 0 ; + } + + c.len = dc->size; + c.buffer =xmalloc_bytes (c.len); +if ( !c.buffer ) +return -ENOMEM ; + + rc =domain_save (d, save_data, &c, dc->mask, false); + + dc->size = c.cur; +if ( !rc &©_to_guest (dc->buffer, c.buffer, dc->size) ) + rc = -EFAULT ; + +xfree (c.buffer); + +return rc; +} + +static int load_data (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + +if ( c->len - c->cur < len ) +return -ENODATA ; + +if ( data ) +memcpy (data, c->buffer + c->cur, len); + + c->cur += len; + +return 0 ; +} + +static int setdomaincontext (struct domain *d, +const struct xen_domctl_domaincontext *dc) +{ +struct domctl_context c = { .len = dc->size }; +int rc; + +if ( d == current->domain ) +return -EPERM ; + + c.buffer =xmalloc_bytes (c.len); +if ( !c.buffer ) +return -ENOMEM ; + + rc = !copy_from_guest (c.buffer, dc->buffer, c.len) ? +domain_load (d, load_data, &c, dc->mask) : -EFAULT ; + +xfree (c.buffer); + +return rc; +} +long do_domctl (XEN_GUEST_HANDLE_PARAM (xen_domctl_t) u_domctl) {long ret =0 ;@@ -942,6 +1048,15 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) copyback =1 ;break ; +case XEN_DOMCTL_getdomaincontext: + ret =getdomaincontext (d, &op->u.domaincontext); + copyback = !ret; +break ; + +case XEN_DOMCTL_setdomaincontext: + ret =setdomaincontext (d, &op->u.domaincontext); +break ; +default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/include/public/domctl.h +25
-1@@ -38,7 +38,7 @@#include "hvm/save.h" #include "memory.h" -#define XEN_DOMCTL_INTERFACE_VERSION 0x00000012 +#define XEN_DOMCTL_INTERFACE_VERSION 0x00000013/* * NB. xen_domctl.domain is an IN/OUT parameter for this operation. @@ -1129,6 +1129,42 @@ struct xen_domctl_vuart_op {*/ }; +/* +* Get/Set domain PV context. The same struct xen_domctl_domaincontext +* is used for both commands but with slightly different field semantics +* as follows: +* +* XEN_DOMCTL_getdomaincontext +* --------------------------- +* +* buffer (IN): The buffer into which the context data should be +* copied, or NULL to query the buffer size that should +* be allocated. +* size (IN/OUT): If 'buffer' is NULL then the value passed in must be +* zero, and the value passed out will be the size of the +* buffer to allocate. +* If 'buffer' is non-NULL then the value passed in must +* be the size of the buffer into which data may be copied. +* mask (IN): See comment on domain_save/load() in xen/save.h. +* +* XEN_DOMCTL_setdomaincontext +* --------------------------- +* +* buffer (IN): The buffer from which the context data should be +* copied. +* size (IN): The size of the buffer from which data may be copied.
xen/include/public/domctl.h struct xen_domctl_vuart_op { · line 1165
Quoted 10 lines
xen/include/public/domctl.h +10
-0+* This data must include DOMAIN_SAVE_CODE_HEADER at the +* start and terminate with a DOMAIN_SAVE_CODE_END record. +* Any data beyond the DOMAIN_SAVE_CODE_END record will be +* ignored. +* mask (IN): See comment on domain_save/load() in xen/save.h. +*/ +struct xen_domctl_domaincontext { +uint32_t size; +uint32_t mask; +XEN_GUEST_HANDLE_64 (uint8) buffer;
Should not it be a void handle?
Also, in the case of XEN_DOMCTL_setdomaincontext, the buffer is not meant to be modified by the hypervisor. So I would rather introduce two separate structure so we can enforce the const.
Quoted 45 lines
xen/include/public/domctl.h +5
-0+}; +struct xen_domctl {uint32_t cmd;#define XEN_DOMCTL_createdomain 1@@ -1210,6 +1246,8 @@ struct xen_domctl {#define XEN_DOMCTL_vuart_op 81#define XEN_DOMCTL_get_cpu_policy 82#define XEN_DOMCTL_set_cpu_policy 83 +#define XEN_DOMCTL_getdomaincontext 84 +#define XEN_DOMCTL_setdomaincontext 85#define XEN_DOMCTL_gdbsx_guestmemio 1000#define XEN_DOMCTL_gdbsx_pausevcpu 1001#define XEN_DOMCTL_gdbsx_unpausevcpu 1002@@ -1270,6 +1308,7 @@ struct xen_domctl {struct xen_domctl_monitor_op monitor_op;struct xen_domctl_psr_alloc psr_alloc;struct xen_domctl_vuart_op vuart_op; +struct xen_domctl_domaincontext domaincontext;uint8_t pad[128 ]; } u; };xen/xsm/flask/hooks.c +6
-0@@ -744,6 +744,12 @@ static int flask_domctl(struct domain *d, int cmd)case XEN_DOMCTL_get_cpu_policy:return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__GET_CPU_POLICY ); +case XEN_DOMCTL_setdomaincontext: +return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__SETCONTEXT ); + +case XEN_DOMCTL_getdomaincontext: +return current_has_perm (d,SECCLASS_DOMAIN2 ,DOMAIN2__GETCONTEXT ); +default :return avc_unknown_permission ("domctl" , cmd); }xen/xsm/flask/policy/access_vectors +0
-0
xen/xsm/flask/policy/access_vectors class domain2 · line 254
Quoted 10 lines
xen/xsm/flask/policy/access_vectors +4
-0@@ -245,6 +245,10 @@ class domain2 resource_map # XEN_DOMCTL_get_cpu_policy get_cpu_policy +# XEN_DOMCTL_setdomaincontext + setcontext +# XEN_DOMCTL_getdomaincontext + getcontext } # Similar to class domain, but primarily contains domctls related to HVM domains
Cheers,
Signature · 1 line
Julien Grall
RE: [PATCH 2/5] xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext
Quoted 119 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 01 April 2020 14:42
To: Paul Durrant <[email protected]>; [email protected]
Cc: Daniel De Graaf <[email protected]>; Ian Jackson <[email protected]>; Wei Liu <[email protected]>; Andrew Cooper <[email protected]>; George Dunlap <[email protected]>; Jan Beulich <[email protected]>; Stefano Stabellini <[email protected]>
Subject: Re: [PATCH 2/5] xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontextHi Paul,
Quoted 50 lines · Paul Durrant
These domctls provide a mechanism to get and set domain context from the toolstack.
Signed-off-by Paul Durrant <[email protected]>---
Cc: Daniel De Graaf <[email protected]>
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>--- tools/flask/policy/modules/xen.if | 4 +- tools/libxc/include/xenctrl.h | 11 +++ tools/libxc/xc_domain.c | 52 +++++++++++++ xen/common/domctl.c | 115 ++++++++++++++++++++++++++++ xen/include/public/domctl.h | 41 +++++++++- xen/xsm/flask/hooks.c | 6 ++ xen/xsm/flask/policy/access_vectors | 4 + 7 files changed, 230 insertions(+), 3 deletions(-)tools/flask/policy/modules/xen.if +2
-2@@ -53,7 +53,7 @@ define(`create_domain_common', ` allow $1 $2:domain2 { set_cpu_policy settsc setscheduler setclaim set_vnumainfo get_vnumainfo cacheflush psr_cmt_op psr_alloc soft_reset - resource_map get_cpu_policy }; + resource_map get_cpu_policy setcontext }; allow $1 $2:security check_context; allow $1 $2:shadow enable; allow $1 $2:mmu { map_read map_write adjust memorymap physmap pinpage mmuext_op updatemp };@@ -97,7 +97,7 @@ define(`migrate_domain_out', ` allow $1 $2:hvm { gethvmc getparam }; allow $1 $2:mmu { stat pageinfo map_read }; allow $1 $2:domain { getaddrsize getvcpucontext pause destroy }; - allow $1 $2:domain2 gettsc; + allow $1 $2:domain2 { gettsc getcontext }; allow $1 $2:shadow { enable disable logdirty }; ')tools/libxc/include/xenctrl.h +0
-0tools/libxc/include/xenctrl.h int xc_domain_hvm_setcontext(xc_interface *xch, · line 873
Quoted 7 lines
tools/libxc/include/xenctrl.h +4
-0@@ -867,6 +867,17 @@ int xc_domain_hvm_setcontext(xc_interface *xch,uint8_t *hvm_ctxt, uint32_t size); +int xc_domain_getcontext(xc_interface *xch, + uint32_t domid, + uint32_t mask, + uint8_t *ctxt_buf,Why do you use
uint8_trather than void here?Quoted 32 lines
tools/libxc/include/xenctrl.h +7
-0+ uint32_t size); +int xc_domain_setcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size); +/** * This function will return guest IO ABI protocol * tools/libxc/xc_domain.c +14
-0@@ -537,6 +537,58 @@ int xc_domain_hvm_setcontext(xc_interface *xch,return ret; } +int xc_domain_getcontext (xc_interface *xch, +uint32_t domid, +uint32_t mask, +uint8_t *ctxt_buf, +uint32_t size) +{ +int ret; +DECLARE_DOMCTL ; +DECLARE_HYPERCALL_BOUNCE (ctxt_buf, size,XC_HYPERCALL_BUFFER_BOUNCE_OUT ); + +if (xc_hypercall_bounce_pre (xch, ctxt_buf) ) +return -1 ; + + domctl.cmd = XEN_DOMCTL_getdomaincontext;tools/libxc/xc_domain.c int xc_domain_hvm_setcontext(xc_interface *xch, · line 563
Quoted 10 lines
tools/libxc/xc_domain.c +10
-0+ domctl.domain = domid; + domctl.u.domaincontext.mask = mask; + domctl.u.domaincontext.size = size; +set_xen_guest_handle (domctl.u.domaincontext.buffer, ctxt_buf); + + ret =do_domctl (xch, &domctl); + +xc_hypercall_bounce_post (xch, ctxt_buf); + +return !ret ? domctl.u.domaincontext.size :-1 ;Looking at the domctl interface, size would be a 32-bit unsigned value. However the return is a 32-bit signed value. This is a call for trouble if the size is too big.
Quoted 9 lines
tools/libxc/xc_domain.c int xc_domain_hvm_setcontext(xc_interface *xch, · line 569
Quoted 6 lines
tools/libxc/xc_domain.c +6
-0+} + +int xc_domain_setcontext(xc_interface *xch, + uint32_t domid, + uint32_t mask, + uint8_t *ctxt_buf,The context is not meant to be modified. So this should really be const.
All of the above were basically inherited via cut'n'paste from the HVM save code. I'll re-work it.
Quoted 41 lines
Quoted 41 lines
tools/libxc/xc_domain.c +22
-0+ uint32_t size) +{ +int ret; +DECLARE_DOMCTL ; +DECLARE_HYPERCALL_BOUNCE (ctxt_buf, size,XC_HYPERCALL_BUFFER_BOUNCE_IN ); + +if (xc_hypercall_bounce_pre (xch, ctxt_buf) ) +return -1 ; + + domctl.cmd = XEN_DOMCTL_setdomaincontext; + domctl.domain = domid; + domctl.u.domaincontext.mask = mask; + domctl.u.domaincontext.size = size; +set_xen_guest_handle (domctl.u.domaincontext.buffer, ctxt_buf); + + ret =do_domctl (xch, &domctl); + +xc_hypercall_bounce_post (xch, ctxt_buf); + +return ret; +} + int xc_vcpu_getcontext(xc_interface *xch, uint32_t domid, uint32_t vcpu,xen/common/domctl.c +1
-0@@ -25,6 +25,7 @@#include <xen/hypercall.h> #include <xen/vm_event.h> #include <xen/monitor.h> +#include <xen/save.h> #include <asm/current.h> #include <asm/irq.h> #include <asm/page.h> @@ -358,6 +359,111 @@ static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo,return ERR_PTR (ret); }
Quoted 15 lines
xen/common/domctl.c static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo, · line 373
Quoted 12 lines
xen/common/domctl.c +12
-0+struct domctl_context +{ +void *buffer; +size_t len; +size_t cur; +}; + +static int accumulate_size (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + + c->len += len;What does prevent overflow?
Good point. That needs to be checked.
Quoted 33 lines
Quoted 33 lines
xen/common/domctl.c +33
-0+ +return 0 ; +} + +static int save_data (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + +if ( c->len - c->cur < len ) +return -ENOSPC ; + +memcpy (c->buffer + c->cur, data, len); + c->cur += len; + +return 0 ; +} + +static int getdomaincontext (struct domain *d, +struct xen_domctl_domaincontext *dc) +{ +struct domctl_context c = { }; +int rc; + +if ( d == current->domain ) +return -EPERM ; + +if (guest_handle_is_null (dc->buffer) )/* query for buffer size */ + + { +if ( dc->size ) +return -EINVAL ; + +/* dry run to acquire buffer size */
Quoted 9 lines
xen/common/domctl.c static struct vnuma_info *vnuma_init(const struct xen_domctl_vnuma *uinfo, · line 411
Quoted 5 lines
xen/common/domctl.c +5
-0+ rc =domain_save (d, accumulate_size, &c, dc->mask, true); +if ( rc ) +return rc; + + dc->size = c.len;len is a
size_t(so 64-bit on 64-bit arch) value, but size is 32-bit. So we return an error if the stream is going to be bigger than 4G?
I'd hope a 4G stream is unlikely but I'll expand size to 64 bits.
Quoted 119 lines
Quoted 119 lines
xen/common/domctl.c +64
-0+return 0 ; + } + + c.len = dc->size; + c.buffer =xmalloc_bytes (c.len); +if ( !c.buffer ) +return -ENOMEM ; + + rc =domain_save (d, save_data, &c, dc->mask, false); + + dc->size = c.cur; +if ( !rc &©_to_guest (dc->buffer, c.buffer, dc->size) ) + rc = -EFAULT ; + +xfree (c.buffer); + +return rc; +} + +static int load_data (void *priv,void *data,size_t len) +{ +struct domctl_context *c = priv; + +if ( c->len - c->cur < len ) +return -ENODATA ; + +if ( data ) +memcpy (data, c->buffer + c->cur, len); + + c->cur += len; + +return 0 ; +} + +static int setdomaincontext (struct domain *d, +const struct xen_domctl_domaincontext *dc) +{ +struct domctl_context c = { .len = dc->size }; +int rc; + +if ( d == current->domain ) +return -EPERM ; + + c.buffer =xmalloc_bytes (c.len); +if ( !c.buffer ) +return -ENOMEM ; + + rc = !copy_from_guest (c.buffer, dc->buffer, c.len) ? +domain_load (d, load_data, &c, dc->mask) : -EFAULT ; + +xfree (c.buffer); + +return rc; +} +long do_domctl (XEN_GUEST_HANDLE_PARAM (xen_domctl_t) u_domctl) {long ret =0 ;@@ -942,6 +1048,15 @@ long do_domctl(XEN_GUEST_HANDLE_PARAM(xen_domctl_t) u_domctl) copyback =1 ;break ; +case XEN_DOMCTL_getdomaincontext: + ret =getdomaincontext (d, &op->u.domaincontext); + copyback = !ret; +break ; + +case XEN_DOMCTL_setdomaincontext: + ret =setdomaincontext (d, &op->u.domaincontext); +break ; +default : ret =arch_do_domctl (op, d, u_domctl);break ;xen/include/public/domctl.h +29
-1@@ -38,7 +38,7 @@#include "hvm/save.h" #include "memory.h" -#define XEN_DOMCTL_INTERFACE_VERSION 0x00000012 +#define XEN_DOMCTL_INTERFACE_VERSION 0x00000013/* * NB. xen_domctl.domain is an IN/OUT parameter for this operation. @@ -1129,6 +1129,42 @@ struct xen_domctl_vuart_op {*/ }; +/* +* Get/Set domain PV context. The same struct xen_domctl_domaincontext +* is used for both commands but with slightly different field semantics +* as follows: +* +* XEN_DOMCTL_getdomaincontext +* --------------------------- +* +* buffer (IN): The buffer into which the context data should be +* copied, or NULL to query the buffer size that should +* be allocated. +* size (IN/OUT): If 'buffer' is NULL then the value passed in must be +* zero, and the value passed out will be the size of the +* buffer to allocate. +* If 'buffer' is non-NULL then the value passed in must +* be the size of the buffer into which data may be copied. +* mask (IN): See comment on domain_save/load() in xen/save.h. +* +* XEN_DOMCTL_setdomaincontext +* --------------------------- +* +* buffer (IN): The buffer from which the context data should be +* copied. +* size (IN): The size of the buffer from which data may be copied. +* This data must include DOMAIN_SAVE_CODE_HEADER at the +* start and terminate with a DOMAIN_SAVE_CODE_END record. +* Any data beyond the DOMAIN_SAVE_CODE_END record will be +* ignored.
Quoted 9 lines
xen/include/public/domctl.h struct xen_domctl_vuart_op { · line 1165
Quoted 6 lines
xen/include/public/domctl.h +6
-0+* mask (IN): See comment on domain_save/load() in xen/save.h. +*/ +struct xen_domctl_domaincontext { +uint32_t size; +uint32_t mask; +XEN_GUEST_HANDLE_64 (uint8) buffer;Should not it be a void handle?
Perhaps.
Quoted 4 lines
Also, in the case of
XEN_DOMCTL_setdomaincontext, the buffer is not meant to be modified by the hypervisor. So I would rather introduce two separate structure so we can enforce the const.
Can handles be meaningfully const?
Paul
Re: [PATCH 2/5] xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext
Quoted 10 lines · Paul Durrant
Quoted 8 lines
-----Original Message-----
From: Julien Grall <[email protected]>
Sent: 01 April 2020 14:42On 27/03/2020 18:50, Paul Durrant wrote:
Also, in the case ofXEN_DOMCTL_setdomaincontext, the buffer is not meant to be modified by the hypervisor. So I would rather introduce two separate structure so we can enforce the const.Can handles be meaningfully const?
Yes, see e.g. Julien's recent patch to force honoring const-ness in some cases where it wasn't enforced so far. Luckily there look to not have crept in any violators.
Jan
Re: [PATCH 2/5] xen/common/domctl: introduce XEN_DOMCTL_get/setdomaincontext
In reply to Paul DurrantQuoted 25 lines · Paul Durrant
+ *
XEN_DOMCTL_getdomaincontext
+ * ---------------------------
+ *+ * buffer (IN): The buffer into which the context data should be + * copied, or NULL to query the buffer size that should + * be allocated. + * size (IN/OUT): If 'buffer' is NULL then the value passed in must be + * zero, and the value passed out will be the size of the + * buffer to allocate. + * If 'buffer' is non-NULL then the value passed in must + * be the size of the buffer into which data may be copied. + * mask (IN): See comment on domain_save/load() in xen/save.h.+ *
+ *XEN_DOMCTL_setdomaincontext
+ * ---------------------------
+ *+ * buffer (IN): The buffer from which the context data should be + * copied. + * size (IN): The size of the buffer from which data may be copied. + * This data must include DOMAIN_SAVE_CODE_HEADER at the + * start and terminate with a DOMAIN_SAVE_CODE_END record. + * Any data beyond the DOMAIN_SAVE_CODE_END record will be + * ignored. + * mask (IN): See comment on domain_save/load() in xen/save.h.+ */
xen/include/public/domctl.h struct xen_domctl_vuart_op { · line 1164
Quoted 3 lines
xen/include/public/domctl.h +3
-0+struct xen_domctl_domaincontext { +uint32_t size; +uint32_t mask;
I actually forgot to comment on the mask. We are restricting ourself to 32 different type of records. I don't think this is going to fly very far.
But what is the expected use of 'mask'. Are we expecting the toolstack to say which records should be saved/restored?
Cheers,
Signature · 1 line
Julien Grall
PATCH 3/5 tools/misc: add xen-ctx to present domain context
In reply to Paul DurrantThis tools is analogous to 'xen-hvmctx' which presents HVM context. Subsequent patches will add 'dump' functions when new records are introduced.
---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
--- .gitignore | 1 + tools/misc/Makefile | 4 ++ tools/misc/xen-ctx.c | 144 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 149 insertions(+) create mode 100644 tools/misc/xen-ctx.c
.gitignore +1 -0
@@ -206,6 +206,7 @@ tools/misc/cpuperf/cpuperf-xen tools/misc/xc_shadow tools/misc/xen_cpuperf tools/misc/xen-cpuid +tools/misc/xen-ctx tools/misc/xen-detect tools/misc/xen-diag tools/misc/xen-tmem-list-parse
tools/misc/Makefile +4 -0
@@ -30,6 +30,7 @@ INSTALL_SBIN += xenpm INSTALL_SBIN += xenwatchdogd INSTALL_SBIN += xen-livepatch INSTALL_SBIN += xen-diag +INSTALL_SBIN += xen-ctx INSTALL_SBIN += $(INSTALL_SBIN-y) # Everything to be installed in a private bin/
@@ -108,6 +109,9 @@ xen-livepatch: xen-livepatch.o xen-diag: xen-diag.o $(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(APPEND_LDFLAGS) +xen-ctx: xen-ctx.o + $(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenctrl) $(APPEND_LDFLAGS) + xen-lowmemd: xen-lowmemd.o $(CC) $(LDFLAGS) -o $@ $< $(LDLIBS_libxenevtchn) $(LDLIBS_libxenctrl) $(LDLIBS_libxenstore) $(APPEND_LDFLAGS)
tools/misc/xen-ctx.c new file +144 -0
144 lines not on this page · raw message
Signature · 1 line
2.20.1
Re: [Xen-devel] [PATCH 3/5] tools/misc: add xen-ctx to present domain context
Quoted 4 lines · Paul Durrant
This tools is analogous to 'xen-hvmctx' which presents HVM context. Subsequent patches will add 'dump' functions when new records are introduced.
Quoted 8 lines
Signed-off-by Paul Durrant <[email protected]>---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>--- .gitignore | 1 + tools/misc/Makefile | 4 ++ tools/misc/xen-ctx.c | 144 +++++++++++++++++++++++++++++++++++++++++++
Is xen-ctx a good choice of a name, considering we already have not only xen-hvmctx, but also xenctx? If the new functionality isn't a good fit for either, perhaps its name would better reflect its connection to save/restore records? xen-sr-dump looks pretty clumsy to me, but still seems better than a name easily mixed up with others.
Jan
RE: [PATCH 3/5] tools/misc: add xen-ctx to present domain context
Quoted 21 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 30 March 2020 11:54
To: Paul Durrant <[email protected]>
Cc: [email protected]; Ian Jackson <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 3/5]tools/misc: add xen-ctx to present domain contextQuoted 4 lines · Paul Durrant
This tools is analogous to 'xen-hvmctx' which presents HVM context. Subsequent patches will add 'dump' functions when new records are introduced.
Quoted 8 lines
Signed-off-by Paul Durrant <[email protected]>---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>--- .gitignore | 1 + tools/misc/Makefile | 4 ++ tools/misc/xen-ctx.c | 144 +++++++++++++++++++++++++++++++++++++++++++
Quoted 6 lines
Is xen-ctx a good choice of a name, considering we already have not only xen-hvmctx, but also xenctx? If the new functionality isn't a good fit for either, perhaps its name would better reflect its connection to save/restore records? xen-sr-dump looks pretty clumsy to me, but still seems better than a name easily mixed up with others.
How about xen-domctx?
Paul
Quoted 2 lines
Jan
Re: [PATCH 3/5] tools/misc: add xen-ctx to present domain context
Quoted 21 lines · Paul Durrant
Quoted 21 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 30 March 2020 11:54
To: Paul Durrant <[email protected]>
Cc: [email protected]; Ian Jackson <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 3/5]tools/misc: add xen-ctx to present domain contextQuoted 12 lines · Paul Durrant
This tools is analogous to 'xen-hvmctx' which presents HVM context. Subsequent patches will add 'dump' functions when new records are introduced.
Signed-off-by Paul Durrant <[email protected]>---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>--- .gitignore | 1 + tools/misc/Makefile | 4 ++ tools/misc/xen-ctx.c | 144 +++++++++++++++++++++++++++++++++++++++++++
Quoted 8 lines
Quoted 6 lines
Is xen-ctx a good choice of a name, considering we already have not only xen-hvmctx, but also xenctx? If the new functionality isn't a good fit for either, perhaps its name would better reflect its connection to save/restore records? xen-sr-dump looks pretty clumsy to me, but still seems better than a name easily mixed up with others.
How about xen-domctx?
Hmm, maybe. Seeing this is about PV pieces, xen-pvctx might also be an option.
Jan
RE: [PATCH 3/5] tools/misc: add xen-ctx to present domain context
Quoted 36 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 03 April 2020 16:30
To: [email protected]
Cc: [email protected]; 'Ian Jackson' <[email protected]>; 'Wei Liu' <[email protected]>
Subject: Re: [PATCH 3/5]tools/misc: add xen-ctx to present domain contextQuoted 28 lines · Paul Durrant
Quoted 27 lines
-----Original Message-----
From: Jan Beulich <[email protected]>
Sent: 30 March 2020 11:54
To: Paul Durrant <[email protected]>
Cc: [email protected]; Ian Jackson <[email protected]>; Wei Liu <[email protected]>
Subject: Re: [PATCH 3/5]tools/misc: add xen-ctx to present domain contextQuoted 4 lines · Paul Durrant
This tools is analogous to 'xen-hvmctx' which presents HVM context. Subsequent patches will add 'dump' functions when new records are introduced.
Quoted 8 lines
Signed-off-by Paul Durrant <[email protected]>---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>--- .gitignore | 1 + tools/misc/Makefile | 4 ++ tools/misc/xen-ctx.c | 144 +++++++++++++++++++++++++++++++++++++++++++Is xen-ctx a good choice of a name, considering we already have not only xen-hvmctx, but also xenctx? If the new functionality isn't a good fit for either, perhaps its name would better reflect its connection to save/restore records? xen-sr-dump looks pretty clumsy to me, but still seems better than a name easily mixed up with others.
Quoted 4 lines
Quoted 1 line
How about xen-domctx?
Hmm, maybe. Seeing this is about PV pieces, xen-pvctx might also be an option.
Yes, that would work but it also implies it might only be valid for PV domains. I also prefer 'domctx' since it better matches the name I chose for the framework.
Paul
PATCH 4/5 common/domain: add a domain context record for shared_info...
In reply to Paul Durrant... and update xen-ctx to dump some information describing the record.
NOTE: To allow a sensible definition of the record in public/save.h
this patch also adds a definition of the Xen ABI's de-facto page size into public/xen.h.
---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>
--- tools/misc/xen-ctx.c | 8 ++++++ xen/common/domain.c | 55 +++++++++++++++++++++++++++++++++++++++ xen/include/public/save.h | 10 ++++++- xen/include/public/xen.h | 3 +++ 4 files changed, 75 insertions(+), 1 deletion(-)
tools/misc/xen-ctx.c +8 -0
@@ -57,6 +57,13 @@ static void dump_header(void) h.magic, h.version); } +static void dump_shared_info (void ) +{ +DOMAIN_SAVE_TYPE (SHARED_INFO) s; +READ (s); +printf (" SHARED_INFO: field_width %u\n" , s.field_width); +} +static void dump_end (void ) {DOMAIN_SAVE_TYPE (END) e;
@@ -124,6 +131,7 @@ int main(int argc, char **argv)switch (desc.typecode) {case DOMAIN_SAVE_CODE (HEADER ):dump_header ();break ; +case DOMAIN_SAVE_CODE (SHARED_INFO ):dump_shared_info ();break ;case DOMAIN_SAVE_CODE (END ):dump_end ();return 0 ;default :printf ("Unknown type %u: skipping\n" , desc.typecode);
xen/common/domain.c +55 -0
67 lines not on this page · raw message
xen/include/public/save.h +9 -1
@@ -69,6 +69,14 @@ struct domain_save_header { }; DECLARE_DOMAIN_SAVE_TYPE(HEADER, 1, struct domain_save_header); -#define DOMAIN_SAVE_CODE_MAX 1 +struct domain_shared_info_context { +uint8_t buffer[XEN_PAGE_SIZE ]; +uint8_t field_width; +uint8_t pad[7 ]; +}; + +DECLARE_DOMAIN_SAVE_TYPE(SHARED_INFO, 2, struct domain_shared_info_context); + +#define DOMAIN_SAVE_CODE_MAX 2#endif /* __XEN_PUBLIC_SAVE_H__ */
xen/include/public/xen.h +3 -0
@@ -37,6 +37,9 @@#error "Unsupported architecture"#endif +/* The Xen ABI assumes a page size of 4k. */ +#define XEN_PAGE_SIZE 4096 +#ifndef __ASSEMBLY__/* Guest handles for primitive C types. */ DEFINE_XEN_GUEST_HANDLE (char);
Signature · 1 line
2.20.1
Re: [PATCH 4/5] common/domain: add a domain context record for shared_info...
HI Paul,
Quoted 1 line · Paul Durrant
... and update xen-ctx to dump some information describing the record.
The commit message should contain a little bit more context why we are saving/restore the shared_info.
Quoted 60 lines
NOTE: To allow a sensible definition of the record in
public/save.h
this patch also adds a definition of the Xen ABI's de-facto page size intopublic/xen.h.Signed-off-by Paul Durrant <[email protected]>---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
Cc: Andrew Cooper <[email protected]>
Cc: George Dunlap <[email protected]>
Cc: Jan Beulich <[email protected]>
Cc: Julien Grall <[email protected]>
Cc: Stefano Stabellini <[email protected]>--- tools/misc/xen-ctx.c | 8 ++++++ xen/common/domain.c | 55 +++++++++++++++++++++++++++++++++++++++ xen/include/public/save.h | 10 ++++++- xen/include/public/xen.h | 3 +++ 4 files changed, 75 insertions(+), 1 deletion(-)tools/misc/xen-ctx.c +8
-0@@ -57,6 +57,13 @@ static void dump_header(void) h.magic, h.version); } +static void dump_shared_info (void ) +{ +DOMAIN_SAVE_TYPE (SHARED_INFO) s; +READ (s); +printf (" SHARED_INFO: field_width %u\n" , s.field_width); +} +static void dump_end (void ) {DOMAIN_SAVE_TYPE (END) e;@@ -124,6 +131,7 @@ int main(int argc, char **argv)switch (desc.typecode) {case DOMAIN_SAVE_CODE (HEADER ):dump_header ();break ; +case DOMAIN_SAVE_CODE (SHARED_INFO ):dump_shared_info ();break ;case DOMAIN_SAVE_CODE (END ):dump_end ();return 0 ;default :printf ("Unknown type %u: skipping\n" , desc.typecode);xen/common/domain.c +1
-0@@ -33,6 +33,7 @@#include <xen/xenoprof.h> #include <xen/irq.h> #include <xen/argo.h> +#include <xen/save.h> #include <asm/debugger.h> #include <asm/p2m.h> #include <asm/processor.h>
xen/common/domain.c int continue_hypercall_on_cpu( · line 1654
Quoted 8 lines
xen/common/domain.c +5
-0@@ -1646,6 +1647,60 @@ int continue_hypercall_on_cpu(return 0 ; } +static int save_shared_info (const struct vcpu *v,struct domain_context *c, +bool dry_run) +{ +struct domain *d = v->domain; +struct domain_shared_info_context ctxt = {};
IHMO, storing 4K worth of data on the stack is a pretty bad idea. Looking at the code...
xen/common/domain.c int continue_hypercall_on_cpu( · line 1658
Quoted 4 lines
xen/common/domain.c +4
-0+ +if ( !dry_run ) + { +memcpy (ctxt.buffer, d->shared_info,PAGE_SIZE );
... it feels to me the content of the shared_info should be directly written in the stream.
Quoted 15 lines
xen/common/domain.c +15
-0+ ctxt.field_width =has_32bit_shinfo (d) ?4 :8 ; + } + +return DOMAIN_SAVE_ENTRY (SHARED_INFO , c, v, &ctxt,sizeof (ctxt)); +} + +static int load_shared_info (struct vcpu *v,struct domain_context *c) +{ +struct domain *d = v->domain; +struct domain_shared_info_context ctxt; +unsigned int i; +int rc; + + rc =DOMAIN_LOAD_ENTRY (SHARED_INFO , c, v, &ctxt,sizeof (ctxt), true); +if ( rc )
xen/common/domain.c int continue_hypercall_on_cpu( · line 1683
Quoted 10 lines
xen/common/domain.c +10
-0+return rc; + +for ( i =0 ; i <ARRAY_SIZE (ctxt.pad); i++ ) +if ( ctxt.pad[i] ) +return -EINVAL ; + +switch ( ctxt.field_width ) + { +case 4 : + d->arch.has_32bit_shinfo =1 ;
I don't think this will compile on Arm.
Quoted 28 lines
xen/common/domain.c +20
-0+break ; + +case 8 : + d->arch.has_32bit_shinfo =0 ; +break ; + +default : + rc = -EINVAL ; +break ; + } + +if ( !rc ) +memcpy (d->shared_info, ctxt.buffer,PAGE_SIZE ); + +return rc; +} + +DOMAIN_REGISTER_SAVE_RESTORE (SHARED_INFO , false, save_shared_info, + load_shared_info); +/* * Local variables: * mode: C xen/include/public/save.h +0
-0
xen/include/public/save.h struct domain_save_header { · line 73
Quoted 6 lines
xen/include/public/save.h +2
-1@@ -69,6 +69,14 @@ struct domain_save_header { }; DECLARE_DOMAIN_SAVE_TYPE(HEADER, 1, struct domain_save_header); -#define DOMAIN_SAVE_CODE_MAX 1 +struct domain_shared_info_context { +uint8_t buffer[XEN_PAGE_SIZE ];
In the load/save code, you are using PAGE_SIZE but here you are using XEN_PAGE_SIZE. I don't think there are any promise they will be the same value. In particular...
Quoted 14 lines
xen/include/public/save.h +7
-0+uint8_t field_width; +uint8_t pad[7 ]; +}; + +DECLARE_DOMAIN_SAVE_TYPE(SHARED_INFO, 2, struct domain_shared_info_context); + +#define DOMAIN_SAVE_CODE_MAX 2#endif /* __XEN_PUBLIC_SAVE_H__ */ xen/include/public/xen.h +0
-0
xen/include/public/xen.h line 41
Quoted 5 lines
xen/include/public/xen.h +2
-0@@ -37,6 +37,9 @@#error "Unsupported architecture"#endif +/* The Xen ABI assumes a page size of 4k. */ +#define XEN_PAGE_SIZE 4096
... this is not correct. Xen ABI is based on the page granularity used by Xen. It happens to be 4KB today but that's because no-one build yet a 64KB/16KB hypervisor.
If someone tomorrow decides to add support for 64KB, then using 4KB in the ABI by default is not going to fly.
Imagine the guest only give access to 4KB region, how do you handle it if your minimum granularity in the hypervisor is 64KB?
In theory, it will not require much effort to get Xen on Arm built with 64KB/16KB support as long as guest is using a page granularity higher than what Xen is using.
I would much prefer if we encode the page size in the stream. We can then use to retrieve data based on the page size.
Regarding the shared_info itself. Why do you need to know the size in tools? Would not it be enough to save the real size in domain_shared_info_context?
Cheers,
Signature · 1 line
Julien Grall
PATCH 5/5 tools/libxc: make use of domain context SHARED_INFO record...
In reply to Paul Durrant... in the save/restore code.
This patch replaces direct mapping of the shared_info_frame (retrieved using XEN_DOMCTL_getdomaininfo) with save/load of the domain context SHARED_INFO record.
No modifications are made to the definition of the migration stream at this point. Subsequent patches will define a record in the libxc domain image format for passing domain context and convert the save/restore code to use that.
---
Cc: Ian Jackson <[email protected]>
Cc: Wei Liu <[email protected]>
--- tools/libxc/xc_sr_common.h | 7 +++- tools/libxc/xc_sr_common_x86.c | 58 ++++++++++++++++++++++++++++++ tools/libxc/xc_sr_common_x86.h | 4 +++ tools/libxc/xc_sr_common_x86_pv.c | 52 +++++++++++++++++++++++++++ tools/libxc/xc_sr_common_x86_pv.h | 3 ++ tools/libxc/xc_sr_restore_x86_pv.c | 40 ++++++++------------- tools/libxc/xc_sr_save_x86_pv.c | 26 ++------------ tools/libxc/xg_save_restore.h | 1 + 8 files changed, 142 insertions(+), 49 deletions(-)
tools/libxc/xc_sr_common.h +6 -1
@@ -287,6 +287,11 @@ struct xc_sr_context { struct /* x86 */ { + struct { + void *buffer; + unsigned int len; + } domain_context; + struct /* x86 PV guest. */ { /* 4 or 8; 32 or 64 bit domain */
@@ -314,7 +319,7 @@ struct xc_sr_context/* The guest pfns containing the p2m leaves */ xen_pfn_t *p2m_pfns; -/* Read-only mapping of guests shared info page */ +/* Pointer to shared_info (located in context buffer) */ shared_info_any_t *shinfo;/* p2m generation count for verifying validity of local p2m. */
tools/libxc/xc_sr_common_x86.c +58 -0
64 lines not on this page · raw message
tools/libxc/xc_sr_common_x86.h +4 -0
@@ -14,6 +14,10 @@ int write_x86_tsc_info(struct xc_sr_context *ctx);*/ int handle_x86_tsc_info (struct xc_sr_context *ctx,struct xc_sr_record *rec); +int x86_get_context (struct xc_sr_context *ctx,uint32_t mask); +int x86_set_context (struct xc_sr_context *ctx,uint32_t mask); +void x86_cleanup (struct xc_sr_context *ctx); +#endif /* * Local variables:
tools/libxc/xc_sr_common_x86_pv.c +52 -0
58 lines not on this page · raw message
tools/libxc/xc_sr_common_x86_pv.h +3 -0
@@ -97,6 +97,9 @@ int x86_pv_domain_info(struct xc_sr_context *ctx);*/ int x86_pv_map_m2p (struct xc_sr_context *ctx); +int x86_pv_get_shinfo (struct xc_sr_context *ctx); +int x86_pv_set_shinfo (struct xc_sr_context *ctx); +#endif /* * Local variables:
tools/libxc/xc_sr_restore_x86_pv.c +15 -25
@@ -864,8 +864,7 @@ static int handle_shared_info(struct xc_sr_context *ctx, {xc_interface *xch = ctx->xch;unsigned int i; -int rc =-1 ; -shared_info_any_t *guest_shinfo =NULL ; +int rc;const shared_info_any_t *old_shinfo = rec->data;if ( !ctx->x86.pv.restore.seen_pv_info )
@@ -878,39 +877,30 @@ static int handle_shared_info(struct xc_sr_context *ctx, {ERROR ("X86_PV_SHARED_INFO record wrong size: length %u" ", expected 4096" , rec->length); - goto err; +return -1 ; } - guest_shinfo =xc_map_foreign_range ( - xch, ctx->domid,PAGE_SIZE ,PROT_READ |PROT_WRITE , - ctx->dominfo.shared_info_frame); -if ( !guest_shinfo ) - { -PERROR ("Failed to map Shared Info at mfn %#lx" , - ctx->dominfo.shared_info_frame); - goto err; - } + rc =x86_pv_get_shinfo (ctx); +if ( rc ) +return rc; -MEMCPY_FIELD (guest_shinfo, old_shinfo, vcpu_info, ctx->x86.pv.width); -MEMCPY_FIELD (guest_shinfo, old_shinfo, arch, ctx->x86.pv.width); +MEMCPY_FIELD (ctx->x86.pv.shinfo, old_shinfo, vcpu_info, + ctx->x86.pv.width); +MEMCPY_FIELD (ctx->x86.pv.shinfo, old_shinfo, arch, ctx->x86.pv.width); -SET_FIELD (guest_shinfo, arch.pfn_to_mfn_frame_list_list, +SET_FIELD (ctx->x86.pv.shinfo, arch.pfn_to_mfn_frame_list_list,0 , ctx->x86.pv.width); -MEMSET_ARRAY_FIELD (guest_shinfo, evtchn_pending,0 , ctx->x86.pv.width); +MEMSET_ARRAY_FIELD (ctx->x86.pv.shinfo, evtchn_pending,0 , + ctx->x86.pv.width);for ( i =0 ; i <XEN_LEGACY_MAX_VCPUS ; i++ ) -SET_FIELD (guest_shinfo, vcpu_info[i].evtchn_pending_sel, +SET_FIELD (ctx->x86.pv.shinfo, vcpu_info[i].evtchn_pending_sel,0 , ctx->x86.pv.width); -MEMSET_ARRAY_FIELD (guest_shinfo, evtchn_mask,0xff , ctx->x86.pv.width); - - rc =0 ; +MEMSET_ARRAY_FIELD (ctx->x86.pv.shinfo, evtchn_mask,0xff , + ctx->x86.pv.width); - err: -if ( guest_shinfo ) -munmap (guest_shinfo,PAGE_SIZE ); - -return rc; +return x86_pv_set_shinfo (ctx); }/* restore_ops function. */
tools/libxc/xc_sr_save_x86_pv.c +3 -23
@@ -9,25 +9,6 @@ static inline bool is_canonical_address(xen_vaddr_t vaddr)return ((int64_t )vaddr >>47 ) == ((int64_t )vaddr >>63 ); } -/* -* Maps the guests shared info page. -*/ -static int map_shinfo (struct xc_sr_context *ctx) -{ -xc_interface *xch = ctx->xch; - - ctx->x86.pv.shinfo =xc_map_foreign_range ( - xch, ctx->domid,PAGE_SIZE ,PROT_READ , ctx->dominfo.shared_info_frame); -if ( !ctx->x86.pv.shinfo ) - { -PERROR ("Failed to map shared info frame at mfn %#lx" , - ctx->dominfo.shared_info_frame); -return -1 ; - } - -return 0 ; -} -/* * Copy a list of mfns from a guest, accounting for differences between guest * and toolstack width. Can fail if truncation would occur.
@@ -1041,7 +1022,7 @@ static int x86_pv_setup(struct xc_sr_context *ctx)if ( rc )return rc; - rc =map_shinfo (ctx); + rc =x86_pv_get_shinfo (ctx);if ( rc )return rc;
@@ -1112,12 +1093,11 @@ static int x86_pv_cleanup(struct xc_sr_context *ctx)if ( ctx->x86.pv.p2m )munmap (ctx->x86.pv.p2m, ctx->x86.pv.p2m_frames *PAGE_SIZE ); -if ( ctx->x86.pv.shinfo ) -munmap (ctx->x86.pv.shinfo,PAGE_SIZE ); -if ( ctx->x86.pv.m2p )munmap (ctx->x86.pv.m2p, ctx->x86.pv.nr_m2p_frames *PAGE_SIZE ); +x86_cleanup (ctx); +return 0 ; }
tools/libxc/xg_save_restore.h +1 -0
@@ -19,6 +19,7 @@#include <xen/foreign/x86_32.h> #include <xen/foreign/x86_64.h> +#include <xen/save.h> /* ** We process save/restore/migrate in batches of pages; the below
Signature · 1 line
2.20.1