remus: Checkpoint Compression
PATCH 1/1 remus: Checkpoint Compression
This patch series adds checkpoint compression functionality, while running under Remus.
PATCH 1/1 tools/libxc: Remus Checkpoint Compression
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID 5dbafaf24c7036f3e24e4387797b32e59d732ac6 # Parent 23c068b109236657ededa3e3b7f180346a5cd9f9 tools/libxc: Remus Checkpoint Compression
Instead of sending dirty pages of guest memory as-is, use a simple compression algorithm that sends a RLE-encoded XOR of the page against its last sent copy. A small LRU cache is used to hold recently dirtied pages. Pagetable pages are sent as-is, as they are canonicalized at sender side and uncanonicalized at receiver.
tools/libxc/Makefile +1 -1
@@ -42,7 +42,7 @@ GUEST_SRCS-y := GUEST_SRCS-y += xg_private.c xc_suspend.c GUEST_SRCS-$(CONFIG_MIGRATE) += xc_domain_restore.c xc_domain_save.c -GUEST_SRCS-$(CONFIG_MIGRATE) += xc_offline_page.c +GUEST_SRCS-$(CONFIG_MIGRATE) += xc_offline_page.c xc_remus.c GUEST_SRCS-$(CONFIG_HVM) += xc_hvm_build.c vpath %.c ../../xen/common/libelf
tools/libxc/xc_domain_restore.c +62 -1
129 lines not on this page · raw message
tools/libxc/xc_domain_save.c +162 -9
252 lines not on this page · raw message
tools/libxc/xc_remus.c new file +465 -0
465 lines not on this page · raw message
tools/libxc/xenctrl.h +54 -0
58 lines not on this page · raw message
tools/libxc/xenguest.h +1 -0
@@ -29,6 +29,7 @@#define XCFLAGS_STDVGA 8#define X86_64_B_SIZE 64#define X86_32_B_SIZE 32 +#define XCFLAGS_REMUS_COMPRESS 16/* callbacks provided by xc_domain_save */ struct save_callbacks {
tools/libxc/xg_save_restore.h +1 -0
@@ -134,6 +134,7 @@#define XC_SAVE_ID_HVM_CONSOLE_PFN -8/* (HVM-only) */ #define XC_SAVE_ID_LAST_CHECKPOINT -9/* Commit to restoring after completion of current iteration. */ #define XC_SAVE_ID_HVM_ACPI_IOPORTS_LOCATION -10 +#define XC_SAVE_ID_COMPRESSED_DATA -11/* Marker to indicate arrival of compressed data */ /* ** We process save/restore/migrate in batches of pages; the below
PATCH 1/1 remus: command line switche to enable/disable checkpoint compression
In reply to Shriram Rajagopalan# HG changeset patch
# User Shriram Rajagopalan <[email protected]>
# Date 1308238107 25200
# Node ID ecf2ee11115836a7085fc4b64af4d16b9a653fd4
# Parent 5dbafaf24c7036f3e24e4387797b32e59d732ac6
remus: command line switche to enable/disable checkpoint compression
Add a command line switch to remus script that allows the user to enable or disable checkpoint compression in the libxc code.
tools/python/xen/lowlevel/checkpoint/checkpoint.c +4 -3
@@ -104,13 +104,14 @@PyObject * postcopy_cb =NULL ;PyObject * checkpoint_cb =NULL ;unsigned int interval =0 ; +unsigned int flags =0 ;int fd;struct save_callbacks callbacks;int rc; -if (!PyArg_ParseTuple (args,"O|OOOI" , &iofile, &suspend_cb, &postcopy_cb, - &checkpoint_cb, &interval)) +if (!PyArg_ParseTuple (args,"O|OOOII" , &iofile, &suspend_cb, &postcopy_cb, + &checkpoint_cb, &interval, &flags))return NULL ; self->interval = interval;
@@ -160,7 +161,7 @@ callbacks.data = self; self->threadstate =PyEval_SaveThread (); - rc =checkpoint_start (&self->cps, fd, &callbacks); + rc =checkpoint_start (&self->cps, fd, &callbacks, flags);PyEval_RestoreThread (self->threadstate);if (rc <0 ) {
tools/python/xen/lowlevel/checkpoint/checkpoint.h +3 -1
@@ -40,13 +40,15 @@timer_t timer; } checkpoint_state; +#define REMUS_FLAGS_COMPRESSION 1char *checkpoint_error (checkpoint_state * s);void checkpoint_init (checkpoint_state * s);int checkpoint_open (checkpoint_state * s,unsigned int domid);void checkpoint_close (checkpoint_state * s);int checkpoint_start (checkpoint_state * s,int fd, -struct save_callbacks * callbacks); +struct save_callbacks * callbacks, +unsigned int remus_flags);int checkpoint_suspend (checkpoint_state * s);int checkpoint_resume (checkpoint_state * s);int checkpoint_postflush (checkpoint_state * s);
tools/python/xen/lowlevel/checkpoint/libcheckpoint.c +4 -1
@@ -170,7 +170,8 @@ }int checkpoint_start (checkpoint_state * s,int fd, -struct save_callbacks * callbacks) +struct save_callbacks * callbacks, +unsigned int remus_flags) {int hvm, rc;int flags =XCFLAGS_LIVE ;
@@ -188,6 +189,8 @@if (switch_qemu_logdirty (s,1 ))return -1 ; } +if (remus_flags &REMUS_FLAGS_COMPRESSION ) + flags |=XCFLAGS_REMUS_COMPRESS ; callbacks->switch_qemu_logdirty = noop_switch_logdirty;
tools/python/xen/remus/save.py +5 -2
@@ -133,7 +133,7 @@ class Saver(object): def __init__(self, domid, fd, suspendcb=None, resumecb=None, - checkpointcb=None, interval=0): + checkpointcb=None, interval=0, flags=0): """Create a Saver object for taking guest checkpoints. domid: name, number or UUID of a running domain fd: a stream to which checkpoint data will be written.
@@ -141,12 +141,14 @@ resumecb: callback invoked before guest resumes checkpointcb: callback invoked when a checkpoint is complete. Return True to take another checkpoint, or False to stop. + flags: Remus flags to be passed to xc_domain_save """ self.fd = fd self.suspendcb = suspendcb self.resumecb = resumecb self.checkpointcb = checkpointcb self.interval = interval + self.flags = flags self.vm = vm.VM(domid)
@@ -164,7 +166,8 @@ try: self.checkpointer.open(self.vm.domid) self.checkpointer.start(self.fd, self.suspendcb, self.resumecb, - self.checkpointcb, self.interval) + self.checkpointcb, self.interval, + self.flags) except xen.lowlevel.checkpoint.error, e: raise CheckpointError(e) finally:
tools/remus/remus +9 -1
@@ -16,6 +16,9 @@ class CfgException(Exception): pass class Cfg(object): + + REMUS_FLAGS_COMPRESSION = 1 + def __init__(self): # must be set self.domid = 0
@@ -25,6 +28,7 @@ self.port = XendOptions.instance().get_xend_relocation_port() self.interval = 200 self.netbuffer = True + self.flags = self.REMUS_FLAGS_COMPRESSION self.timer = False parser = optparse.OptionParser()
@@ -38,6 +42,8 @@ help='replicate to /dev/null (no disk checkpoints, only memory & net buffering)') parser.add_option('', '--no-net', dest='nonet', action='store_true', help='run without net buffering (benchmark option)') + parser.add_option('', '--no-compression', dest='nocompress', action='store_true', + help='run without checkpoint compression') parser.add_option('', '--timer', dest='timer', action='store_true', help='force pause at checkpoint interval (experimental)') self.parser = parser
@@ -56,6 +62,8 @@ self.nullremus = True if opts.nonet: self.netbuffer = False + if opts.nocompress: + self.flags &= ~self.REMUS_FLAGS_COMPRESSION if opts.timer: self.timer = True
@@ -190,7 +198,7 @@ rc = 0 checkpointer = save.Saver(cfg.domid, fd, postsuspend, preresume, commit, - interval) + interval, cfg.flags) try: checkpointer.start()
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
In reply to Shriram RajagopalanQuoted 14 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID5dbafaf24c7036f3e24e4387797b32e59d732ac6# Parent23c068b109236657ededa3e3b7f180346a5cd9f9tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple compression algorithm that sends a RLE-encoded XOR of the page against its last sent copy. A small LRU cache is used to hold recently dirtied pages. Pagetable pages are sent as-is, as they are canonicalized at sender side and uncanonicalized at receiver.
Signed-off-by Shriram Rajagopalan <[email protected]>
...
tools/libxc/xc_domain_restore.c line 1461
Quoted 8 lines
tools/libxc/xc_domain_restore.c +1
-0@@ -1395,6 +1455,7 @@ } pagebuf.nr_physpages = pagebuf.nr_pages =0 ; + pagebuf.compbuf_pos = pagebuf.compbuf_size =0 ; n += j;/* crude stats */
Is that zeroing compbuf_size at the end of every round? Does that mean you're doing a realloc for every single page you ever receive? Ouch.
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 26 lines · Brendan Cully
Quoted 8 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID5dbafaf24c7036f3e24e4387797b32e59d732ac6# Parent23c068b109236657ededa3e3b7f180346a5cd9f9tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple
compression
Quoted 1 line
algorithm that sends a RLE-encoded XOR of the page against its last sent
copy.
Quoted 1 line
A small LRU cache is used to hold recently dirtied pages. Pagetable pages
are
Quoted 1 line
sent as-is, as they are canonicalized at sender side and uncanonicalized
at
Quoted 3 lines
receiver.
Signed-off-by Shriram Rajagopalan <[email protected]>...
Quoted 6 lines
tools/libxc/xc_domain_restore.c +1
-0@@ -1395,6 +1455,7 @@ } pagebuf.nr_physpages = pagebuf.nr_pages =0 ; + pagebuf.compbuf_pos = pagebuf.compbuf_size =0 ;
Quoted 7 lines
tools/libxc/xc_domain_restore.c line 1461
Quoted 2 lines
tools/libxc/xc_domain_restore.c +0
-0n += j;/* crude stats */ Is that zeroing
compbuf_sizeat the end of every round? Does that mean you're doing a realloc for every single page you ever receive? Ouch.We have already talked about it offline and I dont mind explaining it
"again" :).
Its a realloc for every compressed chunk. And a compressed chunk is made of as many pages as the 16MB outbuf on sender side can hold.
shriram
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 29 lines · Shriram Rajagopalan
On Thu, Jun 16, 2011 at 12:34 PM, Brendan Cully <[email protected]> wrote:
Quoted 14 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID5dbafaf24c7036f3e24e4387797b32e59d732ac6# Parent23c068b109236657ededa3e3b7f180346a5cd9f9tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple compression algorithm that sends a RLE-encoded XOR of the page against its last sent copy. A small LRU cache is used to hold recently dirtied pages. Pagetable pages are sent as-is, as they are canonicalized at sender side and uncanonicalized at receiver.
Signed-off-by Shriram Rajagopalan <[email protected]>...
tools/libxc/xc_domain_restore.c line 1461
Quoted 8 lines
tools/libxc/xc_domain_restore.c +1
-0@@ -1395,6 +1455,7 @@ } pagebuf.nr_physpages = pagebuf.nr_pages =0 ; + pagebuf.compbuf_pos = pagebuf.compbuf_size =0 ; n += j;/* crude stats */ Is that zeroing
compbuf_sizeat the end of every round? Does that mean you're doing a realloc for every single page you ever receive? Ouch.
Quoted 3 lines
We have already talked about it offline and I dont mind explaining it "again" :). Its a realloc for every compressed chunk. And a compressed chunk is made of as many pages as the 16MB outbuf on sender side can hold.
Actually, I never received a reply yesterday when I asked this exact question, which is why I'm replying on-list.
I still don't understand why you're zeroing this field while you still have a buffer allocated. It's misleading and can (and probably will) cause unnecessary reallocs. Glad to hear you're not doing this at every page though.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 43 lines · Brendan Cully
On 2011-06-16, at 10:46 AM, Shriram Rajagopalan wrote:
Quoted 26 lines · Brendan Cully
Quoted 8 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID5dbafaf24c7036f3e24e4387797b32e59d732ac6# Parent23c068b109236657ededa3e3b7f180346a5cd9f9tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple
compression
Quoted 1 line
algorithm that sends a RLE-encoded XOR of the page against its last sent
copy.
Quoted 1 line
A small LRU cache is used to hold recently dirtied pages. Pagetable
pages are
Quoted 1 line
sent as-is, as they are canonicalized at sender side and uncanonicalized
at
Quoted 3 lines
receiver.
Signed-off-by Shriram Rajagopalan <[email protected]>...
Quoted 6 lines
tools/libxc/xc_domain_restore.c +1
-0@@ -1395,6 +1455,7 @@ } pagebuf.nr_physpages = pagebuf.nr_pages =0 ; + pagebuf.compbuf_pos = pagebuf.compbuf_size =0 ;Quoted 7 lines
tools/libxc/xc_domain_restore.c line 1461
Quoted 2 lines
tools/libxc/xc_domain_restore.c +0
-0n += j;/* crude stats */ Is that zeroing
compbuf_sizeat the end of every round? Does that mean you're doing a realloc for every single page you ever receive? Ouch.We have already talked about it offline and I dont mind explaining it
"again" :).
Its a realloc for every compressed chunk. And a compressed chunk is made of as many pages as the 16MB outbuf on sender side can hold.
Quoted 8 lines
Actually, I never received a reply yesterday when I asked this exact question, which is why I'm replying on-list.
I still don't understand why you're zeroing this field while you still have a buffer allocated. It's misleading and can (and probably will) cause unnecessary reallocs. Glad to hear you're not doing this at every page though.
That code follows the same pattern as the original receive code.
Usual receive code:
buf->nr_physpages += countpages;
if (!buf->pages) {
if (!(buf->pages = malloc(buf->nr_physpages * PAGE_SIZE))) {
ERROR("Could not allocate page buffer");
return -1;
}
} else {
if (!(ptmp = realloc(buf->pages, buf->nr_physpages * PAGE_SIZE))) {
ERROR("Could not reallocate page buffer");
return -1;
}
buf->pages = ptmp;
}
if ( RDEXACT(fd, buf->pages + oldcount * PAGE_SIZE, countpages *
PAGE_SIZE) ) {
PERROR("Error when reading pages");
return -1;
}****
The compression code:
buf->compbuf_size += compbuf_size;
if (!buf->pages) {
if (!(buf->pages = malloc(buf->compbuf_size))) {
ERROR("Could not allocate compression buffer");
return -1;
}
} else {
if (!(ptmp = realloc(buf->pages, buf->compbuf_size))) {
ERROR("Could not reallocate compression buffer");
return -1;
}
buf->pages = ptmp;
}
if ( RDEXACT(fd, buf->pages + (buf->compbuf_size - compbuf_size),
compbuf_size) ) {
PERROR("Error when reading compression buffer");
return -1;
}*****
In usual code, the nr_physpages is set to 0 in the pagebuf_get() and in the main restore loop.
Same logic is applied here. The original code receives pages in batches of 1024 max. The
compression code receives pages in compressed chunks.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 81 lines · Shriram Rajagopalan
On Thu, Jun 16, 2011 at 1:57 PM, Brendan Cully <[email protected]> wrote:
Quoted 29 lines · Shriram Rajagopalan
On Thu, Jun 16, 2011 at 12:34 PM, Brendan Cully <[email protected]> wrote:
Quoted 14 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308238095 25200
# Node ID5dbafaf24c7036f3e24e4387797b32e59d732ac6# Parent23c068b109236657ededa3e3b7f180346a5cd9f9tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple compression algorithm that sends a RLE-encoded XOR of the page against its last sent copy. A small LRU cache is used to hold recently dirtied pages. Pagetable pages are sent as-is, as they are canonicalized at sender side and uncanonicalized at receiver.
Signed-off-by Shriram Rajagopalan <[email protected]>...
tools/libxc/xc_domain_restore.c line 1461
Quoted 8 lines
tools/libxc/xc_domain_restore.c +1
-0@@ -1395,6 +1455,7 @@ } pagebuf.nr_physpages = pagebuf.nr_pages =0 ; + pagebuf.compbuf_pos = pagebuf.compbuf_size =0 ; n += j;/* crude stats */ Is that zeroing
compbuf_sizeat the end of every round? Does that mean you're doing a realloc for every single page you ever receive? Ouch.Quoted 3 lines
We have already talked about it offline and I dont mind explaining it "again" :). Its a realloc for every compressed chunk. And a compressed chunk is made of as many pages as the 16MB outbuf on sender side can hold.
Actually, I never received a reply yesterday when I asked this exact question, which is why I'm replying on-list.
I still don't understand why you're zeroing this field while you still have a buffer allocated. It's misleading and can (and probably will) cause unnecessary reallocs. Glad to hear you're not doing this at every page though. That code follows the same pattern as the original receive code.
Usual receive code:
buf->nr_physpages += countpages; if (!buf->pages) { if (!(buf->pages = malloc(buf->nr_physpages * PAGE_SIZE))) { ERROR("Could not allocate page buffer"); return -1; } } else { if (!(ptmp = realloc(buf->pages, buf->nr_physpages * PAGE_SIZE))) { ERROR("Could not reallocate page buffer"); return -1; } buf->pages = ptmp; } if ( RDEXACT(fd, buf->pages + oldcount * PAGE_SIZE, countpages * PAGE_SIZE) ) { PERROR("Error when reading pages"); return -1; }****
The compression code:buf->compbuf_size += compbuf_size; if (!buf->pages) { if (!(buf->pages = malloc(buf->compbuf_size))) { ERROR("Could not allocate compression buffer"); return -1; } } else { if (!(ptmp = realloc(buf->pages, buf->compbuf_size))) { ERROR("Could not reallocate compression buffer"); return -1; } buf->pages = ptmp; } if ( RDEXACT(fd, buf->pages + (buf->compbuf_size - compbuf_size), compbuf_size) ) { PERROR("Error when reading compression buffer"); return -1; }
Quoted 5 lines
*****
In usual code, the
nr_physpagesis set to 0 in thepagebuf_get()and in the main restore loop. Same logic is applied here. The original code receives pages in batches of 1024 max. The compression code receives pages in compressed chunks.
Alright. Not ideal, but it'll do.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
In reply to Shriram Rajagopalantools/libxc/xc_domain_restore.c line 885
Quoted 6 lines · Shriram Rajagopalan
tools/libxc/xc_domain_restore.c +2
-0@@ -846,6 +881,9 @@ if (!countpages) return count; + if (buf->compression) + return pagebuf_get_one(xch, ctx, buf, fd, dom);
I'm having trouble understanding the logic of this one. What's supposed to be going on here?
tools/libxc/xc_domain_restore.c line 1317
Quoted 9 lines
tools/libxc/xc_domain_restore.c +5
-0@@ -1257,6 +1310,11 @@ PERROR("read: p2m_size"); goto out; } + if ( RDEXACT(io_fd, &remus_flags, sizeof(uint32_t)) ) + { + PERROR("read: remus_flags"); + goto out; + }
Won't this break save/restore/migrate compatibility with older versions of Xen (which won't be sending this word)? This should probably be sent as one of the XC_SAVE_ID_* numbers.
tools/libxc/xc_domain_save.c line 1155
Quoted 9 lines
(quoted diff) +0
-0@@ -1076,9 +1150,11 @@ }copypages:
tools/libxc/xc_domain_save.c +3
-2-#define wrexact (fd, buf, len) write_buffer(xch, last_iter, &ob, (fd), (buf), (len)) -#define wruncached (fd, live, buf, len) write_uncached(xch, last_iter, &ob, (fd), (buf), (len)) +#define wrexact (fd, buf, len) write_buffer(xch, last_iter, ob, (fd), (buf), (len)) +#define wruncached (fd, live, buf, len) write_uncached(xch, last_iter, ob, (fd), (buf), (len)) +#define wrcompressed (fd) write_compressed(xch, remus_ctx, last_iter, ob, (fd))
Hmm -- this seems a bit weird to have "default" values, but you're following the convention here, so I guess OK...
tools/libxc/xenguest.h Thu Jun 16 08:28:15 2011 -0700 line 32
Quoted 8 lines
diff -r
23c068b10923-r5dbafaf24c70tools/libxc/xenguest.htools/libxc/xenguest.h Thu Jun 16 08:28:15 2011 -0700 renamed from tools/libxc/xenguest.h Wed Jun 15 16:16:41 2011 +0100 +1
-0@@ -29,6 +29,7 @@ #define XCFLAGS_STDVGA 8 #define X86_64_B_SIZE 64 #define X86_32_B_SIZE 32 +#define XCFLAGS_REMUS_COMPRESS 16
<nitpick>This should be with the other XCFLAGS, not after the byte size...</nitpick>
-George Dunlap
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 6 lines · George Dunlap
Quoted 4 lines · Shriram Rajagopalan
tools/libxc/xc_domain_restore.c +0
-0@@ -846,6 +881,9 @@if (!countpages)return count;
Quoted 7 lines
tools/libxc/xc_domain_restore.c line 885
Quoted 2 lines
tools/libxc/xc_domain_restore.c +2
-0+if (buf->compression) +return pagebuf_get_one (xch, ctx, buf, fd, dom);I'm having trouble understanding the logic of this one. What's supposed to be going on here?
The original
xc_domain_restorecode expects the list of pages immediately
after the list of pfns for a batch.
With compression, the compressed chunks are attached at the fag end of the pagebuf,
[<pfn list>]+,[XC_SAVE_ID_* stuff like vcpu info, hvm_ident_pt, etc],
[XC_SAVE_ID_COMPRESSED_DATA,compressed chunk]+
So that if block diverts the pagebuf_get_one()'s control flow to consume the rest of pagebuf in usual recursive manner.
original flow:
pagebuf_get_one()
read integer
if integer is positive, its a count of number of pfns in the pfn list,
read count*sizeof(pfn) from io_fd
1.1 parse pfn list and determine number of valid pages (countpages) 1.2. read (countpages * PAGE_SIZE) from io_fd
-- basically read batch of pfns + pages
else if its negative
check for one of XC_SAVE_ID_* elements and read appropriate size fromio_fd
return pagebuf_get_one()
**********
The modified flow:
pagebuf_get_one()
read integer
if integer is positive, its a count of number of pfns in the pfn list,
read count*sizeof(pfn) from io_fd
1.1 parse pfn list and determine number of valid pages (countpages)
Quoted 1 line
Review line 886
Quoted 1 line
(quoted diff) +0
-01.2 if (compression) return pagebuf_get_one()
--basically accumulate all the pfns and the (compressed) pages would
come later
else if its negative
check for one of XC_SAVE_ID_* elements and read appropriate size fromio_fd
includes XC_SAVE_ID_COMPRESSED_DATA.
return pagebuf_get_one()
Quoted 15 lines
tools/libxc/xc_domain_restore.c line 1317
Quoted 10 lines
tools/libxc/xc_domain_restore.c +5
-0@@ -1257,6 +1310,11 @@PERROR ("read: p2m_size" ); goto out; } +if (RDEXACT (io_fd, &remus_flags,sizeof (uint32_t )) ) + { +PERROR ("read: remus_flags" ); + goto out; + }Won't this break save/restore/migrate compatibility with older versions of Xen (which won't be sending this word)? This should probably be sent as one of the XC_SAVE_ID_* numbers.
Quoted 1 line
oh shux. you are right. thanks. I could probably send this as
XC_SAVE_ID_COMPRESSION
after the last iteration and enable pagebuf->compression once its received.
Quoted 31 lines
tools/libxc/xc_domain_save.c line 1079
Quoted 6 lines
tools/libxc/xc_domain_save.c +0
-1@@ -1076,9 +1150,11 @@ } copypages: -#define wrexact (fd, buf, len) write_buffer(xch, last_iter, &ob, (fd),(buf), (len))
Review line 1080
Quoted 1 line
(quoted diff) +0
-1-#define wruncached(fd, live, buf, len) write_uncached(xch, last_iter,&ob, (fd), (buf), (len))
Review line 1153
Quoted 1 line
(quoted diff) +1
-0+#define wrexact(fd, buf, len) write_buffer(xch, last_iter, ob, (fd),(buf), (len))
Review line 1154
Quoted 1 line
(quoted diff) +1
-0+#define wruncached(fd, live, buf, len) write_uncached(xch, last_iter,ob, (fd), (buf), (len))
Review line 1155
Quoted 1 line
(quoted diff) +1
-0+#define wrcompressed(fd) write_compressed(xch, remus_ctx, last_iter, ob,(fd))
Hmm -- this seems a bit weird to have "default" values, but you're following the convention here, so I guess OK...
tools/libxc/xenguest.h Thu Jun 16 08:28:15 2011 -0700 line 32
Quoted 8 lines
diff -r
23c068b10923-r5dbafaf24c70tools/libxc/xenguest.htools/libxc/xenguest.h Thu Jun 16 08:28:15 2011 -0700 renamed from tools/libxc/xenguest.h Wed Jun 15 16:16:41 2011 +0100 +1
-0@@ -29,6 +29,7 @@ #define XCFLAGS_STDVGA 8 #define X86_64_B_SIZE 64 #define X86_32_B_SIZE 32 +#define XCFLAGS_REMUS_COMPRESS 16<nitpick>This should be with the other XCFLAGS, not after the byte size...</nitpick>
Quoted 5 lines
okay. :)
-George Dunlap
shriram
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 48 lines · Shriram Rajagopalan
Quoted 7 lines · George Dunlap
Quoted 4 lines · Shriram Rajagopalan
tools/libxc/xc_domain_restore.c +0
-0@@ -846,6 +881,9 @@ if (!countpages) return count;Quoted 6 lines
tools/libxc/xc_domain_restore.c line 885
Quoted 2 lines
tools/libxc/xc_domain_restore.c +2
-0+ if (buf->compression) + return pagebuf_get_one(xch, ctx, buf, fd, dom);I'm having trouble understanding the logic of this one. What's supposed to be going on here?
The original
xc_domain_restorecode expects the list of pages immediately after the list of pfns for a batch.With compression, the compressed chunks are attached at the fag end of the pagebuf,
[<pfn list>]+,[XC_SAVE_ID_* stuff like vcpu info,hvm_ident_pt, etc], [XC_SAVE_ID_COMPRESSED_DATA,compressed chunk]+So that if block diverts the pagebuf_get_one()'s control flow to consume the rest of pagebuf in usual recursive manner.
original flow:
pagebuf_get_one()
read integer
if integer is positive, its a count of number of pfns in the pfn list, read count*sizeof(pfn) fromio_fd
1.1 parse pfn list and determine number of valid pages (countpages) 1.2. read (countpages *PAGE_SIZE) fromio_fd
-- basically read batch of pfns + pages
else if its negative
check for one of XC_SAVE_ID_* elements and read appropriate size fromio_fd
returnpagebuf_get_one()**********
The modified flow:
pagebuf_get_one()
read integer
if integer is positive, its a count of number of pfns in the pfn list, read count*sizeof(pfn) fromio_fd
1.1 parse pfn list and determine number of valid pages (countpages)
Quoted 8 lines
Quoted 1 line
Review line 886
Quoted 1 line
(quoted diff) +0
-01.2 if (compression) return pagebuf_get_one()--basically accumulate all the pfns and the (compressed) pages would come later
else if its negative
check for one of XC_SAVE_ID_* elements and read appropriate size fromio_fd
includesXC_SAVE_ID_COMPRESSED_DATA.
returnpagebuf_get_one()
OK -- you should add a comment there explaining that the data will come later in an XC_SAVE_ID_COMPRESSED (or whatever) packet.
Quoted 3 lines
oh shux. you are right. thanks. I could probably send this as
XC_SAVE_ID_COMPRESSION
after the last iteration and enablepagebuf->compressiononce its received.
Yeah, I think that should work -- the first round you'll be sending uncompressed pages anyway.
Hmm -- while you're at it, could you add a comment somewhere in this area saying that this is part of the wire protocol that can't be changed without breaking backwards compatibility? Just to make it easier to avoid this kind of error in the future...
Other than that, looks all right to me.
(Keir / IanJ, let me know if you want me to take a close enough look to give a proper ACK.)
Thanks,
-George
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
In reply to Shriram RajagopalanShriram Rajagopalan writes ("[Xen-devel] [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression"):
Quoted 7 lines
tools/libxc: Remus Checkpoint CompressionInstead of sending dirty pages of guest memory as-is, use a simple compression algorithm that sends a RLE-encoded XOR of the page against its last sent copy. A small LRU cache is used to hold recently dirtied pages. Pagetable pages are sent as-is, as they are canonicalized at sender side and uncanonicalized at receiver.
Thanks for this but it's very hard to review due to wrap damage on my terminal:
- if ( wruncached(io_fd, live, page, PAGE_SIZE) !=PAGE_SIZE )
+ if (compression)
+ {
+ /* Add the pagetable page, raw into the page buffer */
+ if (xc_remus_add_page(xch, remus_ctx, page,
pfn, 1) < 0)
+ {
+ /* We are out of buffer space to hold dirty pages.
+ * Compress and flush the current buffer to
make space.
+ * --This is a corner case, that slows down
Please keep your lines down to 75 characters or so.
Ian.
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
In reply to George DunlapQuoted 3 lines · George Dunlap
Hmm -- while you're at it, could you add a comment somewhere in this area saying that this is part of the wire protocol that can't be changed without breaking backwards compatibility?
Also please update tools/libxl/xg_save_restore.h to document the protocol extensions/changes.
Ian.
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
There is no change to live migration protocol. Its just the remus replication
protocol, which I would be happy to document in xg_save_restore.h.
Can I send that as a separate patch? George already acked the code part of the patch (see [email protected])
shriram
Quoted 11 lines · Ian Campbell
Quoted 3 lines · George Dunlap
Hmm -- while you're at it, could you add a comment somewhere in this area saying that this is part of the wire protocol that can't be changed without breaking backwards compatibility?
Also please update
tools/libxl/xg_save_restore.hto document the protocol extensions/changes.Ian.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
Quoted 6 lines · Shriram Rajagopalan
There is no change to live migration protocol. Its just the remus replication
protocol, which I would be happy to document inxg_save_restore.h.Can I send that as a separate patch? George already acked the code part of the patch (see [email protected])
FYI, I am not (and do not wish to be) the maintainer for the save/restore/migration/remus code; I'm just an interested party (since if it breaks, I'm going to have to deal with it when it comes into XenServer). As such, my "Ack" was only meant to mean, "I'm fine with this" (since I had raised an objection before). IanJ's approval is still needed for check-in.
-George
Quoted 23 lines
shriram
Quoted 11 lines · Ian Campbell
Quoted 3 lines · George Dunlap
Hmm -- while you're at it, could you add a comment somewhere in this area saying that this is part of the wire protocol that can't be changed without breaking backwards compatibility?
Also please update
tools/libxl/xg_save_restore.hto document the protocol extensions/changes.Ian.
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
George Dunlap writes ("Re: [Xen-devel] [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression"):
Quoted 6 lines
FYI, I am not (and do not wish to be) the maintainer for the save/restore/migration/remus code; I'm just an interested party (since if it breaks, I'm going to have to deal with it when it comes into XenServer). As such, my "Ack" was only meant to mean, "I'm fine with this" (since I had raised an objection before). IanJ's approval is still needed for check-in.
Right.
Quoted 5 lines · Shriram Rajagopalan
Quoted 4 lines
There is no change to live migration protocol. Its just the remus replication protocol, which I would be happy to document in
xg_save_restore.h.
Right, yes, please.
Quoted 1 line
Quoted 1 line
Can I send that as a separate patch?
That would be fine.
Quoted 2 lines
Quoted 2 lines
George already acked the code part
of the patch (see [email protected])
I will try to give it a proper review myself.
Thanks,
Ian.
Re: [PATCH 1 of 2] tools/libxc: Remus Checkpoint Compression
In reply to Shriram RajagopalanQuoted 3 lines · Shriram Rajagopalan
There is no change to live migration protocol. Its just the remus replication protocol, which I would be happy to document in
xg_save_restore.h.
Yes please!
Quoted 1 line
Can I send that as a separate patch?
That's fine by me.
Ian.