Xen support for Remus
PATCH 1/1 Xen support for Remus
Hi all,
This set of patches adds Remus support to Xen.
Remus extends the Xen live migration machinery in order to provide transparent, comprehensive high availability for ordinary virtual machines. More information can be found at
<http://dsg.cs.ubc.ca/remus/>.
This patch set is against the current tip of xen-unstable (changeset 2522cc95efd2). It provides support for PV guests in 32-on-32 and 64-on-64 configurations -- 32-on-64 and HVM are still in progress.
These patches modify Xen to support Remus -- to actually use it, you'll also need the control tools and instructions available here:
http://dsg.cs.ubc.ca/remus/hg/tools/
I'll investigate integrating the tool stack into the Xen repository's 'tools' directory soon.
I've tested these changes for backward compatibility with unpatched Xen. Specifically, I've tested live migration in both directions (from plain Xen to Remusified Xen and vice versa) for PV and HVM guests, each in 32-on-32, 32-on-64, and 64-on-64 configurations.
I'm maintaining these patches in an hg queue repository here:
http://dsg.cs.ubc.ca/remus/hg/xen-3.4-testing.hg/patches/
Details of the patch series:
The first two patches provide core support for incremental checkpointing:
continuous-checkpoint
continuous-restore
The next 3 provide a new tapdisk module ('remus') which acts as a proxy for any other tapdisk module, replicating writes to the backup node:
tapdisk-more-fds
tapdisk-image-check
tapdisk-remus
All following patches are performance related.
These patches are a request for comment, and as such are optimized to be easy to rebase rather than beautiful, but I'm happy to discuss how they could be polished. With that in mind, I hope that you'll try them out if you're interested in high availability or fast, incremental checkpointing in general. I look forward to your feedback!
Brendan
PATCH 1/1 Add callbacks for suspend, postcopy and preresume in xc_domain_save
# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355494 25200
# Node ID 904729ffa2692482c77e7da5828c4b218a3a51c2
# Parent 7d552e56d105786838ac027f3625486c9c2ea449
Add callbacks for suspend, postcopy and preresume in xc_domain_save. This makes it possible to perform repeated checkpoints.
tools/libxc/xc_domain_save.c +46 -11
@@ -337,11 +337,11 @@return -1 ; } - -static int suspend_and_state (int (*suspend)(void ),int xc_handle,int io_fd, -int dom,xc_dominfo_t *info) +static int suspend_and_state (int (*suspend)(void *),void * data, +int xc_handle,int io_fd,int dom, +xc_dominfo_t *info) { -if ( !(*suspend)() ) +if ( !(*suspend)(data) ) {ERROR ("Suspend request failed" );return -1 ;
@@ -745,14 +745,15 @@ }int xc_domain_save (int xc_handle,int io_fd,uint32_t dom,uint32_t max_iters, -uint32_t max_factor,uint32_t flags,int (*suspend)(void ), +uint32_t max_factor,uint32_t flags, +struct save_callbacks * callbacks,int hvm,void *(*init_qemu_maps)(int ,unsigned ),void (*qemu_flip_buffer)(int ,int )) {xc_dominfo_t info;DECLARE_DOMCTL ; -int rc =1 , frc, i, j, last_iter, iter =0 ; +int rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG );int race =0 , sent_last_iter, skip_this_iter;
@@ -873,7 +874,8 @@else {/* This is a non-live suspend. Suspend the domain .*/ -if (suspend_and_state (suspend, xc_handle, io_fd, dom, &info) ) +if (suspend_and_state (callbacks->suspend, callbacks->data, xc_handle, + io_fd, dom, &info) ) {ERROR ("Domain appears not to have suspended" ); goto out;
@@ -994,6 +996,7 @@print_stats (xc_handle, dom,0 , &stats,0 ); + copypages:/* Now write out each data page, canonicalising page tables as we go... */ for ( ; ; ) {
@@ -1307,7 +1310,8 @@DPRINTF ("Start last iteration\n" ); last_iter =1 ; -if (suspend_and_state (suspend, xc_handle, io_fd, dom, &info) ) +if (suspend_and_state (callbacks->suspend, callbacks->data, + xc_handle, io_fd, dom, &info) ) {ERROR ("Domain appears not to have suspended" ); goto out;
@@ -1602,6 +1606,40 @@ rc =0 ; out: +if ( !rc && callbacks->postcopy ) + callbacks->postcopy (callbacks->data); + +/* Flush last write and discard cache for file. */ +discard_file_cache (io_fd,1 /* flush */ ); + +/* checkpoint_cb can spend arbitrarily long in between rounds */ +if (!rc && callbacks->checkpoint && + callbacks->checkpoint (callbacks->data) >0 ) + { +/* reset stats timer */ +print_stats (xc_handle, dom,0 , &stats,0 ); + + rc =1 ; +/* last_iter = 1; */ +if (suspend_and_state (callbacks->suspend, callbacks->data, xc_handle, + io_fd, dom, &info) ) + { +ERROR ("Domain appears not to have suspended" ); + goto out; + } +DPRINTF ("SUSPEND shinfo %08lx\n" , info.shared_info_frame); +print_stats (xc_handle, dom,0 , &stats,1 ); + +if (xc_shadow_control (xc_handle, dom, +XEN_DOMCTL_SHADOW_OP_CLEAN , to_send, + p2m_size,NULL ,0 , &stats) != p2m_size ) + { +ERROR ("Error flushing shadow PT" ); + } + + goto copypages; + } +if ( live ) {
@@ -1611,9 +1649,6 @@DPRINTF ("Warning - couldn't disable shadow mode" ); } -/* Flush last write and discard cache for file. */ -discard_file_cache (io_fd,1 /* flush */ ); -if ( live_shinfo )munmap (live_shinfo,PAGE_SIZE );
tools/libxc/xenguest.h +15 -1
@@ -14,6 +14,19 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8 +/* callbacks provided by xc_domain_save */ +struct save_callbacks { +int (*suspend)(void * data); +/* callback to rendezvous with external checkpoint functions */ +int (*postcopy)(void * data); +/* returns: +* 0: terminate checkpointing gracefully +* 1: take another checkpoint */ +int (*checkpoint)(void * data); + +/* to be provided as the first argument to each callback function */ +void * data; +};/** * This function will save a running domain.
@@ -25,7 +38,8 @@*/ int xc_domain_save (int xc_handle,int io_fd,uint32_t dom,uint32_t max_iters,uint32_t max_factor,uint32_t flags/* XCFLAGS_xxx */ , -int (*suspend)(void ),int hvm, +struct save_callbacks * callbacks, +int hvm,void *(*init_qemu_maps)(int ,unsigned ),/* HVM only */ void (*qemu_flip_buffer)(int ,int ));/* HVM only */
tools/python/xen/xend/server/XMLRPCServer.py +2 -1
@@ -94,7 +94,8 @@ 'destroyDevice','getDeviceSxprs', 'setMemoryTarget', 'setName', 'setVCpuCount', 'shutdown', 'send_sysrq', 'getVCPUInfo', 'waitForDevices', - 'getRestartCount', 'getBlockDeviceClass'] + 'getRestartCount', 'getBlockDeviceClass', + 'waitForShutdown', 'resumeDomain'] exclude = ['domain_create', 'domain_restore']
tools/xcutils/xc_save.c +5 -2
@@ -71,7 +71,7 @@return 1 ; } -static int suspend (void ) +static int suspend (void * data) {unsigned long sx_state =0 ;
@@ -217,6 +217,7 @@ {unsigned int maxit, max_f;int io_fd, ret, port; +struct save_callbacks callbacks;if (argc !=6 )errx (1 ,"usage: %s iofd domid maxit maxf flags" , argv[0 ]);
@@ -253,8 +254,10 @@ "using slow path"); } } +memset (&callbacks,0 ,sizeof (callbacks)); + callbacks.suspend = suspend; ret =xc_domain_save (si.xc_fd, io_fd, si.domid, maxit, max_f, si.flags, - &suspend, !!(si.flags &XCFLAGS_HVM ), + &callbacks, !!(si.flags &XCFLAGS_HVM ), &init_qemu_maps, &qemu_flip_buffer);if (si.suspend_evtchn >0 )
PATCH 1/1 Make xc_domain_restore loop until the fd is closed
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355507 25200
# Node ID f5c0d3208d8ae9183391398d52c9be5969da24ec
# Parent 904729ffa2692482c77e7da5828c4b218a3a51c2
Make xc_domain_restore loop until the fd is closed.
The tail containing the final PFN table, VCPU contexts and shared_info_page is buffered, then the read loop is restarted. After the first pass, incoming pages are buffered until the next tail is read, completing a new consistent checkpoint. At this point, the memory changes are applied and the loop begins again. When the fd read fails, the tail buffer is processed.
tools/libxc/xc_domain_restore.c +519 -287
@@ -269,6 +269,438 @@return p2m_frame_list; } +typedef struct { +unsigned int pfncount; +unsigned long * pfntab; +unsigned int vcpucount; +unsigned char * vcpubuf; +unsigned char shared_info_page[PAGE_SIZE ]; +}tailbuf_t ; + +static int buffer_tail (tailbuf_t * buf,int fd,unsigned int max_vcpu_id, +uint64_t vcpumap,int ext_vcpucontext) +{ +unsigned int i; +size_t pfnlen, vcpulen; + +/* TODO: handle changing pfntab and vcpu counts */ +/* PFN tab */ +if (read_exact (fd, &buf->pfncount,sizeof (buf->pfncount)) || + (buf->pfncount > (1U <<28 )) )/* up to 1TB of address space */ + { +ERROR ("Error when reading pfn count" ); +return -1 ; + } + pfnlen =sizeof (unsigned long ) * buf->pfncount; +if ( !(buf->pfntab) ) { +if ( !(buf->pfntab =malloc (pfnlen)) ) { +ERROR ("Error allocating PFN tail buffer" ); +return -1 ; + } + } +// DPRINTF("Reading PFN tab: %d bytes\n", pfnlen); +if (read_exact (fd, buf->pfntab, pfnlen) ) { +ERROR ("Error when reading pfntab" ); + goto free_pfntab; + } + +/* VCPU contexts */ + buf->vcpucount =0 ; +for (i =0 ; i <= max_vcpu_id; i++) { +// DPRINTF("vcpumap: %llx, cpu: %d, bit: %llu\n", vcpumap, i, (vcpumap % (1ULL << i))); +if ( (!(vcpumap & (1ULL << i))) ) +continue ; + buf->vcpucount++; + } +// DPRINTF("VCPU count: %d\n", buf->vcpucount); + vcpulen = ((guest_width ==8 ) ?sizeof (vcpu_guest_context_x86_64_t) + :sizeof (vcpu_guest_context_x86_32_t)) * buf->vcpucount; +if ( ext_vcpucontext ) + vcpulen +=128 * buf->vcpucount; + +if ( !(buf->vcpubuf) ) { +if ( !(buf->vcpubuf =malloc (vcpulen)) ) { +ERROR ("Error allocating VCPU ctxt tail buffer" ); + goto free_pfntab; + } + } +// DPRINTF("Reading VCPUS: %d bytes\n", vcpulen); +if (read_exact (fd, buf->vcpubuf, vcpulen) ) { +ERROR ("Error when reading ctxt" ); + goto free_vcpus; + } + +/* load shared_info_page */ +// DPRINTF("Reading shared info: %lu bytes\n", PAGE_SIZE); +if (read_exact (fd, buf->shared_info_page,PAGE_SIZE ) ) { +ERROR ("Error when reading shared info page" ); + goto free_vcpus; + } + +return 0 ; + + free_vcpus: +if (buf->vcpubuf) { +free (buf->vcpubuf); + buf->vcpubuf =NULL ; + } + free_pfntab: +if (buf->pfntab) { +free (buf->pfntab); + buf->pfntab =NULL ; + } + +return -1 ; +} + +static void tailbuf_free (tailbuf_t * buf) +{ +if (buf->vcpubuf) { +free (buf->vcpubuf); + buf->vcpubuf =NULL ; + } +if (buf->pfntab) { +free (buf->pfntab); + buf->pfntab =NULL ; + } +} + +typedef struct { +void * pages; +unsigned int nr_physpages, nr_pages;/* pages is of length nr_physpages, pfn_types is of length nr_pages */ + +/* Types of the pfns in the current region */ +unsigned long * pfn_types; + +int verify; + +int new_ctxt_format; +int max_vcpu_id; +uint64_t vcpumap; +uint64_t identpt; +uint64_t vm86_tss; +}pagebuf_t ; + +static int pagebuf_init (pagebuf_t * buf) +{ +memset (buf,0 ,sizeof (*buf)); +return 0 ; +} + +static void pagebuf_free (pagebuf_t * buf) +{ +if (buf->pages) { +free (buf->pages); + buf->pages =NULL ; + } +if (buf->pfn_types) { +free (buf->pfn_types); + buf->pfn_types =NULL ; + } +} + +static int pagebuf_get_one (pagebuf_t * buf,int fd) +{ +int count, countpages, oldcount, i; +void * ptmp; + +if (read_exact (fd, &count,sizeof (count)) ) + { +ERROR ("Error when reading batch size" ); +return -1 ; + } + +// DPRINTF("reading batch of %d pages\n", count); + +if (!count) { +DPRINTF ("Last batch read\n" ); +return 0 ; + }else if (count ==-1 ) { +DPRINTF ("Entering page verify mode\n" ); + buf->verify =1 ; +return pagebuf_get_one (buf, fd); + }else if (count ==-2 ) { + buf->new_ctxt_format =1 ; +if (read_exact (fd, &buf->max_vcpu_id,sizeof (buf->max_vcpu_id)) || + buf->max_vcpu_id >=64 ||read_exact (fd, &buf->vcpumap, +sizeof (uint64_t )) ) { +ERROR ("Error when reading max_vcpu_id" ); +return -1 ; + } +// DPRINTF("Max VCPU ID: %d, vcpumap: %llx\n", buf->max_vcpu_id, buf->vcpumap); +return pagebuf_get_one (buf, fd); + }else if (count ==-3 ) { +/* Skip padding 4 bytes then read the EPT identity PT location. */ +if (read_exact (fd, &buf->identpt,sizeof (uint32_t )) || +read_exact (fd, &buf->identpt,sizeof (uint64_t )) ) + { +ERROR ("error read the address of the EPT identity map" ); +return -1 ; + } +// DPRINTF("EPT identity map address: %llx\n", buf->identpt); +return pagebuf_get_one (buf, fd); + }else if ( count ==-4 ) { +/* Skip padding 4 bytes then read the vm86 TSS location. */ +if (read_exact (fd, &buf->vm86_tss,sizeof (uint32_t )) || +read_exact (fd, &buf->vm86_tss,sizeof (uint64_t )) ) + { +ERROR ("error read the address of the vm86 TSS" ); +return -1 ; + } +// DPRINTF("VM86 TSS location: %llx\n", buf->vm86_tss); +return pagebuf_get_one (buf, fd); + }else if ( (count >MAX_BATCH_SIZE ) || (count <0 ) ) { +ERROR ("Max batch size exceeded (%d). Giving up." , count); +return -1 ; + } + + oldcount = buf->nr_pages; + buf->nr_pages += count; +if (!buf->pfn_types) { +if (!(buf->pfn_types =malloc (buf->nr_pages *sizeof (*(buf->pfn_types))))) { +ERROR ("Could not allocate PFN type buffer" ); +return -1 ; + } + }else { +if (!(ptmp =realloc (buf->pfn_types, buf->nr_pages *sizeof (*(buf->pfn_types))))) { +ERROR ("Could not reallocate PFN type buffer" ); +return -1 ; + } + buf->pfn_types = ptmp; + } +if (read_exact (fd, buf->pfn_types + oldcount, count *sizeof (*(buf->pfn_types)))) { +ERROR ("Error when reading region pfn types" ); +return -1 ; + } + + countpages = count; +for (i = oldcount; i < buf->nr_pages; ++i) +if ((buf->pfn_types[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ) ==XEN_DOMCTL_PFINFO_XTAB ) + --countpages; + +if (!countpages) +return count; + + oldcount = buf->nr_physpages; + buf->nr_physpages += countpages; +if (!buf->pages) { +if (!(buf->pages =malloc (buf->nr_physpages *PAGE_SIZE ))) { +ERROR ("Could not allocate page buffer" ); +return -1 ; + } + }else { +if (!(ptmp =realloc (buf->pages, buf->nr_physpages *PAGE_SIZE ))) { +ERROR ("Could not reallocate page buffer" ); +return -1 ; + } + buf->pages = ptmp; + } +if (read_exact (fd, buf->pages + oldcount *PAGE_SIZE , countpages *PAGE_SIZE ) ) { +ERROR ("Error when reading pages" ); +return -1 ; + } + +return count; +} + +static int pagebuf_get (pagebuf_t * buf,int fd) +{ +int rc; + + buf->nr_physpages = buf->nr_pages =0 ; + +do { + rc =pagebuf_get_one (buf, fd); + }while (rc >0 ); + +if (rc <0 ) +pagebuf_free (buf); + +return rc; +} + +static int apply_batch (int xc_handle,uint32_t dom,xen_pfn_t * region_mfn, +unsigned long * pfn_type,int pae_extended_cr3, +unsigned int hvm,struct xc_mmu * mmu, +pagebuf_t * pagebuf,int curbatch) +{ +int nr_mfns =0 ; +int i, j, curpage; +/* used by debug verify code */ +unsigned long buf[PAGE_SIZE /sizeof (unsigned long )]; +/* Our mapping of the current region (batch) */ +char *region_base; +/* A temporary mapping, and a copy, of one frame of guest memory. */ +unsigned long *page =NULL ; +int nraces =0 ; + +unsigned long mfn, pfn, pagetype; + + j = pagebuf->nr_pages - curbatch; +if (j >MAX_BATCH_SIZE ) + j =MAX_BATCH_SIZE ; + +/* First pass for this batch: work out how much memory to alloc */ +for ( i =0 ; i < j; i++ ) + { + pfn = pagebuf->pfn_types[i + curbatch] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = pagebuf->pfn_types[i + curbatch] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( (pagetype !=XEN_DOMCTL_PFINFO_XTAB ) && + (p2m[pfn] ==INVALID_P2M_ENTRY ) ) + { +/* Have a live PFN which hasn't had an MFN allocated */ + p2m_batch[nr_mfns++] = pfn; + p2m[pfn]--; + } + } + +/* Now allocate a bunch of mfns for this batch */ +if ( nr_mfns && + (xc_domain_memory_populate_physmap (xc_handle, dom, nr_mfns,0 , +0 , p2m_batch) !=0 ) ) + { +ERROR ("Failed to allocate memory for batch.!\n" ); + errno =ENOMEM ; +return -1 ; + } + +/* Second pass for this batch: update p2m[] and region_mfn[] */ + nr_mfns =0 ; +for ( i =0 ; i < j; i++ ) + { + pfn = pagebuf->pfn_types[i + curbatch] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = pagebuf->pfn_types[i + curbatch] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) + region_mfn[i] = ~0UL ;/* map will fail but we don't care */ +else + { +if ( p2m[pfn] == (INVALID_P2M_ENTRY -1 ) ) + { +/* We just allocated a new mfn above; update p2m */ + p2m[pfn] = p2m_batch[nr_mfns++]; + nr_pfns++; + } + +/* setup region_mfn[] for batch map. +* For HVM guests, this interface takes PFNs, not MFNs */ + region_mfn[i] = hvm ? pfn : p2m[pfn]; + } + } + +/* Map relevant mfns */ + region_base =xc_map_foreign_batch ( + xc_handle, dom,PROT_WRITE , region_mfn, j); + +if ( region_base ==NULL ) + { +ERROR ("map batch failed" ); +return -1 ; + } + +for ( i =0 , curpage =-1 ; i < j; i++ ) + { + pfn = pagebuf->pfn_types[i + curbatch] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = pagebuf->pfn_types[i + curbatch] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) +/* a bogus/unmapped page: skip it */ +continue ; + + ++curpage; + +if ( pfn > p2m_size ) + { +ERROR ("pfn out of range" ); +return -1 ; + } + + pfn_type[pfn] = pagetype; + + mfn = p2m[pfn]; + +/* In verify mode, we use a copy; otherwise we work in place */ + page = pagebuf->verify ? (void *)buf : (region_base + i*PAGE_SIZE ); + +memcpy (page, pagebuf->pages + (curpage + curbatch) *PAGE_SIZE ,PAGE_SIZE ); + + pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; + +if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && + (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) + { +/* +** A page table page - need to 'uncanonicalize' it, i.e. +** replace all the references to pfns with the corresponding +** mfns for the new domain. +** +** On PAE we need to ensure that PGDs are in MFNs < 4G, and +** so we may need to update the p2m after the main loop. +** Hence we defer canonicalization of L1s until then. +*/ +if ((pt_levels !=3 ) || + pae_extended_cr3 || + (pagetype !=XEN_DOMCTL_PFINFO_L1TAB )) { + +if (!uncanonicalize_pagetable (xc_handle, dom, + pagetype, page)) { +/* +** Failing to uncanonicalize a page table can be ok +** under live migration since the pages type may have +** changed by now (and we'll get an update later). +*/ +DPRINTF ("PT L%ld race on pfn=%08lx mfn=%08lx\n" , + pagetype >>28 , pfn, mfn); + nraces++; +continue ; + } + } + } +else if ( pagetype !=XEN_DOMCTL_PFINFO_NOTAB ) + { +ERROR ("Bogus page type %lx page table is out of range: " +"i=%d p2m_size=%lu" , pagetype, i, p2m_size); +return -1 ; + } + +if ( pagebuf->verify ) + { +int res =memcmp (buf, (region_base + i*PAGE_SIZE ),PAGE_SIZE ); +if ( res ) + { +int v; + +DPRINTF ("************** pfn=%lx type=%lx gotcs=%08lx " +"actualcs=%08lx\n" , pfn, pagebuf->pfn_types[pfn], +csum_page (region_base + (i + curbatch)*PAGE_SIZE ), +csum_page (buf)); + +for ( v =0 ; v <4 ; v++ ) + { +unsigned long *p = (unsigned long *) + (region_base + i*PAGE_SIZE ); +if ( buf[v] != p[v] ) +DPRINTF (" %d: %08lx %08lx\n" , v, buf[v], p[v]); + } + } + } + +if ( !hvm && +xc_add_mmu_update (xc_handle, mmu, + (((unsigned long long )mfn) <<PAGE_SHIFT ) + |MMU_MACHPHYS_UPDATE , pfn) ) + { +ERROR ("failed machpys update mfn=%lx pfn=%lx" , mfn, pfn); +return -1 ; + } + }/* end of 'batch' for loop */ + +munmap (region_base, j*PAGE_SIZE ); + +return nraces; +} + int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom, unsigned int store_evtchn, unsigned long *store_mfn, unsigned int console_evtchn, unsigned long *console_mfn,
@@ -278,7 +710,6 @@int rc =1 , frc, i, j, n, m, pae_extended_cr3 =0 , ext_vcpucontext =0 ;unsigned long mfn, pfn;unsigned int prev_pc, this_pc; -int verify =0 ;int nraces =0 ;/* The new domain's shared-info frame number. */
@@ -297,9 +728,6 @@/* A table of MFNs to map in the current region */ xen_pfn_t *region_mfn =NULL ; -/* Types of the pfns in the current region */ -unsigned long region_pfn_type[MAX_BATCH_SIZE ]; -/* A copy of the pfn-to-mfn table frame list. */ xen_pfn_t *p2m_frame_list =NULL ;
@@ -311,9 +739,6 @@struct xc_mmu *mmu =NULL ; -/* used by debug verify code */ -unsigned long buf[PAGE_SIZE /sizeof (unsigned long )]; -struct mmuext_op pin[MAX_PIN_BATCH ];unsigned int nr_pins;
@@ -327,6 +752,14 @@/* Buffer for holding HVM context */ uint8_t *hvm_buf =NULL ; +int completed =0 ; +pagebuf_t pagebuf; +tailbuf_t tailbuf, tmptail; +void * vcpup; + +pagebuf_init (&pagebuf); +memset (&tailbuf,0 ,sizeof (tailbuf)); +/* For info only */ nr_pfns =0 ;
@@ -435,9 +868,10 @@ prev_pc =0 ; n = m =0 ; + loadpages:for ( ; ; ) { -int j, nr_mfns =0 ; +int j, curbatch; this_pc = (n *100 ) / p2m_size;if ( (this_pc - prev_pc) >=5 )
@@ -446,248 +880,49 @@ prev_pc = this_pc; } -if (read_exact (io_fd, &j,sizeof (int )) ) - { -ERROR ("Error when reading batch size" ); - goto out; - } +if ( !completed ) { + pagebuf.nr_physpages = pagebuf.nr_pages =0 ; +if (pagebuf_get_one (&pagebuf, io_fd) <0 ) { +ERROR ("Error when reading batch\n" ); + goto out; + } + } + j = pagebuf.nr_pages;PPRINTF ("batch %d\n" ,j); -if ( j ==-1 ) - { - verify =1 ; -DPRINTF ("Entering page verify mode\n" ); -continue ; - } +if ( j ==0 ) { +/* catch vcpu updates */ +if (pagebuf.new_ctxt_format) { + vcpumap = pagebuf.vcpumap; + max_vcpu_id = pagebuf.max_vcpu_id; + } +/* should this be deferred? does it change? */ +if (pagebuf.identpt) +xc_set_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , pagebuf.identpt); +if (pagebuf.vm86_tss) +xc_set_hvm_param (xc_handle, dom,HVM_PARAM_VM86_TSS , pagebuf.vm86_tss); +break ;/* our work here is done */ + } -if ( j ==-2 ) - { - new_ctxt_format =1 ; -if (read_exact (io_fd, &max_vcpu_id,sizeof (int )) || - (max_vcpu_id >=64 ) || -read_exact (io_fd, &vcpumap,sizeof (uint64_t )) ) - { -ERROR ("Error when reading max_vcpu_id" ); - goto out; - } -continue ; - } +/* break pagebuf into batches */ + curbatch =0 ; +while ( curbatch < j ) { +int brc; -if ( j ==-3 ) - { -uint64_t ident_pt; + brc =apply_batch (xc_handle, dom, region_mfn, pfn_type, + pae_extended_cr3, hvm, mmu, &pagebuf, curbatch); +if ( brc <0 ) + goto out; -/* Skip padding 4 bytes then read the EPT identity PT location. */ -if (read_exact (io_fd, &ident_pt,sizeof (uint32_t )) || -read_exact (io_fd, &ident_pt,sizeof (uint64_t )) ) - { -ERROR ("error read the address of the EPT identity map" ); - goto out; - } + nraces += brc; -xc_set_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , ident_pt); -continue ; - } + curbatch +=MAX_BATCH_SIZE ; + } -if ( j ==-4 ) - { -uint64_t vm86_tss; + pagebuf.nr_physpages = pagebuf.nr_pages =0 ; -/* Skip padding 4 bytes then read the vm86 TSS location. */ -if (read_exact (io_fd, &vm86_tss,sizeof (uint32_t )) || -read_exact (io_fd, &vm86_tss,sizeof (uint64_t )) ) - { -ERROR ("error read the address of the vm86 TSS" ); - goto out; - } - -xc_set_hvm_param (xc_handle, dom,HVM_PARAM_VM86_TSS , vm86_tss); -continue ; - } - -if ( j ==0 ) -break ;/* our work here is done */ - -if ( (j >MAX_BATCH_SIZE ) || (j <0 ) ) - { -ERROR ("Max batch size exceeded. Giving up." ); - goto out; - } - -if (read_exact (io_fd, region_pfn_type, j*sizeof (unsigned long )) ) - { -ERROR ("Error when reading region pfn types" ); - goto out; - } - -/* First pass for this batch: work out how much memory to alloc */ - nr_mfns =0 ; -for ( i =0 ; i < j; i++ ) - { -unsigned long pfn, pagetype; - pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; - pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; - -if ( (pagetype !=XEN_DOMCTL_PFINFO_XTAB ) && - (p2m[pfn] ==INVALID_P2M_ENTRY ) ) - { -/* Have a live PFN which hasn't had an MFN allocated */ - p2m_batch[nr_mfns++] = pfn; - p2m[pfn]--; - } - } - -/* Now allocate a bunch of mfns for this batch */ -if ( nr_mfns && - (xc_domain_memory_populate_physmap (xc_handle, dom, nr_mfns,0 , -0 , p2m_batch) !=0 ) ) - { -ERROR ("Failed to allocate memory for batch.!\n" ); - errno =ENOMEM ; - goto out; - } - -/* Second pass for this batch: update p2m[] and region_mfn[] */ - nr_mfns =0 ; -for ( i =0 ; i < j; i++ ) - { -unsigned long pfn, pagetype; - pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; - pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; - -if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) - region_mfn[i] = ~0UL ;/* map will fail but we don't care */ -else - { -if ( p2m[pfn] == (INVALID_P2M_ENTRY -1 ) ) - { -/* We just allocated a new mfn above; update p2m */ - p2m[pfn] = p2m_batch[nr_mfns++]; - nr_pfns++; - } - -/* setup region_mfn[] for batch map. -* For HVM guests, this interface takes PFNs, not MFNs */ - region_mfn[i] = hvm ? pfn : p2m[pfn]; - } - } - -/* Map relevant mfns */ - region_base =xc_map_foreign_batch ( - xc_handle, dom,PROT_WRITE , region_mfn, j); - -if ( region_base ==NULL ) - { -ERROR ("map batch failed" ); - goto out; - } - -for ( i =0 ; i < j; i++ ) - { -void *page; -unsigned long pagetype; - - pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; - pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; - -if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) -/* a bogus/unmapped page: skip it */ -continue ; - -if ( pfn > p2m_size ) - { -ERROR ("pfn out of range" ); - goto out; - } - - pfn_type[pfn] = pagetype; - - mfn = p2m[pfn]; - -/* In verify mode, we use a copy; otherwise we work in place */ - page = verify ? (void *)buf : (region_base + i*PAGE_SIZE ); - -if (read_exact (io_fd, page,PAGE_SIZE ) ) - { -ERROR ("Error when reading page (type was %lx)" , pagetype); - goto out; - } - - pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; - -if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && - (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) - { -/* -** A page table page - need to 'uncanonicalize' it, i.e. -** replace all the references to pfns with the corresponding -** mfns for the new domain. -** -** On PAE we need to ensure that PGDs are in MFNs < 4G, and -** so we may need to update the p2m after the main loop. -** Hence we defer canonicalization of L1s until then. -*/ -if ((pt_levels !=3 ) || - pae_extended_cr3 || - (pagetype !=XEN_DOMCTL_PFINFO_L1TAB )) { - -if (!uncanonicalize_pagetable (xc_handle, dom, - pagetype, page)) { -/* -** Failing to uncanonicalize a page table can be ok -** under live migration since the pages type may have -** changed by now (and we'll get an update later). -*/ -DPRINTF ("PT L%ld race on pfn=%08lx mfn=%08lx\n" , - pagetype >>28 , pfn, mfn); - nraces++; -continue ; - } - } - } -else if ( pagetype !=XEN_DOMCTL_PFINFO_NOTAB ) - { -ERROR ("Bogus page type %lx page table is out of range: " -"i=%d p2m_size=%lu" , pagetype, i, p2m_size); - goto out; - - } - -if ( verify ) - { -int res =memcmp (buf, (region_base + i*PAGE_SIZE ),PAGE_SIZE ); -if ( res ) - { -int v; - -DPRINTF ("************** pfn=%lx type=%lx gotcs=%08lx " -"actualcs=%08lx\n" , pfn, pfn_type[pfn], -csum_page (region_base + i*PAGE_SIZE ), -csum_page (buf)); - -for ( v =0 ; v <4 ; v++ ) - { -unsigned long *p = (unsigned long *) - (region_base + i*PAGE_SIZE ); -if ( buf[v] != p[v] ) -DPRINTF (" %d: %08lx %08lx\n" , v, buf[v], p[v]); - } - } - } - -if ( !hvm && -xc_add_mmu_update (xc_handle, mmu, - (((unsigned long long )mfn) <<PAGE_SHIFT ) - |MMU_MACHPHYS_UPDATE , pfn) ) - { -ERROR ("failed machpys update mfn=%lx pfn=%lx" , mfn, pfn); - goto out; - } - }/* end of 'batch' for loop */ - -munmap (region_base, j*PAGE_SIZE ); - n+= j;/* crude stats */ + n += j;/* crude stats */ /* * Discard cache for portion of file read so far up to last
@@ -785,6 +1020,32 @@/* Non-HVM guests only from here on */ +if (!completed) { +if (buffer_tail (&tailbuf, io_fd, max_vcpu_id, vcpumap, + ext_vcpucontext) <0 ) { +ERROR ("error buffering image tail" ); + goto out; + } + completed =1 ; + } + +DPRINTF ("Buffered checkpoint\n" ); +if (pagebuf_get (&pagebuf, io_fd)) { +ERROR ("error when buffering batch, finishing\n" ); + goto finish; + } +if (buffer_tail (&tmptail, io_fd, max_vcpu_id, vcpumap, + ext_vcpucontext) <0 ) { +ERROR ("error buffering image tail, finishing" ); + goto finish; + } +tailbuf_free (&tailbuf); +memcpy (&tailbuf, &tmptail,sizeof (tailbuf)); + + goto loadpages; + + finish: +if ( (pt_levels ==3 ) && !pae_extended_cr3 ) {/*
@@ -953,39 +1214,17 @@/* Get the list of PFNs that are not in the psuedo-phys map */ { -unsigned int count =0 ; -unsigned long *pfntab; -int nr_frees; +int nr_frees =0 ; -if (read_exact (io_fd, &count,sizeof (count)) || - (count > (1U <<28 )) )/* up to 1TB of address space */ +for ( i =0 ; i < tailbuf.pfncount; i++ ) { -ERROR ("Error when reading pfn count (= %u)" , count); - goto out; - } - -if ( !(pfntab =malloc (sizeof (unsigned long ) * count)) ) - { -ERROR ("Out of memory" ); - goto out; - } - -if (read_exact (io_fd, pfntab,sizeof (unsigned long )*count) ) - { -ERROR ("Error when reading pfntab" ); - goto out; - } - - nr_frees =0 ; -for ( i =0 ; i < count; i++ ) - { -unsigned long pfn = pfntab[i]; +unsigned long pfn = tailbuf.pfntab[i];if ( p2m[pfn] !=INVALID_P2M_ENTRY ) {/* pfn is not in physmap now, but was at some point during the save/migration process - need to free it */ - pfntab[nr_frees++] = p2m[pfn]; + tailbuf.pfntab[nr_frees++] = p2m[pfn]; p2m[pfn] =INVALID_P2M_ENTRY ;/* not in pseudo-physical map */ } }
@@ -997,7 +1236,7 @@ .extent_order = 0, .domid = dom }; -set_xen_guest_handle (reservation.extent_start, pfntab); +set_xen_guest_handle (reservation.extent_start, tailbuf.pfntab);if ( (frc =xc_memory_op (xc_handle, XENMEM_decrease_reservation, &reservation)) != nr_frees )
@@ -1006,7 +1245,7 @@ goto out; }else -DPRINTF ("Decreased reservation by %d pages\n" , count); +DPRINTF ("Decreased reservation by %d pages\n" , tailbuf.pfncount); } }
@@ -1016,18 +1255,17 @@return 1 ; } + vcpup = tailbuf.vcpubuf;for ( i =0 ; i <= max_vcpu_id; i++ ) {if ( !(vcpumap & (1ULL << i)) )continue ; -if (read_exact (io_fd, &ctxt, ((guest_width ==8 ) - ?sizeof (ctxt.x64) - :sizeof (ctxt.x32))) ) - { -ERROR ("Error when reading ctxt %d" , i); - goto out; - } +memcpy (&ctxt, vcpup, ((guest_width ==8 ) ?sizeof (ctxt.x64) + :sizeof (ctxt.x32))); + vcpup += (guest_width ==8 ) ?sizeof (ctxt.x64) :sizeof (ctxt.x32); + +DPRINTF ("read VCPU %d\n" , i);if ( !new_ctxt_format )SET_FIELD (&ctxt, flags,GET_FIELD (&ctxt, flags) | VGCF_online);
@@ -1132,12 +1370,8 @@if ( !ext_vcpucontext )continue ; -if (read_exact (io_fd, &domctl.u.ext_vcpucontext,128 ) || - (domctl.u.ext_vcpucontext.vcpu != i) ) - { -ERROR ("Error when reading extended ctxt %d" , i); - goto out; - } +memcpy (&domctl.u.ext_vcpucontext, vcpup,128 ); + vcpup +=128 ; domctl.cmd = XEN_DOMCTL_set_ext_vcpucontext; domctl.domain = dom; frc =xc_domctl (xc_handle, &domctl);
@@ -1148,11 +1382,9 @@ } } -if (read_exact (io_fd, shared_info_page,PAGE_SIZE ) ) - { -ERROR ("Error when reading shared info page" ); - goto out; - } +memcpy (shared_info_page, tailbuf.shared_info_page,PAGE_SIZE ); + +DPRINTF ("Completed checkpoint load\n" );/* Restore contents of shared-info page. No checking needed. */ new_shared_info = xc_map_foreign_range(
PATCH 1/1 Initiate failover if a packet is not received every 500ms
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID b51238ea926948383500b94cd227321eb40a82dd
# Parent f5c0d3208d8ae9183391398d52c9be5969da24ec Initiate failover if a packet is not received every 500ms. This breaks checkpoints at lower frequencies, and should be made configurable.
tools/libxc/xc_domain_restore.c +76 -15
@@ -59,6 +59,51 @@/* Address size of the guest, in bytes */ unsigned int guest_width; +/* set when a consistent image is available */ +static int completed =0 ; + +#define HEARTBEAT_MS 500 + +# ifndef __MINIOS__ +static ssize_t read_exact_timed (int fd,void * buf,size_t size) +{ +size_t offset =0 ; +ssize_t len; +struct timeval tv; +fd_set rfds; + +while ( offset < size ) + { +if (completed) { +/* expect a heartbeat every HEARBEAT_MS ms maximum */ + tv.tv_sec =0 ; + tv.tv_usec =HEARTBEAT_MS *1000 ; + +FD_ZERO (&rfds); +FD_SET (fd, &rfds); + len =select (fd +1 , &rfds,NULL ,NULL , &tv); +if ( !FD_ISSET (fd, &rfds) ) { +fprintf (stderr,"read_exact_timed failed (select returned %zd)\n" , len); +return -1 ; + } + } + + len =read (fd, buf + offset, size - offset); +if ( (len ==-1 ) && ((errno ==EINTR ) || (errno ==EAGAIN )) ) +continue ; +if ( len <=0 ) +return -1 ; + offset += len; + } + +return 0 ; +} + +#define read_exact read_exact_timed + +#else +#define read_exact_timed read_exact +#endif /* ** In the state file (or during transfer), all page-table pages are ** converted into a 'canonical' form where references to actual mfns
@@ -413,7 +458,9 @@// DPRINTF("reading batch of %d pages\n", count); if (!count) { +/* DPRINTF("Last batch read\n"); +*/ return 0 ; }else if (count ==-1 ) {DPRINTF ("Entering page verify mode\n" );
@@ -704,7 +751,8 @@int xc_domain_restore (int xc_handle,int io_fd,uint32_t dom,unsigned int store_evtchn,unsigned long *store_mfn,unsigned int console_evtchn,unsigned long *console_mfn, -unsigned int hvm,unsigned int pae) +unsigned int hvm,unsigned int pae, +int (*resume)(void *),void * resumedata) {DECLARE_DOMCTL ;int rc =1 , frc, i, j, n, m, pae_extended_cr3 =0 , ext_vcpucontext =0 ;
@@ -752,7 +800,6 @@/* Buffer for holding HVM context */ uint8_t *hvm_buf =NULL ; -int completed =0 ;pagebuf_t pagebuf;tailbuf_t tailbuf, tmptail;void * vcpup;
@@ -946,7 +993,9 @@ goto out; } +/* DPRINTF("Received all pages (%d races)\n", nraces); +*/ if ( hvm ) {
@@ -1021,28 +1070,40 @@/* Non-HVM guests only from here on */ if (!completed) { +int flags =0 ; +if (buffer_tail (&tailbuf, io_fd, max_vcpu_id, vcpumap, ext_vcpucontext) <0 ) {ERROR ("error buffering image tail" ); goto out; } + completed =1 ; +/* shift into nonblocking mode for the remainder */ +if ((flags =fcntl (io_fd,F_GETFL ,0 )) <0 ) + flags =0 ; +fcntl (io_fd,F_SETFL , flags |O_NONBLOCK ); } - + +/* DPRINTF("Buffered checkpoint\n"); -if (pagebuf_get (&pagebuf, io_fd)) { -ERROR ("error when buffering batch, finishing\n" ); - goto finish; +*/ +if (!resume (resumedata)) { +if (pagebuf_get (&pagebuf, io_fd)) { +ERROR ("error when buffering batch, finishing\n" ); + goto finish; + } +memset (&tmptail,0 ,sizeof (tmptail)); +if (buffer_tail (&tmptail, io_fd, max_vcpu_id, vcpumap, + ext_vcpucontext) <0 ) { +ERROR ("error buffering image tail, finishing" ); + goto finish; + } +tailbuf_free (&tailbuf); +memcpy (&tailbuf, &tmptail,sizeof (tailbuf)); + + goto loadpages; } -if (buffer_tail (&tmptail, io_fd, max_vcpu_id, vcpumap, - ext_vcpucontext) <0 ) { -ERROR ("error buffering image tail, finishing" ); - goto finish; - } -tailbuf_free (&tailbuf); -memcpy (&tailbuf, &tmptail,sizeof (tailbuf)); - - goto loadpages; finish:
tools/libxc/xenguest.h +5 -1
@@ -54,12 +54,16 @@* @parm store_mfn returned with the mfn of the store page * @parm hvm non-zero if this is a HVM restore * @parm pae non-zero if this HVM domain has PAE support enabled +* @parm resume a function returning 1 to resume or 0 to expect +* another checkpoint +* @parm resumedata a void pointer to pass back to the resume function * @return 0 on success, -1 on failure */ int xc_domain_restore (int xc_handle,int io_fd,uint32_t dom,unsigned int store_evtchn,unsigned long *store_mfn,unsigned int console_evtchn,unsigned long *console_mfn, -unsigned int hvm,unsigned int pae); +unsigned int hvm,unsigned int pae, +int (*resume)(void *),void * resumedata);/** * This function will create a domain for a paravirtualized Linux
tools/python/xen/xend/XendCheckpoint.py +2 -1
@@ -318,7 +318,8 @@ restore_image.setCpuid() - os.read(fd, 1) # Wait for source to close connection + #os.read(fd, 1) # Wait for source to close connection + # ^^ breaks failover, and I don't know why it's needed. dominfo.completeRestore(handler.store_mfn, handler.console_mfn)
tools/xcutils/xc_restore.c +54 -1
78 lines not on this page · raw message
PATCH 1/1 Support more than 2 FDs per tapdisk
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID 99c01882edb65e702875afc76319e92b88966402
# Parent b51238ea926948383500b94cd227321eb40a82dd Support more than 2 FDs per tapdisk.
The Remus disk replication module needs this to be able to poll the backup server as well as the underlying disk module.
tools/blktap/drivers/tapdisk.c +14 -9
@@ -132,16 +132,19 @@ {fd_list_entry_t *ptr;struct disk_driver *dd; +int i; ptr = fd_start;while (ptr !=NULL ) {if (ptr->tap_fd) {FD_SET (ptr->tap_fd, readfds);td_for_each_disk (ptr->s, dd) { -if (dd->io_fd[READ ]) -FD_SET (dd->io_fd[READ ], readfds); - maxfds = (dd->io_fd[READ ] > maxfds ? - dd->io_fd[READ ] : maxfds); +for (i =0 ; i <MAX_IOFD ; i++) { +if (dd->io_fd[i]) +FD_SET (dd->io_fd[i], readfds); + maxfds = (dd->io_fd[i] > maxfds ? + dd->io_fd[i] : maxfds); + } } maxfds = (ptr->tap_fd > maxfds ? ptr->tap_fd : maxfds); }
@@ -820,13 +823,15 @@int progress_made =0 ;struct disk_driver *dd;tapdev_info_t *info = ptr->s->ring_info; +int i;td_for_each_disk (ptr->s, dd) { -if (dd->io_fd[READ ] && -FD_ISSET (dd->io_fd[READ ], - &readfds)) { -io_done (dd,READ ); - progress_made =1 ; +for (i =0 ; i <MAX_IOFD ; i++) { +if (dd->io_fd[i] && +FD_ISSET (dd->io_fd[i], &readfds)) { +io_done (dd, i); + progress_made =1 ; + } } }
tools/blktap/drivers/tapdisk.h +1 -1
@@ -74,7 +74,7 @@#define SECTOR_SHIFT 9#define DEFAULT_SECTOR_SIZE 512 -#define MAX_IOFD 2 +#define MAX_IOFD 4#define BLK_NOT_ALLOCATED 99#define TD_NO_PARENT 1
PATCH 1/1 Check tapdisk paths from the last colon instead of the first
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully
# Date 1242260367 25200
# Node ID 42fddb3a8edeb80339618b7f758dc2959cf97115
# Parent 99c01882edb65e702875afc76319e92b88966402
Check tapdisk paths from the last colon instead of the first. This allows the Remus tapdisk proxy to configure itself using the form tap:remus:host:port:aio:....
tools/blktap/lib/xenbus.c +1 -1
@@ -256,7 +256,7 @@ path +=strlen ("ioemu:" ); } - tmp =strchr (path,':' ); + tmp =strrchr (path,':' );if (tmp !=NULL ) path = tmp +1 ;
tools/python/xen/util/blkif.py +2 -1
@@ -78,7 +78,8 @@ if typ in ("phy", "drbd") and not fn.startswith("/"): fn = "/dev/%s" %(fn,) if typ == "tap": - (taptype, fn) = fn.split(":", 1) + flds = fn.split(':') + (taptype, fn) = flds[0], flds[-1] return (fn, taptype) def blkdev_uname_to_file(uname):
PATCH 1/1 Remus tapdisk proxy
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID 03fd0c9729f3d87e7803afb170dfc3cdff184998
# Parent 42fddb3a8edeb80339618b7f758dc2959cf97115
Remus tapdisk proxy.
This proxy forwards local disk writes to a backup server, where they are buffered until the local disk receives a checkpoint signal from the remus control tools, which it forwards to the backup to cause buffered writes to be flushed.
Configuration is of the form
tap:remus:<backup host>:<backup port>:<underlying tapdisk string>
The first write to a disk protected by this proxy will block until the Remus control tools have been activated, which create the backup disk proxy.
tools/blktap/drivers/Makefile +5 -1
@@ -11,6 +11,8 @@ CFLAGS += $(CFLAGS_libxenctrl) CFLAGS += $(CFLAGS_libxenstore) CFLAGS += -I $(LIBAIO_DIR) +# for hashtable_itr.h in gcc 4.2 +CFLAGS += -fgnu89-inline CFLAGS += -D_GNU_SOURCE ifeq ($(shell . ./check_gcrypt $(CC)),yes)
@@ -22,7 +24,8 @@ endif LDFLAGS_blktapctrl := $(LDFLAGS_libxenctrl) $(LDFLAGS_libxenstore) -L../lib -lblktap -LDFLAGS_img := $(LIBAIO_DIR)/libaio.a $(CRYPT_LIB) -lpthread -lz +# hashtable.c uses ceilf from libm +LDFLAGS_img := $(LIBAIO_DIR)/libaio.a $(CRYPT_LIB) -lpthread -lm -lz BLK-OBJS-y := block-aio.o BLK-OBJS-y += block-sync.o
@@ -30,6 +33,7 @@ BLK-OBJS-y += block-ram.o BLK-OBJS-y += block-qcow.o BLK-OBJS-y += block-qcow2.o +BLK-OBJS-y += block-remus.o hashtable.o hashtable_itr.o hashtable_utility.o BLK-OBJS-y += aes.o BLK-OBJS-y += tapaio.o BLK-OBJS-$(CONFIG_Linux) += blk_linux.o
tools/blktap/drivers/block-remus.c new file +1686 -0
1,686 lines not on this page · raw message
tools/blktap/drivers/hashtable.c new file +274 -0
274 lines not on this page · raw message
tools/blktap/drivers/hashtable_itr.c new file +188 -0
188 lines not on this page · raw message
tools/blktap/drivers/hashtable_itr.h new file +112 -0
112 lines not on this page · raw message
tools/blktap/drivers/hashtable_utility.c new file +71 -0
71 lines not on this page · raw message
tools/blktap/drivers/hashtable_utility.h new file +55 -0
55 lines not on this page · raw message
tools/blktap/drivers/tapdisk.h +13 -0
@@ -160,6 +160,7 @@extern struct tap_disk tapdisk_ram;extern struct tap_disk tapdisk_qcow;extern struct tap_disk tapdisk_qcow2; +extern struct tap_disk tapdisk_remus;/*Define Individual Disk Parameters here */
@@ -229,6 +230,17 @@#endif }; +static disk_info_t remus_disk = { +DISK_TYPE_REMUS , +"replicated disk (remus)" , +"remus" , +0 , +0 , +#ifdef TAPDISK + &tapdisk_remus, +#endif +}; +/*Main disk info array */ static disk_info_t *dtypes[] = { &aio_disk,
@@ -237,6 +249,7 @@ &ram_disk, &qcow_disk, &qcow2_disk, + &remus_disk }; typedef struct driver_list_entry {
tools/blktap/lib/blktaplib.h +1 -0
@@ -219,6 +219,7 @@#define DISK_TYPE_RAM 3#define DISK_TYPE_QCOW 4#define DISK_TYPE_QCOW2 5 +#define DISK_TYPE_REMUS 6/* xenstore/xenbus: */ #define DOMNAME "Domain-0"
tools/python/xen/xend/server/BlktapController.py +1 -0
@@ -14,6 +14,7 @@ 'ram', 'qcow', 'qcow2', + 'remus', 'ioemu', 'tapdisk',
PATCH 1/1 Buffer checkpoint data locally until domain has resumed execution
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID dd394ec3ad784da4cfe789c380aa47092dfb0f20
# Parent 03fd0c9729f3d87e7803afb170dfc3cdff184998 Buffer checkpoint data locally until domain has resumed execution.
tools/libxc/xc_domain_save.c +174 -25
@@ -57,6 +57,15 @@/* Address size of the guest */ unsigned int guest_width; +/* buffer for output */ +struct outbuf { +void * buf; +size_t size; +size_t pos; +}; + +#define OUTBUF_SIZE (16384 * 1024) +/* grep fodder: machine_to_phys */ #define mfn_to_pfn (_mfn) (live_m2p[(_mfn)])
@@ -159,6 +168,93 @@return rc; } +static int outbuf_init (struct outbuf * ob,size_t size) +{ +memset (ob,0 ,sizeof (*ob)); + +if (!(ob->buf =malloc (size))) { +DPRINTF ("error allocating output buffer of size %zu\n" , size); +return -1 ; + } + + ob->size = size; + +return 0 ; +} + +static inline int outbuf_resize (struct outbuf * ob,size_t nsize) +{ +void * nbuf; + +if (nsize <= ob->size) +return 0 ; + +if (!(nbuf =realloc (ob->buf, nsize))) { +DPRINTF ("error reallocating output buffer from %zu to %zu\n" , + ob->size, nsize); +return -1 ; + } + + ob->buf = nbuf; + +return 0 ; +} + +static inline int outbuf_write (struct outbuf * ob,void * buf,size_t len) +{ +if (len > ob->size - ob->pos) { +DPRINTF ("outbuf_write: %zu > %zu@%zu\n" , len, ob->size - ob->pos, ob->pos); +return -1 ; + } + +memcpy (ob->buf + ob->pos, buf, len); + ob->pos += len; + +return 0 ; +} + +/* prep for nonblocking I/O */ +static int outbuf_flush (struct outbuf * ob,int fd) +{ +int rc; +int cur =0 ; + +if (!ob->pos) +return 0 ; + + rc =write (fd, ob->buf, ob->pos); +while (rc <0 || cur + rc < ob->pos) { +if (rc <0 && errno !=EAGAIN && errno !=EINTR ) { +DPRINTF ("error flushing output: %d\n" , errno); +return -1 ; + } +if (rc >0 ) + cur += rc; + + rc =write (fd, ob->buf + cur, ob->pos - cur); + } + + ob->pos =0 ; + +return 0 ; +} + +/* if there's no room in the buffer, flush it and try again. */ +static inline int outbuf_hardwrite (struct outbuf * ob,int fd,void * buf, +size_t len) +{ +if (!len) +return 0 ; + +if (!outbuf_write (ob, buf, len)) +return 0 ; + +if (outbuf_flush (ob, fd) <0 ) +return -1 ; + +return outbuf_write (ob, buf, len); +} +#ifdef ADAPTIVE_SAVE /*
@@ -799,6 +895,10 @@unsigned long mfn; +struct outbuf ob; + +outbuf_init (&ob,OUTBUF_SIZE ); +/* If no explicit control parameters given, use defaults */ max_iters = max_iters ? :DEF_MAX_ITERS ; max_factor = max_factor ? :DEF_MAX_FACTOR ;
@@ -1171,13 +1271,21 @@ } } -if (write_exact (io_fd, &batch,sizeof (unsigned int )) ) +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd, &batch,sizeof (unsigned int )); +else + rc =write_exact (io_fd, &batch,sizeof (unsigned int )); +if ( rc ) {PERROR ("Error when writing to state file (2)" ); goto out; } -if (write_exact (io_fd, pfn_type,sizeof (unsigned long )*batch) ) +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd, pfn_type,sizeof (unsigned long )*batch); +else + rc =write_exact (io_fd, pfn_type,sizeof (unsigned long )*batch); +if ( rc ) {PERROR ("Error when writing to state file (3)" ); goto out;
@@ -1199,9 +1307,14 @@run of pages we may have previously acumulated */ if ( run ) { -if (ratewrite (io_fd, live, - (char *)region_base+(PAGE_SIZE *(j-run)), -PAGE_SIZE *run) !=PAGE_SIZE *run ) +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd,(char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run); +else + rc =ratewrite (io_fd, live, + (char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run) !=PAGE_SIZE *run; +if ( rc ) {ERROR ("Error when writing to state file (4a)" " (errno %d)" , errno);
@@ -1231,7 +1344,12 @@ goto out; } -if (ratewrite (io_fd, live, page,PAGE_SIZE ) !=PAGE_SIZE ) +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd, page,PAGE_SIZE ); +else + rc =ratewrite (io_fd, live, page,PAGE_SIZE ) != +PAGE_SIZE ; +if ( rc ) {ERROR ("Error when writing to state file (4b)" " (errno %d)" , errno);
@@ -1240,17 +1358,37 @@ }else { +#if 0 /* We have a normal page: accumulate it for writing. */ run++; +#else +/* We have a normal page: just write it directly. */ +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd, spage,PAGE_SIZE ); +else + rc =ratewrite (io_fd, live, spage,PAGE_SIZE ) != +PAGE_SIZE ; +if ( rc ) + { +ERROR ("Error when writing to state file (5)" +" (errno %d)" , errno); + goto out; + } +#endif } }/* end of the write out for this batch */ if ( run ) {/* write out the last accumulated run of pages */ -if (ratewrite (io_fd, live, - (char *)region_base+(PAGE_SIZE *(j-run)), -PAGE_SIZE *run) !=PAGE_SIZE *run ) +if ( last_iter ) + rc =outbuf_hardwrite (&ob, io_fd, (char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run); +else + rc =ratewrite (io_fd, live, + (char *)region_base+(PAGE_SIZE *(j-run)), +PAGE_SIZE *run) !=PAGE_SIZE *run; +if ( rc ) {ERROR ("Error when writing to state file (4c)" " (errno %d)" , errno);
@@ -1275,9 +1413,11 @@ {print_stats ( xc_handle, dom, sent_this_iter, &stats,1 ); +/* DPRINTF("Total pages sent= %ld (%.2fx)\n", total_sent, ((float)total_sent)/p2m_size ); DPRINTF("(of which %ld were fixups)\n", needed_to_fix ); +*/ }if ( last_iter && debug )
@@ -1288,7 +1428,7 @@DPRINTF ("Entering debug resend-all mode\n" );/* send "-1" to put receiver into debug mode */ -if (write_exact (io_fd, &minusone,sizeof (int )) ) +if (outbuf_hardwrite (&ob, io_fd, &minusone,sizeof (int )) ) {PERROR ("Error when writing to state file (6)" ); goto out;
@@ -1356,7 +1496,9 @@ } }/* end of infinite for loop */ +/* DPRINTF("All memory is saved\n"); +*/ {struct {
@@ -1380,7 +1522,7 @@ } chunk.vcpumap = vcpumap; -if (write_exact (io_fd, &chunk,sizeof (chunk)) ) +if (outbuf_hardwrite (&ob, io_fd, &chunk,sizeof (chunk)) ) {PERROR ("Error when writing to state file" ); goto out;
@@ -1400,7 +1542,7 @@ (unsigned long *)&chunk.data);if ( (chunk.data !=0 ) && -write_exact (io_fd, &chunk,sizeof (chunk)) ) +outbuf_hardwrite (&ob, io_fd, &chunk,sizeof (chunk)) ) {PERROR ("Error when writing the ident_pt for EPT guest" ); goto out;
@@ -1411,7 +1553,7 @@ (unsigned long *)&chunk.data);if ( (chunk.data !=0 ) && -write_exact (io_fd, &chunk,sizeof (chunk)) ) +outbuf_hardwrite (&ob, io_fd, &chunk,sizeof (chunk)) ) {PERROR ("Error when writing the vm86 TSS for guest" ); goto out;
@@ -1420,7 +1562,7 @@/* Zero terminate */ i =0 ; -if (write_exact (io_fd, &i,sizeof (int )) ) +if (outbuf_hardwrite (&ob, io_fd, &i,sizeof (int )) ) {PERROR ("Error when writing to state file (6')" ); goto out;
@@ -1438,7 +1580,7 @@ (unsigned long *)&magic_pfns[1]);xc_get_hvm_param (xc_handle, dom,HVM_PARAM_STORE_PFN , (unsigned long *)&magic_pfns[2 ]); -if (write_exact (io_fd, magic_pfns,sizeof (magic_pfns)) ) +if (outbuf_hardwrite (&ob, io_fd, magic_pfns,sizeof (magic_pfns)) ) {PERROR ("Error when writing to state file (7)" ); goto out;
@@ -1452,13 +1594,13 @@ goto out; } -if (write_exact (io_fd, &rec_size,sizeof (uint32_t )) ) +if (outbuf_hardwrite (&ob, io_fd, &rec_size,sizeof (uint32_t )) ) {PERROR ("error write hvm buffer size" ); goto out; } -if (write_exact (io_fd, hvm_buf, rec_size) ) +if (outbuf_hardwrite (&ob, io_fd, hvm_buf, rec_size) ) {PERROR ("write HVM info failed!\n" ); goto out;
@@ -1482,7 +1624,7 @@ j++; } -if (write_exact (io_fd, &j,sizeof (unsigned int )) ) +if (outbuf_hardwrite (&ob, io_fd, &j,sizeof (unsigned int )) ) {PERROR ("Error when writing to state file (6a)" ); goto out;
@@ -1496,7 +1638,7 @@ i++;if ( (j ==1024 ) || (i == p2m_size) ) { -if (write_exact (io_fd, &pfntab,sizeof (unsigned long )*j) ) +if (outbuf_hardwrite (&ob, io_fd, &pfntab,sizeof (unsigned long )*j) ) {PERROR ("Error when writing to state file (6b)" ); goto out;
@@ -1567,9 +1709,9 @@FOLD_CR3 (mfn_to_pfn (UNFOLD_CR3 (ctxt.x64.ctrlreg[1 ]))); } -if (write_exact (io_fd, &ctxt, ((guest_width==8 ) - ?sizeof (ctxt.x64) - :sizeof (ctxt.x32))) ) +if (outbuf_hardwrite (&ob, io_fd, &ctxt, ((guest_width==8 ) + ?sizeof (ctxt.x64) + :sizeof (ctxt.x32))) ) {PERROR ("Error when writing to state file (1)" ); goto out;
@@ -1583,7 +1725,7 @@ERROR ("No extended context for VCPU%d" , i); goto out; } -if (write_exact (io_fd, &domctl.u.ext_vcpucontext,128 ) ) +if (outbuf_hardwrite (&ob, io_fd, &domctl.u.ext_vcpucontext,128 ) ) {PERROR ("Error when writing to state file (2)" ); goto out;
@@ -1596,7 +1738,7 @@memcpy (page, live_shinfo,PAGE_SIZE );SET_FIELD (((shared_info_any_t *)page), arch.pfn_to_mfn_frame_list_list,0 ); -if (write_exact (io_fd, page,PAGE_SIZE ) ) +if (outbuf_hardwrite (&ob, io_fd, page,PAGE_SIZE ) ) {PERROR ("Error when writing to state file (1)" ); goto out;
@@ -1610,6 +1752,11 @@ callbacks->postcopy (callbacks->data);/* Flush last write and discard cache for file. */ +if (outbuf_flush (&ob, io_fd) <0 ) { +ERROR ("Error when flushing output buffer\n" ); + rc =1 ; + } +discard_file_cache (io_fd,1 /* flush */ );/* checkpoint_cb can spend arbitrarily long in between rounds */
@@ -1627,9 +1774,11 @@ERROR ("Domain appears not to have suspended" ); goto out; } +/* DPRINTF("SUSPEND shinfo %08lx\n", info.shared_info_frame); +print_stats(xc_handle, dom, 0, &stats, 1); - +*/ if (xc_shadow_control (xc_handle, dom,XEN_DOMCTL_SHADOW_OP_CLEAN , to_send, p2m_size,NULL ,0 , &stats) != p2m_size )
PATCH 1/1 Do not bother with to_skip/to_fix bitmaps after the first final round
In reply to Brendan Cully# HG changeset patch
# User Geoffrey Lefebvre <[email protected]>
# Date 1240355511 25200
# Node ID 0f1925415df0a4323d431085a9ee8956b8a95764
# Parent dd394ec3ad784da4cfe789c380aa47092dfb0f20
Do not bother with to_skip/to_fix bitmaps after the first final round.
tools/libxc/xc_domain_save.c +58 -42
@@ -897,6 +897,8 @@struct outbuf ob; +int completed =0 ; +outbuf_init (&ob,OUTBUF_SIZE );/* If no explicit control parameters given, use defaults */
@@ -1153,54 +1155,66 @@ mfn_to_pfn(pfn_to_mfn(n)&0xFFFFF));DPRINTF ("\n" ); } -if ( !last_iter && -test_bit (n, to_send) && -test_bit (n, to_skip) ) - skip_this_iter++;/* stats keeping */ -if ( !((test_bit (n, to_send) && !test_bit (n, to_skip)) || - (test_bit (n, to_send) && last_iter) || - (test_bit (n, to_fix) && last_iter)) ) -continue ; +if ( completed ) + { +if ( !test_bit (n, to_send) ) +continue ; -/* -** we get here if: -** 1. page is marked to_send & hasn't already been re-dirtied -** 2. (ignore to_skip in last iteration) -** 3. add in pages that still need fixup (net bufs) -*/ + pfn_batch[batch] = n; + pfn_type[batch] = live_p2m[n]; + } +else + { +if ( !last_iter && +test_bit (n, to_send) && +test_bit (n, to_skip) ) + skip_this_iter++;/* stats keeping */ - pfn_batch[batch] = n; +if ( !((test_bit (n, to_send) && !test_bit (n, to_skip)) || + (test_bit (n, to_send) && last_iter) || + (test_bit (n, to_fix) && last_iter)) ) +continue ; -/* Hypercall interfaces operate in PFNs for HVM guests -* and MFNs for PV guests */ -if ( hvm ) - pfn_type[batch] = n; -else - pfn_type[batch] =pfn_to_mfn (n); +/* +** we get here if: +** 1. page is marked to_send & hasn't already been re-dirtied +** 2. (ignore to_skip in last iteration) +** 3. add in pages that still need fixup (net bufs) +*/ + + pfn_batch[batch] = n; + +/* Hypercall interfaces operate in PFNs for HVM guests +* and MFNs for PV guests */ +if ( hvm ) + pfn_type[batch] = n; +else + pfn_type[batch] =pfn_to_mfn (n); -if ( !is_mapped (pfn_type[batch]) ) - { -/* -** not currently in psuedo-physical map -- set bit -** in to_fix since we must send this page in last_iter -** unless its sent sooner anyhow, or it never enters -** pseudo-physical map (e.g. for ballooned down doms) -*/ -set_bit (n, to_fix); -continue ; +if ( !is_mapped (pfn_type[batch]) ) + { +/* +** not currently in psuedo-physical map -- set bit +** in to_fix since we must send this page in last_iter +** unless its sent sooner anyhow, or it never enters +** pseudo-physical map (e.g. for ballooned down doms) +*/ +set_bit (n, to_fix); +continue ; + } + +if ( last_iter && +test_bit (n, to_fix) && + !test_bit (n, to_send) ) + { + needed_to_fix++; +DPRINTF ("Fix! iter %d, pfn %x. mfn %lx\n" , + iter, n, pfn_type[batch]); + } + +clear_bit (n, to_fix); } - -if ( last_iter && -test_bit (n, to_fix) && - !test_bit (n, to_send) ) - { - needed_to_fix++; -DPRINTF ("Fix! iter %d, pfn %x. mfn %lx\n" , - iter, n, pfn_type[batch]); - } - -clear_bit (n, to_fix); batch++; }
@@ -1748,6 +1762,8 @@ rc =0 ; out: + completed =1 ; +if ( !rc && callbacks->postcopy ) callbacks->postcopy (callbacks->data);
PATCH 1/1 Do bitmap scan word-by-word before bit-by-bit
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355511 25200
# Node ID 4a9831ea4a7d7bb95cd9584abe94e2fdc50f46f6
# Parent 0f1925415df0a4323d431085a9ee8956b8a95764 Do bitmap scan word-by-word before bit-by-bit.
For sparse bitmaps and large domains this saves a lot of time.
tools/libxc/xc_domain_save.c +10 -0
@@ -99,6 +99,8 @@#define BITMAP_SHIFT (_nr) ((_nr) % BITS_PER_LONG) +#define ORDER_LONG (sizeof(unsigned long) == 4 ? 5 : 6) +static inline int test_bit (int nr,volatile void * addr) {return (BITMAP_ENTRY (nr, addr) >>BITMAP_SHIFT (nr)) &1 ;
@@ -1158,6 +1160,14 @@if ( completed ) { +/* for sparse bitmaps, word-by-word may save time */ +if ( !to_send[N >>ORDER_LONG ] ) + { +/* incremented again in for loop! */ +N +=BITS_PER_LONG -1 ; +continue ; + } +if ( !test_bit (n, to_send) )continue ;