Remus 0.9 released!
PATCH 1/1 Remus 0.9 released!
Dear Xen enthusiasts,
It's my pleasure to announce the release of Remus 0.9!
Remus provides comprehensive fault tolerance for Xen virtual machines. If the physical machine hosting your VM fails, the backup can take over instantly, as if you had migrated it to the backup at the instant before the failure occured. There's no need for recovery, because the backup is always completely up to date. Furthermore, Remus runs completely transparently, requiring no changes to your existing guests.
This release works with the tip of the xen-unstable repository, and supports PV and HVM in 32-on-32, 64-on-64, 32-on-64, and 64-on-32 configurations. It has been tested using Linux (Ubuntu) PV guests, and both Linux and Windows XP under HVM.
I believe that it is now ready for inclusion in Xen, and I would love to have people try it out and let me know how it goes.
You can find more information, including installation and usage instructions, at the web site:
The web site includes a list of improvements I have in mind over the short term. Ideas (and patches) welcome!
-Brendan
PATCH 1/1 Add callbacks for suspend, postcopy and preresume in xc_domain_save
# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355494 25200
# Node ID d325cdd95abb5f675e3d63fa9cc59ca89271489a
# Parent ac9d4ba48b8334f0adc3a928be4e48d2e6fdebd1
Add callbacks for suspend, postcopy and preresume in xc_domain_save. This makes it possible to perform repeated checkpoints.
tools/libxc/xc_domain_save.c +45 -11
@@ -332,11 +332,11 @@return -1 ; } - -static int suspend_and_state (int (*suspend)(void ),int xc_handle,int io_fd, -int dom,xc_dominfo_t *info) +static int suspend_and_state (int (*suspend)(void *),void * data, +int xc_handle,int io_fd,int dom, +xc_dominfo_t *info) { -if ( !(*suspend)() ) +if ( !(*suspend)(data) ) {ERROR ("Suspend request failed" );return -1 ;
@@ -742,13 +742,14 @@ }int xc_domain_save (int xc_handle,int io_fd,uint32_t dom,uint32_t max_iters, -uint32_t max_factor,uint32_t flags,int (*suspend)(void ), +uint32_t max_factor,uint32_t flags, +struct save_callbacks * callbacks,int hvm,void (*switch_qemu_logdirty)(int ,unsigned )) {xc_dominfo_t info;DECLARE_DOMCTL ; -int rc =1 , frc, i, j, last_iter, iter =0 ; +int rc =1 , frc, i, j, last_iter =0 , iter =0 ;int live = (flags &XCFLAGS_LIVE );int debug = (flags &XCFLAGS_DEBUG );int race =0 , sent_last_iter, skip_this_iter;
@@ -864,7 +865,8 @@else {/* This is a non-live suspend. Suspend the domain .*/ -if (suspend_and_state (suspend, xc_handle, io_fd, dom, &info) ) +if (suspend_and_state (callbacks->suspend, callbacks->data, xc_handle, + io_fd, dom, &info) ) {ERROR ("Domain appears not to have suspended" ); goto out;
@@ -992,6 +994,7 @@ goto out; } + copypages:/* Now write out each data page, canonicalising page tables as we go... */ for ( ; ; ) {
@@ -1305,7 +1308,8 @@DPRINTF ("Start last iteration\n" ); last_iter =1 ; -if (suspend_and_state (suspend, xc_handle, io_fd, dom, &info) ) +if (suspend_and_state (callbacks->suspend, callbacks->data, + xc_handle, io_fd, dom, &info) ) {ERROR ("Domain appears not to have suspended" ); goto out;
@@ -1586,6 +1590,39 @@ rc =0 ; out: +if ( !rc && callbacks->postcopy ) + callbacks->postcopy (callbacks->data); + +/* Flush last write and discard cache for file. */ +discard_file_cache (io_fd,1 /* flush */ ); + +/* checkpoint_cb can spend arbitrarily long in between rounds */ +if (!rc && callbacks->checkpoint && + callbacks->checkpoint (callbacks->data) >0 ) + { +/* reset stats timer */ +print_stats (xc_handle, dom,0 , &stats,0 ); + + rc =1 ; +/* last_iter = 1; */ +if (suspend_and_state (callbacks->suspend, callbacks->data, xc_handle, + io_fd, dom, &info) ) + { +ERROR ("Domain appears not to have suspended" ); + goto out; + } +DPRINTF ("SUSPEND shinfo %08lx\n" , info.shared_info_frame); +print_stats (xc_handle, dom,0 , &stats,1 ); + +if (xc_shadow_control (xc_handle, dom, +XEN_DOMCTL_SHADOW_OP_CLEAN , to_send, + p2m_size,NULL ,0 , &stats) != p2m_size ) + { +ERROR ("Error flushing shadow PT" ); + } + + goto copypages; + }if ( tmem_saved !=0 && live )xc_tmem_save_done (xc_handle, dom);
@@ -1600,9 +1637,6 @@switch_qemu_logdirty (dom,0 ); } -/* Flush last write and discard cache for file. */ -discard_file_cache (io_fd,1 /* flush */ ); -if ( live_shinfo )munmap (live_shinfo,PAGE_SIZE );
tools/libxc/xenguest.h +15 -2
@@ -14,6 +14,19 @@#define XCFLAGS_HVM 4#define XCFLAGS_STDVGA 8 +/* callbacks provided by xc_domain_save */ +struct save_callbacks { +int (*suspend)(void * data); +/* callback to rendezvous with external checkpoint functions */ +int (*postcopy)(void * data); +/* returns: +* 0: terminate checkpointing gracefully +* 1: take another checkpoint */ +int (*checkpoint)(void * data); + +/* to be provided as the first argument to each callback function */ +void * data; +};/** * This function will save a running domain.
@@ -25,8 +38,8 @@*/ int xc_domain_save (int xc_handle,int io_fd,uint32_t dom,uint32_t max_iters,uint32_t max_factor,uint32_t flags/* XCFLAGS_xxx */ , -int (*suspend)(void ),int hvm, -void (*switch_qemu_logdirty)(int ,unsigned ));/* HVM only */ +struct save_callbacks * callbacks, +int hvm,void (*switch_qemu_logdirty)(int ,unsigned ));/* HVM only */ /**
tools/xcutils/xc_save.c +5 -2
@@ -71,7 +71,7 @@return 1 ; } -static int suspend (void ) +static int suspend (void * data) {unsigned long sx_state =0 ;
@@ -166,6 +166,7 @@ {unsigned int maxit, max_f;int io_fd, ret, port; +struct save_callbacks callbacks;if (argc !=6 )errx (1 ,"usage: %s iofd domid maxit maxf flags" , argv[0 ]);
@@ -202,8 +203,10 @@ "using slow path"); } } +memset (&callbacks,0 ,sizeof (callbacks)); + callbacks.suspend = suspend; ret =xc_domain_save (si.xc_fd, io_fd, si.domid, maxit, max_f, si.flags, - &suspend, !!(si.flags &XCFLAGS_HVM ), + &callbacks, !!(si.flags &XCFLAGS_HVM ), &switch_qemu_logdirty);if (si.suspend_evtchn >0 )
PATCH 1/1 Make xc_domain_restore loop until the fd is closed
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355507 25200
# Node ID a82f2e189c29acfc26794ea10d4000d0a350bf40
# Parent d325cdd95abb5f675e3d63fa9cc59ca89271489a
Make xc_domain_restore loop until the fd is closed.
The tail containing the final PFN table, VCPU contexts and shared_info_page is buffered, then the read loop is restarted. After the first pass, incoming pages are buffered until the next tail is read, completing a new consistent checkpoint. At this point, the memory changes are applied and the loop begins again. When the fd read fails, the tail buffer is processed.
tools/libxc/xc_domain_restore.c +493 -260
@@ -624,6 +624,410 @@return p2m_frame_list; } +typedef struct { +unsigned int pfncount; +unsigned long * pfntab; +unsigned int vcpucount; +unsigned char * vcpubuf; +unsigned char shared_info_page[PAGE_SIZE ]; +}tailbuf_t ; + +static int buffer_tail (tailbuf_t * buf,int fd,unsigned int max_vcpu_id, +uint64_t vcpumap,int ext_vcpucontext) +{ +unsigned int i; +size_t pfnlen, vcpulen; + +/* TODO: handle changing pfntab and vcpu counts */ +/* PFN tab */ +if (read_exact (fd, &buf->pfncount,sizeof (buf->pfncount)) || + (buf->pfncount > (1U <<28 )) )/* up to 1TB of address space */ + { +ERROR ("Error when reading pfn count" ); +return -1 ; + } + pfnlen =sizeof (unsigned long ) * buf->pfncount; +if ( !(buf->pfntab) ) { +if ( !(buf->pfntab =malloc (pfnlen)) ) { +ERROR ("Error allocating PFN tail buffer" ); +return -1 ; + } + } +// DPRINTF("Reading PFN tab: %d bytes\n", pfnlen); +if (read_exact (fd, buf->pfntab, pfnlen) ) { +ERROR ("Error when reading pfntab" ); + goto free_pfntab; + } + +/* VCPU contexts */ + buf->vcpucount =0 ; +for (i =0 ; i <= max_vcpu_id; i++) { +// DPRINTF("vcpumap: %llx, cpu: %d, bit: %llu\n", vcpumap, i, (vcpumap % (1ULL << i))); +if ( (!(vcpumap & (1ULL << i))) ) +continue ; + buf->vcpucount++; + } +// DPRINTF("VCPU count: %d\n", buf->vcpucount); + vcpulen = ((guest_width ==8 ) ?sizeof (vcpu_guest_context_x86_64_t) + :sizeof (vcpu_guest_context_x86_32_t)) * buf->vcpucount; +if ( ext_vcpucontext ) + vcpulen +=128 * buf->vcpucount; + +if ( !(buf->vcpubuf) ) { +if ( !(buf->vcpubuf =malloc (vcpulen)) ) { +ERROR ("Error allocating VCPU ctxt tail buffer" ); + goto free_pfntab; + } + } +// DPRINTF("Reading VCPUS: %d bytes\n", vcpulen); +if (read_exact (fd, buf->vcpubuf, vcpulen) ) { +ERROR ("Error when reading ctxt" ); + goto free_vcpus; + } + +/* load shared_info_page */ +// DPRINTF("Reading shared info: %lu bytes\n", PAGE_SIZE); +if (read_exact (fd, buf->shared_info_page,PAGE_SIZE ) ) { +ERROR ("Error when reading shared info page" ); + goto free_vcpus; + } + +return 0 ; + + free_vcpus: +if (buf->vcpubuf) { +free (buf->vcpubuf); + buf->vcpubuf =NULL ; + } + free_pfntab: +if (buf->pfntab) { +free (buf->pfntab); + buf->pfntab =NULL ; + } + +return -1 ; +} + +static void tailbuf_free (tailbuf_t * buf) +{ +if (buf->vcpubuf) { +free (buf->vcpubuf); + buf->vcpubuf =NULL ; + } +if (buf->pfntab) { +free (buf->pfntab); + buf->pfntab =NULL ; + } +} + +typedef struct { +void * pages; +/* pages is of length nr_physpages, pfn_types is of length nr_pages */ +unsigned int nr_physpages, nr_pages; + +/* Types of the pfns in the current region */ +unsigned long * pfn_types; + +int verify; + +int new_ctxt_format; +int max_vcpu_id; +uint64_t vcpumap; +uint64_t identpt; +uint64_t vm86_tss; +}pagebuf_t ; + +static int pagebuf_init (pagebuf_t * buf) +{ +memset (buf,0 ,sizeof (*buf)); +return 0 ; +} + +static void pagebuf_free (pagebuf_t * buf) +{ +if (buf->pages) { +free (buf->pages); + buf->pages =NULL ; + } +if (buf->pfn_types) { +free (buf->pfn_types); + buf->pfn_types =NULL ; + } +} + +static int pagebuf_get_one (pagebuf_t * buf,int fd,int xch,uint32_t dom) +{ +int count, countpages, oldcount, i; +void * ptmp; + +if (read_exact (fd, &count,sizeof (count)) ) + { +ERROR ("Error when reading batch size" ); +return -1 ; + } + +// DPRINTF("reading batch of %d pages\n", count); + +if (!count) { +// DPRINTF("Last batch read\n"); +return 0 ; + }else if (count ==-1 ) { +DPRINTF ("Entering page verify mode\n" ); + buf->verify =1 ; +return pagebuf_get_one (buf, fd, xch, dom); + }else if (count ==-2 ) { + buf->new_ctxt_format =1 ; +if (read_exact (fd, &buf->max_vcpu_id,sizeof (buf->max_vcpu_id)) || + buf->max_vcpu_id >=64 ||read_exact (fd, &buf->vcpumap, +sizeof (uint64_t )) ) { +ERROR ("Error when reading max_vcpu_id" ); +return -1 ; + } +// DPRINTF("Max VCPU ID: %d, vcpumap: %llx\n", buf->max_vcpu_id, buf->vcpumap); +return pagebuf_get_one (buf, fd, xch, dom); + }else if (count ==-3 ) { +/* Skip padding 4 bytes then read the EPT identity PT location. */ +if (read_exact (fd, &buf->identpt,sizeof (uint32_t )) || +read_exact (fd, &buf->identpt,sizeof (uint64_t )) ) + { +ERROR ("error read the address of the EPT identity map" ); +return -1 ; + } +// DPRINTF("EPT identity map address: %llx\n", buf->identpt); +return pagebuf_get_one (buf, fd, xch, dom); + }else if ( count ==-4 ) { +/* Skip padding 4 bytes then read the vm86 TSS location. */ +if (read_exact (fd, &buf->vm86_tss,sizeof (uint32_t )) || +read_exact (fd, &buf->vm86_tss,sizeof (uint64_t )) ) + { +ERROR ("error read the address of the vm86 TSS" ); +return -1 ; + } +// DPRINTF("VM86 TSS location: %llx\n", buf->vm86_tss); +return pagebuf_get_one (buf, fd, xch, dom); + }else if ( count ==-5 ) { +DPRINTF ("xc_domain_restore start tmem\n" ); +if (xc_tmem_restore (xch, dom, fd) ) { +ERROR ("error reading/restoring tmem" ); +return -1 ; + } +return pagebuf_get_one (buf, fd, xch, dom); + } +else if ( count ==-6 ) { +if (xc_tmem_restore_extra (xch, dom, fd) ) { +ERROR ("error reading/restoring tmem extra" ); +return -1 ; + } +return pagebuf_get_one (buf, fd, xch, dom); + }else if ( (count >MAX_BATCH_SIZE ) || (count <0 ) ) { +ERROR ("Max batch size exceeded (%d). Giving up." , count); +return -1 ; + } + + oldcount = buf->nr_pages; + buf->nr_pages += count; +if (!buf->pfn_types) { +if (!(buf->pfn_types =malloc (buf->nr_pages *sizeof (*(buf->pfn_types))))) { +ERROR ("Could not allocate PFN type buffer" ); +return -1 ; + } + }else { +if (!(ptmp =realloc (buf->pfn_types, buf->nr_pages *sizeof (*(buf->pfn_types))))) { +ERROR ("Could not reallocate PFN type buffer" ); +return -1 ; + } + buf->pfn_types = ptmp; + } +if (read_exact (fd, buf->pfn_types + oldcount, count *sizeof (*(buf->pfn_types)))) { +ERROR ("Error when reading region pfn types" ); +return -1 ; + } + + countpages = count; +for (i = oldcount; i < buf->nr_pages; ++i) +if ((buf->pfn_types[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ) ==XEN_DOMCTL_PFINFO_XTAB ) + --countpages; + +if (!countpages) +return count; + + oldcount = buf->nr_physpages; + buf->nr_physpages += countpages; +if (!buf->pages) { +if (!(buf->pages =malloc (buf->nr_physpages *PAGE_SIZE ))) { +ERROR ("Could not allocate page buffer" ); +return -1 ; + } + }else { +if (!(ptmp =realloc (buf->pages, buf->nr_physpages *PAGE_SIZE ))) { +ERROR ("Could not reallocate page buffer" ); +return -1 ; + } + buf->pages = ptmp; + } +if (read_exact (fd, buf->pages + oldcount *PAGE_SIZE , countpages *PAGE_SIZE ) ) { +ERROR ("Error when reading pages" ); +return -1 ; + } + +return count; +} + +static int pagebuf_get (pagebuf_t * buf,int fd,int xch,uint32_t dom) +{ +int rc; + + buf->nr_physpages = buf->nr_pages =0 ; + +do { + rc =pagebuf_get_one (buf, fd, xch, dom); + }while (rc >0 ); + +if (rc <0 ) +pagebuf_free (buf); + +return rc; +} + +static int apply_batch (int xc_handle,uint32_t dom,xen_pfn_t * region_mfn, +unsigned long * pfn_type,int pae_extended_cr3, +unsigned int hvm,struct xc_mmu * mmu, +pagebuf_t * pagebuf,int curbatch,int superpages) +{ +int i, j, curpage; +/* used by debug verify code */ +unsigned long buf[PAGE_SIZE /sizeof (unsigned long )]; +/* Our mapping of the current region (batch) */ +char *region_base; +/* A temporary mapping, and a copy, of one frame of guest memory. */ +unsigned long *page =NULL ; +int nraces =0 ; + +unsigned long mfn, pfn, pagetype; + + j = pagebuf->nr_pages - curbatch; +if (j >MAX_BATCH_SIZE ) + j =MAX_BATCH_SIZE ; + +if (allocate_physmem (xc_handle, dom, &pagebuf->pfn_types[curbatch], + j, hvm, region_mfn, superpages) !=0 ) + { +ERROR ("allocate_physmem() failed\n" ); +return -1 ; + } + +/* Map relevant mfns */ + region_base =xc_map_foreign_batch ( + xc_handle, dom,PROT_WRITE , region_mfn, j); + +if ( region_base ==NULL ) + { +ERROR ("map batch failed" ); +return -1 ; + } + +for ( i =0 , curpage =-1 ; i < j; i++ ) + { + pfn = pagebuf->pfn_types[i + curbatch] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; + pagetype = pagebuf->pfn_types[i + curbatch] &XEN_DOMCTL_PFINFO_LTAB_MASK ; + +if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) +/* a bogus/unmapped page: skip it */ +continue ; + + ++curpage; + +if ( pfn > p2m_size ) + { +ERROR ("pfn out of range" ); +return -1 ; + } + + pfn_type[pfn] = pagetype; + + mfn = p2m[pfn]; + +/* In verify mode, we use a copy; otherwise we work in place */ + page = pagebuf->verify ? (void *)buf : (region_base + i*PAGE_SIZE ); + +memcpy (page, pagebuf->pages + (curpage + curbatch) *PAGE_SIZE ,PAGE_SIZE ); + + pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; + +if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && + (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) + { +/* +** A page table page - need to 'uncanonicalize' it, i.e. +** replace all the references to pfns with the corresponding +** mfns for the new domain. +** +** On PAE we need to ensure that PGDs are in MFNs < 4G, and +** so we may need to update the p2m after the main loop. +** Hence we defer canonicalization of L1s until then. +*/ +if ((pt_levels !=3 ) || + pae_extended_cr3 || + (pagetype !=XEN_DOMCTL_PFINFO_L1TAB )) { + +if (!uncanonicalize_pagetable (xc_handle, dom, + pagetype, page, superpages)) { +/* +** Failing to uncanonicalize a page table can be ok +** under live migration since the pages type may have +** changed by now (and we'll get an update later). +*/ +DPRINTF ("PT L%ld race on pfn=%08lx mfn=%08lx\n" , + pagetype >>28 , pfn, mfn); + nraces++; +continue ; + } + } + } +else if ( pagetype !=XEN_DOMCTL_PFINFO_NOTAB ) + { +ERROR ("Bogus page type %lx page table is out of range: " +"i=%d p2m_size=%lu" , pagetype, i, p2m_size); +return -1 ; + } + +if ( pagebuf->verify ) + { +int res =memcmp (buf, (region_base + i*PAGE_SIZE ),PAGE_SIZE ); +if ( res ) + { +int v; + +DPRINTF ("************** pfn=%lx type=%lx gotcs=%08lx " +"actualcs=%08lx\n" , pfn, pagebuf->pfn_types[pfn], +csum_page (region_base + (i + curbatch)*PAGE_SIZE ), +csum_page (buf)); + +for ( v =0 ; v <4 ; v++ ) + { +unsigned long *p = (unsigned long *) + (region_base + i*PAGE_SIZE ); +if ( buf[v] != p[v] ) +DPRINTF (" %d: %08lx %08lx\n" , v, buf[v], p[v]); + } + } + } + +if ( !hvm && +xc_add_mmu_update (xc_handle, mmu, + (((unsigned long long )mfn) <<PAGE_SHIFT ) + |MMU_MACHPHYS_UPDATE , pfn) ) + { +ERROR ("failed machpys update mfn=%lx pfn=%lx" , mfn, pfn); +return -1 ; + } + }/* end of 'batch' for loop */ + +munmap (region_base, j*PAGE_SIZE ); + +return nraces; +} + int xc_domain_restore(int xc_handle, int io_fd, uint32_t dom, unsigned int store_evtchn, unsigned long *store_mfn, unsigned int console_evtchn, unsigned long *console_mfn,
@@ -633,7 +1037,6 @@int rc =1 , frc, i, j, n, m, pae_extended_cr3 =0 , ext_vcpucontext =0 ;unsigned long mfn, pfn;unsigned int prev_pc, this_pc; -int verify =0 ;int nraces =0 ;/* The new domain's shared-info frame number. */
@@ -652,9 +1055,6 @@/* A table of MFNs to map in the current region */ xen_pfn_t *region_mfn =NULL ; -/* Types of the pfns in the current region */ -unsigned long region_pfn_type[MAX_BATCH_SIZE ]; -/* A copy of the pfn-to-mfn table frame list. */ xen_pfn_t *p2m_frame_list =NULL ;
@@ -666,9 +1066,6 @@struct xc_mmu *mmu =NULL ; -/* used by debug verify code */ -unsigned long buf[PAGE_SIZE /sizeof (unsigned long )]; -struct mmuext_op pin[MAX_PIN_BATCH ];unsigned int nr_pins;
@@ -682,6 +1079,14 @@/* Buffer for holding HVM context */ uint8_t *hvm_buf =NULL ; +int completed =0 ; +pagebuf_t pagebuf; +tailbuf_t tailbuf, tmptail; +void * vcpup; + +pagebuf_init (&pagebuf); +memset (&tailbuf,0 ,sizeof (tailbuf)); +/* For info only */ nr_pfns =0 ;
@@ -784,9 +1189,10 @@ prev_pc =0 ; n = m =0 ; + loadpages:for ( ; ; ) { -int j; +int j, curbatch; this_pc = (n *100 ) / p2m_size;if ( (this_pc - prev_pc) >=5 )
@@ -795,221 +1201,49 @@ prev_pc = this_pc; } -if (read_exact (io_fd, &j,sizeof (int )) ) - { -ERROR ("Error when reading batch size" ); - goto out; +if ( !completed ) { + pagebuf.nr_physpages = pagebuf.nr_pages =0 ; +if (pagebuf_get_one (&pagebuf, io_fd, xc_handle, dom) <0 ) { +ERROR ("Error when reading batch\n" ); + goto out; + } } + j = pagebuf.nr_pages;PPRINTF ("batch %d\n" ,j); -if ( j ==-1 ) - { - verify =1 ; -DPRINTF ("Entering page verify mode\n" ); -continue ; +if ( j ==0 ) { +/* catch vcpu updates */ +if (pagebuf.new_ctxt_format) { + vcpumap = pagebuf.vcpumap; + max_vcpu_id = pagebuf.max_vcpu_id; + } +/* should this be deferred? does it change? */ +if ( pagebuf.identpt ) +xc_set_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , pagebuf.identpt); +if ( pagebuf.vm86_tss ) +xc_set_hvm_param (xc_handle, dom,HVM_PARAM_VM86_TSS , pagebuf.vm86_tss); +break ;/* our work here is done */ } -if ( j ==-2 ) - { - new_ctxt_format =1 ; -if (read_exact (io_fd, &max_vcpu_id,sizeof (int )) || - (max_vcpu_id >=64 ) || -read_exact (io_fd, &vcpumap,sizeof (uint64_t )) ) - { -ERROR ("Error when reading max_vcpu_id" ); +/* break pagebuf into batches */ + curbatch =0 ; +while ( curbatch < j ) { +int brc; + + brc =apply_batch (xc_handle, dom, region_mfn, pfn_type, + pae_extended_cr3, hvm, mmu, &pagebuf, curbatch, superpages); +if ( brc <0 ) goto out; - } -continue ; + + nraces += brc; + + curbatch +=MAX_BATCH_SIZE ; } -if ( j ==-3 ) - { -uint64_t ident_pt; + pagebuf.nr_physpages = pagebuf.nr_pages =0 ; -/* Skip padding 4 bytes then read the EPT identity PT location. */ -if (read_exact (io_fd, &ident_pt,sizeof (uint32_t )) || -read_exact (io_fd, &ident_pt,sizeof (uint64_t )) ) - { -ERROR ("error read the address of the EPT identity map" ); - goto out; - } - -xc_set_hvm_param (xc_handle, dom,HVM_PARAM_IDENT_PT , ident_pt); -continue ; - } - -if ( j ==-4 ) - { -uint64_t vm86_tss; - -/* Skip padding 4 bytes then read the vm86 TSS location. */ -if (read_exact (io_fd, &vm86_tss,sizeof (uint32_t )) || -read_exact (io_fd, &vm86_tss,sizeof (uint64_t )) ) - { -ERROR ("error read the address of the vm86 TSS" ); - goto out; - } - -xc_set_hvm_param (xc_handle, dom,HVM_PARAM_VM86_TSS , vm86_tss); -continue ; - } - -if ( j ==-5 ) - { -DPRINTF ("xc_domain_restore start tmem\n" ); -if (xc_tmem_restore (xc_handle, dom, io_fd) ) - { -ERROR ("error reading/restoring tmem" ); - goto out; - } -continue ; - } - -if ( j ==-6 ) - { -if (xc_tmem_restore_extra (xc_handle, dom, io_fd) ) - { -ERROR ("error reading/restoring tmem extra" ); - goto out; - } -continue ; - } - -if ( j ==0 ) -break ;/* our work here is done */ - -if ( (j >MAX_BATCH_SIZE ) || (j <0 ) ) - { -ERROR ("Max batch size exceeded. Giving up." ); - goto out; - } - -if (read_exact (io_fd, region_pfn_type, j*sizeof (unsigned long )) ) - { -ERROR ("Error when reading region pfn types" ); - goto out; - } - -if (allocate_physmem (xc_handle, dom, region_pfn_type, - j, hvm, region_mfn, superpages) !=0 ) - goto out; - -/* Map relevant mfns */ - region_base =xc_map_foreign_batch ( - xc_handle, dom,PROT_WRITE , region_mfn, j); - -if ( region_base ==NULL ) - { -ERROR ("map batch failed" ); - goto out; - } - -for ( i =0 ; i < j; i++ ) - { -void *page; -unsigned long pagetype; - - pfn = region_pfn_type[i] & ~XEN_DOMCTL_PFINFO_LTAB_MASK ; - pagetype = region_pfn_type[i] &XEN_DOMCTL_PFINFO_LTAB_MASK ; - -if ( pagetype ==XEN_DOMCTL_PFINFO_XTAB ) -/* a bogus/unmapped page: skip it */ -continue ; - -if ( pfn > p2m_size ) - { -ERROR ("pfn out of range" ); - goto out; - } - - pfn_type[pfn] = pagetype; - - mfn = p2m[pfn]; - -/* In verify mode, we use a copy; otherwise we work in place */ - page = verify ? (void *)buf : (region_base + i*PAGE_SIZE ); - -if (read_exact (io_fd, page,PAGE_SIZE ) ) - { -ERROR ("Error when reading page (type was %lx)" , pagetype); - goto out; - } - - pagetype &=XEN_DOMCTL_PFINFO_LTABTYPE_MASK ; - -if ( (pagetype >=XEN_DOMCTL_PFINFO_L1TAB ) && - (pagetype <=XEN_DOMCTL_PFINFO_L4TAB ) ) - { -/* -** A page table page - need to 'uncanonicalize' it, i.e. -** replace all the references to pfns with the corresponding -** mfns for the new domain. -** -** On PAE we need to ensure that PGDs are in MFNs < 4G, and -** so we may need to update the p2m after the main loop. -** Hence we defer canonicalization of L1s until then. -*/ -if ((pt_levels !=3 ) || - pae_extended_cr3 || - (pagetype !=XEN_DOMCTL_PFINFO_L1TAB )) { - -if (!uncanonicalize_pagetable (xc_handle, dom, - pagetype, page, superpages)) { -/* -** Failing to uncanonicalize a page table can be ok -** under live migration since the pages type may have -** changed by now (and we'll get an update later). -*/ -DPRINTF ("PT L%ld race on pfn=%08lx mfn=%08lx\n" , - pagetype >>28 , pfn, mfn); - nraces++; -continue ; - } - } - } -else if ( pagetype !=XEN_DOMCTL_PFINFO_NOTAB ) - { -ERROR ("Bogus page type %lx page table is out of range: " -"i=%d p2m_size=%lu" , pagetype, i, p2m_size); - goto out; - - } - -if ( verify ) - { -int res =memcmp (buf, (region_base + i*PAGE_SIZE ),PAGE_SIZE ); -if ( res ) - { -int v; - -DPRINTF ("************** pfn=%lx type=%lx gotcs=%08lx " -"actualcs=%08lx\n" , pfn, pfn_type[pfn], -csum_page (region_base + i*PAGE_SIZE ), -csum_page (buf)); - -for ( v =0 ; v <4 ; v++ ) - { -unsigned long *p = (unsigned long *) - (region_base + i*PAGE_SIZE ); -if ( buf[v] != p[v] ) -DPRINTF (" %d: %08lx %08lx\n" , v, buf[v], p[v]); - } - } - } - -if ( !hvm && -xc_add_mmu_update (xc_handle, mmu, - (((unsigned long long )mfn) <<PAGE_SHIFT ) - |MMU_MACHPHYS_UPDATE , pfn) ) - { -ERROR ("failed machpys update mfn=%lx pfn=%lx" , mfn, pfn); - goto out; - } - }/* end of 'batch' for loop */ - -munmap (region_base, j*PAGE_SIZE ); - n+= j;/* crude stats */ + n += j;/* crude stats */ /* * Discard cache for portion of file read so far up to last
@@ -1033,7 +1267,7 @@ goto out; } -DPRINTF ("Received all pages (%d races)\n" , nraces); +// DPRINTF("Received all pages (%d races)\n", nraces); if ( hvm ) {
@@ -1107,6 +1341,34 @@/* Non-HVM guests only from here on */ +if ( !completed ) { +if (buffer_tail (&tailbuf, io_fd, max_vcpu_id, vcpumap, + ext_vcpucontext) <0 ) { +ERROR ("error buffering image tail" ); + goto out; + } + completed =1 ; + } + +// DPRINTF("Buffered checkpoint\n"); + +if (pagebuf_get (&pagebuf, io_fd, xc_handle, dom) ) { +ERROR ("error when buffering batch, finishing\n" ); + goto finish; + } +memset (&tmptail,0 ,sizeof (tmptail)); +if (buffer_tail (&tmptail, io_fd, max_vcpu_id, vcpumap, + ext_vcpucontext) <0 ) { +ERROR ("error buffering image tail, finishing" ); + goto finish; + } +tailbuf_free (&tailbuf); +memcpy (&tailbuf, &tmptail,sizeof (tailbuf)); + + goto loadpages; + + finish: +if ( (pt_levels ==3 ) && !pae_extended_cr3 ) {/*
@@ -1275,39 +1537,17 @@/* Get the list of PFNs that are not in the psuedo-phys map */ { -unsigned int count =0 ; -unsigned long *pfntab; -int nr_frees; +int nr_frees =0 ; -if (read_exact (io_fd, &count,sizeof (count)) || - (count > (1U <<28 )) )/* up to 1TB of address space */ +for ( i =0 ; i < tailbuf.pfncount; i++ ) { -ERROR ("Error when reading pfn count (= %u)" , count); - goto out; - } - -if ( !(pfntab =malloc (sizeof (unsigned long ) * count)) ) - { -ERROR ("Out of memory" ); - goto out; - } - -if (read_exact (io_fd, pfntab,sizeof (unsigned long )*count) ) - { -ERROR ("Error when reading pfntab" ); - goto out; - } - - nr_frees =0 ; -for ( i =0 ; i < count; i++ ) - { -unsigned long pfn = pfntab[i]; +unsigned long pfn = tailbuf.pfntab[i];if ( p2m[pfn] !=INVALID_P2M_ENTRY ) {/* pfn is not in physmap now, but was at some point during the save/migration process - need to free it */ - pfntab[nr_frees++] = p2m[pfn]; + tailbuf.pfntab[nr_frees++] = p2m[pfn]; p2m[pfn] =INVALID_P2M_ENTRY ;/* not in pseudo-physical map */ } }
@@ -1319,7 +1559,7 @@ .extent_order = 0, .domid = dom }; -set_xen_guest_handle (reservation.extent_start, pfntab); +set_xen_guest_handle (reservation.extent_start, tailbuf.pfntab);if ( (frc =xc_memory_op (xc_handle, XENMEM_decrease_reservation, &reservation)) != nr_frees )
@@ -1328,7 +1568,7 @@ goto out; }else -DPRINTF ("Decreased reservation by %d pages\n" , count); +DPRINTF ("Decreased reservation by %d pages\n" , tailbuf.pfncount); } }
@@ -1338,18 +1578,17 @@return 1 ; } + vcpup = tailbuf.vcpubuf;for ( i =0 ; i <= max_vcpu_id; i++ ) {if ( !(vcpumap & (1ULL << i)) )continue ; -if (read_exact (io_fd, &ctxt, ((guest_width ==8 ) - ?sizeof (ctxt.x64) - :sizeof (ctxt.x32))) ) - { -ERROR ("Error when reading ctxt %d" , i); - goto out; - } +memcpy (&ctxt, vcpup, ((guest_width ==8 ) ?sizeof (ctxt.x64) + :sizeof (ctxt.x32))); + vcpup += (guest_width ==8 ) ?sizeof (ctxt.x64) :sizeof (ctxt.x32); + +DPRINTF ("read VCPU %d\n" , i);if ( !new_ctxt_format )SET_FIELD (&ctxt, flags,GET_FIELD (&ctxt, flags) | VGCF_online);
@@ -1454,12 +1693,8 @@if ( !ext_vcpucontext )continue ; -if (read_exact (io_fd, &domctl.u.ext_vcpucontext,128 ) || - (domctl.u.ext_vcpucontext.vcpu != i) ) - { -ERROR ("Error when reading extended ctxt %d" , i); - goto out; - } +memcpy (&domctl.u.ext_vcpucontext, vcpup,128 ); + vcpup +=128 ; domctl.cmd = XEN_DOMCTL_set_ext_vcpucontext; domctl.domain = dom; frc =xc_domctl (xc_handle, &domctl);
@@ -1470,11 +1705,9 @@ } } -if (read_exact (io_fd, shared_info_page,PAGE_SIZE ) ) - { -ERROR ("Error when reading shared info page" ); - goto out; - } +memcpy (shared_info_page, tailbuf.shared_info_page,PAGE_SIZE ); + +DPRINTF ("Completed checkpoint load\n" );/* Restore contents of shared-info page. No checking needed. */ new_shared_info = xc_map_foreign_range(
tools/python/xen/xend/XendCheckpoint.py +1 -1
@@ -345,7 +345,7 @@ restore_image.setCpuid() - os.read(fd, 1) # Wait for source to close connection + # xc_restore will wait for source to close connection dominfo.completeRestore(handler.store_mfn, handler.console_mfn)
PATCH 1/1 Initiate failover if a packet is not received every 500ms
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID 7a96f23c0e7d1bb142bace2d348d80043bfa518b
# Parent a82f2e189c29acfc26794ea10d4000d0a350bf40 Initiate failover if a packet is not received every 500ms. This breaks checkpoints at lower frequencies, and should be made optional.
tools/libxc/xc_domain_restore.c +51 -1
@@ -453,6 +453,51 @@ } +/* set when a consistent image is available */ +static int completed =0 ; + +#define HEARTBEAT_MS 500 + +#ifndef __MINIOS__ +static ssize_t read_exact_timed (int fd,void * buf,size_t size) +{ +size_t offset =0 ; +ssize_t len; +struct timeval tv; +fd_set rfds; + +while ( offset < size ) + { +if ( completed ) { +/* expect a heartbeat every HEARBEAT_MS ms maximum */ + tv.tv_sec =0 ; + tv.tv_usec =HEARTBEAT_MS *1000 ; + +FD_ZERO (&rfds); +FD_SET (fd, &rfds); + len =select (fd +1 , &rfds,NULL ,NULL , &tv); +if ( !FD_ISSET (fd, &rfds) ) { +fprintf (stderr,"read_exact_timed failed (select returned %zd)\n" , len); +return -1 ; + } + } + + len =read (fd, buf + offset, size - offset); +if ( (len ==-1 ) && ((errno ==EINTR ) || (errno ==EAGAIN )) ) +continue ; +if ( len <=0 ) +return -1 ; + offset += len; + } + +return 0 ; +} + +#define read_exact read_exact_timed + +#else +#define read_exact_timed read_exact +#endif /* ** In the state file (or during transfer), all page-table pages are ** converted into a 'canonical' form where references to actual mfns
@@ -1079,7 +1124,6 @@/* Buffer for holding HVM context */ uint8_t *hvm_buf =NULL ; -int completed =0 ;pagebuf_t pagebuf;tailbuf_t tailbuf, tmptail;void * vcpup;
@@ -1342,12 +1386,18 @@/* Non-HVM guests only from here on */ if ( !completed ) { +int flags =0 ; +if (buffer_tail (&tailbuf, io_fd, max_vcpu_id, vcpumap, ext_vcpucontext) <0 ) {ERROR ("error buffering image tail" ); goto out; } completed =1 ; +/* shift into nonblocking mode for the remainder */ +if ( (flags =fcntl (io_fd,F_GETFL ,0 )) <0 ) + flags =0 ; +fcntl (io_fd,F_SETFL , flags |O_NONBLOCK ); }// DPRINTF("Buffered checkpoint\n");
PATCH 1/1 Buffer checkpoint data locally until domain has resumed execution
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355510 25200
# Node ID 4771b6cb3a82ae2b7da86388a9cfb472858cd90e
# Parent 7a96f23c0e7d1bb142bace2d348d80043bfa518b Buffer checkpoint data locally until domain has resumed execution.
tools/libxc/xc_domain_save.c +112 -1
@@ -52,6 +52,15 @@/* Address size of the guest */ unsigned int guest_width; +/* buffer for output */ +struct outbuf { +void * buf; +size_t size; +size_t pos; +}; + +#define OUTBUF_SIZE (16384 * 1024) +/* grep fodder: machine_to_phys */ #define mfn_to_pfn (_mfn) (live_m2p[(_mfn)])
@@ -154,6 +163,85 @@return rc; } +static int outbuf_init (struct outbuf * ob,size_t size) +{ +memset (ob,0 ,sizeof (*ob)); + +if ( !(ob->buf =malloc (size)) ) { +DPRINTF ("error allocating output buffer of size %zu\n" , size); +return -1 ; + } + + ob->size = size; + +return 0 ; +} + +static inline int outbuf_write (struct outbuf * ob,void * buf,size_t len) +{ +if ( len > ob->size - ob->pos ) { +DPRINTF ("outbuf_write: %zu > %zu@%zu\n" , len, ob->size - ob->pos, ob->pos); +return -1 ; + } + +memcpy (ob->buf + ob->pos, buf, len); + ob->pos += len; + +return 0 ; +} + +/* prep for nonblocking I/O */ +static int outbuf_flush (struct outbuf * ob,int fd) +{ +int rc; +int cur =0 ; + +if ( !ob->pos ) +return 0 ; + + rc =write (fd, ob->buf, ob->pos); +while (rc <0 || cur + rc < ob->pos) { +if (rc <0 && errno !=EAGAIN && errno !=EINTR ) { +DPRINTF ("error flushing output: %d\n" , errno); +return -1 ; + } +if (rc >0 ) + cur += rc; + + rc =write (fd, ob->buf + cur, ob->pos - cur); + } + + ob->pos =0 ; + +return 0 ; +} + +/* if there's no room in the buffer, flush it and try again. */ +static inline int outbuf_hardwrite (struct outbuf * ob,int fd,void * buf, +size_t len) +{ +if ( !len ) +return 0 ; + +if ( !outbuf_write (ob, buf, len) ) +return 0 ; + +if (outbuf_flush (ob, fd) <0 ) +return -1 ; + +return outbuf_write (ob, buf, len); +} + +/* start buffering output once we've reached checkpoint mode. */ +static inline int write_buffer (int dobuf,struct outbuf * ob,int fd,void * buf, +size_t len) +{ +if ( dobuf ) +return outbuf_hardwrite (ob, fd, buf, len); +else +return write_exact (fd, buf, len); +} +#ifdef ADAPTIVE_SAVE /*
@@ -246,6 +334,16 @@#endif +/* like write_buffer for ratewrite, which returns number of bytes written */ +static inline int ratewrite_buffer (int dobuf,struct outbuf * ob,int fd, +int live,void * buf,size_t len) +{ +if ( dobuf ) +return outbuf_hardwrite (ob, fd, buf, len) ?-1 : len; +else +return ratewrite (fd, live, buf, len); +} +static int print_stats (int xc_handle,uint32_t domid,int pages_sent,xc_shadow_op_stats_t *stats,int print) {
@@ -796,6 +894,10 @@unsigned long mfn; +struct outbuf ob; + +outbuf_init (&ob,OUTBUF_SIZE ); +/* If no explicit control parameters given, use defaults */ max_iters = max_iters ? :DEF_MAX_ITERS ; max_factor = max_factor ? :DEF_MAX_FACTOR ;
@@ -995,6 +1097,12 @@ } copypages: +#define write_exact (fd, buf, len) write_buffer(last_iter, &ob, (fd), (buf), (len)) +#ifdef ratewrite +#undef ratewrite +#endif +#define ratewrite (fd, live, buf, len) ratewrite_buffer(last_iter, &ob, (fd), (live), (buf), (len)) +/* Now write out each data page, canonicalising page tables as we go... */ for ( ; ; ) {
@@ -1594,7 +1702,10 @@ callbacks->postcopy (callbacks->data);/* Flush last write and discard cache for file. */ -discard_file_cache (io_fd,1 /* flush */ ); +if (outbuf_flush (&ob, io_fd) <0 ) { +ERROR ("Error when flushing output buffer\n" ); + rc =1 ; + }/* checkpoint_cb can spend arbitrarily long in between rounds */ if (!rc && callbacks->checkpoint &&
PATCH 1/1 Do not bother with to_skip/to_fix bitmaps after the first final round
In reply to Brendan Cully# HG changeset patch
# User Geoffrey Lefebvre <[email protected]>
# Date 1240355511 25200
# Node ID d6c00f82239b2527cbdb8cc0140f67d7b168ae20
# Parent 4771b6cb3a82ae2b7da86388a9cfb472858cd90e
Do not bother with to_skip/to_fix bitmaps after the first final round.
tools/libxc/xc_domain_save.c +63 -42
@@ -896,6 +896,8 @@struct outbuf ob; +int completed =0 ; +outbuf_init (&ob,OUTBUF_SIZE );/* If no explicit control parameters given, use defaults */
@@ -1159,54 +1161,69 @@ mfn_to_pfn(pfn_to_mfn(n)&0xFFFFF));DPRINTF ("\n" ); } -if ( !last_iter && -test_bit (n, to_send) && -test_bit (n, to_skip) ) - skip_this_iter++;/* stats keeping */ -if ( !((test_bit (n, to_send) && !test_bit (n, to_skip)) || - (test_bit (n, to_send) && last_iter) || - (test_bit (n, to_fix) && last_iter)) ) -continue ; +if ( completed ) + { +if ( !test_bit (n, to_send) ) +continue ; -/* -** we get here if: -** 1. page is marked to_send & hasn't already been re-dirtied -** 2. (ignore to_skip in last iteration) -** 3. add in pages that still need fixup (net bufs) -*/ + pfn_batch[batch] = n; +if ( hvm ) + pfn_type[batch] = n; +else + pfn_type[batch] =pfn_to_mfn (n); + } +else + { +if ( !last_iter && +test_bit (n, to_send) && +test_bit (n, to_skip) ) + skip_this_iter++;/* stats keeping */ - pfn_batch[batch] = n; +if ( !((test_bit (n, to_send) && !test_bit (n, to_skip)) || + (test_bit (n, to_send) && last_iter) || + (test_bit (n, to_fix) && last_iter)) ) +continue ; -/* Hypercall interfaces operate in PFNs for HVM guests -* and MFNs for PV guests */ -if ( hvm ) - pfn_type[batch] = n; -else - pfn_type[batch] =pfn_to_mfn (n); +/* +** we get here if: +** 1. page is marked to_send & hasn't already been re-dirtied +** 2. (ignore to_skip in last iteration) +** 3. add in pages that still need fixup (net bufs) +*/ + + pfn_batch[batch] = n; + +/* Hypercall interfaces operate in PFNs for HVM guests +* and MFNs for PV guests */ +if ( hvm ) + pfn_type[batch] = n; +else + pfn_type[batch] =pfn_to_mfn (n); -if ( !is_mapped (pfn_type[batch]) ) - { -/* -** not currently in psuedo-physical map -- set bit -** in to_fix since we must send this page in last_iter -** unless its sent sooner anyhow, or it never enters -** pseudo-physical map (e.g. for ballooned down doms) -*/ -set_bit (n, to_fix); -continue ; +if ( !is_mapped (pfn_type[batch]) ) + { +/* +** not currently in psuedo-physical map -- set bit +** in to_fix since we must send this page in last_iter +** unless its sent sooner anyhow, or it never enters +** pseudo-physical map (e.g. for ballooned down doms) +*/ +set_bit (n, to_fix); +continue ; + } + +if ( last_iter && +test_bit (n, to_fix) && + !test_bit (n, to_send) ) + { + needed_to_fix++; +DPRINTF ("Fix! iter %d, pfn %x. mfn %lx\n" , + iter, n, pfn_type[batch]); + } + +clear_bit (n, to_fix); } - -if ( last_iter && -test_bit (n, to_fix) && - !test_bit (n, to_send) ) - { - needed_to_fix++; -DPRINTF ("Fix! iter %d, pfn %x. mfn %lx\n" , - iter, n, pfn_type[batch]); - } - -clear_bit (n, to_fix); batch++; }
@@ -1698,6 +1715,8 @@ rc =0 ; out: + completed =1 ; +if ( !rc && callbacks->postcopy ) callbacks->postcopy (callbacks->data);
@@ -1707,6 +1726,8 @@ rc =1 ; } +discard_file_cache (io_fd,1 /* flush */ ); +/* checkpoint_cb can spend arbitrarily long in between rounds */ if (!rc && callbacks->checkpoint && callbacks->checkpoint (callbacks->data) >0 )
PATCH 1/1 Do bitmap scan word-by-word before bit-by-bit
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1240355511 25200
# Node ID ae725eba611ff93dd9a09cfa995f61b6db28cf86
# Parent d6c00f82239b2527cbdb8cc0140f67d7b168ae20 Do bitmap scan word-by-word before bit-by-bit.
For sparse bitmaps and large domains this saves a lot of time.
tools/libxc/xc_domain_save.c +10 -0
@@ -94,6 +94,8 @@#define BITMAP_SHIFT (_nr) ((_nr) % BITS_PER_LONG) +#define ORDER_LONG (sizeof(unsigned long) == 4 ? 5 : 6) +static inline int test_bit (int nr,volatile void * addr) {return (BITMAP_ENTRY (nr, addr) >>BITMAP_SHIFT (nr)) &1 ;
@@ -1164,6 +1166,14 @@if ( completed ) { +/* for sparse bitmaps, word-by-word may save time */ +if ( !to_send[N >>ORDER_LONG ] ) + { +/* incremented again in for loop! */ +N +=BITS_PER_LONG -1 ; +continue ; + } +if ( !test_bit (n, to_send) )continue ;
PATCH 1/1 Make checkpoint buffering HVM-aware
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1253640427 25200
# Node ID 6ca67fe3514ada809a62282c30fe46d5df5ce265 # Parent ae725eba611ff93dd9a09cfa995f61b6db28cf86 Make checkpoint buffering HVM-aware
tools/libxc/xc_domain_restore.c +287 -96
@@ -670,15 +670,204 @@ }typedef struct { -unsigned int pfncount; -unsigned long * pfntab; -unsigned int vcpucount; -unsigned char * vcpubuf; -unsigned char shared_info_page[PAGE_SIZE ]; +int ishvm; +union { +struct tailbuf_pv { +unsigned int pfncount; +unsigned long * pfntab; +unsigned int vcpucount; +unsigned char * vcpubuf; +unsigned char shared_info_page[PAGE_SIZE ]; + } pv; +struct tailbuf_hvm { +uint64_t magicpfns[3 ]; +uint32_t hvmbufsize, reclen; +uint8_t * hvmbuf; +struct { +uint32_t magic; +uint32_t version; +uint64_t len; + } qemuhdr; +uint32_t qemubufsize; +uint8_t * qemubuf; + } hvm; + } u; }tailbuf_t ; -static int buffer_tail (tailbuf_t * buf,int fd,unsigned int max_vcpu_id, -uint64_t vcpumap,int ext_vcpucontext) +/* read stream until EOF, growing buffer as necssary */ +static int compat_buffer_qemu (int fd,struct tailbuf_hvm *buf) +{ +uint8_t *qbuf, *tmp; +int blen =0 , dlen =0 ; +int rc; + +/* currently save records tend to be about 7K */ + blen =8192 ; +if ( !(qbuf =malloc (blen)) ) { +ERROR ("Error allocating QEMU buffer" ); +return -1 ; + } + +while ( (rc =read (fd, qbuf+dlen, blen-dlen)) >0 ) { +DPRINTF ("Read %d bytes of QEMU data\n" , rc); + dlen += rc; + +if (dlen == blen) { +DPRINTF ("%d-byte QEMU buffer full, reallocating...\n" , dlen); + blen +=4096 ; + tmp =realloc (qbuf, blen); +if ( !tmp ) { +ERROR ("Error growing QEMU buffer to %d bytes" , blen); +free (qbuf); +return -1 ; + } + qbuf = tmp; + } + } + +if ( rc <0 ) { +ERROR ("Error reading QEMU data" ); +free (qbuf); +return -1 ; + } + +if (memcmp (qbuf,"QEVM" ,4 ) ) { +ERROR ("Invalid QEMU magic: 0x%08x" , *(unsigned long *)qbuf); +free (qbuf); +return -1 ; + } + + buf->qemubuf = qbuf; + buf->qemubufsize = dlen; + +return 0 ; +} + +static int buffer_qemu (int fd,struct tailbuf_hvm *buf) +{ +uint32_t qlen; +uint8_t *tmp; + +if (read_exact (fd, &qlen,sizeof (qlen)) ) { +ERROR ("Error reading QEMU header length" ); +return -1 ; + } + +if ( qlen > buf->qemubufsize ) { +if ( buf->qemubuf) { + tmp =realloc (buf->qemubuf, qlen); +if ( tmp ) + buf->qemubuf = tmp; +else { +ERROR ("Error reallocating QEMU state buffer" ); +return -1 ; + } + }else { + buf->qemubuf =malloc (qlen); +if ( !buf->qemubuf ) { +ERROR ("Error allocating QEMU state buffer" ); +return -1 ; + } + } + } + buf->qemubufsize = qlen; + +if (read_exact (fd, buf->qemubuf, buf->qemubufsize) ) { +ERROR ("Error reading QEMU state" ); +return -1 ; + } + +return 0 ; +} + +static int dump_qemu (uint32_t dom,struct tailbuf_hvm *buf) +{ +int saved_errno; +char path[256 ]; +FILE *fp; + +sprintf (path,"/var/lib/xen/qemu-save.%u" , dom); + fp =fopen (path,"wb" ); +if ( !fp ) +return -1 ; + +DPRINTF ("Writing %d bytes of QEMU data\n" , buf->qemubufsize); +if (fwrite (buf->qemubuf,1 , buf->qemubufsize, fp) != buf->qemubufsize) { + saved_errno = errno; +fclose (fp); + errno = saved_errno; +return -1 ; + } + +fclose (fp); + +return 0 ; +} + +static int buffer_tail_hvm (struct tailbuf_hvm *buf,int fd, +unsigned int max_vcpu_id,uint64_t vcpumap, +int ext_vcpucontext) +{ +uint8_t *tmp; +unsigned char qemusig[21 ]; + +if (read_exact (fd, buf->magicpfns,sizeof (buf->magicpfns)) ) { +ERROR ("Error reading magic PFNs" ); +return -1 ; + } + +if (read_exact (fd, &buf->reclen,sizeof (buf->reclen)) ) { +ERROR ("Error reading HVM params size" ); +return -1 ; + } + +if ( buf->reclen > buf->hvmbufsize ) { +if ( buf->hvmbuf) { + tmp =realloc (buf->hvmbuf, buf->reclen); +if ( tmp ) { + buf->hvmbuf = tmp; + buf->hvmbufsize = buf->reclen; + }else { +ERROR ("Error reallocating HVM param buffer" ); +return -1 ; + } + }else { + buf->hvmbuf =malloc (buf->reclen); +if ( !buf->hvmbuf ) { +ERROR ("Error allocating HVM param buffer" ); +return -1 ; + } + buf->hvmbufsize = buf->reclen; + } + } + +if (read_exact (fd, buf->hvmbuf, buf->reclen) ) { +ERROR ("Error reading HVM params" ); +return -1 ; + } + +if (read_exact (fd, qemusig,sizeof (qemusig)) ) { +ERROR ("Error reading QEMU signature" ); +return -1 ; + } + +/* The normal live-migration QEMU record has no length information. +* Short of reimplementing the QEMU parser, we're forced to just read +* until EOF. Remus gets around this by sending a different signature +* which includes a length prefix */ +if ( !memcmp (qemusig,"QemuDeviceModelRecord" ,sizeof (qemusig)) ) +return compat_buffer_qemu (fd, buf); +else if ( !memcmp (qemusig,"RemusDeviceModelState" ,sizeof (qemusig)) ) +return buffer_qemu (fd, buf); + + qemusig[20 ] ='\0' ; +ERROR ("Invalid QEMU signature: %s" , qemusig); +return -1 ; +} + +static int buffer_tail_pv (struct tailbuf_pv *buf,int fd, +unsigned int max_vcpu_id,uint64_t vcpumap, +int ext_vcpucontext) {unsigned int i;size_t pfnlen, vcpulen;
@@ -753,18 +942,49 @@return -1 ; } -static void tailbuf_free (tailbuf_t * buf) +static int buffer_tail (tailbuf_t *buf,int fd,unsigned int max_vcpu_id, +uint64_t vcpumap,int ext_vcpucontext) { -if (buf->vcpubuf) { +if ( buf->ishvm ) +return buffer_tail_hvm (&buf->u.hvm, fd, max_vcpu_id, vcpumap, + ext_vcpucontext); +else +return buffer_tail_pv (&buf->u.pv, fd, max_vcpu_id, vcpumap, + ext_vcpucontext); +} + +static void tailbuf_free_hvm (struct tailbuf_hvm *buf) +{ +if ( buf->hvmbuf ) { +free (buf->hvmbuf); + buf->hvmbuf =NULL ; + } +if ( buf->qemubuf ) { +free (buf->qemubuf); + buf->qemubuf =NULL ; + } +} + +static void tailbuf_free_pv (struct tailbuf_pv *buf) +{ +if ( buf->vcpubuf ) {free (buf->vcpubuf); buf->vcpubuf =NULL ; } -if (buf->pfntab) { +if ( buf->pfntab ) {free (buf->pfntab); buf->pfntab =NULL ; } } +static void tailbuf_free (tailbuf_t *buf) +{ +if ( buf->ishvm ) +tailbuf_free_hvm (&buf->u.hvm); +else +tailbuf_free_pv (&buf->u.pv); +} + typedef struct { void* pages; /* pages is of length nr_physpages, pfn_types is of length nr_pages */
@@ -1118,18 +1338,13 @@unsigned int max_vcpu_id =0 ;int new_ctxt_format =0 ; -/* Magic frames in HVM guests: ioreqs and xenstore comms. */ -uint64_t magic_pfns[3 ];/* ioreq_pfn, bufioreq_pfn, store_pfn */ - -/* Buffer for holding HVM context */ -uint8_t *hvm_buf =NULL ; -pagebuf_t pagebuf;tailbuf_t tailbuf, tmptail;void * vcpup;pagebuf_init (&pagebuf);memset (&tailbuf,0 ,sizeof (tailbuf)); + tailbuf.ishvm = hvm;/* For info only */ nr_pfns =0 ;
@@ -1313,78 +1528,6 @@// DPRINTF("Received all pages (%d races)\n", nraces); -if ( hvm ) - { -uint32_t rec_len; - -/* Set HVM-specific parameters */ -if (read_exact (io_fd, magic_pfns,sizeof (magic_pfns)) ) - { -ERROR ("error reading magic page addresses" ); - goto out; - } - -/* These comms pages need to be zeroed at the start of day */ -if (xc_clear_domain_page (xc_handle, dom, magic_pfns[0 ]) || -xc_clear_domain_page (xc_handle, dom, magic_pfns[1 ]) || -xc_clear_domain_page (xc_handle, dom, magic_pfns[2 ]) ) - { -ERROR ("error zeroing magic pages" ); - goto out; - } - -if ( (frc =xc_set_hvm_param (xc_handle, dom, -HVM_PARAM_IOREQ_PFN , magic_pfns[0 ])) - || (frc =xc_set_hvm_param (xc_handle, dom, -HVM_PARAM_BUFIOREQ_PFN , magic_pfns[1 ])) - || (frc =xc_set_hvm_param (xc_handle, dom, -HVM_PARAM_STORE_PFN , magic_pfns[2 ])) - || (frc =xc_set_hvm_param (xc_handle, dom, -HVM_PARAM_PAE_ENABLED , pae)) - || (frc =xc_set_hvm_param (xc_handle, dom, -HVM_PARAM_STORE_EVTCHN , - store_evtchn)) ) - { -ERROR ("error setting HVM params: %i" , frc); - goto out; - } - *store_mfn = magic_pfns[2 ]; - -/* Read HVM context */ -if (read_exact (io_fd, &rec_len,sizeof (uint32_t )) ) - { -ERROR ("error read hvm context size!\n" ); - goto out; - } - - hvm_buf =malloc (rec_len); -if ( hvm_buf ==NULL ) - { -ERROR ("memory alloc for hvm context buffer failed" ); - errno =ENOMEM ; - goto out; - } - -if (read_exact (io_fd, hvm_buf, rec_len) ) - { -ERROR ("error loading the HVM context" ); - goto out; - } - - frc =xc_domain_hvm_setcontext (xc_handle, dom, hvm_buf, rec_len); -if ( frc ) - { -ERROR ("error setting the HVM context" ); - goto out; - } - -/* HVM success! */ - rc =0 ; - goto out; - } - -/* Non-HVM guests only from here on */ -if ( !completed ) {int flags =0 ;
@@ -1407,6 +1550,7 @@ goto finish; }memset (&tmptail,0 ,sizeof (tmptail)); + tmptail.ishvm = hvm;if (buffer_tail (&tmptail, io_fd, max_vcpu_id, vcpumap, ext_vcpucontext) <0 ) {ERROR ("error buffering image tail, finishing" );
@@ -1418,6 +1562,8 @@ goto loadpages; finish: +if ( hvm ) + goto finish_hvm;if ( (pt_levels ==3 ) && !pae_extended_cr3 ) {
@@ -1589,15 +1735,15 @@ {int nr_frees =0 ; -for ( i =0 ; i < tailbuf.pfncount; i++ ) +for ( i =0 ; i < tailbuf.u.pv.pfncount; i++ ) { -unsigned long pfn = tailbuf.pfntab[i]; +unsigned long pfn = tailbuf.u.pv.pfntab[i];if ( p2m[pfn] !=INVALID_P2M_ENTRY ) {/* pfn is not in physmap now, but was at some point during the save/migration process - need to free it */ - tailbuf.pfntab[nr_frees++] = p2m[pfn]; + tailbuf.u.pv.pfntab[nr_frees++] = p2m[pfn]; p2m[pfn] =INVALID_P2M_ENTRY ;/* not in pseudo-physical map */ } }
@@ -1609,7 +1755,7 @@ .extent_order = 0, .domid = dom }; -set_xen_guest_handle (reservation.extent_start, tailbuf.pfntab); +set_xen_guest_handle (reservation.extent_start, tailbuf.u.pv.pfntab);if ( (frc =xc_memory_op (xc_handle, XENMEM_decrease_reservation, &reservation)) != nr_frees )
@@ -1618,7 +1764,7 @@ goto out; }else -DPRINTF ("Decreased reservation by %d pages\n" , tailbuf.pfncount); +DPRINTF ("Decreased reservation by %d pages\n" , tailbuf.u.pv.pfncount); } }
@@ -1628,7 +1774,7 @@return 1 ; } - vcpup = tailbuf.vcpubuf; + vcpup = tailbuf.u.pv.vcpubuf;for ( i =0 ; i <= max_vcpu_id; i++ ) {if ( !(vcpumap & (1ULL << i)) )
@@ -1755,7 +1901,7 @@ } } -memcpy (shared_info_page, tailbuf.shared_info_page,PAGE_SIZE ); +memcpy (shared_info_page, tailbuf.u.pv.shared_info_page,PAGE_SIZE );DPRINTF ("Completed checkpoint load\n" );
@@ -1812,6 +1958,51 @@DPRINTF ("Domain ready to be built.\n" ); rc =0 ; + goto out; + + finish_hvm: +/* Dump the QEMU state to a state file for QEMU to load */ +if (dump_qemu (dom, &tailbuf.u.hvm) ) { +ERROR ("Error dumping QEMU state to file" ); + goto out; + } + +/* These comms pages need to be zeroed at the start of day */ +if (xc_clear_domain_page (xc_handle, dom, tailbuf.u.hvm.magicpfns[0 ]) || +xc_clear_domain_page (xc_handle, dom, tailbuf.u.hvm.magicpfns[1 ]) || +xc_clear_domain_page (xc_handle, dom, tailbuf.u.hvm.magicpfns[2 ]) ) + { +ERROR ("error zeroing magic pages" ); + goto out; + } + +if ( (frc =xc_set_hvm_param (xc_handle, dom, +HVM_PARAM_IOREQ_PFN , tailbuf.u.hvm.magicpfns[0 ])) + || (frc =xc_set_hvm_param (xc_handle, dom, +HVM_PARAM_BUFIOREQ_PFN , tailbuf.u.hvm.magicpfns[1 ])) + || (frc =xc_set_hvm_param (xc_handle, dom, +HVM_PARAM_STORE_PFN , tailbuf.u.hvm.magicpfns[2 ])) + || (frc =xc_set_hvm_param (xc_handle, dom, +HVM_PARAM_PAE_ENABLED , pae)) + || (frc =xc_set_hvm_param (xc_handle, dom, +HVM_PARAM_STORE_EVTCHN , + store_evtchn)) ) + { +ERROR ("error setting HVM params: %i" , frc); + goto out; + } + *store_mfn = tailbuf.u.hvm.magicpfns[2 ]; + + frc =xc_domain_hvm_setcontext (xc_handle, dom, tailbuf.u.hvm.hvmbuf, + tailbuf.u.hvm.reclen); +if ( frc ) + { +ERROR ("error setting the HVM context" ); + goto out; + } + +/* HVM success! */ + rc =0 ; out:if ( (rc !=0 ) && (dom !=0 ) )
@@ -1819,7 +2010,7 @@free (mmu);free (p2m);free (pfn_type); -free (hvm_buf); +tailbuf_free (&tailbuf);/* discard cache for save file */ discard_file_cache (io_fd,1 /*flush*/ );
tools/python/xen/xend/XendCheckpoint.py +0 -20
@@ -323,26 +323,6 @@ if not is_hvm and handler.console_mfn is None: raise XendError('Could not read console MFN') - # get qemu state and create a tmp file for dm restore - # Even PV guests may have QEMU stat, but its not currently - # used so only bother with HVM currently. - if is_hvm: - qemu_signature = read_exact(fd, len(QEMU_SIGNATURE), - "invalid device model signature read") - if qemu_signature != QEMU_SIGNATURE: - raise XendError("not a valid device model state: found '%s'" % - qemu_signature) - qemu_fd = os.open("/var/lib/xen/qemu-save.%d" % dominfo.getDomid(), - os.O_WRONLY | os.O_CREAT | os.O_TRUNC) - while True: - buf = os.read(fd, dm_batch) - if len(buf): - write_exact(qemu_fd, buf, - "could not write dm state to tmp file") - else: - break - os.close(qemu_fd) - restore_image.setCpuid() # xc_restore will wait for source to close connection
PATCH 1/1 blktap2: configurable driver chains
In reply to Brendan Cully# HG changeset patch
# User Ryan O'Connor <[email protected]>
# Date 1252530408 25200
# Node ID 4ca00ee41ce9731b8725c736b27c841b484dce5d # Parent 6ca67fe3514ada809a62282c30fe46d5df5ce265
blktap2: configurable driver chains
Blktap2 allows block device drivers to be layered to create more advanced virtual block devices. However, composing a layered driver is not exposed to the user. This patch fixes this, and allows the user to explicitly specify a driver chain when starting a tapdisk process, using the pipe character ('|') to explicitly seperate layers in a blktap2 configuration string.
for example, the command:
~$ tapdisk2 -n "log:|aio:/path/to/file.img"
will create a blktap2 device where read and write requests are passed to the 'log' driver, then forwarded to the 'aio' driver.
tools/blktap2/drivers/tapdisk-vbd.c +197 -42
@@ -106,6 +106,7 @@ vbd->callback = tapdisk_vbd_callback; vbd->argument = vbd; +INIT_LIST_HEAD (&vbd->driver_stack);INIT_LIST_HEAD (&vbd->images);INIT_LIST_HEAD (&vbd->new_requests);INIT_LIST_HEAD (&vbd->pending_requests);
@@ -541,6 +542,105 @@ goto out; } +/* TODO: ugh, lets not call it parent info... */ +static struct list_head * +tapdisk_vbd_open_level (td_vbd_t *vbd,char * params,int driver_type,td_disk_info_t *parent_info,td_flag_t flags) +{ +char *name; +int type, err; +td_image_t *image; +td_disk_id_t id; +struct list_head *images; +td_driver_t *driver; + + images =calloc (1 ,sizeof (struct list_head )); +INIT_LIST_HEAD (images); + + name = params; + type = driver_type; + +for (;;) { + err = -ENOMEM ; + image =tapdisk_image_allocate (name, type, + vbd->storage, flags, vbd); + +/* free 'name' if it was created by td_get_parent_id() */ +if (name != params) { +free (name); + name =NULL ; + } + +if (!image) +return NULL ; + + +/* We have to do this to set the driver info for child drivers. this conflicts with td_open */ + driver = image->driver; +if (!driver) { + driver =tapdisk_driver_allocate (image->type, + image->name, + image->flags, + image->storage); +if (!driver) +return NULL ; + } +/* the image has a driver, set the info and driver */ + image->driver = driver; + image->info = driver->info; + +/* XXX: we don't touch driver->refcount, broken? */ +/* XXX: we've replicated about 90% of td_open() gross! */ +/* XXX: this breaks if a driver modifies its info within a layer */ + +/* if the parent info is set, pass it to the child */ +if (parent_info) + { + image->driver->info = *parent_info; + } + + err =td_load (image); +if (err) { +if (err != -ENODEV ) +return NULL ; + + err =td_open (image); +if (err) +return NULL ; + } + +/* TODO: non-sink drivers that don't care about their child +* currently return EINVAL. Could return TD_PARENT_OK or +* TD_ANY_PARENT */ + + err =td_get_parent_id (image, &id); +if (err && (err !=TD_NO_PARENT && err != -EINVAL )) { +td_close (image); +return NULL ; + } + +if (!image->storage) + image->storage = vbd->storage; + +/* add this image to the end of the list */ +list_add_tail (&image->next, images); + + image =NULL ; + +/* if the image does not have a parent we return the +* list of images generated by this level of the stack */ +if (err ==TD_NO_PARENT || err == -EINVAL ) +break ; + + name = id.name; + type = id.drivertype; +#if 0 +/* catch this by validate, not here */ + flags |= (TD_OPEN_RDONLY |TD_OPEN_SHAREABLE ); +#endif + } +return images; +} +static int __tapdisk_vbd_open_vdi (td_vbd_t *vbd,td_flag_t extra_flags) {
@@ -548,58 +648,36 @@int err, type;td_flag_t flags;td_disk_id_t id; -td_image_t *image, *tmp; +td_image_t *tmp;struct tfilter *filter =NULL ; +td_vbd_driver_info_t *driver_info; +struct list_head *images; +td_disk_info_t *parent_info =NULL ; err =tapdisk_vbd_reactivate_volumes (vbd,0 );if (err)return err; flags = (vbd->flags & ~TD_OPEN_SHAREABLE ) | extra_flags; - file = vbd->name; - type = vbd->type; -for (;;) { - err = -ENOMEM ; - image =tapdisk_image_allocate (file, type, - vbd->storage, flags, vbd); -if (file != vbd->name) { -free (file); - file =NULL ; - } +/* loop on each user specified driver. +* NOTE: driver_info is in reverse order. That is, the first +* item is the 'parent' or 'sink' driver */ +list_for_each_entry (driver_info, &vbd->driver_stack, next) { + file = driver_info->params; + type = driver_info->type; + images =tapdisk_vbd_open_level (vbd, file, type, parent_info, flags); +if (!images) +return -EINVAL ; -if (!image) - goto fail; +/* after each loop, append the created stack to the result stack */ +list_splice (images, &vbd->images); +free (images); - err =td_load (image); -if (err) { -if (err != -ENODEV ) - goto fail; - - err =td_open (image); -if (err) - goto fail; - } - - err =td_get_parent_id (image, &id); -if (err && err !=TD_NO_PARENT ) { -td_close (image); - goto fail; - } - -if (!image->storage) - image->storage = vbd->storage; - -tapdisk_vbd_add_image (vbd, image); - image =NULL ; - -if (err ==TD_NO_PARENT ) -break ; - - file = id.name; - type = id.drivertype; - flags |= (TD_OPEN_RDONLY |TD_OPEN_SHAREABLE ); +/* set the parent_info to the first diskinfo on the stack */ + tmp =tapdisk_vbd_first_image (vbd); + parent_info = &tmp->info; }if (td_flag_test (vbd->flags,TD_OPEN_LOG_DIRTY )) {
@@ -623,14 +701,91 @@return 0 ; fail: + +/* TODO: loop over vbd to free images? maybe do that in vbd_close_vdi */ +#if 0 if (image)tapdisk_image_free (image); +#endif +/* TODO: handle partial stack createion? */ tapdisk_vbd_close_vdi (vbd);return err; } +/* this populates a vbd type based on path */ +int +tapdisk_vbd_parse_stack (td_vbd_t *vbd,const char *path) +{ +int err; +char *params, *driver_str; +td_vbd_driver_info_t *driver; + +/* make a copy of path */ +/* TODO: check against MAX_NAME_LEM ? */ + err =tapdisk_namedup (¶ms, path); +if (err) + goto error; + + +/* tokenize params based on pipe '|' */ + driver_str =strtok (params,"|" ); +while (driver_str !=NULL ) + { +/* parse driver info and add to vbd */ + driver =calloc (1 ,sizeof (td_vbd_driver_info_t)); +INIT_LIST_HEAD (&driver->next); + err =tapdisk_parse_disk_type (driver_str, &driver->params, &driver->type); +if (err) + goto error; + +/* build the list backwards as the last driver will be the first +* driver to open in the stack */ +list_add (&driver->next, &vbd->driver_stack); + +/* get next driver string */ + driver_str =strtok (NULL ,"|" ); + } + +return 0 ; + +/* error: free any driver_info's and params */ +error: +while (!list_empty (&vbd->driver_stack)) { + driver =list_entry (vbd->driver_stack.next, td_vbd_driver_info_t, next); +list_del (&driver->next); +free (driver); + } + +return err; +} + +/* NOTE: driver type, etc. must be set */ +static int +tapdisk_vbd_open_stack (td_vbd_t *vbd,uint16_t storage,td_flag_t flags) +{ +int i, err; + + vbd->flags = flags; + vbd->storage = storage; + +for (i =0 ; i <TD_VBD_EIO_RETRIES ; i++) { + err =__tapdisk_vbd_open_vdi (vbd,0 ); +if (err != -EIO ) +break ; + +sleep (TD_VBD_EIO_SLEEP ); + } +if (err) + goto fail; + +return 0 ; + +fail: +return err; +} +int tapdisk_vbd_open_vdi (td_vbd_t *vbd,const char *path,uint16_t drivertype,uint16_t storage,td_flag_t flags)
@@ -759,7 +914,7 @@ {int err; - err =tapdisk_vbd_open_vdi (vbd, name, type, storage, flags); + err =tapdisk_vbd_open_stack (vbd, storage, flags);if (err) goto out;
tools/blktap2/drivers/tapdisk-vbd.h +10 -0
@@ -53,6 +53,7 @@typedef struct td_ring td_ring_t ;typedef struct td_vbd_request td_vbd_request_t ; +typedef struct td_vbd_driver_info td_vbd_driver_info_t ;typedef struct td_vbd_handle td_vbd_t ;typedef void (*td_vbd_cb_t ) (void *,blkif_response_t *);
@@ -79,12 +80,20 @@struct list_head next; }; +struct td_vbd_driver_info { +char *params; +int type; +struct list_head next; +}; +struct td_vbd_handle {char *name;td_uuid_t uuid;int type; +struct list_head driver_stack; +int storage;uint8_t reopened;
@@ -164,6 +173,7 @@int tapdisk_vbd_initialize (int ,int ,td_uuid_t );void tapdisk_vbd_set_callback (td_vbd_t *,td_vbd_cb_t ,void *); +int tapdisk_vbd_parse_stack (td_vbd_t *vbd,const char *path);int tapdisk_vbd_open (td_vbd_t *,const char *,uint16_t ,uint16_t ,const char *,td_flag_t );int tapdisk_vbd_close (td_vbd_t *);
tools/blktap2/drivers/tapdisk2.c +7 -0
@@ -264,6 +264,13 @@return err; } + err =tapdisk_vbd_parse_stack (vbd, name); +if (err) { +CHILD_ERR (err,"vbd_parse_stack failed: %d\n" , err); +return err; + } + +/* TODO: clean this up */ err =tapdisk_vbd_open (vbd, path, type,TAPDISK_STORAGE_TYPE_DEFAULT , devname,0 );
tools/blktap2/include/list.h +20 -0
@@ -87,6 +87,26 @@return list->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *head) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; +struct list_head *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +static inline void list_splice (struct list_head *list,struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head); +} +#define list_entry (ptr, type, member) \ ((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))
Re: [PATCH 08 of 11] blktap2: configurable driver chains
Hey Ryan.
The command line idea is ok, but how about "!"?
Also, please check the line width.
Also, please don't alter the kernel list macros [..or submit yours to lkml].
Daniel
Quoted 409 lines · Brendan Cully
# HG changeset patch
# User Ryan O'Connor <[email protected]>
# Date 1252530408 25200
# Node ID4ca00ee41ce9731b8725c736b27c841b484dce5d# Parent6ca67fe3514ada809a62282c30fe46d5df5ce265
blktap2: configurable driver chainsBlktap2 allows block device drivers to be layered to create more advanced virtual block devices. However, composing a layered driver is not exposed to the user. This patch fixes this, and allows the user to explicitly specify a driver chain when starting a tapdisk process, using the pipe character ('|') to explicitly seperate layers in a blktap2 configuration string.
for example, the command:
~$ tapdisk2 -n "log:|aio:/path/to/file.img"
will create a blktap2 device where read and write requests are passed to the 'log' driver, then forwarded to the 'aio' driver.Signed-off-by Ryan O'Connor <[email protected]>tools/blktap2/drivers/tapdisk-vbd.c +197
-42@@ -106,6 +106,7 @@ vbd->callback = tapdisk_vbd_callback; vbd->argument = vbd; +INIT_LIST_HEAD (&vbd->driver_stack);INIT_LIST_HEAD (&vbd->images);INIT_LIST_HEAD (&vbd->new_requests);INIT_LIST_HEAD (&vbd->pending_requests);@@ -541,6 +542,105 @@ goto out; } +/* TODO: ugh, lets not call it parent info... */ +static struct list_head * +tapdisk_vbd_open_level (td_vbd_t *vbd,char * params,int driver_type,td_disk_info_t *parent_info,td_flag_t flags) +{ +char *name; +int type, err; +td_image_t *image; +td_disk_id_t id; +struct list_head *images; +td_driver_t *driver; + + images =calloc (1 ,sizeof (struct list_head )); +INIT_LIST_HEAD (images); + + name = params; + type = driver_type; + +for (;;) { + err = -ENOMEM ; + image =tapdisk_image_allocate (name, type, + vbd->storage, flags, vbd); + +/* free 'name' if it was created by td_get_parent_id() */ +if (name != params) { +free (name); + name =NULL ; + } + +if (!image) +return NULL ; + + +/* We have to do this to set the driver info for child drivers. this conflicts with td_open */ + driver = image->driver; +if (!driver) { + driver =tapdisk_driver_allocate (image->type, + image->name, + image->flags, + image->storage); +if (!driver) +return NULL ; + } +/* the image has a driver, set the info and driver */ + image->driver = driver; + image->info = driver->info; + +/* XXX: we don't touch driver->refcount, broken? */ +/* XXX: we've replicated about 90% of td_open() gross! */ +/* XXX: this breaks if a driver modifies its info within a layer */ + +/* if the parent info is set, pass it to the child */ +if (parent_info) + { + image->driver->info = *parent_info; + } + + err =td_load (image); +if (err) { +if (err != -ENODEV ) +return NULL ; + + err =td_open (image); +if (err) +return NULL ; + } + +/* TODO: non-sink drivers that don't care about their child +* currently return EINVAL. Could return TD_PARENT_OK or +* TD_ANY_PARENT */ + + err =td_get_parent_id (image, &id); +if (err && (err !=TD_NO_PARENT && err != -EINVAL )) { +td_close (image); +return NULL ; + } + +if (!image->storage) + image->storage = vbd->storage; + +/* add this image to the end of the list */ +list_add_tail (&image->next, images); + + image =NULL ; + +/* if the image does not have a parent we return the +* list of images generated by this level of the stack */ +if (err ==TD_NO_PARENT || err == -EINVAL ) +break ; + + name = id.name; + type = id.drivertype; +#if 0 +/* catch this by validate, not here */ + flags |= (TD_OPEN_RDONLY |TD_OPEN_SHAREABLE ); +#endif + } +return images; +} +static int __tapdisk_vbd_open_vdi (td_vbd_t *vbd,td_flag_t extra_flags) {@@ -548,58 +648,36 @@int err, type;td_flag_t flags;td_disk_id_t id; -td_image_t *image, *tmp; +td_image_t *tmp;struct tfilter *filter =NULL ; +td_vbd_driver_info_t *driver_info; +struct list_head *images; +td_disk_info_t *parent_info =NULL ; err =tapdisk_vbd_reactivate_volumes (vbd,0 );if (err)return err; flags = (vbd->flags & ~TD_OPEN_SHAREABLE ) | extra_flags; - file = vbd->name; - type = vbd->type; -for (;;) { - err = -ENOMEM ; - image =tapdisk_image_allocate (file, type, - vbd->storage, flags, vbd); -if (file != vbd->name) { -free (file); - file =NULL ; - } +/* loop on each user specified driver. +* NOTE: driver_info is in reverse order. That is, the first +* item is the 'parent' or 'sink' driver */ +list_for_each_entry (driver_info, &vbd->driver_stack, next) { + file = driver_info->params; + type = driver_info->type; + images =tapdisk_vbd_open_level (vbd, file, type, parent_info, flags); +if (!images) +return -EINVAL ; -if (!image) - goto fail; +/* after each loop, append the created stack to the result stack */ +list_splice (images, &vbd->images); +free (images); - err =td_load (image); -if (err) { -if (err != -ENODEV ) - goto fail; - - err =td_open (image); -if (err) - goto fail; - } - - err =td_get_parent_id (image, &id); -if (err && err !=TD_NO_PARENT ) { -td_close (image); - goto fail; - } - -if (!image->storage) - image->storage = vbd->storage; - -tapdisk_vbd_add_image (vbd, image); - image =NULL ; - -if (err ==TD_NO_PARENT ) -break ; - - file = id.name; - type = id.drivertype; - flags |= (TD_OPEN_RDONLY |TD_OPEN_SHAREABLE ); +/* set the parent_info to the first diskinfo on the stack */ + tmp =tapdisk_vbd_first_image (vbd); + parent_info = &tmp->info; }if (td_flag_test (vbd->flags,TD_OPEN_LOG_DIRTY )) {@@ -623,14 +701,91 @@return 0 ; fail: + +/* TODO: loop over vbd to free images? maybe do that in vbd_close_vdi */ +#if 0 if (image)tapdisk_image_free (image); +#endif +/* TODO: handle partial stack createion? */ tapdisk_vbd_close_vdi (vbd);return err; } +/* this populates a vbd type based on path */ +int +tapdisk_vbd_parse_stack (td_vbd_t *vbd,const char *path) +{ +int err; +char *params, *driver_str; +td_vbd_driver_info_t *driver; + +/* make a copy of path */ +/* TODO: check against MAX_NAME_LEM ? */ + err =tapdisk_namedup (¶ms, path); +if (err) + goto error; + + +/* tokenize params based on pipe '|' */ + driver_str =strtok (params,"|" ); +while (driver_str !=NULL ) + { +/* parse driver info and add to vbd */ + driver =calloc (1 ,sizeof (td_vbd_driver_info_t)); +INIT_LIST_HEAD (&driver->next); + err =tapdisk_parse_disk_type (driver_str, &driver->params, &driver->type); +if (err) + goto error; + +/* build the list backwards as the last driver will be the first +* driver to open in the stack */ +list_add (&driver->next, &vbd->driver_stack); + +/* get next driver string */ + driver_str =strtok (NULL ,"|" ); + } + +return 0 ; + +/* error: free any driver_info's and params */ +error: +while (!list_empty (&vbd->driver_stack)) { + driver =list_entry (vbd->driver_stack.next, td_vbd_driver_info_t, next); +list_del (&driver->next); +free (driver); + } + +return err; +} + +/* NOTE: driver type, etc. must be set */ +static int +tapdisk_vbd_open_stack (td_vbd_t *vbd,uint16_t storage,td_flag_t flags) +{ +int i, err; + + vbd->flags = flags; + vbd->storage = storage; + +for (i =0 ; i <TD_VBD_EIO_RETRIES ; i++) { + err =__tapdisk_vbd_open_vdi (vbd,0 ); +if (err != -EIO ) +break ; + +sleep (TD_VBD_EIO_SLEEP ); + } +if (err) + goto fail; + +return 0 ; + +fail: +return err; +} +int tapdisk_vbd_open_vdi (td_vbd_t *vbd,const char *path,uint16_t drivertype,uint16_t storage,td_flag_t flags)@@ -759,7 +914,7 @@ {int err; - err =tapdisk_vbd_open_vdi (vbd, name, type, storage, flags); + err =tapdisk_vbd_open_stack (vbd, storage, flags);if (err) goto out;tools/blktap2/drivers/tapdisk-vbd.h +10
-0@@ -53,6 +53,7 @@typedef struct td_ring td_ring_t ;typedef struct td_vbd_request td_vbd_request_t ; +typedef struct td_vbd_driver_info td_vbd_driver_info_t ;typedef struct td_vbd_handle td_vbd_t ;typedef void (*td_vbd_cb_t ) (void *,blkif_response_t *);@@ -79,12 +80,20 @@struct list_head next; }; +struct td_vbd_driver_info { +char *params; +int type; +struct list_head next; +}; +struct td_vbd_handle {char *name;td_uuid_t uuid;int type; +struct list_head driver_stack; +int storage;uint8_t reopened;@@ -164,6 +173,7 @@int tapdisk_vbd_initialize (int ,int ,td_uuid_t );void tapdisk_vbd_set_callback (td_vbd_t *,td_vbd_cb_t ,void *); +int tapdisk_vbd_parse_stack (td_vbd_t *vbd,const char *path);int tapdisk_vbd_open (td_vbd_t *,const char *,uint16_t ,uint16_t ,const char *,td_flag_t );int tapdisk_vbd_close (td_vbd_t *);tools/blktap2/drivers/tapdisk2.c +7
-0@@ -264,6 +264,13 @@return err; } + err =tapdisk_vbd_parse_stack (vbd, name); +if (err) { +CHILD_ERR (err,"vbd_parse_stack failed: %d\n" , err); +return err; + } + +/* TODO: clean this up */ err =tapdisk_vbd_open (vbd, path, type,TAPDISK_STORAGE_TYPE_DEFAULT , devname,0 );tools/blktap2/include/list.h +20
-0@@ -87,6 +87,26 @@return list->next == head; } +static inline void __list_splice (struct list_head *list, +struct list_head *head) +{ +struct list_head *first = list->next; +struct list_head *last = list->prev; +struct list_head *at = head->next; + + first->prev = head; + head->next = first; + + last->next = at; + at->prev = last; +} + +static inline void list_splice (struct list_head *list,struct list_head *head) +{ +if (!list_empty (list)) +__list_splice (list, head); +} +#define list_entry (ptr, type, member) \ ((type *)((char *)(ptr)-(unsigned long)(&((type *)0)->member)))_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
PATCH 1/1 blktap2: only open driver stack once
In reply to Brendan Cully# HG changeset patch
# User Ryan O'Connor <[email protected]>
# Date 1252530408 25200
# Node ID b14bb76b36dd239586b41c47e6f2531cbcbf4b47 # Parent 4ca00ee41ce9731b8725c736b27c841b484dce5d
blktap2: only open driver stack once
Currently blktap2 opens a driver stack, closes it, and re-opens it. This causes problems with our remus driver: the primary may connect to the backup in between the first and second open.
This is a temporary fix.
tools/blktap2/drivers/tapdisk-vbd.c +2 -0
@@ -1561,9 +1561,11 @@gettimeofday (&vreq->last_try,NULL );tapdisk_vbd_move_request (vreq, &vbd->pending_requests); +#if 0 err =tapdisk_vbd_check_queue (vbd);if (err) goto fail; +#endif err =tapdisk_image_check_ring_request (image, req);if (err)
PATCH 1/1 Fixup for tap:tapdisk syntax in remus uname
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1252530408 25200
# Node ID 5e0779189ef3b9382675b2e574b75936a6e9fb15 # Parent b14bb76b36dd239586b41c47e6f2531cbcbf4b47 Fixup for tap:tapdisk syntax in remus uname
tools/python/xen/xend/server/BlktapController.py +2 -0
@@ -172,6 +172,8 @@ uname = config.get('uname', '') try: (typ, subtyp, params, file) = string.split(uname, ':', 3) + if subtyp not in ('tapdisk', 'ioemu'): + raise ValueError('invalid subtype') except: (typ, params, file) = string.split(uname, ':', 2) subtyp = 'tapdisk'
PATCH 1/1 blktap2: add remus driver
In reply to Brendan Cully# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1252530408 25200
# Node ID aaf56934865afa3f5611cd891347953cdd1e5729 # Parent 5e0779189ef3b9382675b2e574b75936a6e9fb15
blktap2: add remus driver
Blktap2 port of remus disk driver. Backwards compatable with blktap1 implementation.
tools/blktap2/drivers/Makefile +10 -1
@@ -36,7 +36,7 @@ CRYPT_LIB += -lcrypto endif -LDFLAGS_img := $(CRYPT_LIB) -lpthread -lz +LDFLAGS_img := $(CRYPT_LIB) -lpthread -lz -lm LIBS += -L$(LIBVHDDIR) -lvhd
@@ -44,6 +44,14 @@ LIBS += -luuid endif +REMUS-OBJS := block-remus.o +REMUS-OBJS += hashtable.o +REMUS-OBJS += hashtable_itr.o +REMUS-OBJS += hashtable_utility.o + +$(REMUS-OBJS): CFLAGS += -fgnu89-inline -I$(XEN_XENSTORE) + + LIBAIO_DIR = $(XEN_ROOT)/tools/libaio/src tapdisk2 tapdisk-stream tapdisk-diff $(QCOW_UTIL): AIOLIBS := $(LIBAIO_DIR)/libaio.a tapdisk-client tapdisk-stream tapdisk-diff $(QCOW_UTIL): CFLAGS += -I$(LIBAIO_DIR) -I$(XEN_LIBXC)
@@ -81,6 +89,7 @@ BLK-OBJS-y += block-qcow.o BLK-OBJS-y += aes.o BLK-OBJS-y += $(PORTABLE-OBJS-y) +BLK-OBJS-y += $(REMUS-OBJS) all: $(IBIN) lock-util qcow-util
tools/blktap2/drivers/block-remus.c new file +1670 -0
1,670 lines not on this page · raw message
tools/blktap2/drivers/disktypes.h +13 -0
@@ -49,6 +49,7 @@extern struct tap_disk tapdisk_qcow;extern struct tap_disk tapdisk_block_cache;extern struct tap_disk tapdisk_log; +extern struct tap_disk tapdisk_remus;#define MAX_DISK_TYPES 20
@@ -61,6 +62,7 @@#define DISK_TYPE_QCOW 6#define DISK_TYPE_BLOCK_CACHE 7#define DISK_TYPE_LOG 9 +#define DISK_TYPE_REMUS 10/*Define Individual Disk Parameters here */ static disk_info_t null_disk = {
@@ -167,6 +169,16 @@#endif }; +static disk_info_t remus_disk = { +DISK_TYPE_REMUS , +"remus disk replicator (remus)" , +"remus" , +0 , +#ifdef TAPDISK + &tapdisk_remus, +#endif +}; +/*Main disk info array */ static disk_info_t *dtypes[] = { &aio_disk,
@@ -179,6 +191,7 @@ &block_cache_disk, &null_disk, &log_disk, + &remus_disk, };#endif
tools/blktap2/drivers/hashtable.c new file +274 -0
274 lines not on this page · raw message
tools/blktap2/drivers/hashtable_itr.c new file +188 -0
188 lines not on this page · raw message
tools/blktap2/drivers/hashtable_itr.h new file +112 -0
112 lines not on this page · raw message
tools/blktap2/drivers/hashtable_utility.c new file +71 -0
71 lines not on this page · raw message
tools/blktap2/drivers/hashtable_utility.h new file +55 -0
55 lines not on this page · raw message
tools/python/xen/xend/server/BlktapController.py +1 -0
@@ -28,6 +28,7 @@ 'ram', 'qcow', 'vhd', + 'remus', ] blktap_disk_types = blktap1_disk_types + blktap2_disk_types
Quoted 4 lines · Brendan Cully
It's my pleasure to announce the release of Remus 0.9!
I believe that it is now ready for inclusion in Xen, and I would love to have people try it out and let me know how it goes.
Does anyone on the list have any comments to make on Remus?
Personally I would like to get this in asap ahead of stabilisation for the next major Xen release. If there are comments/criticisms/support for Remus, or Kemari, or HA solutions in general, then they need to be aired now!
-- Keir
Quoted 4 lines · Brendan Cully
This release works with the tip of the xen-unstable repository, and supports PV and HVM in 32-on-32, 64-on-64, 32-on-64, and 64-on-32 configurations. It has been tested using Linux (Ubuntu) PV guests, and both Linux and Windows XP under HVM.
It should be noted that HVM guests running PV drivers are unsupported.
Keith Coleman
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel