remus: cleanup python code
PATCH RESEND v2 1/1 remus: cleanup python code
This patch series removes unused classes/code from Remus python modules and adds some exception handling to ensure a cleaner exit.
Changes since last revision:
Patch 3/3: Handle specific exceptions when installing/uninstalling net buffers, instead of blank exceptions.
PATCH 1/1 remus: remove dead code and unused modules
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683501 25200
# Node ID ca4a8d0d504344a84f64bc7e939f8910baac236e # Parent c31e9249893d309655a8e739ba2ecb07d2c0148b
remus: remove dead code and unused modules
remove unused modules (profile, tapdisk, vdi) and unused thread classes/qdisc classes from the python code base.
tools/python/xen/remus/profile.py deleted +0 -56
@@ -1,56 +0,0 @@ -"""Simple profiling module -""" - -import time - -class ProfileBlock(object): - """A section of code to be profiled""" - def __init__(self, name): - self.name = name - - def enter(self): - print "PROF: entered %s at %f" % (self.name, time.time()) - - def exit(self): - print "PROF: exited %s at %f" % (self.name, time.time()) - -class NullProfiler(object): - def enter(self, name): - pass - - def exit(self, name=None): - pass - -class Profiler(object): - def __init__(self): - self.blocks = {} - self.running = [] - - def enter(self, name): - try: - block = self.blocks[name] - except KeyError: - block = ProfileBlock(name) - self.blocks[name] = block - - block.enter() - self.running.append(block) - - def exit(self, name=None): - if name is not None: - block = None - while self.running: - tmp = self.running.pop() - if tmp.name == name: - block = tmp - break - tmp.exit() - if not block: - raise KeyError('block %s not running' % name) - else: - try: - block = self.running.pop() - except IndexError: - raise KeyError('no block running') - - block.exit()
tools/python/xen/remus/qdisc.py +0 -37
@@ -109,43 +109,6 @@ qdisc_kinds['prio'] = PrioQdisc qdisc_kinds['pfifo_fast'] = PrioQdisc -class CfifoQdisc(Qdisc): - fmt = 'II' - - def __init__(self, qdict): - super(CfifoQdisc, self).__init__(qdict) - - if qdict.get('options'): - self.unpack(qdict['options']) - else: - self.epoch = 0 - self.vmid = 0 - - def pack(self): - return struct.pack(self.fmt, self.epoch, self.vmid) - - def unpack(self, opts): - self.epoch, self.vmid = struct.unpack(self.fmt, opts) - - def parse(self, opts): - args = list(opts) - try: - while args: - arg = args.pop(0) - if arg == 'epoch': - self.epoch = int(args.pop(0)) - continue - if arg.lower() == 'vmid': - self.vmid = int(args.pop(0)) - continue - except Exception, inst: - raise QdiscException(str(inst)) - - def optstr(self): - return 'epoch %d vmID %d' % (self.epoch, self.vmid) - -qdisc_kinds['cfifo'] = CfifoQdisc - TC_PLUG_CHECKPOINT = 0 TC_PLUG_RELEASE = 1
tools/python/xen/remus/save.py +1 -93
123 lines not on this page · raw message
tools/python/xen/remus/tapdisk.py deleted +0 -4
@@ -1,4 +0,0 @@ -import blkdev - -class TapDisk(BlkDev): - pass
tools/python/xen/remus/vdi.py deleted +0 -121
121 lines not on this page · raw message
tools/python/xen/remus/vm.py +0 -4
@@ -143,10 +143,6 @@ return [blkdev.parse(disk) for disk in disks] -def fromxend(domid): - "create a VM object from xend information" - return VM(domid) - def getshadowmem(vm): "Balloon down domain0 to create free memory for shadow paging." maxmem = int(vm.dom['maxmem'])
tools/remus/remus +0 -3
@@ -86,12 +86,9 @@ # I am not sure what the best way to die is. xm destroy is another option, # or we could attempt to trigger some instant reboot. print "dying..." - print util.runcmd(['sudo', 'ifdown', 'eth2']) - # dangling imq0 handle on vif locks up the system for buf in bufs: buf.uninstall() print util.runcmd(['sudo', 'xm', 'destroy', cfg.domid]) - print util.runcmd(['sudo', 'ifup', 'eth2']) def getcommand(): """Get a command to execute while running.
Re: [PATCH 1 of 3] remus: remove dead code and unused modules
Some of this code is indeed obsolete or otherwise not useful, but a lot of it (like CheckpointingFile) is, I think, useful for people who might want to hack on Remus. Specifically, I agree with removing vdi.py and with your change to tools/remus/remus, I am neutral on tapdisk.py, and I would prefer to keep the rest.
Quoted 420 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683501 25200
# Node IDca4a8d0d504344a84f64bc7e939f8910baac236e# Parentc31e9249893d309655a8e739ba2ecb07d2c0148b
remus: remove dead code and unused modulesremove unused modules (profile, tapdisk, vdi) and unused thread classes/qdisc classes from the python code base.
Signed-off-by Shriram Rajagopalan <[email protected]>diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/profile.pytools/python/xen/remus/profile.py deleted +0
-56@@ -1,56 +0,0 @@ -"""Simple profiling module -""" - -import time - -class ProfileBlock(object): - """A section of code to be profiled""" - def __init__(self, name): - self.name = name - - def enter(self): - print "PROF: entered %s at %f" % (self.name, time.time()) - - def exit(self): - print "PROF: exited %s at %f" % (self.name, time.time()) - -class NullProfiler(object): - def enter(self, name): - pass - - def exit(self, name=None): - pass - -class Profiler(object): - def __init__(self): - self.blocks = {} - self.running = [] - - def enter(self, name): - try: - block = self.blocks[name] - except KeyError: - block = ProfileBlock(name) - self.blocks[name] = block - - block.enter() - self.running.append(block) - - def exit(self, name=None): - if name is not None: - block = None - while self.running: - tmp = self.running.pop() - if tmp.name == name: - block = tmp - break - tmp.exit() - if not block: - raise KeyError('block %s not running' % name) - else: - try: - block = self.running.pop() - except IndexError: - raise KeyError('no block running') - - block.exit()diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/qdisc.pytools/python/xen/remus/qdisc.py +0
-0@@ -109,43 +109,6 @@qdisc_kinds['prio'] = PrioQdisc
qdisc_kinds['pfifo_fast'] = PrioQdisctools/python/xen/remus/qdisc.py +0
-37-class CfifoQdisc(Qdisc): - fmt = 'II' - - def __init__(self, qdict): - super(CfifoQdisc, self).__init__(qdict) - - if qdict.get('options'): - self.unpack(qdict['options']) - else: - self.epoch = 0 - self.vmid = 0 - - def pack(self): - return struct.pack(self.fmt, self.epoch, self.vmid) - - def unpack(self, opts): - self.epoch, self.vmid = struct.unpack(self.fmt, opts) - - def parse(self, opts): - args = list(opts) - try: - while args: - arg = args.pop(0) - if arg == 'epoch': - self.epoch = int(args.pop(0)) - continue - if arg.lower() == 'vmid': - self.vmid = int(args.pop(0)) - continue - except Exception, inst: - raise QdiscException(str(inst)) - - def optstr(self): - return 'epoch %d vmID %d' % (self.epoch, self.vmid) - -qdisc_kinds['cfifo'] = CfifoQdisc -
TC_PLUG_CHECKPOINT= 0TC_PLUG_RELEASE= 1diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/save.pytools/python/xen/remus/save.py +1
-2@@ -1,8 +1,7 @@ #!/usr/bin/env python -import os, select, socket, threading, time, signal, xmlrpclib +import os, socket, xmlrpclib -from xen.xend.XendClient import serverfrom xen.xend.xenstore.xswatch import xswatch
import xen.lowlevel.xc
tools/python/xen/remus/save.py +0
-0@@ -13,10 +12,6 @@import vm, image
tools/python/xen/remus/save.py +0
-4-XCFLAGS_LIVE = 1 - -xcsave = '/usr/lib/xen/bin/xc_save' -class
_proxy(object):
"proxy simulates an object without inheritance" def__init__(self, obj):tools/python/xen/remus/save.py +0
-0@@ -30,58 +25,6 @@class CheckpointError(Exception): pass
tools/python/xen/remus/save.py +0
-52-class CheckpointingFile(_proxy): - """Tee writes into separate file objects for each round. - This is necessary because xc_save gets a single file descriptor - for the duration of checkpointing. - """ - def __init__(self, path): - self.path = path - - self.round = 0 - self.rfd, self.wfd = os.pipe() - self.fd = file(path, 'wb') - - # this pipe is used to notify the writer thread of checkpoints - self.cprfd, self.cpwfd = os.pipe() - - super(CheckpointingFile, self).__init__(self.fd) - - wt = threading.Thread(target=self._wrthread, name='disk-write-thread') - wt.setDaemon(True) - wt.start() - self.wt = wt - - def fileno(self): - return self.wfd - - def close(self): - os.close(self.wfd) - # closing wfd should signal writer to stop - self.wt.join() - os.close(self.rfd) - os.close(self.cprfd) - os.close(self.cpwfd) - self.fd.close() - self.wt = None - - def checkpoint(self): - os.write(self.cpwfd, '1') - - def _wrthread(self): - while True: - r, o, e = select.select((self.rfd, self.cprfd), (), ()) - if self.rfd in r: - data = os.read(self.rfd, 256 * 1024) - if not data: - break - self.fd.write(data) - if self.cprfd in r: - junk = os.read(self.cprfd, 1) - self.round += 1 - self.fd = file('%s.%d' % (self.path, self.round), 'wb') - self.proxy(self.fd) -class
MigrationSocket(_proxy):
def__init__(self, address):
sock =socket.socket(socket.AF_INET, socket.SOCK_STREAM)tools/python/xen/remus/save.py +0
-30@@ -101,36 +44,6 @@ fd = os.fdopen(filedesc, 'w+') super(NullSocket, self).__init__(fd) -class Keepalive(object): - "Call a keepalive method at intervals" - def __init__(self, method, interval=0.1): - self.keepalive = method - self.interval = interval - - self.thread = None - self.running = False - - def start(self): - if not self.interval: - return - self.thread = threading.Thread(target=self.run, name='keepalive-thread') - self.thread.setDaemon(True) - self.running = True - self.thread.start() - - def stop(self): - if not self.thread: - return - self.running = False - self.thread.join() - self.thread = None - - def run(self): - while self.running: - self.keepalive() - time.sleep(self.interval) - self.keepalive(stop=True) -class Saver(object):
def __init__(self, domid, fd, suspendcb=None, resumecb=None,
checkpointcb=None, interval=0, flags=0):tools/python/xen/remus/save.py +0
-5@@ -177,10 +90,5 @@ pass def _resume(self): - """low-overhead version of XendDomainInfo.resumeDomain""" - # TODO: currently assumes SUSPEND_CANCEL is available - if True: xc.domain_resume(self.vm.domid, 1) xsutil.ResumeDomain(self.vm.domid) - else: - server.xend.domain.resumeDomain(self.vm.domid)diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/tapdisk.pytools/python/xen/remus/tapdisk.py deleted +0
-4@@ -1,4 +0,0 @@ -import blkdev - -class TapDisk(BlkDev): - passdiff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vdi.pytools/python/xen/remus/vdi.py deleted +0
-121@@ -1,121 +0,0 @@ -#code to play with vdis and snapshots - -import os - -def run(cmd): - fd = os.popen(cmd) - res = [l for l in fd if l.rstrip()] - return not fd.close(), res - - -_blockstore = '/blockstore.dat' - -def set_blockstore(blockstore): - global _blockstore - __blockstore = blockstore - - -class SnapShot: - def __init__(self, vdi, block, index): - self.__vdi = vdi - self.__block = block - self.__index = index - - #TODO add snapshot date and radix - - def __str__(self): - return '%d %d %d' % (self.__vdi.id(), self.__block, self.__index) - - def vdi(self): - return self.__vdi - - def block(self): - return self.__block - - def index(self): - return self.__index - - def match(self, block, index): - return self.__block == block and self.__index == index - - -class VDIException(Exception): - pass - - -class VDI: - def __init__(self, id, name): - self.__id = id - self.__name = name - - def __str__(self): - return 'vdi: %d %s' % (self.__id, self.__name) - - def id(self): - return self.__id - - def name(self): - return self.__name - - def list_snapshots(self): - res, ls = run('vdi_snap_list %s %d' % (_blockstore, self.__id)) - if res: - return [SnapShot(self, int(l[0]), int(l[1])) for l in [l.split() for l in ls[1:]]] - else: - raise VDIException("Error reading snapshot list") - - def snapshot(self): - res, ls = run('vdi_checkpoint %s %d' % (_blockstore, self.__id)) - if res: - _, block, idx = ls[0].split() - return SnapShot(self, int(block), int(idx)) - else: - raise VDIException("Error taking vdi snapshot") - - -def create(name, snap): - res, _ = run('vdi_create %s %s %d %d' - % (_blockstore, name, snap.block(), snap.index())) - if res: - return lookup_by_name(name) - else: - raise VDIException('Unable to create vdi from snapshot') - - -def fill(name, img_file): - res, _ = run('vdi_create %s %s' % (_blockstore, name)) - - if res: - vdi = lookup_by_name(name) - res, _ = run('vdi_fill %d %s' % (vdi.id(), img_file)) - if res: - return vdi - raise VDIException('Unable to create vdi from disk img file') - - -def list_vdis(): - vdis = [] - res, lines = run('vdi_list %s' % _blockstore) - if res: - for l in lines: - r = l.split() - vdis.append(VDI(int(r[0]), r[1])) - return vdis - else: - raise VDIException("Error doing vdi list") - - -def lookup_by_id(id): - vdis = list_vdis() - for v in vdis: - if v.id() == id: - return v - raise VDIException("No match from vdi id") - - -def lookup_by_name(name): - vdis = list_vdis() - for v in vdis: - if v.name() == name: - return v - raise VDIException("No match for vdi name")diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vm.pytools/python/xen/remus/vm.py +0
-4@@ -143,10 +143,6 @@ return [blkdev.parse(disk) for disk in disks] -def fromxend(domid): - "create a VM object from xend information" - return VM(domid) -def getshadowmem(vm):
"Balloon down domain0 to create free memory for shadow paging." maxmem =int(vm.dom['maxmem'])
diff -rc31e9249893d-rca4a8d0d5043tools/remus/remustools/remus/remus +0
-3@@ -86,12 +86,9 @@ # I am not sure what the best way to die is. xm destroy is another option, # or we could attempt to trigger some instant reboot. print "dying..." - print util.runcmd(['sudo', 'ifdown', 'eth2']) - # dangling imq0 handle on vif locks up the system for buf in bufs: buf.uninstall() print util.runcmd(['sudo', 'xm', 'destroy', cfg.domid]) - print util.runcmd(['sudo', 'ifup', 'eth2']) def getcommand(): """Get a command to execute while running._______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 3] remus: remove dead code and unused modules
Quoted 6 lines · Brendan Cully
Some of this code is indeed obsolete or otherwise not useful, but a lot of it (like CheckpointingFile) is, I think, useful for people who might want to hack on Remus. Specifically, I agree with removing
vdi.pyand with your change totools/remus/remus, I am neutral ontapdisk.py, and I would prefer to keep the rest.
Please specify what goes into "a lot of it" and "rest"
Quoted 398 lines · Shriram Rajagopalan
Quoted 8 lines
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683501 25200
# Node IDca4a8d0d504344a84f64bc7e939f8910baac236e# Parentc31e9249893d309655a8e739ba2ecb07d2c0148b
remus: remove dead code and unused modulesQuoted 8 lines
remove unused modules (profile, tapdisk, vdi) and unused thread classes/qdisc classes from the python code base.
Signed-off-by Shriram Rajagopalan <[email protected]>diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/profile.py
---a/tools/python/xen/remus/profile.pySat Jun 18 20:52:07 2011-0700
Quoted 141 lines
+++
/dev/nullThu Jan 01 00:00:00 1970 +0000tools/python/xen/remus/profile.py deleted +0
-56@@ -1,56 +0,0 @@ -"""Simple profiling module -""" - -import time - -class ProfileBlock(object): - """A section of code to be profiled""" - def __init__(self, name): - self.name = name - - def enter(self): - print "PROF: entered %s at %f" % (self.name, time.time()) - - def exit(self): - print "PROF: exited %s at %f" % (self.name, time.time()) - -class NullProfiler(object): - def enter(self, name): - pass - - def exit(self, name=None): - pass - -class Profiler(object): - def __init__(self): - self.blocks = {} - self.running = [] - - def enter(self, name): - try: - block = self.blocks[name] - except KeyError: - block = ProfileBlock(name) - self.blocks[name] = block - - block.enter() - self.running.append(block) - - def exit(self, name=None): - if name is not None: - block = None - while self.running: - tmp = self.running.pop() - if tmp.name == name: - block = tmp - break - tmp.exit() - if not block: - raise KeyError('block %s not running' % name) - else: - try: - block = self.running.pop() - except IndexError: - raise KeyError('no block running') - - block.exit()diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/qdisc.pytools/python/xen/remus/qdisc.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/qdisc.py Sat Jun 18 20:52:07 2011 -0700 +0
-0@@ -109,43 +109,6 @@qdisc_kinds['prio'] = PrioQdisc
qdisc_kinds['pfifo_fast'] = PrioQdisctools/python/xen/remus/qdisc.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/qdisc.py Sat Jun 18 20:52:07 2011 -0700 +0
-37-class CfifoQdisc(Qdisc): - fmt = 'II' - - def __init__(self, qdict): - super(CfifoQdisc, self).__init__(qdict) - - if qdict.get('options'): - self.unpack(qdict['options']) - else: - self.epoch = 0 - self.vmid = 0 - - def pack(self): - return struct.pack(self.fmt, self.epoch, self.vmid) - - def unpack(self, opts): - self.epoch, self.vmid = struct.unpack(self.fmt, opts) - - def parse(self, opts): - args = list(opts) - try: - while args: - arg = args.pop(0) - if arg == 'epoch': - self.epoch = int(args.pop(0)) - continue - if arg.lower() == 'vmid': - self.vmid = int(args.pop(0)) - continue - except Exception, inst: - raise QdiscException(str(inst)) - - def optstr(self): - return 'epoch %d vmID %d' % (self.epoch, self.vmid) - -qdisc_kinds['cfifo'] = CfifoQdisc -
TC_PLUG_CHECKPOINT= 0TC_PLUG_RELEASE= 1diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/save.pytools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +1
-2@@ -1,8 +1,7 @@ #!/usr/bin/env python -import os, select, socket, threading, time, signal, xmlrpclib +import os, socket, xmlrpclib -from xen.xend.XendClient import serverfrom xen.xend.xenstore.xswatch import xswatch
import xen.lowlevel.xc
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-0@@ -13,10 +12,6 @@import vm, image
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-4-XCFLAGS_LIVE = 1 - -xcsave = '/usr/lib/xen/bin/xc_save' -class
_proxy(object):
"proxy simulates an object without inheritance" def__init__(self, obj):tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-0@@ -30,58 +25,6 @@class CheckpointError(Exception): pass
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-8-class CheckpointingFile(_proxy): - """Tee writes into separate file objects for each round. - This is necessary because xc_save gets a single file descriptor - for the duration of checkpointing. - """ - def __init__(self, path): - self.path = path -tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700
Quoted 10 lines
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-10- self.round = 0 - self.rfd, self.wfd = os.pipe() - self.fd = file(path, 'wb') - - # this pipe is used to notify the writer thread of checkpoints - self.cprfd, self.cpwfd = os.pipe() - - super(CheckpointingFile, self).__init__(self.fd) - - wt = threading.Thread(target=self._wrthread,name='disk-write-thread')
Quoted 44 lines
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-34- wt.setDaemon(True) - wt.start() - self.wt = wt - - def fileno(self): - return self.wfd - - def close(self): - os.close(self.wfd) - # closing wfd should signal writer to stop - self.wt.join() - os.close(self.rfd) - os.close(self.cprfd) - os.close(self.cpwfd) - self.fd.close() - self.wt = None - - def checkpoint(self): - os.write(self.cpwfd, '1') - - def _wrthread(self): - while True: - r, o, e = select.select((self.rfd, self.cprfd), (), ()) - if self.rfd in r: - data = os.read(self.rfd, 256 * 1024) - if not data: - break - self.fd.write(data) - if self.cprfd in r: - junk = os.read(self.cprfd, 1) - self.round += 1 - self.fd = file('%s.%d' % (self.path, self.round), 'wb') - self.proxy(self.fd) -class
MigrationSocket(_proxy):
def__init__(self, address):
sock =socket.socket(socket.AF_INET, socket.SOCK_STREAM)tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-3@@ -101,36 +44,6 @@ fd = os.fdopen(filedesc, 'w+') super(NullSocket, self).__init__(fd) -class Keepalive(object): - "Call a keepalive method at intervals" - def __init__(self, method, interval=0.1):tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 line 116
Quoted 10 lines
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-10- self.keepalive = method - self.interval = interval - - self.thread = None - self.running = False - - def start(self): - if not self.interval: - return - self.thread = threading.Thread(target=self.run,name='keepalive-thread')
Quoted 31 lines
tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-17- self.thread.setDaemon(True) - self.running = True - self.thread.start() - - def stop(self): - if not self.thread: - return - self.running = False - self.thread.join() - self.thread = None - - def run(self): - while self.running: - self.keepalive() - time.sleep(self.interval) - self.keepalive(stop=True) -class Saver(object):
def __init__(self, domid, fd, suspendcb=None, resumecb=None,
checkpointcb=None, interval=0, flags=0):tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-5@@ -177,10 +90,5 @@ pass def _resume(self): - """low-overhead version of XendDomainInfo.resumeDomain""" - # TODO: currently assumes SUSPEND_CANCEL is available - if True: xc.domain_resume(self.vm.domid, 1) xsutil.ResumeDomain(self.vm.domid) - else: - server.xend.domain.resumeDomain(self.vm.domid)Quoted 2 lines
diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/tapdisk.py
---a/tools/python/xen/remus/tapdisk.pySat Jun 18 20:52:07 2011-0700
Quoted 27 lines
+++
/dev/nullThu Jan 01 00:00:00 1970 +0000tools/python/xen/remus/save.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/save.py Sat Jun 18 20:52:07 2011 -0700 +0
-4@@ -1,4 +0,0 @@ -import blkdev - -class TapDisk(BlkDev): - passdiff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vdi.pydev/null Thu Jan 01 00:00:00 1970 +0000 renamed from tools/python/xen/remus/vdi.py Sat Jun 18 20:52:07 2011 -0700 +0
-17@@ -1,121 +0,0 @@ -#code to play with vdis and snapshots - -import os - -def run(cmd): - fd = os.popen(cmd) - res = [l for l in fd if l.rstrip()] - return not fd.close(), res - - -_blockstore = '/blockstore.dat' - -def set_blockstore(blockstore): - global _blockstore - __blockstore = blockstore - -dev/null Thu Jan 01 00:00:00 1970 +0000 line 27
Quoted 10 lines
dev/null Thu Jan 01 00:00:00 1970 +0000 renamed from tools/python/xen/remus/vdi.py Sat Jun 18 20:52:07 2011 -0700 +0
-10-class SnapShot: - def __init__(self, vdi, block, index): - self.__vdi = vdi - self.__block = block - self.__index = index - - #TODO add snapshot date and radix - - def __str__(self): - return '%d %d %d' % (self.__vdi.id(), self.__block,self.__index)
Quoted 26 lines
dev/null Thu Jan 01 00:00:00 1970 +0000 renamed from tools/python/xen/remus/vdi.py Sat Jun 18 20:52:07 2011 -0700 +0
-26- - def vdi(self): - return self.__vdi - - def block(self): - return self.__block - - def index(self): - return self.__index - - def match(self, block, index): - return self.__block == block and self.__index == index - - -class VDIException(Exception): - pass - - -class VDI: - def __init__(self, id, name): - self.__id = id - self.__name = name - - def __str__(self): - return 'vdi: %d %s' % (self.__id, self.__name) -dev/null Thu Jan 01 00:00:00 1970 +0000 line 63
Quoted 10 lines
dev/null Thu Jan 01 00:00:00 1970 +0000 renamed from tools/python/xen/remus/vdi.py Sat Jun 18 20:52:07 2011 -0700 +0
-10- def id(self): - return self.__id - - def name(self): - return self.__name - - def list_snapshots(self): - res, ls = run('vdi_snap_list %s %d' % (_blockstore, self.__id)) - if res: - return [SnapShot(self, int(l[0]), int(l[1])) for l in[
l.split()for l in ls[1:]]]Quoted 65 lines
dev/null Thu Jan 01 00:00:00 1970 +0000 renamed from tools/python/xen/remus/vdi.py Sat Jun 18 20:52:07 2011 -0700 +0
-58- else: - raise VDIException("Error reading snapshot list") - - def snapshot(self): - res, ls = run('vdi_checkpoint %s %d' % (_blockstore, self.__id)) - if res: - _, block, idx = ls[0].split() - return SnapShot(self, int(block), int(idx)) - else: - raise VDIException("Error taking vdi snapshot") - - -def create(name, snap): - res, _ = run('vdi_create %s %s %d %d' - % (_blockstore, name, snap.block(), snap.index())) - if res: - return lookup_by_name(name) - else: - raise VDIException('Unable to create vdi from snapshot') - - -def fill(name, img_file): - res, _ = run('vdi_create %s %s' % (_blockstore, name)) - - if res: - vdi = lookup_by_name(name) - res, _ = run('vdi_fill %d %s' % (vdi.id(), img_file)) - if res: - return vdi - raise VDIException('Unable to create vdi from disk img file') - - -def list_vdis(): - vdis = [] - res, lines = run('vdi_list %s' % _blockstore) - if res: - for l in lines: - r = l.split() - vdis.append(VDI(int(r[0]), r[1])) - return vdis - else: - raise VDIException("Error doing vdi list") - - -def lookup_by_id(id): - vdis = list_vdis() - for v in vdis: - if v.id() == id: - return v - raise VDIException("No match from vdi id") - - -def lookup_by_name(name): - vdis = list_vdis() - for v in vdis: - if v.name() == name: - return v - raise VDIException("No match for vdi name")diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vm.pytools/python/xen/remus/vm.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/vm.py Sat Jun 18 20:52:07 2011 -0700 +0
-0@@ -143,10 +143,6 @@ return [blkdev.parse(disk) for disk in disks]
Quoted 32 lines
tools/remus/remus Tue Jun 21 12:11:41 2011 -0700 line 86
Quoted 12 lines
tools/python/xen/remus/vm.py Tue Jun 21 12:11:41 2011 -0700 renamed from tools/python/xen/remus/vm.py Sat Jun 18 20:52:07 2011 -0700 +0
-4-def fromxend(domid): - "create a VM object from xend information" - return VM(domid) -def getshadowmem(vm):
"Balloon down domain0 to create free memory for shadow paging." maxmem =int(vm.dom['maxmem'])
diff -rc31e9249893d-rca4a8d0d5043tools/remus/remustools/remus/remus Tue Jun 21 12:11:41 2011 -0700 renamed from tools/remus/remus Sat Jun 18 20:52:07 2011 -0700 +0
-0@@ -86,12 +86,9 @@ # I am not sure what the best way to die is. xm destroy isanother option,
Quoted 17 lines
tools/remus/remus Tue Jun 21 12:11:41 2011 -0700 renamed from tools/remus/remus Sat Jun 18 20:52:07 2011 -0700 +0
-3# or we could attempt to trigger some instant reboot. print "dying..." - print util.runcmd(['sudo', 'ifdown', 'eth2']) - # dangling imq0 handle on vif locks up the system for buf in bufs: buf.uninstall() print util.runcmd(['sudo', 'xm', 'destroy', cfg.domid]) - print util.runcmd(['sudo', 'ifup', 'eth2']) def getcommand(): """Get a command to execute while running._______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 1 of 3] remus: remove dead code and unused modules
Quoted 10 lines · Shriram Rajagopalan
Quoted 6 lines · Brendan Cully
Some of this code is indeed obsolete or otherwise not useful, but a lot of it (like CheckpointingFile) is, I think, useful for people who might want to hack on Remus. Specifically, I agree with removing
vdi.pyand with your change totools/remus/remus, I am neutral ontapdisk.py, and I would prefer to keep the rest.Please specify what goes into "a lot of it" and "rest"
Did the second sentence of the original email not make that clear?
Quoted 424 lines · Shriram Rajagopalan
Quoted 423 lines
Quoted 8 lines
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683501 25200
# Node IDca4a8d0d504344a84f64bc7e939f8910baac236e# Parentc31e9249893d309655a8e739ba2ecb07d2c0148b
remus: remove dead code and unused modulesQuoted 7 lines
remove unused modules (profile, tapdisk, vdi) and unused thread classes/qdisc classes from the python code base.
Signed-off-by Shriram Rajagopalan <[email protected]>diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/profile.py-0700
Quoted 139 lines
+++
/dev/nullThu Jan 01 00:00:00 1970 +0000tools/python/xen/remus/profile.py deleted +0
-56@@ -1,56 +0,0 @@ -"""Simple profiling module -""" - -import time - -class ProfileBlock(object): - """A section of code to be profiled""" - def __init__(self, name): - self.name = name - - def enter(self): - print "PROF: entered %s at %f" % (self.name, time.time()) - - def exit(self): - print "PROF: exited %s at %f" % (self.name, time.time()) - -class NullProfiler(object): - def enter(self, name): - pass - - def exit(self, name=None): - pass - -class Profiler(object): - def __init__(self): - self.blocks = {} - self.running = [] - - def enter(self, name): - try: - block = self.blocks[name] - except KeyError: - block = ProfileBlock(name) - self.blocks[name] = block - - block.enter() - self.running.append(block) - - def exit(self, name=None): - if name is not None: - block = None - while self.running: - tmp = self.running.pop() - if tmp.name == name: - block = tmp - break - tmp.exit() - if not block: - raise KeyError('block %s not running' % name) - else: - try: - block = self.running.pop() - except IndexError: - raise KeyError('no block running') - - block.exit()diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/qdisc.py+++b/tools/python/xen/remus/qdisc.pyTue Jun 21 12:11:41 2011 -0700(quoted diff) +0
-0@@ -109,43 +109,6 @@qdisc_kinds['prio'] = PrioQdisc
qdisc_kinds['pfifo_fast'] = PrioQdisctools/python/xen/remus/qdisc.py +0
-37-class CfifoQdisc(Qdisc): - fmt = 'II' - - def __init__(self, qdict): - super(CfifoQdisc, self).__init__(qdict) - - if qdict.get('options'): - self.unpack(qdict['options']) - else: - self.epoch = 0 - self.vmid = 0 - - def pack(self): - return struct.pack(self.fmt, self.epoch, self.vmid) - - def unpack(self, opts): - self.epoch, self.vmid = struct.unpack(self.fmt, opts) - - def parse(self, opts): - args = list(opts) - try: - while args: - arg = args.pop(0) - if arg == 'epoch': - self.epoch = int(args.pop(0)) - continue - if arg.lower() == 'vmid': - self.vmid = int(args.pop(0)) - continue - except Exception, inst: - raise QdiscException(str(inst)) - - def optstr(self): - return 'epoch %d vmID %d' % (self.epoch, self.vmid) - -qdisc_kinds['cfifo'] = CfifoQdisc -
TC_PLUG_CHECKPOINT= 0TC_PLUG_RELEASE= 1diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/save.py+++b/tools/python/xen/remus/save.pyTue Jun 21 12:11:41 2011 -0700tools/python/xen/remus/save.py +1
-2@@ -1,8 +1,7 @@ #!/usr/bin/env python -import os, select, socket, threading, time, signal, xmlrpclib +import os, socket, xmlrpclib -from xen.xend.XendClient import serverfrom xen.xend.xenstore.xswatch import xswatch
import xen.lowlevel.xc
(quoted diff) +0
-0@@ -13,10 +12,6 @@import vm, image
tools/python/xen/remus/save.py +0
-4-XCFLAGS_LIVE = 1 - -xcsave = '/usr/lib/xen/bin/xc_save' -class
_proxy(object):
"proxy simulates an object without inheritance" def__init__(self, obj):(quoted diff) +0
-0@@ -30,58 +25,6 @@class CheckpointError(Exception): pass
tools/python/xen/remus/save.py +0
-8-class CheckpointingFile(_proxy): - """Tee writes into separate file objects for each round. - This is necessary because xc_save gets a single file descriptor - for the duration of checkpointing. - """ - def __init__(self, path): - self.path = path -tools/python/xen/remus/save.py
Quoted 10 lines
tools/python/xen/remus/save.py +0
-10- self.round = 0 - self.rfd, self.wfd = os.pipe() - self.fd = file(path, 'wb') - - # this pipe is used to notify the writer thread of checkpoints - self.cprfd, self.cpwfd = os.pipe() - - super(CheckpointingFile, self).__init__(self.fd) - - wt = threading.Thread(target=self._wrthread,name='disk-write-thread')
Quoted 44 lines
tools/python/xen/remus/save.py +0
-34- wt.setDaemon(True) - wt.start() - self.wt = wt - - def fileno(self): - return self.wfd - - def close(self): - os.close(self.wfd) - # closing wfd should signal writer to stop - self.wt.join() - os.close(self.rfd) - os.close(self.cprfd) - os.close(self.cpwfd) - self.fd.close() - self.wt = None - - def checkpoint(self): - os.write(self.cpwfd, '1') - - def _wrthread(self): - while True: - r, o, e = select.select((self.rfd, self.cprfd), (), ()) - if self.rfd in r: - data = os.read(self.rfd, 256 * 1024) - if not data: - break - self.fd.write(data) - if self.cprfd in r: - junk = os.read(self.cprfd, 1) - self.round += 1 - self.fd = file('%s.%d' % (self.path, self.round), 'wb') - self.proxy(self.fd) -class
MigrationSocket(_proxy):
def__init__(self, address):
sock =socket.socket(socket.AF_INET, socket.SOCK_STREAM)tools/python/xen/remus/save.py +0
-3@@ -101,36 +44,6 @@ fd = os.fdopen(filedesc, 'w+') super(NullSocket, self).__init__(fd) -class Keepalive(object): - "Call a keepalive method at intervals" - def __init__(self, method, interval=0.1):tools/python/xen/remus/save.py line 116
Quoted 10 lines
tools/python/xen/remus/save.py +0
-10- self.keepalive = method - self.interval = interval - - self.thread = None - self.running = False - - def start(self): - if not self.interval: - return - self.thread = threading.Thread(target=self.run,name='keepalive-thread')
Quoted 31 lines
tools/python/xen/remus/save.py +0
-17- self.thread.setDaemon(True) - self.running = True - self.thread.start() - - def stop(self): - if not self.thread: - return - self.running = False - self.thread.join() - self.thread = None - - def run(self): - while self.running: - self.keepalive() - time.sleep(self.interval) - self.keepalive(stop=True) -class Saver(object):
def __init__(self, domid, fd, suspendcb=None, resumecb=None,
checkpointcb=None, interval=0, flags=0):tools/python/xen/remus/save.py +0
-5@@ -177,10 +90,5 @@ pass def _resume(self): - """low-overhead version of XendDomainInfo.resumeDomain""" - # TODO: currently assumes SUSPEND_CANCEL is available - if True: xc.domain_resume(self.vm.domid, 1) xsutil.ResumeDomain(self.vm.domid) - else: - server.xend.domain.resumeDomain(self.vm.domid)Quoted 1 line
diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/tapdisk.py-0700
Quoted 26 lines
+++
/dev/nullThu Jan 01 00:00:00 1970 +0000tools/python/xen/remus/tapdisk.py deleted +0
-4@@ -1,4 +0,0 @@ -import blkdev - -class TapDisk(BlkDev): - passdiff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vdi.py+++/dev/nullThu Jan 01 00:00:00 1970 +0000tools/python/xen/remus/vdi.py deleted +0
-17@@ -1,121 +0,0 @@ -#code to play with vdis and snapshots - -import os - -def run(cmd): - fd = os.popen(cmd) - res = [l for l in fd if l.rstrip()] - return not fd.close(), res - - -_blockstore = '/blockstore.dat' - -def set_blockstore(blockstore): - global _blockstore - __blockstore = blockstore - -tools/python/xen/remus/vdi.py line 27
Quoted 10 lines
tools/python/xen/remus/vdi.py deleted +0
-10-class SnapShot: - def __init__(self, vdi, block, index): - self.__vdi = vdi - self.__block = block - self.__index = index - - #TODO add snapshot date and radix - - def __str__(self): - return '%d %d %d' % (self.__vdi.id(), self.__block,self.__index)
Quoted 26 lines
tools/python/xen/remus/vdi.py deleted +0
-26- - def vdi(self): - return self.__vdi - - def block(self): - return self.__block - - def index(self): - return self.__index - - def match(self, block, index): - return self.__block == block and self.__index == index - - -class VDIException(Exception): - pass - - -class VDI: - def __init__(self, id, name): - self.__id = id - self.__name = name - - def __str__(self): - return 'vdi: %d %s' % (self.__id, self.__name) -tools/python/xen/remus/vdi.py
Quoted 10 lines
tools/python/xen/remus/vdi.py deleted +0
-10- def id(self): - return self.__id - - def name(self): - return self.__name - - def list_snapshots(self): - res, ls = run('vdi_snap_list %s %d' % (_blockstore, self.__id)) - if res: - return [SnapShot(self, int(l[0]), int(l[1])) for l in[
l.split()for l in ls[1:]]]Quoted 74 lines
tools/python/xen/remus/vdi.py deleted +0
-58- else: - raise VDIException("Error reading snapshot list") - - def snapshot(self): - res, ls = run('vdi_checkpoint %s %d' % (_blockstore, self.__id)) - if res: - _, block, idx = ls[0].split() - return SnapShot(self, int(block), int(idx)) - else: - raise VDIException("Error taking vdi snapshot") - - -def create(name, snap): - res, _ = run('vdi_create %s %s %d %d' - % (_blockstore, name, snap.block(), snap.index())) - if res: - return lookup_by_name(name) - else: - raise VDIException('Unable to create vdi from snapshot') - - -def fill(name, img_file): - res, _ = run('vdi_create %s %s' % (_blockstore, name)) - - if res: - vdi = lookup_by_name(name) - res, _ = run('vdi_fill %d %s' % (vdi.id(), img_file)) - if res: - return vdi - raise VDIException('Unable to create vdi from disk img file') - - -def list_vdis(): - vdis = [] - res, lines = run('vdi_list %s' % _blockstore) - if res: - for l in lines: - r = l.split() - vdis.append(VDI(int(r[0]), r[1])) - return vdis - else: - raise VDIException("Error doing vdi list") - - -def lookup_by_id(id): - vdis = list_vdis() - for v in vdis: - if v.id() == id: - return v - raise VDIException("No match from vdi id") - - -def lookup_by_name(name): - vdis = list_vdis() - for v in vdis: - if v.name() == name: - return v - raise VDIException("No match for vdi name")diff -r
c31e9249893d-rca4a8d0d5043tools/python/xen/remus/vm.py
+++b/tools/python/xen/remus/vm.pyTue Jun 21 12:11:41 2011 -0700tools/python/xen/remus/vm.py +0
-4@@ -143,10 +143,6 @@ return [blkdev.parse(disk) for disk in disks] -def fromxend(domid): - "create a VM object from xend information" - return VM(domid) -def getshadowmem(vm):
"Balloon down domain0 to create free memory for shadow paging." maxmem =int(vm.dom['maxmem'])
diff -rc31e9249893d-rca4a8d0d5043tools/remus/remus
+++ b/tools/remus/remus Tue Jun 21 12:11:41 2011 -0700Review line 86
Quoted 1 line
(quoted diff) +0
-0@@ -86,12 +86,9 @@ # I am not sure what the best way to die is. xm destroy isanother option,
Quoted 17 lines
tools/remus/remus +0
-3# or we could attempt to trigger some instant reboot. print "dying..." - print util.runcmd(['sudo', 'ifdown', 'eth2']) - # dangling imq0 handle on vif locks up the system for buf in bufs: buf.uninstall() print util.runcmd(['sudo', 'xm', 'destroy', cfg.domid]) - print util.runcmd(['sudo', 'ifup', 'eth2']) def getcommand(): """Get a command to execute while running._______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
PATCH 1/1 remus: move makeheader from image.py to vm.py
In reply to Shriram Rajagopalan# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683504 25200
# Node ID 794ead1a2be0578d70c38f006e7ab61c7abe9203 # Parent ca4a8d0d504344a84f64bc7e939f8910baac236e
remus: move makeheader from image.py to vm.py
makeheader is the only function used from image.py. Move it to vm.py and remove image.py file.
tools/python/xen/remus/image.py deleted +0 -227
@@ -1,227 +0,0 @@ -# VM image file manipulation - -import logging, struct - -import vm - -SIGNATURE = 'LinuxGuestRecord' -LONGLEN = struct.calcsize('L') -INTLEN = struct.calcsize('i') -PAGE_SIZE = 4096 -# ~0L -P2M_EXT_SIG = 4294967295L -# frames per page -FPP = 1024 -LTAB_MASK = 0xf << 28 -BATCH_SIZE = 1024 -IDXLEN = INTLEN + BATCH_SIZE * LONGLEN - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger() - -class VMParseException(Exception): pass - -class VMImage(object): - def __init__(self, img=None): - """img may be a path or a file object. - If compact is True, apply checkpoints to base image instead - of simply concatenating them. - """ - self.img = img - - self.dom = None - self.fd = None - self.header = None - self.nr_pfns = 0 - # p2m extension header (unparsed) - self.p2mext = None - - if self.img: - self.open(self.img) - - def open(self, img): - if isinstance(img, str): - self.fd = file(img, 'rb') - else: - self.fd = img - - self.readheader() - - def readheader(self): - sig = self.fd.read(len(SIGNATURE)) - if sig != SIGNATURE: - raise VMParseException("Bad signature in image") - - hlen = self.fd.read(INTLEN) - hlen, = struct.unpack('!i', hlen) - - self.header = self.fd.read(hlen) - self.dom = parseheader(self.header) - - def readp2mfl(self): - "read the P2M frame list" - pfnlen = self.fd.read(LONGLEN) - self.nr_pfns, = struct.unpack('L', pfnlen) - p2m0 = self.fd.read(LONGLEN) - - p2mhdr = p2m0 - p2m0, = struct.unpack('L', p2m0) - if p2m0 == P2M_EXT_SIG: - elen = self.fd.read(INTLEN) - elen, = struct.unpack('I', elen) - - self.p2mext = self.fd.read(elen) - - p2m0 = self.fd.read(LONGLEN) - p2m0, = struct.unpack('L', p2m0) - p2mfl = [p2m0] - - p2mfle = (self.nr_pfns + FPP - 1)/FPP - 1 - p2ms = self.fd.read(LONGLEN * p2mfle) - p2mfl.extend(struct.unpack('%dL' % p2mfle, p2ms)) - - self.p2mfl = p2mfl - - def flush(self): - self.ofd.write(self.tail) - -class Writer(object): - """compress a stream of checkpoints into a single image of the - last checkpoint""" - def __init__(self, fd, compact=False): - self.fd = fd - self.compact = compact - - self.vm = None - self.tail = None - # offset to first batch of pages - self.imgstart = 0 - # PFN mappings - self.pfns = [] - - def __del__(self): - self.close() - - def writeheader(self): - hlen = struct.pack('!i', len(self.vm.header)) - header = ''.join([SIGNATURE, hlen, self.vm.header]) - self.fd.write(header) - - def writep2mfl(self): - p2m = [struct.pack('L', self.vm.nr_pfns)] - if self.vm.p2mext: - p2m.extend([struct.pack('L', P2M_EXT_SIG), self.vm.p2mext]) - p2m.append(struct.pack('%dL' % len(self.vm.p2mfl), *self.vm.p2mfl)) - self.fd.write(''.join(p2m)) - - def writebatch(self, batch): - def offset(pfn): - isz = (pfn / BATCH_SIZE + 1) * IDXLEN - return self.imgstart + isz + pfn * PAGE_SIZE - - if not self.compact: - return self.fd.write(batch) - - batch = parsebatch(batch) - # sort pages for better disk seek behaviour - batch.sort(lambda x, y: cmp(x[0] & ~LTAB_MASK, y[0] & ~LTAB_MASK)) - - for pfndesc, page in batch: - pfn = pfndesc & ~LTAB_MASK - if pfn > self.vm.nr_pfns: - log.error('INVALID PFN: %d' % pfn) - if len(self.pfns) <= pfn: - self.pfns.extend([0] * (pfn - len(self.pfns) + 1)) - self.pfns[pfn] = pfndesc - self.fd.seek(offset(pfn)) - self.fd.write(page) - - #print "max offset: %d, %d" % (len(self.pfns), offset(self.pfns[-1])) - - def writeindex(self): - "Write batch header in front of each page" - hdrlen = INTLEN + BATCH_SIZE * LONGLEN - batches = (len(self.pfns) + BATCH_SIZE - 1) / BATCH_SIZE - - for i in xrange(batches): - offset = self.imgstart + i * (hdrlen + (PAGE_SIZE * BATCH_SIZE)) - pfnoff = i * BATCH_SIZE - # python auto-clamps overreads - pfns = self.pfns[pfnoff:pfnoff + BATCH_SIZE] - - self.fd.seek(offset) - self.fd.write(struct.pack('i', len(pfns))) - self.fd.write(struct.pack('%dL' % len(pfns), *pfns)) - - def slurp(self, ifd): - """Apply an incremental checkpoint to a loaded image. - accepts a path or a file object.""" - if isinstance(ifd, str): - ifd = file(ifd, 'rb') - - if not self.vm: - self.vm = VMImage(ifd) - self.writeheader() - - self.vm.readp2mfl() - self.writep2mfl() - self.imgstart = self.fd.tell() - - while True: - l, batch = readbatch(ifd) - if l <= 0: - break - self.writebatch(batch) - self.tail = batch + ifd.read() - - def flush(self): - if self.tail: - self.fd.seek(0, 2) - self.fd.write(self.tail) - if self.compact: - self.writeindex() - self.tail = None - - def close(self): - self.flush() - -def parseheader(header): - "parses a header sexpression" - return vm.parsedominfo(vm.strtosxpr(header)) - -def makeheader(dominfo): - "create an image header from a VM dominfo sxpr" - items = [SIGNATURE] - sxpr = vm.sxprtostr(dominfo) - items.append(struct.pack('!i', len(sxpr))) - items.append(sxpr) - return ''.join(items) - -def readbatch(fd): - batch = [] - batchlen = fd.read(INTLEN) - batch.append(batchlen) - batchlen, = struct.unpack('i', batchlen) - log.info("batch length: %d" % batchlen) - if batchlen <= 0: - return (batchlen, batch[0]) - - batchfns = fd.read(LONGLEN * batchlen) - batch.append(batchfns) - pages = fd.read(PAGE_SIZE * batchlen) - if len(pages) != PAGE_SIZE * batchlen: - log.error('SHORT READ: %d' % len(pages)) - batch.append(pages) - - return (batchlen, ''.join(batch)) - -def parsebatch(batch): - "parse a batch string into pages" - batchlen, batch = batch[:INTLEN], batch[INTLEN:] - batchlen, = struct.unpack('i', batchlen) - #print 'batch length: %d' % batchlen - pfnlen = batchlen * LONGLEN - pfns = struct.unpack('%dL' % batchlen, batch[:pfnlen]) - pagebuf = batch[pfnlen:] - pages = [pagebuf[i*PAGE_SIZE:(i+1)*PAGE_SIZE] for i in xrange(batchlen)] - return zip(pfns, pages)
tools/python/xen/remus/save.py +2 -2
@@ -10,7 +10,7 @@ import xen.lowlevel.checkpoint -import vm, image +import vm class _proxy(object): "proxy simulates an object without inheritance"
@@ -70,7 +70,7 @@ def start(self): vm.getshadowmem(self.vm) - hdr = image.makeheader(self.vm.dominfo) + hdr = vm.makeheader(self.vm.dominfo) self.fd.write(hdr) self.fd.flush()
tools/python/xen/remus/vm.py +10 -1
@@ -1,6 +1,6 @@ #!/usr/bin/env python -import xmlrpclib +import xmlrpclib, struct from xen.xend.XendClient import server from xen.xend import sxp, osdep
@@ -11,6 +11,7 @@ # need a nicer way to load disk drivers import vbd +SIGNATURE = 'LinuxGuestRecord' class VMException(Exception): pass class VM(object):
@@ -162,3 +163,11 @@ # target is in MB, not KB target = (dom0cur - needed) / 1024 server.xend.domain.setMemoryTarget(0, target) + +def makeheader(dominfo): + "create an image header from a VM dominfo sxpr" + items = [SIGNATURE] + sxpr = sxprtostr(dominfo) + items.append(struct.pack('!i', len(sxpr))) + items.append(sxpr) + return ''.join(items)
Re: [PATCH 2 of 3] remus: move makeheader from image.py to vm.py
I think this code is useful for Remus or live migration hackers too (we don't have python parsers for the save format elsewhere that I know of), and would prefer to keep it as is.
Quoted 301 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]> # Date 1308683504 25200
# Node ID794ead1a2be0578d70c38f006e7ab61c7abe9203# Parentca4a8d0d504344a84f64bc7e939f8910baac236e
remus: move makeheader fromimage.pytovm.pymakeheader is the only function used from
image.py. Move it tovm.pyand removeimage.pyfile.Signed-off-by Shriram Rajagopalan <[email protected]>diff -r
ca4a8d0d5043-r794ead1a2be0tools/python/xen/remus/image.pytools/python/xen/remus/image.py deleted +0
-227@@ -1,227 +0,0 @@ -# VM image file manipulation - -import logging, struct - -import vm - -SIGNATURE = 'LinuxGuestRecord' -LONGLEN = struct.calcsize('L') -INTLEN = struct.calcsize('i') -PAGE_SIZE = 4096 -# ~0L -P2M_EXT_SIG = 4294967295L -# frames per page -FPP = 1024 -LTAB_MASK = 0xf << 28 -BATCH_SIZE = 1024 -IDXLEN = INTLEN + BATCH_SIZE * LONGLEN - -logging.basicConfig(level=logging.DEBUG) -log = logging.getLogger() - -class VMParseException(Exception): pass - -class VMImage(object): - def __init__(self, img=None): - """img may be a path or a file object. - If compact is True, apply checkpoints to base image instead - of simply concatenating them. - """ - self.img = img - - self.dom = None - self.fd = None - self.header = None - self.nr_pfns = 0 - # p2m extension header (unparsed) - self.p2mext = None - - if self.img: - self.open(self.img) - - def open(self, img): - if isinstance(img, str): - self.fd = file(img, 'rb') - else: - self.fd = img - - self.readheader() - - def readheader(self): - sig = self.fd.read(len(SIGNATURE)) - if sig != SIGNATURE: - raise VMParseException("Bad signature in image") - - hlen = self.fd.read(INTLEN) - hlen, = struct.unpack('!i', hlen) - - self.header = self.fd.read(hlen) - self.dom = parseheader(self.header) - - def readp2mfl(self): - "read the P2M frame list" - pfnlen = self.fd.read(LONGLEN) - self.nr_pfns, = struct.unpack('L', pfnlen) - p2m0 = self.fd.read(LONGLEN) - - p2mhdr = p2m0 - p2m0, = struct.unpack('L', p2m0) - if p2m0 == P2M_EXT_SIG: - elen = self.fd.read(INTLEN) - elen, = struct.unpack('I', elen) - - self.p2mext = self.fd.read(elen) - - p2m0 = self.fd.read(LONGLEN) - p2m0, = struct.unpack('L', p2m0) - p2mfl = [p2m0] - - p2mfle = (self.nr_pfns + FPP - 1)/FPP - 1 - p2ms = self.fd.read(LONGLEN * p2mfle) - p2mfl.extend(struct.unpack('%dL' % p2mfle, p2ms)) - - self.p2mfl = p2mfl - - def flush(self): - self.ofd.write(self.tail) - -class Writer(object): - """compress a stream of checkpoints into a single image of the - last checkpoint""" - def __init__(self, fd, compact=False): - self.fd = fd - self.compact = compact - - self.vm = None - self.tail = None - # offset to first batch of pages - self.imgstart = 0 - # PFN mappings - self.pfns = [] - - def __del__(self): - self.close() - - def writeheader(self): - hlen = struct.pack('!i', len(self.vm.header)) - header = ''.join([SIGNATURE, hlen, self.vm.header]) - self.fd.write(header) - - def writep2mfl(self): - p2m = [struct.pack('L', self.vm.nr_pfns)] - if self.vm.p2mext: - p2m.extend([struct.pack('L', P2M_EXT_SIG), self.vm.p2mext]) - p2m.append(struct.pack('%dL' % len(self.vm.p2mfl), *self.vm.p2mfl)) - self.fd.write(''.join(p2m)) - - def writebatch(self, batch): - def offset(pfn): - isz = (pfn / BATCH_SIZE + 1) * IDXLEN - return self.imgstart + isz + pfn * PAGE_SIZE - - if not self.compact: - return self.fd.write(batch) - - batch = parsebatch(batch) - # sort pages for better disk seek behaviour - batch.sort(lambda x, y: cmp(x[0] & ~LTAB_MASK, y[0] & ~LTAB_MASK)) - - for pfndesc, page in batch: - pfn = pfndesc & ~LTAB_MASK - if pfn > self.vm.nr_pfns: - log.error('INVALID PFN: %d' % pfn) - if len(self.pfns) <= pfn: - self.pfns.extend([0] * (pfn - len(self.pfns) + 1)) - self.pfns[pfn] = pfndesc - self.fd.seek(offset(pfn)) - self.fd.write(page) - - #print "max offset: %d, %d" % (len(self.pfns), offset(self.pfns[-1])) - - def writeindex(self): - "Write batch header in front of each page" - hdrlen = INTLEN + BATCH_SIZE * LONGLEN - batches = (len(self.pfns) + BATCH_SIZE - 1) / BATCH_SIZE - - for i in xrange(batches): - offset = self.imgstart + i * (hdrlen + (PAGE_SIZE * BATCH_SIZE)) - pfnoff = i * BATCH_SIZE - # python auto-clamps overreads - pfns = self.pfns[pfnoff:pfnoff + BATCH_SIZE] - - self.fd.seek(offset) - self.fd.write(struct.pack('i', len(pfns))) - self.fd.write(struct.pack('%dL' % len(pfns), *pfns)) - - def slurp(self, ifd): - """Apply an incremental checkpoint to a loaded image. - accepts a path or a file object.""" - if isinstance(ifd, str): - ifd = file(ifd, 'rb') - - if not self.vm: - self.vm = VMImage(ifd) - self.writeheader() - - self.vm.readp2mfl() - self.writep2mfl() - self.imgstart = self.fd.tell() - - while True: - l, batch = readbatch(ifd) - if l <= 0: - break - self.writebatch(batch) - self.tail = batch + ifd.read() - - def flush(self): - if self.tail: - self.fd.seek(0, 2) - self.fd.write(self.tail) - if self.compact: - self.writeindex() - self.tail = None - - def close(self): - self.flush() - -def parseheader(header): - "parses a header sexpression" - return vm.parsedominfo(vm.strtosxpr(header)) - -def makeheader(dominfo): - "create an image header from a VM dominfo sxpr" - items = [SIGNATURE] - sxpr = vm.sxprtostr(dominfo) - items.append(struct.pack('!i', len(sxpr))) - items.append(sxpr) - return ''.join(items) - -def readbatch(fd): - batch = [] - batchlen = fd.read(INTLEN) - batch.append(batchlen) - batchlen, = struct.unpack('i', batchlen) - log.info("batch length: %d" % batchlen) - if batchlen <= 0: - return (batchlen, batch[0]) - - batchfns = fd.read(LONGLEN * batchlen) - batch.append(batchfns) - pages = fd.read(PAGE_SIZE * batchlen) - if len(pages) != PAGE_SIZE * batchlen: - log.error('SHORT READ: %d' % len(pages)) - batch.append(pages) - - return (batchlen, ''.join(batch)) - -def parsebatch(batch): - "parse a batch string into pages" - batchlen, batch = batch[:INTLEN], batch[INTLEN:] - batchlen, = struct.unpack('i', batchlen) - #print 'batch length: %d' % batchlen - pfnlen = batchlen * LONGLEN - pfns = struct.unpack('%dL' % batchlen, batch[:pfnlen]) - pagebuf = batch[pfnlen:] - pages = [pagebuf[i*PAGE_SIZE:(i+1)*PAGE_SIZE] for i in xrange(batchlen)] - return zip(pfns, pages)diff -r
ca4a8d0d5043-r794ead1a2be0tools/python/xen/remus/save.pytools/python/xen/remus/save.py +0
-0@@ -10,7 +10,7 @@import xen.lowlevel.checkpoint
tools/python/xen/remus/save.py +1
-1-import vm, image +import vmclass
_proxy(object):
"proxy simulates an object without inheritance"tools/python/xen/remus/save.py +1
-1@@ -70,7 +70,7 @@ def start(self): vm.getshadowmem(self.vm) - hdr = image.makeheader(self.vm.dominfo) + hdr = vm.makeheader(self.vm.dominfo) self.fd.write(hdr) self.fd.flush()diff -r
ca4a8d0d5043-r794ead1a2be0tools/python/xen/remus/vm.pytools/python/xen/remus/vm.py +1
-1@@ -1,6 +1,6 @@ #!/usr/bin/env python -import xmlrpclib +import xmlrpclib, structfrom xen.xend.XendClient import server
from xen.xend import sxp, osdeptools/python/xen/remus/vm.py +0
-0@@ -11,6 +11,7 @@ # need a nicer way to load disk driversimport vbd
+SIGNATURE = 'LinuxGuestRecord'
class VMException(Exception): passclass VM(object):
tools/python/xen/remus/vm.py +8
-0@@ -162,3 +163,11 @@ # target is in MB, not KB target = (dom0cur - needed) / 1024 server.xend.domain.setMemoryTarget(0, target) + +def makeheader(dominfo): + "create an image header from a VM dominfo sxpr" + items = [SIGNATURE] + sxpr = sxprtostr(dominfo) + items.append(struct.pack('!i', len(sxpr))) + items.append(sxpr) + return ''.join(items)_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
PATCH 1/1 remus: handle exceptions while installing/unstalling net buffer
In reply to Shriram Rajagopalan# HG changeset patch
# User Shriram Rajagopalan <[email protected]>
# Date 1308749695 25200
# Node ID 9eed27800ff6a2e6d73f138f20af072c1b41925e
# Parent 794ead1a2be0578d70c38f006e7ab61c7abe9203
remus: handle exceptions while installing/unstalling net buffer
tools/python/xen/remus/device.py +24 -8
@@ -169,15 +169,25 @@ self.vif = vif # voodoo from http://www.linuxfoundation.org/collaborate/workgroups/networking/ifb#Typical_Usage util.runcmd('ip link set %s up' % self.devname) - util.runcmd('tc qdisc add dev %s ingress' % vif.dev) + try: + util.runcmd('tc qdisc add dev %s ingress' % vif.dev) + except util.PipeException, e: + # check if error indicates that ingress qdisc + # already exists on the vif. If so, ignore it. + ignoreme = 'RTNETLINK answers: File exists' + if ignoreme in str(e): + pass + else: + raise e util.runcmd('tc filter add dev %s parent ffff: proto ip pref 10 ' 'u32 match u32 0 0 action mirred egress redirect ' 'dev %s' % (vif.dev, self.devname)) def uninstall(self): - util.runcmd('tc filter del dev %s parent ffff: proto ip pref 10 u32' \ - % self.vif.dev) - util.runcmd('tc qdisc del dev %s ingress' % self.vif.dev) + try: + util.runcmd('tc qdisc del dev %s ingress' % self.vif.dev) + except util.PipeException, e: + pass util.runcmd('ip link set %s down' % self.devname) class IMQBuffer(Netbuf):
@@ -373,9 +383,15 @@ def uninstall(self): if self.installed: - req = qdisc.delrequest(self.bufdevno, self.handle) - self.rth.talk(req.pack()) + try: + req = qdisc.delrequest(self.bufdevno, self.handle) + self.rth.talk(req.pack()) + except IOError, e: + pass self.installed = False - self.bufdev.uninstall() - self.pool.put(self.bufdev) + try: + self.bufdev.uninstall() + except util.PipeException, e: + pass + self.pool.put(self.bufdev)
tools/python/xen/remus/util.py +4 -2
@@ -65,8 +65,10 @@ proc.wait() if proc.returncode: print ' '.join(args) - print stderr.strip() - raise PipeException('%s failed' % args[0], proc.returncode) + errmsg = stderr.strip() + print errmsg + raise PipeException('%s failed (errmsg: %s)' % (args[0], errmsg), + proc.returncode) return stdout except (OSError, IOError), inst: raise PipeException('could not run %s' % args[0], inst.errno)
Re: [PATCH 3 of 3] remus: handle exceptions while installing/unstalling net buffer
Quoted 82 lines · Shriram Rajagopalan
# HG changeset patch
# User Shriram Rajagopalan <[email protected]>
# Date 1308749695 25200
# Node ID9eed27800ff6a2e6d73f138f20af072c1b41925e
# Parent794ead1a2be0578d70c38f006e7ab61c7abe9203
remus: handle exceptions while installing/unstalling net bufferSigned-off-by Shriram Rajagopalan <[email protected]>diff -r
794ead1a2be0-r9eed27800ff6tools/python/xen/remus/device.pytools/python/xen/remus/device.py Wed Jun 22 06:34:55 2011 -0700 renamed from tools/python/xen/remus/device.py Tue Jun 21 12:11:44 2011 -0700 +0
-0@@ -169,15 +169,25 @@ self.vif = vif # voodoo fromhttp://www.linuxfoundation.org/collaborate/workgroups/networking/ifb#Typical_Usage
util.runcmd('ip link set %s up' % self.devname)tools/python/xen/remus/device.py Wed Jun 22 06:34:55 2011 -0700 renamed from tools/python/xen/remus/device.py Tue Jun 21 12:11:44 2011 -0700 +10
-2- util.runcmd('tc qdisc add dev %s ingress' % vif.dev) + try: + util.runcmd('tc qdisc add dev %s ingress' % vif.dev) + except util.PipeException, e: + # check if error indicates that ingress qdisc + # already exists on the vif. If so, ignore it. + ignoreme = 'RTNETLINK answers: File exists' + if ignoreme in str(e): + pass + else: + raise e util.runcmd('tc filter add dev %s parent ffff: proto ip pref 10 ' 'u32 match u32 0 0 action mirred egress redirect ' 'dev %s' % (vif.dev, self.devname)) def uninstall(self): - util.runcmd('tc filter del dev %s parent ffff: proto ip pref 10u32' \
tools/python/xen/remus/device.py Wed Jun 22 06:34:55 2011 -0700 renamed from tools/python/xen/remus/device.py Tue Jun 21 12:11:44 2011 -0700 +14
-6- % self.vif.dev) - util.runcmd('tc qdisc del dev %s ingress' % self.vif.dev) + try: + util.runcmd('tc qdisc del dev %s ingress' % self.vif.dev) + except util.PipeException, e: + pass util.runcmd('ip link set %s down' % self.devname) class IMQBuffer(Netbuf):@@ -373,9 +383,15 @@ def uninstall(self): if self.installed: - req = qdisc.delrequest(self.bufdevno, self.handle) - self.rth.talk(req.pack()) + try: + req = qdisc.delrequest(self.bufdevno, self.handle) + self.rth.talk(req.pack()) + except IOError, e: + pass self.installed = False - self.bufdev.uninstall() - self.pool.put(self.bufdev) + try: + self.bufdev.uninstall() + except util.PipeException, e: + pass + self.pool.put(self.bufdev)diff -r
794ead1a2be0-r9eed27800ff6tools/python/xen/remus/util.pytools/python/xen/remus/util.py Wed Jun 22 06:34:55 2011 -0700 renamed from tools/python/xen/remus/util.py Tue Jun 21 12:11:44 2011 -0700 +4
-2@@ -65,8 +65,10 @@ proc.wait() if proc.returncode: print ' '.join(args) - print stderr.strip() - raise PipeException('%s failed' % args[0], proc.returncode) + errmsg = stderr.strip() + print errmsg + raise PipeException('%s failed (errmsg: %s)' % (args[0], errmsg), + proc.returncode) return stdout except (OSError, IOError), inst:raise
PipeException('could not run %s' % args[0], inst.errno)
Quoted 5 lines
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
As Brendan pointed out, 90% of what I proposed to remove
in patches 1 & 2 of this series might be useful for folks who want to hack on Remus
or live migration code. I ll do the clean up later, more carefully.
Ian, if you have no objection to this patch, could you take in this one alone and
reject patches 1 & 2? Or do you want me to send this one separately again?
shriram
_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel
Re: [PATCH 3 of 3] remus: handle exceptions while installing/unstalling net buffer
Shriram Rajagopalan writes ("Re: [Xen-devel] [PATCH 3 of 3] remus: handle exceptions while installing/unstalling net buffer"):
Quoted 4 lines
As Brendan pointed out, 90% of what I proposed to remove in patches 1 & 2 of this series might be useful for folks who want to hack on Remus or live migration code. I ll do the clean up later, more carefully.
Right, OK.
Quoted 3 lines
Ian, if you have no objection to this patch, could you take in this one alone and reject patches 1 & 2? Or do you want me to send this one separately again?
I have acked and applied the error exception patch.
Thanks,
Ian.