Skip to contentSkip to conversation

Remus: pvops dom0 support

PATCH In review Brendan Cully 4 participants 11 messages May 3, 2010

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 Remus: pvops dom0 support

The following patch series makes it possible to run Remus on top of pvops dom0. Specifically, it uses the IFB device (included with newer kernels) for buffering outbound network traffic, instead of the old 3rd-party IMQ patch.

It also cleans up some bugs in the python rtnetlink bindings, supports guests with multiple VIFs, and allocates unused IFB/IMQ devices to allow concurrent protection of multiple guests.

No non-Remus code is touched by these patches. With these patches applied, the IMQ patch can be dropped from the pvops tree.

#1 lore raw

PATCH 1/1 Remus: python netlink fixes

Fix deprecation warning in Qdisc class under python 2.6. Fix rtattr length and padding (rta_len is unaligned). Null-terminate qdisc name in rtnl messages.

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694696 25200
# Node ID c0ef3e29c328f52cc9fe1896d3056b42af4ea115 # Parent 40334179f45b9de88520fe082ba592d9ab0424b8
Remus: python netlink fixes

Fix deprecation warning in Qdisc class under python 2.6. Fix rtattr length and padding (rta_len is unaligned). Null-terminate qdisc name in rtnl messages.

tools/python/xen/remus/netlink.py +7 -3
@@ -1,5 +1,7 @@
 # netlink wrappers
 
+# See include/linux/netlink.h and rtnetlink.h
+
 import socket, struct
 import xen.lowlevel.netlink
 
@@ -77,9 +79,9 @@
         return align(self.rta_len)
 
     def pack(self):
-        self.rta_len = self.fmtlen + align(len(self.body), 2)
+        self.rta_len = self.fmtlen + len(self.body)
         s = struct.pack(self.fmt, self.rta_len, self.rta_type) + self.body
-        pad = self.rta_len - len(s)
+        pad = align(self.rta_len) - len(s)
         if pad:
             s += '\0' * pad
         return s
@@ -127,14 +129,16 @@
         attr.rta_type = type
         attr.body = data
         self.rta += attr.pack()
+        self.nlmsg_len = len(self)
 
     def settype(self, cmd):
         self.nlmsg_type = cmd
 
     def pack(self):
-        return struct.pack(self.fmt, len(self), self.nlmsg_type,
+        s = struct.pack(self.fmt, len(self), self.nlmsg_type,
                            self.nlmsg_flags, self.nlmsg_seq,
                            self.nlmsg_pid) + self.body + self.rta
+        return s
 
     def unpack(self, msg):
         args = struct.unpack(self.fmt, msg[:self.fmtlen])
tools/python/xen/remus/qdisc.py +3 -3
@@ -35,7 +35,7 @@
         flags = netlink.NLM_F_EXCL|netlink.NLM_F_CREATE
         super(addrequest, self).__init__(netlink.RTM_NEWQDISC, flags=flags,
                                          dev=dev, handle=handle)
-        self.n.addattr(netlink.TCA_KIND, qdisc.kind)
+        self.n.addattr(netlink.TCA_KIND, qdisc.kind + '\0')
         opts = qdisc.pack()
         if opts:
             self.n.addattr(netlink.TCA_OPTIONS, opts)
@@ -49,7 +49,7 @@
     def __init__(self, dev, handle, qdisc):
         super(changerequest, self).__init__(netlink.RTM_NEWQDISC,
                                             dev=dev, handle=handle)
-        self.n.addattr(netlink.TCA_KIND, qdisc.kind)
+        self.n.addattr(netlink.TCA_KIND, qdisc.kind + '\0')
         opts = qdisc.pack()
         if opts:
             self.n.addattr(netlink.TCA_OPTIONS, opts)
@@ -59,7 +59,7 @@
         if qdict:
             kind = qdict.get('kind')
             cls = qdisc_kinds.get(kind, cls)
-        obj = super(Qdisc, cls).__new__(cls, qdict=qdict, *args, **opts)
+        obj = super(Qdisc, cls).__new__(cls)
         return obj
 
     def __init__(self, qdict):

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#2 lore raw

PATCH 1/1 Remus: remove obsolete code

In reply to Brendan Cully

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694699 25200
# Node ID f0786e8c7fd7b56f0c5712649fe52870d84488de # Parent c0ef3e29c328f52cc9fe1896d3056b42af4ea115
Remus: remove obsolete code

tools/remus/remus +0 -18
@@ -22,7 +22,6 @@
         self.port = XendOptions.instance().get_xend_relocation_port()
         self.interval = 200
         self.netbuffer = True
-        self.nobackup = False
         self.timer = False
 
         parser = optparse.OptionParser()
@@ -36,10 +35,6 @@
                           help='run without net buffering (benchmark option)')
         parser.add_option('', '--timer', dest='timer', action='store_true',
                           help='force pause at checkpoint interval (experimental)')
-        parser.add_option('', '--no-backup', dest='nobackup',
-                          action='store_true',
-                          help='prevent backup from starting up (benchmark '
-                          'option)')
         self.parser = parser
 
     def usage(self):
@@ -106,20 +101,12 @@
     def __del__(self):
         self.uninstall()
 
-    def setup(self):
-        #self.ctlfd.write('buffer')
-        #self.ctlfd.flush()
-        self.installed = True
-
     def uninstall(self):
         if self.ctlfd:
             self.ctlfd.close()
             self.ctlfd = None
 
     def postsuspend(self):
-        if not self.installed:
-            self.setup()
-
         os.write(self.ctlfd.fileno(), 'flush')
 
     def commit(self):
@@ -340,11 +327,6 @@
     for buf in bufs:
         buf.uninstall()
 
-    if cfg.nobackup:
-        # lame attempt to kill backup if protection is stopped deliberately.
-        # It would be much better to move this into the heartbeat "protocol".
-        print util.runcmd(['sudo', '-u', os.getlogin(), 'ssh', cfg.host, 'sudo', 'xm', 'destroy', dom.name])
-
     sys.exit(rc)
 
 cfg = Cfg()

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#3 lore raw

PATCH 1/1 Remus: move device handling into its own module

In reply to Brendan Cully

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694699 25200
# Node ID 35a9de6bca43769acf76cbd3a4a1f8c4f245da4b # Parent f0786e8c7fd7b56f0c5712649fe52870d84488de
Remus: move device handling into its own module

tools/python/xen/remus/device.py renamed from tools/remus/remus +13 -217
copy from tools/remus/remus
copy to tools/python/xen/remus/device.py
@@ -1,66 +1,12 @@
-#!/usr/bin/env python
+# Remus device interface
 #
-# This is a save process which also buffers outgoing I/O between
-# rounds, so that external viewers never see anything that hasn't
-# been committed at the backup
-#
-# TODO: fencing.
+# Coordinates with devices at suspend, resume, and commit hooks
 
-import optparse, os, re, select, signal, sys, time
-from xen.remus import save, vm
-from xen.xend import XendOptions
-from xen.remus import netlink, qdisc, util
+import os
 
-class CfgException(Exception): pass
+import netlink, qdisc, util
 
-class Cfg(object):
-    def __init__(self):
-        # must be set
-        self.domid = 0
-
-        self.host = 'localhost'
-        self.port = XendOptions.instance().get_xend_relocation_port()
-        self.interval = 200
-        self.netbuffer = True
-        self.timer = False
-
-        parser = optparse.OptionParser()
-        parser.usage = '%prog [options] domain [destination]'
-        parser.add_option('-i', '--interval', dest='interval', type='int',
-                          metavar='MS',
-                          help='checkpoint every MS milliseconds')
-        parser.add_option('-p', '--port', dest='port', type='int',
-                          help='send stream to port PORT', metavar='PORT')
-        parser.add_option('', '--no-net', dest='nonet', action='store_true',
-                          help='run without net buffering (benchmark option)')
-        parser.add_option('', '--timer', dest='timer', action='store_true',
-                          help='force pause at checkpoint interval (experimental)')
-        self.parser = parser
-
-    def usage(self):
-        self.parser.print_help()
-
-    def getargs(self):
-        opts, args = self.parser.parse_args()
-
-        if opts.interval:
-            self.interval = opts.interval
-        if opts.port:
-            self.port = opts.port
-        if opts.nonet:
-            self.netbuffer = False
-        if opts.timer:
-            self.timer = True
-
-        if not args:
-            raise CfgException('Missing domain')
-        self.domid = args[0]
-        if (len(args) > 1):
-            self.host = args[1]
-
-class ReplicatedDiskException(Exception): pass
-
-class BufferedDevice(object):
+class CheckpointedDevice(object):
     'Base class for buffered devices'
 
     def postsuspend(self):
@@ -75,7 +21,9 @@
         'called when backup has acknowledged checkpoint reception'
         pass
 
-class ReplicatedDisk(BufferedDevice):
+class ReplicatedDiskException(Exception): pass
+
+class ReplicatedDisk(CheckpointedDevice):
     """
     Send a checkpoint message to a replicated disk while the domain
     is paused between epochs.
@@ -114,9 +62,9 @@
         if msg != 'done':
             print 'Unknown message: %s' % msg
 
-class NetbufferException(Exception): pass
+class BufferedNICException(Exception): pass
 
-class Netbuffer(BufferedDevice):
+class BufferedNIC(CheckpointedDevice):
     """
     Buffer a protected domain's network output between rounds so that
     nothing is issued that a failover might not know about.
@@ -133,7 +81,7 @@
         self.devname = self._startimq(domid)
         dev = self.rth.getlink(self.devname)
         if not dev:
-            raise NetbufferException('could not find device %s' % self.devname)
+            raise BufferedNICException('could not find device %s' % self.devname)
         self.dev = dev['index']
         self.handle = qdisc.TC_H_ROOT
         self.q = qdisc.QueueQdisc()
@@ -164,8 +112,8 @@
                 self.installed = True
                 return
             if q['kind'] != 'pfifo_fast':
-                raise NetbufferException('there is already a queueing '
-                                         'discipline on %s' % self.devname)
+                raise BufferedNICException('there is already a queueing '
+                                           'discipline on %s' % self.devname)
 
         print 'installing buffer on %s' % self.devname
         req = qdisc.addrequest(self.dev, self.handle, self.q)
@@ -190,155 +138,3 @@
         util.runcmd("%s -A FORWARD -i %s -j imq --todev %s" % (imqebt, vid, imqdev))
 
         return imqdev
-
-class SignalException(Exception): pass
-
-def run(cfg):
-    closure = lambda: None
-    closure.cmd = None
-
-    def sigexception(signo, frame):
-        raise SignalException(signo)
-
-    def die():
-        # I am not sure what the best way to die is. xm destroy is another option,
-        # or we could attempt to trigger some instant reboot.
-        print "dying..."
-        print util.runcmd(['sudo', 'ifdown', 'eth2'])
-        # dangling imq0 handle on vif locks up the system
-        for buf in bufs:
-            buf.uninstall()
-        print util.runcmd(['sudo', 'xm', 'destroy', cfg.domid])
-        print util.runcmd(['sudo', 'ifup', 'eth2'])
-
-    def getcommand():
-        """Get a command to execute while running.
-        Commands include:
-          s: die prior to postsuspend hook
-          s2: die after postsuspend hook
-          r: die prior to preresume hook
-          r2: die after preresume hook
-          c: die prior to commit hook
-          c2: die after commit hook
-          """
-        r, w, x = select.select([sys.stdin], [], [], 0)
-        if sys.stdin not in r:
-            return
-
-        cmd = sys.stdin.readline().strip()
-        if cmd not in ('s', 's2', 'r', 'r2', 'c', 'c2'):
-            print "unknown command: %s" % cmd
-        closure.cmd = cmd
-
-    signal.signal(signal.SIGTERM, sigexception)
-
-    dom = vm.VM(cfg.domid)
-
-    # set up I/O buffers
-    bufs = []
-
-    # disks must commit before network can be released
-    for disk in dom.disks:
-        try:
-            bufs.append(ReplicatedDisk(disk))
-        except ReplicatedDiskException, e:
-            print e
-            continue
-
-    if cfg.netbuffer:
-        for vif in dom.vifs:
-            bufs.append(Netbuffer(dom.domid))
-
-    fd = save.MigrationSocket((cfg.host, cfg.port))
-
-    def postsuspend():
-        'Begin external checkpointing after domain has paused'
-        if not cfg.timer:
-            # when not using a timer thread, sleep until now + interval
-            closure.starttime = time.time()
-
-        if closure.cmd == 's':
-            die()
-
-        for buf in bufs:
-            buf.postsuspend()
-
-        if closure.cmd == 's2':
-            die()
-
-    def preresume():
-        'Complete external checkpointing before domain resumes'
-        if closure.cmd == 'r':
-            die()
-
-        for buf in bufs:
-            buf.preresume()
-
-        if closure.cmd == 'r2':
-            die()
-
-    def commit():
-        'commit network buffer'
-        if closure.cmd == 'c':
-            die()
-
-        print >> sys.stderr, "PROF: flushed memory at %0.6f" % (time.time())
-
-        for buf in bufs:
-            buf.commit()
-
-        if closure.cmd == 'c2':
-            die()
-
-        # Since the domain is running at this point, it's a good time to
-        # check for control channel commands
-        getcommand()
-
-        if not cfg.timer:
-            endtime = time.time()
-            elapsed = (endtime - closure.starttime) * 1000
-
-            if elapsed < cfg.interval:
-                time.sleep((cfg.interval - elapsed) / 1000.0)
-
-        # False ends checkpointing
-        return True
-
-    if cfg.timer:
-        interval = cfg.interval
-    else:
-        interval = 0
-
-    rc = 0
-
-    checkpointer = save.Saver(cfg.domid, fd, postsuspend, preresume, commit,
-                              interval)
-
-    try:
-        checkpointer.start()
-    except save.CheckpointError, e:
-        print e
-        rc = 1
-    except KeyboardInterrupt:
-        pass
-    except SignalException:
-        print '*** signalled ***'
-
-    for buf in bufs:
-        buf.uninstall()
-
-    sys.exit(rc)
-
-cfg = Cfg()
-try:
-    cfg.getargs()
-except CfgException, inst:
-    print str(inst)
-    cfg.usage()
-    sys.exit(1)
-
-try:
-    run(cfg)
-except vm.VMException, inst:
-    print str(inst)
-    sys.exit(1)
tools/remus/remus +3 -135

151 lines not on this page · raw message

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#4 lore raw

PATCH 1/1 Remus: fix VM stringification

In reply to Brendan Cully

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694699 25200
# Node ID d72ec2ffeb67195d55e942f50c04f98365eaec74 # Parent 35a9de6bca43769acf76cbd3a4a1f8c4f245da4b
Remus: fix VM stringification

tools/python/xen/remus/vm.py +3 -2
@@ -42,8 +42,9 @@
         self.vifs = getvifs(self.dom)
 
     def __str__(self):
-        return 'VM %d (%s), MACs: [%s], disks: [%s]' % \
-               (self.domid, self.name, self.epoch, ', '.join(self.macs),
+        return 'VM %d (%s), vifs: [%s], disks: [%s]' % \
+               (self.domid, self.name,
+                ', '.join([str(v) for v in self.vifs]),
                 ', '.join([str(d) for d in self.disks]))
 
 def parsedominfo(dominfo):

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#5 lore raw

Measuring the amount of memory read/written (basically touched) by a domU

Hi,

We want to measure the amount of memory that a domU touches while its running, and if possible even get a breakdown between memory read/written. We want to basically do these measurements for some windows domUs (XP, vista, 7) and perhaps even linux domU's on top of XEN... Does anyone know how do I can get this information easily? Has anyone done something like this already?

My test system is using XEN 4.0 + 2.6.32 pvops kernel -- (Ubuntu 9.10)

Thanks!
Yuvraj

#6 lore raw

Re: Measuring the amount of memory read/written (basically touched) by a domU

The live migration shadow-pagetable logic could perhaps be used for this. Live migration uses it for dirty page tracking, but you could extend it no doubt to both accessed- and dirty-page tracking, and take counts periodically. Definitely some hacking involved, however!

-- Keir

Quoted 19 lines · Yuvraj Agarwal

Hi,

We want to measure the amount of memory that a domU touches while its running, and if possible even get a breakdown between memory read/written. We want to basically do these measurements for some windows domUs (XP, vista, 7) and perhaps even linux domU's on top of XEN... Does anyone know how do I can get this information easily? Has anyone done something like this already?

My test system is using XEN 4.0 + 2.6.32 pvops kernel -- (Ubuntu 9.10)

Thanks!
Yuvraj

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#7 lore raw

PATCH 1/1 Remus: include device name in vif objects

In reply to Brendan Cully

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694699 25200
# Node ID c10edb7433f4546351dca7b47de0821fa0da795e # Parent d72ec2ffeb67195d55e942f50c04f98365eaec74
Remus: include device name in vif objects

tools/python/xen/remus/vif.py +6 -2
@@ -2,6 +2,7 @@
 
 class VIF(object):
     def __init__(self, **props):
+        self.dev = 'unknown'
         self.__dict__.update(props)
         if 'mac' in props:
             self.mac = canonifymac(props['mac'])
@@ -9,6 +10,9 @@
     def __str__(self):
         return self.mac
 
-def parse(props):
+def parse(props, domid, index):
     "turn a vm device dictionary into a vif object"
-    return VIF(**props)
+    vif = VIF(**props)
+    vif.dev = 'vif%d.%d' % (domid, index)
+
+    return vif
tools/python/xen/remus/vm.py +7 -1
@@ -113,7 +113,13 @@
     if type(vifs) != list:
         vifs = [vifs]
 
-    return [vif.parse(v) for v in vifs]
+    vifno = 0
+    parsed = []
+    for v in vifs:
+        parsed.append(vif.parse(v, dom['domid'], vifno))
+        vifno += 1
+
+    return parsed
 
 def getdisks(dom):
     "return block device objects for devices in dom"

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#8 lore raw

PATCH 1/1 Remus: add file locking and modprobe utility functions

In reply to Brendan Cully

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272694699 25200
# Node ID 95c7229ae3f411dfe57b9d92203336c75fd628e8 # Parent c10edb7433f4546351dca7b47de0821fa0da795e
Remus: add file locking and modprobe utility functions

tools/python/xen/remus/util.py +50 -1
@@ -1,6 +1,6 @@
 # utility functions
 
-import os, subprocess
+import fcntl, os, subprocess
 
 class PipeException(Exception):
     def __init__(self, message, errno):
@@ -8,9 +8,50 @@
         message = '%s: %d, %s' % (message, errno, os.strerror(errno))
         Exception.__init__(self, message)
 
+class Lock(object):
+    """advisory lock"""
+
+    def __init__(self, filename):
+        """lock using filename for synchronization"""
+        self.filename = filename + '.lock'
+
+        self.fd = None
+
+        self.lock()
+
+    def __del__(self):
+        self.unlock()
+
+    def lock(self):
+        if self.fd:
+            return
+
+        self.fd = open(self.filename, 'w')
+        fcntl.lockf(self.fd, fcntl.LOCK_EX)
+
+    def unlock(self):
+        if not self.fd:
+            return
+
+        fcntl.lockf(self.fd, fcntl.LOCK_UN)
+        self.fd = None
+        try:
+            os.remove(self.filename)
+        except OSError:
+            # harmless race
+            pass
+
 def canonifymac(mac):
     return ':'.join(['%02x' % int(field, 16) for field in mac.split(':')])
 
+def checkpid(pid):
+    """return True if pid is live"""
+    try:
+        os.kill(pid, 0)
+        return True
+    except OSError:
+        return False
+
 def runcmd(args, cwd=None):
     # TODO: stdin handling
     if type(args) == str:
@@ -29,3 +70,11 @@
         return stdout
     except (OSError, IOError), inst:
         raise PipeException('could not run %s' % args[0], inst.errno)
+
+def modprobe(modname):
+    """attempt to load kernel module modname"""
+    try:
+        runcmd(['modprobe', '-q', modname])
+        return True
+    except PipeException:
+        return False

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#9 lore raw

PATCH 1/1 Remus: use IFB for net buffer on newer kernels

In reply to Brendan Cully

IMQ does not work with ebtables on 2.6.31, and IFB is not a third-party patch.

# HG changeset patch
# User Brendan Cully <[email protected]>
# Date 1272912695 25200
# Node ID c288e90c1cf76899761da1dbc86a9cd70ef0f904 # Parent 95c7229ae3f411dfe57b9d92203336c75fd628e8
Remus: use IFB for net buffer on newer kernels

IMQ does not work with ebtables on 2.6.31, and IFB is not a third-party patch.

tools/python/xen/remus/device.py +235 -35
@@ -6,6 +6,9 @@
 
 import netlink, qdisc, util
 
+class ReplicatedDiskException(Exception): pass
+class BufferedNICException(Exception): pass
+
 class CheckpointedDevice(object):
     'Base class for buffered devices'
 
@@ -21,8 +24,6 @@
         'called when backup has acknowledged checkpoint reception'
         pass
 
-class ReplicatedDiskException(Exception): pass
-
 class ReplicatedDisk(CheckpointedDevice):
     """
     Send a checkpoint message to a replicated disk while the domain
@@ -62,36 +63,223 @@
         if msg != 'done':
             print 'Unknown message: %s' % msg
 
-class BufferedNICException(Exception): pass
+### Network
+
+# shared rtnl handle
+_rth = None
+def getrth():
+    global _rth
+
+    if not _rth:
+        _rth = netlink.rtnl()
+    return _rth
+
+class Netbuf(object):
+    "Proxy for netdev with a queueing discipline"
+
+    @staticmethod
+    def devclass():
+        "returns the name of this device class"
+        return 'unknown'
+
+    @classmethod
+    def available(cls):
+        "returns True if this module can proxy the device"
+        return cls._hasdev(cls.devclass())
+
+    def __init__(self, devname):
+        self.devname = devname
+        self.vif = None
+
+    # override in subclasses
+    def install(self, vif):
+        "set up proxy on device"
+        raise BufferedNICException('unimplemented')
+
+    def uninstall(self):
+        "remove proxy on device"
+        raise BufferedNICException('unimplemented')
+
+    # protected
+    @staticmethod
+    def _hasdev(devclass):
+        """check for existence of device, attempting to load kernel
+        module if not present"""
+        devname = '%s0' % devclass
+        rth = getrth()
+
+        if rth.getlink(devname):
+            return True
+        if util.modprobe(devclass) and rth.getlink(devname):
+            return True
+
+        return False
+
+class IFBBuffer(Netbuf):
+    """Capture packets arriving on a VIF using an ingress filter and tc
+    mirred action to forward them to an IFB device.
+    """
+
+    @staticmethod
+    def devclass():
+        return 'ifb'
+
+    def install(self, vif):
+        self.vif = vif
+        # voodoo from http://www.linuxfoundation.org/collaborate/workgroups/networking/ifb#Typical_Usage
+        util.runcmd('ip link set %s up' % self.devname)
+        util.runcmd('tc qdisc add dev %s ingress' % vif.dev)
+        util.runcmd('tc filter add dev %s parent ffff: proto ip pref 10 '
+                    'u32 match u32 0 0 action mirred egress redirect '
+                    'dev %s' % (vif.dev, self.devname))
+
+    def uninstall(self):
+        util.runcmd('tc filter del dev %s parent ffff: proto ip pref 10 u32' \
+                        % self.vif.dev)
+        util.runcmd('tc qdisc del dev %s ingress' % self.vif.dev)
+        util.runcmd('ip link set %s down' % self.devname)
+
+class IMQBuffer(Netbuf):
+    """Redirect packets coming in on vif to an IMQ device."""
+
+    imqebt = '/usr/lib/xen/bin/imqebt'
+
+    @staticmethod
+    def devclass():
+        return 'imq'
+
+    def install(self, vif):
+        # stopgap hack to set up IMQ for an interface. Wrong in many ways.
+        self.vif = vif
+
+        for mod in ['imq', 'ebt_imq']:
+            util.runcmd(['modprobe', mod])
+        util.runcmd("ip link set %s up" % self.devname)
+        util.runcmd("%s -F FORWARD" % self.imqebt)
+        util.runcmd("%s -A FORWARD -i %s -j imq --todev %s" % (self.imqebt, vif.dev, self.devname))
+
+    def uninstall(self):
+        util.runcmd("%s -F FORWARD" % self.imqebt)
+        util.runcmd('ip link set %s down' % self.devname)
+
+# in order of desirability
+netbuftypes = [IFBBuffer, IMQBuffer]
+
+def selectnetbuf():
+    "Find the best available buffer type"
+    for driver in netbuftypes:
+        if driver.available():
+            return driver
+
+    raise BufferedNICException('no net buffer available')
+
+class Netbufpool(object):
+    """Allocates/releases proxy netdevs (IMQ/IFB)
+
+    A file contains a list of entries of the form <pid>:<device>\n
+    To allocate a device, lock the file, then claim a new device if
+    one is free. If there are no free devices, check each PID for liveness
+    and take a device if the PID is dead, otherwise return failure.
+    Add an entry to the file before releasing the lock.
+    """
+    def __init__(self, netbufclass):
+        "Create a pool of Device"
+        self.netbufclass = netbufclass
+        self.path = '/var/run/remus/' + self.netbufclass.devclass()
+
+        self.devices = self.getdevs()
+
+        pooldir = os.path.dirname(self.path)
+        if not os.path.exists(pooldir):
+            os.makedirs(pooldir, 0755)
+
+    def get(self):
+        "allocate a free device"
+        def getfreedev(table):
+            for dev in self.devices:
+                if dev not in table or not util.checkpid(table[dev]):
+                    return dev
+
+            return None
+
+        lock = util.Lock(self.path)
+        table = self.load()
+
+        dev = getfreedev(table)
+        if not dev:
+            lock.unlock()
+            raise BufferedNICException('no free devices')
+        dev = self.netbufclass(dev)
+
+        table[dev.devname] = os.getpid()
+
+        self.save(table)
+        lock.unlock()
+
+        return dev
+
+    def put(self, dev):
+        "release claim on device"
+        lock = util.Lock(self.path)
+        table = self.load()
+
+        del table[dev.devname]
+
+        self.save(table)
+        lock.unlock()
+
+    # private
+    def load(self):
+        """load and parse allocation table"""
+        table = {}
+        if not os.path.exists(self.path):
+            return table
+
+        fd = open(self.path)
+        for line in fd.readlines():
+            iface, pid = line.strip().split()
+            table[iface] = int(pid)
+        fd.close()
+        return table
+
+    def save(self, table):
+        """write table to disk"""
+        lines = ['%s %d\n' % (iface, table[iface]) for iface in sorted(table)]
+        fd = open(self.path, 'w')
+        fd.writelines(lines)
+        fd.close()
+
+    def getdevs(self):
+        """find all available devices of our device type"""
+        ifaces = []
+        for line in util.runcmd('ifconfig -a -s').splitlines():
+            iface = line.split()[0]
+            if iface.startswith(self.netbufclass.devclass()):
+                ifaces.append(iface)
+
+        return ifaces
 
 class BufferedNIC(CheckpointedDevice):
     """
     Buffer a protected domain's network output between rounds so that
     nothing is issued that a failover might not know about.
     """
-    # shared rtnetlink handle
-    rth = None
 
-    def __init__(self, domid):
+    def __init__(self, vif):
         self.installed = False
+        self.vif = vif
 
-        if not self.rth:
-            self.rth = netlink.rtnl()
+        self.pool = Netbufpool(selectnetbuf())
+        self.rth = getrth()
 
-        self.devname = self._startimq(domid)
-        dev = self.rth.getlink(self.devname)
-        if not dev:
-            raise BufferedNICException('could not find device %s' % self.devname)
-        self.dev = dev['index']
-        self.handle = qdisc.TC_H_ROOT
-        self.q = qdisc.QueueQdisc()
+        self.setup()
 
     def __del__(self):
         self.uninstall()
 
     def postsuspend(self):
         if not self.installed:
-            self._setup()
+            self.install()
 
         self._sendqmsg(qdisc.TC_QUEUE_CHECKPOINT)
 
@@ -100,41 +288,53 @@
         the backup'''
         self._sendqmsg(qdisc.TC_QUEUE_RELEASE)
 
+    # private
     def _sendqmsg(self, action):
         self.q.action = action
-        req = qdisc.changerequest(self.dev, self.handle, self.q)
+        req = qdisc.changerequest(self.bufdevno, self.handle, self.q)
         self.rth.talk(req.pack())
+        return True
 
-    def _setup(self):
-        q = self.rth.getqdisc(self.dev)
+    def setup(self):
+        """install Remus queue on VIF outbound traffic"""
+        self.bufdev = self.pool.get()
+
+        devname = self.bufdev.devname
+        bufdev = self.rth.getlink(devname)
+        if not bufdev:
+            raise BufferedNICException('could not find device %s' % devname)
+
+        self.bufdev.install(self.vif)
+
+        self.bufdevno = bufdev['index']
+        self.handle = qdisc.TC_H_ROOT
+        self.q = qdisc.QueueQdisc()
+
+        if not util.modprobe('sch_queue'):
+            raise BufferedNICException('could not load sch_queue module')
+
+    def install(self):
+        devname = self.bufdev.devname
+        q = self.rth.getqdisc(self.bufdevno)
         if q:
             if q['kind'] == 'queue':
                 self.installed = True
                 return
             if q['kind'] != 'pfifo_fast':
                 raise BufferedNICException('there is already a queueing '
-                                           'discipline on %s' % self.devname)
+                                           'discipline on %s' % devname)
 
-        print 'installing buffer on %s' % self.devname
-        req = qdisc.addrequest(self.dev, self.handle, self.q)
+        print ('installing buffer on %s... ' % devname),
+        req = qdisc.addrequest(self.bufdevno, self.handle, self.q)
         self.rth.talk(req.pack())
         self.installed = True
+        print 'done.'
 
     def uninstall(self):
         if self.installed:
-            req = qdisc.delrequest(self.dev, self.handle)
+            req = qdisc.delrequest(self.bufdevno, self.handle)
             self.rth.talk(req.pack())
             self.installed = False
 
-    def _startimq(self, domid):
-        # stopgap hack to set up IMQ for an interface. Wrong in many ways.
-        imqebt = '/usr/lib/xen/bin/imqebt'
-        imqdev = 'imq0'
-        vid = 'vif%d.0' % domid
-        for mod in ['sch_queue', 'imq', 'ebt_imq']:
-            util.runcmd(['modprobe', mod])
-        util.runcmd("ip link set %s up" % (imqdev))
-        util.runcmd("%s -F FORWARD" % (imqebt))
-        util.runcmd("%s -A FORWARD -i %s -j imq --todev %s" % (imqebt, vid, imqdev))
-
-        return imqdev
+        self.bufdev.uninstall()
+        self.pool.put(self.bufdev)
tools/remus/remus +3 -2
@@ -9,7 +9,8 @@
 import optparse, os, re, select, signal, sys, time
 
 from xen.remus import save, util, vm
-from xen.remus.device import ReplicatedDisk, BufferedNIC
+from xen.remus.device import ReplicatedDisk, ReplicatedDiskException
+from xen.remus.device import BufferedNIC, BufferedNICException
 from xen.xend import XendOptions
 
 class CfgException(Exception): pass
@@ -115,7 +116,7 @@
 
     if cfg.netbuffer:
         for vif in dom.vifs:
-            bufs.append(Netbuffer(dom.domid))
+            bufs.append(BufferedNIC(vif))
 
     fd = save.MigrationSocket((cfg.host, cfg.port))
 

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

#10 lore raw
In reply to Brendan Cully

Thanks Brendan...

I can't wait to test this patches...

Regards

Em Seg, 2010-05-03 às 11:53 -0700, Brendan Cully escreveu:

Quoted 8 lines

The following patch series makes it possible to run Remus on top of pvops dom0. Specifically, it uses the IFB device (included with newer kernels) for buffering outbound network traffic, instead of the old 3rd-party IMQ patch.

It also cleans up some bugs in the python rtnetlink bindings, supports guests with multiple VIFs, and allocates unused IFB/IMQ devices to allow concurrent protection of multiple guests.

Quoted 7 lines

No non-Remus code is touched by these patches. With these patches applied, the IMQ patch can be dropped from the pvops tree.

_______________________________________________
Xen-devel mailing list
[email protected]
http://lists.xensource.com/xen-devel

Gilberto Nunes Ferreira
Suporte T.I.
Selbetti Gestão de Documentos

(47) 3441-6004 / (47) 8861-6672
Fax (47) 3441-6027
[email protected]