Skip to contentSkip to conversation

tools/python: Python 3 compatibility

PATCH Superseded Andrew Cooper 2 participants 4 messages Dec 18, 2019

v1 of 2 Superseded Latest is v2 →

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 tools/python: Python 3 compatibility

convert-legacy-stream is only used for incomming migration from pre Xen 4.7, and verify-stream-v2 appears to only be used by me during migration development - it is little surprise that they missed the main converstion effort in Xen 4.13.

Fix it all up.

Move open_file_or_fd() into a new util.py to avoid duplication, making it a more generic wrapper around open() or fdopen().

---
CC: Ian Jackson <[email protected]>
CC: Wei Liu <[email protected]>

This needs backporting to 4.13 ASAP

---
 tools/python/scripts/convert-legacy-stream | 49 +++++++-----------------------
 tools/python/scripts/verify-stream-v2      | 43 +++++---------------------
 tools/python/xen/migration/libxc.py        |  2 +-
 tools/python/xen/migration/libxl.py        |  2 +-
 tools/python/xen/migration/verify.py       |  4 +--
 tools/python/xen/util.py                   | 23 ++++++++++++++
 6 files changed, 46 insertions(+), 77 deletions(-)
 create mode 100644 tools/python/xen/util.py
tools/python/scripts/convert-legacy-stream +11 -38
index 5f80f13654..b0d81aa92e 100755
@@ -5,6 +5,8 @@
 Convert a legacy migration stream to a v2 stream.
 """
 
+from __future__ import print_function
+
 import sys
 import os, os.path
 import syslog
@@ -12,6 +14,7 @@ import traceback
 
 from struct import calcsize, unpack, pack
 
+from xen.util import open_file_or_fd as open_file_or_fd
 from xen.migration import legacy, public, libxc, libxl, xl
 
 __version__ = 1
@@ -39,16 +42,16 @@ def info(msg):
             for line in msg.split("\n"):
                 syslog.syslog(syslog.LOG_INFO, line)
         else:
-            print msg
+            print(msg)
 
 def err(msg):
     """Error message, routed to appropriate destination"""
     if log_to_syslog:
         for line in msg.split("\n"):
             syslog.syslog(syslog.LOG_ERR, line)
-    print >> sys.stderr, msg
+    print(msg, file = sys.stderr)
 
-class StreamError(StandardError):
+class StreamError(Exception):
     """Error with the incoming migration stream"""
     pass
 
@@ -70,7 +73,7 @@ class VM(object):
 
         # libxl
         self.libxl = fmt == "libxl"
-        self.emu_xenstore = "" # NUL terminated key&val pairs from "toolstack" records
+        self.emu_xenstore = b"" # NUL terminated key&val pairs from "toolstack" records
 
 def write_libxc_ihdr():
     stream_write(pack(libxc.IHDR_FORMAT,
@@ -336,7 +339,7 @@ def read_libxl_toolstack(vm, data):
         if twidth == 64:
             name = name[:-4]
 
-        if name[-1] != '\x00':
+        if name[-1] != b'\x00':
             raise StreamError("physmap name not NUL terminated")
 
         root = "physmap/%x" % (phys,)
@@ -347,7 +350,7 @@ def read_libxl_toolstack(vm, data):
         for key, val in zip(kv[0::2], kv[1::2]):
             info("    '%s' = '%s'" % (key, val))
 
-        vm.emu_xenstore += '\x00'.join(kv) + '\x00'
+        vm.emu_xenstore += b'\x00'.join(kv) + b'\x00'
 
 
 def read_chunks(vm):
@@ -534,7 +537,7 @@ def read_qemu(vm):
     sig, = unpack("21s", rawsig)
     info("Qemu signature: %s" % (sig, ))
 
-    if sig == "DeviceModelRecord0002":
+    if sig == b"DeviceModelRecord0002":
         rawsz = rdexact(4)
         sz, = unpack("I", rawsz)
         qdata = rdexact(sz)
@@ -617,36 +620,6 @@ def read_legacy_stream(vm):
         return 2
     return 0
 
-def open_file_or_fd(val, mode):
-    """
-    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
-    open it as a regular file.
-    """
-
-    fd = -1
-    try:
-        # Does it look like an integer?
-        try:
-            fd = int(val, 10)
-        except ValueError:
-            pass
-
-        # Try to open it...
-        if fd != -1:
-            return os.fdopen(fd, mode, 0)
-        else:
-            return open(val, mode, 0)
-
-    except StandardError, e:
-        if fd != -1:
-            err("Unable to open fd %d: %s: %s" %
-                (fd, e.__class__.__name__, e))
-        else:
-            err("Unable to open file '%s': %s: %s" %
-                (val, e.__class__.__name__, e))
-
-    raise SystemExit(1)
-
 
 def main():
     from optparse import OptionParser
@@ -723,7 +696,7 @@ def main():
 if __name__ == "__main__":
     try:
         sys.exit(main())
-    except SystemExit, e:
+    except SystemExit as e:
         sys.exit(e.code)
     except KeyboardInterrupt:
         sys.exit(1)
tools/python/scripts/verify-stream-v2 +8 -35

91 lines not on this page · raw message

tools/python/xen/migration/libxc.py +1 -1
index f24448a9ef..cbffd1975e 100644
@@ -223,7 +223,7 @@ def verify_record(self):
             self.squashed_pagedata_records += 1
 
         padding = content[length:]
-        if padding != "\x00" * len(padding):
+        if padding != b"\x00" * len(padding):
             raise StreamError("Padding containing non0 bytes found")
 
         if rtype not in record_verifiers:
tools/python/xen/migration/libxl.py +1 -1
index d5f54dc489..79f4024e72 100644
@@ -128,7 +128,7 @@ def verify_record(self):
         content = self.rdexact(contentsz)
 
         padding = content[length:]
-        if padding != "\x00" * len(padding):
+        if padding != b"\x00" * len(padding):
             raise StreamError("Padding containing non0 bytes found")
 
         if rtype not in record_verifiers:
tools/python/xen/migration/verify.py +2 -2
index 7a42dbfc58..1e38f4a3c0 100644
@@ -7,11 +7,11 @@
 
 from struct import calcsize, unpack
 
-class StreamError(StandardError):
+class StreamError(Exception):
     """Error with the stream"""
     pass
 
-class RecordError(StandardError):
+class RecordError(Exception):
     """Error with a record in the stream"""
     pass
 
tools/python/xen/util.py new file +23 -0
new file mode 100644
index 0000000000..a11358eefa
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+
+def open_file_or_fd(val, *argl, **kwargs):
+    """
+    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
+    open it as a regular file.
+    """
+
+    fd = -1
+    try:
+        # Does it look like an integer?
+        fd = int(val, 10)
+    except ValueError:
+        pass
+
+    # Try to open it...
+    if fd != -1:
+        return os.fdopen(fd, *argl, **kwargs)
+    else:
+        return open(val, *argl, **kwargs)
Signature · 7 lines
2.11.0


_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel
#1 lore raw
Quoted 68 lines · Andrew Cooper

convert-legacy-stream is only used for incomming migration from pre Xen 4.7, and verify-stream-v2 appears to only be used by me during migration development - it is little surprise that they missed the main converstion effort in Xen 4.13.

Fix it all up.

Move open_file_or_fd() into a new util.py to avoid duplication, making it a more generic wrapper around open() or fdopen().

---
CC: Ian Jackson <[email protected]>
CC: Wei Liu <[email protected]>

This needs backporting to 4.13 ASAP

---
 tools/python/scripts/convert-legacy-stream | 49 +++++++-----------------------
 tools/python/scripts/verify-stream-v2      | 43 +++++---------------------
 tools/python/xen/migration/libxc.py        |  2 +-
 tools/python/xen/migration/libxl.py        |  2 +-
 tools/python/xen/migration/verify.py       |  4 +--
 tools/python/xen/util.py                   | 23 ++++++++++++++
 6 files changed, 46 insertions(+), 77 deletions(-)
 create mode 100644 tools/python/xen/util.py
tools/python/scripts/convert-legacy-stream +6 -3
index 5f80f13654..b0d81aa92e 100755
@@ -5,6 +5,8 @@
 Convert a legacy migration stream to a v2 stream.
 """
 
+from __future__ import print_function
+
 import sys
 import os, os.path
 import syslog
@@ -12,6 +14,7 @@ import traceback
 
 from struct import calcsize, unpack, pack
 
+from xen.util import open_file_or_fd as open_file_or_fd
 from xen.migration import legacy, public, libxc, libxl, xl
 
 __version__ = 1
@@ -39,16 +42,16 @@ def info(msg):
             for line in msg.split("\n"):
                 syslog.syslog(syslog.LOG_INFO, line)
         else:
-            print msg
+            print(msg)
 
 def err(msg):
     """Error message, routed to appropriate destination"""
     if log_to_syslog:
         for line in msg.split("\n"):
             syslog.syslog(syslog.LOG_ERR, line)
-    print >> sys.stderr, msg
+    print(msg, file = sys.stderr)
 
-class StreamError(StandardError):
+class StreamError(Exception):
     """Error with the incoming migration stream"""
     pass
 

tools/python/scripts/convert-legacy-stream class VM(object): · line 79

Quoted 8 lines
tools/python/scripts/convert-legacy-stream +1 -1
@@ -70,7 +73,7 @@ class VM(object):
 
         # libxl
         self.libxl = fmt == "libxl"
-        self.emu_xenstore = "" # NUL terminated key&val pairs from "toolstack" records
+        self.emu_xenstore = b"" # NUL terminated key&val pairs from "toolstack" records
 
 def write_libxc_ihdr():
     stream_write(pack(libxc.IHDR_FORMAT,

You also need to update write_record (string constants). And few calls to it with string constants (write_libxl_end, write_libxl_libxc_context, read_pv_tail, read_hvm_tail). And blkid == ... in read_pv_extended_info().

Quoted 248 lines
tools/python/scripts/convert-legacy-stream +4 -34
@@ -336,7 +339,7 @@ def read_libxl_toolstack(vm, data):
         if twidth == 64:
             name = name[:-4]
 
-        if name[-1] != '\x00':
+        if name[-1] != b'\x00':
             raise StreamError("physmap name not NUL terminated")
 
         root = "physmap/%x" % (phys,)
@@ -347,7 +350,7 @@ def read_libxl_toolstack(vm, data):
         for key, val in zip(kv[0::2], kv[1::2]):
             info("    '%s' = '%s'" % (key, val))
 
-        vm.emu_xenstore += '\x00'.join(kv) + '\x00'
+        vm.emu_xenstore += b'\x00'.join(kv) + b'\x00'
 
 
 def read_chunks(vm):
@@ -534,7 +537,7 @@ def read_qemu(vm):
     sig, = unpack("21s", rawsig)
     info("Qemu signature: %s" % (sig, ))
 
-    if sig == "DeviceModelRecord0002":
+    if sig == b"DeviceModelRecord0002":
         rawsz = rdexact(4)
         sz, = unpack("I", rawsz)
         qdata = rdexact(sz)
@@ -617,36 +620,6 @@ def read_legacy_stream(vm):
         return 2
     return 0
 
-def open_file_or_fd(val, mode):
-    """
-    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
-    open it as a regular file.
-    """
-
-    fd = -1
-    try:
-        # Does it look like an integer?
-        try:
-            fd = int(val, 10)
-        except ValueError:
-            pass
-
-        # Try to open it...
-        if fd != -1:
-            return os.fdopen(fd, mode, 0)
-        else:
-            return open(val, mode, 0)
-
-    except StandardError, e:
-        if fd != -1:
-            err("Unable to open fd %d: %s: %s" %
-                (fd, e.__class__.__name__, e))
-        else:
-            err("Unable to open file '%s': %s: %s" %
-                (val, e.__class__.__name__, e))
-
-    raise SystemExit(1)
-
 
 def main():
     from optparse import OptionParser
@@ -723,7 +696,7 @@ def main():
 if __name__ == "__main__":
     try:
         sys.exit(main())
-    except SystemExit, e:
+    except SystemExit as e:
         sys.exit(e.code)
     except KeyboardInterrupt:
         sys.exit(1)
tools/python/scripts/verify-stream-v2 +8 -35
index 3daf25791e..8355c2d206 100755
@@ -3,12 +3,15 @@
 
 """ Verify a v2 format migration stream """
 
+from __future__ import print_function
+
 import sys
 import struct
 import os, os.path
 import syslog
 import traceback
 
+from xen.util import open_file_or_fd as open_file_or_fd
 from xen.migration.verify import StreamError, RecordError
 from xen.migration.libxc import VerifyLibxc
 from xen.migration.libxl import VerifyLibxl
@@ -25,7 +28,7 @@ def info(msg):
             for line in msg.split("\n"):
                 syslog.syslog(syslog.LOG_INFO, line)
         else:
-            print msg
+            print(msg)
 
 def err(msg):
     """Error message, routed to appropriate destination"""
@@ -33,7 +36,7 @@ def err(msg):
         if log_to_syslog:
             for line in msg.split("\n"):
                 syslog.syslog(syslog.LOG_ERR, line)
-        print >> sys.stderr, msg
+        print(msg, file = sys.stderr)
 
 def stream_read(_ = None):
     """Read from input"""
@@ -56,7 +59,7 @@ def skip_xl_header():
     """Skip over an xl header in the stream"""
 
     hdr = rdexact(32)
-    if hdr != "Xen saved domain, xl format\n \0 \r":
+    if hdr != b"Xen saved domain, xl format\n \0 \r":
         raise StreamError("No xl header")
 
     _, mflags, _, optlen = unpack_exact("=IIII")
@@ -86,7 +89,7 @@ def read_stream(fmt):
         err(traceback.format_exc())
         return 1
 
-    except StandardError:
+    except Exception:
         err("Script Error:")
         err(traceback.format_exc())
         err("Please fix me")
@@ -94,36 +97,6 @@ def read_stream(fmt):
 
     return 0
 
-def open_file_or_fd(val, mode, buffering):
-    """
-    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
-    open it as a regular file.
-    """
-
-    fd = -1
-    try:
-        # Does it look like an integer?
-        try:
-            fd = int(val, 10)
-        except ValueError:
-            pass
-
-        # Try to open it...
-        if fd != -1:
-            return os.fdopen(fd, mode, buffering)
-        else:
-            return open(val, mode, buffering)
-
-    except StandardError, e:
-        if fd != -1:
-            err("Unable to open fd %d: %s: %s" %
-                (fd, e.__class__.__name__, e))
-        else:
-            err("Unable to open file '%s': %s: %s" %
-                (val, e.__class__.__name__, e))
-
-    raise SystemExit(2)
-
 def main():
     """ main """
     from optparse import OptionParser
@@ -168,7 +141,7 @@ def main():
 if __name__ == "__main__":
     try:
         sys.exit(main())
-    except SystemExit, e:
+    except SystemExit as e:
         sys.exit(e.code)
     except KeyboardInterrupt:
         sys.exit(2)
tools/python/xen/migration/libxc.py +1 -1
index f24448a9ef..cbffd1975e 100644
@@ -223,7 +223,7 @@ def verify_record(self):
             self.squashed_pagedata_records += 1
 
         padding = content[length:]
-        if padding != "\x00" * len(padding):
+        if padding != b"\x00" * len(padding):
             raise StreamError("Padding containing non0 bytes found")
 
         if rtype not in record_verifiers:
tools/python/xen/migration/libxl.py +1 -1
index d5f54dc489..79f4024e72 100644
@@ -128,7 +128,7 @@ def verify_record(self):
         content = self.rdexact(contentsz)
 
         padding = content[length:]
-        if padding != "\x00" * len(padding):
+        if padding != b"\x00" * len(padding):
             raise StreamError("Padding containing non0 bytes found")
 
         if rtype not in record_verifiers:
tools/python/xen/migration/verify.py +2 -2
index 7a42dbfc58..1e38f4a3c0 100644
@@ -7,11 +7,11 @@
 
 from struct import calcsize, unpack
 
-class StreamError(StandardError):
+class StreamError(Exception):
     """Error with the stream"""
     pass
 
-class RecordError(StandardError):
+class RecordError(Exception):
     """Error with a record in the stream"""
     pass
 
tools/python/xen/util.py new file +23 -0
new file mode 100644
index 0000000000..a11358eefa
@@ -0,0 +1,23 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import os
+
+def open_file_or_fd(val, *argl, **kwargs):
+    """
+    If 'val' looks like a decimal integer, open it as an fd.  If not, try to
+    open it as a regular file.
+    """
+
+    fd = -1
+    try:
+        # Does it look like an integer?
+        fd = int(val, 10)
+    except ValueError:
+        pass
+
+    # Try to open it...
+    if fd != -1:
+        return os.fdopen(fd, *argl, **kwargs)
+    else:
+        return open(val, *argl, **kwargs)
Signature · 5 lines
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab
A: Because it messes up the order in which people normally read text.
Q: Why is top-posting such a bad thing?

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#2 lore raw
Quoted 7 lines · Marek Marczykowski-Górecki
Quoted 7 lines
tools/python/scripts/convert-legacy-stream +1 -1
@@ -70,7 +73,7 @@ class VM(object):
 
         # libxl
         self.libxl = fmt == "libxl"
-        self.emu_xenstore = "" # NUL terminated key&val pairs from "toolstack" records
+        self.emu_xenstore = b"" # NUL terminated key&val pairs from "toolstack" records
 
Quoted 6 lines

tools/python/scripts/convert-legacy-stream class VM(object): · line 79

Quoted 2 lines
tools/python/scripts/convert-legacy-stream +0 -0
 def write_libxc_ihdr():
     stream_write(pack(libxc.IHDR_FORMAT,

You also need to update write_record (string constants). And few calls to it with string constants (write_libxl_end, write_libxl_libxc_context, read_pv_tail, read_hvm_tail). And blkid == ... in read_pv_extended_info().

Urgh - well spotted.

Was this manual inspection, or something else?  (I probably should complete and upstream write-legacy-stream for the purpose of dev-testing the convert-legacy-stream script now that 4.6 is waaay in the past.)

~Andrew

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel

#3 lore raw
Quoted 15 lines · Andrew Cooper
Quoted 7 lines · Marek Marczykowski-Górecki
Quoted 7 lines
tools/python/scripts/convert-legacy-stream +1 -1
@@ -70,7 +73,7 @@ class VM(object):
 
         # libxl
         self.libxl = fmt == "libxl"
-        self.emu_xenstore = "" # NUL terminated key&val pairs from "toolstack" records
+        self.emu_xenstore = b"" # NUL terminated key&val pairs from "toolstack" records
 
Quoted 6 lines

tools/python/scripts/convert-legacy-stream class VM(object): · line 79

Quoted 2 lines
tools/python/scripts/convert-legacy-stream +0 -0
 def write_libxc_ihdr():
     stream_write(pack(libxc.IHDR_FORMAT,

You also need to update write_record (string constants). And few calls to it with string constants (write_libxl_end, write_libxl_libxc_context, read_pv_tail, read_hvm_tail). And blkid == ... in read_pv_extended_info().

Quoted 3 lines

Urgh - well spotted.

Was this manual inspection, or something else?

Manual search for " and '.

Quoted 5 lines

(I probably should
complete and upstream write-legacy-stream for the purpose of dev-testing the convert-legacy-stream script now that 4.6 is waaay in the past.)

~Andrew

Signature · 5 lines
Best Regards,
Marek Marczykowski-Górecki
Invisible Things Lab
A: Because it messes up the order in which people normally read text.
Q: Why is top-posting such a bad thing?

_______________________________________________
Xen-devel mailing list
[email protected]
https://lists.xenproject.org/mailman/listinfo/xen-devel