scripts: Support building with Python 3
PATCH 1/1 scripts: Support building with Python 3
This allows building with "./configure --python=python3", where the python3 program is at least version 3.6. It preserves compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modules
---
Hi,
I've been applying these changes when building on Fedora 26, which does not include any Python 2 packages by default. It was tested with Python 2.7 and 3.6.
I just saw the list of scripts that need updating on the mailing list, and this doesn't cover all of them, but it is enough to build a binary for running virtual machines with KVM. Maybe it is still useful as a starting point.
Thanks.
David
configure | 6 ++++-- scripts/qapi.py | 31 ++++++++++++++++++++----------- scripts/qapi2texi.py | 10 +++++----- scripts/signrom.py | 4 ++-- 4 files changed, 31 insertions(+), 20 deletions(-)
configure +4 -2
@@ -1548,9 +1548,11 @@ fi # Note that if the Python conditional here evaluates True we will exit # with status 1 which is a shell 'false' value. -if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then +if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then + error_exit "Cannot use '$python', Python 3.6 or later is required." \ + "Use --python=/path/to/python3 to specify a supported Python 3." +elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then error_exit "Cannot use '$python', Python 2.6 or later is required." \ - "Note that Python 3 or later is not yet supported." \ "Use --python=/path/to/python to specify a supported Python." fi
scripts/qapi.py +20 -11
@@ -15,9 +15,11 @@ import errno import getopt import os import re -import string import sys -from ordereddict import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict builtin_types = { 'null': 'QTYPE_QNULL',
@@ -252,7 +254,7 @@ class QAPIDoc(object): "'Returns:' is only valid for commands") def check(self): - bogus = [name for name, section in self.args.iteritems() + bogus = [name for name, section in self.args.items() if not section.member] if bogus: raise QAPISemError(
@@ -308,7 +310,7 @@ class QAPISchemaParser(object): if not isinstance(pragma, dict): raise QAPISemError( info, "Value of 'pragma' must be a dictionary") - for name, value in pragma.iteritems(): + for name, value in pragma.items(): self._pragma(name, value, info) else: expr_elem = {'expr': expr,
@@ -1574,7 +1576,7 @@ class QAPISchema(object): def _make_members(self, data, info): return [self._make_member(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] def _def_struct_type(self, expr, info, doc): name = expr['struct']
@@ -1606,11 +1608,11 @@ class QAPISchema(object): name, info, doc, 'base', self._make_members(base, info))) if tag_name: variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] members = [] else: variants = [self._make_simple_variant(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] typ = self._make_implicit_enum_type(name, info, [v.name for v in variants]) tag_member = QAPISchemaObjectTypeMember('type', typ, False)
@@ -1625,7 +1627,7 @@ class QAPISchema(object): name = expr['alternate'] data = expr['data'] variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] tag_member = QAPISchemaObjectTypeMember('type', 'QType', False) self._def_entity( QAPISchemaAlternateType(name, info, doc,
@@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None): type_name = prefix return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() -c_name_trans = string.maketrans('.-', '__') +try: + c_name_trans = str.maketrans('.-', '__') +except AttributeError: + import string + c_name_trans = string.maketrans('.-', '__') # Map @name to a valid C identifier.
@@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, if really: return open(name, opt) else: - import StringIO - return StringIO.StringIO() + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + return StringIO() fdef = maybe_open(do_c, c_file, 'w') fdecl = maybe_open(do_h, h_file, 'w')
scripts/qapi2texi.py +5 -5
@@ -147,7 +147,7 @@ def texi_member(member, suffix=''): def texi_members(doc, what, base, variants, member_func): """Format the table of members""" items = '' - for section in doc.args.itervalues(): + for section in doc.args.values(): # TODO Drop fallbacks when undocumented members are outlawed if section.content: desc = texi_format(str(section))
@@ -285,15 +285,15 @@ def texi_schema(schema): def main(argv): """Takes schema argument, prints result to stdout""" if len(argv) != 2: - print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0] + sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0]) sys.exit(1) schema = qapi.QAPISchema(argv[1]) if not qapi.doc_required: - print >>sys.stderr, ("%s: need pragma 'doc-required' " - "to generate documentation" % argv[0]) + sys.stderr.write("%s: need pragma 'doc-required' " + "to generate documentation\n" % argv[0]) sys.exit(1) - print texi_schema(schema) + sys.stdout.write(texi_schema(schema) + "\n") if __name__ == '__main__':
scripts/signrom.py +2 -2
@@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb') fout = open(sys.argv[2], 'wb') magic = fin.read(2) -if magic != '\x55\xaa': +if magic != b'\x55\xaa': sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1]) size_byte = ord(fin.read(1))
@@ -33,7 +33,7 @@ elif len(data) < size: # Add padding if necessary, rounding the whole input to a multiple of # 512 bytes according to the third byte of the input. # size-1 because a final byte is added below to store the checksum. - data = data.ljust(size-1, '\0') + data = data.ljust(size-1, b'\0') else: if ord(data[-1:]) != 0: sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
Signature · 1 line
2.13.5
Quoted 41 lines · David Michael
This allows building with "./configure --python=python3", where the python3 program is at least version 3.6. It preserves compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modulesSigned-off-by David Michael <[email protected]>---
Hi,
I've been applying these changes when building on Fedora 26, which does not include any Python 2 packages by default. It was tested with Python 2.7 and 3.6.
I just saw the list of scripts that need updating on the mailing list, and this doesn't cover all of them, but it is enough to build a binary for running virtual machines with KVM. Maybe it is still useful as a starting point.
Thanks.
David
configure | 6 ++++-- scripts/qapi.py | 31 ++++++++++++++++++++----------- scripts/qapi2texi.py | 10 +++++----- scripts/signrom.py | 4 ++-- 4 files changed, 31 insertions(+), 20 deletions(-)configure +0
-0@@ -1548,9 +1548,11 @@ fi # Note that if the Python conditional here evaluates True we will exit # with status 1 which is a shell 'false' value.
configure fi · line 1558
Quoted 10 lines
configure +4
-2-if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then +if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then + error_exit "Cannot use '$python', Python 3.6 or later is required." \ + "Use --python=/path/to/python3 to specify a supported Python 3." +elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then error_exit "Cannot use '$python', Python 2.6 or later is required." \ - "Note that Python 3 or later is not yet supported." \ "Use --python=/path/to/python to specify a supported Python." fi
Hmm. If we detect a Python 2 that is too old, we ask the user to provide a sufficiently new Python 2. We don't ask him for a sufficiently new Python 3, even though that would work, too. Should we tweak the error message a bit?
Quoted 19 lines
scripts/qapi.py +4
-2@@ -15,9 +15,11 @@ import errno import getopt import os import re -import string import sys -from ordereddict import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict builtin_types = { 'null': 'QTYPE_QNULL',
scripts/qapi.py class QAPIDoc(object): · line 260
Quoted 8 lines
scripts/qapi.py +1
-1@@ -252,7 +254,7 @@ class QAPIDoc(object): "'Returns:' is only valid for commands") def check(self): - bogus = [name for name, section in self.args.iteritems() + bogus = [name for name, section in self.args.items() if not section.member] if bogus: raise QAPISemError(
Efficiency loss with Python 2. Not critical, just sad.
Quoted 115 lines
scripts/qapi.py +15
-8@@ -308,7 +310,7 @@ class QAPISchemaParser(object): if not isinstance(pragma, dict): raise QAPISemError( info, "Value of 'pragma' must be a dictionary") - for name, value in pragma.iteritems(): + for name, value in pragma.items(): self._pragma(name, value, info) else: expr_elem = {'expr': expr,@@ -1574,7 +1576,7 @@ class QAPISchema(object): def _make_members(self, data, info): return [self._make_member(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] def _def_struct_type(self, expr, info, doc): name = expr['struct']@@ -1606,11 +1608,11 @@ class QAPISchema(object): name, info, doc, 'base', self._make_members(base, info))) if tag_name: variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] members = [] else: variants = [self._make_simple_variant(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] typ = self._make_implicit_enum_type(name, info, [v.name for v in variants]) tag_member = QAPISchemaObjectTypeMember('type', typ, False)@@ -1625,7 +1627,7 @@ class QAPISchema(object): name = expr['alternate'] data = expr['data'] variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] tag_member = QAPISchemaObjectTypeMember('type', 'QType', False) self._def_entity( QAPISchemaAlternateType(name, info, doc,@@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None): type_name = prefix return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() -c_name_trans = string.maketrans('.-', '__') +try: + c_name_trans = str.maketrans('.-', '__') +except AttributeError: + import string + c_name_trans = string.maketrans('.-', '__') # Map @name to a valid C identifier.@@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, if really: return open(name, opt) else: - import StringIO - return StringIO.StringIO() + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + return StringIO() fdef = maybe_open(do_c, c_file, 'w') fdecl = maybe_open(do_h, h_file, 'w')scripts/qapi2texi.py +5
-5@@ -147,7 +147,7 @@ def texi_member(member, suffix=''): def texi_members(doc, what, base, variants, member_func): """Format the table of members""" items = '' - for section in doc.args.itervalues(): + for section in doc.args.values(): # TODO Drop fallbacks when undocumented members are outlawed if section.content: desc = texi_format(str(section))@@ -285,15 +285,15 @@ def texi_schema(schema): def main(argv): """Takes schema argument, prints result to stdout""" if len(argv) != 2: - print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0] + sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0]) sys.exit(1) schema = qapi.QAPISchema(argv[1]) if not qapi.doc_required: - print >>sys.stderr, ("%s: need pragma 'doc-required' " - "to generate documentation" % argv[0]) + sys.stderr.write("%s: need pragma 'doc-required' " + "to generate documentation\n" % argv[0]) sys.exit(1) - print texi_schema(schema) + sys.stdout.write(texi_schema(schema) + "\n") if __name__ == '__main__':scripts/signrom.py +1
-1@@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb') fout = open(sys.argv[2], 'wb') magic = fin.read(2) -if magic != '\x55\xaa': +if magic != b'\x55\xaa': sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1]) size_byte = ord(fin.read(1))
scripts/signrom.py elif len(data) < size: · line 39
Quoted 8 lines
scripts/signrom.py +1
-1@@ -33,7 +33,7 @@ elif len(data) < size: # Add padding if necessary, rounding the whole input to a multiple of # 512 bytes according to the third byte of the input. # size-1 because a final byte is added below to store the checksum. - data = data.ljust(size-1, '\0') + data = data.ljust(size-1, b'\0') else: if ord(data[-1:]) != 0: sys.stderr.write('WARNING: ROM includes nonzero checksum\n')
What is our Python 2 -> 3 migration strategy?
Don't support Python 3 until a flag day, then flip and don't support Python 2?
Keep the source in Python 2 and support 3 via conversion with 2to3?
Port the source to Python 3 and support 2 via conversion with 3to2?
Port to Python 3, but keep it working with Python 2 with help of the six module?
Support both with ad hoc hackery (like this patch does)?
I think we should make up our minds before we hack up things too much.
Quoted 4 lines · Markus Armbruster
What is our Python 2 -> 3 migration strategy?
Don't support Python 3 until a flag day, then flip and don't support Python 2?
Add support for Python 3 so that both Python 2.6+ and Python 3 are supported.
When Python 2 is EOL it will need to be dropped and the code becomes Python 3-only.
Quoted 8 lines
Keep the source in Python 2 and support 3 via conversion with 2to3?
Port the source to Python 3 and support 2 via conversion with 3to2?
Port to Python 3, but keep it working with Python 2 with help of the six module?
Support both with ad hoc hackery (like this patch does)?
Yes, please see for details:
https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg01758.html
Stefan
Quoted 25 lines · David Michael
This allows building with "./configure --python=python3", where the python3 program is at least version 3.6. It preserves compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modulesSigned-off-by David Michael <[email protected]>---
Hi,
I've been applying these changes when building on Fedora 26, which does not include any Python 2 packages by default. It was tested with Python 2.7 and 3.6.
I just saw the list of scripts that need updating on the mailing list, and this doesn't cover all of them, but it is enough to build a binary for running virtual machines with KVM. Maybe it is still useful as a starting point.
Thanks.
Quoted 7 lines
David
configure | 6 ++++-- scripts/qapi.py | 31 ++++++++++++++++++++----------- scripts/qapi2texi.py | 10 +++++----- scripts/signrom.py | 4 ++-- 4 files changed, 31 insertions(+), 20 deletions(-)
Cool, thanks for contributing this!
Quoted 8 lines · Stefan Hajnoczi
Quoted 4 lines · Markus Armbruster
What is our Python 2 -> 3 migration strategy?
Don't support Python 3 until a flag day, then flip and don't support Python 2?
Add support for Python 3 so that both Python 2.6+ and Python 3 are supported.
Python 2.7 is the end of the Python 2 line of development (PEP 404). It'll be maintained until 2020. Downstream support for Python 2 may or may not last longer.
Quoted 7 lines
When Python 2 is EOL it will need to be dropped and the code becomes Python 3-only.
Quoted 4 lines
Keep the source in Python 2 and support 3 via conversion with 2to3?
Port the source to Python 3 and support 2 via conversion with 3to2?
Quoted 7 lines
Quoted 4 lines
Port to Python 3, but keep it working with Python 2 with help of the six module?
Support both with ad hoc hackery (like this patch does)?
Yes, please see for details:
https://lists.gnu.org/archive/html/qemu-devel/2017-08/msg01758.html
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
Quoted 10 lines · Markus Armbruster
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
My take is that we have the following goals we want to achieve:
(1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6)
(2) We need to be able to build and run on newer distros that have dropped Python 2 altogether in favour of Python 3
(I don't know if there are any such today, but presumably by 2020 there will be)
Unless we can confidently say that either (1) or (2) is the empty set, we need to handle both 2 and 3 in the same codebase. This is a pain, but unfortunately Python upstream have forced us into it by breaking source code compatibility.
I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history".
thanks
-- PMM
Quoted 12 lines · Peter Maydell
Quoted 10 lines · Markus Armbruster
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
Quoted 8 lines
My take is that we have the following goals we want to achieve:
(1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6)
(2) We need to be able to build and run on newer distros that have dropped Python 2 altogether in favour of Python 3
(I don't know if there are any such today, but presumably by 2020 there will be)
Fedora has dropped Python 2 in the default install now, so needs to be manually pulled in. While python 2 won't be dropped entirely in the near future, its days are definitely numbered and so once upstream officially dedclares it dead, I'd expect Fedora to follow.
Quoted 9 lines
Unless we can confidently say that either (1) or (2) is the empty set, we need to handle both 2 and 3 in the same codebase. This is a pain, but unfortunately Python upstream have forced us into it by breaking source code compatibility.
I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history".
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Quoted 1 line
From QEMU pov, I don't think supporting py2 and py3 at the same time
is actually as bad as you might expect from reading about the pain in the broader community. We're lucky that pretty much all our python code is self-contained not relying on 3rd party modules, and it doesn't use particularly advanced features of python either.
I worked on OpenStack Nova during the conversion to support py2 & py3 in parallel, and the resulting code was not really any harder to maintain than the original code. The hard bit was Nova's use of 3rd party modules whose maintainers hadn't converted.
So, IMHO, provided we make sure our CI system is giving us test coverage across the various py2 & py3 versions, the burden on us will not be significant. Writing code that's py2 & py3 compatible is not hard as long as you have good testing. So we just have a lump for the initial conversion, then CI will take care of avoiding regressions going forward.
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 3 lines · Daniel P. Berrange
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020...
thanks
-- PMM
Quoted 7 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020...
Who is actually requiring that we support RHEL6 until EOL though ?
AFAIK, from Red Hat side, we don't need QEMU git master to be buildable on RHEL-6 machines anymore, since QEMU there is long since in bug-fix only mode, no rebases or feature backports.
I assume there's probably some community interest in building git master on RHEL-6, but I think it is reasonable for us to say no at some point, rather than waiting until final EOL (2020 for normal subscribers, 2024 for people who paid for extended lifetime).
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 6 lines · Daniel P. Berrange
Quoted 5 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Quoted 8 lines
Quoted 2 lines
Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020...
Who is actually requiring that we support RHEL6 until EOL though ?
AFAIK, from Red Hat side, we don't need QEMU git master to be buildable on RHEL-6 machines anymore, since QEMU there is long since in bug-fix only mode, no rebases or feature backports.
Even in the unlikely case that a bug fix involves backporting some Python, having to port just those bits to RHEL-6's version of Python would be less work than making *all* upstream Python code work with both Python dialects.
Quoted 4 lines
I assume there's probably some community interest in building git master on RHEL-6, but I think it is reasonable for us to say no at some point, rather than waiting until final EOL (2020 for normal subscribers, 2024 for people who paid for extended lifetime).
"Interest" by itself doesn't pay bills or write code.
If you have to have upstream QEMU on RHEL-6, and you're fine with getting it by building it yourself (there being no other way), then you should also be fine with getting its build requirements. Python 3.4 is in EPEL.
Quoted 10 lines · Daniel P. Berrange
Quoted 7 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020...
Who is actually requiring that we support RHEL6 until EOL though ?
OK, if we don't need to retain RHEL6 support that would help (we should in that case not bother pulling in that big lump of argument parsing code that just went past in a patchset purely for the benefit of 2.6 support). How are the other LTS distros for Python versions?
thanks
-- PMM
Quoted 9 lines · Peter Maydell
Quoted 8 lines · Markus Armbruster
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Quoted 7 lines
Quoted 2 lines
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
My take is that we have the following goals we want to achieve:
(1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6)
That's an assertion, not an answer to my question :)
What exactly are we hoping to accomplish by supporting what distros exactly?
Quoted 4 lines
(2) We need to be able to build and run on newer distros that have dropped Python 2 altogether in favour of Python 3 (I don't know if there are any such today, but presumably by 2020 there will be)
Yup.
Quoted 9 lines
Unless we can confidently say that either (1) or (2) is the empty set, we need to handle both 2 and 3 in the same codebase. This is a pain, but unfortunately Python upstream have forced us into it by breaking source code compatibility.
I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history".
I don't buy the "clearly" in "pretty clearly not (yet) an empty set", because I don't understand *why* we need to support "older (long-term-support) distros".
If we can leave Python 2 behind, I'm all for porting to Python 3 now.
If we can't yet, then I don't see why I should do work *now* just because "some" distro might drop support for Python 2 "before all the LTS 2-only distro versions vanish into history".
If it ain't broke, don't fix it!
Quoted 45 lines · Markus Armbruster
Quoted 9 lines · Peter Maydell
Quoted 8 lines · Markus Armbruster
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Quoted 7 lines
Quoted 2 lines
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
My take is that we have the following goals we want to achieve:
(1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6)
That's an assertion, not an answer to my question :)
What exactly are we hoping to accomplish by supporting what distros exactly?
Quoted 4 lines
(2) We need to be able to build and run on newer distros that have dropped Python 2 altogether in favour of Python 3 (I don't know if there are any such today, but presumably by 2020 there will be)
Yup.
Quoted 9 lines
Unless we can confidently say that either (1) or (2) is the empty set, we need to handle both 2 and 3 in the same codebase. This is a pain, but unfortunately Python upstream have forced us into it by breaking source code compatibility.
I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history".
I don't buy the "clearly" in "pretty clearly not (yet) an empty set", because I don't understand *why* we need to support "older (long-term-support) distros".
Quoted 5 lines
If we can leave Python 2 behind, I'm all for porting to Python 3 now.
If we can't yet, then I don't see why I should do work *now* just because "some" distro might drop support for Python 2 "before all the LTS 2-only distro versions vanish into history".
As mentioned before, while for many projects, maintaining py2 & py3 in parallel is hard, I don't think that is going to be the case for QEMU due to the simplicity of much of our python usage. IOW, for QEMU I tend to view this as more like the work we do to fix up compiler warnings when new GCC comes out. There's some cleanup needed, but the result is rarely harder to maintain.
It is fine if you don't want to do the work to convert QAPI code to support py3, but if some other contributor submits patches for parallel py2 & py3 support that don't appreciably harm maintainability I see little reason to reject them.
Quoted 1 line
If it ain't broke, don't fix it!
NB, some distros have already changed /usr/bin/python to point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros.
In theory you could change the shebang to #!/usr/bin/python2, but the /usr/bin/python2 file symlink is not standard in py2.6 - though certain distros have backported support for it.
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 3 lines · Daniel P. Berrange
NB, some distros have already changed
/usr/bin/pythonto point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros.
This is a bug in those distros and they should fix it. Python 3 is not Python 2 and making /usr/bin/python point to the wrong interpreter will have predictably bad results for their users...
thanks
-- PMM
Quoted 27 lines · Markus Armbruster
Quoted 6 lines · Daniel P. Berrange
Quoted 5 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
If we can update to python 2.7 as our minimum, then supporting py2 and py3 gets simpler, avoiding some of the nastier hacks, even without that though it isn't too hard.
Quoted 8 lines
Quoted 2 lines
Unfortunately RHEL6 is what's holding us to retaining 2.6 support, and googling suggests that doesn't go EOL until 2020...
Who is actually requiring that we support RHEL6 until EOL though ?
AFAIK, from Red Hat side, we don't need QEMU git master to be buildable on RHEL-6 machines anymore, since QEMU there is long since in bug-fix only mode, no rebases or feature backports.
Even in the unlikely case that a bug fix involves backporting some Python, having to port just those bits to RHEL-6's version of Python would be less work than making *all* upstream Python code work with both Python dialects.
Quoted 4 lines
I assume there's probably some community interest in building git master on RHEL-6, but I think it is reasonable for us to say no at some point, rather than waiting until final EOL (2020 for normal subscribers, 2024 for people who paid for extended lifetime).
Quoted 6 lines
"Interest" by itself doesn't pay bills or write code.
If you have to have upstream QEMU on RHEL-6, and you're fine with getting it by building it yourself (there being no other way), then you should also be fine with getting its build requirements. Python 3.4 is in EPEL.
That sounds like a good argument (to ditch Python 2.6) to me -- EL-6 users getting it from EPEL[*], one of its purposes is to facilate cases exactly like the one above.
[*] https://fedoraproject.org/wiki/EPEL
Signature · 1 line
/kashyap
Quoted 8 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
NB, some distros have already changed
/usr/bin/pythonto point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros.This is a bug in those distros and they should fix it. Python 3 is not Python 2 and making
/usr/bin/pythonpoint to the wrong interpreter will have predictably bad results for their users...
Having /usr/bin/python point to py3 is what upstream Python will eventually recommend. Some distros jumped a bit early, but the others will eventually follow, and I don't see those early distros moving back.
https://www.python.org/dev/peps/pep-0394/
"* python2 will refer to some version of Python 2.x.
* python3 will refer to some version of Python 3.x.
* for the time being, all distributions should ensure that python refers to the same target as python2.
* however, end users should be aware that python refers to python3 on at least Arch Linux (that change is what prompted the creation of this PEP), so python should be used in the shebang line only for scripts that are source compatible with both Python 2 and 3.
* in preparation for an eventual change in the default version of Python, Python 2 only scripts should either be updated to be source compatible with Python 3 or else to use python2 in the shebang line."
Regards,
Daniel
Signature · 3 lines
|: https://berrange.com -o- https://www.flickr.com/photos/dberrange :| |: https://libvirt.org -o- https://fstop138.berrange.com :| |: https://entangle-photo.org -o- https://www.instagram.com/dberrange :|
Quoted 16 lines · Markus Armbruster
Quoted 14 lines · Peter Maydell
Quoted 10 lines · Markus Armbruster
So, first we'll invest in work-arounds to make both 2 and 3 work. Once 2 is gone, we can invest some more to clean them up. Which probably won't happen, so we'll continue to carry work-arounds that no longer make sense.
I maintain roughly one fourth of all Python code in qemu, and I'm not looking forward to this hoop-jumping at all.
Are we really, really sure we want to go this way? What exactly are we hoping to accomplish by it?
My take is that we have the following goals we want to achieve:
Quoted 7 lines
Quoted 2 lines
(1) We need to continue to build and run on older (long-term-support) distros that still ship only Python 2.x (alas even back to 2.6)
That's an assertion, not an answer to my question :)
What exactly are we hoping to accomplish by supporting what distros exactly?
Of my current build-test machines:
OSX is python 2.7
Ubuntu 16.04 (s390) has both 2.7 and 3.5 FreeBSD 11.0 has python 2.7 but I could install 3.3 OpenBSD 6.0 has python 2.7 but I could install 3.5 NetBSD 7.1 has python 2.7 but I could install 3.6 The ppc64 box is Centos 7 which has python 2.7 but
I could ask the admins to install 3.4
The aarch64 box is Ubuntu 14.04 and has 2.7 and 3.4
So OSX is the major laggard in having no official Python 3 at all, but the majority still seem to have their stock python be a version 2, even if a version 3 is optionally available.
Last time we discussed Python versions (in 2015) the three cited reasons for 2.6 were RHEL6, SLES 11, and OSX 10.6. We can maybe move forward to 2.7 (RHEL6 apparently has a 2.7 backport, I think our two or three OSX 10.6 users can cope with installing a 2.7 from homebrew, and I dunno what the SLES11 situation is). I don't think we should move further forward than that yet, I don't see the need.
Quoted 8 lines
Quoted 4 lines
I think (1) is pretty clearly not (yet) an empty set, so the only alternative I see to "support 2 and 3 now" is "keep supporting only 2 for the moment and hope that no distro drops 2 support before all the LTS 2-only distro versions vanish into history".
I don't buy the "clearly" in "pretty clearly not (yet) an empty set", because I don't understand *why* we need to support "older (long-term-support) distros".
I like that you can build QEMU on a lot of distros without having to install from source a whole pile of dependent libraries and tools. I think that's a service to our users (and to the distros that we run on) which we shouldn't give up without a good reason.
thanks
-- PMM
Quoted 10 lines · Daniel P. Berrange
Quoted 8 lines · Peter Maydell
Quoted 3 lines · Daniel P. Berrange
NB, some distros have already changed
/usr/bin/pythonto point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros.This is a bug in those distros and they should fix it. Python 3 is not Python 2 and making
/usr/bin/pythonpoint to the wrong interpreter will have predictably bad results for their users...
Quoted 6 lines
Having
/usr/bin/pythonpoint to py3 is what upstream Python will eventually recommend. Some distros jumped a bit early, but the others will eventually follow, and I don't see those early distros moving back.
That appears to be "Arch Linux broke this, and rather than just telling them to fix this we decided to make everybody else deal with the breakage". Water under the bridge, I guess :-(
thanks
-- PMM
On 2017-08-31 15:14, Daniel P. Berrange wrote:
[...]
Quoted 3 lines
NB, some distros have already changed
/usr/bin/pythonto point to py3 so all the files with #!/usr/bin/python in QEMU are already broken on those distros.
For what it's worth, I (at least used to) build qemu regularly on Arch Linux and the only issue I encountered with this is that the iotests wouldn't work. As a workaround, I made the check script detect that interpreter line and just run the script through the interpreter configured through --python, so it works now...
So from what I could tell, everything worked when you specify --python=python2.
Max
Quoted 18 lines · David Michael
This allows building with "./configure --python=python3", where the python3 program is at least version 3.6. It preserves compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modulesSigned-off-by David Michael <[email protected]>---
Hi,
I've been applying these changes when building on Fedora 26, which does not include any Python 2 packages by default. It was tested with Python 2.7 and 3.6.
Quoted 6 lines
I just saw the list of scripts that need updating on the mailing list, and this doesn't cover all of them, but it is enough to build a binary for running virtual machines with KVM. Maybe it is still useful as a starting point.
Thanks.
Markus,
You wanted to discuss the general direction of Python version support before applying this.
Has consensus been reached?
Stefan
Quoted 181 lines
David
configure | 6 ++++-- scripts/qapi.py | 31 ++++++++++++++++++++----------- scripts/qapi2texi.py | 10 +++++----- scripts/signrom.py | 4 ++-- 4 files changed, 31 insertions(+), 20 deletions(-)configure +4
-2@@ -1548,9 +1548,11 @@ fi # Note that if the Python conditional here evaluates True we will exit # with status 1 which is a shell 'false' value. -if ! $python -c 'import sys; sys.exit(sys.version_info < (2,6) or sys.version_info >= (3,))'; then +if ! $python -c 'import sys; sys.exit(sys.version_info >= (3,) and sys.version_info < (3,6))'; then + error_exit "Cannot use '$python', Python 3.6 or later is required." \ + "Use --python=/path/to/python3 to specify a supported Python 3." +elif ! $python -c 'import sys; sys.exit(sys.version_info < (2,6))'; then error_exit "Cannot use '$python', Python 2.6 or later is required." \ - "Note that Python 3 or later is not yet supported." \ "Use --python=/path/to/python to specify a supported Python." fiscripts/qapi.py +20
-11@@ -15,9 +15,11 @@ import errno import getopt import os import re -import string import sys -from ordereddict import OrderedDict +try: + from collections import OrderedDict +except ImportError: + from ordereddict import OrderedDict builtin_types = { 'null': 'QTYPE_QNULL',@@ -252,7 +254,7 @@ class QAPIDoc(object): "'Returns:' is only valid for commands") def check(self): - bogus = [name for name, section in self.args.iteritems() + bogus = [name for name, section in self.args.items() if not section.member] if bogus: raise QAPISemError(@@ -308,7 +310,7 @@ class QAPISchemaParser(object): if not isinstance(pragma, dict): raise QAPISemError( info, "Value of 'pragma' must be a dictionary") - for name, value in pragma.iteritems(): + for name, value in pragma.items(): self._pragma(name, value, info) else: expr_elem = {'expr': expr,@@ -1574,7 +1576,7 @@ class QAPISchema(object): def _make_members(self, data, info): return [self._make_member(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] def _def_struct_type(self, expr, info, doc): name = expr['struct']@@ -1606,11 +1608,11 @@ class QAPISchema(object): name, info, doc, 'base', self._make_members(base, info))) if tag_name: variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] members = [] else: variants = [self._make_simple_variant(key, value, info) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] typ = self._make_implicit_enum_type(name, info, [v.name for v in variants]) tag_member = QAPISchemaObjectTypeMember('type', typ, False)@@ -1625,7 +1627,7 @@ class QAPISchema(object): name = expr['alternate'] data = expr['data'] variants = [self._make_variant(key, value) - for (key, value) in data.iteritems()] + for (key, value) in data.items()] tag_member = QAPISchemaObjectTypeMember('type', 'QType', False) self._def_entity( QAPISchemaAlternateType(name, info, doc,@@ -1735,7 +1737,11 @@ def c_enum_const(type_name, const_name, prefix=None): type_name = prefix return camel_to_upper(type_name) + '_' + c_name(const_name, False).upper() -c_name_trans = string.maketrans('.-', '__') +try: + c_name_trans = str.maketrans('.-', '__') +except AttributeError: + import string + c_name_trans = string.maketrans('.-', '__') # Map @name to a valid C identifier.@@ -1997,8 +2003,11 @@ def open_output(output_dir, do_c, do_h, prefix, c_file, h_file, if really: return open(name, opt) else: - import StringIO - return StringIO.StringIO() + try: + from StringIO import StringIO + except ImportError: + from io import StringIO + return StringIO() fdef = maybe_open(do_c, c_file, 'w') fdecl = maybe_open(do_h, h_file, 'w')scripts/qapi2texi.py +5
-5@@ -147,7 +147,7 @@ def texi_member(member, suffix=''): def texi_members(doc, what, base, variants, member_func): """Format the table of members""" items = '' - for section in doc.args.itervalues(): + for section in doc.args.values(): # TODO Drop fallbacks when undocumented members are outlawed if section.content: desc = texi_format(str(section))@@ -285,15 +285,15 @@ def texi_schema(schema): def main(argv): """Takes schema argument, prints result to stdout""" if len(argv) != 2: - print >>sys.stderr, "%s: need exactly 1 argument: SCHEMA" % argv[0] + sys.stderr.write("%s: need exactly 1 argument: SCHEMA\n" % argv[0]) sys.exit(1) schema = qapi.QAPISchema(argv[1]) if not qapi.doc_required: - print >>sys.stderr, ("%s: need pragma 'doc-required' " - "to generate documentation" % argv[0]) + sys.stderr.write("%s: need pragma 'doc-required' " + "to generate documentation\n" % argv[0]) sys.exit(1) - print texi_schema(schema) + sys.stdout.write(texi_schema(schema) + "\n") if __name__ == '__main__':scripts/signrom.py +2
-2@@ -18,7 +18,7 @@ fin = open(sys.argv[1], 'rb') fout = open(sys.argv[2], 'wb') magic = fin.read(2) -if magic != '\x55\xaa': +if magic != b'\x55\xaa': sys.exit("%s: option ROM does not begin with magic 55 aa" % sys.argv[1]) size_byte = ord(fin.read(1))@@ -33,7 +33,7 @@ elif len(data) < size: # Add padding if necessary, rounding the whole input to a multiple of # 512 bytes according to the third byte of the input. # size-1 because a final byte is added below to store the checksum. - data = data.ljust(size-1, '\0') + data = data.ljust(size-1, b'\0') else: if ord(data[-1:]) != 0: sys.stderr.write('WARNING: ROM includes nonzero checksum\n')Signature · 1 line
2.13.5
Quoted 24 lines · Stefan Hajnoczi
Quoted 23 lines · David Michael
This allows building with "./configure --python=python3", where the python3 program is at least version 3.6. It preserves compatibility with Python 2. The changes include:
- Avoiding "print" usage
- Using bytes with files opened in binary mode
- Switching .iteritems() to .items()
- Adding fallback imports for functions moved to other modulesSigned-off-by David Michael <[email protected]>---
Hi,
I've been applying these changes when building on Fedora 26, which does not include any Python 2 packages by default. It was tested with Python 2.7 and 3.6.
I just saw the list of scripts that need updating on the mailing list, and this doesn't cover all of them, but it is enough to build a binary for running virtual machines with KVM. Maybe it is still useful as a starting point.
Quoted 6 lines
Quoted 1 line
Thanks.
Markus,
You wanted to discuss the general direction of Python version support before applying this.
Dan has since posted competing patches.
Quoted 1 line
Has consensus been reached?
I need to decide how much I hate supporting both Python 2 and Python 3, and that involves looking at both patch sets in more detail.
QAPI is a rather busy subsystem, and right now my review queue is in a state where I have to decide whom to annoy and disappoint with overly long delays.
Given the choice between addressing a problem that doesn't really exist, yet (Python 2 going away) and addressing problems that are hurting us *now* (such as QAPI's inability to express compile-time conditionals), I have to prioritize the latter. I wish I wouldn't have to make such choices, and I apologize for the inconvenience.