make fio scripts python3-ready

Many distributions are moving to python3 by default. Here's an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas were hand fixed: remove superfluous parentheses introduced by 2to3 converter in print function calls, shebang modifications to use environment variable for python version, and byte-string decode correction in steadystate_tests.py following 2to3 conversion.

The modified scripts pass rudimentary testing when run under python2.7 as well as python3.

Signed-off-by Bill O'Donnell <[email protected]>

---
 doc/conf.py                     |  3 +++
 tools/fiologparser.py           |  6 +++++-
 tools/hist/fiologparser_hist.py | 17 +++++++++++------
 unit_tests/steadystate_tests.py | 21 ++++++++++++---------
 4 files changed, 31 insertions(+), 16 deletions(-)

doc/conf.py +3 -0

index d4dd9d20..087a9a11 100644
@@ -22,6 +22,9 @@
 
 # -- General configuration ------------------------------------------------
 
+from __future__ import absolute_import
+from __future__ import print_function
+
 # If your documentation needs a minimal Sphinx version, state it here.
 #
 # needs_sphinx = '1.0'

tools/fiologparser.py +0 -0

index 8549859f..14ddc417 100755

tools/hist/fiologparser_hist.py +11 -6

@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/env python
 """
     Utility for converting *_clat_hist* files generated by fio into latency statistics.
 

@@ -13,11 +13,16 @@
 
     @author Karl Cronburg <[email protected]>
 """
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import pandas
 import re
 import numpy as np
+from six.moves import map
+from six.moves import range
+from six.moves import zip
 
 runascmd = False
 

@@ -124,7 +129,7 @@ def gen_output_columns(ctx):
         columns = ["end-time", "dir", "samples", "min", "avg", "median"]
     else:
         columns = ["end-time", "samples", "min", "avg", "median"]
-    columns.extend(list(map(lambda x: x+'%', strpercs)))
+    columns.extend(list([x+'%' for x in strpercs]))
     columns.append("max")
 
 def fmt_float_list(ctx, num=1):

@@ -254,7 +259,7 @@ def print_all_stats(ctx, end, mn, ss_cnt, vs, ws, mx, dir=dir):
         # max and min are decimal values if no divisor
         fmt = fmt + "%d, " + fmt_float_list(ctx, len(percs)+1) + ", %d"
 
-    print (fmt % tuple(row))
+    print(fmt % tuple(row))
 
 def update_extreme(val, fncn, new_val):
     """ Calculate min / max in the presence of None values """

@@ -338,8 +343,8 @@ def guess_max_from_bins(ctx, hist_cols):
         bins = [ctx.group_nr * (1 << 6)]
     else:
         bins = [1216,1280,1344,1408,1472,1536,1600,1664]
-    coarses = range(max_coarse + 1)
-    fncn = lambda z: list(map(lambda x: z/2**x if z % 2**x == 0 else -10, coarses))
+    coarses = list(range(max_coarse + 1))
+    fncn = lambda z: list([z/2**x if z % 2**x == 0 else -10 for x in coarses])
 
     arr = np.transpose(list(map(fncn, bins)))
     idx = np.where(arr == hist_cols)

@@ -473,7 +478,7 @@ def main(ctx):
         try:
             from configparser import SafeConfigParser, NoOptionError
         except ImportError:
-            from ConfigParser import SafeConfigParser, NoOptionError
+            from six.moves.configparser import SafeConfigParser, NoOptionError
 
         cp = SafeConfigParser(allow_no_value=True)
         with open(ctx.job_file, 'r') as fp:

unit_tests/steadystate_tests.py +12 -10

index 5a74f956..8a5189ce 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/env python
 #
 # steadystate_tests.py
 #

@@ -18,6 +18,8 @@
 # if ss attained: min runtime = ss_dur + ss_ramp
 # if not attained: runtime = timeout
 
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import json

@@ -26,11 +28,12 @@ import pprint
 import argparse
 import subprocess
 from scipy import stats
+from six.moves import range
 
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('fio',
-                        help='path to fio executable');
+                        help='path to fio executable')
     parser.add_argument('--read',
                         help='target for read testing')
     parser.add_argument('--write',

@@ -45,7 +48,7 @@ def check(data, iops, slope, pct, limit, dur, criterion):
     data = data[measurement]
     mean = sum(data) / len(data)
     if slope:
-        x = range(len(data))
+        x = list(range(len(data)))
         m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
         m = abs(m)
         if pct:

@@ -89,11 +92,11 @@ if __name__ == '__main__':
                   'output': "set steady state BW threshold to 12" },
               ]
     for test in parsing:
-        output = subprocess.check_output([args.fio] + test['args']);
-        if test['output'] in output:
-            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
+        output = subprocess.check_output([args.fio] + test['args'])
+        if test['output'] in output.decode():
+            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
         else:
-            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
+            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
 
 #
 # test some read workloads

@@ -117,7 +120,7 @@ if __name__ == '__main__':
             args.read = '/dev/zero'
             extra = [ "--size=134217728" ]  # 128 MiB
         else:
-            print "ERROR: file for read testing must be specified on non-posix systems"
+            print("ERROR: file for read testing must be specified on non-posix systems")
             sys.exit(1)
     else:
         extra = []

@@ -216,7 +219,7 @@ if __name__ == '__main__':
                 else:
                     result = 'FAILED '
                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
-            print line
+            print(line)
             if 'steadystate' in jsonjob:
                 pp.pprint(jsonjob['steadystate'])
         jobnum += 1
--

2.14.3

--
To unsubscribe from this list: send the line "unsubscribe fio" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Hi,

Quoted 40 lines · Bill O'Donnell

Many distributions are moving to python3 by default. Here's an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas were hand fixed: remove superfluous parentheses introduced by 2to3 converter in print function calls, shebang modifications to use environment variable for python version, and byte-string decode correction in steadystate_tests.py following 2to3 conversion.

The modified scripts pass rudimentary testing when run under python2.7 as well as python3.

Signed-off-by Bill O'Donnell <[email protected]>

---
 doc/conf.py                     |  3 +++
 tools/fiologparser.py           |  6 +++++-
 tools/hist/fiologparser_hist.py | 17 +++++++++++------
 unit_tests/steadystate_tests.py | 21 ++++++++++++---------
 4 files changed, 31 insertions(+), 16 deletions(-)

doc/conf.py +3 -0

index d4dd9d20..087a9a11 100644
@@ -22,6 +22,9 @@
 
 # -- General configuration ------------------------------------------------
 
+from __future__ import absolute_import
+from __future__ import print_function
+
 # If your documentation needs a minimal Sphinx version, state it here.
 #
 # needs_sphinx = '1.0'

tools/fiologparser.py +0 -0

index 8549859f..14ddc417 100755

tools/fiologparser.py line 1

Quoted 2 lines

tools/fiologparser.py +1 -1

@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/env python

Apparently there's a distro or operating system that discourages or doesn't like every type of python #! line you can come up with (see

https://github.com/axboe/fio/commit/60023ade47e7817db1c18d9b7e511839de5c2c99
). For now let's leave it at #!/usr/bin/python2.7 but add a comment beneath saying that the script actually works with python2 or python3 (also see the discussion in https://www.spinics.net/lists/fio/msg06746.html
, https://www.spinics.net/lists/fio/msg06748.html and
https://www.spinics.net/lists/fio/msg06977.html ). Distros can then use a uniform tool at packaging time to convert all the scripts to whatever their preferred syntax is.

Quoted 8 lines

tools/fiologparser.py +0 -0

 #
 # fiologparser.py
 #

@@ -13,8 +13,12 @@
 #
 # to see per-interval average completion latency.
 

tools/fiologparser.py line 21

Quoted 6 lines

tools/fiologparser.py +4 -0

+from __future__ import absolute_import
+from __future__ import print_function
 import argparse
 import math
+from six.moves import range
+from functools import reduce

I wonder if some of these imports are actually redundant and could be skipped...

tools/hist/fiologparser_hist.py

Quoted 7 lines

tools/fiologparser.py +0 -0

 def parse_args():
     parser = argparse.ArgumentParser()

tools/hist/fiologparser_hist.py +0 -0

index 8910d5fa..958c9141 100755

Isn't this one already python 3 compatible? (see the thread in
https://www.spinics.net/lists/fio/msg06709.html for where Kris[[ converted it).

Quoted 145 lines

tools/hist/fiologparser_hist.py +11 -6

@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/env python
 """
     Utility for converting *_clat_hist* files generated by fio into latency statistics.
 

@@ -13,11 +13,16 @@
 
     @author Karl Cronburg <[email protected]>
 """
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import pandas
 import re
 import numpy as np
+from six.moves import map
+from six.moves import range
+from six.moves import zip
 
 runascmd = False
 

@@ -124,7 +129,7 @@ def gen_output_columns(ctx):
         columns = ["end-time", "dir", "samples", "min", "avg", "median"]
     else:
         columns = ["end-time", "samples", "min", "avg", "median"]
-    columns.extend(list(map(lambda x: x+'%', strpercs)))
+    columns.extend(list([x+'%' for x in strpercs]))
     columns.append("max")
 
 def fmt_float_list(ctx, num=1):

@@ -254,7 +259,7 @@ def print_all_stats(ctx, end, mn, ss_cnt, vs, ws, mx, dir=dir):
         # max and min are decimal values if no divisor
         fmt = fmt + "%d, " + fmt_float_list(ctx, len(percs)+1) + ", %d"
 
-    print (fmt % tuple(row))
+    print(fmt % tuple(row))
 
 def update_extreme(val, fncn, new_val):
     """ Calculate min / max in the presence of None values """

@@ -338,8 +343,8 @@ def guess_max_from_bins(ctx, hist_cols):
         bins = [ctx.group_nr * (1 << 6)]
     else:
         bins = [1216,1280,1344,1408,1472,1536,1600,1664]
-    coarses = range(max_coarse + 1)
-    fncn = lambda z: list(map(lambda x: z/2**x if z % 2**x == 0 else -10, coarses))
+    coarses = list(range(max_coarse + 1))
+    fncn = lambda z: list([z/2**x if z % 2**x == 0 else -10 for x in coarses])
 
     arr = np.transpose(list(map(fncn, bins)))
     idx = np.where(arr == hist_cols)

@@ -473,7 +478,7 @@ def main(ctx):
         try:
             from configparser import SafeConfigParser, NoOptionError
         except ImportError:
-            from ConfigParser import SafeConfigParser, NoOptionError
+            from six.moves.configparser import SafeConfigParser, NoOptionError
 
         cp = SafeConfigParser(allow_no_value=True)
         with open(ctx.job_file, 'r') as fp:

unit_tests/steadystate_tests.py +12 -10

index 5a74f956..8a5189ce 100755
@@ -1,4 +1,4 @@
-#!/usr/bin/python2.7
+#!/usr/bin/env python
 #
 # steadystate_tests.py
 #

@@ -18,6 +18,8 @@
 # if ss attained: min runtime = ss_dur + ss_ramp
 # if not attained: runtime = timeout
 
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import json

@@ -26,11 +28,12 @@ import pprint
 import argparse
 import subprocess
 from scipy import stats
+from six.moves import range
 
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('fio',
-                        help='path to fio executable');
+                        help='path to fio executable')
     parser.add_argument('--read',
                         help='target for read testing')
     parser.add_argument('--write',

@@ -45,7 +48,7 @@ def check(data, iops, slope, pct, limit, dur, criterion):
     data = data[measurement]
     mean = sum(data) / len(data)
     if slope:
-        x = range(len(data))
+        x = list(range(len(data)))
         m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
         m = abs(m)
         if pct:

@@ -89,11 +92,11 @@ if __name__ == '__main__':
                   'output': "set steady state BW threshold to 12" },
               ]
     for test in parsing:
-        output = subprocess.check_output([args.fio] + test['args']);
-        if test['output'] in output:
-            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
+        output = subprocess.check_output([args.fio] + test['args'])
+        if test['output'] in output.decode():
+            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
         else:
-            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
+            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
 
 #
 # test some read workloads

@@ -117,7 +120,7 @@ if __name__ == '__main__':
             args.read = '/dev/zero'
             extra = [ "--size=134217728" ]  # 128 MiB
         else:
-            print "ERROR: file for read testing must be specified on non-posix systems"
+            print("ERROR: file for read testing must be specified on non-posix systems")
             sys.exit(1)
     else:
         extra = []

@@ -216,7 +219,7 @@ if __name__ == '__main__':
                 else:
                     result = 'FAILED '
                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
-            print line
+            print(line)
             if 'steadystate' in jsonjob:
                 pp.pprint(jsonjob['steadystate'])
         jobnum += 1
--

2.14.3

--
To unsubscribe from this list: send the line "unsubscribe fio" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Signature · 1 line

Sitsofe | http://sucs.org/~sits/

Many distributions are moving to python3 by default. Here's an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas were hand fixed: remove superfluous parentheses introduced by 2to3 converter in print function calls, shebang modifications to use environment variable for python version, and byte-string decode correction in steadystate_tests.py following 2to3 conversion.

The modified scripts pass rudimentary testing when run under python2.7 as well as python3.

Signed-off-by Bill O'Donnell <[email protected]>

---
v2: Remove unnecessary changes to fiologparser_hist.py as
its conversion has been covered in earlier patches. Keep shebangs at previous python2.7 state. Remove superfluous imports.

 doc/conf.py                     |  3 +++
 tools/fiologparser.py           |  3 +++
 unit_tests/steadystate_tests.py | 20 ++++++++++++--------
 3 files changed, 18 insertions(+), 8 deletions(-)

doc/conf.py +0 -0

index d4dd9d20..087a9a11 100644

doc/conf.py +0 -0

 # If your documentation needs a minimal Sphinx version, state it here.
 #
 # needs_sphinx = '1.0'

tools/fiologparser.py +3 -0

index 8549859f..cc29f1c7 100755
@@ -1,4 +1,5 @@
 #!/usr/bin/python2.7
+# Note: this script is python2 and python 3 compatible.
 #
 # fiologparser.py
 #

@@ -13,6 +14,8 @@
 #
 # to see per-interval average completion latency.
 
+from __future__ import absolute_import
+from __future__ import print_function
 import argparse
 import math
 

unit_tests/steadystate_tests.py +11 -7

index 5a74f956..50254dcc 100755
@@ -1,4 +1,5 @@
 #!/usr/bin/python2.7
+# Note: this script is python2 and python 3 compatible.
 #
 # steadystate_tests.py
 #

@@ -18,6 +19,8 @@
 # if ss attained: min runtime = ss_dur + ss_ramp
 # if not attained: runtime = timeout
 
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import json

@@ -26,11 +29,12 @@ import pprint
 import argparse
 import subprocess
 from scipy import stats
+from six.moves import range
 
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('fio',
-                        help='path to fio executable');
+                        help='path to fio executable')
     parser.add_argument('--read',
                         help='target for read testing')
     parser.add_argument('--write',

@@ -45,7 +49,7 @@ def check(data, iops, slope, pct, limit, dur, criterion):
     data = data[measurement]
     mean = sum(data) / len(data)
     if slope:
-        x = range(len(data))
+        x = list(range(len(data)))
         m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
         m = abs(m)
         if pct:

@@ -89,11 +93,11 @@ if __name__ == '__main__':
                   'output': "set steady state BW threshold to 12" },
               ]
     for test in parsing:
-        output = subprocess.check_output([args.fio] + test['args']);
-        if test['output'] in output:
-            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
+        output = subprocess.check_output([args.fio] + test['args'])
+        if test['output'] in output.decode():
+            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
         else:
-            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
+            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
 
 #
 # test some read workloads

@@ -117,7 +121,7 @@ if __name__ == '__main__':
             args.read = '/dev/zero'
             extra = [ "--size=134217728" ]  # 128 MiB
         else:
-            print "ERROR: file for read testing must be specified on non-posix systems"
+            print("ERROR: file for read testing must be specified on non-posix systems")
             sys.exit(1)
     else:
         extra = []

Hi,

Quoted 32 lines · Bill O'Donnell

Many distributions are moving to python3 by default. Here's an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas were hand fixed: remove superfluous parentheses introduced by 2to3 converter in print function calls, shebang modifications to use environment variable for python version, and byte-string decode correction in steadystate_tests.py following 2to3 conversion.

The modified scripts pass rudimentary testing when run under python2.7 as well as python3.

Signed-off-by Bill O'Donnell <[email protected]>

---
v2: Remove unnecessary changes to fiologparser_hist.py as
its conversion has been covered in earlier patches. Keep shebangs at previous python2.7 state. Remove superfluous imports.

 doc/conf.py                     |  3 +++
 tools/fiologparser.py           |  3 +++
 unit_tests/steadystate_tests.py | 20 ++++++++++++--------
 3 files changed, 18 insertions(+), 8 deletions(-)

doc/conf.py +0 -0

index d4dd9d20..087a9a11 100644
@@ -22,6 +22,9 @@
 

doc/conf.py line 27

Quoted 5 lines

doc/conf.py +3 -0

 # -- General configuration ------------------------------------------------
 
+from __future__ import absolute_import
+from __future__ import print_function
+

IMHO these imports are needed only for python < 2.7. I do not think it worth it to support such an ancient versions. The same for the other files.

Quoted 91 lines

doc/conf.py +0 -0

 # If your documentation needs a minimal Sphinx version, state it here.
 #
 # needs_sphinx = '1.0'

tools/fiologparser.py +3 -0

index 8549859f..cc29f1c7 100755
@@ -1,4 +1,5 @@
 #!/usr/bin/python2.7
+# Note: this script is python2 and python 3 compatible.
 #
 # fiologparser.py
 #

@@ -13,6 +14,8 @@
 #
 # to see per-interval average completion latency.
 
+from __future__ import absolute_import
+from __future__ import print_function
 import argparse
 import math
 

unit_tests/steadystate_tests.py +11 -7

index 5a74f956..50254dcc 100755
@@ -1,4 +1,5 @@
 #!/usr/bin/python2.7
+# Note: this script is python2 and python 3 compatible.
 #
 # steadystate_tests.py
 #

@@ -18,6 +19,8 @@
 # if ss attained: min runtime = ss_dur + ss_ramp
 # if not attained: runtime = timeout
 
+from __future__ import absolute_import
+from __future__ import print_function
 import os
 import sys
 import json

@@ -26,11 +29,12 @@ import pprint
 import argparse
 import subprocess
 from scipy import stats
+from six.moves import range
 
 def parse_args():
     parser = argparse.ArgumentParser()
     parser.add_argument('fio',
-                        help='path to fio executable');
+                        help='path to fio executable')
     parser.add_argument('--read',
                         help='target for read testing')
     parser.add_argument('--write',

@@ -45,7 +49,7 @@ def check(data, iops, slope, pct, limit, dur, criterion):
     data = data[measurement]
     mean = sum(data) / len(data)
     if slope:
-        x = range(len(data))
+        x = list(range(len(data)))
         m, intercept, r_value, p_value, std_err = stats.linregress(x,data)
         m = abs(m)
         if pct:

@@ -89,11 +93,11 @@ if __name__ == '__main__':
                   'output': "set steady state BW threshold to 12" },
               ]
     for test in parsing:
-        output = subprocess.check_output([args.fio] + test['args']);
-        if test['output'] in output:
-            print "PASSED '{0}' found with arguments {1}".format(test['output'], test['args'])
+        output = subprocess.check_output([args.fio] + test['args'])
+        if test['output'] in output.decode():
+            print("PASSED '{0}' found with arguments {1}".format(test['output'], test['args']))
         else:
-            print "FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args'])
+            print("FAILED '{0}' NOT found with arguments {1}".format(test['output'], test['args']))
 
 #
 # test some read workloads

@@ -117,7 +121,7 @@ if __name__ == '__main__':
             args.read = '/dev/zero'
             extra = [ "--size=134217728" ]  # 128 MiB
         else:
-            print "ERROR: file for read testing must be specified on non-posix systems"
+            print("ERROR: file for read testing must be specified on non-posix systems")
             sys.exit(1)
     else:
         extra = []

unit_tests/steadystate_tests.py if __name__ == '__main__': · line 226

Quoted 8 lines

unit_tests/steadystate_tests.py +1 -1

@@ -216,7 +220,7 @@ if __name__ == '__main__':
                 else:
                     result = 'FAILED '
                 line = result + line + ' no ss, expected runtime {0} ~= actual runtime {1}'.format(expected, actual)
-            print line
+            print(line)
             if 'steadystate' in jsonjob:
                 pp.pprint(jsonjob['steadystate'])
         jobnum += 1

Regards,
Mikhail
--
To unsubscribe from this list: send the line "unsubscribe fio" in the body of a message to [email protected]
More majordomo info at http://vger.kernel.org/majordomo-info.html

Many distributions are moving to python3 by default. Here's an attempt to make the python scripts in fio python3-ready.

Conversion was facilitated with automated tools. A few areas were hand fixed: remove superfluous parentheses introduced by 2to3 converter in print function calls, shebang modifications to use environment variable for python version, and byte-string decode correction in steadystate_tests.py following 2to3 conversion.

The modified scripts pass rudimentary testing when run under python2.7 as well as python3.