Skip to contentSkip to conversation

scripts: convert-overrides: Allow command-line customizations

PATCH In review Joshua Watt 2 participants 5 messages Nov 11, 2022 – Dec 6, 2022

lore
Share & tools
lore
#0 lore raw

PATCH 1/1 scripts: convert-overrides: Allow command-line customizations

Adds argument parsing to the conversion script so that the fields that the script uses to do conversions can be customized on the command line. The intention is to allows easier customization without having to fork the script, and allow automated checking on 3rd party layers via CI without false positives

---
 scripts/contrib/convert-overrides.py | 103 +++++++++++++++------------
 1 file changed, 57 insertions(+), 46 deletions(-)
scripts/contrib/convert-overrides.py +57 -46
index 4d41a4c4759..1939757f1bc 100755
@@ -22,50 +22,62 @@ import sys
 import tempfile
 import shutil
 import mimetypes
+import argparse
 
-if len(sys.argv) < 2:
-    print("Please specify a directory to run the conversion script against.")
-    sys.exit(1)
+parser = argparse.ArgumentParser(description="Convert override syntax")
+parser.add_argument("--override", "-o", action="append", default=[], help="Add additional strings to consider as an override (e.g. custom machines/distros")
+parser.add_argument("--skip", "-s", action="append", default=[], help="Add additional string to skip and not consider an override")
+parser.add_argument("--skip-ext", "-e", action="append", default=[], help="Additional file suffixes to skip when processing (e.g. '.foo')")
+parser.add_argument("--package-vars", action="append", default=[], help="Additional variables to treat as package variables")
+parser.add_argument("--image-vars", action="append", default=[], help="Additional variables to treat as image variables")
+parser.add_argument("--short-override", action="append", default=[], help="Additional strings to treat as short overrides")
+parser.add_argument("path", nargs="+", help="Paths to convert")
+
+args = parser.parse_args()
 
 # List of strings to treat as overrides
-vars = ["append", "prepend", "remove"]
-vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
-vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
-vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
-vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
-vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
-vars = vars + ["tune-",  "pn-", "forcevariable"]
-vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
-vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
-vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
-vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
-vars = vars + ["linux-gnueabi", "eabi"]
-vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
+vars = args.override
+vars += ["append", "prepend", "remove"]
+vars += ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
+vars += ["genericx86", "edgerouter", "beaglebone-yocto"]
+vars += ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
+vars += ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
+vars += ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
+vars += ["tune-",  "pn-", "forcevariable"]
+vars += ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
+vars += ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
+vars += ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
+vars += ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
+vars += ["linux-gnueabi", "eabi"]
+vars += ["virtclass-multilib", "virtclass-mcextend"]
 
 # List of strings to treat as overrides but only with whitespace following or another override (more restricted matching).
 # Handles issues with arc matching arch.
-shortvars = ["arc", "mips", "mipsel", "sh4"]
+shortvars = ["arc", "mips", "mipsel", "sh4"] + args.short_override
 
 # Variables which take packagenames as an override
 packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",
               "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",
               "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",
               "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",
-              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
+              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] + args.package_vars
 
 # Expressions to skip if encountered, these are not overrides
-skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
-skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
-skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
-skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
-skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
-skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
-skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
-skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
-skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
-
-imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"]
-packagevars = packagevars + imagevars
+skips = args.skip
+skips += ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
+skips += ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
+skips += ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
+skips += ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
+skips += ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
+skips += ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
+skips += ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
+skips += ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
+skips += ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
+
+imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] + args.image_vars
+packagevars += imagevars
+
+skip_ext = [".html", ".patch", ".m4", ".diff"] + args.skip_ext
 
 vars_re = {}
 for exp in vars:
@@ -124,21 +136,20 @@ def processfile(fn):
 ourname = os.path.basename(sys.argv[0])
 ourversion = "0.9.3"
 
-if os.path.isfile(sys.argv[1]):
-    processfile(sys.argv[1])
-    sys.exit(0)
-
-for targetdir in sys.argv[1:]:
-    print("processing directory '%s'" % targetdir)
-    for root, dirs, files in os.walk(targetdir):
-        for name in files:
-            if name == ourname:
-                continue
-            fn = os.path.join(root, name)
-            if os.path.islink(fn):
-                continue
-            if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):
-                continue
-            processfile(fn)
+for p in args.path:
+    if os.path.isfile(p):
+        processfile(p)
+    else:
+        print("processing directory '%s'" % p)
+        for root, dirs, files in os.walk(p):
+            for name in files:
+                if name == ourname:
+                    continue
+                fn = os.path.join(root, name)
+                if os.path.islink(fn):
+                    continue
+                if "/.git/" in fn or any(fn.endswith(ext) for ext in skip_ext):
+                    continue
+                processfile(fn)
 
 print("All files processed with version %s" % ourversion)
Signature · 1 line
2.33.0
#1 lore raw

Steve,

Please backport this to kirkstone and langdale.

Thanks!

Quoted 149 lines · Joshua Watt

Adds argument parsing to the conversion script so that the fields that the script uses to do conversions can be customized on the command line. The intention is to allows easier customization without having to fork the script, and allow automated checking on 3rd party layers via CI without false positives

---
 scripts/contrib/convert-overrides.py | 103 +++++++++++++++------------
 1 file changed, 57 insertions(+), 46 deletions(-)
scripts/contrib/convert-overrides.py +57 -47
index 4d41a4c4759..1939757f1bc 100755
@@ -22,50 +22,62 @@ import sys
 import tempfile
 import shutil
 import mimetypes
+import argparse
 
-if len(sys.argv) < 2:
-    print("Please specify a directory to run the conversion script against.")
-    sys.exit(1)
+parser = argparse.ArgumentParser(description="Convert override syntax")
+parser.add_argument("--override", "-o", action="append", default=[], help="Add additional strings to consider as an override (e.g. custom machines/distros")
+parser.add_argument("--skip", "-s", action="append", default=[], help="Add additional string to skip and not consider an override")
+parser.add_argument("--skip-ext", "-e", action="append", default=[], help="Additional file suffixes to skip when processing (e.g. '.foo')")
+parser.add_argument("--package-vars", action="append", default=[], help="Additional variables to treat as package variables")
+parser.add_argument("--image-vars", action="append", default=[], help="Additional variables to treat as image variables")
+parser.add_argument("--short-override", action="append", default=[], help="Additional strings to treat as short overrides")
+parser.add_argument("path", nargs="+", help="Paths to convert")
+
+args = parser.parse_args()
 
 # List of strings to treat as overrides
-vars = ["append", "prepend", "remove"]
-vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
-vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
-vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
-vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
-vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
-vars = vars + ["tune-",  "pn-", "forcevariable"]
-vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
-vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
-vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
-vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
-vars = vars + ["linux-gnueabi", "eabi"]
-vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
+vars = args.override
+vars += ["append", "prepend", "remove"]
+vars += ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
+vars += ["genericx86", "edgerouter", "beaglebone-yocto"]
+vars += ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
+vars += ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
+vars += ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
+vars += ["tune-",  "pn-", "forcevariable"]
+vars += ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
+vars += ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
+vars += ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
+vars += ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
+vars += ["linux-gnueabi", "eabi"]
+vars += ["virtclass-multilib", "virtclass-mcextend"]
 
 # List of strings to treat as overrides but only with whitespace following or another override (more restricted matching).
 # Handles issues with arc matching arch.
-shortvars = ["arc", "mips", "mipsel", "sh4"]
+shortvars = ["arc", "mips", "mipsel", "sh4"] + args.short_override
 
 # Variables which take packagenames as an override
 packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",
               "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",
               "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",
               "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",
-              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
+              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] + args.package_vars
 
 # Expressions to skip if encountered, these are not overrides
-skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
-skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
-skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
-skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
-skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
-skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
-skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
-skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
-skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
-
-imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"]
-packagevars = packagevars + imagevars
+skips = args.skip
+skips += ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
+skips += ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
+skips += ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
+skips += ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
+skips += ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
+skips += ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
+skips += ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
+skips += ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
+skips += ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
+
+imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] + args.image_vars
+packagevars += imagevars
+
+skip_ext = [".html", ".patch", ".m4", ".diff"] + args.skip_ext
 
 vars_re = {}
 for exp in vars:
@@ -124,21 +136,20 @@ def processfile(fn):
 ourname = os.path.basename(sys.argv[0])
 ourversion = "0.9.3"
 
-if os.path.isfile(sys.argv[1]):
-    processfile(sys.argv[1])
-    sys.exit(0)
-
-for targetdir in sys.argv[1:]:
-    print("processing directory '%s'" % targetdir)
-    for root, dirs, files in os.walk(targetdir):
-        for name in files:
-            if name == ourname:
-                continue
-            fn = os.path.join(root, name)
-            if os.path.islink(fn):
-                continue
-            if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):
-                continue
-            processfile(fn)
+for p in args.path:
+    if os.path.isfile(p):
+        processfile(p)
+    else:
+        print("processing directory '%s'" % p)
+        for root, dirs, files in os.walk(p):
+            for name in files:
+                if name == ourname:
+                    continue
+                fn = os.path.join(root, name)
+                if os.path.islink(fn):
+                    continue
+                if "/.git/" in fn or any(fn.endswith(ext) for ext in skip_ext):
+                    continue
+                processfile(fn)
 
 print("All files processed with version %s" % ourversion)
--

2.33.0

#2 lore raw
Quoted 6 lines · Joshua Watt

Steve,

Please backport this to kirkstone and langdale.

Thanks!

Ok, will do!

Steve

Quoted 224 lines · Joshua Watt
Quoted 7 lines

Adds argument parsing to the conversion script so that the fields that the script uses to do conversions can be customized on the command line. The intention is to allows easier customization without having to fork the script, and allow automated checking on 3rd party layers via CI without false positives

Review

Quoted 6 lines
---
 scripts/contrib/convert-overrides.py | 103 +++++++++++++++------------
 1 file changed, 57 insertions(+), 46 deletions(-)
(quoted diff) +0 -0

b/scripts/contrib/convert-overrides.py

scripts/contrib/convert-overrides.py import sys · line 27

Quoted 11 lines

index 4d41a4c4759..1939757f1bc 100755

scripts/contrib/convert-overrides.py +1 -2
@@ -22,50 +22,62 @@ import sys
 import tempfile
 import shutil
 import mimetypes
+import argparse
 
-if len(sys.argv) < 2:
-    print("Please specify a directory to run the conversion script

against.")

scripts/contrib/convert-overrides.py import sys

Quoted 3 lines
scripts/contrib/convert-overrides.py +2 -1
-    sys.exit(1)
+parser = argparse.ArgumentParser(description="Convert override syntax")
+parser.add_argument("--override", "-o", action="append", default=[],

help="Add additional strings to consider as an override (e.g. custom machines/distros")

scripts/contrib/convert-overrides.py import sys · line 29

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+parser.add_argument("--skip", "-s", action="append", default=[],

help="Add additional string to skip and not consider an override")

scripts/contrib/convert-overrides.py import sys · line 30

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+parser.add_argument("--skip-ext", "-e", action="append", default=[],

help="Additional file suffixes to skip when processing (e.g. '.foo')")

scripts/contrib/convert-overrides.py import sys · line 31

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+parser.add_argument("--package-vars", action="append", default=[],

help="Additional variables to treat as package variables")

scripts/contrib/convert-overrides.py import sys · line 32

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+parser.add_argument("--image-vars", action="append", default=[],

help="Additional variables to treat as image variables")

scripts/contrib/convert-overrides.py import sys · line 33

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+parser.add_argument("--short-override", action="append", default=[],

help="Additional strings to treat as short overrides")

scripts/contrib/convert-overrides.py import sys

Quoted 7 lines
scripts/contrib/convert-overrides.py +3 -2
+parser.add_argument("path", nargs="+", help="Paths to convert")
+
+args = parser.parse_args()
 
 # List of strings to treat as overrides
-vars = ["append", "prepend", "remove"]
-vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc",

"qemuriscv", "qemuall"]

scripts/contrib/convert-overrides.py import sys · line 34

Quoted 2 lines
scripts/contrib/convert-overrides.py +0 -2
-vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
-vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64",

"aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]

scripts/contrib/convert-overrides.py import sys · line 35

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze",

"e5500-64b", "mipsisa32", "mipsisa64"]

scripts/contrib/convert-overrides.py import sys · line 36

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-vars = vars + ["class-native", "class-target", "class-cross-canadian",

"class-cross", "class-devupstream"]

scripts/contrib/convert-overrides.py import sys · line 38

Quoted 2 lines
scripts/contrib/convert-overrides.py +0 -2
-vars = vars + ["tune-",  "pn-", "forcevariable"]
-vars = vars + ["libc-musl", "libc-glibc",

"libc-newlib","libc-baremetal"]

scripts/contrib/convert-overrides.py import sys · line 39

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-vars = vars + ["task-configure", "task-compile", "task-install",

"task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]

scripts/contrib/convert-overrides.py import sys · line 40

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32",

"overrideone", "overridetwo"]

scripts/contrib/convert-overrides.py import sys · line 41

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32",

"mingw32", "poky", "darwin", "linuxstdbase"]

scripts/contrib/convert-overrides.py import sys

Quoted 5 lines
scripts/contrib/convert-overrides.py +3 -2
-vars = vars + ["linux-gnueabi", "eabi"]
-vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
+vars = args.override
+vars += ["append", "prepend", "remove"]
+vars += ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv",

"qemuall"]

scripts/contrib/convert-overrides.py import sys · line 43

Quoted 2 lines
scripts/contrib/convert-overrides.py +2 -0
+vars += ["genericx86", "edgerouter", "beaglebone-yocto"]
+vars += ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64",

"aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]

scripts/contrib/convert-overrides.py import sys · line 44

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+vars += ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b",

"mipsisa32", "mipsisa64"]

scripts/contrib/convert-overrides.py import sys · line 45

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+vars += ["class-native", "class-target", "class-cross-canadian",

"class-cross", "class-devupstream"]

scripts/contrib/convert-overrides.py import sys

Quoted 3 lines
scripts/contrib/convert-overrides.py +3 -0
+vars += ["tune-",  "pn-", "forcevariable"]
+vars += ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
+vars += ["task-configure", "task-compile", "task-install",

"task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]

scripts/contrib/convert-overrides.py import sys · line 49

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+vars += ["toolchain-clang", "mydistro", "nios2", "sdkmingw32",

"overrideone", "overridetwo"]

scripts/contrib/convert-overrides.py import sys · line 50

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+vars += ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32",

"poky", "darwin", "linuxstdbase"]

scripts/contrib/convert-overrides.py import sys

Quoted 4 lines
scripts/contrib/convert-overrides.py +2 -0
+vars += ["linux-gnueabi", "eabi"]
+vars += ["virtclass-multilib", "virtclass-mcextend"]
 
 # List of strings to treat as overrides but only with whitespace

following or another override (more restricted matching).

scripts/contrib/convert-overrides.py import sys

Quoted 6 lines
scripts/contrib/convert-overrides.py +1 -1
 # Handles issues with arc matching arch.
-shortvars = ["arc", "mips", "mipsel", "sh4"]
+shortvars = ["arc", "mips", "mipsel", "sh4"] + args.short_override
 
 # Variables which take packagenames as an override
 packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY",

"DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",

scripts/contrib/convert-overrides.py import sys · line 60

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -0
               "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst",

"INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",

scripts/contrib/convert-overrides.py import sys · line 61

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -0
               "PKGE", "PKGV", "PKGR", "USERADD_PARAM",

"GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",

scripts/contrib/convert-overrides.py import sys · line 62

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -0
               "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM",

"SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",

scripts/contrib/convert-overrides.py import sys · line 63

Quoted 2 lines
scripts/contrib/convert-overrides.py +1 -1
-              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
+              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] +

args.package_vars

scripts/contrib/convert-overrides.py import sys · line 56

Quoted 3 lines
scripts/contrib/convert-overrides.py +0 -1
 # Expressions to skip if encountered, these are not overrides
-skips = ["parser_append", "recipe_to_append", "extra_append",

"to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]

scripts/contrib/convert-overrides.py import sys · line 57

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["expanded_removes", "color_remove", "test_remove",

"empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]

scripts/contrib/convert-overrides.py import sys · line 58

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["no_report_remove", "test_prepend", "test_append",

"multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]

scripts/contrib/convert-overrides.py import sys · line 59

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["parser_remove", "to_append", "no_remove",

"bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]

scripts/contrib/convert-overrides.py import sys · line 60

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["autotools_dep_prepend", "go_map_arm",

"alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]

scripts/contrib/convert-overrides.py import sys · line 61

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["run_loaddata_poky", "determine_if_poky_env",

"do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]

scripts/contrib/convert-overrides.py import sys · line 62

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["get_appends_for_files", "test_doubleref_remove",

"test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]

scripts/contrib/convert-overrides.py import sys · line 63

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove",

"recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]

scripts/contrib/convert-overrides.py import sys · line 64

Quoted 1 line
scripts/contrib/convert-overrides.py +0 -1
-skips = skips + ["recipetool_append", "changetype_remove",

"try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]

scripts/contrib/convert-overrides.py import sys · line 66

Quoted 2 lines
scripts/contrib/convert-overrides.py +0 -2
-
-imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP",

"CONVERSION_CMD", "COMPRESS_CMD"]

scripts/contrib/convert-overrides.py import sys

Quoted 3 lines
scripts/contrib/convert-overrides.py +2 -1
-packagevars = packagevars + imagevars
+skips = args.skip
+skips += ["parser_append", "recipe_to_append", "extra_append",

"to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]

scripts/contrib/convert-overrides.py import sys · line 67

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["expanded_removes", "color_remove", "test_remove",

"empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]

scripts/contrib/convert-overrides.py import sys · line 68

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["no_report_remove", "test_prepend", "test_append",

"multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]

scripts/contrib/convert-overrides.py import sys · line 69

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["parser_remove", "to_append", "no_remove",

"bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]

scripts/contrib/convert-overrides.py import sys · line 70

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["autotools_dep_prepend", "go_map_arm", "alt_remove_links",

"systemd_append_file", "file_append", "process_file_darwin"]

scripts/contrib/convert-overrides.py import sys · line 71

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["run_loaddata_poky", "determine_if_poky_env",

"do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]

scripts/contrib/convert-overrides.py import sys · line 72

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["get_appends_for_files", "test_doubleref_remove",

"test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]

scripts/contrib/convert-overrides.py import sys · line 73

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["test_rpm_remove", "test_bitbakelayers_add_remove",

"recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]

scripts/contrib/convert-overrides.py import sys · line 74

Quoted 1 line
scripts/contrib/convert-overrides.py +1 -0
+skips += ["recipetool_append", "changetype_remove",

"try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]

scripts/contrib/convert-overrides.py import sys · line 76

Quoted 2 lines
scripts/contrib/convert-overrides.py +2 -0
+
+imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP",

"CONVERSION_CMD", "COMPRESS_CMD"] + args.image_vars

Quoted 14 lines
scripts/contrib/convert-overrides.py +3 -4
+packagevars += imagevars
+
+skip_ext = [".html", ".patch", ".m4", ".diff"] + args.skip_ext
 
 vars_re = {}
 for exp in vars:
@@ -124,21 +136,20 @@ def processfile(fn):
 ourname = os.path.basename(sys.argv[0])
 ourversion = "0.9.3"
 
-if os.path.isfile(sys.argv[1]):
-    processfile(sys.argv[1])
-    sys.exit(0)
-

scripts/contrib/convert-overrides.py def processfile(fn): · line 140

Quoted 10 lines
scripts/contrib/convert-overrides.py +0 -10
-for targetdir in sys.argv[1:]:
-    print("processing directory '%s'" % targetdir)
-    for root, dirs, files in os.walk(targetdir):
-        for name in files:
-            if name == ourname:
-                continue
-            fn = os.path.join(root, name)
-            if os.path.islink(fn):
-                continue
-            if "/.git/" in fn or fn.endswith(".html") or

fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):

Quoted 5 lines
scripts/contrib/convert-overrides.py +3 -2
-                continue
-            processfile(fn)
+for p in args.path:
+    if os.path.isfile(p):
+        processfile(p)

scripts/contrib/convert-overrides.py def processfile(fn):

Quoted 10 lines
scripts/contrib/convert-overrides.py +10 -0
+    else:
+        print("processing directory '%s'" % p)
+        for root, dirs, files in os.walk(p):
+            for name in files:
+                if name == ourname:
+                    continue
+                fn = os.path.join(root, name)
+                if os.path.islink(fn):
+                    continue
+                if "/.git/" in fn or any(fn.endswith(ext) for ext in

skip_ext):

Quoted 7 lines
scripts/contrib/convert-overrides.py +2 -1
+                    continue
+                processfile(fn)
 
 print("All files processed with version %s" % ourversion)
--

2.33.0

#3 lore raw

Steve,

I don't see this one in (or pending for) kirkstone yet?

Thanks,
Joshua Watt

Quoted 168 lines · Steve Sakoman
Quoted 6 lines · Joshua Watt

Steve,

Please backport this to kirkstone and langdale.

Thanks!

Ok, will do!

Steve

Quoted 151 lines · Joshua Watt
Quoted 149 lines

Adds argument parsing to the conversion script so that the fields that the script uses to do conversions can be customized on the command line. The intention is to allows easier customization without having to fork the script, and allow automated checking on 3rd party layers via CI without false positives

---
 scripts/contrib/convert-overrides.py | 103 +++++++++++++++------------
 1 file changed, 57 insertions(+), 46 deletions(-)
scripts/contrib/convert-overrides.py +57 -47
index 4d41a4c4759..1939757f1bc 100755
@@ -22,50 +22,62 @@ import sys
 import tempfile
 import shutil
 import mimetypes
+import argparse
 
-if len(sys.argv) < 2:
-    print("Please specify a directory to run the conversion script against.")
-    sys.exit(1)
+parser = argparse.ArgumentParser(description="Convert override syntax")
+parser.add_argument("--override", "-o", action="append", default=[], help="Add additional strings to consider as an override (e.g. custom machines/distros")
+parser.add_argument("--skip", "-s", action="append", default=[], help="Add additional string to skip and not consider an override")
+parser.add_argument("--skip-ext", "-e", action="append", default=[], help="Additional file suffixes to skip when processing (e.g. '.foo')")
+parser.add_argument("--package-vars", action="append", default=[], help="Additional variables to treat as package variables")
+parser.add_argument("--image-vars", action="append", default=[], help="Additional variables to treat as image variables")
+parser.add_argument("--short-override", action="append", default=[], help="Additional strings to treat as short overrides")
+parser.add_argument("path", nargs="+", help="Paths to convert")
+
+args = parser.parse_args()
 
 # List of strings to treat as overrides
-vars = ["append", "prepend", "remove"]
-vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
-vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
-vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
-vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
-vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
-vars = vars + ["tune-",  "pn-", "forcevariable"]
-vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
-vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
-vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
-vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
-vars = vars + ["linux-gnueabi", "eabi"]
-vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
+vars = args.override
+vars += ["append", "prepend", "remove"]
+vars += ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
+vars += ["genericx86", "edgerouter", "beaglebone-yocto"]
+vars += ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
+vars += ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
+vars += ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
+vars += ["tune-",  "pn-", "forcevariable"]
+vars += ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
+vars += ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
+vars += ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
+vars += ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
+vars += ["linux-gnueabi", "eabi"]
+vars += ["virtclass-multilib", "virtclass-mcextend"]
 
 # List of strings to treat as overrides but only with whitespace following or another override (more restricted matching).
 # Handles issues with arc matching arch.
-shortvars = ["arc", "mips", "mipsel", "sh4"]
+shortvars = ["arc", "mips", "mipsel", "sh4"] + args.short_override
 
 # Variables which take packagenames as an override
 packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",
               "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",
               "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",
               "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",
-              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
+              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] + args.package_vars
 
 # Expressions to skip if encountered, these are not overrides
-skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
-skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
-skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
-skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
-skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
-skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
-skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
-skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
-skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
-
-imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"]
-packagevars = packagevars + imagevars
+skips = args.skip
+skips += ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
+skips += ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
+skips += ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
+skips += ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
+skips += ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
+skips += ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
+skips += ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
+skips += ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
+skips += ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
+
+imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] + args.image_vars
+packagevars += imagevars
+
+skip_ext = [".html", ".patch", ".m4", ".diff"] + args.skip_ext
 
 vars_re = {}
 for exp in vars:
@@ -124,21 +136,20 @@ def processfile(fn):
 ourname = os.path.basename(sys.argv[0])
 ourversion = "0.9.3"
 
-if os.path.isfile(sys.argv[1]):
-    processfile(sys.argv[1])
-    sys.exit(0)
-
-for targetdir in sys.argv[1:]:
-    print("processing directory '%s'" % targetdir)
-    for root, dirs, files in os.walk(targetdir):
-        for name in files:
-            if name == ourname:
-                continue
-            fn = os.path.join(root, name)
-            if os.path.islink(fn):
-                continue
-            if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):
-                continue
-            processfile(fn)
+for p in args.path:
+    if os.path.isfile(p):
+        processfile(p)
+    else:
+        print("processing directory '%s'" % p)
+        for root, dirs, files in os.walk(p):
+            for name in files:
+                if name == ourname:
+                    continue
+                fn = os.path.join(root, name)
+                if os.path.islink(fn):
+                    continue
+                if "/.git/" in fn or any(fn.endswith(ext) for ext in skip_ext):
+                    continue
+                processfile(fn)
 
 print("All files processed with version %s" % ourversion)
--

2.33.0

#4 lore raw
Quoted 1 line · Joshua Watt

I don't see this one in (or pending for) kirkstone yet?

Oops, I thought I had done that. Sorry! I've got it now for both langdale and kirkstone.

Steve

Quoted 169 lines · Steve Sakoman
Quoted 168 lines · Joshua Watt
Quoted 6 lines

Steve,

Please backport this to kirkstone and langdale.

Thanks!

Ok, will do!

Steve

Quoted 151 lines · Joshua Watt
Quoted 149 lines

Adds argument parsing to the conversion script so that the fields that the script uses to do conversions can be customized on the command line. The intention is to allows easier customization without having to fork the script, and allow automated checking on 3rd party layers via CI without false positives

---
 scripts/contrib/convert-overrides.py | 103 +++++++++++++++------------
 1 file changed, 57 insertions(+), 46 deletions(-)
scripts/contrib/convert-overrides.py +57 -47
index 4d41a4c4759..1939757f1bc 100755
@@ -22,50 +22,62 @@ import sys
 import tempfile
 import shutil
 import mimetypes
+import argparse
 
-if len(sys.argv) < 2:
-    print("Please specify a directory to run the conversion script against.")
-    sys.exit(1)
+parser = argparse.ArgumentParser(description="Convert override syntax")
+parser.add_argument("--override", "-o", action="append", default=[], help="Add additional strings to consider as an override (e.g. custom machines/distros")
+parser.add_argument("--skip", "-s", action="append", default=[], help="Add additional string to skip and not consider an override")
+parser.add_argument("--skip-ext", "-e", action="append", default=[], help="Additional file suffixes to skip when processing (e.g. '.foo')")
+parser.add_argument("--package-vars", action="append", default=[], help="Additional variables to treat as package variables")
+parser.add_argument("--image-vars", action="append", default=[], help="Additional variables to treat as image variables")
+parser.add_argument("--short-override", action="append", default=[], help="Additional strings to treat as short overrides")
+parser.add_argument("path", nargs="+", help="Paths to convert")
+
+args = parser.parse_args()
 
 # List of strings to treat as overrides
-vars = ["append", "prepend", "remove"]
-vars = vars + ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
-vars = vars + ["genericx86", "edgerouter", "beaglebone-yocto"]
-vars = vars + ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
-vars = vars + ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
-vars = vars + ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
-vars = vars + ["tune-",  "pn-", "forcevariable"]
-vars = vars + ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
-vars = vars + ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
-vars = vars + ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
-vars = vars + ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
-vars = vars + ["linux-gnueabi", "eabi"]
-vars = vars + ["virtclass-multilib", "virtclass-mcextend"]
+vars = args.override
+vars += ["append", "prepend", "remove"]
+vars += ["qemuarm", "qemux86", "qemumips", "qemuppc", "qemuriscv", "qemuall"]
+vars += ["genericx86", "edgerouter", "beaglebone-yocto"]
+vars += ["armeb", "arm", "armv5", "armv6", "armv4", "powerpc64", "aarch64", "riscv32", "riscv64", "x86", "mips64", "powerpc"]
+vars += ["mipsarch", "x86-x32", "mips16e", "microblaze", "e5500-64b", "mipsisa32", "mipsisa64"]
+vars += ["class-native", "class-target", "class-cross-canadian", "class-cross", "class-devupstream"]
+vars += ["tune-",  "pn-", "forcevariable"]
+vars += ["libc-musl", "libc-glibc", "libc-newlib","libc-baremetal"]
+vars += ["task-configure", "task-compile", "task-install", "task-clean", "task-image-qa", "task-rm_work", "task-image-complete", "task-populate-sdk"]
+vars += ["toolchain-clang", "mydistro", "nios2", "sdkmingw32", "overrideone", "overridetwo"]
+vars += ["linux-gnux32", "linux-muslx32", "linux-gnun32", "mingw32", "poky", "darwin", "linuxstdbase"]
+vars += ["linux-gnueabi", "eabi"]
+vars += ["virtclass-multilib", "virtclass-mcextend"]
 
 # List of strings to treat as overrides but only with whitespace following or another override (more restricted matching).
 # Handles issues with arc matching arch.
-shortvars = ["arc", "mips", "mipsel", "sh4"]
+shortvars = ["arc", "mips", "mipsel", "sh4"] + args.short_override
 
 # Variables which take packagenames as an override
 packagevars = ["FILES", "RDEPENDS", "RRECOMMENDS", "SUMMARY", "DESCRIPTION", "RSUGGESTS", "RPROVIDES", "RCONFLICTS", "PKG", "ALLOW_EMPTY",
               "pkg_postrm", "pkg_postinst_ontarget", "pkg_postinst", "INITSCRIPT_NAME", "INITSCRIPT_PARAMS", "DEBIAN_NOAUTONAME", "ALTERNATIVE",
               "PKGE", "PKGV", "PKGR", "USERADD_PARAM", "GROUPADD_PARAM", "CONFFILES", "SYSTEMD_SERVICE", "LICENSE", "SECTION", "pkg_preinst",
               "pkg_prerm", "RREPLACES", "GROUPMEMS_PARAM", "SYSTEMD_AUTO_ENABLE", "SKIP_FILEDEPS", "PRIVATE_LIBS", "PACKAGE_ADD_METADATA",
-              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"]
+              "INSANE_SKIP", "DEBIANNAME", "SYSTEMD_SERVICE_ESCAPED"] + args.package_vars
 
 # Expressions to skip if encountered, these are not overrides
-skips = ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
-skips = skips + ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
-skips = skips + ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
-skips = skips + ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
-skips = skips + ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
-skips = skips + ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
-skips = skips + ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
-skips = skips + ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
-skips = skips + ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
-
-imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"]
-packagevars = packagevars + imagevars
+skips = args.skip
+skips += ["parser_append", "recipe_to_append", "extra_append", "to_remove", "show_appends", "applied_appends", "file_appends", "handle_remove"]
+skips += ["expanded_removes", "color_remove", "test_remove", "empty_remove", "toaster_prepend", "num_removed", "licfiles_append", "_write_append"]
+skips += ["no_report_remove", "test_prepend", "test_append", "multiple_append", "test_remove", "shallow_remove", "do_remove_layer", "first_append"]
+skips += ["parser_remove", "to_append", "no_remove", "bblayers_add_remove", "bblayers_remove", "apply_append", "is_x86", "base_dep_prepend"]
+skips += ["autotools_dep_prepend", "go_map_arm", "alt_remove_links", "systemd_append_file", "file_append", "process_file_darwin"]
+skips += ["run_loaddata_poky", "determine_if_poky_env", "do_populate_poky_src", "libc_cv_include_x86_isa_level", "test_rpm_remove", "do_install_armmultilib"]
+skips += ["get_appends_for_files", "test_doubleref_remove", "test_bitbakelayers_add_remove", "elf32_x86_64", "colour_remove", "revmap_remove"]
+skips += ["test_rpm_remove", "test_bitbakelayers_add_remove", "recipe_append_file", "log_data_removed", "recipe_append", "systemd_machine_unit_append"]
+skips += ["recipetool_append", "changetype_remove", "try_appendfile_wc", "test_qemux86_directdisk", "test_layer_appends", "tgz_removed"]
+
+imagevars = ["IMAGE_CMD", "EXTRA_IMAGECMD", "IMAGE_TYPEDEP", "CONVERSION_CMD", "COMPRESS_CMD"] + args.image_vars
+packagevars += imagevars
+
+skip_ext = [".html", ".patch", ".m4", ".diff"] + args.skip_ext
 
 vars_re = {}
 for exp in vars:
@@ -124,21 +136,20 @@ def processfile(fn):
 ourname = os.path.basename(sys.argv[0])
 ourversion = "0.9.3"
 
-if os.path.isfile(sys.argv[1]):
-    processfile(sys.argv[1])
-    sys.exit(0)
-
-for targetdir in sys.argv[1:]:
-    print("processing directory '%s'" % targetdir)
-    for root, dirs, files in os.walk(targetdir):
-        for name in files:
-            if name == ourname:
-                continue
-            fn = os.path.join(root, name)
-            if os.path.islink(fn):
-                continue
-            if "/.git/" in fn or fn.endswith(".html") or fn.endswith(".patch") or fn.endswith(".m4") or fn.endswith(".diff"):
-                continue
-            processfile(fn)
+for p in args.path:
+    if os.path.isfile(p):
+        processfile(p)
+    else:
+        print("processing directory '%s'" % p)
+        for root, dirs, files in os.walk(p):
+            for name in files:
+                if name == ourname:
+                    continue
+                fn = os.path.join(root, name)
+                if os.path.islink(fn):
+                    continue
+                if "/.git/" in fn or any(fn.endswith(ext) for ext in skip_ext):
+                    continue
+                processfile(fn)
 
 print("All files processed with version %s" % ourversion)
--

2.33.0