Add support to handle INCOMPATIBLE_LICENSE

Reimplemented license_manifest_create from shell to python for INCOMPATIBLE_LICENSE handle using oe.license module.

Optimizations are made to avoid license copy now uses a hardlink and symbolic link this helps to save space during build.

Signed-off-by Aníbal Limón <[email protected]>

---
 meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
 1 file changed, 82 insertions(+), 81 deletions(-)

meta/classes/license.bbclass +1 -81

index d9409a9..975867d 100644
@@ -25,87 +25,88 @@ python write_package_manifest() {
         'w+').write(image_list_installed_packages(d))
 }
 
-license_create_manifest() {
-        # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
-        if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
-          exit 0
-        fi
-
-       INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
-       LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
-       # remove existing license.manifest file
-       if [ -f ${LICENSE_MANIFEST} ]; then
-               rm ${LICENSE_MANIFEST}
-       fi
-       touch ${LICENSE_MANIFEST}
-       for pkg in ${INSTALLED_PKGS}; do
-               filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
-               pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
-
-               # check to see if the package name exists in the manifest. if so, bail.
-               if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
-                       continue
-               fi
-
-               pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
-               pkged_name="$(basename $(readlink ${filename}))"
-               pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
-               pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
-               if [ -z "${pkged_lic}" ]; then
-                       # fallback checking value of LICENSE
-                       pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
-               fi
-
-               echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
-               echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
-               echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
-               echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
-               echo "" >> ${LICENSE_MANIFEST}
-
-               # If the package doesn't contain any file, that is, its size is 0, the license
-               # isn't relevant as far as the final image is concerned. So doing license check
-               # doesn't make much sense, skip it.
-               if [ "$pkged_size" = "0" ]; then
-                       continue
-               fi
-
-               lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/  */ /g" )"
-               for lic in ${lics}; do
-                       # to reference a license file trim trailing + symbol
-                       if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
-                               bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
-                       fi
-               done
-       done
-
-       # Two options here:
-       # - Just copy the manifest
-       # - Copy the manifest and the license directories
-       # With both options set we see a .5 M increase in core-image-minimal
-       if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
-               mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
-               cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
-               if [ "${COPY_LIC_DIRS}" = "1" ]; then
-                       for pkg in ${INSTALLED_PKGS}; do
-                               mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
-                               pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
-                               for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
-                                       # Really don't need to copy the generics as they're
-                                       # represented in the manifest and in the actual pkg licenses
-                                       # Doing so would make your image quite a bit larger
-                                       if [ "${lic#generic_}" = "${lic}" ]; then
-                                               cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-                                       else
-                                               if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
-                                                       cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
-                                               fi
-                                               ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-                                       fi
-                               done
-                       done
-               fi
-       fi
-
+python license_create_manifest() {

meta/classes/license.bbclass +71 -1

+    with open(package_manifest, "r") as package_file:
+        pkg_list = package_file.read().split()
+        for pkg in pkg_list:
+            pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
+                                    'runtime-reverse', pkg)
+            pkg_name = os.path.basename(os.readlink(pkg_info))
+
+            pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
+            if not "LICENSE" in pkg_dic[pkg_name].keys():
+                pkg_lic_name = "LICENSE_" + pkg_name
+                pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
+
+    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                        d.getVar('IMAGE_NAME', True), 'license.manifest')
+    with open(license_manifest, "w") as license_file:
+        for pkg in sorted(pkg_dic):
+            license_file.write("PACKAGE NAME: %s\n" % pkg)
+            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+            license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+            # If the package doesn't contain any file, that is, its size is 0, the license
+            # isn't relevant as far as the final image is concerned. So doing license check
+            # doesn't make much sense, skip it.
+            if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+                continue
+
+            licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+            licenses = re.sub('  *', ' ', licenses)
+            for lic in licenses.split():
+                lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                        pkg_dic[pkg]["PN"], "generic_%s" %
+                                        re.sub('\+', '', lic))
+                if not os.path.exists(lic_file):
+                   bb.warn("The license listed %s was not in the "\
+                            "licenses collected for recipe %s"
+                            % (lic, pkg_dic[pkg]["PN"]))
+
+    # Two options here:
+    # - Just copy the manifest
+    # - Copy the manifest and the license directories
+    # With both options set we see a .5 M increase in core-image-minimal
+    copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
+    copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
+    if copy_lic_manifest == "1":
+        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
+                                'usr', 'share', 'common-licenses')
+        os.makedirs(rootfs_license_dir)
+        rootfs_license_manifest = os.path.join(rootfs_license_dir,
+                                                'license.manifest')
+        os.link(license_manifest, rootfs_license_manifest)
+
+        if copy_lic_dirs == "1":
+            for pkg in sorted(pkg_dic):
+                pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
+                os.makedirs(pkg_rootfs_license_dir)
+                pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                            pkg_dic[pkg]["PN"])
+                licenses = os.listdir(pkg_license_dir)
+                for lic in licenses:
+                    rootfs_license = os.path.join(rootfs_license_dir, lic)
+                    pkg_license = os.path.join(pkg_license_dir, lic)
+                    pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
+
+                    if re.match("^generic_.*$", lic):
+                        if not os.path.exists(rootfs_license):
+                            os.link(pkg_license, rootfs_license)
+
+                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+                    else:
+                        os.link(pkg_license, pkg_rootfs_license)
 }
 
 python do_populate_lic() {
--

1.8.4.5

Quoted 105 lines · Aníbal Limón

Reimplemented license_manifest_create from shell to python for INCOMPATIBLE_LICENSE handle using oe.license module.

Optimizations are made to avoid license copy now uses a hardlink and symbolic link this helps to save space during build.

Signed-off-by Aníbal Limón <[email protected]>

---
  meta/classes/license.bbclass | 163 ++++++++++++++++++++++---------------------
  1 file changed, 82 insertions(+), 81 deletions(-)

meta/classes/license.bbclass +5 -81

index d9409a9..975867d 100644
@@ -25,87 +25,88 @@ python write_package_manifest() {
          'w+').write(image_list_installed_packages(d))
  }
 
-license_create_manifest() {
-        # Test if BUILD_IMAGES_FROM_FEEDS is defined in env
-        if [ -n "${BUILD_IMAGES_FROM_FEEDS}" ]; then
-          exit 0
-        fi
-
-       INSTALLED_PKGS=`cat ${LICENSE_DIRECTORY}/${IMAGE_NAME}/package.manifest`
-       LICENSE_MANIFEST="${LICENSE_DIRECTORY}/${IMAGE_NAME}/license.manifest"
-       # remove existing license.manifest file
-       if [ -f ${LICENSE_MANIFEST} ]; then
-               rm ${LICENSE_MANIFEST}
-       fi
-       touch ${LICENSE_MANIFEST}
-       for pkg in ${INSTALLED_PKGS}; do
-               filename=`ls ${PKGDATA_DIR}/runtime-reverse/${pkg}| head -1`
-               pkged_pn="$(sed -n 's/^PN: //p' ${filename})"
-
-               # check to see if the package name exists in the manifest. if so, bail.
-               if grep -q "^PACKAGE NAME: ${pkg}" ${LICENSE_MANIFEST}; then
-                       continue
-               fi
-
-               pkged_pv="$(sed -n 's/^PV: //p' ${filename})"
-               pkged_name="$(basename $(readlink ${filename}))"
-               pkged_lic="$(sed -n "/^LICENSE_${pkged_name}: /{ s/^LICENSE_${pkged_name}: //; p }" ${filename})"
-               pkged_size="$(sed -n "/^PKGSIZE_${pkged_name}: /{ s/^PKGSIZE_${pkged_name}: //; p }" ${filename})"
-               if [ -z "${pkged_lic}" ]; then
-                       # fallback checking value of LICENSE
-                       pkged_lic="$(sed -n "/^LICENSE: /{ s/^LICENSE: //; p }" ${filename})"
-               fi
-
-               echo "PACKAGE NAME:" ${pkg} >> ${LICENSE_MANIFEST}
-               echo "PACKAGE VERSION:" ${pkged_pv} >> ${LICENSE_MANIFEST}
-               echo "RECIPE NAME:" ${pkged_pn} >> ${LICENSE_MANIFEST}
-               echo "LICENSE:" ${pkged_lic} >> ${LICENSE_MANIFEST}
-               echo "" >> ${LICENSE_MANIFEST}
-
-               # If the package doesn't contain any file, that is, its size is 0, the license
-               # isn't relevant as far as the final image is concerned. So doing license check
-               # doesn't make much sense, skip it.
-               if [ "$pkged_size" = "0" ]; then
-                       continue
-               fi
-
-               lics="$(echo ${pkged_lic} | sed "s/[|&()*]/ /g" | sed "s/  */ /g" )"
-               for lic in ${lics}; do
-                       # to reference a license file trim trailing + symbol
-                       if ! [ -e "${LICENSE_DIRECTORY}/${pkged_pn}/generic_${lic%+}" ]; then
-                               bbwarn "The license listed ${lic} was not in the licenses collected for ${pkged_pn}"
-                       fi
-               done
-       done
-
-       # Two options here:
-       # - Just copy the manifest
-       # - Copy the manifest and the license directories
-       # With both options set we see a .5 M increase in core-image-minimal
-       if [ "${COPY_LIC_MANIFEST}" = "1" ]; then
-               mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/
-               cp ${LICENSE_MANIFEST} ${IMAGE_ROOTFS}/usr/share/common-licenses/license.manifest
-               if [ "${COPY_LIC_DIRS}" = "1" ]; then
-                       for pkg in ${INSTALLED_PKGS}; do
-                               mkdir -p ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}
-                               pkged_pn="$(oe-pkgdata-util -p ${PKGDATA_DIR} lookup-recipe ${pkg})"
-                               for lic in `ls ${LICENSE_DIRECTORY}/${pkged_pn}`; do
-                                       # Really don't need to copy the generics as they're
-                                       # represented in the manifest and in the actual pkg licenses
-                                       # Doing so would make your image quite a bit larger
-                                       if [ "${lic#generic_}" = "${lic}" ]; then
-                                               cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-                                       else
-                                               if [ ! -f ${IMAGE_ROOTFS}/usr/share/common-licenses/${lic} ]; then
-                                                       cp ${LICENSE_DIRECTORY}/${pkged_pn}/${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/
-                                               fi
-                                               ln -sf ../${lic} ${IMAGE_ROOTFS}/usr/share/common-licenses/${pkg}/${lic}
-                                       fi
-                               done
-                       done
-               fi
-       fi
-
+python license_create_manifest() {
+    import re
+    import oe.packagedata
+
+    build_images_from_feeds = d.getVar('BUILD_IMAGES_FROM_FEEDS', True)

meta/classes/license.bbclass python write_package_manifest() { · line 38

Quoted 6 lines

meta/classes/license.bbclass +6 -0

+    if build_images_from_feeds == "1":
+        return 0
+
+    pkg_dic = {}
+    package_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                        d.getVar('IMAGE_NAME', True), 'package.manifest')

Why aren't you using [image|sdk]_list_installed_packages to get the package list here?

Quoted 77 lines

meta/classes/license.bbclass +71 -1

+    with open(package_manifest, "r") as package_file:
+        pkg_list = package_file.read().split()
+        for pkg in pkg_list:
+            pkg_info = os.path.join(d.getVar('PKGDATA_DIR', True),
+                                    'runtime-reverse', pkg)
+            pkg_name = os.path.basename(os.readlink(pkg_info))
+
+            pkg_dic[pkg_name] = oe.packagedata.read_pkgdatafile(pkg_info)
+            if not "LICENSE" in pkg_dic[pkg_name].keys():
+                pkg_lic_name = "LICENSE_" + pkg_name
+                pkg_dic[pkg_name]["LICENSE"] = pkg_dic[pkg_name][pkg_lic_name]
+
+    license_manifest = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                        d.getVar('IMAGE_NAME', True), 'license.manifest')
+    with open(license_manifest, "w") as license_file:
+        for pkg in sorted(pkg_dic):
+            license_file.write("PACKAGE NAME: %s\n" % pkg)
+            license_file.write("PACKAGE VERSION: %s\n" % pkg_dic[pkg]["PV"])
+            license_file.write("RECIPE NAME: %s\n" % pkg_dic[pkg]["PN"])
+            license_file.write("LICENSE: %s\n\n" % pkg_dic[pkg]["LICENSE"])
+
+            # If the package doesn't contain any file, that is, its size is 0, the license
+            # isn't relevant as far as the final image is concerned. So doing license check
+            # doesn't make much sense, skip it.
+            if pkg_dic[pkg]["PKGSIZE_%s" % pkg] == "0":
+                continue
+
+            licenses = re.sub('[|&()*]', '', pkg_dic[pkg]["LICENSE"])
+            licenses = re.sub('  *', ' ', licenses)
+            for lic in licenses.split():
+                lic_file = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                        pkg_dic[pkg]["PN"], "generic_%s" %
+                                        re.sub('\+', '', lic))
+                if not os.path.exists(lic_file):
+                   bb.warn("The license listed %s was not in the "\
+                            "licenses collected for recipe %s"
+                            % (lic, pkg_dic[pkg]["PN"]))
+
+    # Two options here:
+    # - Just copy the manifest
+    # - Copy the manifest and the license directories
+    # With both options set we see a .5 M increase in core-image-minimal
+    copy_lic_manifest = d.getVar('COPY_LIC_MANIFEST', True)
+    copy_lic_dirs = d.getVar('COPY_LIC_DIRS', True)
+    if copy_lic_manifest == "1":
+        rootfs_license_dir = os.path.join(d.getVar('IMAGE_ROOTFS', 'True'),
+                                'usr', 'share', 'common-licenses')
+        os.makedirs(rootfs_license_dir)
+        rootfs_license_manifest = os.path.join(rootfs_license_dir,
+                                                'license.manifest')
+        os.link(license_manifest, rootfs_license_manifest)
+
+        if copy_lic_dirs == "1":
+            for pkg in sorted(pkg_dic):
+                pkg_rootfs_license_dir = os.path.join(rootfs_license_dir, pkg)
+                os.makedirs(pkg_rootfs_license_dir)
+                pkg_license_dir = os.path.join(d.getVar('LICENSE_DIRECTORY', True),
+                                            pkg_dic[pkg]["PN"])
+                licenses = os.listdir(pkg_license_dir)
+                for lic in licenses:
+                    rootfs_license = os.path.join(rootfs_license_dir, lic)
+                    pkg_license = os.path.join(pkg_license_dir, lic)
+                    pkg_rootfs_license = os.path.join(pkg_rootfs_license_dir, lic)
+
+                    if re.match("^generic_.*$", lic):
+                        if not os.path.exists(rootfs_license):
+                            os.link(pkg_license, rootfs_license)
+
+                        os.symlink(os.path.join('..', lic), pkg_rootfs_license)
+                    else:
+                        os.link(pkg_license, pkg_rootfs_license)
  }
 
  python do_populate_lic() {
--

1.8.4.5