icu: Add knobs to generate a subset of ICU data
PATCH v4 1/1 icu: Add knobs to generate a subset of ICU data
Recent versions of ICU (64+) provides a tool for configuring ICU locale data file with finer granularity [1]
Default generated size for libicudata.so.66.1 is ~27M, which is quite large for embedded systems and all of them may not even need all locale data.
This patch calls the icudata buildtool during configure on the icudata and utilizes a filter called 'filter.json` ( empty by default) to create the data, default behavior should remain same but someone can add a filter.json in own layer to configure this data, e.g.
{
"localeFilter": {
"filterType": "language",
"whitelist": [
"en",
"de",
"it"
] }
}
would only generate the locale data for english/german/italian
This would reduce the size of libicudata.so.66.1 to 12M
Ensure that icudata is generated using host-tools so it can deal with endianness correctly, when host and target systems have different endianness
install the icudtata file back into in/ folder so that main build can now pickup this data file instead of regenerating it and wiping out the filter changes that are expected to take effect
[1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md
---
v2: Fix build for BE systems
v3: Unset AR and RANLIB too
v4: Install fresh icudata back into data/in folder
meta/recipes-support/icu/icu/filter.json | 2 ++ meta/recipes-support/icu/icu_66.1.bb | 26 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 meta/recipes-support/icu/icu/filter.json
meta/recipes-support/icu/icu/filter.json new file +2 -0
@@ -0,0 +1,2 @@ +{ +}
meta/recipes-support/icu/icu_66.1.bb +23 -3
@@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm" ARM_INSTRUCTION_SET_armv5 = "arm" BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz" -SRC_URI = "${BASE_SRC_URI} \ +DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip" +SRC_URI = "${BASE_SRC_URI};name=code \ + ${DATA_SRC_URI};name=data \ + file://filter.json \ file://icu-pkgdata-large-cmd.patch \ file://fix-install-manx.patch \ file://0001-Fix-big-endian-build.patch \
@@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \ SRC_URI_append_class-target = "\ file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \ " -SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8" -SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e" UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src" UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases" + +do_make_icudata_class-target () { + rm -rf ${S}/data + cp -a ${WORKDIR}/data ${S} + cd ${S} + unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS + ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \ + ./runConfigureICU Linux --with-data-packaging=archive + oe_runmake ${PARALLEL_MAKE} + install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat +} + +do_make_icudata() { + : +} + +addtask make_icudata before do_configure after do_patch
Signature · 1 line
2.26.0
ignore this one
Quoted 105 lines · Khem Raj
Recent versions of ICU (64+) provides a tool for configuring ICU locale data file with finer granularity [1]
Default generated size for libicudata.so.66.1 is ~27M, which is quite large for embedded systems and all of them may not even need all locale data.
This patch calls the icudata buildtool during configure on the icudata and utilizes a filter called 'filter.json` ( empty by default) to create the data, default behavior should remain same but someone can add a filter.json in own layer to configure this data, e.g.
{
"localeFilter": { "filterType": "language", "whitelist": [ "en", "de", "it" ]}
}would only generate the locale data for english/german/italian
This would reduce the size of libicudata.so.66.1 to 12M
Ensure that icudata is generated using host-tools so it can deal with endianness correctly, when host and target systems have different endianness
install the icudtata file back into in/ folder so that main build can now pickup this data file instead of regenerating it and wiping out the filter changes that are expected to take effect
[1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md
Signed-off-by Khem Raj <[email protected]>Suggested-by Wouter Meek <[email protected]>---
v2: Fix build for BE systems
v3: Unset AR and RANLIB too
v4: Install fresh icudata back into data/in foldermeta/recipes-support/icu/icu/filter.json | 2 ++ meta/recipes-support/icu/icu_66.1.bb | 26 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 meta/recipes-support/icu/icu/filter.jsonmeta/recipes-support/icu/icu/filter.json new file +2
-0@@ -0,0 +1,2 @@ +{ +}meta/recipes-support/icu/icu_66.1.bb +23
-4@@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm" ARM_INSTRUCTION_SET_armv5 = "arm" BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz" -SRC_URI = "${BASE_SRC_URI} \ +DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip" +SRC_URI = "${BASE_SRC_URI};name=code \ + ${DATA_SRC_URI};name=data \ + file://filter.json \ file://icu-pkgdata-large-cmd.patch \ file://fix-install-manx.patch \ file://0001-Fix-big-endian-build.patch \@@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \ SRC_URI_append_class-target = "\ file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \ " -SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8" -SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e" UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src" UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases" + +do_make_icudata_class-target () { + rm -rf ${S}/data + cp -a ${WORKDIR}/data ${S} + cd ${S} + unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS + ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \ + ./runConfigureICU Linux --with-data-packaging=archive + oe_runmake ${PARALLEL_MAKE} + install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat +} + +do_make_icudata() { + : +} + +addtask make_icudata before do_configure after do_patch --2.26.0
Quoted 3 lines · Khem Raj
ignore this one
This now should be ok to take in, I was seeing a issue on musl which I have sent a separate patch on top
Quoted 106 lines · Khem Raj
Quoted 105 lines
Recent versions of ICU (64+) provides a tool for configuring ICU locale data file with finer granularity [1]
Default generated size for libicudata.so.66.1 is ~27M, which is quite large for embedded systems and all of them may not even need all locale data.
This patch calls the icudata buildtool during configure on the icudata and utilizes a filter called 'filter.json` ( empty by default) to create the data, default behavior should remain same but someone can add a filter.json in own layer to configure this data, e.g.
{
"localeFilter": { "filterType": "language", "whitelist": [ "en", "de", "it" ]}
}would only generate the locale data for english/german/italian
This would reduce the size of libicudata.so.66.1 to 12M
Ensure that icudata is generated using host-tools so it can deal with endianness correctly, when host and target systems have different endianness
install the icudtata file back into in/ folder so that main build can now pickup this data file instead of regenerating it and wiping out the filter changes that are expected to take effect
[1] https://github.com/unicode-org/icu/blob/master/docs/userguide/icu_data/buildtool.md
Signed-off-by Khem Raj <[email protected]>Suggested-by Wouter Meek <[email protected]>---
v2: Fix build for BE systems
v3: Unset AR and RANLIB too
v4: Install fresh icudata back into data/in foldermeta/recipes-support/icu/icu/filter.json | 2 ++ meta/recipes-support/icu/icu_66.1.bb | 26 +++++++++++++++++++++--- 2 files changed, 25 insertions(+), 3 deletions(-) create mode 100644 meta/recipes-support/icu/icu/filter.jsonmeta/recipes-support/icu/icu/filter.json new file +2
-0@@ -0,0 +1,2 @@ +{ +}meta/recipes-support/icu/icu_66.1.bb +23
-4@@ -18,7 +18,10 @@ ARM_INSTRUCTION_SET_armv4 = "arm" ARM_INSTRUCTION_SET_armv5 = "arm" BASE_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-src.tgz" -SRC_URI = "${BASE_SRC_URI} \ +DATA_SRC_URI = "https://github.com/unicode-org/icu/releases/download/release-${ICU_FOLDER}/icu4c-${ICU_PV}-data.zip" +SRC_URI = "${BASE_SRC_URI};name=code \ + ${DATA_SRC_URI};name=data \ + file://filter.json \ file://icu-pkgdata-large-cmd.patch \ file://fix-install-manx.patch \ file://0001-Fix-big-endian-build.patch \@@ -28,8 +31,25 @@ SRC_URI = "${BASE_SRC_URI} \ SRC_URI_append_class-target = "\ file://0001-Disable-LDFLAGSICUDT-for-Linux.patch \ " -SRC_URI[md5sum] = "b33dc6766711517c98d318447e5110f8" -SRC_URI[sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[code.sha256sum] = "52a3f2209ab95559c1cf0a14f24338001f389615bf00e2585ef3dbc43ecf0a2e" +SRC_URI[data.sha256sum] = "8be647f738891d2beb79d48f99077b3499948430eae6f1be112553b15ab0243e" UPSTREAM_CHECK_REGEX = "icu4c-(?P<pver>\d+(_\d+)+)-src" UPSTREAM_CHECK_URI = "https://github.com/unicode-org/icu/releases" + +do_make_icudata_class-target () { + rm -rf ${S}/data + cp -a ${WORKDIR}/data ${S} + cd ${S} + unset AR CC CPP CXX RANLIB CFLAGS CPPFLAGS CXXFLAGS LDFLAGS + ICU_DATA_FILTER_FILE=${WORKDIR}/filter.json \ + ./runConfigureICU Linux --with-data-packaging=archive + oe_runmake ${PARALLEL_MAKE} + install -Dm644 ${S}/data/out/icudt${ICU_MAJOR_VER}l.dat ${S}/data/in/icudt${ICU_MAJOR_VER}l.dat +} + +do_make_icudata() { + : +} + +addtask make_icudata before do_configure after do_patch --2.26.0