firmware: Add support for loading compressed files
Subsystems (3)
v2 cover firmware: Add support for loading compressed files
Hi,
here are the rest and the main part of patches to add the support for loading the compressed firmware files. The patch was slightly refactored for more easily enhancing for other compression formats (if anyone wants). Also the selftest patch is included. The functionality doesn't change from the previous patchset.
thanks,
Takashi
===
Takashi Iwai (2):
firmware: Add support for loading compressed files
selftests: firmware: Add compressed firmware tests
drivers/base/firmware_loader/Kconfig | 18 +++ drivers/base/firmware_loader/firmware.h | 8 +- drivers/base/firmware_loader/main.c | 147 ++++++++++++++++++++-- tools/testing/selftests/firmware/fw_filesystem.sh | 73 +++++++++-- tools/testing/selftests/firmware/fw_lib.sh | 7 ++ tools/testing/selftests/firmware/fw_run_tests.sh | 1 + 6 files changed, 232 insertions(+), 22 deletions(-)
Signature · 1 line
2.16.4
PATCH v2 1/2 firmware: Add support for loading compressed files
This patch adds the support for loading compressed firmware files. The primary motivation is to reduce the storage size; e.g. currently the files in /lib/firmware on my machine counts up to 419MB, while they can be reduced to 130MB by file compression.
The patch introduces a new kconfig option CONFIG_FW_LOADER_COMPRESS. Even with this option set, the firmware loader still tries to load the original firmware file as-is at first, but then falls back to the file with ".xz" extension when it's not found, and the decompressed file content is returned to the caller of request_firmware(). So, no change is needed for the rest.
Currently only XZ format is supported. A caveat is that the kernel XZ helper code supports only CRC32 (or none) integrity check type, so you'll have to compress the files via xz -C crc32 option.
Since we can't determine the expanded size immediately from an XZ file, the patch re-uses the paged buffer that was used for the user-mode fallback; it puts the decompressed content page, which are vmapped at the end. The paged buffer code is conditionally built with a new Kconfig that is selected automatically.
---v1->v2: Slight code refactoring, no functional changes
drivers/base/firmware_loader/Kconfig | 18 ++++ drivers/base/firmware_loader/firmware.h | 8 +- drivers/base/firmware_loader/main.c | 147 ++++++++++++++++++++++++++++++-- 3 files changed, 161 insertions(+), 12 deletions(-)
drivers/base/firmware_loader/Kconfig +18 -0
@@ -26,6 +26,9 @@ config FW_LOADER if FW_LOADER +config FW_LOADER_PAGED_BUF + bool + config EXTRA_FIRMWARE string "Build named firmware blobs into the kernel binary" help
@@ -67,6 +70,7 @@ config EXTRA_FIRMWARE_DIR config FW_LOADER_USER_HELPER bool "Enable the firmware sysfs fallback mechanism" + select FW_LOADER_PAGED_BUF help This option enables a sysfs loading facility to enable firmware loading to the kernel through userspace as a fallback mechanism
@@ -151,5 +155,19 @@ config FW_LOADER_USER_HELPER_FALLBACK If you are unsure about this, say N here. +config FW_LOADER_COMPRESS + bool "Enable compressed firmware support" + select FW_LOADER_PAGED_BUF + select XZ_DEC + help + This option enables the support for loading compressed firmware + files. The caller of firmware API receives the decompressed file + content. The compressed file is loaded as a fallback, only after + loading the raw file failed at first. + + Currently only XZ-compressed files are supported, and they have to + be compressed with either none or crc32 integrity check type (pass + "-C crc32" option to xz command). + endif # FW_LOADER endmenu
drivers/base/firmware_loader/firmware.h +5 -3
@@ -64,12 +64,14 @@ struct fw_priv {void *data;size_t size;size_t allocated_size; -#ifdef CONFIG_FW_LOADER_USER_HELPER +#ifdef CONFIG_FW_LOADER_PAGED_BUF bool is_paged_buf; -bool need_uevent;struct page **pages;int nr_pages;int page_array_size; +#endif +#ifdef CONFIG_FW_LOADER_USER_HELPER +bool need_uevent;struct list_head pending_list;#endif const char *fw_name;
@@ -133,7 +135,7 @@ static inline void fw_state_done(struct fw_priv *fw_priv)int assign_fw (struct firmware *fw,struct device *device,enum fw_opt opt_flags); -#ifdef CONFIG_FW_LOADER_USER_HELPER +#ifdef CONFIG_FW_LOADER_PAGED_BUF void fw_free_paged_buf (struct fw_priv *fw_priv);int fw_grow_paged_buf (struct fw_priv *fw_priv,int pages_needed);int fw_map_paged_buf (struct fw_priv *fw_priv);
drivers/base/firmware_loader/main.c +138 -9
203 lines not on this page · raw message
Signature · 1 line
2.16.4
PATCH v2 2/2 selftests: firmware: Add compressed firmware tests
In reply to Takashi IwaiThis patch adds the test cases for checking compressed firmware load. Two more cases are added to fw_filesystem.sh:
- Both a plain file and an xz file are present, and load the former
- Only an xz file is present, and load without '.xz' suffix
The tests are enabled only when CONFIG_FW_LOADER_COMPRESS is enabled and xz program is installed.
--- tools/testing/selftests/firmware/fw_filesystem.sh | 73 +++++++++++++++++++---- tools/testing/selftests/firmware/fw_lib.sh | 7 +++ tools/testing/selftests/firmware/fw_run_tests.sh | 1 + 3 files changed, 71 insertions(+), 10 deletions(-)
tools/testing/selftests/firmware/fw_filesystem.sh +63 -10
@@ -153,13 +153,18 @@ config_set_read_fw_idx() read_firmwares() { + if [ "$1" = "xzonly" ]; then + fwfile="${FW}-orig" + else + fwfile="$FW" + fi for i in $(seq 0 3); do config_set_read_fw_idx $i # Verify the contents are what we expect. # -Z required for now -- check for yourself, md5sum # on $FW and DIR/read_firmware will yield the same. Even # cmp agrees, so something is off. - if ! diff -q -Z "$FW" $DIR/read_firmware 2>/dev/null ; then + if ! diff -q -Z "$fwfile" $DIR/read_firmware 2>/dev/null ; then echo "request #$i: firmware was not loaded" >&2 exit 1 fi
@@ -246,17 +251,17 @@ test_request_firmware_nowait_custom_nofile() test_batched_request_firmware() { - echo -n "Batched request_firmware() try #$1: " + echo -n "Batched request_firmware() $2 try #$1: " config_reset config_trigger_sync - read_firmwares + read_firmwares $2 release_all_firmware echo "OK" } test_batched_request_firmware_direct() { - echo -n "Batched request_firmware_direct() try #$1: " + echo -n "Batched request_firmware_direct() $2 try #$1: " config_reset config_set_sync_direct config_trigger_sync
@@ -266,7 +271,7 @@ test_batched_request_firmware_direct() test_request_firmware_nowait_uevent() { - echo -n "Batched request_firmware_nowait(uevent=true) try #$1: " + echo -n "Batched request_firmware_nowait(uevent=true) $2 try #$1: " config_reset config_trigger_async release_all_firmware
@@ -275,11 +280,16 @@ test_request_firmware_nowait_uevent() test_request_firmware_nowait_custom() { - echo -n "Batched request_firmware_nowait(uevent=false) try #$1: " + echo -n "Batched request_firmware_nowait(uevent=false) $2 try #$1: " config_reset config_unset_uevent RANDOM_FILE_PATH=$(setup_random_file) RANDOM_FILE="$(basename $RANDOM_FILE_PATH)" + if [ "$2" = "both" ]; then + xz -9 -C crc32 -k $RANDOM_FILE_PATH + elif [ "$2" = "xzonly" ]; then + xz -9 -C crc32 $RANDOM_FILE_PATH + fi config_set_name $RANDOM_FILE config_trigger_async release_all_firmware
@@ -294,19 +304,19 @@ test_config_present echo echo "Testing with the file present..." for i in $(seq 1 5); do - test_batched_request_firmware $i + test_batched_request_firmware $i normal done for i in $(seq 1 5); do - test_batched_request_firmware_direct $i + test_batched_request_firmware_direct $i normal done for i in $(seq 1 5); do - test_request_firmware_nowait_uevent $i + test_request_firmware_nowait_uevent $i normal done for i in $(seq 1 5); do - test_request_firmware_nowait_custom $i + test_request_firmware_nowait_custom $i normal done # Test for file not found, errors are expected, the failure would be
@@ -329,4 +339,47 @@ for i in $(seq 1 5); do test_request_firmware_nowait_custom_nofile $i done +test "$HAS_FW_LOADER_COMPRESS" != "yes" && exit 0 + +# test with both files present +xz -9 -C crc32 -k $FW +config_set_name $NAME +echo +echo "Testing with both plain and xz files present..." +for i in $(seq 1 5); do + test_batched_request_firmware $i both +done + +for i in $(seq 1 5); do + test_batched_request_firmware_direct $i both +done + +for i in $(seq 1 5); do + test_request_firmware_nowait_uevent $i both +done + +for i in $(seq 1 5); do + test_request_firmware_nowait_custom $i both +done + +# test with only xz file present +mv "$FW" "${FW}-orig" +echo +echo "Testing with only xz file present..." +for i in $(seq 1 5); do + test_batched_request_firmware $i xzonly +done + +for i in $(seq 1 5); do + test_batched_request_firmware_direct $i xzonly +done + +for i in $(seq 1 5); do + test_request_firmware_nowait_uevent $i xzonly +done + +for i in $(seq 1 5); do + test_request_firmware_nowait_custom $i xzonly +done + exit 0
tools/testing/selftests/firmware/fw_lib.sh +7 -0
@@ -50,6 +50,7 @@ check_setup() { HAS_FW_LOADER_USER_HELPER="$(kconfig_has CONFIG_FW_LOADER_USER_HELPER=y)" HAS_FW_LOADER_USER_HELPER_FALLBACK="$(kconfig_has CONFIG_FW_LOADER_USER_HELPER_FALLBACK=y)" + HAS_FW_LOADER_COMPRESS="$(kconfig_has CONFIG_FW_LOADER_COMPRESS=y)" PROC_FW_IGNORE_SYSFS_FALLBACK="0" PROC_FW_FORCE_SYSFS_FALLBACK="0"
@@ -84,6 +85,12 @@ check_setup() fi OLD_FWPATH="$(cat /sys/module/firmware_class/parameters/path)" + + if [ "$HAS_FW_LOADER_COMPRESS" = "yes" ]; then + if ! which xz 2> /dev/null > /dev/null; then + HAS_FW_LOADER_COMPRESS="" + fi + fi } verify_reqs()
tools/testing/selftests/firmware/fw_run_tests.sh +1 -0
@@ -11,6 +11,7 @@ source $TEST_DIR/fw_lib.sh export HAS_FW_LOADER_USER_HELPER="" export HAS_FW_LOADER_USER_HELPER_FALLBACK="" +export HAS_FW_LOADER_COMPRESS="" run_tests() {
Signature · 1 line
2.16.4
Quoted 12 lines · Takashi Iwai
Hi,
here are the rest and the main part of patches to add the support for loading the compressed firmware files. The patch was slightly refactored for more easily enhancing for other compression formats (if anyone wants). Also the selftest patch is included. The functionality doesn't change from the previous patchset.
thanks,
Quoted 7 lines
Takashi
===
Takashi Iwai (2):
firmware: Add support for loading compressed files
selftests: firmware: Add compressed firmware tests
Ouch, one preliminary patch is missing in the series. Will resubmit again.
Sorry for the mess.
Takashi