Skip to contentSkip to conversation

Add support for compressing firmware in copy-firmware.sh

PATCH In review Peter Robinson 2 participants 3 messages Jan 23, 2021 – Feb 8, 2021 Kbuild

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 Add support for compressing firmware in copy-firmware.sh

As of kernel 5.3 there's initial support for loading compressed firmware. At this stage the only supported compression methis is "xz -C crc32" but this option brings significant benefits.

---

v2: quote filename for xz command

 Makefile         |  4 ++++
 copy-firmware.sh | 47 +++++++++++++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 16 deletions(-)
Makefile +4 -0
index e1c362f..9a48471 100644
@@ -11,3 +11,7 @@ check:
 install:
 	mkdir -p $(DESTDIR)$(FIRMWAREDIR)
 	./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR)
+
+installcompress:
+	mkdir -p $(DESTDIR)$(FIRMWAREDIR)
+	./copy-firmware.sh -C $(DESTDIR)$(FIRMWAREDIR)
copy-firmware.sh +31 -16

83 lines not on this page · raw message

Signature · 1 line
2.29.2
#1 lore raw
Quoted 9 lines · Peter Robinson

As of kernel 5.3 there's initial support for loading compressed firmware. At this stage the only supported compression methis is "xz -C crc32" but this option brings significant benefits.

---

v2: quote filename for xz command

That seems fixed, I ran into another issue.

Quoted 42 lines
 Makefile         |  4 ++++
 copy-firmware.sh | 47 +++++++++++++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 16 deletions(-)
Makefile +4 -0
index e1c362f..9a48471 100644
@@ -11,3 +11,7 @@ check:
 install:
        mkdir -p $(DESTDIR)$(FIRMWAREDIR)
        ./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR)
+
+installcompress:
+       mkdir -p $(DESTDIR)$(FIRMWAREDIR)
+       ./copy-firmware.sh -C $(DESTDIR)$(FIRMWAREDIR)
copy-firmware.sh +6 -0
index 9b46b63..0dd2e5c 100755
@@ -6,6 +6,7 @@
 
 verbose=:
 prune=no
+compress=no
 
 while test $# -gt 0; do
     case $1 in
@@ -19,6 +20,11 @@ while test $# -gt 0; do
             shift
             ;;
 
+        -C | --compress)
+            compress=yes
+            shift
+            ;;
+
         *)
             if test "x$destdir" != "x"; then
                 echo "ERROR: unknown command-line options: $@"
@@ -31,40 +37,49 @@ while test $# -gt 0; do
     esac

copy-firmware.sh while test $# -gt 0; do · line 47

Quoted 10 lines
copy-firmware.sh +8 -0
 done
 
+if test "x$compress" = "xyes"; then
+    cmpxtn=.xz
+    grep '^File:' WHENCE | sed -e's/^File: *//g' -e's/"//g' | while read f; do
+       test -f "$f" || continue
+       $verbose "compressing $f"
+       xz -C crc32 "$f"
+    done
+fi

This compresses the firmware in the local git directory. If you ever run this once, it'll never compress any updated files if you run it again and instead you'll get a bunch of "xz: <file>: File exists" errors:

https://paste.centos.org/view/75c479da

That seems counter-intuitive and leaves everyone's git tree with a ton of untracked files. Options I see:

1) Compress in place and copy, as-is in the patch, but then go back and remove all the compressed files after the copy
2) Compress to a temp location, move to the destination from there
3) Use xz to directly output to the destination file instead of compress/copy

I kind of like 3. Thoughts?

josh

Quoted 57 lines
copy-firmware.sh +17 -17
+
 grep '^File:' WHENCE | sed -e's/^File: *//g' -e's/"//g' | while read f; do
-    test -f "$f" || continue
-    $verbose "copying file $f"
-    mkdir -p $destdir/$(dirname "$f")
-    cp -d "$f" $destdir/"$f"
+    test -f "$f$cmpxtn" || continue
+    $verbose "copying file $f$cmpxtn"
+    mkdir -p $destdir/$(dirname "$f$cmpxtn")
+    cp -d "$f$cmpxtn" $destdir/"$f$cmpxtn"
 done
 
 grep -E '^Link:' WHENCE | sed -e's/^Link: *//g' -e's/-> //g' | while read f d; do
-    if test -L "$f"; then
-        test -f "$destdir/$f" && continue
-        $verbose "copying link $f"
-        mkdir -p $destdir/$(dirname "$f")
+    if test -L "$f$cmpxtn"; then
+        test -f "$destdir/$f$cmpxtn" && continue
+        $verbose "copying link $f$cmpxtn"
+        mkdir -p $destdir/$(dirname "$f$cmpxtn")
         cp -d "$f" $destdir/"$f"
 
         if test "x$d" != "x"; then
-            target=`readlink "$f"`
+            target=`readlink "$f$cmpxtn"`
 
             if test "x$target" != "x$d"; then
                 $verbose "WARNING: inconsistent symlink target: $target != $d"
             else
                 if test "x$prune" != "xyes"; then
-                    $verbose "WARNING: unneeded symlink detected: $f"
+                    $verbose "WARNING: unneeded symlink detected: $f$cmpxtn"
                 else
-                    $verbose "WARNING: pruning unneeded symlink $f"
-                    rm -f "$f"
+                    $verbose "WARNING: pruning unneeded symlink $f$cmpxtn"
+                    rm -f "$f$cmpxtn"
                 fi
             fi
         else
-            $verbose "WARNING: missing target for symlink $f"
+            $verbose "WARNING: missing target for symlink $f$cmpxtn"
         fi
     else
-        $verbose "creating link $f -> $d"
-        mkdir -p $destdir/$(dirname "$f")
-        ln -sf "$d" "$destdir/$f"
+        $verbose "creating link $f$cmpxtn -> $d$cmpxtn"
+        mkdir -p $destdir/$(dirname "$f$cmpxtn")
+        ln -sf "$d$cmpxtn" "$destdir/$f$cmpxtn"
     fi
 done
 
--

2.29.2

#2 lore raw
Quoted 77 lines · Josh Boyer
Quoted 9 lines · Peter Robinson

As of kernel 5.3 there's initial support for loading compressed firmware. At this stage the only supported compression methis is "xz -C crc32" but this option brings significant benefits.

---

v2: quote filename for xz command

That seems fixed, I ran into another issue.

Quoted 42 lines
 Makefile         |  4 ++++
 copy-firmware.sh | 47 +++++++++++++++++++++++++++++++----------------
 2 files changed, 35 insertions(+), 16 deletions(-)
Makefile +4 -0
index e1c362f..9a48471 100644
@@ -11,3 +11,7 @@ check:
 install:
        mkdir -p $(DESTDIR)$(FIRMWAREDIR)
        ./copy-firmware.sh $(DESTDIR)$(FIRMWAREDIR)
+
+installcompress:
+       mkdir -p $(DESTDIR)$(FIRMWAREDIR)
+       ./copy-firmware.sh -C $(DESTDIR)$(FIRMWAREDIR)
copy-firmware.sh +6 -0
index 9b46b63..0dd2e5c 100755
@@ -6,6 +6,7 @@
 
 verbose=:
 prune=no
+compress=no
 
 while test $# -gt 0; do
     case $1 in
@@ -19,6 +20,11 @@ while test $# -gt 0; do
             shift
             ;;
 
+        -C | --compress)
+            compress=yes
+            shift
+            ;;
+
         *)
             if test "x$destdir" != "x"; then
                 echo "ERROR: unknown command-line options: $@"
@@ -31,40 +37,49 @@ while test $# -gt 0; do
     esac

copy-firmware.sh while test $# -gt 0; do · line 47

Quoted 10 lines
copy-firmware.sh +8 -0
 done
 
+if test "x$compress" = "xyes"; then
+    cmpxtn=.xz
+    grep '^File:' WHENCE | sed -e's/^File: *//g' -e's/"//g' | while read f; do
+       test -f "$f" || continue
+       $verbose "compressing $f"
+       xz -C crc32 "$f"
+    done
+fi

This compresses the firmware in the local git directory. If you ever run this once, it'll never compress any updated files if you run it again and instead you'll get a bunch of "xz: <file>: File exists" errors:

https://paste.centos.org/view/75c479da

That seems counter-intuitive and leaves everyone's git tree with a ton of untracked files. Options I see:

Quoted 6 lines

1) Compress in place and copy, as-is in the patch, but then go back and remove all the compressed files after the copy
2) Compress to a temp location, move to the destination from there
3) Use xz to directly output to the destination file instead of compress/copy

I kind of like 3. Thoughts?

Yes, I think that makes sense, I'd been looking at this from a distro packaging perspective where it's generally run from the tarfile output and then that is disgarded so less of an issue. Will take another pass at it.

Quoted 59 lines

josh

Quoted 57 lines
copy-firmware.sh +17 -17
+
 grep '^File:' WHENCE | sed -e's/^File: *//g' -e's/"//g' | while read f; do
-    test -f "$f" || continue
-    $verbose "copying file $f"
-    mkdir -p $destdir/$(dirname "$f")
-    cp -d "$f" $destdir/"$f"
+    test -f "$f$cmpxtn" || continue
+    $verbose "copying file $f$cmpxtn"
+    mkdir -p $destdir/$(dirname "$f$cmpxtn")
+    cp -d "$f$cmpxtn" $destdir/"$f$cmpxtn"
 done
 
 grep -E '^Link:' WHENCE | sed -e's/^Link: *//g' -e's/-> //g' | while read f d; do
-    if test -L "$f"; then
-        test -f "$destdir/$f" && continue
-        $verbose "copying link $f"
-        mkdir -p $destdir/$(dirname "$f")
+    if test -L "$f$cmpxtn"; then
+        test -f "$destdir/$f$cmpxtn" && continue
+        $verbose "copying link $f$cmpxtn"
+        mkdir -p $destdir/$(dirname "$f$cmpxtn")
         cp -d "$f" $destdir/"$f"
 
         if test "x$d" != "x"; then
-            target=`readlink "$f"`
+            target=`readlink "$f$cmpxtn"`
 
             if test "x$target" != "x$d"; then
                 $verbose "WARNING: inconsistent symlink target: $target != $d"
             else
                 if test "x$prune" != "xyes"; then
-                    $verbose "WARNING: unneeded symlink detected: $f"
+                    $verbose "WARNING: unneeded symlink detected: $f$cmpxtn"
                 else
-                    $verbose "WARNING: pruning unneeded symlink $f"
-                    rm -f "$f"
+                    $verbose "WARNING: pruning unneeded symlink $f$cmpxtn"
+                    rm -f "$f$cmpxtn"
                 fi
             fi
         else
-            $verbose "WARNING: missing target for symlink $f"
+            $verbose "WARNING: missing target for symlink $f$cmpxtn"
         fi
     else
-        $verbose "creating link $f -> $d"
-        mkdir -p $destdir/$(dirname "$f")
-        ln -sf "$d" "$destdir/$f"
+        $verbose "creating link $f$cmpxtn -> $d$cmpxtn"
+        mkdir -p $destdir/$(dirname "$f$cmpxtn")
+        ln -sf "$d$cmpxtn" "$destdir/$f$cmpxtn"
     fi
 done
 
--

2.29.2