Skip to contentSkip to conversation

toolchain/external: allow custom toolchains to use newer headers

PATCH Superseded Yann E. MORIN 1 participant 3 messages Jan 6, 2020 – Jan 7, 2020 1 message not in the archive

v2 of 3 Superseded Latest is v3 →

lore
Share & tools
lore
#0 lore raw

PATCH v2 1/1 toolchain/external: allow custom toolchains to use newer headers

From: Vincent Fazio <[email protected]>

When Buildroot is released, it knows up to a certain kernel header version, and no later. However, it is possible that an external toolchain will be used, that uses headers newer than the latest version Buildroot knows about.

This may also happen when testing a development, rc-class kernel version.

In the current state, Buildroot would refuse to use such toolchains, because the test is for strict equality.

We'd like to make that situation possible, but we also want the user not to be lenient at the same time, and select the right headers version when it is known.

So, we add a new Kconfig blind option that the latest kernel headers version selects. This options is then used to decide whether we do a strict or loose check of the kernel headers.

[yann.morin.1998 at free.fr: only do a loose check for the latest version]

---
Changes v1 -> v2: (Yann)
- make it loose only for the latest version

---
 package/linux-headers/linux-headers.mk        |  3 +-
 support/scripts/check-kernel-headers.sh       | 31 ++++++++++++++++---
 toolchain/helpers.mk                          |  3 +-
 .../pkg-toolchain-external.mk                 |  3 +-
 .../Config.in.options                         | 15 +++++++++
 5 files changed, 48 insertions(+), 7 deletions(-)
package/linux-headers/linux-headers.mk +2 -1
index 676c8c44ea..ee1a6c27d7 100644
@@ -135,7 +135,8 @@ define LINUX_HEADERS_CHECK_VERSION
 	$(call check_kernel_headers_version,\
 		$(BUILD_DIR),\
 		$(STAGING_DIR),\
-		$(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)))
+		$(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)),
+		strict)
 endef
 LINUX_HEADERS_POST_INSTALL_STAGING_HOOKS += LINUX_HEADERS_CHECK_VERSION
 endif
support/scripts/check-kernel-headers.sh +27 -4
index 9d23c00feb..aafa7785c1 100755
@@ -1,9 +1,29 @@
 #!/bin/sh
 
+# This script (and the embedded C code) will check that the actual
+# headers version match the user told us they were:
+#
+# - if both versions are the same, all is well.
+#
+# - if the actual headers are older than the user told us, this is
+#   an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a strict check, then this is an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a loose check, then a warning is printed, but this is
+#   not an error.
+#
+# That last conditions allows a user to provide a pre-built external
+# toolchain that uses kernel headers more recent than what Buildroot
+# knew when it was released.
+
 BUILDDIR="${1}"
 SYSROOT="${2}"
 # Make sure we have enough version components
 HDR_VER="${3}.0.0"
+CHECK="${4}"  # 'strict' or 'loose'
 
 HDR_M="${HDR_VER%%.*}"
 HDR_V="${HDR_VER#*.}"
@@ -32,16 +52,19 @@ ${HOSTCC} -imacros "${SYSROOT}/usr/include/linux/version.h" \
 int main(int argc __attribute__((unused)),
          char** argv __attribute__((unused)))
 {
-    if((LINUX_VERSION_CODE & ~0xFF)
-        != KERNEL_VERSION(${HDR_M},${HDR_m},0))
+    int ret = 0;
+    int l = LINUX_VERSION_CODE & ~0xFF;
+    int h = KERNEL_VERSION(${HDR_M},${HDR_m},0);
+
+    if(l != h)
     {
         printf("Incorrect selection of kernel headers: ");
         printf("expected %d.%d.x, got %d.%d.x\n", ${HDR_M}, ${HDR_m},
                ((LINUX_VERSION_CODE>>16) & 0xFF),
                ((LINUX_VERSION_CODE>>8) & 0xFF));
-        return 1;
+        ret = ((l >= h ) && ("${CHECK}"[0] == 'l')) ? 0 : 1;
     }
-    return 0;
+    return ret;
 }
 _EOF_
 
toolchain/helpers.mk +2 -1
index 996cc70d44..9a1f0495a5 100644
@@ -161,9 +161,10 @@ copy_toolchain_sysroot = \
 # $1: build directory
 # $2: sysroot directory
 # $3: kernel version string, in the form: X.Y
+# $4: test to do, 'strict' or 'loose'
 #
 check_kernel_headers_version = \
-	if ! support/scripts/check-kernel-headers.sh $(1) $(2) $(3); then \
+	if ! support/scripts/check-kernel-headers.sh $(1) $(2) $(3) $(4); then \
 		exit 1; \
 	fi
 
toolchain/toolchain-external/pkg-toolchain-external.mk +2 -1
index 1c43409514..0f1a7a5cd3 100644
@@ -542,7 +542,8 @@ define $(2)_CONFIGURE_CMDS
 	$$(call check_kernel_headers_version,\
 		$$(BUILD_DIR)\
 		$$(call toolchain_find_sysroot,$$(TOOLCHAIN_EXTERNAL_CC)),\
-		$$(call qstrip,$$(BR2_TOOLCHAIN_HEADERS_AT_LEAST))); \
+		$$(call qstrip,$$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)),
+		$(if $(BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST),loose,strict)); \
 	$$(call check_gcc_version,$$(TOOLCHAIN_EXTERNAL_CC),\
 		$$(call qstrip,$$(BR2_TOOLCHAIN_GCC_AT_LEAST))); \
 	if test "$$(BR2_arm)" = "y" ; then \
toolchain/toolchain-external/toolchain-external-custom/Config.in.options +15 -0
index 665765a104..ed0a1b4421 100644
@@ -92,6 +92,13 @@ config BR2_TOOLCHAIN_EXTERNAL_GCC_OLD
 
 endchoice
 
+# This should be selected by a single version, below, to indicate that
+# Buildroot does not know of mor erecent headers than the ones selected.
+# This allows using toolchains with headers more recent than Buildroot
+# knows about, while still enforcing strict check for older headers.
+config BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST
+	bool
+
 choice
 	bool "External toolchain kernel headers series"
 	default BR2_TOOLCHAIN_EXTERNAL_HEADERS_REALLY_OLD
@@ -109,9 +116,17 @@ choice
 	    m = ( LINUX_VERSION_CODE >> 8  ) & 0xFF
 	    p = ( LINUX_VERSION_CODE >> 0  ) & 0xFF
 
+	  If your toolchain uses headers newer than the latest version
+	  in the choice, then select the latest version.
+
+# Note: when adding a new version here, be sure to move the
+# 'select BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST' from the
+# current "latest version" to that new "latest version".
+
 config BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_4
 	bool "5.4.x"
 	select BR2_TOOLCHAIN_HEADERS_AT_LEAST_5_4
+	select BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST
 
 config BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_3
 	bool "5.3.x"
Signature · 1 line
2.20.1
#1 lore raw

PATCH v2 1/1 toolchain/external: allow custom toolchains to use newer headers

All,

On 2020-01-06 21:10 +0100, Yann E. MORIN spake thusly:

Quoted 13 lines

From: Vincent Fazio <[email protected]>

When Buildroot is released, it knows up to a certain kernel header version, and no later. However, it is possible that an external toolchain will be used, that uses headers newer than the latest version Buildroot knows about.

This may also happen when testing a development, rc-class kernel version.

In the current state, Buildroot would refuse to use such toolchains, because the test is for strict equality.

Quoted 7 lines

We'd like to make that situation possible, but we also want the user not to be lenient at the same time, and select the right headers version when it is known.

So, we add a new Kconfig blind option that the latest kernel headers version selects. This options is then used to decide whether we do a strict or loose check of the kernel headers.

As Vincent noticed (on IRC), this is missing the case for internal toolchain with headers-same-as=kernel with kernels more recent than what Buildroot knows about.

I'll send a fixed up version soonish.

Regards,
Yann E. MORIN.

Quoted 157 lines
[yann.morin.1998 at free.fr: only do a loose check for the latest version]

---
Changes v1 -> v2: (Yann)
- make it loose only for the latest version

---
 package/linux-headers/linux-headers.mk        |  3 +-
 support/scripts/check-kernel-headers.sh       | 31 ++++++++++++++++---
 toolchain/helpers.mk                          |  3 +-
 .../pkg-toolchain-external.mk                 |  3 +-
 .../Config.in.options                         | 15 +++++++++
 5 files changed, 48 insertions(+), 7 deletions(-)
package/linux-headers/linux-headers.mk +2 -1
index 676c8c44ea..ee1a6c27d7 100644
@@ -135,7 +135,8 @@ define LINUX_HEADERS_CHECK_VERSION
 	$(call check_kernel_headers_version,\
 		$(BUILD_DIR),\
 		$(STAGING_DIR),\
-		$(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)))
+		$(call qstrip,$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)),
+		strict)
 endef
 LINUX_HEADERS_POST_INSTALL_STAGING_HOOKS += LINUX_HEADERS_CHECK_VERSION
 endif
support/scripts/check-kernel-headers.sh +27 -4
index 9d23c00feb..aafa7785c1 100755
@@ -1,9 +1,29 @@
 #!/bin/sh
 
+# This script (and the embedded C code) will check that the actual
+# headers version match the user told us they were:
+#
+# - if both versions are the same, all is well.
+#
+# - if the actual headers are older than the user told us, this is
+#   an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a strict check, then this is an error.
+#
+# - if the actual headers are more recent than the user told us, and
+#   we are doing a loose check, then a warning is printed, but this is
+#   not an error.
+#
+# That last conditions allows a user to provide a pre-built external
+# toolchain that uses kernel headers more recent than what Buildroot
+# knew when it was released.
+
 BUILDDIR="${1}"
 SYSROOT="${2}"
 # Make sure we have enough version components
 HDR_VER="${3}.0.0"
+CHECK="${4}"  # 'strict' or 'loose'
 
 HDR_M="${HDR_VER%%.*}"
 HDR_V="${HDR_VER#*.}"
@@ -32,16 +52,19 @@ ${HOSTCC} -imacros "${SYSROOT}/usr/include/linux/version.h" \
 int main(int argc __attribute__((unused)),
          char** argv __attribute__((unused)))
 {
-    if((LINUX_VERSION_CODE & ~0xFF)
-        != KERNEL_VERSION(${HDR_M},${HDR_m},0))
+    int ret = 0;
+    int l = LINUX_VERSION_CODE & ~0xFF;
+    int h = KERNEL_VERSION(${HDR_M},${HDR_m},0);
+
+    if(l != h)
     {
         printf("Incorrect selection of kernel headers: ");
         printf("expected %d.%d.x, got %d.%d.x\n", ${HDR_M}, ${HDR_m},
                ((LINUX_VERSION_CODE>>16) & 0xFF),
                ((LINUX_VERSION_CODE>>8) & 0xFF));
-        return 1;
+        ret = ((l >= h ) && ("${CHECK}"[0] == 'l')) ? 0 : 1;
     }
-    return 0;
+    return ret;
 }
 _EOF_
 
toolchain/helpers.mk +2 -1
index 996cc70d44..9a1f0495a5 100644
@@ -161,9 +161,10 @@ copy_toolchain_sysroot = \
 # $1: build directory
 # $2: sysroot directory
 # $3: kernel version string, in the form: X.Y
+# $4: test to do, 'strict' or 'loose'
 #
 check_kernel_headers_version = \
-	if ! support/scripts/check-kernel-headers.sh $(1) $(2) $(3); then \
+	if ! support/scripts/check-kernel-headers.sh $(1) $(2) $(3) $(4); then \
 		exit 1; \
 	fi
 
toolchain/toolchain-external/pkg-toolchain-external.mk +2 -1
index 1c43409514..0f1a7a5cd3 100644
@@ -542,7 +542,8 @@ define $(2)_CONFIGURE_CMDS
 	$$(call check_kernel_headers_version,\
 		$$(BUILD_DIR)\
 		$$(call toolchain_find_sysroot,$$(TOOLCHAIN_EXTERNAL_CC)),\
-		$$(call qstrip,$$(BR2_TOOLCHAIN_HEADERS_AT_LEAST))); \
+		$$(call qstrip,$$(BR2_TOOLCHAIN_HEADERS_AT_LEAST)),
+		$(if $(BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST),loose,strict)); \
 	$$(call check_gcc_version,$$(TOOLCHAIN_EXTERNAL_CC),\
 		$$(call qstrip,$$(BR2_TOOLCHAIN_GCC_AT_LEAST))); \
 	if test "$$(BR2_arm)" = "y" ; then \
toolchain/toolchain-external/toolchain-external-custom/Config.in.options +15 -0
index 665765a104..ed0a1b4421 100644
@@ -92,6 +92,13 @@ config BR2_TOOLCHAIN_EXTERNAL_GCC_OLD
 
 endchoice
 
+# This should be selected by a single version, below, to indicate that
+# Buildroot does not know of mor erecent headers than the ones selected.
+# This allows using toolchains with headers more recent than Buildroot
+# knows about, while still enforcing strict check for older headers.
+config BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST
+	bool
+
 choice
 	bool "External toolchain kernel headers series"
 	default BR2_TOOLCHAIN_EXTERNAL_HEADERS_REALLY_OLD
@@ -109,9 +116,17 @@ choice
 	    m = ( LINUX_VERSION_CODE >> 8  ) & 0xFF
 	    p = ( LINUX_VERSION_CODE >> 0  ) & 0xFF
 
+	  If your toolchain uses headers newer than the latest version
+	  in the choice, then select the latest version.
+
+# Note: when adding a new version here, be sure to move the
+# 'select BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST' from the
+# current "latest version" to that new "latest version".
+
 config BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_4
 	bool "5.4.x"
 	select BR2_TOOLCHAIN_HEADERS_AT_LEAST_5_4
+	select BR2_TOOLCHAIN_EXTERNAL_HEADERS_LATEST
 
 config BR2_TOOLCHAIN_EXTERNAL_HEADERS_5_3
 	bool "5.3.x"
Signature · 1 line
2.20.1
Signature · 6 lines
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'

A message that is not in the archive (sent off-list or not yet received): [email protected]

#3 lore raw

PATCH v2 1/1 toolchain/external: allow custom toolchains to use newer headers

Aaron, Vincent, All,

On 2020-01-06 18:04 -0600, Aaron Sierra spake thusly:

Quoted 10 lines
Quoted 7 lines

From: "Yann E. MORIN" <[email protected]> On 2020-01-06 21:10 +0100, Yann E. MORIN spake thusly:

Quoted 5 lines

From: Vincent Fazio <[email protected]>
When Buildroot is released, it knows up to a certain kernel header version, and no later. However, it is possible that an external toolchain will be used, that uses headers newer than the latest version Buildroot knows about.

Vincent and I were looking through your v2 patch and started putting together a truth table. I finished it after he left for the day, so be aware that he hasn't reviewed it at this point:

Thanks for the feedback and the complete tables. :-)

Quoted 20 lines

= equals latest BR is aware
< less than latest BR is aware

Quoted 1 line

greater than latest BR is aware

building   | selected | toolchain | OK?
toolchain? | headers  | headers   |
-------------------------------------------
no         | =        |  =        | yes
no         | =        |  <        | no (user lied about toolchain headers)
no         | =        |  >        | yes (primary goal of patchset)
no         | <        |  =        | no (for BR support purposes)
no         | <        |  <        | yes, if equal (for BR support purposes)
no         | <        |  >        | no (for BR support purposes)
no         | >        |  =        | not possible
no         | >        |  <        | not possible
no         | >        |  >        | not possible
yes        | =        |  =        | yes (non-custom are self-consistent)

These are all custom kernel headers.

Quoted 8 lines
yes        | =        |  <        | no (user lied about toolchain headers)
yes        | =        |  >        | yes (desirable side-effect of primary goal)
yes        | <        |  =        | no (for BR support purposes)
yes        | <        |  <        | yes, if equal (for BR support purposes)
yes        | <        |  >        | no (for BR support purposes)
yes        | >        |  =        | not possible
yes        | >        |  <        | not possible
yes        | >        |  >        | not possible

Those tables are based on comparing the latest version known to Buildroot, with the selected version and the actual version.

What we really want to test in Buildroot, is that he selected version is the same as the actual version. This is the core of the check.

Now we want to relax that check a little, when the latest version is custom and higher than the actual version and the latest know to Buildroot.
So the logic is as such:

if selected version == actual version:
return OK

if not latest:
return KO

if selected version < actual version:
return OK

return KO

Quoted 5 lines
Quoted 1 line
Quoted 1 line

+ ret = ((l >= h ) && ("${CHECK}"[0] == 'l')) ? 0 : 1;

Based on the truth table and descriptive summaries, this test appears to be inverted for the case that Vincent and I are most interested in. This version is subtly different, but seems easier to maintain to me:

I tested this script as follows, with sysroot set to '' (empty) to use my distros' headers, which are 5.0.0:

    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  4.9.0 loose; echo $?
    Incorrect selection of kernel headers: expected 4.9.x, got 5.0.x
    0
    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  5.0.0 loose; echo $?
    0
    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  5.1.0 loose; echo $?
    Incorrect selection of kernel headers: expected 5.1.x, got 5.0.x
    1
    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  4.9.0 strict; echo $?
    Incorrect selection of kernel headers: expected 4.9.x, got 5.0.x
    1
    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  5.0.0 strict; echo $?
    0
    $ HOSTCC=gcc ./support/scripts/check-kernel-headers.sh /home/ymorin/dev/buildroot/buildroot  5.1.0 strict; echo $?
    Incorrect selection of kernel headers: expected 5.1.x, got 5.0.x
    1

So as far as I can see, this script has the expected behaviour, with the first case equivalent to what you expect (custom headers newer than latest version known to BR).

Quoted 2 lines

+ if ("${CHECK}"[0] == 's' || (l > h))
+ ret = 1;

Actually, I think you got it wrong: 'l' is the actual version, and 'h' is the version specified by the user, so you want to allow that.

Of course, I may have entirely missed something (which seems to be quite usual these days...)

Quoted 9 lines

[snip]

Quoted 8 lines
Quoted 8 lines

diff --git a/toolchain/toolchain-external/toolchain-external-custom/Config.in.options b/toolchain/toolchain-external/toolchain-external-custom/Config.in.options index 665765a104..ed0a1b4421 100644

toolchain/toolchain-external/toolchain-external-custom/Config.in.options +0 -0
@@ -92,6 +92,13 @@ config BR2_TOOLCHAIN_EXTERNAL_GCC_OLD
 
Quoted 6 lines
Quoted 4 lines

toolchain/toolchain-external/toolchain-external-custom/Config.in.options config BR2_TOOLCHAIN_EXTERNAL_GCC_OLD · line 96

Quoted 4 lines
toolchain/toolchain-external/toolchain-external-custom/Config.in.options +2 -0
 endchoice
 
+# This should be selected by a single version, below, to indicate that
+# Buildroot does not know of mor erecent headers than the ones selected.

I'm sure you meant "more recent" ;)

I've reworked that part in the new commit, and the code has moved somewhere else, and this comment is no longer needed.

Thanks for the review! :-)

Regards,
Yann E. MORIN.

Quoted 1 line

Aaron

Signature · 6 lines
.-----------------.--------------------.------------------.--------------------.
|  Yann E. MORIN  | Real-Time Embedded | /"\ ASCII RIBBON | Erics' conspiracy: |
| +33 662 376 056 | Software  Designer | \ / CAMPAIGN     |  ___               |
| +33 561 099 427 `------------.-------:  X  AGAINST      |  \e/  There is no  |
| http://ymorin.is-a-geek.org/ | _/*\_ | / \ HTML MAIL    |   v   conspiracy.  |
'------------------------------^-------^------------------^--------------------'