efi_loader: EFI_UNICODE_COLLATION_PROTOCOL

charset.o is only needed for the EFI subsystem

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/Makefile   |  5 ++++-
 lib/vsprintf.c | 12 ++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

lib/Makefile +4 -1

index 5f583aed37..2fd32798a0 100644
@@ -19,7 +19,10 @@ obj-$(CONFIG_ARCH_AT91) += at91/
 obj-$(CONFIG_OPTEE) += optee/
 
 obj-$(CONFIG_AES) += aes.o
-obj-y += charset.o
+
+ifndef API_BUILD
+obj-$(CONFIG_EFI_LOADER) += charset.o
+endif
 obj-$(CONFIG_USB_TTY) += circbuf.o
 obj-y += crc7.o
 obj-y += crc8.o

lib/vsprintf.c +0 -0

index 914fbd30cb..6100357858 100644

lib/vsprintf.c +0 -2

 static char *string16(char *buf, char *end, u16 *s, int field_width,
 		int precision, int flags)
 {

@@ -294,8 +296,6 @@ static char *string16(char *buf, char *end, u16 *s, int field_width,
 	return buf;
 }
 
-#if defined(CONFIG_EFI_LOADER) && \
-	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)
 static char *device_path_string(char *buf, char *end, void *dp, int field_width,
 				int precision, int flags)
 {

lib/vsprintf.c +4 -1

+			if (qualifier == 'l') {
 				str = string16(str, end, va_arg(args, u16 *),
 					       field_width, precision, flags);
-			} else {
+			} else
+#endif
+			{
 				str = string(str, end, va_arg(args, char *),
 					     field_width, precision, flags);
 			}

Quoted 32 lines · Heinrich Schuchardt

charset.o is only needed for the EFI subsystem

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/Makefile   |  5 ++++-
 lib/vsprintf.c | 12 ++++++++----
 2 files changed, 12 insertions(+), 5 deletions(-)

lib/Makefile +4 -1

index 5f583aed37..2fd32798a0 100644
@@ -19,7 +19,10 @@ obj-$(CONFIG_ARCH_AT91) += at91/
 obj-$(CONFIG_OPTEE) += optee/
 
 obj-$(CONFIG_AES) += aes.o
-obj-y += charset.o
+
+ifndef API_BUILD
+obj-$(CONFIG_EFI_LOADER) += charset.o
+endif
 obj-$(CONFIG_USB_TTY) += circbuf.o
 obj-y += crc7.o
 obj-y += crc8.o

lib/vsprintf.c +0 -0

index 914fbd30cb..6100357858 100644
@@ -274,6 +274,8 @@ static char *string(char *buf, char *end, char *s, int field_width,
 	return buf;
 }
 

lib/vsprintf.c static char *string(char *buf, char *end, char *s, int field_width, · line 278

Quoted 2 lines

lib/vsprintf.c +2 -0

+#if defined(CONFIG_EFI_LOADER) && \
+	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)

I believe you want #if CONFIG_IS_ENABLED(EFI_LOADER) here, right? We probably should move to that at other places where we explicitly check for SPL_BUILD too.

Quoted 16 lines

lib/vsprintf.c +0 -2

 static char *string16(char *buf, char *end, u16 *s, int field_width,
 		int precision, int flags)
 {

@@ -294,8 +296,6 @@ static char *string16(char *buf, char *end, u16 *s, int field_width,
 	return buf;
 }
 
-#if defined(CONFIG_EFI_LOADER) && \
-	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)
 static char *device_path_string(char *buf, char *end, void *dp, int field_width,
 				int precision, int flags)
 {

@@ -612,10 +612,14 @@ repeat:
 			continue;
 

lib/vsprintf.c repeat: · line 616

Quoted 4 lines

lib/vsprintf.c +2 -1

 		case 's':
-			if (qualifier == 'l' && !IS_ENABLED(CONFIG_SPL_BUILD)) {
+#if defined(CONFIG_EFI_LOADER) && \
+	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)

This #if deserves a comment.

Alex

Quoted 11 lines

lib/vsprintf.c +4 -1

+			if (qualifier == 'l') {
 				str = string16(str, end, va_arg(args, u16 *),
 					       field_width, precision, flags);
-			} else {
+			} else
+#endif
+			{
 				str = string(str, end, va_arg(args, char *),
 					     field_width, precision, flags);
 			}

The function names utf16_strlen() and utf16_strnlen() are misnomers. The functions do not count utf-16 characters but non-zero words. So let's rename them to u16_strlen and u16_strnlen().

In utf16_dup() avoid assignment in if clause.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 include/charset.h             | 28 +++++++++++-----------------
 lib/charset.c                 | 10 +++++++---
 lib/efi_loader/efi_bootmgr.c  |  2 +-
 lib/efi_loader/efi_console.c  |  2 +-
 lib/efi_loader/efi_file.c     |  2 +-
 lib/efi_loader/efi_variable.c |  2 +-
 lib/vsprintf.c                |  2 +-
 7 files changed, 23 insertions(+), 25 deletions(-)

include/charset.h +0 -0

index 11832cbd12..2307559890 100644

include/charset.h +2 -1

+ * ReturnValue:		number of non-zero words.
+ *			This is not the number of utf-16 letters!
  */
-size_t utf16_strlen(const uint16_t *in);

Quoted 22 lines · Heinrich Schuchardt

The function names utf16_strlen() and utf16_strnlen() are misnomers. The functions do not count utf-16 characters but non-zero words. So let's rename them to u16_strlen and u16_strnlen().

In utf16_dup() avoid assignment in if clause.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 include/charset.h             | 28 +++++++++++-----------------
 lib/charset.c                 | 10 +++++++---
 lib/efi_loader/efi_bootmgr.c  |  2 +-
 lib/efi_loader/efi_console.c  |  2 +-
 lib/efi_loader/efi_file.c     |  2 +-
 lib/efi_loader/efi_variable.c |  2 +-
 lib/vsprintf.c                |  2 +-
 7 files changed, 23 insertions(+), 25 deletions(-)

include/charset.h +0 -0

index 11832cbd12..2307559890 100644

include/charset.h line 16

Quoted 5 lines

include/charset.h +1 -1

@@ -13,29 +13,23 @@
 #define MAX_UTF8_PER_UTF16 3
 
 /**
- * utf16_strlen() - Get the length of an utf16 string
+ * u16_strlen - count non-zero words

This really just implements wcslen() now, right?

include/charset.h line 18

Quoted 7 lines

include/charset.h +1 -5

  *
- * Returns the number of 16 bit characters in an utf16 string, not
- * including the terminating NULL character.
- *
- * @in     the string to measure
- * @return the string length
+ * @in:			utf-16 string

Is "in" really a utf-16 string? Probably rather a null-terminated string of words.

Quoted 7 lines

include/charset.h +3 -1

+ * ReturnValue:		number of non-zero words.
+ *			This is not the number of utf-16 letters!
  */
-size_t utf16_strlen(const uint16_t *in);
+size_t u16_strlen(const u16 *in);
 

include/charset.h line 25

Quoted 8 lines

include/charset.h +1 -6

 /**
- * utf16_strnlen() - Get the length of a fixed-size utf16 string.
- *
- * Returns the number of 16 bit characters in an utf16 string,
- * not including the terminating NULL character, but at most
- * 'count' number of characters.  In doing this, utf16_strnlen()
- * looks at only the first 'count' characters.
+ * u16_strlen - count non-zero words

This really just implements wcsnlen() now, right?

include/charset.h line 27

Quoted 6 lines

include/charset.h +1 -3

  *
- * @in     the string to measure
- * @count  the maximum number of characters to count
- * @return the string length, up to a maximum of 'count'
+ * @in:			utf-16 string

Same comment here.

Alex

Quoted 41 lines · Alexander Graf

Quoted 22 lines · Heinrich Schuchardt

The function names utf16_strlen() and utf16_strnlen() are misnomers. The functions do not count utf-16 characters but non-zero words. So let's rename them to u16_strlen and u16_strnlen().

In utf16_dup() avoid assignment in if clause.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 include/charset.h             | 28 +++++++++++-----------------
 lib/charset.c                 | 10 +++++++---
 lib/efi_loader/efi_bootmgr.c  |  2 +-
 lib/efi_loader/efi_console.c  |  2 +-
 lib/efi_loader/efi_file.c     |  2 +-
 lib/efi_loader/efi_variable.c |  2 +-
 lib/vsprintf.c                |  2 +-
 7 files changed, 23 insertions(+), 25 deletions(-)

include/charset.h +0 -0

index 11832cbd12..2307559890 100644

include/charset.h line 16

Quoted 5 lines

include/charset.h +1 -1

@@ -13,29 +13,23 @@
 #define MAX_UTF8_PER_UTF16 3
 
 /**
- * utf16_strlen() - Get the length of an utf16 string
+ * u16_strlen - count non-zero words

This really just implements wcslen() now, right?

include/charset.h line 18

Quoted 7 lines

include/charset.h +1 -5

  *
- * Returns the number of 16 bit characters in an utf16 string, not
- * including the terminating NULL character.
- *
- * @in     the string to measure
- * @return the string length
+ * @in:			utf-16 string

Quoted 2 lines

Is "in" really a utf-16 string? Probably rather a null-terminated string of words.

I will update the comment.

Quoted 7 lines

Quoted 7 lines

include/charset.h +3 -1

+ * ReturnValue:		number of non-zero words.
+ *			This is not the number of utf-16 letters!
  */
-size_t utf16_strlen(const uint16_t *in);
+size_t u16_strlen(const u16 *in);
 

Quoted 10 lines

include/charset.h line 25

Quoted 8 lines

include/charset.h +1 -6

 /**
- * utf16_strnlen() - Get the length of a fixed-size utf16 string.
- *
- * Returns the number of 16 bit characters in an utf16 string,
- * not including the terminating NULL character, but at most
- * 'count' number of characters.  In doing this, utf16_strnlen()
- * looks at only the first 'count' characters.
+ * u16_strlen - count non-zero words

This really just implements wcsnlen() now, right?

Currently we have set wchar size to 16bit using a compiler flag. In my opinion this was not necessary. In C11 we could have use the u"text" notation for utf-8 string constants instead of L"text".

This function really is for u16[] and not for wchar_t[].

I would hesitate to call this function wcsnlen() as the working of wcsnlen() depends on said compiler setting.

Quoted 65 lines · Heinrich Schuchardt

Quoted 41 lines · Alexander Graf

Quoted 22 lines · Heinrich Schuchardt

The function names utf16_strlen() and utf16_strnlen() are misnomers. The functions do not count utf-16 characters but non-zero words. So let's rename them to u16_strlen and u16_strnlen().

In utf16_dup() avoid assignment in if clause.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 include/charset.h             | 28 +++++++++++-----------------
 lib/charset.c                 | 10 +++++++---
 lib/efi_loader/efi_bootmgr.c  |  2 +-
 lib/efi_loader/efi_console.c  |  2 +-
 lib/efi_loader/efi_file.c     |  2 +-
 lib/efi_loader/efi_variable.c |  2 +-
 lib/vsprintf.c                |  2 +-
 7 files changed, 23 insertions(+), 25 deletions(-)

include/charset.h +0 -0

index 11832cbd12..2307559890 100644

include/charset.h line 16

Quoted 5 lines

include/charset.h +1 -1

@@ -13,29 +13,23 @@
 #define MAX_UTF8_PER_UTF16 3
 
 /**
- * utf16_strlen() - Get the length of an utf16 string
+ * u16_strlen - count non-zero words

This really just implements wcslen() now, right?

include/charset.h line 18

Quoted 7 lines

include/charset.h +1 -5

  *
- * Returns the number of 16 bit characters in an utf16 string, not
- * including the terminating NULL character.
- *
- * @in     the string to measure
- * @return the string length
+ * @in:			utf-16 string

Quoted 2 lines

Is "in" really a utf-16 string? Probably rather a null-terminated string of words.

I will update the comment.

Quoted 17 lines

Quoted 5 lines

include/charset.h +2 -1

+ * ReturnValue:		number of non-zero words.
+ *			This is not the number of utf-16 letters!
  */
-size_t utf16_strlen(const uint16_t *in);

include/charset.h line 25

Quoted 10 lines

include/charset.h +2 -6

+size_t u16_strlen(const u16 *in);
 
 /**
- * utf16_strnlen() - Get the length of a fixed-size utf16 string.
- *
- * Returns the number of 16 bit characters in an utf16 string,
- * not including the terminating NULL character, but at most
- * 'count' number of characters.  In doing this, utf16_strnlen()
- * looks at only the first 'count' characters.
+ * u16_strlen - count non-zero words

This really just implements wcsnlen() now, right?

Quoted 3 lines

Currently we have set wchar size to 16bit using a compiler flag. In my opinion this was not necessary. In C11 we could have use the u"text" notation for utf-8 string constants instead of L"text".

I thought the idea was to get utf-16 string constants?

Quoted 6 lines

This function really is for u16[] and not for wchar_t[].

I would hesitate to call this function wcsnlen() as the working of wcsnlen() depends on said compiler setting.

*shrug* either way works for me. By calling them their official names we could've potentially given gcc the chance to optimize/inline them better.

Alex

Quoted 72 lines · Alexander Graf

Quoted 68 lines · Heinrich Schuchardt

Quoted 41 lines · Alexander Graf

Quoted 22 lines · Heinrich Schuchardt

The function names utf16_strlen() and utf16_strnlen() are misnomers. The functions do not count utf-16 characters but non-zero words. So let's rename them to u16_strlen and u16_strnlen().

In utf16_dup() avoid assignment in if clause.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 include/charset.h             | 28 +++++++++++-----------------
 lib/charset.c                 | 10 +++++++---
 lib/efi_loader/efi_bootmgr.c  |  2 +-
 lib/efi_loader/efi_console.c  |  2 +-
 lib/efi_loader/efi_file.c     |  2 +-
 lib/efi_loader/efi_variable.c |  2 +-
 lib/vsprintf.c                |  2 +-
 7 files changed, 23 insertions(+), 25 deletions(-)

include/charset.h +0 -0

index 11832cbd12..2307559890 100644

include/charset.h line 16

Quoted 5 lines

include/charset.h +1 -1

@@ -13,29 +13,23 @@
 #define MAX_UTF8_PER_UTF16 3
 
 /**
- * utf16_strlen() - Get the length of an utf16 string
+ * u16_strlen - count non-zero words

This really just implements wcslen() now, right?

include/charset.h line 18

Quoted 7 lines

include/charset.h +1 -5

  *
- * Returns the number of 16 bit characters in an utf16 string, not
- * including the terminating NULL character.
- *
- * @in     the string to measure
- * @return the string length
+ * @in:			utf-16 string

Quoted 2 lines

Is "in" really a utf-16 string? Probably rather a null-terminated string of words.

I will update the comment.

Quoted 7 lines

Quoted 7 lines

include/charset.h +3 -1

+ * ReturnValue:		number of non-zero words.
+ *			This is not the number of utf-16 letters!
  */
-size_t utf16_strlen(const uint16_t *in);
+size_t u16_strlen(const u16 *in);
 

Quoted 10 lines

include/charset.h line 25

Quoted 8 lines

include/charset.h +1 -6

 /**
- * utf16_strnlen() - Get the length of a fixed-size utf16 string.
- *
- * Returns the number of 16 bit characters in an utf16 string,
- * not including the terminating NULL character, but at most
- * 'count' number of characters.  In doing this, utf16_strnlen()
- * looks at only the first 'count' characters.
+ * u16_strlen - count non-zero words

This really just implements wcsnlen() now, right?

Currently we have set wchar size to 16bit using a compiler flag. In my opinion this was not necessary. In C11 we could have use the u"text" notation for utf-8 string constants instead of L"text".

Quoted 1 line

I thought the idea was to get utf-16 string constants?

Yes, but the C11 way is not using a compiler flag but using the right prefix (which was not available with C99). So once we do not have to support outdated gcc versions anymore we could get rid of the compiler flag.

Quoted 5 lines

Quoted 5 lines

This function really is for u16[] and not for wchar_t[].

Quoted 5 lines

Quoted 2 lines

I would hesitate to call this function wcsnlen() as the working of wcsnlen() depends on said compiler setting.

*shrug* either way works for me. By calling them their official names we could've potentially given gcc the chance to optimize/inline them better.

Do have any hint that gcc really provides better compilation results based on the function name?

utf8_get() - get next UTF-8 code point from buffer
utf8_put() - write UTF-8 code point to buffer utf8_utf16_strnlen() - length of a utf-8 string after conversion to utf-16 utf8_utf16_strncpy() - copy a utf-8 string to utf-16 utf16_get() - get next UTF-16 code point from buffer utf16_put() - write UTF-16 code point to buffer utf16_utf8_strnlen() - length of a utf-16 string after conversion to utf-8 utf16_utf8_strncpy() - copy a utf-16 string to utf-8

Signed-off-by Heinrich Schuchardt <[email protected]>

Provide unit tests for Unicode functions.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS           |   1 +
 include/test/suites.h |   3 +-
 test/Kconfig          |   8 +
 test/Makefile         |   1 +
 test/cmd_ut.c         |  14 +-
 test/unicode_ut.c     | 470 ++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 493 insertions(+), 4 deletions(-)
 create mode 100644 test/unicode_ut.c

MAINTAINERS +1 -0

index 51a1472cf8..a324139471 100644
@@ -374,6 +374,7 @@ F:	include/asm-generic/pe.h
 F:	lib/charset.c
 F:	lib/efi*/
 F:	test/py/tests/test_efi*
+F:	test/unicode_ut.c
 F:	cmd/bootefi.c
 F:	tools/file2include.c
 

include/test/suites.h +2 -1

index b5019a7cd2..8e4eac60eb 100644
@@ -23,10 +23,11 @@ struct unit_test;
 int cmd_ut_category(const char *name, struct unit_test *tests, int n_ents,
 		    int argc, char * const argv[]);
 
+int do_ut_compression(cmd_tbl_t *cmdtp, int flag, int argc, char *const argv[]);
 int do_ut_dm(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_env(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_overlay(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_time(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_compression(cmd_tbl_t *cmdtp, int flag, int argc, char *const argv[]);
+int do_ut_unicode(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 
 #endif /* __TEST_SUITES_H__ */

test/Kconfig +0 -0

index 3643761bc6..de16d179d0 100644

test/Kconfig +7 -0

+	bool "Unit tests for Unicode functions"
+	depends on UNIT_TEST
+	default y
+	help
+	  Enables the 'ut unicode' command which tests that the functions for
+	  manipulating Unicode strings work correctly.
+
 source "test/dm/Kconfig"
 source "test/env/Kconfig"
 source "test/overlay/Kconfig"

test/Makefile +1 -0

index 1092011fdb..a5f52fd5ad 100644
@@ -8,4 +8,5 @@ obj-$(CONFIG_SANDBOX) += command_ut.o
 obj-$(CONFIG_SANDBOX) += compression.o
 obj-$(CONFIG_SANDBOX) += print_ut.o
 obj-$(CONFIG_UT_TIME) += time_ut.o
+obj-$(CONFIG_UT_UNICODE) += unicode_ut.o
 obj-$(CONFIG_$(SPL_)LOG) += log/

test/cmd_ut.c +0 -0

index 934a5a931b..d6a2593850 100644

Quoted 50 lines · Heinrich Schuchardt

Provide unit tests for Unicode functions.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS           |   1 +
 include/test/suites.h |   3 +-
 test/Kconfig          |   8 +
 test/Makefile         |   1 +
 test/cmd_ut.c         |  14 +-
 test/unicode_ut.c     | 470 ++++++++++++++++++++++++++++++++++++++++++
 6 files changed, 493 insertions(+), 4 deletions(-)
 create mode 100644 test/unicode_ut.c

MAINTAINERS +1 -0

index 51a1472cf8..a324139471 100644
@@ -374,6 +374,7 @@ F:	include/asm-generic/pe.h
 F:	lib/charset.c
 F:	lib/efi*/
 F:	test/py/tests/test_efi*
+F:	test/unicode_ut.c
 F:	cmd/bootefi.c
 F:	tools/file2include.c
 

include/test/suites.h +2 -1

index b5019a7cd2..8e4eac60eb 100644
@@ -23,10 +23,11 @@ struct unit_test;
 int cmd_ut_category(const char *name, struct unit_test *tests, int n_ents,
 		    int argc, char * const argv[]);
 
+int do_ut_compression(cmd_tbl_t *cmdtp, int flag, int argc, char *const argv[]);
 int do_ut_dm(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_env(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_overlay(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 int do_ut_time(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
-int do_ut_compression(cmd_tbl_t *cmdtp, int flag, int argc, char *const argv[]);
+int do_ut_unicode(cmd_tbl_t *cmdtp, int flag, int argc, char * const argv[]);
 
 #endif /* __TEST_SUITES_H__ */

test/Kconfig +0 -0

index 3643761bc6..de16d179d0 100644
@@ -15,6 +15,14 @@ config UT_TIME
 	  problems. But if you are having problems with udelay() and the like,
 	  this is a good place to start.
 

test/Kconfig config UT_TIME · line 18

Quoted 1 line

test/Kconfig +1 -0

+config UT_UNICODE

You probably want to move the hunk that actually builds charset.o with CONFIG_UT_UNICODE set to this patch as well ;).

Quoted 25 lines

test/Kconfig +7 -0

+	bool "Unit tests for Unicode functions"
+	depends on UNIT_TEST
+	default y
+	help
+	  Enables the 'ut unicode' command which tests that the functions for
+	  manipulating Unicode strings work correctly.
+
 source "test/dm/Kconfig"
 source "test/env/Kconfig"
 source "test/overlay/Kconfig"

test/Makefile +1 -0

index 1092011fdb..a5f52fd5ad 100644
@@ -8,4 +8,5 @@ obj-$(CONFIG_SANDBOX) += command_ut.o
 obj-$(CONFIG_SANDBOX) += compression.o
 obj-$(CONFIG_SANDBOX) += print_ut.o
 obj-$(CONFIG_UT_TIME) += time_ut.o
+obj-$(CONFIG_UT_UNICODE) += unicode_ut.o
 obj-$(CONFIG_$(SPL_)LOG) += log/

test/cmd_ut.c +0 -0

index 934a5a931b..d6a2593850 100644

test/cmd_ut.c static cmd_tbl_t cmd_ut_sub[] = { · line 53

Quoted 6 lines

test/cmd_ut.c +2 -0

@@ -49,6 +49,10 @@ static cmd_tbl_t cmd_ut_sub[] = {
 #ifdef CONFIG_UT_TIME
 	U_BOOT_CMD_MKENT(time, CONFIG_SYS_MAXARGS, 1, do_ut_time, "", ""),
 #endif
+#if defined(CONFIG_UT_UNICODE) && \
+	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)

CONFIG_IS_ENABLED()

Alex

The width and precision of the printf() function refer to the number of characters not to the number of bytes printed.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/vsprintf.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

lib/vsprintf.c +2 -8

index a07128ad96..b7eb9d5f5e 100644
@@ -280,18 +280,22 @@ static char *string16(char *buf, char *end, u16 *s, int field_width,
 		int precision, int flags)
 {
 	u16 *str = s ? s : L"<NULL>";
-	int utf16_len = u16_strnlen(str, precision);
-	u8 utf8[utf16_len * MAX_UTF8_PER_UTF16];
-	int utf8_len, i;
-
-	utf8_len = utf16_to_utf8(utf8, str, utf16_len) - utf8;
+	ssize_t i, len = utf16_strnlen(str, precision);
 
 	if (!(flags & LEFT))
-		while (utf8_len < field_width--)
+		for (; len < field_width; --field_width)
 			ADDCH(buf, ' ');
-	for (i = 0; i < utf8_len; ++i)
-		ADDCH(buf, utf8[i]);

Quoted 35 lines · Heinrich Schuchardt

The width and precision of the printf() function refer to the number of characters not to the number of bytes printed.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/vsprintf.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

lib/vsprintf.c +7 -9

index a07128ad96..b7eb9d5f5e 100644
@@ -280,18 +280,22 @@ static char *string16(char *buf, char *end, u16 *s, int field_width,
 		int precision, int flags)
 {
 	u16 *str = s ? s : L"<NULL>";
-	int utf16_len = u16_strnlen(str, precision);
-	u8 utf8[utf16_len * MAX_UTF8_PER_UTF16];
-	int utf8_len, i;
-
-	utf8_len = utf16_to_utf8(utf8, str, utf16_len) - utf8;
+	ssize_t i, len = utf16_strnlen(str, precision);
 
 	if (!(flags & LEFT))
-		while (utf8_len < field_width--)
+		for (; len < field_width; --field_width)
 			ADDCH(buf, ' ');
-	for (i = 0; i < utf8_len; ++i)
-		ADDCH(buf, utf8[i]);
-	while (utf8_len < field_width--)
+	for (i = 0; i < len; ++i) {
+		s32 code = utf16_get((const u16 **)&str);
+
+		if (code < 0) {
+			code = '?';

lib/vsprintf.c static char *string16(char *buf, char *end, u16 *s, int field_width, · line 296

Quoted 4 lines

lib/vsprintf.c +4 -0

+			if (*str)
+				++str;
+		}
+		utf8_put(code, &buf);

Can you introduce or reuse a strcpy() helper in charset.c for this? That way the compiler has the chance to inline utf16_get() and utf8_put() and make the function fast.

Alex

Quoted 38 lines · Alexander Graf

Quoted 35 lines · Heinrich Schuchardt

The width and precision of the printf() function refer to the number of characters not to the number of bytes printed.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/vsprintf.c | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

lib/vsprintf.c +7 -9

index a07128ad96..b7eb9d5f5e 100644
@@ -280,18 +280,22 @@ static char *string16(char *buf, char *end, u16 *s, int field_width,
 		int precision, int flags)
 {
 	u16 *str = s ? s : L"<NULL>";
-	int utf16_len = u16_strnlen(str, precision);
-	u8 utf8[utf16_len * MAX_UTF8_PER_UTF16];
-	int utf8_len, i;
-
-	utf8_len = utf16_to_utf8(utf8, str, utf16_len) - utf8;
+	ssize_t i, len = utf16_strnlen(str, precision);
 
 	if (!(flags & LEFT))
-		while (utf8_len < field_width--)
+		for (; len < field_width; --field_width)
 			ADDCH(buf, ' ');
-	for (i = 0; i < utf8_len; ++i)
-		ADDCH(buf, utf8[i]);
-	while (utf8_len < field_width--)
+	for (i = 0; i < len; ++i) {
+		s32 code = utf16_get((const u16 **)&str);
+
+		if (code < 0) {
+			code = '?';

Quoted 8 lines

lib/vsprintf.c static char *string16(char *buf, char *end, u16 *s, int field_width, · line 296

Quoted 4 lines

lib/vsprintf.c +4 -0

+			if (*str)
+				++str;
+		}
+		utf8_put(code, &buf);

Can you introduce or reuse a strcpy() helper in charset.c for this? That way the compiler has the chance to inline utf16_get() and utf8_put() and make the function fast.

strcpy() works on bytes not on multi-byte utf-8 characters. So it is unclear to me how I should make use of strcpy() here.

Of cause we could define utf8_put() and utf8_get() as inline function. But that would increase code size. Is this what you would prefer? I would guess that the serial interface is always the slowest part of text output anyway.

Test printing of Unicode strings

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 test/unicode_ut.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

test/unicode_ut.c +23 -0

index 29316606c4..8e8c4d189e 100644
@@ -34,6 +34,42 @@ static const char d3[] = {0xe6, 0xbd, 0x9c, 0xe6, 0xb0, 0xb4, 0xe8, 0x89,
 static const char d4[] = {0xf0, 0x90, 0x92, 0x8d, 0xf0, 0x90, 0x92, 0x96,
 			  0xf0, 0x90, 0x92, 0x87, 0x00};
 
+static int ut_string16(void)
+{
+#if defined(CONFIG_EFI_LOADER) && \
+	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)
+	char buf[20];
+
+	memset(buf, 0xff, sizeof(buf));
+	sprintf(buf, "%8.6ls", c2);
+	if (buf[1] != ' ')
+		return -1;
+	if (strncmp(&buf[2], d2, 7))
+		return -1;
+	if (buf[9])
+		return -1;
+
+	memset(buf, 0xff, sizeof(buf));
+	sprintf(buf, "%8.6ls", c4);
+	if (buf[4] != ' ')
+		return -1;
+	if (strncmp(&buf[5], d4, 12))
+		return -1;
+	if (buf[17])
+		return -1;

Quoted 42 lines · Heinrich Schuchardt

Test printing of Unicode strings

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 test/unicode_ut.c | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

test/unicode_ut.c +27 -0

index 29316606c4..8e8c4d189e 100644
@@ -34,6 +34,42 @@ static const char d3[] = {0xe6, 0xbd, 0x9c, 0xe6, 0xb0, 0xb4, 0xe8, 0x89,
 static const char d4[] = {0xf0, 0x90, 0x92, 0x8d, 0xf0, 0x90, 0x92, 0x96,
 			  0xf0, 0x90, 0x92, 0x87, 0x00};
 
+static int ut_string16(void)
+{
+#if defined(CONFIG_EFI_LOADER) && \
+	!defined(CONFIG_SPL_BUILD) && !defined(API_BUILD)
+	char buf[20];
+
+	memset(buf, 0xff, sizeof(buf));
+	sprintf(buf, "%8.6ls", c2);
+	if (buf[1] != ' ')
+		return -1;
+	if (strncmp(&buf[2], d2, 7))
+		return -1;
+	if (buf[9])
+		return -1;
+
+	memset(buf, 0xff, sizeof(buf));
+	sprintf(buf, "%8.6ls", c4);
+	if (buf[4] != ' ')
+		return -1;
+	if (strncmp(&buf[5], d4, 12))
+		return -1;
+	if (buf[17])
+		return -1;
+
+	memset(buf, 0xff, sizeof(buf));
+	sprintf(buf, "%-8.2ls", c4);
+	if (strncmp(buf, d4, 8))

test/unicode_ut.c static const char d3[] = {0xe6, 0xbd, 0x9c, 0xe6, 0xb0, 0xb4, 0xe8, 0x89, · line 69

Quoted 6 lines

test/unicode_ut.c +6 -0

+		return -1;
+	if (buf[8] != ' ')
+		return -1;
+	if (buf[14])
+		return -1;
+#endif

Does this check the illegal character '?' case as well?

Alex

The EFI spec does not provide a length limit for variables.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/efi_loader/efi_variable.c | 52 ++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 22 deletions(-)

lib/efi_loader/efi_variable.c +10 -11

index 770c67abb9..495738884b 100644
@@ -44,10 +44,7 @@
  * converted to utf16?
  */
 
-#define MAX_VAR_NAME 31
-#define MAX_NATIVE_VAR_NAME \
-	(strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_") + \
-		(MAX_VAR_NAME * MAX_UTF8_PER_UTF16))
+#define PREFIX_LEN (strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_"))
 
 static int hex(int ch)
 {

@@ -101,18 +98,20 @@ static char *mem2hex(char *hexstr, const u8 *mem, int count)
 	return hexstr;
 }
 
-static efi_status_t efi_to_native(char *native, u16 *variable_name,
+static efi_status_t efi_to_native(char **native, const u16 *variable_name,
 				  efi_guid_t *vendor)
 {
 	size_t len;
+	char *pos;
 
-	len = u16_strlen((u16 *)variable_name);
-	if (len >= MAX_VAR_NAME)
-		return EFI_DEVICE_ERROR;
+	len = PREFIX_LEN + utf16_utf8_strlen(variable_name) + 1;
+	*native = malloc(len);
+	if (!*native)
+		return EFI_OUT_OF_RESOURCES;
 
-	native += sprintf(native, "efi_%pUl_", vendor);
-	native  = (char *)utf16_to_utf8((u8 *)native, (u16 *)variable_name, len);
-	*native = '\0';
+	pos = *native;
+	pos += sprintf(pos, "efi_%pUl_", vendor);
+	utf16_utf8_strcpy(&pos, variable_name);
 
 	return EFI_SUCCESS;
 }

Quoted 49 lines · Heinrich Schuchardt

The EFI spec does not provide a length limit for variables.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/efi_loader/efi_variable.c | 52 ++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 22 deletions(-)

lib/efi_loader/efi_variable.c +10 -11

index 770c67abb9..495738884b 100644
@@ -44,10 +44,7 @@
  * converted to utf16?
  */
 
-#define MAX_VAR_NAME 31
-#define MAX_NATIVE_VAR_NAME \
-	(strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_") + \
-		(MAX_VAR_NAME * MAX_UTF8_PER_UTF16))
+#define PREFIX_LEN (strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_"))
 
 static int hex(int ch)
 {

@@ -101,18 +98,20 @@ static char *mem2hex(char *hexstr, const u8 *mem, int count)
 	return hexstr;
 }
 
-static efi_status_t efi_to_native(char *native, u16 *variable_name,
+static efi_status_t efi_to_native(char **native, const u16 *variable_name,
 				  efi_guid_t *vendor)
 {
 	size_t len;
+	char *pos;
 
-	len = u16_strlen((u16 *)variable_name);
-	if (len >= MAX_VAR_NAME)
-		return EFI_DEVICE_ERROR;
+	len = PREFIX_LEN + utf16_utf8_strlen(variable_name) + 1;
+	*native = malloc(len);
+	if (!*native)
+		return EFI_OUT_OF_RESOURCES;
 
-	native += sprintf(native, "efi_%pUl_", vendor);
-	native  = (char *)utf16_to_utf8((u8 *)native, (u16 *)variable_name, len);
-	*native = '\0';
+	pos = *native;
+	pos += sprintf(pos, "efi_%pUl_", vendor);
+	utf16_utf8_strcpy(&pos, variable_name);
 

lib/efi_loader/efi_variable.c efi_status_t EFIAPI efi_get_variable(u16 *variable_name, efi_guid_t *vendor, · line 170

Quoted 8 lines

lib/efi_loader/efi_variable.c +1 -1

 	return EFI_SUCCESS;
 }

@@ -168,7 +167,7 @@ efi_status_t EFIAPI efi_get_variable(u16 *variable_name, efi_guid_t *vendor,
 				     u32 *attributes, efi_uintn_t *data_size,
 				     void *data)
 {
-	char native_name[MAX_NATIVE_VAR_NAME + 1];
+	char *native_name;

I think you want to predefine this to = NULL to make sure that an error path doesn't give you uninitialized values on free().

Alex

Quoted 52 lines · Alexander Graf

Quoted 49 lines · Heinrich Schuchardt

The EFI spec does not provide a length limit for variables.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 lib/efi_loader/efi_variable.c | 52 ++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 22 deletions(-)

lib/efi_loader/efi_variable.c +10 -11

index 770c67abb9..495738884b 100644
@@ -44,10 +44,7 @@
  * converted to utf16?
  */
 
-#define MAX_VAR_NAME 31
-#define MAX_NATIVE_VAR_NAME \
-	(strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_") + \
-		(MAX_VAR_NAME * MAX_UTF8_PER_UTF16))
+#define PREFIX_LEN (strlen("efi_xxxxxxxx-xxxx-xxxx-xxxxxxxxxxxxxxxx_"))
 
 static int hex(int ch)
 {

@@ -101,18 +98,20 @@ static char *mem2hex(char *hexstr, const u8 *mem, int count)
 	return hexstr;
 }
 
-static efi_status_t efi_to_native(char *native, u16 *variable_name,
+static efi_status_t efi_to_native(char **native, const u16 *variable_name,
 				  efi_guid_t *vendor)
 {
 	size_t len;
+	char *pos;
 
-	len = u16_strlen((u16 *)variable_name);
-	if (len >= MAX_VAR_NAME)
-		return EFI_DEVICE_ERROR;
+	len = PREFIX_LEN + utf16_utf8_strlen(variable_name) + 1;
+	*native = malloc(len);
+	if (!*native)
+		return EFI_OUT_OF_RESOURCES;
 
-	native += sprintf(native, "efi_%pUl_", vendor);
-	native  = (char *)utf16_to_utf8((u8 *)native, (u16 *)variable_name, len);
-	*native = '\0';
+	pos = *native;
+	pos += sprintf(pos, "efi_%pUl_", vendor);
+	utf16_utf8_strcpy(&pos, variable_name);
 

Quoted 11 lines

lib/efi_loader/efi_variable.c efi_status_t EFIAPI efi_get_variable(u16 *variable_name, efi_guid_t *vendor, · line 170

Quoted 8 lines

lib/efi_loader/efi_variable.c +1 -1

 	return EFI_SUCCESS;
 }

@@ -168,7 +167,7 @@ efi_status_t EFIAPI efi_get_variable(u16 *variable_name, efi_guid_t *vendor,
 				     u32 *attributes, efi_uintn_t *data_size,
 				     void *data)
 {
-	char native_name[MAX_NATIVE_VAR_NAME + 1];
+	char *native_name;

I think you want to predefine this to = NULL to make sure that an error path doesn't give you uninitialized values on free().

efi_to_native() returns EFI_OUT_OF_RESOURCES if the pointer cannot be assigned and the return value is checked. So how should I reach free(native_name) in this case?

The length of a string printed to the console by the EFI_SIMPLE_TEXT_OUTPUT_PROTOCOL is not limited by the UEFI spec. Hence should not allocate a buffer for it on the stack.

Signed-off-by Heinrich Schuchardt <[email protected]>

The number of bytes in an utf-8 string is an upper limit for the number of words in the equivalent utf-16 string. In so far the inumbant coding works correctly. For non-ASCII characters the utf-16 string is shorter. With the patch only the necessary buffer size is allocated for the load options.

Signed-off-by Heinrich Schuchardt <[email protected]>

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS              |    1 +
 include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 1910 insertions(+)
 create mode 100644 include/capitalization.h

MAINTAINERS +1 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:	doc/DocBook/efi.tmpl
 F:	doc/README.uefi
 F:	doc/README.iscsi
 F:	Documentation/efi.rst
+F:	include/capitalization.h
 F:	include/efi*
 F:	include/pe.h
 F:	include/asm-generic/pe.h

include/capitalization.h new file +2 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*

include/capitalization.h new file +7 -0

+	{ 0x0531, /* ARMENIAN CAPITAL LETTER AYB */ \
+	  0x0561, /* ARMENIAN SMALL LETTER AYB */ }, \
+	{ 0x0532, /* ARMENIAN CAPITAL LETTER BEN */ \
+	  0x0562, /* ARMENIAN SMALL LETTER BEN */ }, \
+	{ 0x053E, /* ARMENIAN CAPITAL LETTER CA */ \
+	  0x056E, /* ARMENIAN SMALL LETTER CA */ }, \
+	{ 0x0549, /* ARMENIAN CAPITAL LETTER CHA */ \

Quoted 30 lines · Heinrich Schuchardt

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS              |    1 +
 include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 1910 insertions(+)
 create mode 100644 include/capitalization.h

MAINTAINERS +1 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:	doc/DocBook/efi.tmpl
 F:	doc/README.uefi
 F:	doc/README.iscsi
 F:	Documentation/efi.rst
+F:	include/capitalization.h
 F:	include/efi*
 F:	include/pe.h
 F:	include/asm-generic/pe.h

include/capitalization.h new file +2 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*

include/capitalization.h line 12

Quoted 10 lines

include/capitalization.h new file +10 -0

+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {
+	u16 upper;
+	u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

Is there any slightly more sophisticated pattern in the table maybe that we could just express as code? Would that turn out smaller maybe?

Alex

Quoted 7 lines

include/capitalization.h new file +7 -0

+	{ 0x0531, /* ARMENIAN CAPITAL LETTER AYB */ \
+	  0x0561, /* ARMENIAN SMALL LETTER AYB */ }, \
+	{ 0x0532, /* ARMENIAN CAPITAL LETTER BEN */ \
+	  0x0562, /* ARMENIAN SMALL LETTER BEN */ }, \
+	{ 0x053E, /* ARMENIAN CAPITAL LETTER CA */ \
+	  0x056E, /* ARMENIAN SMALL LETTER CA */ }, \
+	{ 0x0549, /* ARMENIAN CAPITAL LETTER CHA */ \

[...]

Quoted 47 lines · Alexander Graf

Quoted 30 lines · Heinrich Schuchardt

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS              |    1 +
 include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 1910 insertions(+)
 create mode 100644 include/capitalization.h

MAINTAINERS +1 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:	doc/DocBook/efi.tmpl
 F:	doc/README.uefi
 F:	doc/README.iscsi
 F:	Documentation/efi.rst
+F:	include/capitalization.h
 F:	include/efi*
 F:	include/pe.h
 F:	include/asm-generic/pe.h

include/capitalization.h new file +2 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*

include/capitalization.h line 12

Quoted 10 lines

include/capitalization.h new file +10 -0

+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {
+	u16 upper;
+	u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

Quoted 2 lines

Is there any slightly more sophisticated pattern in the table maybe that we could just express as code? Would that turn out smaller maybe?

This is 3792 bytes of data. Unicode capitalization is quite random in arranging lower and upper letters.

We could resort to zlib or gzip. But these libraries are not built by default.

Most urgently we will need the capitalization table for generating and checking short FAT filenames, so we could create a configuration switch that would reduce this table to codepage 437 or codepage 1250 letters depending on the chosen native character set.

In EDK2 I only found code for codepage 1250.

Quoted 39 lines · Alexander Graf

Quoted 38 lines · Heinrich Schuchardt

Quoted 35 lines

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
 MAINTAINERS              |    1 +
 include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++
 2 files changed, 1910 insertions(+)
 create mode 100644 include/capitalization.h

MAINTAINERS +1 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:	doc/DocBook/efi.tmpl
 F:	doc/README.uefi
 F:	doc/README.iscsi
 F:	Documentation/efi.rst
+F:	include/capitalization.h
 F:	include/efi*
 F:	include/pe.h
 F:	include/asm-generic/pe.h

include/capitalization.h new file +7 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*
+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {

Quoted 8 lines

Quoted 8 lines

include/capitalization.h line 12

Quoted 5 lines

include/capitalization.h new file +5 -0

+	u16 upper;
+	u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

Quoted 8 lines

Quoted 2 lines

Is there any slightly more sophisticated pattern in the table maybe that we could just express as code? Would that turn out smaller maybe?

This is 3792 bytes of data. Unicode capitalization is quite random in arranging lower and upper letters.

We could resort to zlib or gzip. But these libraries are not built by default.

Yeah, and that only adds to more overhead.

Quoted 4 lines

Most urgently we will need the capitalization table for generating and checking short FAT filenames, so we could create a configuration switch that would reduce this table to codepage 437 or codepage 1250 letters depending on the chosen native character set.

I think that's a great idea. There probably is a lot of overlap even between the two, so maybe just make it a config option for "non-latin upper/lower case conversion".

Quoted 1 line

In EDK2 I only found code for codepage 1250.

Yeah, I'd be surprised if people really needed more. In fact, how about you just default the config option to =n by default?

Alex

Am 27.08.2018 um 10:30 schrieb Mike FABIAN <[email protected]>:

Alexander Graf <[email protected]> さんは書きました:

Quoted 40 lines

Quoted 40 lines · Heinrich Schuchardt

Quoted 39 lines · Alexander Graf

Quoted 38 lines · Heinrich Schuchardt

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
MAINTAINERS | 1 +
include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++

2 files changed, 1910 insertions(+)

create mode 100644 include/capitalization.h

MAINTAINERS +0 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:    doc/DocBook/efi.tmpl

F:    doc/README.uefi
F:    doc/README.iscsi
F:    Documentation/efi.rst
+F:    include/capitalization.h
F:    include/efi*
F:    include/pe.h
F:    include/asm-generic/pe.h

include/capitalization.h new file +7 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*
+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {

Quoted 8 lines

Quoted 8 lines

Quoted 8 lines

include/capitalization.h line 12

Quoted 5 lines

include/capitalization.h new file +5 -0

+    u16 upper;
+    u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

That data is also in glibc. I don’t know whether you use glibc though
...

Quoted 55 lines

Am 27.08.2018 um 10:30 schrieb Mike FABIAN <[email protected]>:

Alexander Graf <[email protected]> さんは書きました:

Quoted 48 lines

Quoted 48 lines · Heinrich Schuchardt

Quoted 47 lines · Alexander Graf

Quoted 33 lines · Heinrich Schuchardt

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
MAINTAINERS | 1 +
include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++

2 files changed, 1910 insertions(+)

create mode 100644 include/capitalization.h

MAINTAINERS +0 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:    doc/DocBook/efi.tmpl

F:    doc/README.uefi
F:    doc/README.iscsi
F:    Documentation/efi.rst
+F:    include/capitalization.h
F:    include/efi*
F:    include/pe.h
F:    include/asm-generic/pe.h

include/capitalization.h new file +2 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*

include/capitalization.h line 12

Quoted 10 lines

include/capitalization.h new file +10 -0

+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {
+    u16 upper;
+    u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

Quoted 2 lines

That data is also in glibc. I don’t know whether you use glibc though
...

U-Boot is a standalone OS so to say, we do not use glibc (except for sandbox, but that's a special target).

The main problem is that some times people cram U-Boot into very tight spaces, like small flash chips or even on-chip RAM. So we have to be very cautious on space requirements.

Hi,

Quoted 55 lines · Alexander Graf

Quoted 55 lines

Am 27.08.2018 um 10:30 schrieb Mike FABIAN <[email protected]>:

Alexander Graf <[email protected]> さんは書きました:

Quoted 48 lines

Quoted 48 lines · Heinrich Schuchardt

Quoted 47 lines · Alexander Graf

Quoted 33 lines · Heinrich Schuchardt

This patch provides a define to initialize a table that maps lower to capital letters for Unicode code point 0x0000 - 0xffff.

Signed-off-by Heinrich Schuchardt <[email protected]>

---
MAINTAINERS | 1 +
include/capitalization.h | 1909 ++++++++++++++++++++++++++++++++++++++

2 files changed, 1910 insertions(+)

create mode 100644 include/capitalization.h

MAINTAINERS +0 -0

index a324139471..0a543309f2 100644
@@ -368,6 +368,7 @@ F:    doc/DocBook/efi.tmpl

F:    doc/README.uefi
F:    doc/README.iscsi
F:    Documentation/efi.rst
+F:    include/capitalization.h
F:    include/efi*
F:    include/pe.h
F:    include/asm-generic/pe.h

include/capitalization.h new file +2 -0

new file mode 100644
index 0000000000..50d5108f98
@@ -0,0 +1,1909 @@
+/* SPDX-License-Identifier: Unicode-DFS-2016 */
+/*

include/capitalization.h line 12

Quoted 10 lines

include/capitalization.h new file +10 -0

+ * Correspondence table for small and capital Unicode letters in the range of
+ * 0x0000 - 0xffff based on http://www.unicode.org/Public/UCA/11.0.0/allkeys.txt
+ */
+
+struct capitalization_table {
+    u16 upper;
+    u16 lower;
+};
+
+#define UNICODE_CAPITALIZATION_TABLE { \

Ugh, that is a *lot* of data. How much does the binary size grow with the table compiled in?

Quoted 6 lines

Quoted 2 lines

That data is also in glibc. I don’t know whether you use glibc though
...

U-Boot is a standalone OS so to say, we do not use glibc (except for sandbox, but that's a special target).

The main problem is that some times people cram U-Boot into very tight spaces, like small flash chips or even on-chip RAM. So we have to be very cautious on space requirements.

Indeed.

Most archs use a private glib so do not build with one that supports this table.

Wouldn't it be better to put this in a C file as a const struct?

If it is big that's OK, we just need to add a CONFIG option for it. If we already have EFI enabled, perhaps it doesn't matter, since it is pretty big.

The patch implements the EFI_UNICODE_COLLATION_PROTOCOL.

Signed-off-by Heinrich Schuchardt <[email protected]>