unicode cleanups, and split the data table into a separate module
Subsystems (4)
Hi all,
this series is an alternate idea to split the utf8 table into a separate module. It comes with a whole lot of cleanups preloaded.
Diffstat:
fs/ext4/super.c | 31 +--- fs/f2fs/super.c | 30 +--- fs/f2fs/sysfs.c | 3 fs/unicode/Kconfig | 13 +- fs/unicode/Makefile | 13 +- fs/unicode/mkutf8data.c | 24 +++ fs/unicode/utf8-core.c | 105 ++++++---------- fs/unicode/utf8-norm.c | 262 +++++------------------------------------- fs/unicode/utf8-selftest.c | 94 +++++++-------- fs/unicode/utf8data.c_shipped | 22 ++- fs/unicode/utf8n.h | 81 ++++-------- include/linux/unicode.h | 35 +++++ 12 files changed, 260 insertions(+), 453 deletions(-)
PATCH 1/11 ext4: simplify ext4_sb_read_encoding
Return the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
--- fs/ext4/super.c | 21 +++++++-------------- 1 file changed, 7 insertions(+), 14 deletions(-)
fs/ext4/super.c +7 -14
@@ -2021,24 +2021,17 @@ static const struct ext4_sb_encodings { {EXT4_ENC_UTF8_12_1, "utf8", "12.1.0"}, }; -static int ext4_sb_read_encoding(const struct ext4_super_block *es, - const struct ext4_sb_encodings **encoding, - __u16 *flags) +static const struct ext4_sb_encodings * +ext4_sb_read_encoding(const struct ext4_super_block *es) { __u16 magic = le16_to_cpu(es->s_encoding); int i; for (i = 0; i < ARRAY_SIZE(ext4_sb_encoding_map); i++) if (magic == ext4_sb_encoding_map[i].magic) - break; - - if (i >= ARRAY_SIZE(ext4_sb_encoding_map)) - return -EINVAL; + return &ext4_sb_encoding_map[i]; - *encoding = &ext4_sb_encoding_map[i]; - *flags = le16_to_cpu(es->s_encoding_flags); - - return 0; + return NULL; } #endif
@@ -4303,10 +4296,10 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent)if (ext4_has_feature_casefold (sb) && !sb->s_encoding) {const struct ext4_sb_encodings *encoding_info;struct unicode_map *encoding; -__u16 encoding_flags; +__u16 encoding_flags =le16_to_cpu (es->s_encoding_flags); -if (ext4_sb_read_encoding (es, &encoding_info, - &encoding_flags)) { + encoding_info =ext4_sb_read_encoding (es); +if (!encoding_info) {ext4_msg (sb,KERN_ERR ,"Encoding requested by superblock is unknown" ); goto failed_mount;
Signature · 1 line
2.30.2
Re: [PATCH 01/11] ext4: simplify ext4_sb_read_encoding
Quoted 5 lines · Christoph Hellwig
Return the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
Signed-off-by Christoph Hellwig <[email protected]>
Looks good to me. Feel free to add
Signature · 1 line
Gabriel Krisman Bertazi
Re: [PATCH 01/11] ext4: simplify ext4_sb_read_encoding
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
Return the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
Signed-off-by Christoph Hellwig <[email protected]>
Thanks,
- Ted
_______________________________________________
Linux-f2fs-devel mailing list
[email protected]
https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 2/11 f2fs: simplify f2fs_sb_read_encoding
In reply to Christoph HellwigReturn the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
--- fs/f2fs/super.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-)
fs/f2fs/super.c +7 -13
@@ -260,24 +260,17 @@ static const struct f2fs_sb_encodings { {F2FS_ENC_UTF8_12_1, "utf8", "12.1.0"}, }; -static int f2fs_sb_read_encoding (const struct f2fs_super_block *sb, -const struct f2fs_sb_encodings **encoding, -__u16 *flags) +static const struct f2fs_sb_encodings * +f2fs_sb_read_encoding (const struct f2fs_super_block *sb) {__u16 magic =le16_to_cpu (sb->s_encoding);int i;for (i =0 ; i <ARRAY_SIZE (f2fs_sb_encoding_map); i++)if (magic == f2fs_sb_encoding_map[i].magic) -break ; - -if (i >=ARRAY_SIZE (f2fs_sb_encoding_map)) -return -EINVAL ; +return &f2fs_sb_encoding_map[i]; - *encoding = &f2fs_sb_encoding_map[i]; - *flags =le16_to_cpu (sb->s_encoding_flags); - -return 0 ; +return NULL ; }struct kmem_cache *f2fs_cf_name_slab;
@@ -3730,13 +3723,14 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi)struct unicode_map *encoding;__u16 encoding_flags; -if (f2fs_sb_read_encoding (sbi->raw_super, &encoding_info, - &encoding_flags)) { + encoding_info =f2fs_sb_read_encoding (sbi->raw_super); +if (!encoding_info) {f2fs_err (sbi,"Encoding requested by superblock is unknown" );return -EINVAL ; } + encoding_flags =le16_to_cpu (sbi->raw_super->s_encoding_flags); encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) { f2fs_err(sbi,
Signature · 1 line
2.30.2
Re: [PATCH 02/11] f2fs: simplify f2fs_sb_read_encoding
Quoted 5 lines · Christoph Hellwig
Return the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
Signed-off-by Christoph Hellwig <[email protected]>
Looks good:
Signature · 1 line
Gabriel Krisman Bertazi
Re: [f2fs-dev] [PATCH 02/11] f2fs: simplify f2fs_sb_read_encoding
In reply to Christoph HellwigQuoted 5 lines · Christoph Hellwig
Return the encoding table as the return value instead of as an argument, and don't bother with the encoding flags as the caller can handle that trivially.
Signed-off-by Christoph Hellwig <[email protected]>
Thanks,
PATCH 3/11 unicode: remove the charset field from struct unicode_map
In reply to Christoph HellwigIt is hardcoded and only used for a f2fs sysfs file where it can be hardcoded just as easily.
--- fs/f2fs/sysfs.c | 3 +-- fs/unicode/utf8-core.c | 3 --- include/linux/unicode.h | 1 - 3 files changed, 1 insertion(+), 6 deletions(-)
fs/f2fs/sysfs.c +1 -2
@@ -195,8 +195,7 @@ static ssize_t encoding_show(struct f2fs_attr *a,struct super_block *sb = sbi->sb;if (f2fs_sb_has_casefold (sbi)) -return snprintf (buf,PAGE_SIZE ,"%s (%d.%d.%d)\n" , - sb->s_encoding->charset, +return snprintf (buf,PAGE_SIZE ,"UTF-8 (%d.%d.%d)\n" , (sb->s_encoding->version >>16 ) &0xff , (sb->s_encoding->version >>8 ) &0xff , sb->s_encoding->version &0xff );
fs/unicode/utf8-core.c +0 -3
@@ -219,10 +219,7 @@ struct unicode_map *utf8_load(const char *version) um =kzalloc (sizeof (struct unicode_map ),GFP_KERNEL );if (!um)return ERR_PTR (-ENOMEM ); - - um->charset ="UTF-8" ; um->version = unicode_version; -return um; }EXPORT_SYMBOL (utf8_load);
include/linux/unicode.h +0 -1
@@ -6,7 +6,6 @@#include <linux/dcache.h> struct unicode_map { -const char *charset;int version; };
Signature · 1 line
2.30.2
Re: [PATCH 03/11] unicode: remove the charset field from struct unicode_map
Quoted 4 lines · Christoph Hellwig
It is hardcoded and only used for a f2fs sysfs file where it can be hardcoded just as easily.
Signed-off-by Christoph Hellwig <[email protected]>
Signature · 1 line
Gabriel Krisman Bertazi
PATCH 4/11 unicode: mark the version field in struct unicode_map unsigned
In reply to Christoph Hellwigunicode version tripplets are always unsigned.
--- include/linux/unicode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
include/linux/unicode.h +1 -1
@@ -6,7 +6,7 @@#include <linux/dcache.h> struct unicode_map { -int version; +unsigned int version; };int utf8_validate (const struct unicode_map *um,const struct qstr *str);
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [PATCH 04/11] unicode: mark the version field in struct unicode_map unsigned
Quoted 3 lines · Christoph Hellwig
unicode version tripplets are always unsigned.
Signed-off-by Christoph Hellwig <[email protected]>
Thanks,
Quoted 17 lines
--- include/linux/unicode.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)include/linux/unicode.h +1
-1@@ -6,7 +6,7 @@#include <linux/dcache.h> struct unicode_map { -int version; +unsigned int version; };int utf8_validate (const struct unicode_map *um,const struct qstr *str);
Signature · 1 line
Gabriel Krisman Bertazi
PATCH 5/11 unicode: pass a UNICODE_AGE() tripple to utf8_load
In reply to Christoph HellwigDon't bother with pointless string parsing when the caller can just pass the version in the format that the core expects. Also remove the fallback to the latest version that none of the callers actually uses.
--- fs/ext4/super.c | 10 ++++---- fs/f2fs/super.c | 10 ++++---- fs/unicode/utf8-core.c | 50 ++++---------------------------------- fs/unicode/utf8-norm.c | 11 ++------- fs/unicode/utf8-selftest.c | 15 ++++++------ fs/unicode/utf8n.h | 14 ++--------- include/linux/unicode.h | 11 ++++++++- 7 files changed, 37 insertions(+), 84 deletions(-)
fs/ext4/super.c +5 -5
@@ -2016,9 +2016,9 @@ static const struct mount_opts {static const struct ext4_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } ext4_sb_encoding_map[] = { - {EXT4_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {EXT4_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct ext4_sb_encodings *
@@ -4308,15 +4308,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {ext4_msg (sb,KERN_ERR , -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags); goto failed_mount; }ext4_msg (sb,KERN_INFO ,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sb->s_encoding = encoding; sb->s_encoding_flags = encoding_flags;
fs/f2fs/super.c +5 -5
@@ -255,9 +255,9 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)static const struct f2fs_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } f2fs_sb_encoding_map[] = { - {F2FS_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {F2FS_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct f2fs_sb_encodings *
@@ -3734,15 +3734,15 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {f2fs_err (sbi, -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags);return PTR_ERR (encoding); }f2fs_info (sbi,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sbi->sb->s_encoding = encoding; sbi->sb->s_encoding_flags = encoding_flags;
fs/unicode/utf8-core.c +5 -45
64 lines not on this page · raw message
fs/unicode/utf8-norm.c +2 -9
@@ -15,13 +15,12 @@ struct utf8data {#include "utf8data.h" #undef __INCLUDED_FROM_UTF8NORM_C__ -int utf8version_is_supported (u8 maj,u8 min,u8 rev) +int utf8version_is_supported (unsigned int version) {int i =ARRAY_SIZE (utf8agetab) -1 ; -unsigned int sb_utf8version =UNICODE_AGE (maj, min, rev);while (i >=0 && utf8agetab[i] !=0 ) { -if (sb_utf8version == utf8agetab[i]) +if (version == utf8agetab[i])return 1 ; i--; }
@@ -29,12 +28,6 @@ int utf8version_is_supported(u8 maj, u8 min, u8 rev) }EXPORT_SYMBOL (utf8version_is_supported); -int utf8version_latest (void ) -{ -return utf8vers; -} -EXPORT_SYMBOL (utf8version_latest); -/* * UTF-8 valid ranges. *
fs/unicode/utf8-selftest.c +8 -7
@@ -235,7 +235,7 @@ static void check_utf8_nfdicf(void)static void check_utf8_comparisons (void ) {int i; -struct unicode_map *table =utf8_load ("12.1.0" ); +struct unicode_map *table =utf8_load (UNICODE_AGE (12 ,1 ,0 ));if (IS_ERR (table)) { pr_err("%s: Unable to load utf8 %d.%d.%d. Skipping.\n",
@@ -269,18 +269,19 @@ static void check_utf8_comparisons(void)static void check_supported_versions (void ) {/* Unicode 7.0.0 should be supported. */ -test (utf8version_is_supported (7 ,0 ,0 )); +test (utf8version_is_supported (UNICODE_AGE (7 ,0 ,0 )));/* Unicode 9.0.0 should be supported. */ -test (utf8version_is_supported (9 ,0 ,0 )); +test (utf8version_is_supported (UNICODE_AGE (9 ,0 ,0 )));/* Unicode 1x.0.0 (the latest version) should be supported. */ -test (utf8version_is_supported (latest_maj, latest_min, latest_rev)); +test (utf8version_is_supported ( +UNICODE_AGE (latest_maj, latest_min, latest_rev)));/* Next versions don't exist. */ -test (!utf8version_is_supported (13 ,0 ,0 )); -test (!utf8version_is_supported (0 ,0 ,0 )); -test (!utf8version_is_supported (-1 ,-1 ,-1 )); +test (!utf8version_is_supported (UNICODE_AGE (13 ,0 ,0 ))); +test (!utf8version_is_supported (UNICODE_AGE (0 ,0 ,0 ))); +test (!utf8version_is_supported (UNICODE_AGE (-1 ,-1 ,-1 ))); }static int __init init_test_ucd (void )
fs/unicode/utf8n.h +2 -12
@@ -11,19 +11,9 @@#include <linux/export.h> #include <linux/string.h> #include <linux/module.h> +#include <linux/unicode.h> -/* Encoding a unicode version number as a single unsigned int. */ -#define UNICODE_MAJ_SHIFT (16) -#define UNICODE_MIN_SHIFT (8) - -#define UNICODE_AGE (MAJ ,MIN ,REV ) \ - (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ - ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ - ((unsigned int)(REV))) - -/* Highest unicode version supported by the data tables. */ -extern int utf8version_is_supported (u8 maj,u8 min,u8 rev); -extern int utf8version_latest (void ); +int utf8version_is_supported (unsigned int version);/* * Look for the correct const struct utf8data for a unicode version.
include/linux/unicode.h +10 -1
@@ -5,6 +5,15 @@#include <linux/init.h> #include <linux/dcache.h> +/* Encoding a unicode version number as a single unsigned int. */ +#define UNICODE_MAJ_SHIFT (16) +#define UNICODE_MIN_SHIFT (8) + +#define UNICODE_AGE (MAJ ,MIN ,REV ) \ + (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ + ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ + ((unsigned int)(REV))) +struct unicode_map {unsigned int version; };
@@ -29,7 +38,7 @@ int utf8_casefold(const struct unicode_map *um, const struct qstr *str,int utf8_casefold_hash (const struct unicode_map *um,const void *salt,struct qstr *str); -struct unicode_map *utf8_load (const char *version); +struct unicode_map *utf8_load (unsigned int version);void utf8_unload (struct unicode_map *um);#endif /* _LINUX_UNICODE_H */
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [PATCH 05/11] unicode: pass a UNICODE_AGE() tripple to utf8_load
Quoted 132 lines · Christoph Hellwig
Don't bother with pointless string parsing when the caller can just pass the version in the format that the core expects. Also remove the fallback to the latest version that none of the callers actually uses.
Signed-off-by Christoph Hellwig <[email protected]>--- fs/ext4/super.c | 10 ++++---- fs/f2fs/super.c | 10 ++++---- fs/unicode/utf8-core.c | 50 ++++---------------------------------- fs/unicode/utf8-norm.c | 11 ++------- fs/unicode/utf8-selftest.c | 15 ++++++------ fs/unicode/utf8n.h | 14 ++--------- include/linux/unicode.h | 11 ++++++++- 7 files changed, 37 insertions(+), 84 deletions(-)fs/ext4/super.c +5
-5@@ -2016,9 +2016,9 @@ static const struct mount_opts {static const struct ext4_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } ext4_sb_encoding_map[] = { - {EXT4_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {EXT4_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct ext4_sb_encodings *@@ -4308,15 +4308,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {ext4_msg (sb,KERN_ERR , -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags); goto failed_mount; }ext4_msg (sb,KERN_INFO ,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sb->s_encoding = encoding; sb->s_encoding_flags = encoding_flags;fs/f2fs/super.c +5
-5@@ -255,9 +255,9 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)static const struct f2fs_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } f2fs_sb_encoding_map[] = { - {F2FS_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {F2FS_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct f2fs_sb_encodings *@@ -3734,15 +3734,15 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {f2fs_err (sbi, -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags);return PTR_ERR (encoding); }f2fs_info (sbi,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sbi->sb->s_encoding = encoding; sbi->sb->s_encoding_flags = encoding_flags;fs/unicode/utf8-core.c +2
-33@@ -167,59 +167,19 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str, }return -EINVAL ; } -EXPORT_SYMBOL (utf8_normalize); -static int utf8_parse_version (const char *version,unsigned int *maj, -unsigned int *min,unsigned int *rev) +struct unicode_map *utf8_load (unsigned int version) { -substring_t args[3 ]; -char version_string[12 ]; -static const struct match_token token[] = { - {1 ,"%d.%d.%d" }, - {0 ,NULL } - }; - -strncpy (version_string, version,sizeof (version_string)); - -if (match_token (version_string, token, args) !=1 ) -return -EINVAL ; - -if (match_int (&args[0 ], maj) ||match_int (&args[1 ], min) || -match_int (&args[2 ], rev)) -return -EINVAL ; +struct unicode_map *um; -return 0 ; -} - -struct unicode_map *utf8_load (const char *version) -{ -struct unicode_map *um =NULL ; -int unicode_version; - -if (version) { -unsigned int maj, min, rev; - -if (utf8_parse_version (version, &maj, &min, &rev) <0 ) -return ERR_PTR (-EINVAL ); - -if (!utf8version_is_supported (maj, min, rev))
fs/unicode/utf8-core.c int utf8_normalize(const struct unicode_map *um, const struct qstr *str, · line 216
Quoted 10 lines
fs/unicode/utf8-core.c +0
-10-return ERR_PTR (-EINVAL ); - - unicode_version =UNICODE_AGE (maj, min, rev); - }else { - unicode_version =utf8version_latest (); -printk (KERN_WARNING "UTF-8 version not specified. " -"Assuming latest supported version (%d.%d.%d)." , - (unicode_version >>16 ) &0xff , - (unicode_version >>8 ) &0xff , - (unicode_version &0xff ));
I remember this fallback was specifically requested during a review or in a discussion, but I honestly cannot remember the reason.
One advantage I can think is if we have a filesystem that requires a newer unicode version than the current kernel has, and strict mode flag is set, we can fallback to the latest version and still mount the fs read/write.
That is not implemented though, so I'm fine with this removal.
Signature · 1 line
Gabriel Krisman Bertazi
Re: [PATCH 05/11] unicode: pass a UNICODE_AGE() tripple to utf8_load
Quoted 7 lines · Gabriel Krisman Bertazi
I remember this fallback was specifically requested during a review or in a discussion, but I honestly cannot remember the reason.
One advantage I can think is if we have a filesystem that requires a newer unicode version than the current kernel has, and strict mode flag is set, we can fallback to the latest version and still mount the fs read/write.
Well, that seems a little pointless. If we add such a file system we can just upgrade the unіcode data files. (FYI, I did an upgrade to 1.13 to test this series, but without any file system that needs it I did not bother to submit the patch).
Re: [PATCH 05/11] unicode: pass a UNICODE_AGE() tripple to utf8_load
In reply to Christoph HellwigQuoted 32 lines · Christoph Hellwig
Don't bother with pointless string parsing when the caller can just pass the version in the format that the core expects. Also remove the fallback to the latest version that none of the callers actually uses.
Signed-off-by Christoph Hellwig <[email protected]>--- fs/ext4/super.c | 10 ++++---- fs/f2fs/super.c | 10 ++++---- fs/unicode/utf8-core.c | 50 ++++---------------------------------- fs/unicode/utf8-norm.c | 11 ++------- fs/unicode/utf8-selftest.c | 15 ++++++------ fs/unicode/utf8n.h | 14 ++--------- include/linux/unicode.h | 11 ++++++++- 7 files changed, 37 insertions(+), 84 deletions(-)fs/ext4/super.c +2
-2@@ -2016,9 +2016,9 @@ static const struct mount_opts {static const struct ext4_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } ext4_sb_encoding_map[] = { - {EXT4_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {EXT4_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct ext4_sb_encodings *
fs/ext4/super.c static int ext4_fill_super(struct super_block *sb, void *data, int silent) · line 4315
Quoted 9 lines
fs/ext4/super.c +1
-1@@ -4308,15 +4308,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {ext4_msg (sb,KERN_ERR , -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags); goto failed_mount;
"Using encoding defined by superblock: utf8-0xc0100 with flags 0x0"
This is much less readable than what we previously had:
"Using encoding defined by superblock: utf8-12.1.0 with flags 0x0"
It is minor, but can we do instead:
ext4_msg("... %u.%u.%u\n", (encoding_info->version>>12) & 0xff,
(encoding_info->version>>8) & 0xff), encoding_info->version & 0xff))
The rest of the series looks good and I can pick it up for 5.15, unless someone has anything else to say? It has lived on the list for a while now.
Quoted 240 lines
fs/ext4/super.c +2
-2}ext4_msg (sb,KERN_INFO ,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sb->s_encoding = encoding; sb->s_encoding_flags = encoding_flags;fs/f2fs/super.c +5
-5@@ -255,9 +255,9 @@ void f2fs_printk(struct f2fs_sb_info *sbi, const char *fmt, ...)static const struct f2fs_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } f2fs_sb_encoding_map[] = { - {F2FS_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {F2FS_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct f2fs_sb_encodings *@@ -3734,15 +3734,15 @@ static int f2fs_setup_casefold(struct f2fs_sb_info *sbi) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {f2fs_err (sbi, -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags);return PTR_ERR (encoding); }f2fs_info (sbi,"Using encoding defined by superblock: " -"%s-%s with flags 0x%hx" , encoding_info->name, - encoding_info->version?:"\b" , encoding_flags); +"%s-0x%x with flags 0x%hx" , encoding_info->name, + encoding_info->version, encoding_flags); sbi->sb->s_encoding = encoding; sbi->sb->s_encoding_flags = encoding_flags;fs/unicode/utf8-core.c +5
-45@@ -167,59 +167,19 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str, }return -EINVAL ; } -EXPORT_SYMBOL (utf8_normalize); -static int utf8_parse_version (const char *version,unsigned int *maj, -unsigned int *min,unsigned int *rev) +struct unicode_map *utf8_load (unsigned int version) { -substring_t args[3 ]; -char version_string[12 ]; -static const struct match_token token[] = { - {1 ,"%d.%d.%d" }, - {0 ,NULL } - }; - -strncpy (version_string, version,sizeof (version_string)); - -if (match_token (version_string, token, args) !=1 ) -return -EINVAL ; - -if (match_int (&args[0 ], maj) ||match_int (&args[1 ], min) || -match_int (&args[2 ], rev)) -return -EINVAL ; +struct unicode_map *um; -return 0 ; -} - -struct unicode_map *utf8_load (const char *version) -{ -struct unicode_map *um =NULL ; -int unicode_version; - -if (version) { -unsigned int maj, min, rev; - -if (utf8_parse_version (version, &maj, &min, &rev) <0 ) -return ERR_PTR (-EINVAL ); - -if (!utf8version_is_supported (maj, min, rev)) -return ERR_PTR (-EINVAL ); - - unicode_version =UNICODE_AGE (maj, min, rev); - }else { - unicode_version =utf8version_latest (); -printk (KERN_WARNING "UTF-8 version not specified. " -"Assuming latest supported version (%d.%d.%d)." , - (unicode_version >>16 ) &0xff , - (unicode_version >>8 ) &0xff , - (unicode_version &0xff )); - } +if (!utf8version_is_supported (version)) +return ERR_PTR (-EINVAL ); um =kzalloc (sizeof (struct unicode_map ),GFP_KERNEL );if (!um)return ERR_PTR (-ENOMEM ); - um->version = unicode_version; + um->version = version;return um; }EXPORT_SYMBOL (utf8_load);fs/unicode/utf8-norm.c +2
-9@@ -15,13 +15,12 @@ struct utf8data {#include "utf8data.h" #undef __INCLUDED_FROM_UTF8NORM_C__ -int utf8version_is_supported (u8 maj,u8 min,u8 rev) +int utf8version_is_supported (unsigned int version) {int i =ARRAY_SIZE (utf8agetab) -1 ; -unsigned int sb_utf8version =UNICODE_AGE (maj, min, rev);while (i >=0 && utf8agetab[i] !=0 ) { -if (sb_utf8version == utf8agetab[i]) +if (version == utf8agetab[i])return 1 ; i--; }@@ -29,12 +28,6 @@ int utf8version_is_supported(u8 maj, u8 min, u8 rev) }EXPORT_SYMBOL (utf8version_is_supported); -int utf8version_latest (void ) -{ -return utf8vers; -} -EXPORT_SYMBOL (utf8version_latest); -/* * UTF-8 valid ranges. * fs/unicode/utf8-selftest.c +8
-7@@ -235,7 +235,7 @@ static void check_utf8_nfdicf(void)static void check_utf8_comparisons (void ) {int i; -struct unicode_map *table =utf8_load ("12.1.0" ); +struct unicode_map *table =utf8_load (UNICODE_AGE (12 ,1 ,0 ));if (IS_ERR (table)) { pr_err("%s: Unable to load utf8 %d.%d.%d. Skipping.\n",@@ -269,18 +269,19 @@ static void check_utf8_comparisons(void)static void check_supported_versions (void ) {/* Unicode 7.0.0 should be supported. */ -test (utf8version_is_supported (7 ,0 ,0 )); +test (utf8version_is_supported (UNICODE_AGE (7 ,0 ,0 )));/* Unicode 9.0.0 should be supported. */ -test (utf8version_is_supported (9 ,0 ,0 )); +test (utf8version_is_supported (UNICODE_AGE (9 ,0 ,0 )));/* Unicode 1x.0.0 (the latest version) should be supported. */ -test (utf8version_is_supported (latest_maj, latest_min, latest_rev)); +test (utf8version_is_supported ( +UNICODE_AGE (latest_maj, latest_min, latest_rev)));/* Next versions don't exist. */ -test (!utf8version_is_supported (13 ,0 ,0 )); -test (!utf8version_is_supported (0 ,0 ,0 )); -test (!utf8version_is_supported (-1 ,-1 ,-1 )); +test (!utf8version_is_supported (UNICODE_AGE (13 ,0 ,0 ))); +test (!utf8version_is_supported (UNICODE_AGE (0 ,0 ,0 ))); +test (!utf8version_is_supported (UNICODE_AGE (-1 ,-1 ,-1 ))); }static int __init init_test_ucd (void )fs/unicode/utf8n.h +2
-12@@ -11,19 +11,9 @@#include <linux/export.h> #include <linux/string.h> #include <linux/module.h> +#include <linux/unicode.h> -/* Encoding a unicode version number as a single unsigned int. */ -#define UNICODE_MAJ_SHIFT (16) -#define UNICODE_MIN_SHIFT (8) - -#define UNICODE_AGE (MAJ ,MIN ,REV ) \ - (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ - ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ - ((unsigned int)(REV))) - -/* Highest unicode version supported by the data tables. */ -extern int utf8version_is_supported (u8 maj,u8 min,u8 rev); -extern int utf8version_latest (void ); +int utf8version_is_supported (unsigned int version);/* * Look for the correct const struct utf8data for a unicode version. include/linux/unicode.h +10
-1@@ -5,6 +5,15 @@#include <linux/init.h> #include <linux/dcache.h> +/* Encoding a unicode version number as a single unsigned int. */ +#define UNICODE_MAJ_SHIFT (16) +#define UNICODE_MIN_SHIFT (8) + +#define UNICODE_AGE (MAJ ,MIN ,REV ) \ + (((unsigned int)(MAJ) << UNICODE_MAJ_SHIFT) | \ + ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ + ((unsigned int)(REV))) +struct unicode_map {unsigned int version; };@@ -29,7 +38,7 @@ int utf8_casefold(const struct unicode_map *um, const struct qstr *str,int utf8_casefold_hash (const struct unicode_map *um,const void *salt,struct qstr *str); -struct unicode_map *utf8_load (const char *version); +struct unicode_map *utf8_load (unsigned int version);void utf8_unload (struct unicode_map *um);#endif /* _LINUX_UNICODE_H */
Signature · 1 line
Gabriel Krisman Bertazi
Re: [PATCH 05/11] unicode: pass a UNICODE_AGE() tripple to utf8_load
Quoted 50 lines · Gabriel Krisman Bertazi
Quoted 32 lines · Christoph Hellwig
Don't bother with pointless string parsing when the caller can just pass the version in the format that the core expects. Also remove the fallback to the latest version that none of the callers actually uses.
Signed-off-by Christoph Hellwig <[email protected]>--- fs/ext4/super.c | 10 ++++---- fs/f2fs/super.c | 10 ++++---- fs/unicode/utf8-core.c | 50 ++++---------------------------------- fs/unicode/utf8-norm.c | 11 ++------- fs/unicode/utf8-selftest.c | 15 ++++++------ fs/unicode/utf8n.h | 14 ++--------- include/linux/unicode.h | 11 ++++++++- 7 files changed, 37 insertions(+), 84 deletions(-)fs/ext4/super.c +2
-2@@ -2016,9 +2016,9 @@ static const struct mount_opts {static const struct ext4_sb_encodings {__u16 magic;char *name; -char *version; +unsigned int version; } ext4_sb_encoding_map[] = { - {EXT4_ENC_UTF8_12_1 ,"utf8" ,"12.1.0" }, + {EXT4_ENC_UTF8_12_1 ,"utf8" ,UNICODE_AGE (12 ,1 ,0 )}, }; static const struct ext4_sb_encodings *fs/ext4/super.c static int ext4_fill_super(struct super_block *sb, void *data, int silent) · line 4315
Quoted 9 lines
fs/ext4/super.c +1
-1@@ -4308,15 +4308,15 @@ static int ext4_fill_super(struct super_block *sb, void *data, int silent) encoding =utf8_load (encoding_info->version);if (IS_ERR (encoding)) {ext4_msg (sb,KERN_ERR , -"can't mount with superblock charset: %s-%s " +"can't mount with superblock charset: %s-0x%x " "not supported by the kernel. flags: 0x%x." , encoding_info->name, encoding_info->version, encoding_flags); goto failed_mount;"Using encoding defined by superblock: utf8-0xc0100 with flags
0x0"This is much less readable than what we previously had:
"Using encoding defined by superblock: utf8-12.1.0 with flags
0x0"
Quoted 9 lines
It is minor, but can we do instead:
ext4_msg("... %u.%u.%u\n", (encoding_info->version>>12) &
0xff,(encoding_info->version>>8) & 0xff), encoding_info->version & 0xff))The rest of the series looks good and I can pick it up for 5.15, unless someone has anything else to say? It has lived on the list for a while now.
Ugh, pressed reply too quickly. Sorry for the multiple email reply.
In the summary line: tripple -> triple.
Signature · 1 line
Gabriel Krisman Bertazi
Re: [PATCH 05/11] unicode: pass a UNICODE_AGE() tripple to utf8_load
Quoted 2 lines · Gabriel Krisman Bertazi
Quoted 2 lines
ext4_msg("... %u.%u.%u\n", (encoding_info->version>>12) &
0xff,(encoding_info->version>>8) & 0xff), encoding_info->version & 0xff))
I'd rather not open code these shifts and add helpers for them.
Quoted 8 lines
Quoted 4 lines
The rest of the series looks good and I can pick it up for 5.15, unless someone has anything else to say? It has lived on the list for a while now.
Ugh, pressed reply too quickly. Sorry for the multiple email reply.
In the summary line: tripple -> triple.
Let me resend with the above change.
PATCH 6/11 unicode: remove the unused utf8{,n}age{min,max} functions
In reply to Christoph HellwigNo actually used anywhere.
--- fs/unicode/utf8-norm.c | 113 ----------------------------------------- fs/unicode/utf8n.h | 16 ------ 2 files changed, 129 deletions(-)
fs/unicode/utf8-norm.c +0 -113
@@ -391,119 +391,6 @@ static utf8leaf_t *utf8lookup(const struct utf8data *data,return utf8nlookup (data, hangul, s, (size_t )-1 ); } -/* -* Maximum age of any character in s. -* Return -1 if s is not valid UTF-8 unicode. -* Return 0 if only non-assigned code points are used. -*/ -int utf8agemax (const struct utf8data *data,const char *s) -{ -utf8leaf_t *leaf; -int age =0 ; -int leaf_age; -unsigned char hangul[UTF8HANGULLEAF ]; - -if (!data) -return -1 ; - -while (*s) { - leaf =utf8lookup (data, hangul, s); -if (!leaf) -return -1 ; - - leaf_age = utf8agetab[LEAF_GEN (leaf)]; -if (leaf_age <= data->maxage && leaf_age > age) - age = leaf_age; - s +=utf8clen (s); - } -return age; -} -EXPORT_SYMBOL (utf8agemax); - -/* -* Minimum age of any character in s. -* Return -1 if s is not valid UTF-8 unicode. -* Return 0 if non-assigned code points are used. -*/ -int utf8agemin (const struct utf8data *data,const char *s) -{ -utf8leaf_t *leaf; -int age; -int leaf_age; -unsigned char hangul[UTF8HANGULLEAF ]; - -if (!data) -return -1 ; - age = data->maxage; -while (*s) { - leaf =utf8lookup (data, hangul, s); -if (!leaf) -return -1 ; - leaf_age = utf8agetab[LEAF_GEN (leaf)]; -if (leaf_age <= data->maxage && leaf_age < age) - age = leaf_age; - s +=utf8clen (s); - } -return age; -} -EXPORT_SYMBOL (utf8agemin); - -/* -* Maximum age of any character in s, touch at most len bytes. -* Return -1 if s is not valid UTF-8 unicode. -*/ -int utf8nagemax (const struct utf8data *data,const char *s,size_t len) -{ -utf8leaf_t *leaf; -int age =0 ; -int leaf_age; -unsigned char hangul[UTF8HANGULLEAF ]; - -if (!data) -return -1 ; - -while (len && *s) { - leaf =utf8nlookup (data, hangul, s, len); -if (!leaf) -return -1 ; - leaf_age = utf8agetab[LEAF_GEN (leaf)]; -if (leaf_age <= data->maxage && leaf_age > age) - age = leaf_age; - len -=utf8clen (s); - s +=utf8clen (s); - } -return age; -} -EXPORT_SYMBOL (utf8nagemax); - -/* -* Maximum age of any character in s, touch at most len bytes. -* Return -1 if s is not valid UTF-8 unicode. -*/ -int utf8nagemin (const struct utf8data *data,const char *s,size_t len) -{ -utf8leaf_t *leaf; -int leaf_age; -int age; -unsigned char hangul[UTF8HANGULLEAF ]; - -if (!data) -return -1 ; - age = data->maxage; -while (len && *s) { - leaf =utf8nlookup (data, hangul, s, len); -if (!leaf) -return -1 ; - leaf_age = utf8agetab[LEAF_GEN (leaf)]; -if (leaf_age <= data->maxage && leaf_age < age) - age = leaf_age; - len -=utf8clen (s); - s +=utf8clen (s); - } -return age; -} -EXPORT_SYMBOL (utf8nagemin); -/* * Length of the normalization of s. * Return -1 if s is not valid UTF-8 unicode.
fs/unicode/utf8n.h +0 -16
@@ -33,22 +33,6 @@ int utf8version_is_supported(unsigned int version);extern const struct utf8data *utf8nfdi (unsigned int maxage);extern const struct utf8data *utf8nfdicf (unsigned int maxage); -/* -* Determine the maximum age of any unicode character in the string. -* Returns 0 if only unassigned code points are present. -* Returns -1 if the input is not valid UTF-8. -*/ -extern int utf8agemax (const struct utf8data *data,const char *s); -extern int utf8nagemax (const struct utf8data *data,const char *s,size_t len); - -/* -* Determine the minimum age of any unicode character in the string. -* Returns 0 if any unassigned code points are present. -* Returns -1 if the input is not valid UTF-8. -*/ -extern int utf8agemin (const struct utf8data *data,const char *s); -extern int utf8nagemin (const struct utf8data *data,const char *s,size_t len); -/* * Determine the length of the normalized from of the string, * excluding any terminating NULL byte.
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 7/11 unicode: simplify utf8len
In reply to Christoph HellwigJust use the utf8nlen implementation with a (size_t)-1 len argument, similar to utf8_lookup. Also move the function to utf8-selftest.c, as it isn't used anywhere else.
--- fs/unicode/utf8-norm.c | 30 ------------------------------ fs/unicode/utf8-selftest.c | 5 +++++ fs/unicode/utf8n.h | 1 - 3 files changed, 5 insertions(+), 31 deletions(-)
fs/unicode/utf8-norm.c +0 -30
@@ -391,36 +391,6 @@ static utf8leaf_t *utf8lookup(const struct utf8data *data,return utf8nlookup (data, hangul, s, (size_t )-1 ); } -/* -* Length of the normalization of s. -* Return -1 if s is not valid UTF-8 unicode. -* -* A string of Default_Ignorable_Code_Point has length 0. -*/ -ssize_t utf8len (const struct utf8data *data,const char *s) -{ -utf8leaf_t *leaf; -size_t ret =0 ; -unsigned char hangul[UTF8HANGULLEAF ]; - -if (!data) -return -1 ; -while (*s) { - leaf =utf8lookup (data, hangul, s); -if (!leaf) -return -1 ; -if (utf8agetab[LEAF_GEN (leaf)] > data->maxage) - ret +=utf8clen (s); -else if (LEAF_CCC (leaf) ==DECOMPOSE ) - ret +=strlen (LEAF_STR (leaf)); -else - ret +=utf8clen (s); - s +=utf8clen (s); - } -return ret; -} -EXPORT_SYMBOL (utf8len); -/* * Length of the normalization of s, touch at most len bytes. * Return -1 if s is not valid UTF-8 unicode.
fs/unicode/utf8-selftest.c +5 -0
@@ -160,6 +160,11 @@ static const struct { } }; +static ssize_t utf8len (const struct utf8data *data,const char *s) +{ +return utf8nlen (data, s, (size_t )-1 ); +} +static void check_utf8_nfdi (void ) {int i;
fs/unicode/utf8n.h +0 -1
@@ -39,7 +39,6 @@ extern const struct utf8data *utf8nfdicf(unsigned int maxage);* Returns 0 if only ignorable code points are present. * Returns -1 if the input is not valid UTF-8. */ -extern ssize_t utf8len (const struct utf8data *data,const char *s);extern ssize_t utf8nlen (const struct utf8data *data,const char *s,size_t len);/* Needed in struct utf8cursor below. */
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 8/11 unicode: move utf8cursor to utf8-selftest.c
In reply to Christoph HellwigOnly used by the tests, so no need to keep it in the core.
--- fs/unicode/utf8-norm.c | 16 ---------------- fs/unicode/utf8-selftest.c | 6 ++++++ fs/unicode/utf8n.h | 2 -- 3 files changed, 6 insertions(+), 18 deletions(-)
fs/unicode/utf8-norm.c +0 -16
@@ -456,22 +456,6 @@ int utf8ncursor(struct utf8cursor *u8c, const struct utf8data *data, }EXPORT_SYMBOL (utf8ncursor); -/* -* Set up an utf8cursor for use by utf8byte(). -* -* u8c : pointer to cursor. -* data : const struct utf8data to use for normalization. -* s : NUL-terminated string. -* -* Returns -1 on error, 0 on success. -*/ -int utf8cursor (struct utf8cursor *u8c,const struct utf8data *data, -const char *s) -{ -return utf8ncursor (u8c, data, s, (unsigned int )-1 ); -} -EXPORT_SYMBOL (utf8cursor); -/* * Get one byte from the normalized form of the string described by u8c. *
fs/unicode/utf8-selftest.c +6 -0
@@ -165,6 +165,12 @@ static ssize_t utf8len(const struct utf8data *data, const char *s)return utf8nlen (data, s, (size_t )-1 ); } +static int utf8cursor (struct utf8cursor *u8c,const struct utf8data *data, +const char *s) +{ +return utf8ncursor (u8c, data, s, (unsigned int )-1 ); +} +static void check_utf8_nfdi (void ) {int i;
fs/unicode/utf8n.h +0 -2
@@ -65,8 +65,6 @@ struct utf8cursor {* Returns 0 on success. * Returns -1 on failure. */ -extern int utf8cursor (struct utf8cursor *u8c,const struct utf8data *data, -const char *s);extern int utf8ncursor (struct utf8cursor *u8c,const struct utf8data *data,const char *s,size_t len);
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 9/11 unicode: cache the normalization tables in struct unicode_map
In reply to Christoph HellwigInstead of repeatedly looking up the version add pointers to the NFD and NFD+CF tables to struct unicode_map, and pass a unicode_map plus index to the functions using the normalization tables.
--- fs/unicode/utf8-core.c | 37 +++++++++--------- fs/unicode/utf8-norm.c | 45 ++++++++++----------- fs/unicode/utf8-selftest.c | 80 ++++++++++++++++---------------------- fs/unicode/utf8n.h | 10 +++-- include/linux/unicode.h | 19 +++++++++ 5 files changed, 97 insertions(+), 94 deletions(-)
fs/unicode/utf8-core.c +19 -18
@@ -5,16 +5,13 @@#include <linux/slab.h> #include <linux/parser.h> #include <linux/errno.h> -#include <linux/unicode.h> #include <linux/stringhash.h> #include "utf8n.h" int utf8_validate (const struct unicode_map *um,const struct qstr *str) { -const struct utf8data *data =utf8nfdi (um->version); - -if (utf8nlen (data, str->name, str->len) <0 ) +if (utf8nlen (um,UTF8_NFDI , str->name, str->len) <0 )return -1 ;return 0 ; }
@@ -23,14 +20,13 @@ EXPORT_SYMBOL(utf8_validate);int utf8_strncmp (const struct unicode_map *um,const struct qstr *s1,const struct qstr *s2) { -const struct utf8data *data =utf8nfdi (um->version);struct utf8cursor cur1, cur2;int c1, c2; -if (utf8ncursor (&cur1, data, s1->name, s1->len) <0 ) +if (utf8ncursor (&cur1, um,UTF8_NFDI , s1->name, s1->len) <0 )return -EINVAL ; -if (utf8ncursor (&cur2, data, s2->name, s2->len) <0 ) +if (utf8ncursor (&cur2, um,UTF8_NFDI , s2->name, s2->len) <0 )return -EINVAL ; do {
@@ -50,14 +46,13 @@ EXPORT_SYMBOL(utf8_strncmp);int utf8_strncasecmp (const struct unicode_map *um,const struct qstr *s1,const struct qstr *s2) { -const struct utf8data *data =utf8nfdicf (um->version);struct utf8cursor cur1, cur2;int c1, c2; -if (utf8ncursor (&cur1, data, s1->name, s1->len) <0 ) +if (utf8ncursor (&cur1, um,UTF8_NFDICF , s1->name, s1->len) <0 )return -EINVAL ; -if (utf8ncursor (&cur2, data, s2->name, s2->len) <0 ) +if (utf8ncursor (&cur2, um,UTF8_NFDICF , s2->name, s2->len) <0 )return -EINVAL ; do {
@@ -81,12 +76,11 @@ int utf8_strncasecmp_folded(const struct unicode_map *um,const struct qstr *cf, const struct qstr *s1) { - const struct utf8data *data = utf8nfdicf(um->version); struct utf8cursor cur1;int c1, c2;int i =0 ; -if (utf8ncursor (&cur1, data, s1->name, s1->len) <0 ) +if (utf8ncursor (&cur1, um,UTF8_NFDICF , s1->name, s1->len) <0 )return -EINVAL ; do {
@@ -105,11 +99,10 @@ EXPORT_SYMBOL(utf8_strncasecmp_folded);int utf8_casefold (const struct unicode_map *um,const struct qstr *str,unsigned char *dest,size_t dlen) { -const struct utf8data *data =utf8nfdicf (um->version);struct utf8cursor cur;size_t nlen =0 ; -if (utf8ncursor (&cur, data, str->name, str->len) <0 ) +if (utf8ncursor (&cur, um,UTF8_NFDICF , str->name, str->len) <0 )return -EINVAL ;for (nlen =0 ; nlen < dlen; nlen++) {
@@ -128,12 +121,11 @@ EXPORT_SYMBOL(utf8_casefold);int utf8_casefold_hash (const struct unicode_map *um,const void *salt,struct qstr *str) { -const struct utf8data *data =utf8nfdicf (um->version);struct utf8cursor cur;int c;unsigned long hash =init_name_hash (salt); -if (utf8ncursor (&cur, data, str->name, str->len) <0 ) +if (utf8ncursor (&cur, um,UTF8_NFDICF , str->name, str->len) <0 )return -EINVAL ;while ((c =utf8byte (&cur))) {
@@ -149,11 +141,10 @@ EXPORT_SYMBOL(utf8_casefold_hash);int utf8_normalize (const struct unicode_map *um,const struct qstr *str,unsigned char *dest,size_t dlen) { -const struct utf8data *data =utf8nfdi (um->version);struct utf8cursor cur;ssize_t nlen =0 ; -if (utf8ncursor (&cur, data, str->name, str->len) <0 ) +if (utf8ncursor (&cur, um,UTF8_NFDI , str->name, str->len) <0 )return -EINVAL ;for (nlen =0 ; nlen < dlen; nlen++) {
@@ -180,7 +171,17 @@ struct unicode_map *utf8_load(unsigned int version)if (!um)return ERR_PTR (-ENOMEM ); um->version = version; + um->ntab[UTF8_NFDI ] =utf8nfdi (version); +if (!um->ntab[UTF8_NFDI ]) + goto out_free_um; + um->ntab[UTF8_NFDICF ] =utf8nfdicf (version); +if (!um->ntab[UTF8_NFDICF ]) + goto out_free_um;return um; + +out_free_um: +kfree (um); +return ERR_PTR (-EINVAL ); }EXPORT_SYMBOL (utf8_load);
fs/unicode/utf8-norm.c +20 -25
118 lines not on this page · raw message
fs/unicode/utf8-selftest.c +33 -47
160 lines not on this page · raw message
fs/unicode/utf8n.h +6 -4
@@ -39,7 +39,8 @@ extern const struct utf8data *utf8nfdicf(unsigned int maxage);* Returns 0 if only ignorable code points are present. * Returns -1 if the input is not valid UTF-8. */ -extern ssize_t utf8nlen (const struct utf8data *data,const char *s,size_t len); +ssize_t utf8nlen (const struct unicode_map *um,enum utf8_normalization n, +const char *s,size_t len);/* Needed in struct utf8cursor below. */ #define UTF8HANGULLEAF (12)
@@ -48,7 +49,8 @@ extern ssize_t utf8nlen(const struct utf8data *data, const char *s, size_t len);* Cursor structure used by the normalizer. */ struct utf8cursor { -const struct utf8data *data; +const struct unicode_map *um; +enum utf8_normalization n;const char *s;const char *p;const char *ss;
@@ -65,8 +67,8 @@ struct utf8cursor {* Returns 0 on success. * Returns -1 on failure. */ -extern int utf8ncursor (struct utf8cursor *u8c,const struct utf8data *data, -const char *s,size_t len); +int utf8ncursor (struct utf8cursor *u8c,const struct unicode_map *um, +enum utf8_normalization n,const char *s,size_t len);/* * Get the next byte in the normalization.
include/linux/unicode.h +19 -0
@@ -5,6 +5,8 @@#include <linux/init.h> #include <linux/dcache.h> +struct utf8data ; +/* Encoding a unicode version number as a single unsigned int. */ #define UNICODE_MAJ_SHIFT (16)#define UNICODE_MIN_SHIFT (8)
@@ -14,8 +16,25 @@ ((unsigned int)(MIN) << UNICODE_MIN_SHIFT) | \ ((unsigned int)(REV))) +/* +* Two normalization forms are supported: +* 1) NFDI +* - Apply unicode normalization form NFD. +* - Remove any Default_Ignorable_Code_Point. +* 2) NFDICF +* - Apply unicode normalization form NFD. +* - Remove any Default_Ignorable_Code_Point. +* - Apply a full casefold (C + F). +*/ +enum utf8_normalization { +UTF8_NFDI =0 , +UTF8_NFDICF , +UTF8_NMAX , +}; +struct unicode_map {unsigned int version; +const struct utf8data *ntab[UTF8_NMAX ]; };int utf8_validate (const struct unicode_map *um,const struct qstr *str);
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 10/11 unicode: Add utf8-data module
In reply to Christoph Hellwigutf8data.h contains a large database table which is an auto-generated decodification trie for the unicode normalization functions.
Allow building it into a separate module.
Based on a patch from Shreeya Patel <[email protected]>.
---
fs/unicode/Kconfig | 13 ++++-
fs/unicode/Makefile | 13 ++---
fs/unicode/mkutf8data.c | 24 ++++++++--
fs/unicode/utf8-core.c | 35 +++++++++++---
fs/unicode/utf8-norm.c | 48 ++++---------------
fs/unicode/utf8-selftest.c | 16 +++----
...{utf8data.h_shipped => utf8data.c_shipped} | 22 +++++++--
fs/unicode/utf8n.h | 40 ++++++++--------
include/linux/unicode.h | 2 +
9 files changed, 123 insertions(+), 90 deletions(-)
rename fs/unicode/{utf8data.h_shipped => utf8data.c_shipped} (99%)fs/unicode/Kconfig +11 -2
@@ -8,7 +8,16 @@ config UNICODE Say Y here to enable UTF-8 NFD normalization and NFD+CF casefolding support. +config UNICODE_UTF8_DATA + tristate "UTF-8 normalization and casefolding tables" + depends on UNICODE + default UNICODE + help + This contains a large table of case foldings, which can be loaded as + a separate module if you say M here. To be on the safe side stick + to the default of Y. Saying N here makes no sense, if you do not want + utf8 casefolding support, disable CONFIG_UNICODE instead. + config UNICODE_NORMALIZATION_SELFTEST tristate "Test UTF-8 normalization support" - depends on UNICODE - default n + depends on UNICODE_UTF8_DATA
fs/unicode/Makefile +7 -6
@@ -2,14 +2,15 @@ obj-$(CONFIG_UNICODE) += unicode.o obj-$(CONFIG_UNICODE_NORMALIZATION_SELFTEST) += utf8-selftest.o +obj-$(CONFIG_UNICODE_UTF8_DATA) += utf8data.o unicode-y := utf8-norm.o utf8-core.o -$(obj)/utf8-norm.o: $(obj)/utf8data.h +$(obj)/utf8-data.o: $(obj)/utf8data.c -# In the normal build, the checked-in utf8data.h is just shipped. +# In the normal build, the checked-in utf8data.c is just shipped. # -# To generate utf8data.h from UCD, put *.txt files in this directory +# To generate utf8data.c from UCD, put *.txt files in this directory # and pass REGENERATE_UTF8DATA=1 from the command line. ifdef REGENERATE_UTF8DATA
@@ -24,15 +25,15 @@ quiet_cmd_utf8data = GEN $@ -t $(srctree)/$(src)/NormalizationTest.txt \ -o $@ -$(obj)/utf8data.h: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE +$(obj)/utf8data.c: $(obj)/mkutf8data $(filter %.txt, $(cmd_utf8data)) FORCE $(call if_changed,utf8data) else -$(obj)/utf8data.h: $(src)/utf8data.h_shipped FORCE +$(obj)/utf8data.c: $(src)/utf8data.c_shipped FORCE $(call if_changed,shipped) endif -targets += utf8data.h +targets += utf8data.c hostprogs += mkutf8data
fs/unicode/mkutf8data.c +19 -5
@@ -3287,12 +3287,10 @@ static void write_file(void)open_fail (utf8_name, errno);fprintf (file,"/* This file is generated code, do not edit. */\n" ); -fprintf (file,"#ifndef __INCLUDED_FROM_UTF8NORM_C__\n" ); -fprintf (file,"#error Only nls_utf8-norm.c should include this file.\n" ); -fprintf (file,"#endif\n" );fprintf (file,"\n" ); -fprintf (file,"static const unsigned int utf8vers = %#x;\n" , - unicode_maxage); +fprintf (file,"#include <linux/module.h>\n" ); +fprintf (file,"#include <linux/kernel.h>\n" ); +fprintf (file,"#include \"utf8n.h\"\n" );fprintf (file,"\n" );fprintf (file,"static const unsigned int utf8agetab[] = {\n" );for (i =0 ; i != ages_count; i++)
@@ -3339,6 +3337,22 @@ static void write_file(void)fprintf (file,"\n" ); }fprintf (file,"};\n" ); +fprintf (file,"\n" ); +fprintf (file,"struct utf8data_table utf8_data_table = {\n" ); +fprintf (file,"\t.utf8agetab = utf8agetab,\n" ); +fprintf (file,"\t.utf8agetab_size = ARRAY_SIZE(utf8agetab),\n" ); +fprintf (file,"\n" ); +fprintf (file,"\t.utf8nfdicfdata = utf8nfdicfdata,\n" ); +fprintf (file,"\t.utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata),\n" ); +fprintf (file,"\n" ); +fprintf (file,"\t.utf8nfdidata = utf8nfdidata,\n" ); +fprintf (file,"\t.utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata),\n" ); +fprintf (file,"\n" ); +fprintf (file,"\t.utf8data = utf8data,\n" ); +fprintf (file,"};\n" ); +fprintf (file,"EXPORT_SYMBOL_GPL(utf8_data_table);" ); +fprintf (file,"\n" ); +fprintf (file,"MODULE_LICENSE(\"GPL v2\");\n" );fclose (file); }
fs/unicode/utf8-core.c +28 -7
@@ -160,25 +160,45 @@ int utf8_normalize(const struct unicode_map *um, const struct qstr *str, }EXPORT_SYMBOL (utf8_normalize); +static const struct utf8data *find_table_version (const struct utf8data *table, +size_t nr_entries,unsigned int version) +{ +size_t i = nr_entries -1 ; + +while (version < table[i].maxage) + i--; +if (version > table[i].maxage) +return NULL ; +return &table[i]; +} +struct unicode_map *utf8_load (unsigned int version) {struct unicode_map *um; -if (!utf8version_is_supported (version)) -return ERR_PTR (-EINVAL ); - um =kzalloc (sizeof (struct unicode_map ),GFP_KERNEL );if (!um)return ERR_PTR (-ENOMEM ); um->version = version; - um->ntab[UTF8_NFDI ] =utf8nfdi (version); -if (!um->ntab[UTF8_NFDI ]) + + um->tables =symbol_request (utf8_data_table); +if (!um->tables) goto out_free_um; - um->ntab[UTF8_NFDICF ] =utf8nfdicf (version); + +if (!utf8version_is_supported (um, version)) + goto out_symbol_put; + um->ntab[UTF8_NFDI ] =find_table_version (um->tables->utf8nfdidata, + um->tables->utf8nfdidata_size, um->version); +if (!um->ntab[UTF8_NFDI ]) + goto out_symbol_put; + um->ntab[UTF8_NFDICF ] =find_table_version (um->tables->utf8nfdicfdata, + um->tables->utf8nfdicfdata_size, um->version);if (!um->ntab[UTF8_NFDICF ]) - goto out_free_um; + goto out_symbol_put;return um; +out_symbol_put: +symbol_put (um->tables); out_free_um:kfree (um);return ERR_PTR (-EINVAL );
@@ -187,6 +207,7 @@ EXPORT_SYMBOL(utf8_load);void utf8_unload (struct unicode_map *um) { +symbol_put (utf8_data_table);kfree (um); }EXPORT_SYMBOL (utf8_unload);
fs/unicode/utf8-norm.c +9 -39
83 lines not on this page · raw message
fs/unicode/utf8-selftest.c +8 -8
@@ -255,21 +255,21 @@ static void check_utf8_comparisons(struct unicode_map *table) } } -static void check_supported_versions (void ) +static void check_supported_versions (struct unicode_map *um) {/* Unicode 7.0.0 should be supported. */ -test (utf8version_is_supported (UNICODE_AGE (7 ,0 ,0 ))); +test (utf8version_is_supported (um,UNICODE_AGE (7 ,0 ,0 )));/* Unicode 9.0.0 should be supported. */ -test (utf8version_is_supported (UNICODE_AGE (9 ,0 ,0 ))); +test (utf8version_is_supported (um,UNICODE_AGE (9 ,0 ,0 )));/* Unicode 1x.0.0 (the latest version) should be supported. */ -test (utf8version_is_supported (UTF8_LATEST )); +test (utf8version_is_supported (um,UTF8_LATEST ));/* Next versions don't exist. */ -test (!utf8version_is_supported (UNICODE_AGE (13 ,0 ,0 ))); -test (!utf8version_is_supported (UNICODE_AGE (0 ,0 ,0 ))); -test (!utf8version_is_supported (UNICODE_AGE (-1 ,-1 ,-1 ))); +test (!utf8version_is_supported (um,UNICODE_AGE (13 ,0 ,0 ))); +test (!utf8version_is_supported (um,UNICODE_AGE (0 ,0 ,0 ))); +test (!utf8version_is_supported (um,UNICODE_AGE (-1 ,-1 ,-1 ))); }static int __init init_test_ucd (void )
@@ -285,7 +285,7 @@ static int __init init_test_ucd(void)return PTR_ERR (um); } -check_supported_versions (); +check_supported_versions (um);check_utf8_nfdi (um);check_utf8_nfdicf (um);check_utf8_comparisons (um);
fs/unicode/utf8data.c_shipped renamed from fs/unicode/utf8data.h_shipped +18 -4
@@ -1,9 +1,8 @@ /* This file is generated code, do not edit. */ -#ifndef __INCLUDED_FROM_UTF8NORM_C__ -#error Only nls_utf8-norm.c should include this file. -#endif -static const unsigned int utf8vers = 0xc0100; +#include <linux/module.h> +#include <linux/kernel.h> +#include "utf8n.h" static const unsigned int utf8agetab[] = { 0,
@@ -4107,3 +4106,18 @@ static const unsigned char utf8data[64256] = { 0x52,0x04,0x00,0x00,0x11,0x04,0x00,0x00,0x02,0x00,0xcf,0x86,0xcf,0x06,0x02,0x00, 0x81,0x80,0xcf,0x86,0x85,0x84,0xcf,0x86,0xcf,0x06,0x02,0x00,0x00,0x00,0x00,0x00 }; + +struct utf8data_table utf8_data_table = { + .utf8agetab = utf8agetab, + .utf8agetab_size = ARRAY_SIZE(utf8agetab), + + .utf8nfdicfdata = utf8nfdicfdata, + .utf8nfdicfdata_size = ARRAY_SIZE(utf8nfdicfdata), + + .utf8nfdidata = utf8nfdidata, + .utf8nfdidata_size = ARRAY_SIZE(utf8nfdidata), + + .utf8data = utf8data, +}; +EXPORT_SYMBOL_GPL(utf8_data_table); +MODULE_LICENSE("GPL v2");
fs/unicode/utf8n.h +21 -19
@@ -13,25 +13,7 @@#include <linux/module.h> #include <linux/unicode.h> -int utf8version_is_supported (unsigned int version); - -/* -* Look for the correct const struct utf8data for a unicode version. -* Returns NULL if the version requested is too new. -* -* Two normalization forms are supported: nfdi and nfdicf. -* -* nfdi: -* - Apply unicode normalization form NFD. -* - Remove any Default_Ignorable_Code_Point. -* -* nfdicf: -* - Apply unicode normalization form NFD. -* - Remove any Default_Ignorable_Code_Point. -* - Apply a full casefold (C + F). -*/ -extern const struct utf8data *utf8nfdi (unsigned int maxage); -extern const struct utf8data *utf8nfdicf (unsigned int maxage); +int utf8version_is_supported (const struct unicode_map *um,unsigned int version);/* * Determine the length of the normalized from of the string,
@@ -78,4 +60,24 @@ int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um,*/ extern int utf8byte (struct utf8cursor *u8c); +struct utf8data { +unsigned int maxage; +unsigned int offset; +}; + +struct utf8data_table { +const unsigned int *utf8agetab; +int utf8agetab_size; + +const struct utf8data *utf8nfdicfdata; +int utf8nfdicfdata_size; + +const struct utf8data *utf8nfdidata; +int utf8nfdidata_size; + +const unsigned char *utf8data; +}; + +extern struct utf8data_table utf8_data_table; +#endif /* UTF8NORM_H */
include/linux/unicode.h +2 -0
@@ -6,6 +6,7 @@#include <linux/dcache.h> struct utf8data ; +struct utf8data_table ;/* Encoding a unicode version number as a single unsigned int. */ #define UNICODE_MAJ_SHIFT (16)
@@ -35,6 +36,7 @@ enum utf8_normalization {struct unicode_map {unsigned int version;const struct utf8data *ntab[UTF8_NMAX ]; +const struct utf8data_table *tables; };int utf8_validate (const struct unicode_map *um,const struct qstr *str);
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
PATCH 11/11 unicode: only export internal symbols for the selftests
In reply to Christoph HellwigThe exported symbols in utf8-norm.c are not needed for normal file system consumers, so move them to conditional _GPL exports just for the selftest.
--- fs/unicode/utf8-norm.c | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-)
fs/unicode/utf8-norm.c +7 -4
@@ -17,7 +17,6 @@ int utf8version_is_supported(const struct unicode_map *um, unsigned int version) }return 0 ; } -EXPORT_SYMBOL (utf8version_is_supported);/* * UTF-8 valid ranges.
@@ -407,7 +406,6 @@ ssize_t utf8nlen(const struct unicode_map *um, enum utf8_normalization n, }return ret; } -EXPORT_SYMBOL (utf8nlen);/* * Set up an utf8cursor for use by utf8byte().
@@ -442,7 +440,6 @@ int utf8ncursor(struct utf8cursor *u8c, const struct unicode_map *um,return -1 ;return 0 ; } -EXPORT_SYMBOL (utf8ncursor);/* * Get one byte from the normalized form of the string described by u8c.
@@ -588,4 +585,10 @@ int utf8byte(struct utf8cursor *u8c) } } } -EXPORT_SYMBOL (utf8byte); + +#ifdef CONFIG_UNICODE_NORMALIZATION_SELFTEST_MODULE +EXPORT_SYMBOL_GPL (utf8version_is_supported); +EXPORT_SYMBOL_GPL (utf8nlen); +EXPORT_SYMBOL_GPL (utf8ncursor); +EXPORT_SYMBOL_GPL (utf8byte); +#endif
Signature · 8 lines
2.30.2 _______________________________________________ Linux-f2fs-devel mailing list [email protected] https://lists.sourceforge.net/lists/listinfo/linux-f2fs-devel
Re: [PATCH 11/11] unicode: only export internal symbols for the selftests
Quoted 3 lines · Christoph Hellwig
The exported symbols in
utf8-norm.care not needed for normal file system consumers, so move them to conditional_GPLexports just for the selftest.
Would it be better to use EXPORT_SYMBOL_NS_GPL()?
Re: [PATCH 11/11] unicode: only export internal symbols for the selftests
Quoted 6 lines · Matthew Wilcox
Quoted 3 lines · Christoph Hellwig
The exported symbols in
utf8-norm.care not needed for normal file system consumers, so move them to conditional_GPLexports just for the selftest.Would it be better to use
EXPORT_SYMBOL_NS_GPL()?
Maybe. Even better would be the EXPORT_SYMBOL_FOR I've been prototyping for a while but which is still not ready.