nls: add surrogate pair support in nls utf8.
PATCH 1/1 nls: add surrogate pair support in nls utf8.
It allows surrogate pair in nls utf8.
--- fs/nls/nls_base.c | 8 -------- fs/nls/nls_utf8.c | 15 +++++++++++++-- include/linux/nls.h | 8 ++++++++ 3 files changed, 21 insertions(+), 10 deletions(-)
fs/nls/nls_base.c +0 -8
@@ -44,14 +44,6 @@ static const struct utf8_table utf8_table[] = {0, /* end of table */} }; -#define UNICODE_MAX 0x0010ffff -#define PLANE_SIZE 0x00010000 - -#define SURROGATE_MASK 0xfffff800 -#define SURROGATE_PAIR 0x0000d800 -#define SURROGATE_LOW 0x00000400 -#define SURROGATE_BITS 0x000003ff -int utf8_to_utf32 (const u8 *s,int len,unicode_t *pu) {unsigned long l;
fs/nls/nls_utf8.c +13 -2
@@ -30,13 +30,24 @@ static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) {int n;unicode_t u; +u16 *op; + op = uni; n =utf8_to_utf32 (rawstring, boundlen, &u); -if (n <0 || u >MAX_WCHAR_T ) { +if (n <0 || u >UNICODE_MAX ) { *uni =0x003f ;/* ? */ return -EINVAL ; } - *uni = (wchar_t ) u; + +if (u >=PLANE_SIZE ) { + u -=PLANE_SIZE ; + *op++ = (wchar_t) (SURROGATE_PAIR | + ((u >>10 ) &SURROGATE_BITS )); + *op++ = (wchar_t) (SURROGATE_PAIR | +SURROGATE_LOW | (u &SURROGATE_BITS )); + }else + *op++ = (wchar_t ) u; +return n; }
include/linux/nls.h +8 -0
@@ -18,6 +18,14 @@typedef u16 wchar_t ;#define MAX_WCHAR_T 0xffff +/* Plane-1 Unicode surrogate pair characters */ +#define UNICODE_MAX 0x0010ffff +#define PLANE_SIZE 0x00010000 +#define SURROGATE_MASK 0xfffff800 +#define SURROGATE_PAIR 0x0000d800 +#define SURROGATE_LOW 0x00000400 +#define SURROGATE_BITS 0x000003ff +/* Arbitrary Unicode character */ typedef u32 unicode_t ;
Signature · 1 line
1.7.5.4
Quoted 1 line · Namjae Jeon
It allows surrogate pair in nls utf8.
That's a pretty brief description.
Quoted 18 lines
fs/nls/nls_utf8.c +5
-2@@ -30,13 +30,24 @@ static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) {int n;unicode_t u; +u16 *op; + op = uni; n =utf8_to_utf32 (rawstring, boundlen, &u); -if (n <0 || u >MAX_WCHAR_T ) { +if (n <0 || u >UNICODE_MAX ) { *uni =0x003f ;/* ? */ return -EINVAL ; } - *uni = (wchar_t ) u; + +if (u >=PLANE_SIZE ) {
fs/nls/nls_utf8.c static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) · line 52
Quoted 10 lines
fs/nls/nls_utf8.c +8
-0+ u -=PLANE_SIZE ; + *op++ = (wchar_t) (SURROGATE_PAIR | + ((u >>10 ) &SURROGATE_BITS )); + *op++ = (wchar_t) (SURROGATE_PAIR | +SURROGATE_LOW | (u &SURROGATE_BITS )); + }else + *op++ = (wchar_t ) u; +return n; }
Firstly, have you checked whether the callers of this function expect to receive back more than one 16-bit value? Maybe you will overrun their buffers by doing this.
Secondly, you shouldn't have to make all these changes. Just call utf8s_to_utf16s(); then all you have to worry about is changing an invalid character to a '?'.
Alan Stern
2011/12/5 Alan Stern <[email protected]>:
Quoted 5 lines · Namjae Jeon
Quoted 2 lines
It allows surrogate pair in nls utf8.
That's a pretty brief description.
okay~ I will rewrite more.
Quoted 25 lines
Quoted 25 lines
fs/nls/nls_utf8.c +11
-2@@ -30,13 +30,24 @@ static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) { int n; unicode_t u; + u16 *op; + op = uni; n = utf8_to_utf32(rawstring, boundlen, &u); - if (n < 0 || u > MAX_WCHAR_T) { + if (n < 0 || u > UNICODE_MAX) { *uni = 0x003f; /* ? */ return -EINVAL; } - *uni = (wchar_t) u; + + if (u >= PLANE_SIZE) { + u -= PLANE_SIZE; + *op++ = (wchar_t) (SURROGATE_PAIR | + ((u >> 10) & SURROGATE_BITS)); + *op++ = (wchar_t) (SURROGATE_PAIR | + SURROGATE_LOW | (u & SURROGATE_BITS)); + } else
Quoted 8 lines
fs/nls/nls_utf8.c static int char2uni(const unsigned char *rawstring, int boundlen, wchar_t *uni) · line 52
Quoted 4 lines
fs/nls/nls_utf8.c +2
-0+ *op++ = (wchar_t) u; + return n; }Firstly, have you checked whether the callers of this function expect to receive back more than one 16-bit value? Maybe you will overrun their buffers by doing this.
Hi Alan.
first Thanks for your review.
yes, you're right. and yes I have checked it on FAT fs, I will try to post the below FAT patch after this patch is applied.
-------------------------------------------------------------------------------------------------------------------------------------------------------------------------
fs/fat/namei_vfat.c +0 -0
@@ -555,7 +555,10 @@ xlate_to_uni(const unsigned char *name, int len,
unsigned char *outname,
return -EINVAL;
ip += charlen;
i += charlen;- op += 2;
+ if (charlen == sizeof(unicode_t))
+ op += 4;
+ else
+ op += 2;
}
}-----------------------------------------------------------------------------------------------------------------------------------
Quoted 4 lines
Secondly, you shouldn't have to make all these changes. Just call
utf8s_to_utf16s(); then all you have to worry about is changing an invalid character to a '?'.
Currently there are two paths along mount option. if using -o utf8 mount option, utf8s_to_utf16s have been used in xlate_to_uni of FAT. and if using iocharset=utf8, char2uni have been used.. nls->char2uni works on a single charachter while utf8s_to_utf16s works on whole string.
when the mount option(uni_xlate) that require scanning character by character is used it is needed.
Quoted 3 lines
Alan Stern
Quoted 7 lines · Namjae Jeon
Quoted 3 lines
Firstly, have you checked whether the callers of this function expect to receive back more than one 16-bit value? Maybe you will overrun their buffers by doing this.
Hi Alan.
first Thanks for your review.
yes, you're right. and yes I have checked it on FAT fs, I will try to post the below FAT patch after this patch is applied.
But what about other callers? Have you checked the entire kernel source to see if char2uni is used anywhere else?
fs/fat/namei_vfat.c xlate_to_uni(const unsigned char *name, int len, · line 562
Quoted 12 lines
fs/fat/namei_vfat.c +4
-1@@ -555,7 +555,10 @@ xlate_to_uni(const unsigned char *name, int len,unsigned char *outname, return -EINVAL; ip += charlen; i += charlen; - op +=2 ; +if (charlen ==sizeof (unicode_t)) + op +=4 ; +else + op +=2 ;
This seems completely wrong. The amount you increment the output pointer doesn't depend on the length of the input character; it depends on whether the output needed to use a surrogate pair.
Furthermore, it doesn't answer my question. What happens if _every_ character in the input filename has to be converted to a surrogate pair? Then the output string will be twice as long as xlate_to_uni() expects, so it might overrun the output buffer.
Quoted 10 lines
Quoted 3 lines
Secondly, you shouldn't have to make all these changes. Just call
utf8s_to_utf16s(); then all you have to worry about is changing an invalid character to a '?'.Currently there are two paths along mount option. if using -o utf8 mount option,
utf8s_to_utf16shave been used inxlate_to_uniof FAT. and if using iocharset=utf8, char2uni have been used..nls->char2uniworks on a single charachter whileutf8s_to_utf16sworks on whole string.
when the mountoption(uni_xlate)that require scanning character by character is used it is needed.
That doesn't matter. The point is that you have duplicated the code in nls_base.c, which is a bad idea. Instead of making a copy of the code, you should use the code that is already there. You may find that the easiest way is to add a new utf32_to_utf16 function in nls_base.c.
Alan Stern