Fix to convert SURROGATE PAIR filename
Subsystems (2)
PATCH 1/1 Fix to convert SURROGATE PAIR filename
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)
This causes
-- inode number duplication (when ?? and ?? exist) . -- can't remove file, directory.
-- can't operate under directory.
-- auto-remount with noserverino.
[Steps to Reproduce]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .
[BUG description]
Some functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()
-- return incorrect length, because of not considering about surrogate pair.
This causes problem about SymbolicLink with surrogate pair filename.
cifs_mapchar()
-- not considering about surrogate pair.
cifs_from_utf16()
-- not convert surrogate pair from SMB response (from UTF-16 to UTF-8)
, then ls shows garbled characters.
cifsConvertToUTF16()
-- not convert surrogate pair when mapchars(SFM/SFU).
From c477f22597f5a1c2f3fa8f3c5f24173b5fcf5469 Mon Sep 17 00:00:00 2001
From: Nakajima Akira <[email protected]>
Date: Wed, 4 Feb 2015 16:00:42 +0900
Subject: [PATCH] Fix to convert SURROGATE PAIR
--- fs/cifs/cifs_unicode.c | 183 ++++++++++++++++++++++++++++++++++++------------ 1 files changed, 137 insertions(+), 46 deletions(-)
fs/cifs/cifs_unicode.c +137 -46
@@ -27,41 +27,6 @@#include "cifsglob.h" #include "cifs_debug.h" -/* -* cifs_utf16_bytes - how long will a string be after conversion? -* @utf16 - pointer to input string -* @maxbytes - don't go past this many bytes of input string -* @codepage - destination codepage -* -* Walk a utf16le string and return the number of bytes that the string will -* be after being converted to the given charset, not including any null -* termination required. Don't walk past maxbytes in the source buffer. -*/ -int -cifs_utf16_bytes (const __le16 *from,int maxbytes, -const struct nls_table *codepage) -{ -int i; -int charlen, outlen =0 ; -int maxwords = maxbytes /2 ; -char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; - -for (i =0 ; i < maxwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) -break ; - - charlen = codepage->uni2char (ftmp, tmp,NLS_MAX_CHARSET_SIZE ); -if (charlen >0 ) - outlen += charlen; -else - outlen++; - } - -return outlen; -} -int cifs_remap (struct cifs_sb_info *cifs_sb) {int map_type;
@@ -155,10 +120,13 @@ convert_sfm_char(const __u16 src_char, char *target)* enough to hold the result of the conversion (at least NLS_MAX_CHARSET_SIZE). */ static int -cifs_mapchar (char *target,const __u16 src_char,const struct nls_table *cp, +cifs_mapchar (char *target,const __u16 *from,const struct nls_table *cp,int maptype) {int len =1 ; +__u16 src_char; + + src_char = *from;if ((maptype ==SFM_MAP_UNI_RSVD ) &&convert_sfm_char (src_char, target))return len;
@@ -168,10 +136,23 @@ cifs_mapchar(char *target, const __u16 src_char, const struct nls_table *cp,/* if character not one of seven in special remap set */ len = cp->uni2char (src_char, target,NLS_MAX_CHARSET_SIZE ); -if (len <=0 ) { - *target ='?' ; - len =1 ; - } +if (len <=0 ) + goto surrogate_pair; + +return len; + +surrogate_pair: +/* convert SURROGATE_PAIR and IVS */ +if (strcmp (cp->charset,"utf8" )) + goto unknown; + len =utf16s_to_utf8s (from,3 ,UTF16_LITTLE_ENDIAN , target,6 ); +if (len <=0 ) + goto unknown; +return len; + +unknown: + *target ='?' ; + len =1 ;return len; }
@@ -206,7 +187,7 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,int nullsize =nls_nullsize (codepage);int fromwords = fromlen /2 ;char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; +__u16 ftmp[3 ];/* ftmp[3] = 3array x 2bytes = 6bytes UTF-16 */ /* * because the chars can be of varying widths, we need to take care
@@ -217,9 +198,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, safelen = tolen - (NLS_MAX_CHARSET_SIZE + nullsize);for (i =0 ; i < fromwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 )break ; +if (i +1 < fromwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < fromwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ;/* * check to see if converting this character might make the
@@ -234,6 +223,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,/* put converted char into 'to' buffer */ charlen =cifs_mapchar (&to[outlen], ftmp, codepage, map_type); outlen += charlen; + +/* charlen (=bytes of UTF-8 for 1 character) +* 4bytes UTF-8(surrogate pair) is charlen=4 +* (4bytes UTF-16 code) +* 7-8bytes UTF-8(IVS) is charlen=3+4 or 4+4 +* (2 UTF-8 pairs divided to 2 UTF-16 pairs) */ +if (charlen ==4 ) + i++; +else if (charlen >=5 ) +/* 5-6bytes UTF-8 */ + i +=2 ; }/* properly null-terminate string */
@@ -296,6 +296,46 @@ success: }/* +* cifs_utf16_bytes - how long will a string be after conversion? +* @utf16 - pointer to input string +* @maxbytes - don't go past this many bytes of input string +* @codepage - destination codepage +* +* Walk a utf16le string and return the number of bytes that the string will +* be after being converted to the given charset, not including any null +* termination required. Don't walk past maxbytes in the source buffer. +*/ +int +cifs_utf16_bytes (const __le16 *from,int maxbytes, +const struct nls_table *codepage) +{ +int i; +int charlen, outlen =0 ; +int maxwords = maxbytes /2 ; +char tmp[NLS_MAX_CHARSET_SIZE ]; +__u16 ftmp[3 ]; + +for (i =0 ; i < maxwords; i++) { + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 ) +break ; +if (i +1 < maxwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < maxwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ; + + charlen =cifs_mapchar (tmp, ftmp, codepage,NO_MAP_UNI_RSVD ); + outlen += charlen; + } + +return outlen; +} + +/* * cifs_strndup_from_utf16 - copy a string from wire format to the local * codepage * @src - source string
@@ -412,10 +452,15 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,char src_char;__le16 dst_char;wchar_t tmp; +wchar_t *wchar_to;/* UTF-16 */ +int ret; +unicode_t u;if (map_chars ==NO_MAP_UNI_RSVD )return cifs_strtoUTF16 (target, source,PATH_MAX , cp); + wchar_to =kzalloc (6 ,GFP_KERNEL ); +for (i =0 ; i < srclen; j++) { src_char = source[i]; charlen =1 ;
@@ -444,11 +489,55 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,* if no match, use question mark, which at least in * some cases serves as wild card */ -if (charlen <1 ) { - dst_char =cpu_to_le16 (0x003f ); - charlen =1 ; +if (charlen >0 ) + goto ctoUTF16; + +/* convert SURROGATE_PAIR */ +if (strcmp (cp->charset,"utf8" ) || !wchar_to) + goto unknown; +if (*(source + i) &0x80 ) { + charlen =utf8_to_utf32 (source + i,6 , &u); +if (charlen <0 ) + goto unknown; + }else + goto unknown; + ret =utf8s_to_utf16s (source + i, charlen, +UTF16_LITTLE_ENDIAN , + wchar_to,6 ); +if (ret <0 ) + goto unknown; + + i += charlen; + dst_char = *wchar_to; +if (charlen <=3 ) +/* 1-3bytes UTF-8 to 2bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); +else if (charlen ==4 ) { +/* 4bytes UTF-8(surrogate pair) to 4bytes UTF-16 +* 7-8bytes UTF-8(IVS) divided to 2 UTF-16 +* (charlen=3+4 or 4+4) */ +put_unaligned (dst_char, &target[j]); + dst_char = *(wchar_to +1 ); + j++; +put_unaligned (dst_char, &target[j]); + }else if (charlen >=5 ) { +/* 5-6bytes UTF-8 to 6bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); + dst_char = *(wchar_to +1 ); + j++; +put_unaligned (dst_char, &target[j]); + dst_char = *(wchar_to +2 ); + j++; +put_unaligned (dst_char, &target[j]); } +continue ; + +unknown: + dst_char =cpu_to_le16 (0x003f ); + charlen =1 ; } + +ctoUTF16:/* * character may take more than one byte in the source string, * but will take exactly two bytes in the target string
@@ -459,6 +548,8 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen, ctoUTF16_out:put_unaligned (0 , &target[j]);/* Null terminate target unicode string */ +if (wchar_to) +kfree (wchar_to);return j; }
Signature · 1 line
1.7.1
Note build warnings (when running normal sparse checks during compile) looks like need to resolve endian errors so that this is sure to work in big endian, not just little endian boxes
e.g.
make C=1 M=fs/cifs modules CF=-D__CHECK_ENDIAN__
CHECK fs/cifs/cifs_unicode.cfs/cifs/cifs_unicode.c:508:34: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:508:34: expected restricted __le16
[assigned] [usertype] dst_charfs/cifs/cifs_unicode.c:508:34: got unsigned short [unsigned]
[short] [usertype] <noident> fs/cifs/cifs_unicode.c:517:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:517:42: expected restricted __le16
[assigned] [usertype] dst_charfs/cifs/cifs_unicode.c:517:42: got unsigned short [unsigned]
[short] [usertype] <noident> fs/cifs/cifs_unicode.c:523:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:523:42: expected restricted __le16
[assigned] [usertype] dst_charfs/cifs/cifs_unicode.c:523:42: got unsigned short [unsigned]
[short] [usertype] <noident> fs/cifs/cifs_unicode.c:526:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:526:42: expected restricted __le16
[assigned] [usertype] dst_charfs/cifs/cifs_unicode.c:526:42: got unsigned short [unsigned]
[short] [usertype] <noident>
Quoted 34 lines · Nakajima Akira
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)This causes
-- inode number duplication (when ?? and ?? exist) . -- can't remove file, directory.
-- can't operate under directory.
-- auto-remount with noserverino.[Steps to Reproduce]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .[BUG description]
Some functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()
-- return incorrect length, because of not considering about surrogate pair.
This causes problem about SymbolicLink with surrogate pair filename.
cifs_mapchar()
-- not considering about surrogate pair.
cifs_from_utf16()
-- not convert surrogate pair from SMB response (from UTF-16 to UTF-8)
, then ls shows garbled characters.
cifsConvertToUTF16()
-- not convert surrogate pair when mapchars(SFM/SFU).
Signature · 3 lines
Thanks, Steve
Other than the endian bug, the patch tested out fine.
Quoted 77 lines · Steve French
Note build warnings (when running normal sparse checks during compile) looks like need to resolve endian errors so that this is sure to work in big endian, not just little endian boxes
e.g.
make C=1 M=fs/cifs modules CF=-D__CHECK_ENDIAN__
CHECK
fs/cifs/cifs_unicode.cfs/cifs/cifs_unicode.c:508:34: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:508:34: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:508:34: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:517:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:517:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:517:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:523:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:523:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:523:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:526:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:526:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:526:42: got unsigned short [unsigned]
[short] [usertype] <noident>Quoted 28 lines · Nakajima Akira
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)This causes
-- inode number duplication (when ?? and ?? exist) . -- can't remove file, directory.
-- can't operate under directory.
-- auto-remount with noserverino.[Steps to Reproduce]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .[BUG description]
Some functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()
-- return incorrect length, because of not considering about surrogate pair.
This causes problem about SymbolicLink with surrogate pair filename.
cifs_mapchar()
-- not considering about surrogate pair.Quoted 6 lines
cifs_from_utf16()
-- not convert surrogate pair from SMB response (from UTF-16 to UTF-8)
, then ls shows garbled characters.
cifsConvertToUTF16()
-- not convert surrogate pair when mapchars(SFM/SFU).--
Thanks,Steve
Signature · 3 lines
Thanks, Steve
Quoted 34 lines · Steve French
Note build warnings (when running normal sparse checks during compile) looks like need to resolve endian errors so that this is sure to work in big endian, not just little endian boxes
e.g.
make C=1 M=fs/cifs modules CF=-D__CHECK_ENDIAN__
CHECK
fs/cifs/cifs_unicode.cfs/cifs/cifs_unicode.c:508:34: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:508:34: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:508:34: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:517:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:517:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:517:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:523:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:523:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:523:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:526:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:526:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:526:42: got unsigned short [unsigned]
[short] [usertype] <noident>
Quoted 36 lines
Quoted 34 lines · Nakajima Akira
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)This causes
-- inode number duplication (when ?? and ?? exist) . -- can't remove file, directory.
-- can't operate under directory.
-- auto-remount with noserverino.[Steps to Reproduce]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .[BUG description]
Some functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()
-- return incorrect length, because of not considering about surrogate pair.
This causes problem about SymbolicLink with surrogate pair filename.
cifs_mapchar()
-- not considering about surrogate pair.
cifs_from_utf16()
-- not convert surrogate pair from SMB response (from UTF-16 to UTF-8)
, then ls shows garbled characters.
cifsConvertToUTF16()
-- not convert surrogate pair when mapchars(SFM/SFU).
I fixed endian bug by using function cpu_to_le16(). But I don't have Big Endian machine.
Could someone check on Big Endian machine?
I'm trying pearpc(PowerPC Emulator), but pearpc doesn't work.
(pearpc need yaboot?)
[Procedure]
client# touch printf '\xf0\x9d\x9f\xa3'
client# touch printf '\xf0\x9d\x9f\xa4'
client# ls -li
--Before patch
You see same inode number, same filename(=?? and ??) .
--After patch
You see correct inode numbers, correct filenames(look like 1 and 2)
Quoted 1 line
From
4b0db586e09916a88ac2ac7a1714e52c28781ff6Mon Sep 17 00:00:00 2001
From: Nakajima Akira <nakajima.akira-o7dWnD6vFTHqq2nvvmkE/[email protected]>
Date: Thu, 9 Apr 2015 17:27:39 +0900
Subject: [PATCH] Fix to convert SURROGATE PAIR
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)
[Steps to Reproduce for bug]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .
Fix the bug about these functions do not consider about surrogate pair (and IVS). cifs_utf16_bytes()cifs_mapchar()cifs_from_utf16()cifsConvertToUTF16()
--- fs/cifs/cifs_unicode.c | 182 ++++++++++++++++++++++++++++++++++++------------ 1 files changed, 136 insertions(+), 46 deletions(-)
fs/cifs/cifs_unicode.c +136 -46
@@ -27,41 +27,6 @@#include "cifsglob.h" #include "cifs_debug.h" -/* -* cifs_utf16_bytes - how long will a string be after conversion? -* @utf16 - pointer to input string -* @maxbytes - don't go past this many bytes of input string -* @codepage - destination codepage -* -* Walk a utf16le string and return the number of bytes that the string will -* be after being converted to the given charset, not including any null -* termination required. Don't walk past maxbytes in the source buffer. -*/ -int -cifs_utf16_bytes (const __le16 *from,int maxbytes, -const struct nls_table *codepage) -{ -int i; -int charlen, outlen =0 ; -int maxwords = maxbytes /2 ; -char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; - -for (i =0 ; i < maxwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) -break ; - - charlen = codepage->uni2char (ftmp, tmp,NLS_MAX_CHARSET_SIZE ); -if (charlen >0 ) - outlen += charlen; -else - outlen++; - } - -return outlen; -} -int cifs_remap (struct cifs_sb_info *cifs_sb) {int map_type;
@@ -155,10 +120,13 @@ convert_sfm_char(const __u16 src_char, char *target)* enough to hold the result of the conversion (at least NLS_MAX_CHARSET_SIZE). */ static int -cifs_mapchar (char *target,const __u16 src_char,const struct nls_table *cp, +cifs_mapchar (char *target,const __u16 *from,const struct nls_table *cp,int maptype) {int len =1 ; +__u16 src_char; + + src_char = *from;if ((maptype ==SFM_MAP_UNI_RSVD ) &&convert_sfm_char (src_char, target))return len;
@@ -168,10 +136,23 @@ cifs_mapchar(char *target, const __u16 src_char, const struct nls_table *cp,/* if character not one of seven in special remap set */ len = cp->uni2char (src_char, target,NLS_MAX_CHARSET_SIZE ); -if (len <=0 ) { - *target ='?' ; - len =1 ; - } +if (len <=0 ) + goto surrogate_pair; + +return len; + +surrogate_pair: +/* convert SURROGATE_PAIR and IVS */ +if (strcmp (cp->charset,"utf8" )) + goto unknown; + len =utf16s_to_utf8s (from,3 ,UTF16_LITTLE_ENDIAN , target,6 ); +if (len <=0 ) + goto unknown; +return len; + +unknown: + *target ='?' ; + len =1 ;return len; }
@@ -206,7 +187,7 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,int nullsize =nls_nullsize (codepage);int fromwords = fromlen /2 ;char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; +__u16 ftmp[3 ];/* ftmp[3] = 3array x 2bytes = 6bytes UTF-16 */ /* * because the chars can be of varying widths, we need to take care
@@ -217,9 +198,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, safelen = tolen - (NLS_MAX_CHARSET_SIZE + nullsize);for (i =0 ; i < fromwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 )break ; +if (i +1 < fromwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < fromwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ;/* * check to see if converting this character might make the
@@ -234,6 +223,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,/* put converted char into 'to' buffer */ charlen =cifs_mapchar (&to[outlen], ftmp, codepage, map_type); outlen += charlen; + +/* charlen (=bytes of UTF-8 for 1 character) +* 4bytes UTF-8(surrogate pair) is charlen=4 +* (4bytes UTF-16 code) +* 7-8bytes UTF-8(IVS) is charlen=3+4 or 4+4 +* (2 UTF-8 pairs divided to 2 UTF-16 pairs) */ +if (charlen ==4 ) + i++; +else if (charlen >=5 ) +/* 5-6bytes UTF-8 */ + i +=2 ; }/* properly null-terminate string */
@@ -296,6 +296,46 @@ success: }/* +* cifs_utf16_bytes - how long will a string be after conversion? +* @utf16 - pointer to input string +* @maxbytes - don't go past this many bytes of input string +* @codepage - destination codepage +* +* Walk a utf16le string and return the number of bytes that the string will +* be after being converted to the given charset, not including any null +* termination required. Don't walk past maxbytes in the source buffer. +*/ +int +cifs_utf16_bytes (const __le16 *from,int maxbytes, +const struct nls_table *codepage) +{ +int i; +int charlen, outlen =0 ; +int maxwords = maxbytes /2 ; +char tmp[NLS_MAX_CHARSET_SIZE ]; +__u16 ftmp[3 ]; + +for (i =0 ; i < maxwords; i++) { + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 ) +break ; +if (i +1 < maxwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < maxwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ; + + charlen =cifs_mapchar (tmp, ftmp, codepage,NO_MAP_UNI_RSVD ); + outlen += charlen; + } + +return outlen; +} + +/* * cifs_strndup_from_utf16 - copy a string from wire format to the local * codepage * @src - source string
@@ -409,10 +449,15 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,char src_char;__le16 dst_char;wchar_t tmp; +wchar_t *wchar_to;/* UTF-16 */ +int ret; +unicode_t u;if (map_chars ==NO_MAP_UNI_RSVD )return cifs_strtoUTF16 (target, source,PATH_MAX , cp); + wchar_to =kzalloc (6 ,GFP_KERNEL ); +for (i =0 ; i < srclen; j++) { src_char = source[i]; charlen =1 ;
@@ -441,11 +486,55 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,* if no match, use question mark, which at least in * some cases serves as wild card */ -if (charlen <1 ) { - dst_char =cpu_to_le16 (0x003f ); - charlen =1 ; +if (charlen >0 ) + goto ctoUTF16; + +/* convert SURROGATE_PAIR */ +if (strcmp (cp->charset,"utf8" ) || !wchar_to) + goto unknown; +if (*(source + i) &0x80 ) { + charlen =utf8_to_utf32 (source + i,6 , &u); +if (charlen <0 ) + goto unknown; + }else + goto unknown; + ret =utf8s_to_utf16s (source + i, charlen, +UTF16_LITTLE_ENDIAN , + wchar_to,6 ); +if (ret <0 ) + goto unknown; + + i += charlen; + dst_char =cpu_to_le16 (*wchar_to); +if (charlen <=3 ) +/* 1-3bytes UTF-8 to 2bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); +else if (charlen ==4 ) { +/* 4bytes UTF-8(surrogate pair) to 4bytes UTF-16 +* 7-8bytes UTF-8(IVS) divided to 2 UTF-16 +* (charlen=3+4 or 4+4) */ +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +1 )); + j++; +put_unaligned (dst_char, &target[j]); + }else if (charlen >=5 ) { +/* 5-6bytes UTF-8 to 6bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +1 )); + j++; +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +2 )); + j++; +put_unaligned (dst_char, &target[j]); } +continue ; + +unknown: + dst_char =cpu_to_le16 (0x003f ); + charlen =1 ; } + +ctoUTF16:/* * character may take more than one byte in the source string, * but will take exactly two bytes in the target string
@@ -456,6 +545,7 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen, ctoUTF16_out:put_unaligned (0 , &target[j]);/* Null terminate target unicode string */ +kfree (wchar_to);return j; }
Signature · 1 line
1.7.1
From 4b0db586e09916a88ac2ac7a1714e52c28781ff6 Mon Sep 17 00:00:00 2001
From: Nakajima Akira <[email protected]>
Date: Thu, 9 Apr 2015 17:27:39 +0900
Subject: [PATCH] Fix to convert SURROGATE PAIR
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)
[Steps to Reproduce for bug]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .
Fix the bug about these functions do not consider about surrogate pair (and IVS). cifs_utf16_bytes()cifs_mapchar()cifs_from_utf16()cifsConvertToUTF16()
--- fs/cifs/cifs_unicode.c | 182 ++++++++++++++++++++++++++++++++++++------------ 1 files changed, 136 insertions(+), 46 deletions(-)
fs/cifs/cifs_unicode.c +136 -46
252 lines not on this page · raw message
Signature · 1 line
1.7.1
Steve French
Appliedmerged into cifs-2.6.git for-next
Quoted 391 lines · Nakajima Akira
Quoted 34 lines · Steve French
Note build warnings (when running normal sparse checks during compile) looks like need to resolve endian errors so that this is sure to work in big endian, not just little endian boxes
e.g.
make C=1 M=fs/cifs modules CF=-D__CHECK_ENDIAN__
CHECK
fs/cifs/cifs_unicode.cfs/cifs/cifs_unicode.c:508:34: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:508:34: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:508:34: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:517:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:517:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:517:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:523:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:523:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:523:42: got unsigned short [unsigned]
[short] [usertype] <noident>fs/cifs/cifs_unicode.c:526:42: warning: incorrect type in assignment (different base types)fs/cifs/cifs_unicode.c:526:42: expected restricted__le16
[assigned] [usertype]dst_charfs/cifs/cifs_unicode.c:526:42: got unsigned short [unsigned]
[short] [usertype] <noident>Quoted 36 lines
Quoted 34 lines · Nakajima Akira
Garbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)This causes
-- inode number duplication (when ?? and ?? exist) . -- can't remove file, directory.
-- can't operate under directory.
-- auto-remount with noserverino.[Steps to Reproduce]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .[BUG description]
Some functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()
-- return incorrect length, because of not considering about surrogate pair.
This causes problem about SymbolicLink with surrogate pair filename.
cifs_mapchar()
-- not considering about surrogate pair.
cifs_from_utf16()
-- not convert surrogate pair from SMB response (from UTF-16 to UTF-8)
, then ls shows garbled characters.
cifsConvertToUTF16()
-- not convert surrogate pair when mapchars(SFM/SFU).I fixed endian bug by using function
cpu_to_le16(). But I don't have Big Endian machine.
Could someone check on Big Endian machine?I'm trying pearpc(PowerPC Emulator), but pearpc doesn't work.
(pearpc need yaboot?)[Procedure]
client# touchprintf '\xf0\x9d\x9f\xa3'
client# touchprintf '\xf0\x9d\x9f\xa4'
client# ls -li--Before patch
You see same inode number, same filename(=?? and ??) .--After patch
You see correct inode numbers, correct filenames(look like 1 and 2)Quoted 1 line
From
4b0db586e09916a88ac2ac7a1714e52c28781ff6Mon Sep 17 00:00:00 2001From: Nakajima Akira <nakajima.akira-o7dWnD6vFTHqq2nvvmkE/[email protected]>
Date: Thu, 9 Apr 2015 17:27:39 +0900
Subject: [PATCH] Fix to convert SURROGATE PAIRGarbled characters happen by using surrogate pair for filename.
(replace each 1 character to ??)[Steps to Reproduce for bug]
client# touch $(echo -e '\xf0\x9d\x9f\xa3') client# touch $(echo -e '\xf0\x9d\x9f\xa4') client# ls -li
You see same inode number, same filename(=?? and ??) .Fix the bug about these functions do not consider about surrogate pair (and IVS).
cifs_utf16_bytes()cifs_mapchar()cifs_from_utf16()cifsConvertToUTF16()--- fs/cifs/cifs_unicode.c | 182 ++++++++++++++++++++++++++++++++++++------------ 1 files changed, 136 insertions(+), 46 deletions(-)fs/cifs/cifs_unicode.c +136
-47@@ -27,41 +27,6 @@#include "cifsglob.h" #include "cifs_debug.h" -/* -* cifs_utf16_bytes - how long will a string be after conversion? -* @utf16 - pointer to input string -* @maxbytes - don't go past this many bytes of input string -* @codepage - destination codepage -* -* Walk a utf16le string and return the number of bytes that the string will -* be after being converted to the given charset, not including any null -* termination required. Don't walk past maxbytes in the source buffer. -*/ -int -cifs_utf16_bytes (const __le16 *from,int maxbytes, -const struct nls_table *codepage) -{ -int i; -int charlen, outlen =0 ; -int maxwords = maxbytes /2 ; -char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; - -for (i =0 ; i < maxwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) -break ; - - charlen = codepage->uni2char (ftmp, tmp,NLS_MAX_CHARSET_SIZE ); -if (charlen >0 ) - outlen += charlen; -else - outlen++; - } - -return outlen; -} -int cifs_remap (struct cifs_sb_info *cifs_sb) {int map_type;@@ -155,10 +120,13 @@ convert_sfm_char(const __u16 src_char, char *target)* enough to hold the result of the conversion (at least NLS_MAX_CHARSET_SIZE). */ static int -cifs_mapchar (char *target,const __u16 src_char,const struct nls_table *cp, +cifs_mapchar (char *target,const __u16 *from,const struct nls_table *cp,int maptype) {int len =1 ; +__u16 src_char; + + src_char = *from;if ((maptype ==SFM_MAP_UNI_RSVD ) &&convert_sfm_char (src_char, target))return len;@@ -168,10 +136,23 @@ cifs_mapchar(char *target, const __u16 src_char, const struct nls_table *cp,/* if character not one of seven in special remap set */ len = cp->uni2char (src_char, target,NLS_MAX_CHARSET_SIZE ); -if (len <=0 ) { - *target ='?' ; - len =1 ; - } +if (len <=0 ) + goto surrogate_pair; + +return len; + +surrogate_pair: +/* convert SURROGATE_PAIR and IVS */ +if (strcmp (cp->charset,"utf8" )) + goto unknown; + len =utf16s_to_utf8s (from,3 ,UTF16_LITTLE_ENDIAN , target,6 ); +if (len <=0 ) + goto unknown; +return len; + +unknown: + *target ='?' ; + len =1 ;return len; }@@ -206,7 +187,7 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,int nullsize =nls_nullsize (codepage);int fromwords = fromlen /2 ;char tmp[NLS_MAX_CHARSET_SIZE ]; -__u16 ftmp; +__u16 ftmp[3 ];/* ftmp[3] = 3array x 2bytes = 6bytes UTF-16 */ /* * because the chars can be of varying widths, we need to take care @@ -217,9 +198,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen, safelen = tolen - (NLS_MAX_CHARSET_SIZE + nullsize);for (i =0 ; i < fromwords; i++) { - ftmp =get_unaligned_le16 (&from[i]); -if (ftmp ==0 ) + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 )break ; +if (i +1 < fromwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < fromwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ;/* * check to see if converting this character might make the @@ -234,6 +223,17 @@ cifs_from_utf16(char *to, const __le16 *from, int tolen, int fromlen,/* put converted char into 'to' buffer */ charlen =cifs_mapchar (&to[outlen], ftmp, codepage, map_type); outlen += charlen; + +/* charlen (=bytes of UTF-8 for 1 character) +* 4bytes UTF-8(surrogate pair) is charlen=4 +* (4bytes UTF-16 code) +* 7-8bytes UTF-8(IVS) is charlen=3+4 or 4+4 +* (2 UTF-8 pairs divided to 2 UTF-16 pairs) */ +if (charlen ==4 ) + i++; +else if (charlen >=5 ) +/* 5-6bytes UTF-8 */ + i +=2 ; }/* properly null-terminate string */ @@ -296,6 +296,46 @@ success: }/* +* cifs_utf16_bytes - how long will a string be after conversion? +* @utf16 - pointer to input string +* @maxbytes - don't go past this many bytes of input string +* @codepage - destination codepage +* +* Walk a utf16le string and return the number of bytes that the string will +* be after being converted to the given charset, not including any null +* termination required. Don't walk past maxbytes in the source buffer. +*/ +int +cifs_utf16_bytes (const __le16 *from,int maxbytes, +const struct nls_table *codepage) +{ +int i; +int charlen, outlen =0 ; +int maxwords = maxbytes /2 ; +char tmp[NLS_MAX_CHARSET_SIZE ]; +__u16 ftmp[3 ]; + +for (i =0 ; i < maxwords; i++) { + ftmp[0 ] =get_unaligned_le16 (&from[i]); +if (ftmp[0 ] ==0 ) +break ; +if (i +1 < maxwords) + ftmp[1 ] =get_unaligned_le16 (&from[i +1 ]); +else + ftmp[1 ] =0 ; +if (i +2 < maxwords) + ftmp[2 ] =get_unaligned_le16 (&from[i +2 ]); +else + ftmp[2 ] =0 ; + + charlen =cifs_mapchar (tmp, ftmp, codepage,NO_MAP_UNI_RSVD ); + outlen += charlen; + } + +return outlen; +} + +/* * cifs_strndup_from_utf16 - copy a string from wire format to the local * codepage * @src - source string @@ -409,10 +449,15 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,char src_char;__le16 dst_char;wchar_t tmp; +wchar_t *wchar_to;/* UTF-16 */ +int ret; +unicode_t u;if (map_chars ==NO_MAP_UNI_RSVD )return cifs_strtoUTF16 (target, source,PATH_MAX , cp); + wchar_to =kzalloc (6 ,GFP_KERNEL ); +for (i =0 ; i < srclen; j++) { src_char = source[i]; charlen =1 ;@@ -441,11 +486,55 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen,* if no match, use question mark, which at least in * some cases serves as wild card */ -if (charlen <1 ) { - dst_char =cpu_to_le16 (0x003f ); - charlen =1 ; +if (charlen >0 ) + goto ctoUTF16; + +/* convert SURROGATE_PAIR */ +if (strcmp (cp->charset,"utf8" ) || !wchar_to) + goto unknown; +if (*(source + i) &0x80 ) { + charlen =utf8_to_utf32 (source + i,6 , &u); +if (charlen <0 ) + goto unknown; + }else + goto unknown; + ret =utf8s_to_utf16s (source + i, charlen, +UTF16_LITTLE_ENDIAN , + wchar_to,6 ); +if (ret <0 ) + goto unknown; + + i += charlen; + dst_char =cpu_to_le16 (*wchar_to); +if (charlen <=3 ) +/* 1-3bytes UTF-8 to 2bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); +else if (charlen ==4 ) { +/* 4bytes UTF-8(surrogate pair) to 4bytes UTF-16 +* 7-8bytes UTF-8(IVS) divided to 2 UTF-16 +* (charlen=3+4 or 4+4) */ +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +1 )); + j++; +put_unaligned (dst_char, &target[j]); + }else if (charlen >=5 ) { +/* 5-6bytes UTF-8 to 6bytes UTF-16 */ +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +1 )); + j++; +put_unaligned (dst_char, &target[j]); + dst_char =cpu_to_le16 (*(wchar_to +2 )); + j++; +put_unaligned (dst_char, &target[j]); } +continue ; + +unknown: + dst_char =cpu_to_le16 (0x003f ); + charlen =1 ; } + +ctoUTF16:/* * character may take more than one byte in the source string, * but will take exactly two bytes in the target string @@ -456,6 +545,7 @@ cifsConvertToUTF16(__le16 *target, const char *source, int srclen, ctoUTF16_out:put_unaligned (0 , &target[j]);/* Null terminate target unicode string */ +kfree (wchar_to);return j; } --1.7.1
Signature · 3 lines
Thanks, Steve