${#var} returns length in bytes, not characters

POSIX:
http://pubs.opengroup.org/onlinepubs/9699919799/utilities/V3_chap02.html#tag_18_06_02

dash does not expand the length in characters; it expands the length in bytes instead. That is invalid for locales that include multi-byte characters, such as the now ubiquitous UTF-8 set.

Test case:

$ locale
LANG="nl_NL.UTF-8"
LC_COLLATE="nl_NL.UTF-8"
LC_CTYPE="nl_NL.UTF-8"
LC_MESSAGES="nl_NL.UTF-8"
LC_MONETARY="nl_NL.UTF-8"
LC_NUMERIC="nl_NL.UTF-8"
LC_TIME="nl_NL.UTF-8"
LC_ALL=
$ word='bètatest'	# length: 8
$ echo ${#word}
9

Expected output: 8
Got output: 9

(bash, ksh93, mksh, and zsh all do this correctly.)

- Martijn

2015-06-03 13:29:33 +0200, Martijn Dekker:

[...]

See also:

http://thread.gmane.org/gmane.comp.standards.posix.austin.general/9972/focus=10040

For a few UTF-8 related variations in behaviour between shells (including this one), though many of them are about "unspecified behaviour".

The script mentioned there is also attached here.

Stephane

euro=$(printf '\342\202\254')
o342=$(printf '\342')
o202=$(printf '\202')
o254=$(printf '\254')

test_shell=$1

n=0
case $test_shell in
zsh) emulate sh;;
zsh5) setopt shwordsplit;;
[lm]ksh) set -o utf8-mode;;
esac

if [ -n "$test_shell" ]; then
printf '%8s:' "$test_shell"

  testing() {
    n=$(($n + 1))
    test
    ret=$?
    [ "$ret" -eq "$na" ] && ret=-
    printf ' %2s' "$ret"
    [ "$ret" = "$1" ] && printf '\342\203\235'
  }
else
  testing() {
    n=$(($n + 1))
    printf '%2d: %s (expected: %d)\n' "$n" "$2" "$1"

}
fi

na=99
if_accept_invalid() { [ "$o254" ] || exit "$na"; } if_printf_builtin() {
case $(type printf 2> /dev/null) in

    *builtin*);;
    (*) exit "$na"

esac
}

################################################################
test() (
exit "${#euro}"
)
testing 1 '${#utf8-character}'
################################################################
test() (
if_accept_invalid
exit "${#o254}"
)
testing 1 '${#single byte, invalid char}'
################################################################
test() (
if_accept_invalid
t=$o342$o202
exit "${#t}"
)
testing 2 '${truncated character, 2 bytes}'
################################################################
test() (
if_accept_invalid
case $euro in

    *"$o254"*) true;;
    *) false

esac
)
testing 0 '$char contains byte component'
################################################################
test() (
if_accept_invalid
t=+$euro-
[ "${t##*"$o254"}" = - ]
)
testing 0 '${##} matching with byte components of $mbchar'
################################################################
test() (
if_accept_invalid
t=+$o254$euro-
[ "${t##*"$euro"}" = - ]
)
testing 0 '${##} works in invalid strings'
################################################################
test() (
if_accept_invalid
IFS=$o254
t=+$euro+
set -- $t
exit "$#"
)
testing 2 'byte component found in $mbchar by IFS'
################################################################
test() (
if_accept_invalid
IFS=$o342$o202
t=+$euro+
set -- $t
exit "$#"
)
testing 3 'byte component found in $mbchar by IFS'
################################################################
test() (
if_accept_invalid
IFS=$o254
t=+$o254+
set -- $t
exit "$#"
)
testing 2 'IFS works with bytes'
################################################################
test() (
IFS=$euro
t=+$euro+
set -- $t
exit "$#"
)
testing 2 'IFS works with mbchars'
################################################################
test() (
if_accept_invalid
IFS=$euro
t=+$o254$o342$euro+
set -- $t
exit "$#"
)
testing 2 'IFS works with mbchars in invalid strings'
################################################################
test() (
if_accept_invalid
IFS=$o342$o202
set / /
[ "$*" = "/$o342/" ]
)
testing 0 'Joining $* with byte forming invalid char'
################################################################
test() (
IFS=$euro
set / /
[ "$*" = "/$euro/" ]
)
testing 0 'Joining $* with mbchar'
################################################################
test() (
if_printf_builtin
exit "$(printf '%4s' "$euro" | wc -c | tr -cd 0-9)"
)
testing 6 'byte length of 4-padded euro'
################################################################
test() (
if_accept_invalid
if_printf_builtin
exit "$(printf '%4s' "$o342$o202" | wc -c | tr -cd 0-9)"
)
testing 4 'byte length of 4-padded truncated mbchar'
################################################################
test() (
case $euro in

    ?) true;;
    (*) false

esac
)
testing 0 '? matches mbchar'
################################################################
test() (
if_accept_invalid
case $o342$o202 in

    ??) true;;
    (*) false

esac
)
testing 0 '?? matches 2-byte truncated mbchar'
################################################################

echo
if [ -z "$test_shell" ]; then
printf '%8s:' tests
i=1; while [ "$i" -le "$n" ]; do

    printf ' %2d' "$i"
    i=$(($i + 1))
  done
  printf '\n\n'
  for test_shell in dash zsh yash bash lksh mksh ksh93 zsh5 posh; do
    "$test_shell" "$0" "$test_shell"

done
fi

dash doesn't handle locales at all.

Cheers,

Email: Herbert Xu <[email protected]>
Home Page: http://gondor.apana.org.au/~herbert/
PGP Key: http://gondor.apana.org.au/~herbert/pubkey.txt