First Try of handling invalid UTF-8

.github/workflows/release.yaml

@@ -83,12 +83,12 @@ jobs:
     - name: Install Shells (linux only)
       if: ${{ matrix.os=='linux' }}
       run: |
-        sudo apt-get install -y -qq ksh zsh yash
+        sudo apt-get install -y -qq ksh zsh yash x11-utils
         sudo apt-get install -y mksh || true
 
     - name: Install Shells & Shellcheck (macos only)
       if: ${{ matrix.os=='macos' }}
-      run: brew install bash ksh93 shellcheck
+      run: brew install bash ksh93 shellcheck luit
 
     - name: Install Shellcheck (windows only)
       if: ${{ matrix.os=='windows' }}

.github/workflows/snapshot-release.yaml

@@ -77,12 +77,12 @@ jobs:
     - name: Install Shells (linux only)
       if: ${{ matrix.os=='linux' }}
       run: |
-        sudo apt-get install -y -qq ksh zsh yash
+        sudo apt-get install -y -qq ksh zsh yash x11-utils
         sudo apt-get install -y mksh || true
 
     - name: Install Shells & Shellcheck (macos only)
       if: ${{ matrix.os=='macos' }}
-      run: brew install bash ksh93 shellcheck
+      run: brew install bash ksh93 shellcheck luit
 
     - name: Install Shellcheck (windows only)
       if: ${{ matrix.os=='windows' }}

.github/workflows/verify.yaml

@@ -38,12 +38,12 @@ jobs:
     - name: Install Shells (linux only)
       if: ${{ matrix.os=='linux' }}
       run: |
-        sudo apt-get install -y -qq ksh zsh yash
+        sudo apt-get install -y -qq ksh zsh yash x11-utils
         sudo apt-get install -y mksh || true
 
     - name: Install Shells & Shellcheck (macos only)
       if: ${{ matrix.os=='macos' }}
-      run: brew install bash ksh93 shellcheck
+      run: brew install bash ksh93 shellcheck luit
 
     - name: Install Shellcheck (windows only)
       if: ${{ matrix.os=='windows' }}

Cargo.toml

@@ -17,6 +17,8 @@ strip = true        # Strip symbols from binary
 
 [dependencies]
 clap = { version = "4.3.0", features = ["derive"] }
+clap_lex = "0.6.0"
+stfu8 = "0.2.6"
 
 [dev-dependencies]
 assert_cmd = "2.0.12"

script-test/_test.shinc

@@ -52,6 +52,12 @@ start_test()
     echo "Testing:     $(command -v parseargs)"
 }
 
+skip_test()
+{
+        printf '\033[01;33mTEST SKIPPED - %s\033[0m\n' "$*"
+        exit 0
+}
+
 end_test()
 {
     rc=0
@@ -99,6 +105,7 @@ test_pa()
         ok "parseargs $args"
     else
         failed "parseargs $args"
+        parseargs "$@"
     fi
 
     return $rc

script-test/run.sh

@@ -13,11 +13,13 @@ echo "Test Environment: $(uname -a)"
 case "$(uname -s | tr '[:upper:]' '[:lower:]')" in
     *cygwin*)
         IS_CYGWIN=TRUE
-        export IS_CYGWIN
+        IS_WINDOWS=TRUE
+        export IS_CYGWIN IS_WINDOWS
         ;;
     *msys* | *mingw*)
         IS_MSYS=TRUE
-        export IS_MSYS
+        IS_WINDOWS=TRUE
+        export IS_MSYS IS_WINDOWS
         ;;
 esac
 

script-test/test-invalid-utf8.sh

@@ -15,23 +15,62 @@ script_name="$(basename "$0")"
 start_test
 
 
-if [ -n "$IS_MSYS" ] || [ -n "$IS_CYGWIN" ]; then
-    echo "Skipped on Windows"
+if [ -n "$IS_WINDOWS" ]; then
+    skip_test "on Windows"
 elif [ "$TEST_SHELL" = "yash" ]; then
-    echo "Skipped with shell Yash, as it can't handle invalid UTF-8"
+    skip_test "shell Yash, as it can't handle invalid UTF-8"
 else
+    inv_1="$(printf '\303\050')"
+    inv_2="$(printf '\240\241')"
+    inv_3="$(printf '\342\050\241')"
+    inv_4="$(printf '\342\202\050')"
+    inv_5="$(printf '\360\050\214\274')"
+    inv_6="$(printf '\360\220\050\274')"
+    inv_7="$(printf '\360\050\214\050')"
 
-    # The following should create some valid code on day
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\303\050')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\240\241')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\342\050\241')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\342\202\050')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\360\050\214\274')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\360\220\050\274')"
-    test_pa_code 'exit 1' -o "n:name=name" -- "$(printf '\360\050\214\050')"
+    test_pa 'test "$1" = "$inv_1"' -o "n:name=name" -- "$inv_1"
+    test_pa 'test "$1" = "$inv_2"' -o "n:name=name" -- "$inv_2"
+    test_pa 'test "$1" = "$inv_3"' -o "n:name=name" -- "$inv_3"
+    test_pa 'test "$1" = "$inv_4"' -o "n:name=name" -- "$inv_4"
+    test_pa 'test "$1" = "$inv_5"' -o "n:name=name" -- "$inv_5"
+    test_pa 'test "$1" = "$inv_6"' -o "n:name=name" -- "$inv_6"
+    test_pa 'test "$1" = "$inv_7"' -o "n:name=name" -- "$inv_7"
 
+    test_pa 'test "$name" = "$inv_1"' -o "n:name=name" -- -n "$inv_1"
+    test_pa 'test "$name" = "$inv_2"' -o "n:name=name" -- -n "$inv_2"
+    test_pa 'test "$name" = "$inv_3"' -o "n:name=name" -- -n "$inv_3"
+    test_pa 'test "$name" = "$inv_4"' -o "n:name=name" -- -n "$inv_4"
+    test_pa 'test "$name" = "$inv_5"' -o "n:name=name" -- -n "$inv_5"
+    test_pa 'test "$name" = "$inv_6"' -o "n:name=name" -- -n "$inv_6"
+    test_pa 'test "$name" = "$inv_7"' -o "n:name=name" -- -n "$inv_7"
 
-    # The following will always do an error exit
+    test_pa 'test "$name" = "$inv_1"' -o "n:name=name" -- -n"$inv_1"
+    test_pa 'test "$name" = "$inv_2"' -o "n:name=name" -- -n"$inv_2"
+    test_pa 'test "$name" = "$inv_3"' -o "n:name=name" -- -n"$inv_3"
+    test_pa 'test "$name" = "$inv_4"' -o "n:name=name" -- -n"$inv_4"
+    test_pa 'test "$name" = "$inv_5"' -o "n:name=name" -- -n"$inv_5"
+    test_pa 'test "$name" = "$inv_6"' -o "n:name=name" -- -n"$inv_6"
+    test_pa 'test "$name" = "$inv_7"' -o "n:name=name" -- -n"$inv_7"
+
+
+    test_pa 'test "$name" = "$inv_1"' -o "n:name=name" -- --name "$inv_1"
+    test_pa 'test "$name" = "$inv_2"' -o "n:name=name" -- --name "$inv_2"
+    test_pa 'test "$name" = "$inv_3"' -o "n:name=name" -- --name "$inv_3"
+    test_pa 'test "$name" = "$inv_4"' -o "n:name=name" -- --name "$inv_4"
+    test_pa 'test "$name" = "$inv_5"' -o "n:name=name" -- --name "$inv_5"
+    test_pa 'test "$name" = "$inv_6"' -o "n:name=name" -- --name "$inv_6"
+    test_pa 'test "$name" = "$inv_7"' -o "n:name=name" -- --name "$inv_7"
+
+    test_pa 'test "$name" = "$inv_1"' -o "n:name=name" -- --name="$inv_1"
+    test_pa 'test "$name" = "$inv_2"' -o "n:name=name" -- --name="$inv_2"
+    test_pa 'test "$name" = "$inv_3"' -o "n:name=name" -- --name="$inv_3"
+    test_pa 'test "$name" = "$inv_4"' -o "n:name=name" -- --name="$inv_4"
+    test_pa 'test "$name" = "$inv_5"' -o "n:name=name" -- --name="$inv_5"
+    test_pa 'test "$name" = "$inv_6"' -o "n:name=name" -- --name="$inv_6"
+    test_pa 'test "$name" = "$inv_7"' -o "n:name=name" -- --name="$inv_7"
+
+
+    # The following will always result in an error exit
     # The parseargs arguments must always be valid UTF-8.
     test_pa_code 'exit 1' -n "X$(printf '\303\050')Y" -o "n:name=name" --
     test_pa_code 'exit 1' -a "X$(printf '\303\050')Y" -o "n:name=name" --

script-test/test-iso8859-1-charset.sh

@@ -0,0 +1,68 @@
+#!/bin/sh
+#
+# Test parseargs with single byte
+#
+# shellcheck disable=SC2016
+
+script_dir="$(cd "$(dirname "$0")" && pwd)" || exit 1
+script_name="$(basename "$0")"
+script_file="$script_dir/$script_name"
+
+. "$script_dir/_test.shinc"
+
+
+check_single_byte()
+{
+    # print the bytes for the UTF-8 Euro sign and check whether this results in
+    # a single character
+    x="$(printf  '\342\202\254' | sed "s/./X/g")"
+    if [ ${#x} = 1 ]; then
+        return 1
+    else
+        return 0
+    fi
+}
+
+if ! check_single_byte; then
+    # set ISO-8859-1 if available
+    SB_LOCALE=$(locale -a | grep 'iso88591$' | head -n1)
+    if [ -n "$SB_LOCALE" ]; then
+        echo "Setting LC_ALL to $SB_LOCALE"
+        export LC_ALL="$SB_LOCALE"
+    fi
+
+    # If still no single byte char set -> exit
+    if ! check_single_byte; then
+        echo "No ISO-8859-1 character encoding available ... skipping"
+        exit 0
+    fi
+    luit -encoding 'ISO 8859-1' "$script_file"
+    exit $?
+fi
+
+start_test
+
+if [ -n "$IS_WINDOWS" ]; then
+    skip_test "on Windows"
+elif [ "$TEST_SHELL" = "yash" ]; then
+    skip_test "shell Yash, can't handle this"
+fi
+
+test_pa 'test "${#1}" = 3' -o "n:name=name" -- "$(printf '\342\202\254')"
+
+test_pa 'test "$1" = "�"' -o "n:name=name" -- �
+test_pa 'test "$name" = "�"' -o "n:name=name" -- -n �
+test_pa 'test "$name" = "�"' -o "n:name=name" -- -n�
+test_pa 'test "$name" = "�"' -o "n:name=name" -- --name �
+test_pa 'test "$name" = "�"' -o "n:name=name" -- --name=�
+
+test_pa 'test "$1" = "�'\''�"' -o "n:name=name" -- �\'�
+test_pa 'test "$name" = "�'\''�"' -o "n:name=name" -- -n �\'�
+test_pa 'test "$name" = "�'\''�"' -o "n:name=name" -- -n�\'�
+test_pa 'test "$name" = "�'\''�"' -o "n:name=name" -- --name �\'�
+test_pa 'test "$name" = "�'\''�"' -o "n:name=name" -- --name=�\'�
+
+end_test
+
+# vim:fileencoding=latin1
+