Commit 8718d948 by Alexander Popov

Fix the bug in the functional tests

`man 1 sh` says about '-e':
```
The shell does not exit if the command that fails is part of the command list
immediately following a while or until keyword, part of the test following
the if or elif reserved words, part of any command executed in a && or || list
except the command following the final && or ||, any command in a pipeline
but the last, or if the command's return value is being inverted with !.

That's why testing error handling in the functional tests didn't check
the exit status at all :(

Let's fix that.
Example before the fix:
! coverage run -a --branch bin/kconfig-hardened-check -l /proc/cmdline
Example after the fix:
coverage run -a --branch bin/kconfig-hardened-check -l /proc/cmdline && exit 1
parent 88c0698e
...@@ -59,62 +59,62 @@ echo "\n>>>>> have checked $COUNT kconfigs <<<<<" ...@@ -59,62 +59,62 @@ echo "\n>>>>> have checked $COUNT kconfigs <<<<<"
echo "Collect coverage for error handling" echo "Collect coverage for error handling"
echo ">>>>> lonely -l <<<<<" echo ">>>>> lonely -l <<<<<"
! coverage run -a --branch bin/kconfig-hardened-check -l /proc/cmdline coverage run -a --branch bin/kconfig-hardened-check -l /proc/cmdline && exit 1
echo ">>>>> wrong modes for -p <<<<<" echo ">>>>> wrong modes for -p <<<<<"
! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_ok coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_ok && exit 1
! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_fail coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -m show_fail && exit 1
echo ">>>>> -p and -c together <<<<<" echo ">>>>> -p and -c together <<<<<"
! coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -c kconfig_hardened_check/config_files/distros/fedora_34.config coverage run -a --branch bin/kconfig-hardened-check -p X86_64 -c kconfig_hardened_check/config_files/distros/fedora_34.config && exit 1
echo ">>>>> wrong mode for -g <<<<<" echo ">>>>> wrong mode for -g <<<<<"
! coverage run -a --branch bin/kconfig-hardened-check -g X86_64 -m show_ok coverage run -a --branch bin/kconfig-hardened-check -g X86_64 -m show_ok && exit 1
echo ">>>>> -g and -c together <<<<<" echo ">>>>> -g and -c together <<<<<"
! coverage run -a --branch bin/kconfig-hardened-check -g X86_64 -c kconfig_hardened_check/config_files/distros/fedora_34.config coverage run -a --branch bin/kconfig-hardened-check -g X86_64 -c kconfig_hardened_check/config_files/distros/fedora_34.config && exit 1
cp kconfig_hardened_check/config_files/distros/fedora_34.config ./test.config cp kconfig_hardened_check/config_files/distros/fedora_34.config ./test.config
echo ">>>>> no kernel version <<<<<" echo ">>>>> no kernel version <<<<<"
sed '3d' test.config > error.config sed '3d' test.config > error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> strange kernel version string <<<<<" echo ">>>>> strange kernel version string <<<<<"
sed '3 s/5./version 5./' test.config > error.config sed '3 s/5./version 5./' test.config > error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> no arch <<<<<" echo ">>>>> no arch <<<<<"
sed '305d' test.config > error.config sed '305d' test.config > error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> more than one arch <<<<<" echo ">>>>> more than one arch <<<<<"
cp test.config error.config cp test.config error.config
echo 'CONFIG_ARM64=y' >> error.config echo 'CONFIG_ARM64=y' >> error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> invalid disabled kconfig option <<<<<" echo ">>>>> invalid disabled kconfig option <<<<<"
sed '28 s/is not set/is not set yet/' test.config > error.config sed '28 s/is not set/is not set yet/' test.config > error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> invalid enabled kconfig option <<<<<" echo ">>>>> invalid enabled kconfig option <<<<<"
cp test.config error.config cp test.config error.config
echo 'CONFIG_FOO=is not set' >> error.config echo 'CONFIG_FOO=is not set' >> error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> one config option multiple times <<<<<" echo ">>>>> one config option multiple times <<<<<"
cp test.config error.config cp test.config error.config
echo 'CONFIG_BUG=y' >> error.config echo 'CONFIG_BUG=y' >> error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> invalid compiler versions <<<<<" echo ">>>>> invalid compiler versions <<<<<"
cp test.config error.config cp test.config error.config
sed '8 s/CONFIG_CLANG_VERSION=0/CONFIG_CLANG_VERSION=120000/' test.config > error.config sed '8 s/CONFIG_CLANG_VERSION=0/CONFIG_CLANG_VERSION=120000/' test.config > error.config
! coverage run -a --branch bin/kconfig-hardened-check -c error.config coverage run -a --branch bin/kconfig-hardened-check -c error.config && exit 1
echo ">>>>> multi-line cmdline file <<<<<" echo ">>>>> multi-line cmdline file <<<<<"
echo 'hey man 1' > cmdline echo 'hey man 1' > cmdline
echo 'hey man 2' >> cmdline echo 'hey man 2' >> cmdline
! coverage run -a --branch bin/kconfig-hardened-check -c test.config -l cmdline coverage run -a --branch bin/kconfig-hardened-check -c test.config -l cmdline && exit 1
echo "The end of the functional tests" echo "The end of the functional tests"
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment