Merge pull request #7 from fkie-cad/acceptance_tests

Acceptance tests

.gitignore

@@ -214,3 +214,5 @@ test/run_real_world_samples.sh
 .pydevproject
 
 src/cwe_checker.plugin
+
+.#*
\ No newline at end of file

CHANGES.md

@@ -4,6 +4,7 @@
 - Refactoring: Unification of cwe_checker function interface
 - Refactoring: Created utils module for JSON functionality
 - Added check for CWE 248: Uncaught Exception
+- Added automated test suite (run with make test)
 
 0.1 (2018-10-08)
 =====

Makefile

-.phony: all
+.PHONY: all clean test uninstall
 all:
 	cd src; bapbuild -r -Is checkers,utils -pkgs yojson,unix cwe_checker.plugin; bapbundle install cwe_checker.plugin; cd ..
 
 test:
-	bapbuild -r -Is src,src/checkers,src/utils,test -pkgs yojson,unix,alcotest test/test_cwe_checker.byte
-	./test/test_cwe_checker.byte
+	pytest -v
 
 clean:
 	bapbuild -clean

README.md

@@ -41,10 +41,12 @@ The three way is to build it using the provided `Makefile`. In this case you mus
 -  Ocaml 4.05.0
 -  Opam 2.0.2
 -  BAP 1.5 (and its dependencies)
--  yojson <= 1.4.1
--  alcotest <= 0.8.3
+-  yojson >= 1.4.1
+-  alcotest >= 0.8.3
 -  Sark for IDA Pro annotations
-Just run `make all` to compile and register the plugin with BAP.
+-  pytest >= 3.5.1
+
+Just run `make all` to compile and register the plugin with BAP. You can run the test suite via `make test`.
 ## How to use cwe_checker? ##
 The usage is straight forward: adjust the `config.json` (if needed) and call BAP with *cwe_checker* as a pass.
 ``` bash

test/acceptance/cwe_checker_testlib.py

+import subprocess
+
+def build_bap_cmd(filename, target, arch):
+        cmd = 'bap test/artificial_samples/build/cwe_%s_%s.out  --pass=callsites,cwe-checker --cwe-checker-partial=CWE%s --cwe-checker-config=src/config.json' % (filename, arch, target)
+        return cmd.split()
+
+def execute_and_check_occurence(filename, target, arch, string):
+    occurence = 0
+    bap_cmd = build_bap_cmd(filename, target, arch)
+    output = subprocess.check_output(bap_cmd)
+    for l in output.splitlines():
+        if string in l:
+            occurence += 1
+    return occurence

test/acceptance/test_cwe190.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe190(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '190'
+        self.string = b'Integer Overflow or Wraparound'
+
+    def test_cwe190_01_arm(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_x86(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_x64(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe190_01_mips(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe190_01_ppc(self):
+        expect_res = 3
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe215.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe215(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '215'
+        self.filename = '476'
+        self.string = b'Information Exposure Through Debug Information'
+
+    def test_cwe215_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    def test_cwe215_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.filename, self.target, 'mips', self.string)
+        assert res == expect_res    

test/acceptance/test_cwe243.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe243(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '243'
+        self.string = b'The program utilizes chroot without dropping privileges and/or changing the directory'
+
+    def test_cwe243_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe243_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe243_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_arm(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Investigate and fix this issue")
+    def test_cwe243_02_x86(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_x64(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'x64', self.string)
+        assert res == expect_res
+
+    def test_cwe243_02_ppc(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'ppc', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe476_02_mips(self):
+        expect_res = 0
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target + "_clean", self.target, 'mips', self.string)
+        assert res == expect_res

test/acceptance/test_cwe248.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe248(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '248'
+        self.string = b'Possibly Uncaught Exception'
+
+    def test_cwe248_01_arm(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Fix CPP compilation issue for x86")
+    def test_cwe248_01_x86(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe248_01_x64(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe248_01_mips(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe248_01_ppc(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe332.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe332(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '332'
+        self.string = b'Insufficient Entropy in PRNG'
+
+    def test_cwe332_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe332_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe332_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe367.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe367(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '367'
+        self.string = b'Time-of-check Time-of-use Race Condition'
+
+    def test_cwe367_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe367_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe367_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe426.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe426(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '426'
+        self.string = b'Untrusted Search Path'
+
+    def test_cwe426_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe426_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe426_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe426_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe426_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe457.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe457(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '457'
+        self.string = b'Use of Uninitialized Variable'
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe457_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe457_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe467.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe467(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '467'
+        self.string = b'Use of sizeof on a Pointer Type'
+
+    def test_cwe467_01_arm(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    @unittest.skip("FIXME")
+    def test_cwe467_01_x86(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe467_01_x64(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe467_01_mips(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe467_01_ppc(self):
+        expect_res = 2
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe476.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe476(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '476'
+        self.string = b'NULL Pointer Dereference'
+
+    def test_cwe476_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe476_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe476_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe476_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Fix issue in CWE476 implementation to support PPC")
+    def test_cwe476_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe676.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe676(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '676'
+        self.string = b'Use of Potentially Dangerous Function'
+
+    def test_cwe676_01_arm(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'arm', self.string)
+        assert res == expect_res
+
+    def test_cwe676_01_x86(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x86', self.string)
+        assert res == expect_res
+
+    def test_cwe676_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+
+    @unittest.skip("Depends on proper MIPS support in BAP")
+    def test_cwe676_01_mips(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'mips', self.string)
+        assert res == expect_res
+
+    def test_cwe676_01_ppc(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'ppc', self.string)
+        assert res == expect_res
+    

test/acceptance/test_cwe782.py

+import unittest
+import cwe_checker_testlib
+
+class TestCwe782(unittest.TestCase):
+
+    def setUp(self):
+        self.target = '782'
+        self.string = b'Exposed IOCTL with Insufficient Access Control'
+
+    def test_cwe782_01_x64(self):
+        expect_res = 1
+        res = cwe_checker_testlib.execute_and_check_occurence(self.target, self.target, 'x64', self.string)
+        assert res == expect_res
+

test/artificial_samples/cwe_676.c

+#include <stdio.h>
+#include <string.h>
+
+int main ()
+{
+  char str1[]="Hello World!";
+  char str2[40];
+  strcpy (str2,str1);
+  return 0;
+}

test/artificial_samples/makefile

@@ -88,6 +88,8 @@ endef
 define compile_all_cpp
 	$(shell mkdir -p "build")
 	$(call compile_x64_cpp,$(1))
+# building fails
+# (call compile_x86_cpp,$(1))
 	$(call compile_arm_cpp,$(1))
 	$(call compile_mips_cpp,$(1))
 	$(call compile_ppc_cpp,$(1))
@@ -107,6 +109,7 @@ all:
 	$(call compile_all,cwe_467)
 	$(call compile_all,cwe_476)
 	$(call compile_all,cwe_478)
+	$(call compile_all,cwe_676)
 	$(call compile_x64,cwe_782)
 	$(call compile_all,arrays)
 	$(call compile_all,memory_access)

test/run_artificial_samples.sh

-#!/bin/bash
-
-function printf_new() {
- v=$(printf "%-80s" "-")
- echo "${v// /-}"
-}
-
-function run_arch() {
-    echo
-    echo
-    echo "Running architecture:" $1
-    printf_new
-    echo "cwe_190_$1"
-    printf_new
-    bap artificial_samples/build/cwe_190_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_243_$1"
-    printf_new
-    bap artificial_samples/build/cwe_243_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_243_$1 (clean)"
-    printf_new
-    bap artificial_samples/build/cwe_243_clean_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_248_$1"
-    printf_new
-    bap artificial_samples/build/cwe_248_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_323_$1"
-    printf_new
-    bap artificial_samples/build/cwe_332_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json 
-    printf_new
-    echo "cwe_367_$1"
-    printf_new
-    bap artificial_samples/build/cwe_367_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json 
-    printf_new
-    echo "cwe_415_$1"
-    printf_new
-    bap artificial_samples/build/cwe_415_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json 
-    printf_new 
-    echo "cwe_426_$1"
-    printf_new
-    bap artificial_samples/build/cwe_426_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_457_$1"
-    printf_new
-    bap artificial_samples/build/cwe_457_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    echo "cwe_467_$1"
-    printf_new
-    bap artificial_samples/build/cwe_467_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json 
-    printf_new
-    echo "cwe_476_$1"
-    printf_new
-    bap artificial_samples/build/cwe_476_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    if [ $1 == "x64" ]; then
-    echo "cwe_782_$1"
-    printf_new
-    bap artificial_samples/build/cwe_782_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json
-    printf_new
-    fi
-    echo "c_constructs_$1"
-    printf_new
-    bap artificial_samples/build/c_constructs_$1.out  --pass=callsites,cwe-checker --cwe-checker-config=../src/config.json 
-    printf_new
-}
-
-function run_all_arch() {
-run_arch x86
-run_arch x64
-run_arch arm
-run_arch mips
-run_arch ppc
-}
-
-function main() {
-    if [ -z "$1" ]; then
-	run_all_arch
-    else
-	run_arch $1
-    fi 
-}
-
-main "$@"