Unittest flexibility (#67)

Each unit test can now be run with a separate test binary, making all unit tests more flexible.

.travis_prepare.sh

@@ -2,5 +2,7 @@
 cd test/artificial_samples/
 ./install_cross_compilers.sh
 scons
+cd ../unit/
+./specify_test_files_for_compilation.sh
 cd ../..
 docker build -t cwe-checker .

Makefile

@@ -8,6 +8,7 @@ all:
 	cd plugins/cwe_checker_type_inference_print; make all; cd ../..
 
 test:
+	cd test/unit/ && ./specify_test_files_for_compilation.sh
 	dune runtest
 	cd test/artificial_samples; scons; cd ../..
 	pytest -v

test/artificial_samples/SConstruct

@@ -48,7 +48,7 @@ target_flags = {'x64': '',
                'mips': ' -target mips-linux-gnu',
                'mipsel': ' -target mipsel-linux-gnu',
                'mips64': ' -target mips64-linux-gnuabi64',
-               'mips64el': ' -target mips64-linux-gnuabi64',
+               'mips64el': ' -target mips64el-linux-gnuabi64',
                'ppc64': ' -target powerpc64-linux-gnu',
                'ppc64le': ' -target powerpc64le-linux-gnu'}
 

test/unit/Makefile

 all:
-	bapbundle remove cwe_checker_unit_tests.plugin
-	bapbuild -r -Is analysis,checkers,utils cwe_checker_unit_tests.plugin -pkgs alcotest,yojson,unix,ppx_jane,cwe_checker_core
-	bapbundle install cwe_checker_unit_tests.plugin
-	bap ../artificial_samples/build/arrays_x64_gcc.out --pass=cwe-checker-unit-tests
-	bapbundle remove cwe_checker_unit_tests.plugin
+	bapbundle remove unit_tests_cwe_checker.plugin
+	bapbuild -r -Is analysis,checkers,utils unit_tests_cwe_checker.plugin -pkgs alcotest,yojson,unix,ppx_jane,cwe_checker_core
+	bapbundle install unit_tests_cwe_checker.plugin
+	bap tmp/arrays_gcc.out --pass=unit-tests-cwe-checker --unit-tests-cwe-checker-tests=MemRegion,Cconv,TypeInference,CWE476
+	bap tmp/c_constructs_gcc.out --pass=unit-tests-cwe-checker --unit-tests-cwe-checker-tests=CWE560,AddrTrans
+	bapbundle remove unit_tests_cwe_checker.plugin
 
 clean:
 	bapbuild -clean

test/unit/compile_testfile.sh

+#!/bin/bash
+
+tmp="tmp/"
+
+# create a tmp directory if not yet created
+if [ ! -d $tmp ]; then
+  mkdir -p $tmp;
+fi
+
+
+c_compiler=(gcc
+            x86_64-w64-mingw32-gcc
+            i686-w64-mingw32-gcc
+            arm-linux-gnueabi-gcc
+            aarch64-linux-gnu-gcc
+            mips-linux-gnu-gcc
+            mipsel-linux-gnu-gcc
+            mips64-linux-gnuabi64-gcc
+            mips64el-linux-gnuabi64-gcc
+            powerpc-linux-gnu-gcc
+            powerpc64-linux-gnu-gcc
+            powerpc64le-linux-gnu-gcc
+            clang)
+
+cpp_compiler=(g++
+              x86_64-w64-mingw32-g++
+              i686-w64-mingw32-g++
+              arm-linux-gnueabi-g++
+              aarch64-linux-gnu-g++
+              mips-linux-gnu-g++
+              mipsel-linux-gnu-g++
+              mips64-linux-gnuabi64-g++
+              mips64el-linux-gnuabi64-g++
+              powerpc-linux-gnu-g++
+              powerpc64-linux-gnu-g++
+              powerpc64le-linux-gnu-g++)
+
+# In clang cross compilation there is no target for ppc 32 bit
+targets=(x86_64-linux-gnuabi64
+         arm-linux-gnueabi
+         aarch64-linux-gnu
+         mips-linux-gnu
+         mipsel-linux-gnu
+         mips64-linux-gnuabi64
+         mips64el-linux-gnuabi64
+         powerpc64-linux-gnu
+         powerpc64le-linux-gnu)
+
+x86_flag="-m32"
+
+c_flag="-std=c11"
+
+flags="-g -fno-stack-protector"
+
+target_flag="-target"
+
+function compile_clang () {
+  for target in ${targets[@]}
+  do
+    build_name="$tmp$2"
+    build_name+="_$(cut -d'-' -f1 <<<$target)_clang.out"
+    $1 $flags $c_flag $target_flag $target $3 -o $build_name
+  done
+}
+
+function compile_x86 () {
+  build_name="$tmp$2"
+  build_name+="_x86_gcc.out"
+  gcc $x86_flag $flags $c_flag $1 -o $build_name
+  build_name="$tmp$2"
+  build_name+="_x86_clang.out"
+  clang $x86_flag $flags $c_flag $1 -o $build_name
+}
+
+
+function compile_x86_for_cpp () {
+  build_name="$tmp${2}_x86_g++.out"
+  g++ $x86_flag $flags $1 -o $build_name
+}
+
+
+function compile_gcc () {
+  build_name="$tmp$2"
+  if [[ $1 == gcc ]]; then
+    build_name+="_$1.out"
+  else
+    if [[ $1 == *w64* ]]; then
+      build_name+="_${1%-*}_gcc.out"
+    else
+      build_name+="_${1%%-*}_gcc.out"
+    fi
+  fi
+  $1 $flags $c_flag $3 -o $build_name
+}
+
+
+function compile_c () {
+  file_name=$2
+  c_file=$1
+  for compiler in ${c_compiler[@]}
+  do
+    if [[ $compiler == clang ]]; then
+      compile_clang "$compiler" "$file_name" "$c_file"
+    else
+      compile_gcc "$compiler" "$file_name" "$c_file"
+    fi
+  done
+
+  compile_x86 "$c_file" "$file_name"
+}
+
+
+function compile_cpp () {
+  for compiler in ${cpp_compiler[@]}
+  do
+    build_name="$tmp$2"
+    if [[ $compiler == g++ ]]; then
+      build_name+="_$compiler.out"
+    else
+      if [[ $compiler == *w64* ]]; then
+        build_name+="_${compiler%-*}_g++.out"
+      else
+        build_name+="_${compiler%%-*}_g++.out"
+      fi
+    fi
+    $compiler $flags $1 -o $build_name
+  done
+  compile_x86_for_cpp "$1" "$2"
+}
+
+
+function main () {
+  test_file="$1"
+  echo "Compiling test file $test_file"
+  file_name="${test_file%.*}"
+  file_name="${file_name##*/}"
+
+  if [[ $test_file == *.c ]]; then
+    compile_c "$test_file" "$file_name"
+  elif [[ $test_file == *.cpp ]]; then
+    compile_cpp "$test_file" "$file_name"
+  fi
+}
+
+main "$@"

test/unit/cwe_checker_unit_tests.ml

-open Bap.Std
-open Core_kernel
-
-
-let run_tests project =
-  Type_inference_test.example_project := Some(project);
-  Cconv_test.example_project := Some(project);
-  Cwe_476_test.example_project := Some(project);
-  Alcotest.run "Unit tests" ~argv:[|"DoNotComplainWhenRunAsABapPlugin";"--color=always";|] [
-    "Mem_region_tests", Mem_region_test.tests;
-    "Type_inference_tests", Type_inference_test.tests;
-    "Cconv_tests", Cconv_test.tests;
-    "CWE_476_tests", Cwe_476_test.tests;
-    "CWE_560_tests", Cwe_560_test.tests;
-    "Address_translation_tests", Address_translation_test.tests;
-     ]
-
-let () =
-  (* Check whether this file is run as an executable (via dune runtest) or
-     as a bap plugin *)
-  if Sys.argv.(0) = "bap" then
-    (* The file was run as a bap plugin. *)
-    Project.register_pass' run_tests
-  else
-    (* The file was run as a standalone executable. Use make to build and run the unit test plugin *)
-    let () = try
-        Sys.chdir (Sys.getenv "PWD" ^ "/test/unit")
-      with _ -> (* In the docker image the environment variable PWD is not set *)
-        Sys.chdir "/home/bap/cwe_checker/test/unit"
-    in
-    exit (Sys.command "make all")

test/unit/dune

 (executable
-  (name cwe_checker_unit_tests)
+  (name unit_tests_cwe_checker)
   (libraries
   alcotest
   yojson
@@ -13,5 +13,5 @@
 
 (alias
  (name runtest)
- (deps cwe_checker_unit_tests.exe)
+ (deps unit_tests_cwe_checker.exe)
 (action (run %{deps} --color=always)))

test/unit/specify_test_files_for_compilation.sh

+#!/usr/bin/env bash
+
+./compile_testfile.sh testfiles/arrays.c
+./compile_testfile.sh testfiles/c_constructs.c

test/unit/testfiles/arrays.c

+#include <stdio.h>
+#include <stdlib.h>
+
+void heap_based_array(){
+        char* a = malloc(20);
+
+        for(int i=0; i<20;i++){
+                *(a + i) = 'A';
+        }
+
+        free(a);
+}
+
+void stack_based_array(){
+        char a[20];
+        for(int i=0; i<20;i++){
+                a[i] = 'A';
+        }
+}
+
+int main(int argc, char *argv[argc])
+{
+        stack_based_array();
+        heap_based_array();
+        return 0;
+}

test/unit/testfiles/c_constructs.c

+#include <stdlib.h>
+
+void if_statement(){
+  void* bla = malloc(89);
+  int a = 2;
+  if (a < 4){
+    a = 33;
+  }
+  free(bla);
+}
+
+void for_loop(){
+  void* bla = malloc(89);
+  int a = 34;
+  for(int i = 0;i < 100; i++){
+    a += i;
+  }
+  free(bla);
+}
+
+void nested_for_loop(){
+  void* bla = malloc(89);
+  int a = 34;
+  for(int i = 0; i < 100; i++){
+    for(int j = 0; j < 200; j++){
+      a += i + j;
+    }
+  }
+  free(bla);
+}
+
+int main(){
+  if_statement();
+  for_loop();
+  nested_for_loop();
+}

test/unit/unit_tests_cwe_checker.ml

+(**
+   This module contains the unit test infrastructure to coordiate
+   each unit test in the cwe_checker/unit/ folder.
+
+   To add an unit test and the corresponding test file,
+   a few steps have to be performed before execution:
+
+   - add the test list from your unit test to unit_test_list in this module
+
+   - if your unit test utilises a example project, it has to be added to the set_example_project
+     in this module.
+
+   - add your corresponding test file to the testfiles folder in cwe_checker/unit/
+
+   - call the unit test as bap plugin in the Makefile contained in the unit folder.
+     The compiled test file will lie in the tmp folder.
+
+   - lastly, run compile_testfile.sh in the specify_test_files_for_compilation contained in the unit folder
+*)
+
+open Bap.Std
+open Core_kernel
+
+include Self()
+
+let cmdline_params = [
+  ("tests", "Comma separated list defining which tests should be executed with the current test file. e.g. MemRegion,TypeInference,CWE476,...")
+]
+
+let unit_test_list = [
+  "MemRegion", Mem_region_test.tests;
+  "TypeInference", Type_inference_test.tests;
+  "Cconv", Cconv_test.tests;
+  "CWE476", Cwe_476_test.tests;
+  "CWE560", Cwe_560_test.tests;
+  "AddrTrans", Address_translation_test.tests;
+]
+
+
+let set_example_project (project : Project.t) (tests : string list) =
+  List.iter tests ~f:(fun test ->
+    match test with
+    | "TypeInference" -> Type_inference_test.example_project := Some(project)
+    | "Cconv" -> Cconv_test.example_project := Some(project)
+    | "CWE476" -> Cwe_476_test.example_project := Some(project)
+    | _ -> ()
+  )
+
+
+let check_user_input (tests : string list) =
+  let test_list = List.map unit_test_list ~f:(fun test -> match test with (name, _) -> name) in
+  List.iter tests ~f:(fun test ->
+    match Stdlib.List.mem test test_list with
+    | true -> ()
+    | false -> failwith (Printf.sprintf "Test %s is not a valid test." test)
+  )
+
+
+let filter_tests (tests : string list) : (string * unit Alcotest.test_case list) list =
+  List.filter unit_test_list ~f:(fun (name, _) ->
+     match Stdlib.List.mem name tests with
+     | true -> true
+     | false -> false
+  )
+
+
+let run_tests (params : String.t String.Map.t) (project : Project.t) =
+  let test_param = match String.Map.find params "tests" with
+  | Some(param) -> param
+  | None -> failwith "No tests were provided to the unittest plugin." in
+  let tests = (String.split test_param ~on: ',') in
+  check_user_input tests;
+  set_example_project project tests;
+  Alcotest.run "Unit tests" ~argv:[|"DoNotComplainWhenRunAsABapPlugin";"--color=always";|] (filter_tests tests)
+
+
+let generate_bap_params params =
+  List.map params ~f:(fun (name, docstring) -> (name, Config.param Config.string name ~doc:docstring))
+
+
+let () =
+  (* Check whether this file is run as an executable (via dune runtest) or
+     as a bap plugin *)
+  if Sys.argv.(0) = "bap" then
+    let cmdline_params = generate_bap_params cmdline_params in
+    let () = Config.when_ready (fun ({get=(!!)}) ->
+      let params: String.t String.Map.t = List.fold cmdline_params ~init:String.Map.empty ~f:(fun param_map (name, bap_param) ->
+        String.Map.set param_map ~key:name ~data:(!!bap_param)) in
+      Project.register_pass' (run_tests params)
+    ) in
+    ()
+  else
+    (* The file was run as a standalone executable. Use make to build and run the unit test plugin *)
+    let () = try
+        Sys.chdir (Sys.getenv "PWD" ^ "/test/unit")
+      with _ -> (* In the docker image the environment variable PWD is not set *)
+        Sys.chdir "/home/bap/cwe_checker/test/unit"
+    in
+    exit (Sys.command "make all")