Refactoring of logging (#30)

* Removes old version of log_utils, prototypes for new version.

* Implemented native logging

* Json-Output basically working.

* Added acceptance test for JSON parsing

* Adds some odoc to log_utils.

* Added support for file output (--cwe-checker-out)

* Add acceptance test for file output

CHANGES.md

@@ -3,6 +3,8 @@ dev
 
 -   Added more documentation to checks (PR #26)
 -   Fixed check CWE367: use symbols defined in config.json (PR #28)
+-   Refactoring of logging and JSON support via --json (PR #30)
+-   Added file output support via --out (PR #30)
 
 0.2 (2019-06-25)
 =====

README.md

@@ -57,6 +57,7 @@ If you plan to develop cwe_checker, it is recommended to build it using the prov
 -   dune >= 1.6
 -   BAP 1.6 (and its dependencies)
 -   yojson >= 1.6.0
+-   ppx_deriving_json >= 3.5.1
 -   alcotest >= 0.8.3 (for tests)
 -   Sark (latest) for IDA Pro annotations
 -   pytest >= 3.5.1 (for tests)

plugins/cwe_checker/cwe_checker.ml

@@ -34,9 +34,9 @@ let build_version_sexp () =
   |> String.concat ~sep:" "
 
 let print_module_versions () =
-  Log_utils.info
-    "[cwe_checker] module_versions: (%s)"
-    (build_version_sexp ())
+  Log_utils.info (sprintf
+                    "[cwe_checker] module_versions: (%s)"
+                    (build_version_sexp ()))
 
 let execute_cwe_module cwe json program project tid_address_map =
   let parameters = match cwe.has_parameters with
@@ -57,7 +57,7 @@ let partial_run project config modules =
   let program = Project.program project in
   let tid_address_map = Address_translation.generate_tid_map program in
   let json = Yojson.Basic.from_file config in
-  Log_utils.info "[cwe_checker] Just running the following analyses: %s." modules;
+  Log_utils.info (sprintf "[cwe_checker] Just running the following analyses: %s." modules);
   List.iter (String.split modules ~on: ',') ~f:(fun cwe ->
     let cwe_mod = match List.find known_modules ~f:(fun x -> x.name = cwe) with
       | Some(module_) -> module_
@@ -74,10 +74,7 @@ let full_run project config =
     List.iter known_modules ~f:(fun cwe -> execute_cwe_module cwe json program project tid_address_map)
   end
 
-let main config module_versions partial_update project =
-  Log_utils.set_log_level Log_utils.DEBUG;
-  Log_utils.set_output stdout;
-  Log_utils.color_on ();
+let main config module_versions partial_update json_output file_output project =
 
   if module_versions then
     begin
@@ -102,19 +99,28 @@ let main config module_versions partial_update project =
           if partial_update = "" then
             full_run project config
           else
-            partial_run project config partial_update
+            partial_run project config partial_update;
+          if json_output then
+            begin
+              match Project.get project filename with
+              | Some fname -> Log_utils.emit_cwe_warnings_json fname file_output
+              | None -> Log_utils.emit_cwe_warnings_json "" file_output
+            end
+          else
+            Log_utils.emit_cwe_warnings_native file_output
         end
     end
 
 module Cmdline = struct
   open Config
   let config = param string "config" ~doc:"Path to configuration file."
-  let module_versions = param bool "module_versions" ~doc:"Prints out the version numbers of all known modules."
+  let module_versions = flag "module_versions" ~doc:"Prints out the version numbers of all known modules."
+  let json_output = flag "json" ~doc:"Outputs the result as JSON."
+  let file_output = param string "out" ~doc:"Path to output file."
   let partial_update = param string "partial" ~doc:"Comma separated list of modules to apply on binary, e.g. 'CWE332,CWE476,CWE782'"
-  let () = when_ready (fun ({get=(!!)}) -> Project.register_pass' ~deps:["callsites"] (main !!config !!module_versions !!partial_update))
+  let () = when_ready (fun ({get=(!!)}) -> Project.register_pass' ~deps:["callsites"] (main !!config !!module_versions !!partial_update !!json_output !!file_output))
   let () = manpage [
                           `S "DESCRIPTION";
-                          `P
-                            "This plugin checks various CWEs such as Insufficient Entropy in PRNG (CWE-332) or Use of Potentially Dangerous Function (CWE-676)"
+                          `P "This plugin checks various CWEs such as Insufficient Entropy in PRNG (CWE-332) or Use of Potentially Dangerous Function (CWE-676)"
                         ]
 end

plugins/cwe_checker_emulation/cwe_checker_emulation.ml

@@ -124,11 +124,7 @@ end
 - for all subroutins we fork a Primus machine
 - all monitored events are collected globally
 - after the last Primus machine has terminated we report all observed incidents *)
-let main {Config.get=(!)} proj =
-  Log_utils.set_log_level Log_utils.DEBUG;
-  Log_utils.set_output stdout;
-  Log_utils.color_on ();
-
+let main json_output file_output proj =
   Primus.Machine.add_component (module Monitor);
   begin
   let prog = (Project.program proj) in
@@ -141,13 +137,21 @@ let main {Config.get=(!)} proj =
      info "program terminated by a signal: %s" (Primus.Exn.to_string exn);
   end;
   analyze_events ();
-  proj
-
-(** At the moment this plugin depends due to Primus on the plugin
-trivial-condition-form. *)
- let deps = [
-  "trivial-condition-form"
-]
-
-let () =
-  Config.when_ready (fun conf -> Project.register_pass ~deps (main conf))
+  if json_output then
+    begin
+      match Project.get proj filename with
+      | Some fname -> Log_utils.emit_cwe_warnings_json fname file_output
+      | None -> Log_utils.emit_cwe_warnings_json "" file_output
+    end
+  else
+    Log_utils.emit_cwe_warnings_native file_output
+
+module Cmdline = struct
+  open Config
+  let json_output = flag "json" ~doc:"Outputs the result as JSON."
+  let file_output = param string "out" ~doc:"Path to output file."
+  let () = when_ready (fun ({get=(!!)}) -> Project.register_pass' ~deps:["trivial-condition-form"] (main !!json_output !!file_output))
+  let () = manpage [
+               `S "DESCRIPTION";
+               `P "This plugin utilizes symbolic execution to find CWEs like Double Free (CWE415) or Use After Free (CWE416)."]
+end

plugins/cwe_checker_emulation/incident_reporter.ml

 open Core_kernel
 open Cwe_checker_core
+open Log_utils
 
 let version = "0.1"
 
@@ -26,19 +27,28 @@ let get_incident_locations_from_ids ids location_tbl =
   let incident_locations = ref [] in
       Sexplib__Sexp_with_layout.List.iter ids ~f:(fun id ->  incident_locations := (map_id_to_location (Sexp.to_string id) location_tbl)::(!incident_locations)); !incident_locations
 
-let report_cwe_125 location_path = 
-      Log_utils.warn "[CWE125] {%s} (Out-of-bounds Read) %s" version location_path;
-      Log_utils.warn "[CWE787] {%s} (Out-of-bounds Write) %s" version location_path
+let report_cwe_125 location_path =
+  let description = sprintf "(Out-of-bounds Read) %s" location_path in
+  let cwe_warning = cwe_warning_factory "CWE125" version ~other:[["path";location_path]] description in
+  collect_cwe_warning cwe_warning;
+  let description = sprintf "(Out-of-bounds Write) %s" location_path in
+  let cwe_warning = cwe_warning_factory "CWE787" version ~other:[["path";location_path]] description in
+  collect_cwe_warning cwe_warning
 
 let report_cwe_415 location_path =
-  Log_utils.warn "[CWE415] {%s} (Double Free) %s" version location_path
+  let description = sprintf "(Double Free) %s" location_path in
+  let cwe_warning = cwe_warning_factory "CWE415" version ~other:[["path";location_path]] description in
+  collect_cwe_warning cwe_warning
 
 let report_cwe_416 location_path =
-  Log_utils.warn "[CWE416] {%s} (Use After Free) %s" version location_path
+  let description = sprintf "(Use After Free) %s" location_path in
+  let cwe_warning = cwe_warning_factory "CWE416" version ~other:[["path";location_path]] description in
+  collect_cwe_warning cwe_warning
 
 let report_cwe_unknown location_path incident_str =
-  Log_utils.warn "[CWE UNKNOWN] {%s} (%s) %s" version incident_str location_path
-
+  let description = sprintf "(%s) %s" incident_str location_path in
+  let cwe_warning = cwe_warning_factory incident_str version ~other:[["path";location_path]] description in
+  collect_cwe_warning cwe_warning
 
 (** Reports an incident. *)
 let report incident location_tbl =

plugins/cwe_checker_type_inference_print/cwe_checker_type_inference_print.ml

@@ -2,13 +2,27 @@ open Bap.Std
 open Core_kernel
 open Cwe_checker_core
 
-let main project =
-  Log_utils.set_log_level Log_utils.DEBUG;
-  Log_utils.set_output stdout;
-  Log_utils.color_on ();
+include Self()
+
+let main json_output file_output project =
 
   let program = Project.program project in
   let tid_map = Address_translation.generate_tid_map program in
-  Type_inference.print_type_info_tags project tid_map
+  Type_inference.print_type_info_tags project tid_map;
+  if json_output then
+    begin
+      match Project.get project filename with
+      | Some fname -> Log_utils.emit_cwe_warnings_json fname file_output
+      | None -> Log_utils.emit_cwe_warnings_json "" file_output
+    end
+  else
+    Log_utils.emit_cwe_warnings_native file_output
 
-let () = Project.register_pass' main ~deps:["cwe-checker-type-inference"]
+module Cmdline = struct
+  open Config
+  let json_output = flag "json" ~doc:"Outputs the result as JSON."
+  let file_output = param string "out" ~doc:"Path to output file."
+  let () = when_ready (fun ({get=(!!)}) -> Project.register_pass' ~deps:["cwe-checker-type-inference"] (main !!json_output !!file_output))
+  let () = manpage [`S "DESCRIPTION";
+                    `P "This plugin prints the results of the type inference plugin."]
+end

src/analysis/type_inference.ml

@@ -617,13 +617,14 @@ let print_type_info_to_debug state block_tid ~tid_map ~sub_tid ~project =
         end
     | _ -> "Unknown"
   in
-  Log_utils.debug
-    "[%s] {%s} TypeInfo at %s:\nRegister: %s\nStackOffset: %s"
-    name
-    version
-    (Address_translation.translate_tid_to_assembler_address_string block_tid tid_map)
-    register_string
-    stack_offset_str
+  let debug_str = sprintf
+                    "[%s] {%s} TypeInfo at %s:\nRegister: %s\nStackOffset: %s"
+                    name
+                    version
+                    (Address_translation.translate_tid_to_assembler_address_string block_tid tid_map)
+                    register_string
+                    stack_offset_str in
+  Log_utils.debug debug_str
 
 let print_type_info_tags ~project ~tid_map =
   let program = Project.program project in
@@ -635,12 +636,13 @@ let print_type_info_tags ~project ~tid_map =
           match Term.get_attr block type_info_tag with
           | Some(start_state) -> print_type_info_to_debug start_state (Term.tid block) ~tid_map ~sub_tid ~project
           | None -> (* block has no type info tag, which should not happen *)
-            Log_utils.error
-              "[%s] {%s} Block has no TypeInfo at %s (block TID %s)"
-              name
-              version
-              (Address_translation.translate_tid_to_assembler_address_string (Term.tid block) tid_map)
-              (Tid.name (Term.tid block))
+             let error_str = sprintf
+                               "[%s] {%s} Block has no TypeInfo at %s (block TID %s)"
+                               name
+                               version
+                               (Address_translation.translate_tid_to_assembler_address_string (Term.tid block) tid_map)
+                               (Tid.name (Term.tid block)) in
+             Log_utils.error error_str
         )
     )
 

src/checkers/cwe_190.ml

 open Core_kernel
 open Bap.Std
 open Symbol_utils
+open Log_utils
 
 let name = "CWE190"
 let version = "0.1"
@@ -20,12 +21,11 @@ let contains_multiplication d =
 let check_multiplication_before_symbol _proj _prog _sub blk jmp tid_map symbols =
   Seq.iter (Term.enum def_t blk)
     ~f:(fun d -> if contains_multiplication d then
-                   Log_utils.warn
-                     "[%s] {%s} (Integer Overflow or Wraparound) Potential overflow due to multiplication %s (%s)."
-                     name
-                     version
-                     (Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map)
-                     (Symbol_utils.get_symbol_name_from_jmp jmp symbols))
+                   let description = "(Integer Overflow or Wraparound) Potential overflow due to multiplication" in
+                   let addresses = [(Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map)] in
+                   let symbols = [(Symbol_utils.get_symbol_name_from_jmp jmp symbols)] in
+                   let cwe_warning = cwe_warning_factory name version description ~addresses ~symbols in
+                   collect_cwe_warning cwe_warning)
 
 let check_cwe prog proj tid_map symbol_names _ =
   match symbol_names with

src/checkers/cwe_215.ml

 open Core_kernel
 open Bap.Std
-
+open Log_utils
 
 (* TODO: IVG via gitter:
 I see, so you need the CU information, and yes BAP doesn't provide this.
@@ -26,9 +26,14 @@ let check_cwe _ project _ _ _ =
       let cmd = Format.sprintf "readelf --debug-dump=decodedline %s | grep CU" fname in
       try
         let in_chan = Unix.open_process_in cmd in
-        In_channel.input_lines in_chan |> List.iter ~f:(fun l -> Log_utils.warn "[%s] {%s} (Information Exposure Through Debug Information) %s" name version l)
+        In_channel.input_lines in_chan |> List.iter ~f:(fun l ->
+                                              let description = sprintf "(Information Exposure Through Debug Information) %s" l in 
+                                              let cwe_warning = cwe_warning_factory name version description ~symbols:[l] in
+                                              collect_cwe_warning cwe_warning)
+
+
       with
         Unix.Unix_error (e,fm,argm) ->
-        Log_utils.error "[%s] {%s} %s %s %s" name version (Unix.error_message e) fm argm
+        Log_utils.error (sprintf "[%s] {%s} %s %s %s" name version (Unix.error_message e) fm argm)
     end
   | _ -> failwith "[CWE215] symbol_names not as expected"

src/checkers/cwe_243.ml

 open Core_kernel
 open Bap.Std
 open Symbol_utils
+open Log_utils
 
 include Self()
 
@@ -73,12 +74,14 @@ let check_subfunction prog tid_map sub pathes =
     begin
       let path_checks = List.map pathes ~f:(fun path -> check_path prog tid_map sub path) in
       if not (List.exists path_checks ~f:(fun x -> x = true)) then
-       Log_utils.warn
-         "[%s] {%s} (The program utilizes chroot without dropping privileges and/or changing the directory) at %s (%s)"
-         name
-         version
-         (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map)
-         (Term.name sub)
+        let address = (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map) in
+        let symbol = (Term.name sub) in
+        let description = sprintf
+                            "(The program utilizes chroot without dropping privileges and/or changing the directory) at %s (%s)"
+                            address
+                            symbol in
+        let cwe_warning = cwe_warning_factory name version description ~addresses:[address] ~symbols:[symbol] in
+        collect_cwe_warning cwe_warning
     end
 
 let check_cwe prog _proj tid_map pathes _ =

src/checkers/cwe_248.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE248"
 let version = "0.1"
 
 (* Print the findings to the log *)
 let print_uncatched_exception block_tid ~tid_map =
-   Log_utils.warn
-     "[%s] {%s} (Possibly Uncaught Exception) (Exception thrown at %s)."
-     name
-     version
-     (Address_translation.translate_tid_to_assembler_address_string block_tid tid_map)
-
+  let address = (Address_translation.translate_tid_to_assembler_address_string block_tid tid_map) in
+  let description = sprintf "(Possibly Uncaught Exception) (Exception thrown at %s)." address in
+  let cwe_warning = cwe_warning_factory name version description in
+  collect_cwe_warning cwe_warning
+                    
 (* Extract the name of a direct call, if the block contains a direct call. *)
 let extract_direct_call_symbol block =
   match Symbol_utils.extract_direct_call_tid_from_block block with

src/checkers/cwe_332.ml

 open Core_kernel
-
 open Symbol_utils
+open Log_utils
 
 let name = "CWE332"
 let version = "0.1"
@@ -10,6 +10,10 @@ let check_cwe program _proj _tid_map _symbol_pairs _ =
   | None -> begin
       match (find_symbol program "rand") with
       | None -> ()
-      | Some _ -> Log_utils.warn "[%s] {%s} (Insufficient Entropy in PRNG) program uses rand without calling srand before" name version
+      | Some _ -> begin
+          let description = "(Insufficient Entropy in PRNG) program uses rand without calling srand before" in
+          let cwe_warning = cwe_warning_factory name version description in
+          collect_cwe_warning cwe_warning
+        end
     end
   | Some (_srand_tid, _rand_tid) -> ()

src/checkers/cwe_367.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE367"
 let version = "0.1"
@@ -45,14 +46,23 @@ let handle_sub sub program tid_map _symbols source_sink_pair =
           Seq.iter source_calls ~f:(fun source_call ->
               Seq.iter sink_calls ~f:(fun sink_call ->
                   if is_reachable sub source_call sink_call then
-                    Log_utils.warn
-                      "[%s] {%s} (Time-of-check Time-of-use Race Condition) %s is reachable from %s at %s (%s). This could lead to a TOCTOU."
-                      name
-                      version
+                    let address = (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map) in
+                    let symbol = (Term.name sub) in
+                    let other = [["source"; source]; ["sink"; sink]] in
+                    let description = sprintf 
+                     "(Time-of-check Time-of-use Race Condition) %s is reachable from %s at %s (%s). This could lead to a TOCTOU."
                       sink
                       source
-                      (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map)
-                      (Term.name sub)
+                      address
+                      symbol in
+                    let cwe_warning = cwe_warning_factory
+                                        name
+                                        version
+                                        description
+                                        ~other:other
+                                        ~addresses:[address]
+                                        ~symbols:[symbol] in
+                    collect_cwe_warning cwe_warning
                   else
                     ()))
         end

src/checkers/cwe_426.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE426"
 let version = "0.1"
@@ -11,11 +12,13 @@ let handle_sub sub program tid_map symbols =
   if calls_privilege_changing_sub sub program symbols then
     begin
       if Symbol_utils.sub_calls_symbol program sub "system" then
-        Log_utils.warn "[%s] {%s} (Untrusted Search Path) sub %s at %s may be vulnerable to PATH manipulation."
-          name
-          version
-          (Term.name sub)
-          (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map)
+        let symbol = Term.name sub in
+        let address = Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map in
+        let description = sprintf "(Untrusted Search Path) sub %s at %s may be vulnerable to PATH manipulation."
+          symbol
+          address in
+        let cwe_warning = cwe_warning_factory name version ~addresses:[address] ~symbols:[symbol] description in
+        collect_cwe_warning cwe_warning
       else
         ()
     end

src/checkers/cwe_457.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE457"
 let version = "0.1"
@@ -47,7 +48,7 @@ let get_fp_of_arch arch =
   | `armv4 | `armv5 | `armv6 | `armv7 | `armv4eb | `armv5eb | `armv6eb | `armv7eb -> "R11"
   | `mips | `mips64 | `mips64el | `mipsel -> "FP"
   | `ppc | `ppc64 | `ppc64le -> "R31"
-  | _ -> Log_utils.error "[%s] {%s} Unknown architecture." name version; "UNKNOWN"
+  | _ -> failwith  "Unknown architecture."
 
 let vars_contain_fp vars fp_pointer =
   let regs = Set.filter vars ~f:(fun var -> Var.to_string var = fp_pointer) in
@@ -62,6 +63,19 @@ let is_interesting_load_store def fp_pointer =
 (*TODO: implement real filtering*)
 let filter_mem_address i min_fp_offset = Set.filter i ~f:(fun elem -> (Word.of_int  ~width:32 min_fp_offset) < elem)
 
+let log_cwe_warning sub i d tid_map =
+   let word = Word.to_string i in
+   let other =[["word"; word]] in 
+   let symbol = Sub.name sub in
+   let address = Address_translation.translate_tid_to_assembler_address_string (Term.tid d) tid_map in
+   let description = sprintf
+                       "(Use of Uninitialized Variable) Found potentially unitialized stack variable (FP + %s) in function %s at %s"
+                       word
+                       symbol
+                       address in
+   let cwe_warning = cwe_warning_factory name version ~other:other ~addresses:[address] ~symbols:[symbol] description in
+   collect_cwe_warning cwe_warning
+
 let check_subfunction _prog proj tid_map sub =
   let fp_pointer = get_fp_of_arch (Project.arch proj) in
   let min_fp_offset = get_min_fp_offset (Project.arch proj) in
@@ -81,14 +95,7 @@ let check_subfunction _prog proj tid_map sub =
             begin
               let filter_mem_addresses = filter_mem_address ints min_fp_offset in
               Set.iter filter_mem_addresses ~f:(fun i -> if not (Array.exists !stores ~f:(fun elem -> elem = i)) then
-                                           begin
-                                           Log_utils.warn "[%s] {%s} (Use of Uninitialized Variable) Found potentially unitialized stack variable (FP + %s) in function %s at %s"
-                                             name
-                                             version
-                                             (Word.to_string i)
-                                             (Sub.name sub)
-                                             (Address_translation.translate_tid_to_assembler_address_string (Term.tid d) tid_map)
-                                           end)
+                log_cwe_warning sub i d tid_map)
             end
         end)
 

src/checkers/cwe_467.ml

 open Core_kernel
 open Bap.Std
 open Symbol_utils
+open Log_utils
 
 let name = "CWE467"
 let version = "0.1"
@@ -14,11 +15,16 @@ let check_input_is_pointer_size proj _prog _sub blk jmp tid_map symbols =
         begin
         try
           if get_pointer_size (Project.arch proj) = (Word.to_int_exn w) then
-            Log_utils.warn "[%s] {%s} (Use of sizeof on a Pointer Type) sizeof on pointer at %s (%s)."
-              name
-              version
-              (Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map)
-              (Symbol_utils.get_symbol_name_from_jmp jmp symbols)
+            begin
+              let address = Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map in
+              let symbol = Symbol_utils.get_symbol_name_from_jmp jmp symbols in
+              let description = sprintf
+                                  "(Use of sizeof on a Pointer Type) sizeof on pointer at %s (%s)."
+                                  address
+                                  symbol in
+              let cwe_warning = cwe_warning_factory name version ~addresses:[address] ~symbols:[symbol] description in
+              collect_cwe_warning cwe_warning
+            end
         with _ -> Log_utils.error "Caught exception in module [CWE467]."
       end
       | _ -> ())

src/checkers/cwe_476.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE476"
 let version = "0.2"
@@ -240,13 +241,21 @@ let print_hit tid ~sub ~function_names ~tid_map =
       | Call(call) -> begin
           match Call.target call with
           | Direct(call_tid) -> Option.is_some (List.find function_names ~f:(fun fn_name ->
-              if fn_name = (Tid.name call_tid) then begin
-                Log_utils.warn "[%s] {%s} (NULL Pointer Dereference) There is no check if the return value is NULL at %s (%s)."
-                  name
-                  version
-                  (Address_translation.translate_tid_to_assembler_address_string tid tid_map)
-                  fn_name;
-                true
+                                                    if fn_name = (Tid.name call_tid) then
+                                                      begin
+                                                      let address = Address_translation.translate_tid_to_assembler_address_string tid tid_map in
+                                                      let description = sprintf
+                                                                          "(NULL Pointer Dereference) There is no check if the return value is NULL at %s (%s)."
+                                                                          address
+                                                                          fn_name in
+                                                      let cwe_warning = cwe_warning_factory
+                                                                          name
+                                                                          version
+                                                                          ~addresses:[address]
+                                                                          ~symbols:[fn_name]
+                                                                          description in
+                                                      collect_cwe_warning cwe_warning;
+                                                      true
               end else
                 false
             ))

src/checkers/cwe_560.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE560"
 let version = "0.1"
@@ -19,11 +20,16 @@ let check_umask_arg tid_map blk w =
   try
     let umask_arg = Word.to_int_exn w in
     if is_chmod_style_arg umask_arg then
-      Log_utils.warn "[%s] {%s} (Use of umask() with chmod-style Argument) Function %s calls umask with argument %d"
-        name
-        version
-        (Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map)
-        umask_arg
+      let address = Address_translation.translate_tid_to_assembler_address_string (Term.tid blk) tid_map in
+      let umask_arg_str = sprintf "%d" umask_arg in
+      let description = sprintf
+                          "(Use of umask() with chmod-style Argument) Function %s calls umask with argument %s"
+                          address
+                          umask_arg_str in
+      let other = [["umask_arg"; umask_arg_str]] in
+      let cwe_warning = cwe_warning_factory name version ~addresses:[address] ~other:other description in
+      collect_cwe_warning cwe_warning
+
   with _ -> Log_utils.error "Caught exception in module [CWE560]."
 
 let check_umask_callsite tid_map blk =

src/checkers/cwe_676.ml

 open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE676"
 let version = "0.1"
@@ -22,13 +23,26 @@ let get_calls_to_symbols cg subfunctions symbols =
 (* FIXME: refactor variable names *)
 let print_calls calls ~tid_map =
    Seq.iter calls ~f:(fun call -> match call with
-      | (a, b, c) -> Log_utils.warn
-                       "[%s] {%s} (Use of Potentially Dangerous Function) %s (%s) -> %s."
-                       name
-                       version
-                       a
-                       (Address_translation.translate_tid_to_assembler_address_string b tid_map)
-                       c)
+                                  | (a, b, c) ->
+                                     begin
+                                       let address = Address_translation.translate_tid_to_assembler_address_string b tid_map in
+                                       let other = [["dangerous_function"; c]] in
+                                       let description = sprintf
+                                                           "(Use of Potentially Dangerous Function) %s (%s) -> %s."
+                                                           a
+                                                           address
+                                                           c in
+                                       let cwe_warning = cwe_warning_factory
+                                                           name
+                                                           version
+                                                           ~other:other
+                                                           ~addresses:[address]
+                                                           ~symbols:[a]
+                                                           description in
+                                       collect_cwe_warning cwe_warning
+
+                                     end
+)
 
 let resolve_symbols prog symbols =
   Term.enum sub_t prog |>

src/checkers/cwe_782.ml

+open Core_kernel
 open Bap.Std
+open Log_utils
 
 let name = "CWE782"
 let version = "0.1"
 
 (*TODO: check if binary is setuid*)
 let handle_sub sub program tid_map _symbols =
-      if Symbol_utils.sub_calls_symbol program sub "ioctl" then
-        Log_utils.warn "[%s] {%s} (Exposed IOCTL with Insufficient Access Control) Program uses ioctl at %s (%s). Be sure to double check the program and the corresponding driver."
-          name
-          version
-          (Term.name sub)
-          (Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map)
+  if Symbol_utils.sub_calls_symbol program sub "ioctl" then
+    begin
+      let address = Address_translation.translate_tid_to_assembler_address_string (Term.tid sub) tid_map in
+      let symbol = Term.name sub in
+      let description = sprintf
+                          "(Exposed IOCTL with Insufficient Access Control) Program uses ioctl at %s (%s). Be sure to double check the program and the corresponding driver."
+                          symbol
+                          address in
+      let cwe_warning = cwe_warning_factory name version ~addresses:[address] ~symbols:[symbol] description in
+      collect_cwe_warning cwe_warning
+    end
       else
         ()
 

src/dune

@@ -4,8 +4,9 @@
   (libraries
   yojson
   bap
-  core_kernel)
-  (preprocess (pps ppx_jane))
+  core_kernel
+  ppx_deriving_yojson.runtime)
+  (preprocess (pps ppx_jane ppx_deriving_yojson))
 )
 
 (include_subdirs unqualified) ; Include all subdirs when looking for source files

src/utils/log_utils.ml

-(* Copyright (c) 2014, INRIA.
- * Copyright (c) 2013, Zhang Initiative Research Unit,
- * Advance Science Institute, RIKEN
- * 2-1 Hirosawa, Wako, Saitama 351-0198, Japan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *)
-
-open Printf
-
-(* localtime is used to date events, _not_ GMT, BEWARE SCIENTIST *)
-
-type log_level =
-  | FATAL
-  | ERROR
-  | WARN
-  | INFO
-  | DEBUG
-
-let int_of_level = function
-  | FATAL -> 4
-  | ERROR -> 3
-  | WARN  -> 2
-  | INFO  -> 1
-  | DEBUG -> 0
-
-let string_of_level = function
-  | FATAL -> "FATAL"
-  | ERROR -> "ERROR"
-  | WARN  -> "WARN "
-  | INFO  -> "INFO "
-  | DEBUG -> "DEBUG"
-
-let level_of_string = function
-  | "FATAL" | "fatal" -> FATAL
-  | "ERROR" | "error" -> ERROR
-  | "WARN"  | "warn"  -> WARN
-  | "INFO"  | "info"  -> INFO
-  | "DEBUG" | "debug" -> DEBUG
-  | str -> failwith ("no such log level: " ^ str)
-
-type color = Black | Red | Green | Yellow | Blue | Magenta | Cyan | White
-           | Default
-
-(* ANSI terminal colors for UNIX *)
-let color_to_string = function
-  | Black   -> "\027[30m"
-  | Red     -> "\027[31m"
-  | Green   -> "\027[32m"
-  | Yellow  -> "\027[33m"
-  | Blue    -> "\027[34m"
-  | Magenta -> "\027[35m"
-  | Cyan    -> "\027[36m"
-  | White   -> "\027[37m"
-  | Default -> "\027[39m"
-
-let color_reset = "\027[0m"
-
-(* default log levels color mapping *)
-let color_of_level = function
-  | FATAL -> Magenta
-  | ERROR -> Red
-  | WARN  -> Yellow
-  | INFO  -> Green
-  | DEBUG -> Cyan
-
-(* defaults *)
-let level          = ref ERROR
-let output         = ref stdout
-let level_to_color = ref color_of_level
-let use_color      = ref false
-let prefix         = ref ""
-
-let set_log_level l =
-  level := l
-
-let get_log_level () =
-  !level
-
-let set_output o =
-  output := o
-
-let set_prefix p =
-  prefix := p
-
-let clear_prefix () =
-  prefix := ""
-
-let set_color_mapping f =
-  level_to_color := f
-
-let color_on () =
-  use_color := true
-
-let color_off () =
-  use_color := false
-
-let level_to_string lvl =
-  let s = string_of_level lvl in
-  if !use_color then
-    let color = !level_to_color lvl in
-    (color_to_string color) ^ s ^ (color_reset)
-  else
-    s
-
-let section_width = ref 0
-
-module type S = sig
-  val log   : log_level -> ('a, out_channel, unit) format -> 'a
-  val fatal : ('a, out_channel, unit) format -> 'a
-  val error : ('a, out_channel, unit) format -> 'a
-  val warn  : ('a, out_channel, unit) format -> 'a
-  val info  : ('a, out_channel, unit) format -> 'a
-  val debug : ('a, out_channel, unit) format -> 'a
+open Core_kernel
+
+module CweWarning = struct
+  type t = {
+    name : string;
+    version : string;
+    addresses: string list;
+    symbols: string list;
+    other : string list list;
+    description : string;
+  } [@@deriving yojson]
 end
 
-module type SECTION = sig
-  val section: string
+module CweWarningResult = struct
+  type t = {
+      binary : string;
+      time : float;
+      warnings : CweWarning.t list;
+    } [@@deriving yojson]
 end
 
-module Make (S: SECTION) = struct
-
-  let () =
-    if S.section <> "" then
-      section_width := max (String.length S.section) !section_width
-
-  let timestamp_str lvl =
-    let section =
-      if !section_width = 0 then ""
-      else sprintf "%-*s " !section_width S.section
-    in
-    let ts = Unix.gettimeofday() in
-    let tm = Unix.localtime ts in
-    let us, _s = modf ts in
-    (* example: "2012-01-13 18:26:52.091" *)
-    sprintf "%04d-%02d-%02d %02d:%02d:%02d.%03d %s%s%s: "
-      (1900 + tm.Unix.tm_year)
-      (1    + tm.Unix.tm_mon)
-      tm.Unix.tm_mday
-      tm.Unix.tm_hour
-      tm.Unix.tm_min
-      tm.Unix.tm_sec
-      (int_of_float (1_000. *. us))
-      section
-      (level_to_string lvl)
-      !prefix
-
-  (* example for a shorter timestamp string *)
-  let _short_timestamp_str lvl =
-    sprintf "%.3f %s: " (Unix.gettimeofday()) (string_of_level lvl)
+let cwe_warning_store = ref [||]
+
+let cwe_warning_factory name version ?(other = []) ?(addresses = []) ?(symbols = []) description =
+  {
+    CweWarning.name = name;
+    CweWarning.version = version;
+    CweWarning.description = description;
+    CweWarning.other = other;
+    CweWarning.addresses = addresses;
+    CweWarning.symbols = symbols;
+  }
+
+let collect_cwe_warning warning = cwe_warning_store := Array.append !cwe_warning_store [|warning|]
+
+let emit_cwe_warnings_json target_path out_path =
+  let cwe_warning_result = {
+      CweWarningResult.binary = target_path;
+      CweWarningResult.time = Unix.time ();
+      CweWarningResult.warnings = Array.to_list !cwe_warning_store
+    } in
+  let output = Yojson.Safe.pretty_to_string (CweWarningResult.to_yojson cwe_warning_result) in
+  if out_path = "" then
+    print_endline output
+  else
+    Out_channel.write_all out_path ~data:output
 
-  let log lvl fmt =
-    if int_of_level lvl >= int_of_level !level then
-      let now = timestamp_str lvl in
-      fprintf !output ("%s" ^^ fmt ^^ "\n%!") now
-    else
-      ifprintf !output fmt
+let emit_cwe_warnings_native out_path =
+  let output_lines = Array.map !cwe_warning_store ~f:(fun (cwe_warning:CweWarning.t) ->
+      sprintf "[%s] (%s) %s" cwe_warning.name cwe_warning.version cwe_warning.description) in
+  if out_path = "" then
+    Array.iter output_lines ~f:print_endline
+  else
+    Out_channel.write_lines out_path (Array.to_list output_lines)
 
-  let fatal fmt = log FATAL fmt
-  let error fmt = log ERROR fmt
-  let warn  fmt = log WARN  fmt
-  let info  fmt = log INFO  fmt
-  let debug fmt = log DEBUG fmt
+let debug message = print_endline ("DEBUG: " ^ message)
 
-end
+let info message = print_endline ("INFO: " ^ message)
 
-include Make (struct
-    let section = ""
-  end)
+let error message = print_endline ("ERROR: " ^ message)

src/utils/log_utils.mli

-(* Copyright (c) 2014, INRIA.
- * Copyright (c) 2013, Zhang Initiative Research Unit,
- * Advance Science Institute, RIKEN
- * 2-1 Hirosawa, Wako, Saitama 351-0198, Japan
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- *
- * Redistributions of source code must retain the above copyright notice,
- * this list of conditions and the following disclaimer.
- * Redistributions in binary form must reproduce the above copyright notice,
- * this list of conditions and the following disclaimer in the documentation
- * and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
- * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
- * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
- * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
- * HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
- * TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
- * PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
- * LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
- * NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
- * SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. *)
-
-(** {2 Logger} *)
-
-(** {4 Log levels} *)
-
-type log_level = FATAL | ERROR | WARN | INFO | DEBUG
-
-val string_of_level : log_level -> string
-val level_of_string : string -> log_level
-
-(** {4 Setup} *)
-
-val set_log_level : log_level -> unit
-val get_log_level : unit -> log_level
-val set_output : out_channel -> unit
-val set_prefix : string -> unit
-val clear_prefix : unit -> unit
-
-(** {4 Printf-like logging primitives} *)
-
-module type S = sig
-
-  val log: log_level -> ('a, out_channel, unit, unit) format4 -> 'a
-
-  val fatal : ('a, out_channel, unit) format -> 'a
-  val error : ('a, out_channel, unit) format -> 'a
-  val warn  : ('a, out_channel, unit) format -> 'a
-  val info  : ('a, out_channel, unit) format -> 'a
-  val debug : ('a, out_channel, unit) format -> 'a
-
+(** This module implements the logging logic or cwe_checker.
+
+    Each check may produce a CweWarning, which holds information regarding the current CWE hit.
+    These CweWarnings are stored globally so that we can output at the very end. This may be
+    necessary, for instance, when the output should be a JSON document.
+
+    CWE checks can utilize the function cwe_warning_factory to create CweWarning objects and
+    the function collect_cwe_warning to store them globally.
+
+    At the moment, cwe_checker supports plain text and JSON output. The corresponding functions
+    are emit_cwe_warnings_native and emit_cwe_warnings_json.
+
+    In addition, there are several functions (debug, error, info) to notify the user of certain
+    events. Note that these functions may pollute the output. 
+ *)
+
+module CweWarning : sig
+  type t = {
+  name : string;
+  version : string;
+  addresses: string list;
+  symbols: string list;
+  other : string list list;
+  description : string;
+}
 end
 
-include S
-
-(** {4 Coloring of log levels (optional)} *)
-
-type color = Black | Red | Green | Yellow | Blue | Magenta | Cyan | White
-           | Default
-
-val color_on  : unit -> unit
-val color_off : unit -> unit
-val set_color_mapping : (log_level -> color) -> unit
-
-(** {4 Functor interface (optional)} *)
+val cwe_warning_factory : string -> string -> ?other:string list list -> ?addresses:string list -> ?symbols:string list -> string -> CweWarning.t
+val collect_cwe_warning : CweWarning.t -> unit
 
-module type SECTION = sig
-
-  (** Signature for the functor parameters. *)
-
-  val section: string
-  (** Section name. *)
-
-end
+val emit_cwe_warnings_json : string -> string -> unit
+val emit_cwe_warnings_native : string -> unit
 
-module Make (Section: SECTION): S
-(**
-   This module aims to be used on the first line of each module:
-   module Log = Log.Make(struct let section = "module-name" end)
-*)
+val debug : string -> unit
+val error : string -> unit
+val info : string -> unit

test/acceptance/test_file_output.py

+import os
+import subprocess
+import json
+import unittest
+
+class TestFileOutput(unittest.TestCase):
+
+    def setUp(self):
+        self.res_file = '/tmp/res.json'
+        self.cmd = 'bap test/artificial_samples/build/cwe_190_x64.out --pass=cwe-checker --cwe-checker-config=src/config.json --cwe-checker-json --cwe-checker-out=%s' % self.res_file
+
+    def test_can_output_file(self):
+        if 'travis' in os.environ['USER']:
+            self.skipTest('Travis detected: can not create files on Travis!')
+
+        subprocess.check_output(self.cmd.split())
+        with open(self.res_file) as f:
+            j = json.load(f)
+        os.remove(self.res_file)
+        self.assertTrue('warnings' in j)

test/acceptance/test_json.py

+import os
+import subprocess
+import json
+import unittest
+
+class TestJson(unittest.TestCase):
+
+    def setUp(self):
+        if 'travis' in os.environ['USER']:
+            abs_path = os.path.abspath('test/artificial_samples/build/cwe_190_x64.out')
+            self.cmd = 'docker run --rm -v %s:/tmp/input cwe-checker:latest bap /tmp/input --pass=cwe-checker  --cwe-checker-config=/home/bap/cwe_checker/src/config.json --cwe-checker-json' % abs_path
+        else:
+            self.cmd = 'bap test/artificial_samples/build/cwe_190_x64.out --pass=cwe-checker --cwe-checker-config=src/config.json --cwe-checker-json'
+
+    def test_can_output_json(self):
+        output = subprocess.check_output(self.cmd.split())
+        j = json.loads(output)
+        self.assertTrue('warnings' in j)