Switch default backend to Ghidra (#128)

.github/workflows/acceptance-tests.yml

@@ -39,3 +39,13 @@ jobs:
         with:
           command: test
           args: --no-fail-fast -p acceptance_tests_ghidra -- --show-output --ignored --test-threads 1
+
+  docker-build:
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@v2
+      - name: Build the docker image
+        run: docker build -t cwe_checker .
+      - name: Check functionality of the image
+        run: docker run --rm cwe_checker /bin/echo | grep -q CWE676
\ No newline at end of file

CHANGES.md

+0.5-dev
+====
+
+-   Switched default backend to Ghidra (PR #128)
+
 0.4 (2021-01)
 ====
 

Dockerfile

-FROM fkiecad/cwe_checker_travis_docker_image:bap
+FROM rust:1.49 AS builder
 
-COPY . /home/cwe/cwe_checker/
+WORKDIR /cwe_checker
 
-RUN sudo chown -R cwe:cwe /home/cwe/cwe_checker \
-    && cd /home/cwe/cwe_checker \
-    && make all \
-    && cargo clean \
-    && dune clean
+COPY . .
+RUN cargo build --release
 
-WORKDIR /home/cwe/cwe_checker
+FROM phusion/baseimage:18.04-1.0.0 as runtime
 
-ENTRYPOINT ["opam", "config", "exec", "--"]
-CMD cwe_checker /tmp/input
+RUN apt-get -y update \
+    && install_clean sudo \
+    && useradd -m cwe \
+    && echo "cwe:cwe" | chpasswd \
+    && adduser cwe sudo \
+    && sed -i.bkp -e 's/%sudo\s\+ALL=(ALL\(:ALL\)\?)\s\+ALL/%sudo ALL=NOPASSWD:ALL/g' /etc/sudoers
+USER cwe
+
+WORKDIR /home/cwe
+
+ENV PATH="/home/cwe/.cargo/bin/:${PATH}"
+ENV GHIDRA_VERSION="9.2.1_PUBLIC"
+
+# Install Ghidra
+RUN sudo apt-get -y update \
+    && sudo install_clean \
+        curl \
+        unzip \
+        openjdk-11-jdk \
+    && curl -fSL https://www.ghidra-sre.org/ghidra_9.2.1_PUBLIC_20201215.zip -o ghidra.zip \
+    && unzip -q ghidra.zip \
+    && sudo mv ghidra_${GHIDRA_VERSION} /opt/ghidra \
+    && rm ghidra.zip
+
+# Install all necessary files from the builder stage
+COPY --from=builder /cwe_checker/target/release/cwe_checker /home/cwe/cwe_checker
+COPY --from=builder /cwe_checker/src/config.json /home/cwe/.config/cwe_checker/config.json
+COPY --from=builder /cwe_checker/ghidra/p_code_extractor /home/cwe/.local/share/cwe_checker/ghidra/p_code_extractor
+RUN echo "{ \"ghidra_path\": \"/opt/ghidra\" }" | sudo tee /home/cwe/.config/cwe_checker/ghidra.json
+
+WORKDIR /
+
+ENTRYPOINT ["/home/cwe/cwe_checker"]

Makefile

@@ -3,14 +3,22 @@ GHIDRA_PATH =
 .PHONY: all clean test uninstall docker
 all:
 	cargo build --release
+ifdef GHIDRA_PATH
 	mkdir -p ${HOME}/.config/cwe_checker
 	cp src/config.json ${HOME}/.config/cwe_checker/config.json
-ifdef GHIDRA_PATH
 	cargo install --path caller --locked
 	echo "{ \"ghidra_path\": \"${GHIDRA_PATH}\" }" > ${HOME}/.config/cwe_checker/ghidra.json
 	mkdir -p ${HOME}/.local/share/cwe_checker
 	cp -r ghidra ${HOME}/.local/share/cwe_checker/ghidra
 else
+	echo "GHIDRA_PATH not specified. Please set it to the path to your local Ghidra installation."
+	false
+endif
+
+with_bap_backend:
+	cargo build --release
+	mkdir -p ${HOME}/.config/cwe_checker
+	cp src/config.json ${HOME}/.config/cwe_checker/config.json
 	cp target/release/libcwe_checker_rs.a src/libcwe_checker_rs.a
 	cp target/release/libcwe_checker_rs.so src/dllcwe_checker_rs.so
 	dune build
@@ -20,7 +28,6 @@ else
 	cd plugins/cwe_checker_type_inference && make all
 	cd plugins/cwe_checker_type_inference_print && make all
 	cd plugins/cwe_checker_pointer_inference_debug && make all
-endif
 
 test:
 	cargo test
@@ -69,8 +76,8 @@ uninstall:
 	cd plugins/cwe_checker_pointer_inference_debug; make uninstall; cd ../..
 
 documentation:
-	dune build @doc
-	cp -r _build/default/_doc/_html doc/html
+	cargo doc --open
 
 docker:
-	./install.sh
+	make clean
+	docker build -t cwe_checker .

README.md

@@ -8,19 +8,34 @@
 ![Docker-Pulls](https://img.shields.io/docker/pulls/fkiecad/cwe_checker.svg)
 [![Documentation](https://img.shields.io/badge/doc-stable-green.svg)](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html)
 
+**NOTE:** We recently changed our default analysis backend from BAP to the newer Ghidra backend.
+The switch causes some changes in both the command line interface and the docker image interface.
+Be sure to update your scripts accordingly!
+Alternatively, the stable version still uses the old interface.
+
 ## What is cwe_checker? ##
-*cwe_checker* is a suite of tools to detect common bug classes such as use of dangerous functions and simple integer overflows. These bug classes are formally known as [Common Weakness Enumerations](https://cwe.mitre.org/) (CWEs). Its main goal is to aid analysts to quickly find vulnerable code paths.
+*cwe_checker* is a suite of checks to detect common bug classes such as use of dangerous functions and simple integer overflows.
+These bug classes are formally known as [Common Weakness Enumerations](https://cwe.mitre.org/) (CWEs).
+Its main goal is to aid analysts to quickly find vulnerable code paths.
 
-Its main focus are ELF binaries that are commonly found on Linux and Unix operating systems. *cwe_checker* is built on top of [BAP](https://github.com/BinaryAnalysisPlatform/bap) (Binary Analysis Platform). By using BAP, we are not restricted to one low level instruction set architectures like Intel x86. BAP lifts several of them to one common intermediate representation (IR). cwe_checker implements its analyses on this IR. At time of writing, BAP 2.2 supports Intel x86/x64, ARM, MIPS, and PPC amongst others. Hence, this makes *cwe_checker* a valuable tool for firmware analysis.
+Its main focus are ELF binaries that are commonly found on Linux and Unix operating systems.
+The *cwe_checker* uses [Ghidra](https://ghidra-sre.org/) to disassemble binaries into one common intermediate representation
+and implements its own analyses on this IR.
+Hence, the analyses can be run on all CPU architectures that Ghidra can disassemble,
+which makes the *cwe_checker* a valuable tool for firmware analysis.
 
 The following arguments should convince you to give *cwe_checker* a try:
 -  it is very easy to set up, just build the Docker container!
 -  it analyzes ELF binaries of several CPU architectures including x86, ARM, MIPS, and PPC
 -  it is extensible due to its plugin-based architecture
 -  it is configureable, e.g. apply analyses to new APIs
--  view results annotated in IDA Pro and Ghidra
+-  view results annotated in Ghidra
 -  *cwe_checker* can be integrated as a plugin into [FACT](https://github.com/fkie-cad/FACT_core)
 
+<p align="center">
+    <img src="doc/images/example_usage.png" alt="Usage Example" width="80%" height="80%"/>
+</p>
+
 ## Installation ##
 
 ### Using the docker image ###
@@ -31,9 +46,17 @@ The simplest way is to pull the latest Docker image from [dockerhub](https://hub
 
 If you want to build the docker image yourself, just run `docker build -t cwe_checker .`
 
+### Local installation ###
+
+The following dependencies must be installed in order to build and install the *cwe_checker* locally:
+-   [Rust](https://www.rust-lang.org) >= 1.49
+-   [Ghidra](https://ghidra-sre.org/) >= 9.2
+
+Run `make all GHIDRA_PATH=path/to/ghidra_folder` (with the correct path to the local Ghidra installation inserted) to compile and install the *cwe_checker*.
+
 ### Local installation with BAP as backend ###
 
-If you plan to develop *cwe_checker*, it is recommended to build it using the provided `Makefile`. In this case you must ensure that all dependencies are fulfilled:
+If you want to use the older [BAP](https://github.com/BinaryAnalysisPlatform/bap) backend instead of Ghidra, you must ensure that the following dependencies are fulfilled:
 -   Ocaml 4.08.0
 -   Opam 2.0.2
 -   dune >= 2.0
@@ -45,82 +68,85 @@ If you plan to develop *cwe_checker*, it is recommended to build it using the pr
 -   pytest >= 3.5.1 (for tests)
 -   SCons >= 3.0.5 (for tests)
 -   odoc >= 1.4 (for documentation)
--   [Rust](https://www.rust-lang.org) >= 1.44.1
+-   [Rust](https://www.rust-lang.org) >= 1.49
 
-Just run `make all` to compile and register the plugin with BAP. You can run the test suite via `make test`. Documentation can be built via `make documentation`.
-
-### Local installation with Ghidra as backend ###
-
-The Ghidra backend is still in early development, thus some checks are not yet available for it. To try it out, the following dependencies must be fulfilled:
--   [Rust](https://www.rust-lang.org) >= 1.44.1
--   [Ghidra](https://ghidra-sre.org/) >= 9.2. If you want to use an earlier version of Ghidra, you need to manually add the Java library `gson` to Ghidra: Download it from https://search.maven.org/artifact/com.google.code.gson/gson/2.8.6/jar and move it to the Ghidra plugin folder located at `$HOME/.ghidra/.ghidra_9.X.X_PUBLIC/plugins` (with the version number depending on your version of Ghidra).
-
-Run `make all GHIDRA_PATH=path/to/ghidra_folder` (with the correct path to the local Ghidra installation inserted) to compile and install the *cwe_checker*.
+Just run `make with_bap_backend` to compile and register the plugin with BAP.
 
 ## Usage ##
 
-The *cwe_checker* takes as input a binary file, runs several [checks](#checks) based on static analysis on the binary and then outputs a list of CWE warnings that have been found during the analysis.
+The *cwe_checker* takes a binary as input,
+runs several [checks](#checks) based on static analysis on the binary
+and then outputs a list of CWE warnings that have been found during the analysis.
 
 If you use the official docker image, just run
 ```bash
-docker run --rm -v /PATH/TO/BINARY:/tmp/input fkiecad/cwe_checker cwe_checker /tmp/input
+docker run --rm -v /PATH/TO/BINARY:/input fkiecad/cwe_checker /input
 ```
-If you installed the *cwe_checker* locally (e.g. via the Opam package), run
+If you installed the *cwe_checker* locally, run
 ```bash
 cwe_checker BINARY
 ```
-You can adjust the behavior of most checks via a configuration file located at `src/config.json`. If you modify it, add the command line flag `-config=src/config.json` to tell the *cwe_checker* to use the modified file.
+You can adjust the behavior of most checks via a configuration file located at `src/config.json`.
+If you modify it, add the command line flag `--config=src/config.json` to tell the *cwe_checker* to use the modified file.
+For information about other available command line flags you can pass the `--help` flag to the *cwe_checker*.
+
+If you use the stable version, you can also look at the [online documentation](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html) for more information.
+
+## Documentation and Tests ##
+
+The test binaries for our test suite can be built with `make compile_test_files` (needs Docker to be installed!). The test suite can then be run with `make test`.
 
-For more information on usage instructions and available command line flags, see the [online documentation](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html). You can also build the documentation locally via `make documentation` and then browse it in the *doc/html/* folder.
+Source code documentation can be built with `make documentation`. For the stable version, the documentation can be found [here](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html).
 
 ## Implemented Checks <a name=checks></a> ##
+
 So far the following analyses are implemented:
 -   [CWE-125](https://cwe.mitre.org/data/definitions/125.html): Out-of-bounds read (via emulation)
 -   [CWE-190](https://cwe.mitre.org/data/definitions/190.html): Integer Overflow or Wraparound
 -   [CWE-215](https://cwe.mitre.org/data/definitions/215.html): Information Exposure Through Debug Information
 -   [CWE-243](https://cwe.mitre.org/data/definitions/243.html): Creation of chroot Jail Without Changing Working Directory
--   [CWE-248](https://cwe.mitre.org/data/definitions/248.html): Uncaught Exception
 -   [CWE-332](https://cwe.mitre.org/data/definitions/332.html): Insufficient Entropy in PRNG
 -   [CWE-367](https://cwe.mitre.org/data/definitions/367.html): Time-of-check Time-of-use (TOCTOU) Race Condition
--   [CWE-415](https://cwe.mitre.org/data/definitions/415.html): Double Free *(still experimental)*
--   [CWE-416](https://cwe.mitre.org/data/definitions/416.html): Use After Free *(still experimental)*
+-   [CWE-415](https://cwe.mitre.org/data/definitions/415.html): Double Free
+-   [CWE-416](https://cwe.mitre.org/data/definitions/416.html): Use After Free
 -   [CWE-426](https://cwe.mitre.org/data/definitions/426.html): Untrusted Search Path
--   [CWE-457](https://cwe.mitre.org/data/definitions/457.html): Use of Uninitialized Variable
 -   [CWE-467](https://cwe.mitre.org/data/definitions/467.html): Use of sizeof() on a Pointer Type
 -   [CWE-476](https://cwe.mitre.org/data/definitions/476.html): NULL Pointer Dereference
 -   [CWE-560](https://cwe.mitre.org/data/definitions/560.html): Use of umask() with chmod-style Argument
 -   [CWE-676](https://cwe.mitre.org/data/definitions/676.html): Use of Potentially Dangerous Function
 -   [CWE-782](https://cwe.mitre.org/data/definitions/782.html): Exposed IOCTL with Insufficient Access Control
 
-Please note that some of the above analyses only are partially implemented at the moment. Furthermore, false positives are to be expected due to shortcuts and the nature of static analysis as well as over-approximation. For more information about the individual checks you can look at the [online documentation](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html).
-
-**NOTE:** The new memory checks for CWEs 415 and 416 are still very experimental and are disabled on a standard run. You can try them out using the `-partial=Memory` command line flag.
-
-**NOTE:** We recently decided to deprecate the support for the old emulation based checks for CWEs 415, 416 and 787. In addition to trying out the new memory checks, users of these checks should also take a look at the [BAP toolkit](https://github.com/BinaryAnalysisPlatform/bap-toolkit), which provides better-maintained (and still emulation based) versions of these checks.
+Please note that some of the above analyses only are partially implemented at the moment.
+Furthermore, false positives are to be expected due to shortcuts and the nature of static analysis as well as over-approximation.
 
 ## Integration into other tools ##
-*cwe_checker* comes with scripts for IDA Pro and Ghidra, which parse the output of *cwe_checker* and annotate the found CWEs in the disassembler for easier manual analysis. See the [online documentation](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html#ToolIntegration) for their usage. The IDA Pro plugin also uses colors to represent the  severeness of the found issues (yellow, orange, or red). The following screenshot shows some results:
+
+*cwe_checker* comes with a script for Ghidra,
+which parses the output of the *cwe_checker* and annotates the found CWEs in the disassembler for easier manual analysis.
+The script is located at `ghidra_plugin/cwe_checker_ghidra_plugin.py`, usage instructions are contained in the file.
 
 <p align="center">
-    <img src="doc/images/example_ida_anotation.png" alt="IDA Pro anotation" width="50%" height="50%"/>
+    <img src="doc/images/example_ghidra_integration.png" alt="Ghidra Integration" width="90%" height="90%"/>
 </p>
 
-## How to extend cwe_checker? ##
-You can find some information about how to write your own check [here](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html#HackingHowto)
-
-If you plan to open a PR, please utilize [precommit](https://pre-commit.com) in your development environment to catch many issues before the code review.
-
-### Contribute ###
-Contributions are always welcome. Just fork it and open a pull request!
-
 ## How does cwe_checker work internally? ##
-See the [online documentation](https://fkie-cad.github.io/cwe_checker/doc/html/cwe_checker/index.html) or build it locally via `make documentation` and then browse it in the *doc/html/* folder. But the most accurate documentation is still the source code. We also provide some slides of conference presentations on cwe_checker in *doc*. These should be of special interest for those who would like to get a quick/initial overview of its internals.
 
+Building the documentation using `cargo doc --open --document-private-items` will give you more information about the internal structure of the *cwe_checker*.
+However, the best documentation is still the source code itself.
+If you have questions, be sure to ask them on our [discussions page](https://github.com/fkie-cad/cwe_checker/discussions)!
+We are constantly striving to improve extensibility and documentation and your questions will help us to achieve that!
+
+To get a quick/initial overview of its internals you can also look at the slides of conference presentations on the *cwe_checker* in the *doc* folder.
 We presented cwe_checker at the following conferences so far:
 -   [Pass The SALT 2019](https://2019.pass-the-salt.org/talks/74.html) ([slides](doc/slides/cwe_checker_pts19.pdf))
 -   [Black Hat USA 2019](https://www.blackhat.com/us-19/arsenal/schedule/index.html#cwe_checker-hunting-binary-code-vulnerabilities-across-cpu-architectures-16782) ([slides](doc/slides/cwe_checker_BlackHatUSA2019.pdf))
 
+### Contribute ###
+
+Contributions are always welcome. Just fork it and open a pull request!
+
 ## Acknowledgements ##
+
 This project is partly financed by [German Federal Office for Information Security (BSI)](https://www.bsi.bund.de).
 
 A special thanks goes out to the BAP community (especially the official gitter) for answering questions and discussing solutions.

ghidra.Dockerfile → bap.Dockerfile

-FROM fkiecad/cwe_checker_travis_docker_image:ghidra
+FROM fkiecad/cwe_checker_travis_docker_image:bap
 
 COPY . /home/cwe/cwe_checker/
 
 RUN sudo chown -R cwe:cwe /home/cwe/cwe_checker \
     && cd /home/cwe/cwe_checker \
-    && make all GHIDRA_PATH=/home/cwe/ghidra \
-    && cargo clean
+    && make with_bap_backend \
+    && cargo clean \
+    && dune clean
 
 WORKDIR /home/cwe/cwe_checker
 
-# ENTRYPOINT ["/bin/sh", "-c"]
+ENTRYPOINT ["opam", "config", "exec", "--"]
 CMD cwe_checker /tmp/input

caller/Cargo.toml

 [package]
 name = "cwe_checker"
-version = "0.4.0"
+version = "0.5.0-dev"
 authors = ["Enkelmann <nils-edvin.enkelmann@fkie.fraunhofer.de>"]
 edition = "2018"
 

caller/src/main.rs

@@ -34,10 +34,6 @@ struct CmdlineArgs {
     #[structopt(long, short)]
     quiet: bool,
 
-    /// Checks if there is a path from an input function to a CWE hit.
-    #[structopt(long)]
-    check_path: bool,
-
     /// Prints out the version numbers of all known modules.
     #[structopt(long)]
     module_versions: bool,
@@ -86,9 +82,6 @@ fn build_bap_command(args: &CmdlineArgs) -> Command {
     if args.quiet {
         command.arg("--cwe-checker-no-logging");
     }
-    if args.check_path {
-        command.arg("--cwe-checker-check-path");
-    }
     if args.module_versions {
         command.arg("--cwe-checker-module-versions");
     }
@@ -119,10 +112,6 @@ fn run_with_ghidra(args: CmdlineArgs) {
         return;
     }
 
-    if args.check_path {
-        panic!("Check-path module not yet implemented for the Ghidra backend");
-    }
-
     // Get the configuration file
     let config: serde_json::Value = if let Some(config_path) = args.config {
         let file = std::io::BufReader::new(std::fs::File::open(config_path).unwrap());

cwe_checker_rs/Cargo.toml

 [package]
 name = "cwe_checker_rs"
-version = "0.4.0"
+version = "0.5.0-dev"
 authors = ["Nils-Edvin Enkelmann <nils-edvin.enkelmann@fkie.fraunhofer.de>"]
 edition = "2018"
 

ghidra_plugin/cwe_checker_ghidra_plugin.py

 # Import the results of the cwe_checker as bookmarks and comments into Ghidra.
 #
 # Usage:
-# - Run the cwe_checker on a binary and save its output as a json file.
 # - Copy this file into the Ghidra scripts folder
+# - Run the cwe_checker on a binary and save its output as a json file, e.g. with
+#   "cwe_checker BINARY --json --out output.json"
 # - Open the binary in Ghidra and run this file as a script. Select the generated json file when prompted.
 
 import json
-from ghidra.app.util.opinion import ElfLoader
 
 
 def bookmark_cwe(ghidra_address, text):
@@ -42,29 +42,12 @@ def get_cwe_checker_output():
         return json.load(json_file)
 
 
-def compute_ghidra_address(address_string):
-    fixed_address_string = address_string.replace(':32u', '').replace(':64u', '')
-    address_int = int(fixed_address_string, 16)
-    # Ghidra sometimes adds an offset to all addresses.
-    try:
-        # try for ELF-files
-        offset = currentProgram.getMinAddress().getOffset() - int(ElfLoader.getElfOriginalImageBase(currentProgram))
-        return currentProgram.getAddressFactory().getAddress(fixed_address_string).add(offset)
-    except:
-        # the file is probably not an ELF file, so we use a workaround that should work in most cases.
-        if address_int < currentProgram.getMinAddress().getOffset():
-            return currentProgram.getMinAddress().add(address_int)
-        else:
-            return currentProgram.getAddressFactory().getAddress(fixed_address_string)
-
-
 def main():
     """
-    Annotate cwe_checker results (including check_path paths) in Ghidra as end-of-line
+    Annotate cwe_checker results in Ghidra as end-of-line
     comments and bookmarks to the corresponding addresses.
     """
-    cwe_checker_output = get_cwe_checker_output()
-    warnings = cwe_checker_output['warnings']
+    warnings = get_cwe_checker_output()
     for warning in warnings:
         if len(warning['addresses']) == 0:
             cwe_text =  '[' + warning['name'] + '] ' + warning['description']
@@ -72,16 +55,9 @@ def main():
             bookmark_cwe(ghidra_address, cwe_text)
             comment_cwe_pre(ghidra_address, cwe_text)
         else:
-            for address_string in warning['addresses']:
-                ghidra_address = compute_ghidra_address(address_string)
-                bookmark_cwe(ghidra_address, warning['description'])
-                comment_cwe_eol(ghidra_address, warning['description'])
-    if 'check_path' in cwe_checker_output:
-        for check_path in cwe_checker_output['check_path']:
-            ghidra_address = compute_ghidra_address(check_path['source_addr'])
-            check_path_string = 'Path to CWE at ' + check_path['destination_addr'] + ': ' + check_path['path_str']
-            bookmark_cwe(ghidra_address, check_path_string)
-            comment_cwe_eol(ghidra_address, check_path_string)
-
+            address_string = warning['addresses'][0]
+            ghidra_address = currentProgram.getAddressFactory().getAddress(address_string)
+            bookmark_cwe(ghidra_address, warning['description'])
+            comment_cwe_eol(ghidra_address, warning['description'])
 
 main()

install.sh

-#!/usr/bin/env bash
-
-echo "Cleaning up"
-make clean
-
-echo "Building docker container"
-docker build --build-arg=http{,s}_proxy --build-arg=HTTP{,S}_PROXY -t cwe-checker .
-
-exit 0