basic scan functionality

.project

+<?xml version="1.0" encoding="UTF-8"?>
+<projectDescription>
+	<name>common_helper_yara</name>
+	<comment></comment>
+	<projects>
+	</projects>
+	<buildSpec>
+		<buildCommand>
+			<name>org.python.pydev.PyDevBuilder</name>
+			<arguments>
+			</arguments>
+		</buildCommand>
+	</buildSpec>
+	<natures>
+		<nature>org.python.pydev.pythonNature</nature>
+	</natures>
+</projectDescription>

.pydevproject

+<?xml version="1.0" encoding="UTF-8" standalone="no"?>
+<?eclipse-pydev version="1.0"?><pydev_project>
+<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH">
+<path>/${PROJECT_DIR_NAME}</path>
+</pydev_pathproperty>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 3.0</pydev_property>
+<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">python3</pydev_property>
+</pydev_project>

LICENSE

+MIT License
+
+Copyright (c) 2017 Fraunhofer FKIE
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
\ No newline at end of file

README.md

 # Common Helper Yara
 
 Yara command line binding
+
+## Requirements
+* [YARA >= 3.6](](https://virustotal.github.io/yara/)

common_helper_yara/__init__.py

+from .yara_scan import scan
+
+__all__ = [
+    'scan'
+    ]

common_helper_yara/yara_scan.py

+import subprocess
+import sys
+import re
+import json
+import logging
+
+
+def scan(signature_path, file_path, external_variables={}):
+    '''
+    Scan files and return matches
+
+    :param signature_path: path to signature file
+    :type signature_path: string
+    :param file_path: files to scan
+    :type file_path: string
+    :return: dict
+    '''
+    variables = _convert_external_variables(external_variables)
+    with subprocess.Popen('yara {} --print-meta --print-strings {} {}'.format(variables, signature_path, file_path), shell=True, stdout=subprocess.PIPE) as process:
+        output = process.stdout.read().decode()
+    try:
+        return _parse_yara_output(output)
+    except Exception as e:
+        logging.error('Could not parse yara result: {} {}'.format(sys.exc_info()[0].__name__, e))
+        return {}
+
+
+def _convert_external_variables(ext_var_dict):
+    output = []
+    for ext_var in ext_var_dict:
+        output.append('-d {}={}'.format(ext_var, ext_var_dict[ext_var]))
+    return " ".join(sorted(output))
+
+
+def _parse_yara_output(output):
+    resulting_matches = dict()
+
+    match_blocks, rules = _split_output_in_rules_and_matches(output)
+
+    matches_regex = re.compile(r'((0x[a-f0-9]*):(\S+):\s(.+))+')
+    for index, rule in enumerate(rules):
+        for match in matches_regex.findall(match_blocks[index]):
+            _append_match_to_result(match, resulting_matches, rule)
+
+    return resulting_matches
+
+
+def _split_output_in_rules_and_matches(output):
+    split_regex = re.compile(r'\n*.*\[.*\]\s\/.+\n*')
+    match_blocks = split_regex.split(output)
+    while '' in match_blocks:
+        match_blocks.remove('')
+
+    rule_regex = re.compile(r'(.*)\s\[(.*)\]\s([\.\.\/]|[\/]|[\.\/])(.+)')
+    rules = rule_regex.findall(output)
+
+    assert len(match_blocks) == len(rules)
+    return match_blocks, rules
+
+
+def _append_match_to_result(match, resulting_matches, rule):
+    assert len(rule) == 4
+    rule_name, meta_string, _, _ = rule
+    assert len(match) == 4
+    _, offset, matched_tag, matched_string = match
+
+    meta_dict = _parse_meta_data(meta_string)
+
+    this_match = resulting_matches[rule_name] if rule_name in resulting_matches else dict(rule=rule_name, matches=True, strings=list(), meta=meta_dict)
+
+    this_match['strings'].append((int(offset, 16), matched_tag, matched_string.encode()))
+    resulting_matches[rule_name] = this_match
+
+
+def _parse_meta_data(meta_data_string):
+    '''
+    Will be of form 'item0=lowercaseboolean0,item1="value1",item2=value2,..'
+    '''
+    meta_data = dict()
+    for item in meta_data_string.split(','):
+        if '=' in item:
+            key, value = item.split('=', maxsplit=1)
+            value = json.loads(value) if value in ['true', 'false'] else value.strip('\"')
+            meta_data[key] = value
+        else:
+            logging.warning('Malformed meta string \'{}\''.format(meta_data_string))
+    return meta_data

setup.py

+from setuptools import setup, find_packages
+
+VERSION = "0.1"
+
+setup(
+    name="common_helper_yara",
+    version=VERSION,
+    packages=find_packages(),
+    install_requires=[
+        'yara-python >= 3.6'
+    ],
+    description="Yara command line binding",
+    author="Fraunhofer FKIE",
+    author_email="peter.weidenbach@fkie.fraunhofer.de",
+    url="http://www.fkie.fraunhofer.de",
+    license="MIT License"
+)

tests/data/scan_file

+foo bartest
+lighttpd-1.2.2
+

tests/data/signatures.yara

+rule lighttpd
+{
+	meta:
+		software_name = "lighttpd"
+		open_source = true
+		website = "https://www.lighttpd.net/"
+		description = "Lighttpd is a web-server optimized for low memory and cpu usage."
+	strings:
+		$a = /lighttpd-\d+\.\d+\.\d+/ nocase ascii wide
+	condition:
+		$a
+}
+
+rule another_test_rule
+{
+	meta:
+		description = "test rule"
+	strings:
+		$a = "test"
+	condition:
+		$a
+}
+
+rule non_matching_rule
+{
+	meta:
+		description = "non-matching test rule"
+	strings:
+		$a = "non-matching"
+	condition:
+		$a
+}
\ No newline at end of file

tests/data/signatures_ext_var.yara

+import "magic"
+
+rule lighttpd
+{
+	meta:
+		software_name = "lighttpd"
+		open_source = true
+		website = "https://www.lighttpd.net/"
+		description = "Lighttpd is a web-server optimized for low memory and cpu usage."
+	strings:
+		$a = /lighttpd-\d+\.\d+\.\d+/ nocase ascii wide
+	condition:
+		$a and ( magic.mime_type() != "text/plain" or test_flag )
+}
\ No newline at end of file

tests/data/yara_matches

+VersionStrings [description="Version String Finder"] /yara_test_file
+0x0:$a: blah test 1.2.3b
+0x1:$a: lah test 1.2.3b
+0x2:$a: ah test 1.2.3b
+0x3:$a: h test 1.2.3b
+0x5:$a: test 1.2.3b
+0x6:$a: est 1.2.3b
+0x7:$a: st 1.2.3b
+0x14:$a: TeST 3.4.2f
+0x15:$a: eST 3.4.2f
+0x16:$a: ST 3.4.2f
+testRule [software_name="Test Software",open_source=false,website="http://www.fkie.fraunhofer.de",description="Generic Software"] /yara_test_file
+0x5:$a: test 1.2.3
+0x14:$a: TeST 3.4.2
+VersionStrings [description="Version String Finder"] /Firmware/FmpUpdateWrapper.efi
+0x54c8:$a: L\x00i\x00n\x00k\x00 \x00w\x00i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54ca:$a: i\x00n\x00k\x00 \x00w\x00i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54cc:$a: n\x00k\x00 \x00w\x00i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54ce:$a: k\x00 \x00w\x00i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54d2:$a: w\x00i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54d4:$a: i\x00t\x00h\x00 \x001\x00.\x005\x00
+0x54d6:$a: t\x00h\x00 \x001\x00.\x005\x00
+r_libjpeg8_8d12b1_0 [package="libjpeg8",version="8d1-2+b1",filename="libjpeg.0"] /Firmware/drone/p3x_1320/P3X_FW_V01.03.0020.bin
+0x4bcb39a:$a: 31 32 00 43 61 75 74 69
+0x4ba015b:$b: 66 64 63 74 5F 31 36 78
+0x4ba016b:$b: 66 64 63 74 5F 31 36 78
+0x4ba035f:$c: 72 33 00 6A 70 65 67 5F
+r_fwupdate_054_0 [package="fwupdate",version="0.5-4",filename="fwupdate.fwupdate"] /assic/fwupdate
+0x2c0:$a: 8A 8B B0 69 BB E3 92 7C
+0x1c13:$b: 6C 79 20 66 69 72 6D 77
+0x858:$c: 78 74 00 66 77 75 70 5F
+genericPublicKey [author="Joerg Stucke",description="Generic Public Key Block",date="2017-03-16",version="2",version_schema_information="Version number is increased whenever something changes."] /crypto_material/generic_public_key
+0x0:$start_string: -----BEGIN PUBLIC KEY-----
+0xf7:$end_string: -----END PUBLIC KEY-----
+PgpPublicKeyBlock [author="Raphael Ernst",description="Find PGP Public key",date="2015-11-27",version="1",version_schema_information="Version number is increased whenever something changes."] /crypto_material/FP_test
+0x0:$start_string: -----BEGIN PGP PUBLIC KEY BLOCK-----
+0x7a1:$start_string: -----BEGIN PGP PUBLIC KEY BLOCK-----
+0x49:$end_string: -----END PGP PUBLIC KEY BLOCK-----
+0x7ea:$end_string: -----END PGP PUBLIC KEY BLOCK-----
+wareanalyseframework/src/test_unittests/data/crypto_material/0x6C2DF2C5-pub.asc
+0x0:$start_string: -----BEGIN PGP PUBLIC KEY BLOCK-----
+0x2d49:$end_string: -----END PGP PUBLIC KEY BLOCK-----
+PgpPublicKeyBlock_GnuPG [author="Raphael Ernst",description="Find PGP Public key from GnuPG",date="2015-11-27",version="1",version_schema_information="Version number is increased whenever something changes."] /crypto_material/0x6C2DF2C5-pub.asc
+0x0:$start_string: -----BEGIN PGP PUBLIC KEY BLOCK-----
+0x2d49:$end_string: -----END PGP PUBLIC KEY BLOCK-----
+0x25:$gnupg_version_string: Version: GnuPG

tests/test_scan.py

+import os
+import unittest
+
+from common_helper_yara.yara_scan import _parse_yara_output, _convert_external_variables, scan
+
+
+DIR_OF_CURRENT_FILE = os.path.dirname(os.path.abspath(__file__))
+
+
+class TestYaraScan(unittest.TestCase):
+
+    def test_parse_yara_output(self):
+        with open(os.path.join(DIR_OF_CURRENT_FILE, 'data', 'yara_matches'), 'r') as fd:
+            match_file = fd.read()
+        matches = _parse_yara_output(match_file)
+
+        self.assertIsInstance(matches, dict, 'matches should be dict')
+        self.assertIn('PgpPublicKeyBlock', matches.keys(), 'Pgp block should have been matched')
+        self.assertIn(0, matches['PgpPublicKeyBlock']['strings'][0], 'first block should start at 0x0')
+
+    def test_convert_external_variables(self):
+        self.assertEqual(_convert_external_variables({'a': 'b'}), '-d a=b', 'converted output not correct')
+        self.assertEqual(_convert_external_variables({'a': 1, 'b': 'c'}), '-d a=1 -d b=c', 'converted output not correct')
+
+    def test_scan(self):
+        signature_file = os.path.join(DIR_OF_CURRENT_FILE, 'data', 'signatures.yara')
+        scan_file = os.path.join(DIR_OF_CURRENT_FILE, 'data', 'scan_file')
+
+        result = scan(signature_file, scan_file)
+        self.assertIsInstance(result, dict, "result is not a dict")
+        self.assertEqual(len(result), 2, "number of matches not correct")
+        self.assertEqual(result['another_test_rule']['meta']['description'], 'test rule', 'meta data not correct')
+
+    def test_scan_ext_variable_and_magic(self):
+        signature_file = os.path.join(DIR_OF_CURRENT_FILE, 'data', 'signatures_ext_var.yara')
+        scan_file = os.path.join(DIR_OF_CURRENT_FILE, 'data', 'scan_file')
+
+        result = scan(signature_file, scan_file, external_variables={'test_flag': "true"})
+        self.assertEqual(len(result), 1, "number of results not correct")
+
+        result = scan(signature_file, scan_file, external_variables={'test_flag': "false"})
+        self.assertEqual(len(result), 0, "number of results not correct")