large refactoring and moving without change of functionality

pdf_generator/pdf_generator.py

+#! /usr/bin/env python3
+
+import argparse
+import logging
+import os
+import shutil
+import sys
+from pathlib import Path
+from tempfile import TemporaryDirectory
+
+from common_helper_files import create_dir_for_file
+from common_helper_process import execute_shell_command_get_return_code
+from pdf_generator.pre_processing.rest import create_request_url, request_firmware_data
+from pdf_generator.tex_generation.template_engine import create_jinja_environment
+
+PROGRAM_NAME = 'FACT PDF Report Generator'
+PROGRAM_VERSION = '0.1'
+PROGRAM_DESCRIPTION = 'Generates an analysis PDF report'
+
+
+def setup_argparser():
+    parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
+    parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
+    parser.add_argument('-l', '--log_file', help='path to log file')
+    parser.add_argument('-L', '--log_level', help='define the log level [DEBUG,INFO,WARNING,ERROR]')
+    parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
+    parser.add_argument('-u', '--uid', help='firmware analysis UID', dest="UID")
+    return parser.parse_args()
+
+
+def setup_logging(debug_flag=False):
+    log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
+    logger = logging.getLogger('')
+    logger.setLevel(logging.DEBUG)
+
+    create_dir_for_file('/tmp/pdf_generator.log')
+    file_log = logging.FileHandler('/tmp/pdf_generator.log')
+    file_log.setLevel(logging.INFO)
+    file_log.setFormatter(log_format)
+    logger.addHandler(file_log)
+
+    console_log = logging.StreamHandler()
+    console_log.setLevel(logging.DEBUG if debug_flag else logging.INFO)
+    console_log.setFormatter(log_format)
+    logger.addHandler(console_log)
+
+
+def execute_pdflatex(tmp_dir):
+    current_dir = os.getcwd()
+    os.chdir(tmp_dir)
+    logging.debug('Creating pdf file')
+    _, _ = execute_shell_command_get_return_code('env buf_size=1000000 pdflatex main.tex')
+    os.chdir(current_dir)
+
+
+def _copy_fact_image(target):
+    shutil.copy(str(Path(__file__).parent.parent / 'templates' / 'fact_logo.png'), str(Path(target) / 'fact_logo.png'))
+
+
+def main(firmware_uid):
+    request_url = create_request_url(firmware_uid)
+    try:
+        firmware_analyses, firmware_meta_data = request_firmware_data(request_url)
+    except KeyError:
+        logging.warning('No firmware found with UID {}'.format(firmware_uid))
+        return None
+
+    environment = create_jinja_environment()
+
+    with TemporaryDirectory() as tmp_dir:
+        Path(tmp_dir, 'meta.tex').write_text(generate_meta_data_code(environment=environment, meta_data=firmware_meta_data))
+
+        for filename, result_code in generate_analysis_codes(environment=environment, analysis=firmware_analyses, tmp_dir=tmp_dir):
+            Path(tmp_dir, filename).write_text(result_code)
+
+        Path(tmp_dir, 'main.tex').write_text(generate_main_code(firmware_analyses, firmware_meta_data, environment))
+
+        _copy_fact_image(tmp_dir)
+
+        execute_pdflatex(tmp_dir)
+
+        pdf_filename = create_report_filename(firmware_meta_data)
+        shutil.move(str(Path(tmp_dir, 'main.pdf')), str(Path('.', pdf_filename)))
+
+    return None
+
+
+if __name__ == '__main__':
+    args = setup_argparser()
+    setup_logging(args.debug)
+
+    main(args.UID)
+
+    sys.exit(0)

pdf_generator/tex_generation/code_generation.py

+import logging
+
+from jinja2 import TemplateNotFound
+
+GENERIC_TEMPLATE = 'generic.tex'
+
+
+def generate_meta_data_code(environment, meta_data):
+    template = environment.get_template('meta_data.tex')
+    logging.debug('Rendering meta data')
+    return template.render(meta_data=meta_data)
+
+
+def generate_main_code(firmware_analyses, firmware_meta_data, jinja_environment):
+    template = jinja_environment.get_template('main.tex')
+    logging.debug('Rendering main page')
+    return template.render(analysis=firmware_analyses, meta_data=firmware_meta_data)
+
+
+def generate_analysis_codes(environment, analysis, tmp_dir):
+    return [
+        ('{}.tex'.format(analysis_plugin), _render_analysis_result(analysis[analysis_plugin], environment, analysis_plugin, tmp_dir)) for analysis_plugin in analysis
+    ]
+
+
+def _render_analysis_result(analysis, environment, analysis_plugin, tmp_dir):
+    try:
+        template = environment.get_template('{}.tex'.format(analysis_plugin))
+    except TemplateNotFound:
+        logging.debug('Falling back on generic template for {}'.format(analysis_plugin))
+        template = environment.get_template(GENERIC_TEMPLATE)
+
+    logging.debug('Rendering {}'.format(analysis_plugin))
+    return template.render(selected_analysis=analysis, tmp_dir=tmp_dir)
+
+
+def create_report_filename(meta_data):
+    main_tex_filename = meta_data['device_name'] + '_analysis_report.pdf'
+    main_tex_filename = main_tex_filename.replace(' ', '_')
+    return main_tex_filename.replace('/', '__')

src/jinja_filters/filter.py → pdf_generator/tex_generation/template_engine.py

@@ -2,6 +2,7 @@ from base64 import decodebytes
 from pathlib import Path
 from time import localtime, strftime
 
+import jinja2
 from common_helper_files import human_readable_file_size
 
 
@@ -116,4 +117,36 @@ def split_output_lines(output_value):
         if line_length > 92:
             line = line[:92] + ' ' + line[92:]
         output += line + '\n'
-    return output
\ No newline at end of file
+    return output
+
+
+def create_jinja_environment(templates_to_use='default'):
+    template_directory = Path(Path(__file__).parent.parent, 'templates', templates_to_use)
+    environment = jinja2.Environment(
+        block_start_string=r'\BLOCK{',
+        block_end_string='}',
+        variable_start_string=r'\VAR{',
+        variable_end_string='}',
+        comment_start_string=r'\#{',
+        comment_end_string='}',
+        line_statement_prefix='%%',
+        line_comment_prefix='%#',
+        trim_blocks=True,
+        autoescape=False,
+        loader=jinja2.FileSystemLoader(str(template_directory))
+    )
+    _add_filters_to_jinja(environment)
+    return environment
+
+
+def _add_filters_to_jinja(environment):
+    environment.filters['number_format'] = byte_number_filter
+    environment.filters['nice_unix_time'] = nice_unix_time
+    environment.filters['nice_number'] = nice_number_filter
+    environment.filters['filter_chars'] = filter_latex_special_chars
+    environment.filters['elements_count'] = count_elements_in_list
+    environment.filters['base64_to_png'] = convert_base64_to_png_filter
+    environment.filters['check_list'] = check_if_list_empty
+    environment.filters['filter_list'] = filter_chars_in_list
+    environment.filters['split_hash'] = split_hash
+    environment.filters['split_output_lines'] = split_output_lines

src/config/main.cfg

-
-[Logging]
-logFile=/tmp/fact_report.log
-logLevel=WARNING
\ No newline at end of file

src/latex_code_generation/code_generation.py

-import logging
-import os
-import shutil
-from pathlib import Path
-from tempfile import TemporaryDirectory
-
-import jinja2
-from common_helper_process import execute_shell_command_get_return_code
-from jinja_filters.filter import (
-    byte_number_filter, check_if_list_empty, convert_base64_to_png_filter, count_elements_in_list, filter_chars_in_list,
-    filter_latex_special_chars, nice_number_filter, nice_unix_time, split_hash, split_output_lines
-)
-from rest_import.rest import create_request_url, request_firmware_data
-
-GENERIC_TEMPLATE = 'generic.tex'
-
-
-def create_jinja_environment(templates_to_use='default'):
-    template_directory = Path(Path(__file__).parent.parent, 'templates', templates_to_use)
-    environment = jinja2.Environment(
-        block_start_string=r'\BLOCK{',
-        block_end_string='}',
-        variable_start_string=r'\VAR{',
-        variable_end_string='}',
-        comment_start_string=r'\#{',
-        comment_end_string='}',
-        line_statement_prefix='%%',
-        line_comment_prefix='%#',
-        trim_blocks=True,
-        autoescape=False,
-        loader=jinja2.FileSystemLoader(str(template_directory))
-    )
-    _add_filters_to_jinja(environment)
-    return environment
-
-
-def _add_filters_to_jinja(environment):
-    environment.filters['number_format'] = byte_number_filter
-    environment.filters['nice_unix_time'] = nice_unix_time
-    environment.filters['nice_number'] = nice_number_filter
-    environment.filters['filter_chars'] = filter_latex_special_chars
-    environment.filters['elements_count'] = count_elements_in_list
-    environment.filters['base64_to_png'] = convert_base64_to_png_filter
-    environment.filters['check_list'] = check_if_list_empty
-    environment.filters['filter_list'] = filter_chars_in_list
-    environment.filters['split_hash'] = split_hash
-    environment.filters['split_output_lines'] = split_output_lines
-
-
-def generate_meta_data_code(environment, meta_data):
-    template = environment.get_template('meta_data.tex')
-    logging.debug('Rendering meta data')
-    return template.render(meta_data=meta_data)
-
-
-def generate_main_code(firmware_analyses, firmware_meta_data, jinja_environment):
-    template = jinja_environment.get_template('main.tex')
-    logging.debug('Rendering main page')
-    return template.render(analysis=firmware_analyses, meta_data=firmware_meta_data)
-
-
-def generate_analysis_codes(environment, analysis, tmp_dir):
-    return [
-        ('{}.tex'.format(analysis_plugin), _render_analysis_result(analysis[analysis_plugin], environment, analysis_plugin, tmp_dir)) for analysis_plugin in analysis
-    ]
-
-
-def _render_analysis_result(analysis, environment, analysis_plugin, tmp_dir):
-    try:
-        template = environment.get_template('{}.tex'.format(analysis_plugin))
-    except jinja2.TemplateNotFound:
-        logging.debug('Falling back on generic template for {}'.format(analysis_plugin))
-        template = environment.get_template(GENERIC_TEMPLATE)
-
-    logging.debug('Rendering {}'.format(analysis_plugin))
-    return template.render(selected_analysis=analysis, tmp_dir=tmp_dir)
-
-
-def create_report_filename(meta_data):
-    main_tex_filename = meta_data['device_name'] + '_analysis_report.pdf'
-    main_tex_filename = main_tex_filename.replace(' ', '_')
-    return main_tex_filename.replace('/', '__')
-
-
-def _copy_fact_image(target):
-    shutil.copy(str(Path(__file__).parent.parent / 'templates' / 'fact_logo.png'), str(Path(target) / 'fact_logo.png'))
-
-
-def execute_pdflatex(tmp_dir):
-    current_dir = os.getcwd()
-    os.chdir(tmp_dir)
-    logging.debug('Creating pdf file')
-    _, _ = execute_shell_command_get_return_code('env buf_size=1000000 pdflatex main.tex')
-    os.chdir(current_dir)
-
-
-def generate_pdf_report(firmware_uid):
-    request_url = create_request_url(firmware_uid)
-    try:
-        firmware_analyses, firmware_meta_data = request_firmware_data(request_url)
-    except KeyError:
-        logging.warning('No firmware found with UID {}'.format(firmware_uid))
-        return None
-
-    environment = create_jinja_environment()
-
-    with TemporaryDirectory() as tmp_dir:
-        Path(tmp_dir, 'meta.tex').write_text(generate_meta_data_code(environment=environment, meta_data=firmware_meta_data))
-
-        for filename, result_code in generate_analysis_codes(environment=environment, analysis=firmware_analyses, tmp_dir=tmp_dir):
-            Path(tmp_dir, filename).write_text(result_code)
-
-        Path(tmp_dir, 'main.tex').write_text(generate_main_code(firmware_analyses, firmware_meta_data, environment))
-
-        _copy_fact_image(tmp_dir)
-
-        execute_pdflatex(tmp_dir)
-
-        pdf_filename = create_report_filename(firmware_meta_data)
-        shutil.move(str(Path(tmp_dir, 'main.pdf')), str(Path('.', pdf_filename)))
-
-    return None

src/pdf_generator.py

-#! /usr/bin/env python3
-'''
-This Template has a separate console and file logging.
-File logging-level can be changed by -L parameter.
-Log file can be changed by -l parameter.
-Standard values for logging can be set in the config file
--> "Logging"
----> "logFile"
----> "logLevel"
-Console logging is INFO or DEBUG according to debug debug_flag.
-Config file path can be changed by -C parameter.
-'''
-
-import argparse
-from common_helper_files import create_dir_for_file
-import configparser
-import logging
-import os
-import sys
-
-from latex_code_generation.code_generation import generate_pdf_report
-
-
-PROGRAM_NAME = 'FACT PDF Report Generator'
-PROGRAM_VERSION = '0.1'
-PROGRAM_DESCRIPTION = 'Generates a analysis PDF report'
-
-STANDARD_CONF_FILE = os.path.dirname(os.path.abspath(__file__)) + '/config/main.cfg'
-
-
-def _setup_argparser():
-    parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
-    parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
-    parser.add_argument('-l', '--log_file', help='path to log file')
-    parser.add_argument('-L', '--log_level', help='define the log level [DEBUG,INFO,WARNING,ERROR]')
-    parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
-    parser.add_argument('-C', '--config_file', help='set path to config File', default=STANDARD_CONF_FILE)
-    parser.add_argument('-u', '--uid', help='firmware analysis UID', dest="UID")
-    return parser.parse_args()
-
-
-def _get_console_output_level(debug_flag):
-    if debug_flag:
-        return logging.DEBUG
-    else:
-        return logging.INFO
-
-
-def _setup_logging(config, debug_flag=False):
-    log_level = getattr(logging, config['Logging']['logLevel'], None)
-    log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
-    logger = logging.getLogger('')
-    logger.setLevel(logging.DEBUG)
-    create_dir_for_file(config['Logging']['logFile'])
-    file_log = logging.FileHandler(config['Logging']['logFile'])
-    file_log.setLevel(log_level)
-    file_log.setFormatter(log_format)
-    console_log = logging.StreamHandler()
-    console_log.setLevel(_get_console_output_level(debug_flag))
-    console_log.setFormatter(log_format)
-    logger.addHandler(file_log)
-    logger.addHandler(console_log)
-
-
-def _load_config(args):
-    config = configparser.ConfigParser()
-    config.read(args.config_file)
-    if args.log_file is not None:
-        config['Logging']['logFile'] = args.log_file
-    if args.log_level is not None:
-        config['Logging']['logLevel'] = args.log_level
-    return config
-
-
-if __name__ == '__main__':
-    args = _setup_argparser()
-    config = _load_config(args)
-    _setup_logging(config, args.debug)
-    logging.info(args.config_file)
-
-    generate_pdf_report(args.UID)
-
-    sys.exit(0)

src/test/acceptance/test_main.py → test/acceptance/test_main.py

-from common_helper_process.fail_safe_subprocess import execute_shell_command_get_return_code
-import os
-import pytest
+from pathlib import Path
 
+import pytest
+from common_helper_process.fail_safe_subprocess import execute_shell_command_get_return_code
 
-SRC_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../../pdf_generator.py'
+SCRIPT_PATH = Path(__file__).parent.parent.parent / 'pdf_generator' / 'pdf_generator.py'
 
 
 @pytest.mark.parametrize('arguments, expected_output, expected_return_code', [
@@ -11,7 +11,7 @@ SRC_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../../pdf_generator.py'
     ('-h', 'usag', 0)
 ])
 def test_main_program(arguments, expected_output, expected_return_code):
-    command_line = SRC_DIR + ' ' + arguments
+    command_line = str(SCRIPT_PATH) + ' ' + arguments
     output, return_code = execute_shell_command_get_return_code(command_line)
     assert output[0:4] == expected_output
     assert return_code == expected_return_code

src/test/data/test_dict.py → test/data/test_dict.py

-test_dict = {
+TEST_DICT = {
     "firmware": {
         "analysis": {
             "binwalk": {

src/test/integration/test_code_generation.py → test/integration/test_code_generation.py

@@ -2,10 +2,10 @@ from pathlib import Path
 
 import pytest
 from jinja2 import Environment, FileSystemLoader
-from latex_code_generation.code_generation import _add_filters_to_jinja, generate_meta_data_code
-from rest_import.rest import request_firmware_data
-
-from ..data.test_dict import test_dict
+from pdf_generator.pre_processing.rest import request_firmware_data
+from pdf_generator.tex_generation.code_generation import generate_meta_data_code
+from pdf_generator.tex_generation.template_engine import _add_filters_to_jinja
+from test.data.test_dict import TEST_DICT
 
 # pylint: disable=redefined-outer-name
 
@@ -29,19 +29,19 @@ def mock_environment():
         line_comment_prefix='%#',
         trim_blocks=True,
         autoescape=False,
-        loader=FileSystemLoader(str(Path(Path(__file__).parent.parent.parent, 'templates', 'default'))),
+        loader=FileSystemLoader(str(Path(Path(__file__).parent.parent.parent, 'pdf_generator', 'templates', 'default'))),
     )
     _add_filters_to_jinja(env)
     return env
 
 
 def test_anything_mocked(monkeypatch):
-    monkeypatch.setattr('rest_import.rest.requests.get', lambda x: MockResponse())
+    monkeypatch.setattr('pdf_generator.pre_processing.rest.requests.get', lambda x: MockResponse())
 
     anything = request_firmware_data('anything')
     assert anything
 
 
 def test_latex_code_generation(mock_environment):
-    result = generate_meta_data_code(mock_environment, test_dict)
+    result = generate_meta_data_code(mock_environment, TEST_DICT)
     assert result

src/test/unit/test_code_generation.py → test/unit/test_code_generation.py

 from tempfile import TemporaryDirectory
 
-from latex_code_generation.code_generation import _render_analysis_result, create_jinja_environment, generate_pdf_report
+from pdf_generator.pdf_generator import main
+from pdf_generator.tex_generation.code_generation import _render_analysis_result
+from pdf_generator.tex_generation.template_engine import create_jinja_environment
 
 TEST_DATA = {
     'analysis': {'file_hashes': {'ssdeep': 'bla', 'sha1': 'blah'}},
@@ -18,6 +20,6 @@ def test_render_template():
 
 
 def test_main(monkeypatch):
-    monkeypatch.setattr('latex_code_generation.code_generation.request_firmware_data', lambda *_: (TEST_DATA['analysis'], TEST_DATA['meta_data']))
-    monkeypatch.setattr('latex_code_generation.code_generation.shutil.move', lambda *_: None)
-    generate_pdf_report(firmware_uid='b79ea608e2f0390744642bad472f8d9fd7e4713791857da5d5fcabf70a009e50_29626948')
+    monkeypatch.setattr('pdf_generator.pdf_generator.request_firmware_data', lambda *_: (TEST_DATA['analysis'], TEST_DATA['meta_data']))
+    monkeypatch.setattr('pdf_generator.pdf_generator.shutil.move', lambda *_: None)
+    main(firmware_uid='b79ea608e2f0390744642bad472f8d9fd7e4713791857da5d5fcabf70a009e50_29626948')