Skip to content

F
fact_pdf_report
Commit 3b98695f by dorp
Options

large refactoring and moving without change of functionality

parent 83acc828
Showing with 187 additions and 229 deletions
pdf_generator/pdf_generator.py 0 → 100755
#! /usr/bin/env python3
import argparse
import logging
import os
import shutil
import sys
from pathlib import Path
from tempfile import TemporaryDirectory
from common_helper_files import create_dir_for_file
from common_helper_process import execute_shell_command_get_return_code
from pdf_generator.pre_processing.rest import create_request_url, request_firmware_data
from pdf_generator.tex_generation.template_engine import create_jinja_environment
PROGRAM_NAME = 'FACT PDF Report Generator'
PROGRAM_VERSION = '0.1'
PROGRAM_DESCRIPTION = 'Generates an analysis PDF report'
def setup_argparser():
parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
parser.add_argument('-l', '--log_file', help='path to log file')
parser.add_argument('-L', '--log_level', help='define the log level [DEBUG,INFO,WARNING,ERROR]')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
parser.add_argument('-u', '--uid', help='firmware analysis UID', dest="UID")
return parser.parse_args()
def setup_logging(debug_flag=False):
log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
create_dir_for_file('/tmp/pdf_generator.log')
file_log = logging.FileHandler('/tmp/pdf_generator.log')
file_log.setLevel(logging.INFO)
file_log.setFormatter(log_format)
logger.addHandler(file_log)
console_log = logging.StreamHandler()
console_log.setLevel(logging.DEBUG if debug_flag else logging.INFO)
console_log.setFormatter(log_format)
logger.addHandler(console_log)
def execute_pdflatex(tmp_dir):
current_dir = os.getcwd()
os.chdir(tmp_dir)
logging.debug('Creating pdf file')
_, _ = execute_shell_command_get_return_code('env buf_size=1000000 pdflatex main.tex')
os.chdir(current_dir)
def _copy_fact_image(target):
shutil.copy(str(Path(__file__).parent.parent / 'templates' / 'fact_logo.png'), str(Path(target) / 'fact_logo.png'))
def main(firmware_uid):
request_url = create_request_url(firmware_uid)
try:
firmware_analyses, firmware_meta_data = request_firmware_data(request_url)
except KeyError:
logging.warning('No firmware found with UID {}'.format(firmware_uid))
return None
environment = create_jinja_environment()
with TemporaryDirectory() as tmp_dir:
Path(tmp_dir, 'meta.tex').write_text(generate_meta_data_code(environment=environment, meta_data=firmware_meta_data))
for filename, result_code in generate_analysis_codes(environment=environment, analysis=firmware_analyses, tmp_dir=tmp_dir):
Path(tmp_dir, filename).write_text(result_code)
Path(tmp_dir, 'main.tex').write_text(generate_main_code(firmware_analyses, firmware_meta_data, environment))
_copy_fact_image(tmp_dir)
execute_pdflatex(tmp_dir)
pdf_filename = create_report_filename(firmware_meta_data)
shutil.move(str(Path(tmp_dir, 'main.pdf')), str(Path('.', pdf_filename)))
return None
if __name__ == '__main__':
args = setup_argparser()
setup_logging(args.debug)
main(args.UID)
sys.exit(0)
pdf_generator/tex_generation/code_generation.py 0 → 100644
import logging
from jinja2 import TemplateNotFound
GENERIC_TEMPLATE = 'generic.tex'
def generate_meta_data_code(environment, meta_data):
template = environment.get_template('meta_data.tex')
logging.debug('Rendering meta data')
return template.render(meta_data=meta_data)
def generate_main_code(firmware_analyses, firmware_meta_data, jinja_environment):
template = jinja_environment.get_template('main.tex')
logging.debug('Rendering main page')
return template.render(analysis=firmware_analyses, meta_data=firmware_meta_data)
def generate_analysis_codes(environment, analysis, tmp_dir):
return [
('{}.tex'.format(analysis_plugin), _render_analysis_result(analysis[analysis_plugin], environment, analysis_plugin, tmp_dir)) for analysis_plugin in analysis
]
def _render_analysis_result(analysis, environment, analysis_plugin, tmp_dir):
try:
template = environment.get_template('{}.tex'.format(analysis_plugin))
except TemplateNotFound:
logging.debug('Falling back on generic template for {}'.format(analysis_plugin))
template = environment.get_template(GENERIC_TEMPLATE)
logging.debug('Rendering {}'.format(analysis_plugin))
return template.render(selected_analysis=analysis, tmp_dir=tmp_dir)
def create_report_filename(meta_data):
main_tex_filename = meta_data['device_name'] + '_analysis_report.pdf'
main_tex_filename = main_tex_filename.replace(' ', '_')
return main_tex_filename.replace('/', '__')
src/jinja_filters/filter.py → pdf_generator/tex_generation/template_engine.py
...@@ -2,6 +2,7 @@ from base64 import decodebytes ...@@ -2,6 +2,7 @@ from base64 import decodebytes
from pathlib import Path from pathlib import Path
from time import localtime, strftime from time import localtime, strftime
import jinja2
from common_helper_files import human_readable_file_size from common_helper_files import human_readable_file_size
...@@ -116,4 +117,36 @@ def split_output_lines(output_value): ...@@ -116,4 +117,36 @@ def split_output_lines(output_value):
if line_length > 92: if line_length > 92:
line = line[:92] + ' ' + line[92:] line = line[:92] + ' ' + line[92:]
output += line + '\n' output += line + '\n'
return output return output
\ No newline at end of file
def create_jinja_environment(templates_to_use='default'):
template_directory = Path(Path(__file__).parent.parent, 'templates', templates_to_use)
environment = jinja2.Environment(
block_start_string=r'\BLOCK{',
block_end_string='}',
variable_start_string=r'\VAR{',
variable_end_string='}',
comment_start_string=r'\#{',
comment_end_string='}',
line_statement_prefix='%%',
line_comment_prefix='%#',
trim_blocks=True,
autoescape=False,
loader=jinja2.FileSystemLoader(str(template_directory))
)
_add_filters_to_jinja(environment)
return environment
def _add_filters_to_jinja(environment):
environment.filters['number_format'] = byte_number_filter
environment.filters['nice_unix_time'] = nice_unix_time
environment.filters['nice_number'] = nice_number_filter
environment.filters['filter_chars'] = filter_latex_special_chars
environment.filters['elements_count'] = count_elements_in_list
environment.filters['base64_to_png'] = convert_base64_to_png_filter
environment.filters['check_list'] = check_if_list_empty
environment.filters['filter_list'] = filter_chars_in_list
environment.filters['split_hash'] = split_hash
environment.filters['split_output_lines'] = split_output_lines
src/config/main.cfg deleted 100644 → 0
[Logging]
logFile=/tmp/fact_report.log
logLevel=WARNING
\ No newline at end of file
src/latex_code_generation/code_generation.py deleted 100644 → 0
import logging
import os
import shutil
from pathlib import Path
from tempfile import TemporaryDirectory
import jinja2
from common_helper_process import execute_shell_command_get_return_code
from jinja_filters.filter import (
byte_number_filter, check_if_list_empty, convert_base64_to_png_filter, count_elements_in_list, filter_chars_in_list,
filter_latex_special_chars, nice_number_filter, nice_unix_time, split_hash, split_output_lines
)
from rest_import.rest import create_request_url, request_firmware_data
GENERIC_TEMPLATE = 'generic.tex'
def create_jinja_environment(templates_to_use='default'):
template_directory = Path(Path(__file__).parent.parent, 'templates', templates_to_use)
environment = jinja2.Environment(
block_start_string=r'\BLOCK{',
block_end_string='}',
variable_start_string=r'\VAR{',
variable_end_string='}',
comment_start_string=r'\#{',
comment_end_string='}',
line_statement_prefix='%%',
line_comment_prefix='%#',
trim_blocks=True,
autoescape=False,
loader=jinja2.FileSystemLoader(str(template_directory))
)
_add_filters_to_jinja(environment)
return environment
def _add_filters_to_jinja(environment):
environment.filters['number_format'] = byte_number_filter
environment.filters['nice_unix_time'] = nice_unix_time
environment.filters['nice_number'] = nice_number_filter
environment.filters['filter_chars'] = filter_latex_special_chars
environment.filters['elements_count'] = count_elements_in_list
environment.filters['base64_to_png'] = convert_base64_to_png_filter
environment.filters['check_list'] = check_if_list_empty
environment.filters['filter_list'] = filter_chars_in_list
environment.filters['split_hash'] = split_hash
environment.filters['split_output_lines'] = split_output_lines
def generate_meta_data_code(environment, meta_data):
template = environment.get_template('meta_data.tex')
logging.debug('Rendering meta data')
return template.render(meta_data=meta_data)
def generate_main_code(firmware_analyses, firmware_meta_data, jinja_environment):
template = jinja_environment.get_template('main.tex')
logging.debug('Rendering main page')
return template.render(analysis=firmware_analyses, meta_data=firmware_meta_data)
def generate_analysis_codes(environment, analysis, tmp_dir):
return [
('{}.tex'.format(analysis_plugin), _render_analysis_result(analysis[analysis_plugin], environment, analysis_plugin, tmp_dir)) for analysis_plugin in analysis
]
def _render_analysis_result(analysis, environment, analysis_plugin, tmp_dir):
try:
template = environment.get_template('{}.tex'.format(analysis_plugin))
except jinja2.TemplateNotFound:
logging.debug('Falling back on generic template for {}'.format(analysis_plugin))
template = environment.get_template(GENERIC_TEMPLATE)
logging.debug('Rendering {}'.format(analysis_plugin))
return template.render(selected_analysis=analysis, tmp_dir=tmp_dir)
def create_report_filename(meta_data):
main_tex_filename = meta_data['device_name'] + '_analysis_report.pdf'
main_tex_filename = main_tex_filename.replace(' ', '_')
return main_tex_filename.replace('/', '__')
def _copy_fact_image(target):
shutil.copy(str(Path(__file__).parent.parent / 'templates' / 'fact_logo.png'), str(Path(target) / 'fact_logo.png'))
def execute_pdflatex(tmp_dir):
current_dir = os.getcwd()
os.chdir(tmp_dir)
logging.debug('Creating pdf file')
_, _ = execute_shell_command_get_return_code('env buf_size=1000000 pdflatex main.tex')
os.chdir(current_dir)
def generate_pdf_report(firmware_uid):
request_url = create_request_url(firmware_uid)
try:
firmware_analyses, firmware_meta_data = request_firmware_data(request_url)
except KeyError:
logging.warning('No firmware found with UID {}'.format(firmware_uid))
return None
environment = create_jinja_environment()
with TemporaryDirectory() as tmp_dir:
Path(tmp_dir, 'meta.tex').write_text(generate_meta_data_code(environment=environment, meta_data=firmware_meta_data))
for filename, result_code in generate_analysis_codes(environment=environment, analysis=firmware_analyses, tmp_dir=tmp_dir):
Path(tmp_dir, filename).write_text(result_code)
Path(tmp_dir, 'main.tex').write_text(generate_main_code(firmware_analyses, firmware_meta_data, environment))
_copy_fact_image(tmp_dir)
execute_pdflatex(tmp_dir)
pdf_filename = create_report_filename(firmware_meta_data)
shutil.move(str(Path(tmp_dir, 'main.pdf')), str(Path('.', pdf_filename)))
return None
src/pdf_generator.py deleted 100755 → 0
#! /usr/bin/env python3
'''
This Template has a separate console and file logging.
File logging-level can be changed by -L parameter.
Log file can be changed by -l parameter.
Standard values for logging can be set in the config file
-> "Logging"
---> "logFile"
---> "logLevel"
Console logging is INFO or DEBUG according to debug debug_flag.
Config file path can be changed by -C parameter.
'''
import argparse
from common_helper_files import create_dir_for_file
import configparser
import logging
import os
import sys
from latex_code_generation.code_generation import generate_pdf_report
PROGRAM_NAME = 'FACT PDF Report Generator'
PROGRAM_VERSION = '0.1'
PROGRAM_DESCRIPTION = 'Generates a analysis PDF report'
STANDARD_CONF_FILE = os.path.dirname(os.path.abspath(__file__)) + '/config/main.cfg'
def _setup_argparser():
parser = argparse.ArgumentParser(description='{} - {}'.format(PROGRAM_NAME, PROGRAM_DESCRIPTION))
parser.add_argument('-V', '--version', action='version', version='{} {}'.format(PROGRAM_NAME, PROGRAM_VERSION))
parser.add_argument('-l', '--log_file', help='path to log file')
parser.add_argument('-L', '--log_level', help='define the log level [DEBUG,INFO,WARNING,ERROR]')
parser.add_argument('-d', '--debug', action='store_true', default=False, help='print debug messages')
parser.add_argument('-C', '--config_file', help='set path to config File', default=STANDARD_CONF_FILE)
parser.add_argument('-u', '--uid', help='firmware analysis UID', dest="UID")
return parser.parse_args()
def _get_console_output_level(debug_flag):
if debug_flag:
return logging.DEBUG
else:
return logging.INFO
def _setup_logging(config, debug_flag=False):
log_level = getattr(logging, config['Logging']['logLevel'], None)
log_format = logging.Formatter(fmt='[%(asctime)s][%(module)s][%(levelname)s]: %(message)s', datefmt='%Y-%m-%d %H:%M:%S')
logger = logging.getLogger('')
logger.setLevel(logging.DEBUG)
create_dir_for_file(config['Logging']['logFile'])
file_log = logging.FileHandler(config['Logging']['logFile'])
file_log.setLevel(log_level)
file_log.setFormatter(log_format)
console_log = logging.StreamHandler()
console_log.setLevel(_get_console_output_level(debug_flag))
console_log.setFormatter(log_format)
logger.addHandler(file_log)
logger.addHandler(console_log)
def _load_config(args):
config = configparser.ConfigParser()
config.read(args.config_file)
if args.log_file is not None:
config['Logging']['logFile'] = args.log_file
if args.log_level is not None:
config['Logging']['logLevel'] = args.log_level
return config
if __name__ == '__main__':
args = _setup_argparser()
config = _load_config(args)
_setup_logging(config, args.debug)
logging.info(args.config_file)
generate_pdf_report(args.UID)
sys.exit(0)
src/test/acceptance/test_main.py → test/acceptance/test_main.py
from common_helper_process.fail_safe_subprocess import execute_shell_command_get_return_code from pathlib import Path
import os
import pytest
import pytest
from common_helper_process.fail_safe_subprocess import execute_shell_command_get_return_code
SRC_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../../pdf_generator.py' SCRIPT_PATH = Path(__file__).parent.parent.parent / 'pdf_generator' / 'pdf_generator.py'
@pytest.mark.parametrize('arguments, expected_output, expected_return_code', [ @pytest.mark.parametrize('arguments, expected_output, expected_return_code', [
...@@ -11,7 +11,7 @@ SRC_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../../pdf_generator.py' ...@@ -11,7 +11,7 @@ SRC_DIR = os.path.dirname(os.path.abspath(__file__)) + '/../../pdf_generator.py'
('-h', 'usag', 0) ('-h', 'usag', 0)
]) ])
def test_main_program(arguments, expected_output, expected_return_code): def test_main_program(arguments, expected_output, expected_return_code):
command_line = SRC_DIR + ' ' + arguments command_line = str(SCRIPT_PATH) + ' ' + arguments
output, return_code = execute_shell_command_get_return_code(command_line) output, return_code = execute_shell_command_get_return_code(command_line)
assert output[0:4] == expected_output assert output[0:4] == expected_output
assert return_code == expected_return_code assert return_code == expected_return_code
src/test/data/test_dict.py → test/data/test_dict.py
test_dict = { TEST_DICT = {
"firmware": { "firmware": {
"analysis": { "analysis": {
"binwalk": { "binwalk": {
......
src/test/integration/test_code_generation.py → test/integration/test_code_generation.py
...@@ -2,10 +2,10 @@ from pathlib import Path ...@@ -2,10 +2,10 @@ from pathlib import Path
import pytest import pytest
from jinja2 import Environment, FileSystemLoader from jinja2 import Environment, FileSystemLoader
from latex_code_generation.code_generation import _add_filters_to_jinja, generate_meta_data_code from pdf_generator.pre_processing.rest import request_firmware_data
from rest_import.rest import request_firmware_data from pdf_generator.tex_generation.code_generation import generate_meta_data_code
from pdf_generator.tex_generation.template_engine import _add_filters_to_jinja
from ..data.test_dict import test_dict from test.data.test_dict import TEST_DICT
# pylint: disable=redefined-outer-name # pylint: disable=redefined-outer-name
...@@ -29,19 +29,19 @@ def mock_environment(): ...@@ -29,19 +29,19 @@ def mock_environment():
line_comment_prefix='%#', line_comment_prefix='%#',
trim_blocks=True, trim_blocks=True,
autoescape=False, autoescape=False,
loader=FileSystemLoader(str(Path(Path(__file__).parent.parent.parent, 'templates', 'default'))), loader=FileSystemLoader(str(Path(Path(__file__).parent.parent.parent, 'pdf_generator', 'templates', 'default'))),
) )
_add_filters_to_jinja(env) _add_filters_to_jinja(env)
return env return env
def test_anything_mocked(monkeypatch): def test_anything_mocked(monkeypatch):
monkeypatch.setattr('rest_import.rest.requests.get', lambda x: MockResponse()) monkeypatch.setattr('pdf_generator.pre_processing.rest.requests.get', lambda x: MockResponse())
anything = request_firmware_data('anything') anything = request_firmware_data('anything')
assert anything assert anything
def test_latex_code_generation(mock_environment): def test_latex_code_generation(mock_environment):
result = generate_meta_data_code(mock_environment, test_dict) result = generate_meta_data_code(mock_environment, TEST_DICT)
assert result assert result
src/test/unit/test_code_generation.py → test/unit/test_code_generation.py
from tempfile import TemporaryDirectory from tempfile import TemporaryDirectory
from latex_code_generation.code_generation import _render_analysis_result, create_jinja_environment, generate_pdf_report from pdf_generator.pdf_generator import main
from pdf_generator.tex_generation.code_generation import _render_analysis_result
from pdf_generator.tex_generation.template_engine import create_jinja_environment
TEST_DATA = { TEST_DATA = {
'analysis': {'file_hashes': {'ssdeep': 'bla', 'sha1': 'blah'}}, 'analysis': {'file_hashes': {'ssdeep': 'bla', 'sha1': 'blah'}},
...@@ -18,6 +20,6 @@ def test_render_template(): ...@@ -18,6 +20,6 @@ def test_render_template():
def test_main(monkeypatch): def test_main(monkeypatch):
monkeypatch.setattr('latex_code_generation.code_generation.request_firmware_data', lambda *_: (TEST_DATA['analysis'], TEST_DATA['meta_data'])) monkeypatch.setattr('pdf_generator.pdf_generator.request_firmware_data', lambda *_: (TEST_DATA['analysis'], TEST_DATA['meta_data']))
monkeypatch.setattr('latex_code_generation.code_generation.shutil.move', lambda *_: None) monkeypatch.setattr('pdf_generator.pdf_generator.shutil.move', lambda *_: None)
generate_pdf_report(firmware_uid='b79ea608e2f0390744642bad472f8d9fd7e4713791857da5d5fcabf70a009e50_29626948') main(firmware_uid='b79ea608e2f0390744642bad472f8d9fd7e4713791857da5d5fcabf70a009e50_29626948')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment