Update display output

src/binwalk/core/display.py

@@ -16,9 +16,8 @@ class Display(object):
     HEADER_WIDTH = 80
     DEFAULT_FORMAT = "%s\n"
 
-    def __init__(self, quiet=False, verbose=False, log=None, csv=False, fit_to_screen=False, filter=None):
+    def __init__(self, quiet=False, verbose=False, log=None, csv=False, fit_to_screen=False):
         self.quiet = quiet
-        self.filter = filter
         self.verbose = verbose
         self.fit_to_screen = fit_to_screen
         self.fp = None
@@ -99,17 +98,12 @@ class Display(object):
     def _fprint(self, fmt, columns, csv=True, stdout=True, filter=True):
         line = fmt % tuple(columns)
 
-        # TODO: Additional filtering was originally done here to support the --grep option,
-        #       which is now depreciated. Seems redundant now, as the result won't get passed
-        #       to the display class unless it has already passed the filter.valid_result check.
-        #if not filter or self.filter.valid_result(line):
-        if True:
-            if not self.quiet and stdout:
-                sys.stdout.write(self._format_line(line.strip()) + "\n")
-                sys.stdout.flush()
-
-            if self.fp and not (self.csv and not csv):
-                self.log(fmt, columns)
+        if not self.quiet and stdout:
+            sys.stdout.write(self._format_line(line.strip()) + "\n")
+            sys.stdout.flush()
+
+        if self.fp and not (self.csv and not csv):
+            self.log(fmt, columns)
 
     def _append_to_data_parts(self, data, start, end):
         '''

src/binwalk/modules/compression.py

@@ -5,6 +5,7 @@ import lzma
 import struct
 import binwalk.core.C
 import binwalk.core.compat
+import binwalk.core.common
 from binwalk.core.module import Option, Kwarg, Module
 
 class LZMAHeader(object):
@@ -16,16 +17,26 @@ class LZMA(object):
 
     DESCRIPTION = "Raw LZMA compression stream"
     FAKE_SIZE = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
+    COMMON_PROPERTIES = [0x5D, 0x6E]
     MAX_PROP = ((4 * 5 + 4) * 9 + 8)
     BLOCK_SIZE = 32*1024
 
     def __init__(self, module):
         self.module = module
+        self.decompressed_data = None
 
         self.build_properties()
         self.build_dictionaries()
         self.build_headers()
 
+        # Add an extraction rule
+        #if self.module.extractor.enabled:
+        #    self.module.extractor.add_rule(regex='^%s' % self.DESCRIPTION.lower(), extension="lzma", cmd=self.extractor)
+
+    # TODO: Reliable extraction is horribly inefficient with the Python lzma module
+    #def extractor(self, file_name):
+    #    compressed_data = binwalk.core.common.BlockFile(file_name).read()
+
     def build_property(self, pb, lp, lc):
         prop = (((pb * 5) + lp) * 9) + lc
         if prop > self.MAX_PROP:
@@ -53,18 +64,25 @@ class LZMA(object):
     def build_properties(self):
         self.properties = set()
 
-        for pb in range(0, 9):
-            for lp in range(0, 5):
-                for lc in range(0, 5):
-                    prop = self.build_property(pb, lp, lc)
-                    if prop is not None:
-                        self.properties.add(chr(prop))
+        if self.module.partial_scan == True:
+            for prop in self.COMMON_PROPERTIES:
+                self.properties.add(chr(prop))
+        else:
+            for pb in range(0, 9):
+                for lp in range(0, 5):
+                    for lc in range(0, 5):
+                        prop = self.build_property(pb, lp, lc)
+                        if prop is not None:
+                            self.properties.add(chr(prop))
 
     def build_dictionaries(self):
-        self.dictionaries = set()
+        self.dictionaries = []
 
-        for n in range(16, 26):
-            self.dictionaries.add(binwalk.core.compat.bytes2str(struct.pack("<I", 2**n)))
+        if self.module.partial_scan == True:
+            self.dictionaries.append(struct.pack("<I", 2**16))
+        else:
+            for n in range(16, 26):
+                self.dictionaries.append(binwalk.core.compat.bytes2str(struct.pack("<I", 2**n)))
 
     def build_headers(self):
         self.headers = set()
@@ -73,34 +91,42 @@ class LZMA(object):
             for dictionary in self.dictionaries:
                 self.headers.add(prop + dictionary + self.FAKE_SIZE)
 
-    def decompress(self, data):
+    def decompress(self, data, complete=False):
         result = None
         description = None
+        self.decompressed_data = None
+        i = 0
 
         for header in self.headers:
+            i += 1
             # The only acceptable exceptions are those indicating that the input data was truncated.
             try:
-                lzma.decompress(binwalk.core.compat.str2bytes(header + data))
+                final_data = binwalk.core.compat.str2bytes(header + data)
+                if complete:
+                    open("test.bin-%d" % i, "wb").write(final_data)
+                self.decompressed_data = lzma.decompress(final_data)
                 result = self.parse_header(header)
                 break
             except IOError as e:
                 # The Python2 module gives this error on truncated input data.
-                if str(e) == "unknown BUF error":
+                if not complete and str(e) == "unknown BUF error":
                     result = self.parse_header(header)
                     break
             except Exception as e:
                 # The Python3 module gives this error on truncated input data.
                 # The inconsistency between modules is a bit worrisome.
-                if str(e) == "Compressed data ended before the end-of-stream marker was reached":
+                if not complete and str(e) == "Compressed data ended before the end-of-stream marker was reached":
                     result = self.parse_header(header)
                     break
 
         if result is not None:
-            description = "%s, pb: %d, lp: %d, lc: %d, dictionary size: %d" % (self.DESCRIPTION,
-                                                                               result.pb,
-                                                                               result.lp,
-                                                                               result.lc,
-                                                                               result.dictionary)
+            prop = self.build_property(result.pb, result.lp, result.lc)
+            description = "%s, properties: 0x%.2X [pb: %d, lp: %d, lc: %d], dictionary size: %d" % (self.DESCRIPTION,
+                                                                                                   prop,
+                                                                                                   result.pb,
+                                                                                                   result.lp,
+                                                                                                   result.lc,
+                                                                                                   result.dictionary)
 
         return description
 
@@ -158,6 +184,10 @@ class RawCompression(Module):
                    long='lzma',
                    kwargs={'enabled' : True, 'scan_for_lzma' : True},
                    description='Scan for raw LZMA compression streams'),
+            Option(short='P',
+                   long='partial',
+                   kwargs={'partial_scan' : True},
+                   description='Perform a superficial, but faster, scan'),
             Option(short='S',
                    long='stop',
                    kwargs={'stop_on_first_hit' : True},
@@ -166,13 +196,12 @@ class RawCompression(Module):
 
     KWARGS = [
             Kwarg(name='enabled', default=False),
+            Kwarg(name='partial_scan', default=False),
             Kwarg(name='stop_on_first_hit', default=False),
             Kwarg(name='scan_for_deflate', default=False),
             Kwarg(name='scan_for_lzma', default=False),
     ]
 
-    #READ_BLOCK_SIZE = 64*1024
-
     def init(self):
         self.decompressors = []
 
@@ -185,7 +214,6 @@ class RawCompression(Module):
         for fp in iter(self.next_file, None):
 
             file_done = False
-            #fp.set_block_size(peek=self.READ_BLOCK_SIZE)
 
             self.header()
 

src/binwalk/modules/general.py

@@ -39,22 +39,6 @@ class General(Module):
                type=int,
                kwargs={'swap_size' : 0},
                description='Reverse every n bytes before scanning'),
-        Option(short='I',
-               long='invalid',
-               kwargs={'show_invalid' : True},
-               description='Show results marked as invalid'),
-        Option(short='x',
-               long='exclude',
-               kwargs={'exclude_filters' : []},
-               type=list,
-               dtype=str.__name__,
-               description='Exclude results that match <str>'),
-        Option(short='y',
-               long='include',
-               kwargs={'include_filters' : []},
-               type=list,
-               dtype=str.__name__,
-               description='Only show results that match <str>'),
         Option(long='log',
                short='f',
                type=argparse.FileType,
@@ -91,9 +75,6 @@ class General(Module):
         Kwarg(name='offset', default=0),
         Kwarg(name='block', default=0),
         Kwarg(name='swap_size', default=0),
-        Kwarg(name='show_invalid', default=False),
-        Kwarg(name='include_filters', default=[]),
-        Kwarg(name='exclude_filters', default=[]),
         Kwarg(name='log_file', default=None),
         Kwarg(name='csv', default=False),
         Kwarg(name='format_to_terminal', default=False),
@@ -113,20 +94,11 @@ class General(Module):
         self._open_target_files()
         self._set_verbosity()
 
-        self.filter = binwalk.core.filter.Filter(self.show_invalid)
-
-        # Set any specified include/exclude filters
-        for regex in self.exclude_filters:
-            self.filter.exclude(regex)
-        for regex in self.include_filters:
-            self.filter.include(regex)
-
         self.settings = binwalk.core.settings.Settings()
         self.display = binwalk.core.display.Display(log=self.log_file,
                                                     csv=self.csv,
                                                     quiet=self.quiet,
                                                     verbose=self.verbose,
-                                                    filter=self.filter,
                                                     fit_to_screen=self.format_to_terminal)
 
         if self.show_help:

src/binwalk/modules/hexdiff.py

@@ -172,10 +172,6 @@ class HexDiff(Module):
             loop_count += 1
 
     def init(self):
-        # Disable the invalid description auto-filtering feature.
-        # This will not affect our own validation.
-        self.config.filter.show_invalid_results = True
-
         # Always disable terminal formatting, as it won't work properly with colorized output
         self.config.display.fit_to_screen = False
 

src/binwalk/modules/signature.py

@@ -38,10 +38,29 @@ class Signature(Module):
                    long='dumb',
                    kwargs={'dumb_scan' : True},
                    description='Disable smart signature keywords'),
+        Option(short='I',
+               long='invalid',
+               kwargs={'show_invalid' : True},
+               description='Show results marked as invalid'),
+        Option(short='x',
+               long='exclude',
+               kwargs={'exclude_filters' : []},
+               type=list,
+               dtype=str.__name__,
+               description='Exclude results that match <str>'),
+        Option(short='y',
+               long='include',
+               kwargs={'include_filters' : []},
+               type=list,
+               dtype=str.__name__,
+               description='Only show results that match <str>'),
     ]
 
     KWARGS = [
             Kwarg(name='enabled', default=False),
+            Kwarg(name='show_invalid', default=False),
+            Kwarg(name='include_filters', default=[]),
+            Kwarg(name='exclude_filters', default=[]),
             Kwarg(name='raw_bytes', default=None),
             Kwarg(name='search_for_opcodes', default=False),
             Kwarg(name='explicit_signature_scan', default=False),
@@ -55,9 +74,18 @@ class Signature(Module):
     def init(self):
         self.keep_going = self.config.keep_going
 
+        # Initialize the filter
+        self.filter = binwalk.core.filter.Filter(self.show_invalid)
+
+        # Set any specified include/exclude filters
+        for regex in self.exclude_filters:
+            self.filter.exclude(regex)
+        for regex in self.include_filters:
+            self.filter.include(regex)
+
         # Create Signature and MagicParser class instances. These are mostly for internal use.
-        self.smart = binwalk.core.smart.Signature(self.config.filter, ignore_smart_signatures=self.dumb_scan)
-        self.parser = binwalk.core.parser.MagicParser(self.config.filter, self.smart)
+        self.smart = binwalk.core.smart.Signature(self.filter, ignore_smart_signatures=self.dumb_scan)
+        self.parser = binwalk.core.parser.MagicParser(self.filter, self.smart)
 
         # If a raw byte sequence was specified, build a magic file from that instead of using the default magic files
         if self.raw_bytes is not None:
@@ -99,7 +127,7 @@ class Signature(Module):
         '''
         Called automatically by self.result.
         '''
-        if self.config.filter.show_invalid_results:
+        if self.filter.show_invalid_results:
             r.valid = True
         else:
             if not r.description:
@@ -111,7 +139,7 @@ class Signature(Module):
             if r.jump and (r.jump + r.offset) > r.file.size:
                 r.valid = False
 
-            r.valid = self.config.filter.valid_result(r.description)
+            r.valid = self.filter.valid_result(r.description)
 
     def scan_file(self, fp):
         current_file_offset = 0