Added hidden attribute to core.module.Option class; added gzip option to entropy module.

src/binwalk/core/magic.py

@@ -242,9 +242,9 @@ class SignatureLine(object):
             retag = re.compile(r'\{.*?\}')
 
             # Parse out tag keywords from the format string
-            for tag in [m.group() for m in retag.finditer(self.format)]:
+            for match in retag.finditer(self.format):
                 # Get rid of the curly braces.
-                tag = tag.replace('{', '').replace('}', '')
+                tag = match.group().replace('{', '').replace('}', '')
 
                 # If the tag specifies a value, it will be colon delimited (e.g., '{name:%s}')
                 if ':' in tag:
@@ -553,7 +553,7 @@ class Magic(object):
                     if line.value is None:
                         # Check to see if this is a string whose size is known and has been specified on a previous
                         # signature line.
-                        if [x for x in line.tags if x.name == 'string'] and binwalk.core.compat.has_key(tags, 'strlen'):
+                        if binwalk.core.compat.has_key(tags, 'strlen') and [x for x in line.tags if x.name == 'string']:
                             dvalue = self.data[start:(start+tags['strlen'])]
                         # Else, just terminate the string at the first newline, carriage return, or NULL byte
                         else:
@@ -718,7 +718,7 @@ class Magic(object):
             for match in signature.regex.finditer(self.data):
                 # Take the offset of the start of the signature into account
                 offset = match.start() - signature.offset
-                # Signatures are orderd based on the length of their magic bytes (largest first).
+                # Signatures are ordered based on the length of their magic bytes (largest first).
                 # If this offset has already been matched to a previous signature, ignore it unless
                 # self.show_invalid has been specified. Also ignore obviously invalid offsets (<1)
                 # as well as those outside the specified self.data range (dlen).
@@ -774,16 +774,16 @@ class Magic(object):
                     # If there is an existing signature, append it to the signature list,
                     # unless the text in its title field has been filtered by user-defined
                     # filter rules.
-                    if signature:
-                        if not self._filtered(signature.title):
-                            self.signatures.append(signature)
+                    if signature and not self._filtered(signature.title):
+                        self.signatures.append(signature)
 
                     # Create a new signature object; use the size of self.signatures to
                     # assign each signature a unique ID.
                     signature = Signature(len(self.signatures), sigline)
                 # Else, just append this line to the existing signature
                 elif signature:
-                    signature.append(sigline)
+                    #signature.append(sigline)
+                    signature.lines.append(sigline)
                 # If this is not the first line of a signature entry and there is no other
                 # existing signature entry, something is very wrong with the signature file.
                 else:

src/binwalk/core/module.py

@@ -19,7 +19,7 @@ class Option(object):
     A container class that allows modules to declare command line options.
     '''
 
-    def __init__(self, kwargs={}, priority=0, description="", short="", long="", type=None, dtype=None):
+    def __init__(self, kwargs={}, priority=0, description="", short="", long="", type=None, dtype=None, hidden=False):
         '''
         Class constructor.
 
@@ -30,6 +30,7 @@ class Option(object):
         @long        - The long option to use (if None, this option will not be displayed in help output).
         @type        - The accepted data type (one of: io.FileIO/argparse.FileType/binwalk.core.common.BlockFile, list, str, int, float).
         @dtype       - The displayed accepted type string, to be shown in help output.
+        @hidden      - If set to True, this option will not be displayed in the help output.
 
         Returns None.
         '''
@@ -40,6 +41,7 @@ class Option(object):
         self.long = long
         self.type = type
         self.dtype = dtype
+        self.hidden = hidden
 
         if not self.dtype and self.type:
             if self.type in [io.FileIO, argparse.FileType, binwalk.core.common.BlockFile]:
@@ -627,7 +629,7 @@ class Modules(object):
             help_string += "\n%s Options:\n" % module.TITLE
 
             for module_option in module.CLI:
-                if module_option.long:
+                if module_option.long and not module_option.hidden:
                     long_opt = '--' + module_option.long
 
                     if module_option.dtype:

src/binwalk/modules/entropy.py

@@ -2,6 +2,7 @@
 
 import os
 import math
+import zlib
 import binwalk.core.common
 from binwalk.core.compat import *
 from binwalk.core.module import Module, Option, Kwarg
@@ -23,12 +24,16 @@ class Entropy(Module):
 
     TITLE = "Entropy Analysis"
     ORDER = 8
-    
+
     CLI = [
             Option(short='E',
                    long='entropy',
                    kwargs={'enabled' : True},
                    description='Calculate file entropy'),
+            Option(short='H',
+                   long='zlib',
+                   kwargs={'use_zlib' : True},
+                   description='Use zlib compression ratios instead of Shannon algorithm'),
             Option(short='J',
                    long='save',
                    kwargs={'save_plot' : True},
@@ -46,6 +51,7 @@ class Entropy(Module):
     KWARGS = [
             Kwarg(name='enabled', default=False),
             Kwarg(name='save_plot', default=False),
+            Kwarg(name='use_zlib', default=False),
             Kwarg(name='display_results', default=True),
             Kwarg(name='do_plot', default=True),
             Kwarg(name='show_legend', default=True),
@@ -57,10 +63,14 @@ class Entropy(Module):
 
     def init(self):
         self.HEADER[-1] = "ENTROPY"
-        self.algorithm = self.shannon
         self.max_description_length = 0
         self.file_markers = {}
 
+        if self.use_zlib:
+            self.algorithm = self.gzip
+        else:
+            self.algorithm = self.shannon
+
         # Get a list of all other module's results to mark on the entropy graph
         for (module, obj) in iterator(self.modules):
             for result in obj.results:
@@ -95,8 +105,8 @@ class Entropy(Module):
 
             if self.display_results:
                 self.footer()
-    
-        if self.do_plot and not self.save_plot:    
+
+        if self.do_plot and not self.save_plot:
             from pyqtgraph.Qt import QtGui
             QtGui.QApplication.instance().exec_()
 
@@ -114,7 +124,11 @@ class Entropy(Module):
             i = 0
             while i < dlen:
                 entropy = self.algorithm(data[i:i+self.block_size])
-                r = self.result(offset=(file_offset + i), file=fp, entropy=entropy, description=("%f" % entropy), display=self.display_results)
+                r = self.result(offset=(file_offset + i),
+                                file=fp,
+                                entropy=entropy,
+                                description=("%f" % entropy),
+                                display=self.display_results)
                 i += self.block_size
 
         if self.do_plot:
@@ -128,7 +142,7 @@ class Entropy(Module):
 
         if data:
             length = len(data)
-            
+
             seen = dict(((chr(x), 0) for x in range(0, 256)))
             for byte in data:
                 seen[byte] += 1
@@ -136,7 +150,7 @@ class Entropy(Module):
             for x in range(0, 256):
                 p_x = float(seen[chr(x)]) / length
                 if p_x > 0:
-                    entropy += - p_x*math.log(p_x, 2)
+                    entropy -= p_x * math.log(p_x, 2)
 
         return (entropy / 8)
 
@@ -146,7 +160,7 @@ class Entropy(Module):
         This is faster than the shannon entropy analysis, but not as accurate.
         '''
         # Entropy is a simple ratio of: <zlib compressed size> / <original size>
-        e = float(float(len(zlib.compress(data, 9))) / float(len(data)))
+        e = float(float(len(zlib.compress(str2bytes(data), 9))) / float(len(data)))
 
         if truncate and e > 1.0:
             e = 1.0
@@ -173,7 +187,7 @@ class Entropy(Module):
             plt.addLegend(size=(self.max_description_length*10, 0))
 
             for (offset, description) in self.file_markers[fname]:
-                # If this description has already been plotted at a different offset, we need to 
+                # If this description has already been plotted at a different offset, we need to
                 # use the same color for the marker, but set the description to None to prevent
                 # duplicate entries in the graph legend.
                 #

src/binwalk/modules/general.py

@@ -71,6 +71,7 @@ class General(Module):
 
         # Hidden, API-only arguments
         Option(long="string",
+               hidden=True,
                kwargs={'subclass' : binwalk.core.common.StringFile}),
     ]