Merge branch 'master' of https://github.com/devttys0/binwalk

INSTALL.md

@@ -31,6 +31,18 @@ Although all binwalk run-time dependencies are optional, the `python-lzma` modul
 $ sudo apt-get install python-lzma
 ```
 
+Binwalk uses the `pycrypto` library to decrypt some known encrypted firmware images:
+
+```bash
+# Python2.7
+$ sudo apt-get install python-crypto
+```
+
+```bash
+# Python3.x
+$ sudo apt-get install python3-crypto
+```
+
 Binwalk uses [pyqtgraph](http://www.pyqtgraph.org) to generate graphs and visualizations, which requires the following: 
 
 ```bash
@@ -70,7 +82,7 @@ $ sudo apt-get install mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabex
 # Install sasquatch to extract non-standard SquashFS images
 $ sudo apt-get install zlib1g-dev liblzma-dev liblzo2-dev
 $ git clone https://github.com/devttys0/sasquatch
-$ (cd sasquatch && make && sudo make install)
+$ (cd sasquatch && ./build.sh)
 ```
 
 ```bash
@@ -81,6 +93,19 @@ $ (cd jefferson && sudo python setup.py install)
 ```
 
 ```bash
+# Install ubi_reader to extract UBIFS file systems
+$ sudo apt-get install liblzo2-dev python-lzo
+$ git clone https://github.com/jrspruitt/ubi_reader
+$ (cd ubi_reader && sudo python setup.py install)
+```
+
+```bash
+# Install yaffshiv to extract YAFFS file systems
+$ git clone https://github.com/devttys0/yaffshiv
+$ (cd yaffshiv && sudo python setup.py install)
+```
+
+```bash
 # Install unstuff (closed source) to extract StuffIt archive files
 $ wget -O - http://my.smithmicro.com/downloads/files/stuffit520.611linux-i386.tar.gz | tar -zxv
 $ sudo cp bin/unstuff /usr/local/bin/

deps.sh

 #!/bin/bash
+set -o nounset
 
 REQUIRED_UTILS="wget tar python"
 APTCMD="apt-get"
 YUMCMD="yum"
-APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev"
-PYTHON2_APT_CANDIDATES="python-lzma python-pip python-opengl python-qt4 python-qt4-gl python-numpy python-scipy"
-PYTHON3_APT_CANDIDATES="python3-pip python3-opengl python3-pyqt4 python3-pyqt4.qtopengl python3-numpy python3-scipy"
+APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit openjdk-7-jdk"
+PYTHON2_APT_CANDIDATES="python-crypto python-lzo python-lzma python-pip python-opengl python-qt4 python-qt4-gl python-numpy python-scipy"
+PYTHON3_APT_CANDIDATES="python3-crypto python3-pip python3-opengl python3-pyqt4 python3-pyqt4.qtopengl python3-numpy python3-scipy"
 PYTHON3_YUM_CANDIDATES=""
 YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma"
 PYTHON2_YUM_CANDIDATES="python-pip python-opengl python-qt4 numpy python-numdisplay numpy-2f python-Bottleneck scipy"
@@ -25,13 +26,13 @@ fi
 function install_sasquatch
 {
     git clone https://github.com/devttys0/sasquatch
-    (cd sasquatch && make && $SUDO make install)
+    (cd sasquatch && $SUDO ./build.sh)
     $SUDO rm -rf sasquatch
 }
 
 function install_jefferson
 {
-    $SUDO pip install cstruct
+    install_pip_package cstruct
     git clone https://github.com/sviehb/jefferson
     (cd jefferson && $SUDO python2 setup.py install)
     $SUDO rm -rf jefferson
@@ -47,6 +48,13 @@ function install_unstuff
     rm -rf /tmp/unstuff
 }
 
+function install_ubireader
+{
+    git clone https://github.com/jrspruitt/ubi_reader
+    (cd ubi_reader && $SUDO python setup.py install)
+    $SUDO rm -rf ubi_reader
+}
+
 function install_pip_package
 {
     PACKAGE="$1"
@@ -137,10 +145,16 @@ fi
 
 # Do the install(s)
 cd /tmp
-sudo $PKGCMD $PKGCMD_OPTS $PKG_CANDIDTES
+sudo $PKGCMD $PKGCMD_OPTS $PKG_CANDIDATES
+if [ $? -ne 0 ]
+    then
+    echo "Package installation failed: $PKG_CANDIDATES"
+    exit 1
+fi
 install_pip_package pyqtgraph
 install_pip_package capstone
 install_sasquatch
 install_jefferson
 install_unstuff
+install_ubireader
 

images/README.md

+This is just a directory to store screenshots used in the github Wiki / documentation.

setup.py

@@ -8,6 +8,7 @@ from distutils.core import setup, Command
 from distutils.dir_util import remove_tree
 
 MODULE_NAME = "binwalk"
+SCRIPT_NAME = MODULE_NAME
 
 # Python2/3 compliance
 try:
@@ -200,7 +201,7 @@ for data_dir in ["magic", "config", "plugins", "modules", "core"]:
 
 # Install the module, script, and support files
 setup(name = MODULE_NAME,
-      version = "2.1.0",
+      version = "2.1.2b",
       description = "Firmware analysis tool",
       author = "Craig Heffner",
       url = "https://github.com/devttys0/%s" % MODULE_NAME,
@@ -208,7 +209,7 @@ setup(name = MODULE_NAME,
       requires = [],
       packages = [MODULE_NAME],
       package_data = {MODULE_NAME : install_data_files},
-      scripts = [os.path.join("scripts", MODULE_NAME)],
+      scripts = [os.path.join("scripts", SCRIPT_NAME)],
 
       cmdclass = {'clean' : CleanCommand, 'uninstall' : UninstallCommand, 'idainstall' : IDAInstallCommand, 'idauninstall' : IDAUnInstallCommand}
 )

src/binwalk/__init__.py

-__all__ = ['scan', 'execute', 'Modules', 'ModuleException']
+__all__ = ['scan', 'execute', 'ModuleException']
 
 from binwalk.core.module import Modules, ModuleException
 
 # Convenience functions
 def scan(*args, **kwargs):
-    return Modules(*args, **kwargs).execute()
+    with Modules(*args, **kwargs) as m:
+        objs = m.execute()
+    return objs
 def execute(*args, **kwargs):
-    return Modules(*args, **kwargs).execute()
+    return scan(*args, **kwargs)

src/binwalk/config/extract.conf

@@ -13,23 +13,23 @@
 #
 #   o zlib
 #   o cpio
-#   o jffs2
 #   o Raw LZMA/deflate streams
+#   o Hilink encrypted uImage firmware
 #
 # There are also alternative extractors for the following file formats, implemented as plugins:
 #
 #   o gzip
 #   o lzma
+#   o xz
 #
 #######################################################################################################################################
 
 # Assumes these utilities are installed in $PATH.
 ^gzip compressed data:gz:gzip -d -f '%e':0,2
 ^lzma compressed data:7z:7z e -y '%e':0,1
-^xz compressed data:tar:tar xJf '%e'
+^xz compressed data:xz:7z e -y '%e':0,1
 ^bzip2 compressed data:bz2:bzip2 -d '%e'
 ^compress'd data:Z:gzip -d '%e'
-^zip archive data:zip:7z x -y '%e' -p '':0,1
 ^posix tar archive:tar:tar xvf '%e'
 ^rar archive data:rar:unrar e '%e'
 ^rar archive data:rar:unrar -x '%e' # This is for the 'free' version
@@ -39,6 +39,17 @@
 ^microsoft cabinet archive:cab:cabextract '%e'
 ^stuffit:sit:unstuff '%e'
 
+# jar just does a better job of extracting zip files than either
+# unzip or 7z.
+#
+# unzip does not support files that are "missing end of central directory header".
+# 
+# 7z handles most zip files, but fails on some zip archives, inexplicably seeing
+# only the *last* entry in the zip archive (though 7z thinks it succeeded). See
+# StarCam firmware CH-sys-48.53.64.67.zip.
+^zip archive data:zip:jar xvf '%e':0
+^zip archive data:zip:7z x -y '%e' -p '':0,1
+
 # Try unsquashfs first, or if not installed, sasquatch
 ^squashfs filesystem:squashfs:unsquashfs -d '%%squashfs-root%%' '%e':0:False
 ^squashfs filesystem:squashfs:sasquatch -p 1 -le -d '%%squashfs-root%%' '%e':0:False
@@ -60,15 +71,21 @@
 # Use sviehb's jefferson.py tool for JFFS2 extraction
 ^jffs2 filesystem:jffs2:jefferson -d '%%jffs2-root%%' '%e':0:False
 
+# Use ubi_reader tool for UBIFS extraction
+^ubifs filesystem superblock node:ubi:ubireader_extract_files -o '%%ubifs-root%%' '%e':0:False
+^ubi erase count header:ubi:ubireader_extract_files -o '%%ubifs-root%%' '%e':0:False
+
+# Experimental yaffs extractor
+^yaffs filesystem:yaffs:yaffshiv --auto --brute-force -f '%e' -d '%%yaffs-root%%':0:False
+
 # These were extractors used from FMK that still need suitable replacements.
 #^bff volume entry:bff:/opt/firmware-mod-kit/src/bff/bffxtractor.py '%e'
 #^wdk file system:wdk:/opt/firmware-mod-kit/src/firmware-tools/unwdk.py '%e'
 
 # Extract, but don't run anything
-^ubifs filesystem:ubi
-^elf,:elf
-private key:key
-certificate:crt
-html document header
-xml document:xml
+#^elf,:elf
+#private key:key
+#certificate:crt
+#html document header
+#xml document:xml
 

src/binwalk/core/common.py

@@ -5,11 +5,15 @@ import os
 import re
 import sys
 import ast
-import hashlib
 import platform
 import operator as op
+import binwalk.core.idb
 from binwalk.core.compat import *
 
+# Don't try to import hashlib when loaded into IDA; it doesn't work.
+if not binwalk.core.idb.LOADED_IN_IDA:
+    import hashlib
+
 # The __debug__ value is a bit backwards; by default it is set to True, but
 # then set to False if the Python interpreter is run with the -O option.
 if not __debug__:
@@ -483,3 +487,4 @@ def BlockFile(fname, mode='r', subclass=io.FileIO, **kwargs):
             return (data, dlen)
 
     return InternalBlockFile(fname, mode=mode, **kwargs)
+

src/binwalk/core/display.py

@@ -33,6 +33,31 @@ class Display(object):
             if csv:
                 self.csv = pycsv.writer(self.fp)
 
+    def _fix_unicode(self, line):
+        '''
+        This is a hack, there must be a better way to handle it.
+        In Python3, if the environment variable LANG=C is set, indicating
+        that the terminal is ASCII only, but unicode characters need to be
+        printed to the screen or to a file (e.g., file path, magic result
+        format string), then an UnicodeEncodError exception will be raised.
+
+        This converts the given line to ASCII, ignoring conversion errors,
+        and returns a str.
+        '''
+        return bytes2str(line.encode('ascii', 'ignore'))
+
+    def _fix_unicode_list(self, columns):
+        '''
+        Convenience wrapper for self.log which is passed a list of format arguments.
+        '''
+        if type(columns) in [list, tuple]:
+            for i in range(0, len(columns)):
+                try:
+                    columns[i] = self._fix_unicode(columns[i])
+                except AttributeError:
+                    pass
+        return columns
+
     def format_strings(self, header, result):
         self.result_format = result
         self.header_format = header
@@ -43,9 +68,15 @@ class Display(object):
     def log(self, fmt, columns):
         if self.fp:
             if self.csv:
+                try:
                     self.csv.writerow(columns)
+                except UnicodeEncodeError:
+                    self.csv.writerow(self._fix_unicode_list(columns))
             else:
+                try:
                     self.fp.write(fmt % tuple(columns))
+                except UnicodeEncodeError:
+                    self.fp.write(fmt % tuple(self._fix_unicode_list(columns)))
 
             self.fp.flush()
 
@@ -100,6 +131,10 @@ class Display(object):
 
         if not self.quiet and stdout:
             try:
+                try:
+                    sys.stdout.write(self._format_line(line.strip()) + "\n")
+                except UnicodeEncodeError:
+                    line = self._fix_unicode(line)
                     sys.stdout.write(self._format_line(line.strip()) + "\n")
                 sys.stdout.flush()
             except IOError as e:

src/binwalk/core/magic.py

@@ -789,6 +789,7 @@ class Magic(object):
 
         Returns None.
         '''
+        # Magic files must be ASCII, else encoding issues can arise.
         fp = open(fname, "r")
         lines = fp.readlines()
         self.parse(lines)

src/binwalk/core/module.py

@@ -6,12 +6,15 @@
 import io
 import os
 import sys
+import time
 import inspect
 import argparse
 import traceback
+import binwalk.core.statuserver
 import binwalk.core.common
 import binwalk.core.settings
 import binwalk.core.plugin
+from threading import Thread
 from binwalk.core.compat import *
 
 class Option(object):
@@ -216,10 +219,11 @@ class Module(object):
     # Set to False if this is not a primary module (e.g., General, Extractor modules)
     PRIMARY = True
 
-    def __init__(self, **kwargs):
+    def __init__(self, parent, **kwargs):
         self.errors = []
         self.results = []
 
+        self.parent = parent
         self.target_file_list = []
         self.status = None
         self.enabled = False
@@ -258,6 +262,13 @@ class Module(object):
         '''
         return None
 
+    def unload(self):
+        '''
+        Invoked at module load time.
+        May be overridden by the module sub-class.
+        '''
+        return None
+
     def reset(self):
         '''
         Invoked only for dependency modules immediately prior to starting a new primary module.
@@ -336,6 +347,17 @@ class Module(object):
 
         return args
 
+    def _unload_dependencies(self):
+        # Calls the unload method for all dependency modules.
+        # These modules cannot be unloaded immediately after being run, as
+        # they must persist until the module that depends on them is finished.
+        # As such, this must be done separately from the Modules.run 'unload' call.
+        for dependency in self.dependencies:
+            try:
+                getattr(self, dependency.attribute).unload()
+            except AttributeError:
+                continue
+
     def next_file(self, close_previous=True):
         '''
         Gets the next file to be scanned (including pending extracted files, if applicable).
@@ -358,7 +380,7 @@ class Module(object):
 
         # Reset all dependencies prior to continuing with another file.
         # This is particularly important for the extractor module, which must be reset
-        # in order to reset it's base output directory path for each file, and the
+        # in order to reset its base output directory path for each file, and the
         # list of pending files.
         self.reset_dependencies()
 
@@ -386,8 +408,10 @@ class Module(object):
 
         if fp is not None:
             self.current_target_file_name = fp.path
+            self.status.fp = fp
         else:
             self.current_target_file_name = None
+            self.status.fp = None
 
         self.previous_next_file_fp = fp
 
@@ -426,6 +450,12 @@ class Module(object):
         self.validate(r)
         self._plugins_result(r)
 
+        # Update the progress status automatically if it is not being done manually by the module
+        if r.offset and r.file and self.AUTO_UPDATE_STATUS:
+            self.status.total = r.file.length
+            self.status.completed = r.offset
+            self.status.fp = r.file
+
         for dependency in self.dependencies:
             try:
                 getattr(self, dependency.attribute).callback(r)
@@ -435,11 +465,6 @@ class Module(object):
         if r.valid:
             self.results.append(r)
 
-            # Update the progress status automatically if it is not being done manually by the module
-            if r.offset and r.file and self.AUTO_UPDATE_STATUS:
-                self.status.total = r.file.length
-                self.status.completed = r.offset
-
             if r.display:
                 display_args = self._build_display_args(r)
                 if display_args:
@@ -499,14 +524,14 @@ class Module(object):
             if hasattr(self, dependency.attribute):
                 getattr(self, dependency.attribute).reset()
 
-    def main(self, parent):
+    def main(self):
         '''
         Responsible for calling self.init, initializing self.config.display, and calling self.run.
 
         Returns the value returned from self.run.
         '''
-        self.status = parent.status
-        self.modules = parent.loaded_modules
+        self.status = self.parent.status
+        self.modules = self.parent.executed_modules
 
         # A special exception for the extractor module, which should be allowed to
         # override the verbose setting, e.g., if --matryoshka has been specified
@@ -584,12 +609,25 @@ class Modules(object):
         Returns None.
         '''
         self.arguments = []
-        self.loaded_modules = {}
+        self.executed_modules = {}
         self.default_dependency_modules = {}
-        self.status = Status(completed=0, total=0)
+        self.status = Status(completed=0, total=0, fp=None, running=False, shutdown=False, finished=False)
+        self.status_server_started = False
+        self.status_service = None
 
         self._set_arguments(list(argv), kargv)
 
+    def cleanup(self):
+        if self.status_service:
+            self.status_service.server.socket.shutdown(1)
+            self.status_service.server.socket.close()
+
+    def __enter__(self):
+        return self
+
+    def __exit__(self, t, v, b):
+        self.cleanup()
+
     def _set_arguments(self, argv=[], kargv={}):
         for (k,v) in iterator(kargv):
             k = self._parse_api_opt(k)
@@ -691,7 +729,7 @@ class Modules(object):
             obj = self.run(module)
 
         # Add all loaded modules that marked themselves as enabled to the run_modules list
-        for (module, obj) in iterator(self.loaded_modules):
+        for (module, obj) in iterator(self.executed_modules):
             # Report the results if the module is enabled and if it is a primary module or if it reported any results/errors
             if obj.enabled and (obj.PRIMARY or obj.results or obj.errors):
                 run_modules.append(obj)
@@ -704,15 +742,29 @@ class Modules(object):
         '''
         Runs a specific module.
         '''
+        try:
             obj = self.load(module, kwargs)
 
             if isinstance(obj, binwalk.core.module.Module) and obj.enabled:
-            obj.main(parent=self)
+                obj.main()
                 self.status.clear()
 
-        # If the module is not being loaded as a dependency, add it to the loaded modules dictionary
+            # If the module is not being loaded as a dependency, add it to the executed modules dictionary.
+            # This is used later in self.execute to determine which objects should be returned.
             if not dependency:
-            self.loaded_modules[module] = obj
+                self.executed_modules[module] = obj
+
+                # The unload method tells the module that we're done with it, and gives it a chance to do
+                # any cleanup operations that may be necessary. We still retain the object instance in self.executed_modules.
+                obj._unload_dependencies()
+                obj.unload()
+        except KeyboardInterrupt as e:
+            # Tell the status server to shut down, and give it time to clean up.
+            if self.status.running:
+                self.status.shutdown = True
+                while not self.status.finished:
+                    time.sleep(0.1)
+            raise e
 
         return obj
 
@@ -720,7 +772,7 @@ class Modules(object):
         argv = self.argv(module, argv=self.arguments)
         argv.update(kwargs)
         argv.update(self.dependencies(module, argv['enabled']))
-        return module(**argv)
+        return module(self, **argv)
 
     def dependencies(self, module, module_enabled):
         import binwalk.modules
@@ -859,6 +911,21 @@ class Modules(object):
         else:
             raise Exception("binwalk.core.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(obj))
 
+    def status_server(self, port):
+        '''
+        Starts the progress bar TCP service on the specified port.
+        This service will only be started once per instance, regardless of the
+        number of times this method is invoked.
+
+        Failure to start the status service is considered non-critical; that is,
+        a warning will be displayed to the user, but normal operation will proceed.
+        '''
+        if self.status_server_started == False:
+            self.status_server_started = True
+            try:
+                self.status_service = binwalk.core.statuserver.StatusServer(port, self)
+            except Exception as e:
+                binwalk.core.common.warning("Failed to start status server on port %d: %s" % (port, str(e)))
 
 def process_kwargs(obj, kwargs):
     '''
@@ -869,7 +936,9 @@ def process_kwargs(obj, kwargs):
 
     Returns None.
     '''
-    return Modules().kwargs(obj, kwargs)
+    with Modules() as m:
+        kwargs = m.kwargs(obj, kwargs)
+    return kwargs
 
 def show_help(fd=sys.stdout):
     '''
@@ -879,6 +948,7 @@ def show_help(fd=sys.stdout):
 
     Returns None.
     '''
-    fd.write(Modules().help())
+    with Modules() as m:
+        fd.write(m.help())
 
 

src/binwalk/core/settings.py

@@ -17,7 +17,7 @@ class Settings:
         o PLUGINS             - Path to the plugins directory.
     '''
     # Release version
-    VERSION = "2.1.0"
+    VERSION = "2.1.2b"
 
     # Sub directories
     BINWALK_USER_DIR = ".binwalk"

src/binwalk/core/statuserver.py

+# Provides scan status information via a TCP socket service.
+# Currently only works for signature scans.
+
+import sys
+import time
+import errno
+import threading
+import binwalk.core.compat
+
+# Python 2/3 compatibility
+try:
+    import SocketServer
+except ImportError:
+    import socketserver as SocketServer
+
+class StatusRequestHandler(SocketServer.BaseRequestHandler):
+
+    def handle(self):
+        message_format = "%s     %3d%%     [ %d / %d ]"
+        last_status_message_len = 0
+        status_message = ''
+        message_sent = False
+
+        self.server.binwalk.status.running = True
+
+        while True:
+            time.sleep(0.1)
+
+            try:
+                self.request.send(binwalk.core.compat.str2bytes('\b' * last_status_message_len))
+                self.request.send(binwalk.core.compat.str2bytes(' ' * last_status_message_len))
+                self.request.send(binwalk.core.compat.str2bytes('\b' * last_status_message_len))
+
+                if self.server.binwalk.status.shutdown:
+                    self.server.binwalk.status.finished = True
+                    break
+
+                if self.server.binwalk.status.total != 0:
+                    percentage = ((float(self.server.binwalk.status.completed) / float(self.server.binwalk.status.total)) * 100)
+                    status_message = message_format % (self.server.binwalk.status.fp.path,
+                                                       percentage,
+                                                       self.server.binwalk.status.completed,
+                                                       self.server.binwalk.status.total)
+                elif not message_sent:
+                    status_message = "No status information available at this time!"
+                else:
+                    continue
+
+                last_status_message_len = len(status_message)
+                self.request.send(binwalk.core.compat.str2bytes(status_message))
+                message_sent = True
+            except IOError as e:
+                if e.errno == errno.EPIPE:
+                    break
+            except Exception as e:
+                binwalk.core.common.debug('StatusRequestHandler exception: ' + str(e) + '\n')
+            except KeyboardInterrupt as e:
+                raise e
+
+        self.server.binwalk.status.running = False
+        return
+
+class ThreadedStatusServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+    daemon_threads = True
+    allow_reuse_address = True
+
+class StatusServer(object):
+
+    def __init__(self, port, binwalk):
+        self.server = ThreadedStatusServer(('127.0.0.1', port), StatusRequestHandler)
+        self.server.binwalk = binwalk
+
+        t = threading.Thread(target=self.server.serve_forever)
+        t.setDaemon(True)
+        t.start()

src/binwalk/magic/binarch

@@ -17,15 +17,15 @@
 #
 # addiu $sp, XX
 # jr $ra
-0	belong	0x03e00008	MIPS instructions, function epilogue
->4	beshort	!0x27BD		{invalid}
-0	beshort	0x27BD		MIPS instructions, function epilogue
->2	belong	!0x03e00008	{invalid}
+0	ubelong	0x03e00008	MIPS instructions, function epilogue
+>4	ubeshort	!0x27BD		{invalid}
+0	ubeshort	0x27BD		MIPS instructions, function epilogue
+>2	ubelong	!0x03e00008	{invalid}
 
-0	lelong	0x03e00008	MIPSEL instructions, function epilogue
->6	leshort	!0x27BD		{invalid}
-0	leshort	0x27BD		MIPS instructions, function epilogue
->2	lelong	!0x03e00008	{invalid}
+0	ulelong	0x03e00008	MIPSEL instructions, function epilogue
+>6	uleshort	!0x27BD		{invalid}
+0	uleshort	0x27BD		MIPS instructions, function epilogue
+>2	ulelong	!0x03e00008	{invalid}
 
 # MIPS16e
 # nop (x4)
@@ -41,63 +41,63 @@
 # move $sp, $s1
 # restore XX, XX, XX
 # jrc $ra
-0	beshort	0x65B9			MIPS16e instructions, function epilogue
+0	ubeshort	0x65B9			MIPS16e instructions, function epilogue
 >3	byte	    !0x64			{invalid}
->4	beshort	!0xE8A0			{invalid}
-0	leshort	0x65B9			MIPSEL16e instructions, function epilogue
+>4	ubeshort	!0xE8A0			{invalid}
+0	uleshort	0x65B9			MIPSEL16e instructions, function epilogue
 >3	byte	    !0x64			{invalid}
->4	leshort	!0xE8A0			{invalid}
+>4	uleshort	!0xE8A0			{invalid}
 
 # jrc $ra
 # nop
-0	belong	0xe8a06500		MIPS16e instructions, function epilogue
-0	lelong	0xe8a06500		MIPSEL16e instructions, function epilogue
+0	ubelong	0xe8a06500		MIPS16e instructions, function epilogue
+0	ulelong	0xe8a06500		MIPSEL16e instructions, function epilogue
 
 # PowerPC prologue
 # mflr r0
-0	belong 0x7C0802A6	PowerPC big endian instructions, function prologue
-0	lelong 0x7C0802A6	PowerPC little endian instructions, function prologue
+0	ubelong 0x7C0802A6	PowerPC big endian instructions, function prologue
+0	ulelong 0x7C0802A6	PowerPC little endian instructions, function prologue
 
 # PowerPC epilogue
 # blr
-0	belong 0x4E800020	PowerPC big endian instructions, function epilogue
-0	lelong 0x4E800020	PowerPC little endian instructions, function epilogue
+0	ubelong 0x4E800020	PowerPC big endian instructions, function epilogue
+0	ulelong 0x4E800020	PowerPC little endian instructions, function epilogue
 
 # TODO: Add ARM Thumb dectection
 # ARM prologue
 # STMFD SP!, {XX}
 # <any instruction whose opcode begins with 0xE>
-0	beshort		0xE92D	ARMEB instructions, function prologue
+0	ubeshort	0xE92D	ARMEB instructions, function prologue
 >4	byte&0xF0	!0xE0	{invalid}
 >8	byte&0xF0	!0xE0	{invalid}
-0	leshort		0xE92D	ARM instructions, function prologue{adjust:-2}
+0	uleshort	0xE92D	ARM instructions, function prologue{adjust:-2}
 >5	byte&0xF0	!0xE0	{invalid}
 >9	byte&0xF0	!0xE0	{invalid}
 
 # ARM epilogue
 # MOV R0, XX
 # LDMFD SP!, {XX}
-0	beshort		0xE1A0	ARMEB instructions, function epilogue
+0	ubeshort	0xE1A0	ARMEB instructions, function epilogue
 >4	beshort		!0xE8BD	{invalid}
-0	leshort		0xE1A0	ARM instructions, function epilogue{adjust:-2}
+0	uleshort	0xE1A0	ARM instructions, function epilogue{adjust:-2}
 >4	leshort		!0xE8BD	{invalid}
 
 
 # Ubicom32 prologue
 # move.4 -4($sp)++, $ra
-0	belong	0x02FF6125	Ubicom32 instructions, function prologue
+0	ubelong	    0x02FF6125	Ubicom32 instructions, function prologue
 
 # Ubicom32 epilogues
 # calli $ra, 0($ra)
 # ret ($sp)4++
-0	belong	0xF0A000A0	Ubicom32 instructions, function epilogue
-0	belong	0x000022E1	Ubicom32 instructions, function epilogue
+0	ubelong	    0xF0A000A0	Ubicom32 instructions, function epilogue
+0	ubelong	    0x000022E1	Ubicom32 instructions, function epilogue
 
 # AVR8 prologue
 # push r28
 # push r29
-0	belong	0x93CF93DF	AVR8 instructions, function prologue
-0	belong	0x93DF93CF	AVR8 instructions, function prologue
+0	ubelong	    0x93CF93DF	AVR8 instructions, function prologue
+0	ubelong	    0x93DF93CF	AVR8 instructions, function prologue
 
 # AVR32 prologue
 # pushm   r7,lr

src/binwalk/magic/compressed

@@ -115,6 +115,7 @@
 >3      byte            &0x02           \b, has header CRC
 >3      byte&0x04       0x04
 >>10    leshort         x               \b, has %d bytes of extra data
+>>10    leshort         <0              {invalid}(invalid extra data size)               
 >3      byte&0xC        =0x08           \b, has original file name
 >>10    string          x               \b: "%s"{name:%s}
 >3      byte            &0x10           \b, has comment
@@ -137,14 +138,12 @@
 >3      byte            &0x20           \b, encrypted{invalid}
 # Dates before 1992 are {invalid}, unless of course you're DD-WRT in which
 # case you don't know how to set a date in your gzip files. Brilliant.
->4      lelong          =0              \b, NULL date (1970-01-01 00:00:00)
->4      lelong          <0              {invalid}
->4      lelong          >0
->>4     lelong          <694224000      {invalid}
->>4     lelong          =694224000      {invalid}
->>4     lelong          >694224000      \b, last modified:
->>>4    ledate          x               %s
->>>4    lelong          x               {epoch:%d}
+>4      ledate          x               \b, last modified: %s
+>4      lelong          =0              (null date)
+>4      lelong          !0
+>>4     lelong          <694224000      (bogus date)
+>>4     lelong          =694224000      (bogus date)
+
 
 # Supplementary magic data for the file(1) command to support
 # rzip(1).  The format is described in magic(5).

src/binwalk/magic/console

@@ -75,7 +75,7 @@
 >113    string  x               (%s)
 
 #------------------------------------------------------------------------------
-## Microsoft Xbox executables .xbe (Esa Hyytiä <ehyytia@cc.hut.fi>)
+## Microsoft Xbox executables .xbe (Esa Hyyti <ehyytia@cc.hut.fi>)
 0                       string              XBEH            Microsoft Xbox executable (XBE),
 ## probabilistic checks whether signed or not
 >0x0004                 ulelong             =0

src/binwalk/magic/crypto

@@ -6,6 +6,7 @@
 0       string  -----BEGIN\x20CERTIFICATE\x20REQ    PEM certificate request
 0       string  -----BEGIN\x20RSA\x20PRIVATE        PEM RSA private key
 0       string  -----BEGIN\x20DSA\x20PRIVATE        PEM DSA private key
+0       string  -----BEGIN\x20EC\x20PRIVATE         PEM EC private key
 
 # Type: OpenSSH key files
 # From: Nicolas Collignon <tsointsoin@gmail.com>
@@ -17,6 +18,9 @@
 
 0       string  ssh-dss\x20             OpenSSH DSA public key
 0       string  ssh-rsa\x20             OpenSSH RSA public key
+0       string  ecdsa-sha2-nistp256\x20 OpenSSH ECDSA (Curve P-256) public key
+0       string  ecdsa-sha2-nistp384\x20 OpenSSH ECDSA (Curve P-384) public key
+0       string  ecdsa-sha2-nistp521\x20 OpenSSH ECDSA (Curve P-521) public key
 
 # Type: Certificates/key files in DER format
 # From: Gert Hulselmans <hulselmansgert@gmail.com>

src/binwalk/magic/efi

+# http://blogs.phoenix.com/phoenix_technologies_bios/\
+# 2007/02/uefi_pi_10_firm.html
+
+40      string     _FVH              UEFI PI Firmware Volume
+>32     ulequad    >0xFFFFFFFF       {invalid} unrealistic size
+>32     ulequad    x                 \b, volume size: %d
+>52     uleshort   x                 \b, header size: %d
+>58     ubyte      !0                {invalid} reserved byte set
+>59     ubyte      >1                {invalid} invalid revision
+>59     ubyte      x                 \b, revision: %d
+
+# GUID: 7A9354D9-0468-444A-81CE-0BF617D890D
+>16     string     \xd9\x54\x93\x7a\x68\x04\x4a\x44\x81\xce\x0b\xf6\x17\xd8\x90\xdf  \b, EFI Firmware File System
+
+# GUID: 8C8CE578-8A3D-4F1C-9935-896185C32DD3
+>16     string     \x78\xe5\x8c\x8c\x3d\x8a\x1c\x4f\x99\x35\x89\x61\x85\xc3\x2d\xd3  \b, EFI Firmware File System v2
+
+# GUID: 04ADEEAD-61FF-4D31-B6BA-64F8BF901F5A
+>16     string     \xad\xee\xad\x04\xff\x61\x31\x4d\xb6\xba\x64\xf8\xbf\x90\x1f\x5a  \b, Apple Boot Volume
+
+# GUID: 8C1B00BD-716A-7B48-A14F-0C2A2DCF7A5D
+>16     string     \x8c\x1b\x00\xbd\x71\x6a\x7b\x48\xa1\x4f\x0c\x2a\x2d\xcf\x7a\x5d  \b, Apple Boot Volume v2
+
+# GUID: AD3FFFFF-D28B-44C4-9F13-9EA98A97F9F0
+>16     string     \xff\xff\x3f\xad\x8b\xd2\xc4\x44\x9f\x13\x9e\xa9\x8a\x97\xf9\xf0  \b, Intel v1
+
+# GUID: D6A1CD70-4B33-4994-A6EA-375F2CCC5437
+>16     string     \x70\xcd\xa1\xd6\x33\x4b\x94\x49\xa6\xea\x37\x5f\x2c\xcc\x54\x37  \b, Intel v2
+
+# GUID: 4F494156-AED6-4D64-A537-B8A5557BCEEC
+>16     string     \x56\x41\x49\x4f\xd6\xae\x64\x4d\xa5\x37\xb8\xa5\x55\x7b\xce\xec  \b, Sony v1
+
+# GUID: FFF12B8D-7696-4C8B-85A9-2747075B4F50
+>16     string     \x8d\x2b\xf1\xff\x96\x76\x8b\x4c\xa9\x85\x27\x47\x07\x5b\x4f\x50  \b, Variable Storage
+
+>16     ulelong    x                 \b, GUID: %.8X-
+>>20    uleshort   x                 \b%.4X-
+>>22    uleshort   x                 \b%.4X-
+>>24    uleshort   x                 \b%.4X-
+>>26    ubyte      x                 \b%.2X
+>>27    ubyte      x                 \b%.2X
+>>28    ubyte      x                 \b%.2X
+>>29    ubyte      x                 \b%.2X
+>>30    ubyte      x                 \b%.2X
+>>31    ubyte      x                 \b%.2X
+
+# http://www.intel.com/content/www/us/en/architecture-and-technology/\
+# unified-extensible-firmware-interface/efi-capsule-specification.html
+# GUID: 3B6686BD-0D76-4030-B70E-B5519E2FC5A0
+0       string     \xBD\x86\x66\x3B\x76\x0D\x30\x40\xB7\x0E\xB5\x51\x9E\x2F\xC5\xA0  EFI capsule v0.9
+>16     lelong     <0                {invalid}
+>16     lelong     x                 \b, header size: %d
+>20     lelong     x                 \b, flags: 0x%.8X
+>24     lelong     <0                {invalid}
+>24     lelong     x                 \b, capsule size: %d
+
+# Intel/UEFI format
+# http://www.uefi.org/sites/default/files/resources/UEFI%202_5.pdf
+# GUID: 539182B9-ABB5-4391-B69A-E3A943F72FCC
+0       string     \xb9\x82\x91\x53\xb5\xab\x91\x43\xb6\x9a\xe3\xa9\x43\xf7\x2f\xcc  UEFI capsule
+>16     lelong     <0                {invalid}
+>16     lelong     x                 \b, header size: %d
+>20     lelong     x                 \b, flags: 0x%.8X
+>24     lelong     <0                {invalid}
+>24     lelong     x                 \b, capsule size: %d
+
+# GUID: 4A3CA68B-7723-48FB-803D-578CC1FEC44D
+0       string     \x8b\xa6\x3c\x4a\x23\x77\xfb\x48\x80\x3d\x57\x8c\xc1\xfe\xc4\x4d  AMI Aptio extended EFI capsule
+>16     lelong     <0                {invalid}
+>16     lelong     x                 \b, header size: %d
+>20     lelong     x                 \b, flags: 0x%.8X
+>24     lelong     <0                {invalid}
+>24     lelong     x                 \b, capsule size: %d
+
+# GUID: 14EEBB90-890A-43DB-AED1-5D3C4588A418
+0       string     \x90\xbb\xee\x14\x0a\x89\xdb\x43\xae\xd1\x5d\x3c\x45\x88\xa4\x18  AMI Aptio unsigned EFI capsule
+>16     lelong     <0                {invalid}
+>16     lelong     x                 \b, header size: %d
+>20     lelong     x                 \b, flags: 0x%.8X
+>24     lelong     <0                {invalid}
+>24     lelong     x                 \b, capsule size: %d
+
+# GUID: 3BE07062-1D51-45D2-2B83-F093257ED461
+0       string     \x62\x70\xe0\x3b\x51\x1d\xd2\x45\x83\x2b\xf0\x93\x25\x7e\xd4\x61  Toshiba EFI capsule
+>16     lelong     <0                {invalid}
+>16     lelong     x                 \b, header size: %d
+>20     lelong     x                 \b, flags: 0x%.8X
+>24     lelong     <0                {invalid}
+>24     lelong     x                 \b, capsule size: %d

src/binwalk/magic/filesystems

@@ -34,7 +34,11 @@
 #>0x1e   string      minix       \b, bootable
 
 # YAFFS
-0    string    \x03\x00\x00\x00\x01\x00\x00\x00\xFF\xFF    YAFFS filesystem
+0    string     \x03\x00\x00\x00\x01\x00\x00\x00\xFF\xFF\x00\x00    YAFFS filesystem, little endian
+# The big endian signature has to be done a bit differently to prevent it from being self-overlapping
+4    string     \x00\x00\x00\x01\xFF\xFF                            YAFFS filesystem, big endian
+>0   string     !\x00\x00\x00\x03                                   {invalid}(first object is not a directory)
+>10  string     !\x00                                               {invalid}(unexpected name in the first object entry)
 
 # EFS2 file system - jojo@utulsa.edu
 0      lelong       0x53000000       EFS2 Qualcomm filesystem super block, little endian,
@@ -68,8 +72,11 @@
 
 # MPFS file system
 0       string      MPFS     MPFS filesystem, Microchop,
+>4      byte        <0       {invalid}
+>5      byte        <0       {invalid}
 >4      byte        x        version %d.
 >5      byte        x        \b%d,
+>6      leshort     <0       {invalid}
 >6      leshort     x        %d file entries
 
 # cramfs filesystem - russell@coker.com.au
@@ -116,15 +123,17 @@
 >28     string      !\x00*12        {invalid}
 
 # http://lxr.free-electrons.com/source/fs/ubifs/ubifs-media.h
-#0       string      UBI\x23         UBI erase count header,
-#>4      ubyte       x               version: %d,
-#>5      string      !\x00*3         {invalid}
-#>8      ubequad     x               EC: 0x%lX,
-#>16     ubelong     x               VID header offset: 0x%X,
-#>20     ubelong     x               data offset: 0x%X
+0       string      UBI\x23         UBI erase count header,
+>4      ubyte       x               version: %d,
+>5      string      !\x00*3         {invalid}
+>8      ubequad     x               EC: 0x%lX,
+>16     ubelong     x               VID header offset: 0x%X,
+>20     ubelong     x               data offset: 0x%X
+# dummy jump - actual jump value is determined in UBIValidPlugin
+>20       ubyte       x               {jump:0}
 
 # http://lxr.free-electrons.com/source/fs/ubifs/ubifs-media.h
-0        lelong      0x06101831     UBIFS
+0        lelong      0x06101831     UBIFS filesystem
 >20      ubyte       <6             {invalid}
 >20      ubyte       >7             {invalid} # Only look for superblock and master nodes
 >22      leshort     !0             {invalid} # 2 bytes of padding should be filled with NULLs

src/binwalk/magic/firmware

@@ -67,6 +67,10 @@
 >31    byte      3              compression type: lzma,
 >32    string    x              image name: "%s"
 
+# Hilink encrypted uImage firmware.
+# Additional validation/processing is done by the hilink.py plugin.
+0x23   string    \x4A\x52\xCA\xDA     Encrypted Hilink uImage firmware header
+
 #IMG0 header, found in VxWorks-based Mercury router firmware
 0       string        IMG0    IMG0 (VxWorks) header,
 >4      belong        <1      {invalid}
@@ -495,8 +499,6 @@
 >35     byte        x           try decryption tool from:
 >35     byte        x           http://download.modem-help.co.uk/mfcs-A/Alcatel/Modems/Misc/
 
-16    string        \xd9\x54\x93\x7a\x68\x04\x4a\x44\x81\xce\x0b\xf6\x17\xd8\x90\xdf    UEFI PI firmware volume
-
 # http://android.stackexchange.com/questions/23357/\
 # is-there-a-way-to-look-inside-and-modify-an-adb-backup-created-file/\
 # 23608#23608
@@ -690,5 +692,13 @@
 >44 lelong &0x4 SLT
 >44 lelong &0xffffff00 {invalid}
 
+# Android bootimg
+# https://android.googlesource.com/platform/system/core.git/+/master/mkbootimg/bootimg.h
+0     string     ANDROID!      Android bootimg
+>8    ulelong    x             \b, kernel size: %d bytes
+>12   ulelong    x             \b, kernel addr: 0x%X
+>16   ulelong    x             \b, ramdisk size: %d bytes
+>20   ulelong    x             \b, ramdisk addr: 0x%X
+>48   string     x             \b, product name: "%s"
 
 

src/binwalk/magic/linux

@@ -3,7 +3,7 @@
 
 # Linux kernel boot images, from Albert Cahalan <acahalan@cs.uml.edu>
 # and others such as Axel Kohlmeyer <akohlmey@rincewind.chemie.uni-ulm.de>
-# and Nicolás Lichtmaier <nick@debian.org>
+# and Nicolas Lichtmaier <nick@debian.org>
 # All known start with: b8 c0 07 8e d8 b8 00 90 8e c0 b9 00 01 29 f6 29
 0       string      \xb8\xc0\x07\x8e\xd8\xb8\x00\x90\x8e\xc0\xb9\x00\x01\x29\xf6\x29    Linux kernel boot image
 >514    string      !HdrS                                                               {invalid}
@@ -16,14 +16,23 @@
 >>14    string      x                       "%s"
 
 # Linux ARM compressed kernel image
-# See arch/arm/boot/compressed/head.S and arch/arm/boot/compressed/vmlinux.lds.S
-0	ulelong	0x016f2818	Linux kernel ARM boot executable zImage (little-endian),
->4	ulelong x		    load address: "0x%.8X",
->8	ulelong	x		    end address: "0x%.8X"
->12 ulelong !0x04030201 {invalid}
-
-0	ubelong	0x016f2818	Linux kernel ARM boot executable zImage (big-endian),
->4	ubelong x		    load address: "0x%.8X",
->8	ubelong	x		    end address: "0x%.8X"
->12 ubelong !0x04030201 {invalid}
+# Starts with 8 NOPs, with 0x016F2818 at offset 0x24
+36  ulelong 0x016F2818                      Linux kernel ARM boot executable zImage (little-endian)
+>0  ulelong !0xE1A00000                     {invalid}(invalid)
+>4  ulelong !0xE1A00000                     {invalid}(invalid)
+>8  ulelong !0xE1A00000                     {invalid}(invalid)
+>12 ulelong !0xE1A00000                     {invalid}(invalid)
+>16 ulelong !0xE1A00000                     {invalid}(invalid)
+>20 ulelong !0xE1A00000                     {invalid}(invalid)
+>24 ulelong !0xE1A00000                     {invalid}(invalid)
+>28 ulelong !0xE1A00000                     {invalid}(invalid)
 
+36  ubelong 0x016F2818                      Linux kernel ARM boot executable zImage (big-endian)
+>0  ubelong !0xE1A00000                     {invalid}(invalid)
+>4  ubelong !0xE1A00000                     {invalid}(invalid)
+>8  ubelong !0xE1A00000                     {invalid}(invalid)
+>12 ubelong !0xE1A00000                     {invalid}(invalid)
+>16 ubelong !0xE1A00000                     {invalid}(invalid)
+>20 ubelong !0xE1A00000                     {invalid}(invalid)
+>24 ubelong !0xE1A00000                     {invalid}(invalid)
+>28 ubelong !0xE1A00000                     {invalid}(invalid)

src/binwalk/magic/misc

@@ -41,8 +41,18 @@
 # CodeGate 2011 http://nopsrus.blogspot.com/2013/05/codegate-ctf-2011-binary-100-points.html
 0    string    \x23\x40\x7e\x5e    Windows Script Encoded Data (screnc.exe)
 
-0   regex   /[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_/].*       Unix path:
+0           regex   /[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_/].*       Unix path:
 >0          string  x                                                                        %s
+>0          string  !/home/
+>>0         string  !/bin/
+>>>0        string  !/sbin/
+>>>>0       string  !/usr/
+>>>>>0      string  !/sys/
+>>>>>>0     string  !/var/
+>>>>>>>0    string  !/opt/
+>>>>>>>>0   string  !/etc/
+>>>>>>>>>0  string  !/lib/
+>>>>>>>>>>0 string  !/dev/                                                                   {invalid}(likely false positive)
 
 0   string      neighbor           Neighborly text,
 >0  string      x                  "%s

src/binwalk/magic/phones

+# From: http://arm.ninja/2016/03/04/reverse-engineering-samsung-s6-modem/
+0           string      TOC\x00\x00\x00\x00     Samsung modem TOC index,
+>20         lelong      x                       size: 0x%X bytes,
+>24         lelong      !0                      invalid TOC CRC,{invalid}
+>28         lelong      !1                      invalid TOC index{invalid}
+>0x20       string      x                       TOC entries: %s
+>0x40       byte        !0
+>>0x40      string      x                       \b, %s
+>0x60       byte        !0
+>>0x60      string      x                       \b, %s
+>0x80       byte        !0
+>>0x80      string      x                       \b, %s
+>0xA0       byte        !0
+>>0xA0      string      x                       \b, %s
+>0xC0       byte        !0
+>>0xC0      string      x                       \b, %s
+>0xE0       byte        !0
+>>0xE0      string      x                       \b, %s
+>0x100      byte        !0
+>>0x100     string      x                       \b, %s

src/binwalk/modules/compression.py

@@ -2,11 +2,14 @@
 
 import os
 import zlib
-import lzma
 import struct
 import binwalk.core.compat
 import binwalk.core.common
 from binwalk.core.module import Option, Kwarg, Module
+try:
+    import lzma
+except ImportError:
+    from backports import lzma
 
 class LZMAHeader(object):
     def __init__(self, **kwargs):
@@ -159,18 +162,38 @@ class Deflate(object):
             self.module.extractor.add_rule(regex='^%s' % self.DESCRIPTION.lower(), extension="deflate", cmd=self.extractor)
 
     def extractor(self, file_name):
+        in_data = ""
+        out_data = ""
+        retval = False
         out_file = os.path.splitext(file_name)[0]
 
+        with binwalk.core.common.BlockFile(file_name, 'r') as fp_in:
+            while True:
+                (data, dlen) = fp_in.read_block()
+                if not data or dlen == 0:
+                    break
+                else:
+                    in_data += data
+
+                try:
+                    out_data = zlib.decompress(binwalk.core.compat.str2bytes(in_data), -15)
+                    with binwalk.core.common.BlockFile(out_file, 'w') as fp_out:
+                        fp_out.write(out_data)
+                    retval = True
+                    break
+                except zlib.error as e:
+                    pass
+
+        return retval
+
     def decompress(self, data):
         valid = True
         description = None
 
-        # Prepend data with a standard zlib header
-        data = "\x78\x9C" + data
-
         # Looking for either a valid decompression, or an error indicating truncated input data
         try:
-            zlib.decompress(binwalk.core.compat.str2bytes(data))
+            # Negative window size (e.g., -15) indicates that raw decompression should be performed
+            zlib.decompress(binwalk.core.compat.str2bytes(data), -15)
         except zlib.error as e:
             if not str(e).startswith("Error -5"):
                 # Bad data.

src/binwalk/modules/entropy.py

@@ -3,7 +3,6 @@
 import os
 import math
 import zlib
-import multiprocessing
 import binwalk.core.common
 from binwalk.core.compat import *
 from binwalk.core.module import Module, Option, Kwarg
@@ -113,20 +112,13 @@ class Entropy(Module):
             else:
                 self.block_size = None
 
+    def _entropy_sigterm_handler(self, *args):
+        print ("FUck it all.")
+
     def run(self):
-        # Need to invoke the pyqtgraph stuff via a separate process, as calling pg.exit
-        # is pretty much required. pg.exit calls os._exit though, and we don't want to
-        # exit out of the main process (especially if being run via the API).
-        if not binwalk.core.common.MSWindows():
-            p = multiprocessing.Process(target=self._run)
-            p.start()
-            p.join()
-        else:
-            # There seem to be all kinds of issues using the multiprocessing module in
-            # Windows, as done above.
-            #
-            # This means that when run in Windows, pg.exit will cause binwalk
-            # to exit.
+        # If generating a graphical plot, this function will never return, as it invokes
+        # pg.exit. Calling pg.exit is pretty much required, but pg.exit calls os._exit in
+        # order to work around QT cleanup issues.
         self._run()
 
     def _run(self):
@@ -135,7 +127,7 @@ class Entropy(Module):
             try:
                 import pyqtgraph as pg
             except ImportError as e:
-                binwalk.core.common.warning("pyqtgraph not found, visual entropy graphing will be disabled")
+                binwalk.core.common.warning("Failed to import pyqtgraph module, visual entropy graphing will be disabled")
                 self.do_plot = False
 
         for fp in iter(self.next_file, None):

src/binwalk/modules/general.py

@@ -80,6 +80,11 @@ class General(Module):
                type=str,
                kwargs={'file_name_exclude_regex' : ""},
                description='Do not scan files whose names match this regex'),
+        Option(short='s',
+               long='status',
+               type=int,
+               kwargs={'status_server_port' : 0},
+               description='Enable the status server on the specified port'),
         Option(long=None,
                short=None,
                type=binwalk.core.common.BlockFile,
@@ -96,6 +101,7 @@ class General(Module):
         Kwarg(name='offset', default=0),
         Kwarg(name='base', default=0),
         Kwarg(name='block', default=0),
+        Kwarg(name='status_server_port', default=0),
         Kwarg(name='swap_size', default=0),
         Kwarg(name='log_file', default=None),
         Kwarg(name='csv', default=False),
@@ -113,6 +119,7 @@ class General(Module):
     PRIMARY = False
 
     def load(self):
+        self.threads_active = False
         self.target_files = []
 
         # A special case for when we're loaded into IDA
@@ -141,6 +148,9 @@ class General(Module):
             if not binwalk.core.idb.LOADED_IN_IDA:
                 sys.exit(0)
 
+        if self.status_server_port > 0:
+            self.parent.status_server(self.status_server_port)
+
     def reset(self):
         pass
 
@@ -181,7 +191,13 @@ class General(Module):
         if swap is None:
             swap = self.swap_size
 
-        return binwalk.core.common.BlockFile(fname, subclass=self.subclass, length=length, offset=offset, swap=swap, block=block, peek=peek)
+        return binwalk.core.common.BlockFile(fname,
+                                             subclass=self.subclass,
+                                             length=length,
+                                             offset=offset,
+                                             swap=swap,
+                                             block=block,
+                                             peek=peek)
 
     def _open_target_files(self):
         '''

src/binwalk/modules/hexdiff.py

@@ -118,6 +118,13 @@ class HexDiff(Module):
         loop_count = 0
         sep_count = 0
 
+        # Figure out the maximum diff size (largest file size)
+        self.status.total = 0
+        for i in range(0, len(target_files)):
+            if target_files[i].size > self.status.total:
+                self.status.total = target_files[i].size
+                self.status.fp = target_files[i]
+
         while True:
             line = ""
             done_files = 0
@@ -168,6 +175,7 @@ class HexDiff(Module):
 
             last_line = line
             loop_count += 1
+            self.status.completed += self.block
 
     def init(self):
         # To mimic expected behavior, if all options are False, we show everything

src/binwalk/plugins/compressd.py

-import binwalk.core.C
+#import binwalk.core.C
 import binwalk.core.plugin
-from binwalk.core.common import *
+#from binwalk.core.common import *
 
 class CompressdPlugin(binwalk.core.plugin.Plugin):
-    '''
-    Searches for and validates compress'd data.
-    '''
+#    '''
+#    Searches for and validates compress'd data.
+#    '''
 
     MODULES = ['Signature']
 
-    READ_SIZE = 64
+    #READ_SIZE = 64
 
-    COMPRESS42 = "compress42"
+    #COMPRESS42 = "compress42"
     #COMPRESS42_FUNCTIONS = [
     #    binwalk.core.C.Function(name="is_compressed", type=bool),
     #]
 
-    comp = None
+    #comp = None
 
-    def init(self):
+    #def init(self):
         #self.comp = binwalk.core.C.Library(self.COMPRESS42, self.COMPRESS42_FUNCTIONS)
         # This plugin is currently disabled due to the need to move away from supporting C
         # libraries and into a pure Python project, for cross-platform support and ease of
         # installation / package maintenance. A Python implementation will likely need to
         # be custom developed in the future, but for now, since this compression format is
         # not very common, especially in firmware, simply disable it.
-        self.comp = None
+        #self.comp = None
 
-    def scan(self, result):
-        if self.comp and result.file and result.description.lower().startswith("compress'd data"):
-            fd = self.module.config.open_file(result.file.name, offset=result.offset, length=self.READ_SIZE)
-            compressed_data = fd.read(self.READ_SIZE)
-            fd.close()
+    #def scan(self, result):
+    #    if self.comp and result.file and result.description.lower().startswith("compress'd data"):
+    #        fd = self.module.config.open_file(result.file.name, offset=result.offset, length=self.READ_SIZE)
+    #        compressed_data = fd.read(self.READ_SIZE)
+    #        fd.close()
 
-            if not self.comp.is_compressed(compressed_data, len(compressed_data)):
-                result.valid = False
+    #        if not self.comp.is_compressed(compressed_data, len(compressed_data)):
+    #            result.valid = False
 
 

src/binwalk/plugins/hilink.py

+#!/usr/bin/env python
+
+import struct
+import string
+import binwalk.core.plugin
+import binwalk.core.compat
+import binwalk.core.common
+try:
+    # Requires the pycrypto library
+    from Crypto.Cipher import DES
+except ImportError as e:
+    DES = None
+
+
+class HilinkDecryptor(binwalk.core.plugin.Plugin):
+    '''
+    Plugin to decrypt, validate, and extract Hilink encrypted firmware.
+    '''
+    MODULES = ["Signature"]
+
+    DES_KEY = "H@L9K*(3"
+    SIGNATURE_DESCRIPTION = "Encrypted Hilink uImage firmware".lower()
+
+    def init(self):
+        if DES is None:
+            self.enabled = False
+        else:
+            self.enabled = True
+
+        if self.enabled is True and self.module.extractor.enabled is True:
+            # Add an extraction rule for encrypted Hilink firmware signature results
+            self.module.extractor.add_rule(regex="^%s" % self.SIGNATURE_DESCRIPTION,
+                                           extension="enc",
+                                           cmd=self._decrypt_and_extract)
+
+
+    def _decrypt_and_extract(self, fname):
+        '''
+        This does the extraction (e.g., it decrypts the image and writes it to a new file on disk).
+        '''
+        with open(fname, "r") as fp_in:
+            encrypted_data = fp_in.read()
+
+            decrypted_data = self._hilink_decrypt(encrypted_data)
+
+            with open(binwalk.core.common.unique_file_name(fname[:-4], "dec"), "w") as fp_out:
+                fp_out.write(decrypted_data)
+
+    def _hilink_decrypt(self, encrypted_firmware):
+        '''
+        This does the actual decryption.
+        '''
+        cipher = DES.new(self.DES_KEY, DES.MODE_ECB)
+
+        p1 = encrypted_firmware[0:3]
+        p2 = encrypted_firmware[3:]
+        p2 += b"\x00" * (8 - (len(p2) % 8))
+
+        d1 = p1 + cipher.decrypt(p2)
+        d1 += b"\x00" * (8 - (len(d1) % 8))
+
+        return cipher.decrypt(d1)
+
+    def scan(self, result):
+        '''
+        Validate signature results.
+        '''
+        if self.enabled is True:
+            if result.valid is True:
+                if result.description.lower().startswith(self.SIGNATURE_DESCRIPTION) is True:
+                    # Read in the first 64 bytes of the suspected encrypted uImage header
+                    fd = self.module.config.open_file(result.file.name, offset=result.offset)
+                    encrypted_header_data = binwalk.core.compat.str2bytes(fd.read(64))
+                    fd.close()
+
+                    # Decrypt the header
+                    decrypted_header_data = self._hilink_decrypt(encrypted_header_data)
+
+                    # Pull out the image size and image name fields from the decrypted uImage header
+                    # and add them to the printed description.
+                    result.size = struct.unpack(b">L", decrypted_header_data[12:16])[0]
+                    result.description += ", size: %d" % (result.size)
+                    # NOTE: The description field should be 32 bytes? Hilink seems to use only 24 bytes for this field,
+                    #       even though the header size is still 64 bytes?
+                    result.description += ', image name: "%s"' % binwalk.core.compat.bytes2str(decrypted_header_data[32:56]).strip("\x00")
+
+                    # Do some basic validation on the decrypted size and image name fields
+                    if result.size > (result.file.size - result.offset):
+                        result.valid = False
+                    if not all(c in string.printable for c in result.description):
+                        result.valid = False
+

src/binwalk/plugins/lzmaextract.py

@@ -9,7 +9,14 @@ class LZMAExtractPlugin(binwalk.core.plugin.Plugin):
 
     def init(self):
         try:
+            # lzma package in Python 2.0 decompress() does not handle multiple
+            # compressed streams, only first stream is extracted.
+            # backports.lzma package could be used to keep consistent behaviour.
+            try:
                 import lzma
+            except ImportError:
+                from backports import lzma
+
             self.decompressor = lzma.decompress
 
             # If the extractor is enabled for the module we're currently loaded
@@ -19,6 +26,10 @@ class LZMAExtractPlugin(binwalk.core.plugin.Plugin):
                                                regex="^lzma compressed data",
                                                extension="7z",
                                                cmd=self.extractor)
+                self.module.extractor.add_rule(txtrule=None,
+                                               regex="^xz compressed data",
+                                               extension="xz",
+                                               cmd=self.extractor)
         except ImportError as e:
             pass
 

src/binwalk/plugins/lzmavalid.py

@@ -17,7 +17,10 @@ class LZMAPlugin(binwalk.core.plugin.Plugin):
 
     def init(self):
         try:
+            try:
                 import lzma
+            except ImportError:
+                from backports import lzma
             self.decompressor = lzma.decompress
         except ImportError as e:
             self.decompressor = None

src/binwalk/plugins/ubivalid.py

+import struct
+import binascii
+import binwalk.core.plugin
+import binwalk.core.compat
+
+class UBIValidPlugin(binwalk.core.plugin.Plugin):
+    '''
+    Helps validate UBI erase count signature results.
+
+    Checks header CRC and calculates jump value
+    '''
+    MODULES = ['Signature']
+    current_file=None
+    last_ec_hdr_offset = None
+    peb_size = None
+
+    def _check_crc(self, ec_header):
+        # Get the header's reported CRC value
+        header_crc = struct.unpack(">I", ec_header[60:64])[0]
+
+        # Calculate the actual CRC
+        calculated_header_crc = ~binascii.crc32(ec_header[0:60]) & 0xffffffff
+
+        # Make sure they match
+        return header_crc == calculated_header_crc
+
+    def _process_result(self, result):
+        if self.current_file == result.file.name:
+            result.display=False
+        else:
+            # Reset everything in case new file is encountered
+            self.peb_size=None
+            self.last_ec_hdr_offset=None
+            self.peb_size=None
+
+            # Display result and trigger extraction
+            result.display=True
+
+        self.current_file = result.file.name
+
+        if not self.peb_size and self.last_ec_hdr_offset:
+            # Calculate PEB size by subtracting last EC block offset
+            self.peb_size = result.offset - self.last_ec_hdr_offset
+        else:
+            # First time plugin is called on file, save EC block offset
+            self.last_ec_hdr_offset = result.offset
+
+        if self.peb_size:
+            # If PEB size has been determined jump PEB size
+            result.jump = self.peb_size
+        else:
+            result.jump = 0
+
+    def scan(self, result):
+        if result.file and result.description.lower().startswith('ubi erase count header'):
+            # Seek to and read the suspected UBI erase count header
+            fd = self.module.config.open_file(result.file.name, offset=result.offset)
+
+            ec_header = binwalk.core.compat.str2bytes(fd.read(1024))
+            fd.close()
+
+            result.valid = self._check_crc(ec_header[0:64])
+            if result.valid:
+                self._process_result(result)

src/scripts/binwalk

@@ -2,7 +2,6 @@
 
 import os
 import sys
-from threading import Thread
 
 # If installed to a custom prefix directory, binwalk may not be in
 # the default module search path(s). Try to resolve the prefix module
@@ -15,7 +14,8 @@ for _module_path in [
     # from build dir: build/scripts-3.4/ -> build/lib/
     os.path.join(_parent_dir, "lib"),
     # installed in non-default path: bin/ -> lib/python3.4/site-packages/
-    os.path.join(_parent_dir, "lib",
+    os.path.join(_parent_dir,
+                 "lib",
                  "python%d.%d" % (sys.version_info[0], sys.version_info[1]),
                  "site-packages")
 ]:
@@ -24,37 +24,13 @@ for _module_path in [
 
 import binwalk
 import binwalk.modules
-from binwalk.core.compat import user_input
-
-def display_status(m):
-    # Display the current scan progress when the enter key is pressed.
-    while True:
-        try:
-            user_input()
-            percentage = ((float(m.status.completed) / float(m.status.total)) * 100)
-            sys.stderr.write("Progress: %.2f%% (%d / %d)\n\n" % (percentage,
-                                                                 m.status.completed,
-                                                                 m.status.total))
-        except KeyboardInterrupt as e:
-            raise e
-        except Exception:
-            pass
-
-def usage(modules):
-    sys.stderr.write(modules.help())
-    sys.exit(1)
 
 def main():
-    modules = binwalk.Modules()
-
-    # Start the display_status function as a daemon thread.
-    t = Thread(target=display_status, args=(modules,))
-    t.setDaemon(True)
-    t.start()
-
+    with binwalk.Modules() as modules:
         try:
             if len(sys.argv) == 1:
-            usage(modules)
+                sys.stderr.write(modules.help())
+                sys.exit(1)
             # If no explicit module was enabled in the command line arguments,
             # run again with the default signature scan explicitly enabled.
             elif not modules.execute():
@@ -66,9 +42,9 @@ def main():
                 else:
                     sys.stderr.write("Error: Signature scans not supported; ")
                     sys.stderr.write("make sure you have python-lzma installed and try again.\n")
-                sys.exit(1)
+                    sys.exit(2)
         except binwalk.ModuleException as e:
-        sys.exit(1)
+            sys.exit(3)
 
 if __name__ == '__main__':
     try: