Skip to content

B
binwalk
Commit d394c490 by Martin Sundhaug
Options

Merge branch 'master' of https://github.com/devttys0/binwalk

parents 617b4a63 85c522b9
Showing with 854 additions and 191 deletions
API.md 100755 → 100644
File mode changed from 100755 to 100644
INSTALL.md
......@@ -31,6 +31,18 @@ Although all binwalk run-time dependencies are optional, the `python-lzma` modul
$ sudo apt-get install python-lzma
```
Binwalk uses the `pycrypto` library to decrypt some known encrypted firmware images:
```bash
# Python2.7
$ sudo apt-get install python-crypto
```
```bash
# Python3.x
$ sudo apt-get install python3-crypto
```
Binwalk uses [pyqtgraph](http://www.pyqtgraph.org) to generate graphs and visualizations, which requires the following:
```bash
......@@ -70,7 +82,7 @@ $ sudo apt-get install mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabex
# Install sasquatch to extract non-standard SquashFS images
$ sudo apt-get install zlib1g-dev liblzma-dev liblzo2-dev
$ git clone https://github.com/devttys0/sasquatch
$ (cd sasquatch && make && sudo make install)
$ (cd sasquatch && ./build.sh)
```
```bash
......@@ -81,6 +93,19 @@ $ (cd jefferson && sudo python setup.py install)
```
```bash
# Install ubi_reader to extract UBIFS file systems
$ sudo apt-get install liblzo2-dev python-lzo
$ git clone https://github.com/jrspruitt/ubi_reader
$ (cd ubi_reader && sudo python setup.py install)
```
```bash
# Install yaffshiv to extract YAFFS file systems
$ git clone https://github.com/devttys0/yaffshiv
$ (cd yaffshiv && sudo python setup.py install)
```
```bash
# Install unstuff (closed source) to extract StuffIt archive files
$ wget -O - http://my.smithmicro.com/downloads/files/stuffit520.611linux-i386.tar.gz | tar -zxv
$ sudo cp bin/unstuff /usr/local/bin/
......
deps.sh
#!/bin/bash
set -o nounset
REQUIRED_UTILS="wget tar python"
APTCMD="apt-get"
YUMCMD="yum"
APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev"
PYTHON2_APT_CANDIDATES="python-lzma python-pip python-opengl python-qt4 python-qt4-gl python-numpy python-scipy"
PYTHON3_APT_CANDIDATES="python3-pip python3-opengl python3-pyqt4 python3-pyqt4.qtopengl python3-numpy python3-scipy"
APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit openjdk-7-jdk"
PYTHON2_APT_CANDIDATES="python-crypto python-lzo python-lzma python-pip python-opengl python-qt4 python-qt4-gl python-numpy python-scipy"
PYTHON3_APT_CANDIDATES="python3-crypto python3-pip python3-opengl python3-pyqt4 python3-pyqt4.qtopengl python3-numpy python3-scipy"
PYTHON3_YUM_CANDIDATES=""
YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma"
PYTHON2_YUM_CANDIDATES="python-pip python-opengl python-qt4 numpy python-numdisplay numpy-2f python-Bottleneck scipy"
......@@ -25,13 +26,13 @@ fi
function install_sasquatch
{
git clone https://github.com/devttys0/sasquatch
(cd sasquatch && make && $SUDO make install)
(cd sasquatch && $SUDO ./build.sh)
$SUDO rm -rf sasquatch
}
function install_jefferson
{
$SUDO pip install cstruct
install_pip_package cstruct
git clone https://github.com/sviehb/jefferson
(cd jefferson && $SUDO python2 setup.py install)
$SUDO rm -rf jefferson
......@@ -47,6 +48,13 @@ function install_unstuff
rm -rf /tmp/unstuff
}
function install_ubireader
{
git clone https://github.com/jrspruitt/ubi_reader
(cd ubi_reader && $SUDO python setup.py install)
$SUDO rm -rf ubi_reader
}
function install_pip_package
{
PACKAGE="$1"
......@@ -137,10 +145,16 @@ fi
# Do the install(s)
cd /tmp
sudo $PKGCMD $PKGCMD_OPTS $PKG_CANDIDTES
sudo $PKGCMD $PKGCMD_OPTS $PKG_CANDIDATES
if [ $? -ne 0 ]
then
echo "Package installation failed: $PKG_CANDIDATES"
exit 1
fi
install_pip_package pyqtgraph
install_pip_package capstone
install_sasquatch
install_jefferson
install_unstuff
install_ubireader
images/README.md 0 → 100644
This is just a directory to store screenshots used in the github Wiki / documentation.
setup.py
......@@ -8,6 +8,7 @@ from distutils.core import setup, Command
from distutils.dir_util import remove_tree
MODULE_NAME = "binwalk"
SCRIPT_NAME = MODULE_NAME
# Python2/3 compliance
try:
......@@ -200,7 +201,7 @@ for data_dir in ["magic", "config", "plugins", "modules", "core"]:
# Install the module, script, and support files
setup(name = MODULE_NAME,
version = "2.1.0",
version = "2.1.2b",
description = "Firmware analysis tool",
author = "Craig Heffner",
url = "https://github.com/devttys0/%s" % MODULE_NAME,
......@@ -208,7 +209,7 @@ setup(name = MODULE_NAME,
requires = [],
packages = [MODULE_NAME],
package_data = {MODULE_NAME : install_data_files},
scripts = [os.path.join("scripts", MODULE_NAME)],
scripts = [os.path.join("scripts", SCRIPT_NAME)],
cmdclass = {'clean' : CleanCommand, 'uninstall' : UninstallCommand, 'idainstall' : IDAInstallCommand, 'idauninstall' : IDAUnInstallCommand}
)
......
src/binwalk/__init__.py
__all__ = ['scan', 'execute', 'Modules', 'ModuleException']
__all__ = ['scan', 'execute', 'ModuleException']
from binwalk.core.module import Modules, ModuleException
# Convenience functions
def scan(*args, **kwargs):
return Modules(*args, **kwargs).execute()
with Modules(*args, **kwargs) as m:
objs = m.execute()
return objs
def execute(*args, **kwargs):
return Modules(*args, **kwargs).execute()
return scan(*args, **kwargs)
src/binwalk/config/extract.conf
......@@ -13,23 +13,23 @@
#
# o zlib
# o cpio
# o jffs2
# o Raw LZMA/deflate streams
# o Hilink encrypted uImage firmware
#
# There are also alternative extractors for the following file formats, implemented as plugins:
#
# o gzip
# o lzma
# o xz
#
#######################################################################################################################################
# Assumes these utilities are installed in $PATH.
^gzip compressed data:gz:gzip -d -f '%e':0,2
^lzma compressed data:7z:7z e -y '%e':0,1
^xz compressed data:tar:tar xJf '%e'
^xz compressed data:xz:7z e -y '%e':0,1
^bzip2 compressed data:bz2:bzip2 -d '%e'
^compress'd data:Z:gzip -d '%e'
^zip archive data:zip:7z x -y '%e' -p '':0,1
^posix tar archive:tar:tar xvf '%e'
^rar archive data:rar:unrar e '%e'
^rar archive data:rar:unrar -x '%e' # This is for the 'free' version
......@@ -39,6 +39,17 @@
^microsoft cabinet archive:cab:cabextract '%e'
^stuffit:sit:unstuff '%e'
# jar just does a better job of extracting zip files than either
# unzip or 7z.
#
# unzip does not support files that are "missing end of central directory header".
#
# 7z handles most zip files, but fails on some zip archives, inexplicably seeing
# only the *last* entry in the zip archive (though 7z thinks it succeeded). See
# StarCam firmware CH-sys-48.53.64.67.zip.
^zip archive data:zip:jar xvf '%e':0
^zip archive data:zip:7z x -y '%e' -p '':0,1
# Try unsquashfs first, or if not installed, sasquatch
^squashfs filesystem:squashfs:unsquashfs -d '%%squashfs-root%%' '%e':0:False
^squashfs filesystem:squashfs:sasquatch -p 1 -le -d '%%squashfs-root%%' '%e':0:False
......@@ -60,15 +71,21 @@
# Use sviehb's jefferson.py tool for JFFS2 extraction
^jffs2 filesystem:jffs2:jefferson -d '%%jffs2-root%%' '%e':0:False
# Use ubi_reader tool for UBIFS extraction
^ubifs filesystem superblock node:ubi:ubireader_extract_files -o '%%ubifs-root%%' '%e':0:False
^ubi erase count header:ubi:ubireader_extract_files -o '%%ubifs-root%%' '%e':0:False
# Experimental yaffs extractor
^yaffs filesystem:yaffs:yaffshiv --auto --brute-force -f '%e' -d '%%yaffs-root%%':0:False
# These were extractors used from FMK that still need suitable replacements.
#^bff volume entry:bff:/opt/firmware-mod-kit/src/bff/bffxtractor.py '%e'
#^wdk file system:wdk:/opt/firmware-mod-kit/src/firmware-tools/unwdk.py '%e'
# Extract, but don't run anything
^ubifs filesystem:ubi
^elf,:elf
private key:key
certificate:crt
html document header
xml document:xml
#^elf,:elf
#private key:key
#certificate:crt
#html document header
#xml document:xml
src/binwalk/core/common.py
......@@ -5,11 +5,15 @@ import os
import re
import sys
import ast
import hashlib
import platform
import operator as op
import binwalk.core.idb
from binwalk.core.compat import *
# Don't try to import hashlib when loaded into IDA; it doesn't work.
if not binwalk.core.idb.LOADED_IN_IDA:
import hashlib
# The __debug__ value is a bit backwards; by default it is set to True, but
# then set to False if the Python interpreter is run with the -O option.
if not __debug__:
......@@ -483,3 +487,4 @@ def BlockFile(fname, mode='r', subclass=io.FileIO, **kwargs):
return (data, dlen)
return InternalBlockFile(fname, mode=mode, **kwargs)
src/binwalk/core/display.py
......@@ -33,6 +33,31 @@ class Display(object):
if csv:
self.csv = pycsv.writer(self.fp)
def _fix_unicode(self, line):
'''
This is a hack, there must be a better way to handle it.
In Python3, if the environment variable LANG=C is set, indicating
that the terminal is ASCII only, but unicode characters need to be
printed to the screen or to a file (e.g., file path, magic result
format string), then an UnicodeEncodError exception will be raised.
This converts the given line to ASCII, ignoring conversion errors,
and returns a str.
'''
return bytes2str(line.encode('ascii', 'ignore'))
def _fix_unicode_list(self, columns):
'''
Convenience wrapper for self.log which is passed a list of format arguments.
'''
if type(columns) in [list, tuple]:
for i in range(0, len(columns)):
try:
columns[i] = self._fix_unicode(columns[i])
except AttributeError:
pass
return columns
def format_strings(self, header, result):
self.result_format = result
self.header_format = header
......@@ -43,9 +68,15 @@ class Display(object):
def log(self, fmt, columns):
if self.fp:
if self.csv:
try:
self.csv.writerow(columns)
except UnicodeEncodeError:
self.csv.writerow(self._fix_unicode_list(columns))
else:
try:
self.fp.write(fmt % tuple(columns))
except UnicodeEncodeError:
self.fp.write(fmt % tuple(self._fix_unicode_list(columns)))
self.fp.flush()
......@@ -100,6 +131,10 @@ class Display(object):
if not self.quiet and stdout:
try:
try:
sys.stdout.write(self._format_line(line.strip()) + "\n")
except UnicodeEncodeError:
line = self._fix_unicode(line)
sys.stdout.write(self._format_line(line.strip()) + "\n")
sys.stdout.flush()
except IOError as e:
......
src/binwalk/core/magic.py
......@@ -789,6 +789,7 @@ class Magic(object):
Returns None.
'''
# Magic files must be ASCII, else encoding issues can arise.
fp = open(fname, "r")
lines = fp.readlines()
self.parse(lines)
......
src/binwalk/core/module.py
......@@ -6,12 +6,15 @@
import io
import os
import sys
import time
import inspect
import argparse
import traceback
import binwalk.core.statuserver
import binwalk.core.common
import binwalk.core.settings
import binwalk.core.plugin
from threading import Thread
from binwalk.core.compat import *
class Option(object):
......@@ -216,10 +219,11 @@ class Module(object):
# Set to False if this is not a primary module (e.g., General, Extractor modules)
PRIMARY = True
def __init__(self, **kwargs):
def __init__(self, parent, **kwargs):
self.errors = []
self.results = []
self.parent = parent
self.target_file_list = []
self.status = None
self.enabled = False
......@@ -258,6 +262,13 @@ class Module(object):
'''
return None
def unload(self):
'''
Invoked at module load time.
May be overridden by the module sub-class.
'''
return None
def reset(self):
'''
Invoked only for dependency modules immediately prior to starting a new primary module.
......@@ -336,6 +347,17 @@ class Module(object):
return args
def _unload_dependencies(self):
# Calls the unload method for all dependency modules.
# These modules cannot be unloaded immediately after being run, as
# they must persist until the module that depends on them is finished.
# As such, this must be done separately from the Modules.run 'unload' call.
for dependency in self.dependencies:
try:
getattr(self, dependency.attribute).unload()
except AttributeError:
continue
def next_file(self, close_previous=True):
'''
Gets the next file to be scanned (including pending extracted files, if applicable).
......@@ -358,7 +380,7 @@ class Module(object):
# Reset all dependencies prior to continuing with another file.
# This is particularly important for the extractor module, which must be reset
# in order to reset it's base output directory path for each file, and the
# in order to reset its base output directory path for each file, and the
# list of pending files.
self.reset_dependencies()
......@@ -386,8 +408,10 @@ class Module(object):
if fp is not None:
self.current_target_file_name = fp.path
self.status.fp = fp
else:
self.current_target_file_name = None
self.status.fp = None
self.previous_next_file_fp = fp
......@@ -426,6 +450,12 @@ class Module(object):
self.validate(r)
self._plugins_result(r)
# Update the progress status automatically if it is not being done manually by the module
if r.offset and r.file and self.AUTO_UPDATE_STATUS:
self.status.total = r.file.length
self.status.completed = r.offset
self.status.fp = r.file
for dependency in self.dependencies:
try:
getattr(self, dependency.attribute).callback(r)
......@@ -435,11 +465,6 @@ class Module(object):
if r.valid:
self.results.append(r)
# Update the progress status automatically if it is not being done manually by the module
if r.offset and r.file and self.AUTO_UPDATE_STATUS:
self.status.total = r.file.length
self.status.completed = r.offset
if r.display:
display_args = self._build_display_args(r)
if display_args:
......@@ -499,14 +524,14 @@ class Module(object):
if hasattr(self, dependency.attribute):
getattr(self, dependency.attribute).reset()
def main(self, parent):
def main(self):
'''
Responsible for calling self.init, initializing self.config.display, and calling self.run.
Returns the value returned from self.run.
'''
self.status = parent.status
self.modules = parent.loaded_modules
self.status = self.parent.status
self.modules = self.parent.executed_modules
# A special exception for the extractor module, which should be allowed to
# override the verbose setting, e.g., if --matryoshka has been specified
......@@ -584,12 +609,25 @@ class Modules(object):
Returns None.
'''
self.arguments = []
self.loaded_modules = {}
self.executed_modules = {}
self.default_dependency_modules = {}
self.status = Status(completed=0, total=0)
self.status = Status(completed=0, total=0, fp=None, running=False, shutdown=False, finished=False)
self.status_server_started = False
self.status_service = None
self._set_arguments(list(argv), kargv)
def cleanup(self):
if self.status_service:
self.status_service.server.socket.shutdown(1)
self.status_service.server.socket.close()
def __enter__(self):
return self
def __exit__(self, t, v, b):
self.cleanup()
def _set_arguments(self, argv=[], kargv={}):
for (k,v) in iterator(kargv):
k = self._parse_api_opt(k)
......@@ -691,7 +729,7 @@ class Modules(object):
obj = self.run(module)
# Add all loaded modules that marked themselves as enabled to the run_modules list
for (module, obj) in iterator(self.loaded_modules):
for (module, obj) in iterator(self.executed_modules):
# Report the results if the module is enabled and if it is a primary module or if it reported any results/errors
if obj.enabled and (obj.PRIMARY or obj.results or obj.errors):
run_modules.append(obj)
......@@ -704,15 +742,29 @@ class Modules(object):
'''
Runs a specific module.
'''
try:
obj = self.load(module, kwargs)
if isinstance(obj, binwalk.core.module.Module) and obj.enabled:
obj.main(parent=self)
obj.main()
self.status.clear()
# If the module is not being loaded as a dependency, add it to the loaded modules dictionary
# If the module is not being loaded as a dependency, add it to the executed modules dictionary.
# This is used later in self.execute to determine which objects should be returned.
if not dependency:
self.loaded_modules[module] = obj
self.executed_modules[module] = obj
# The unload method tells the module that we're done with it, and gives it a chance to do
# any cleanup operations that may be necessary. We still retain the object instance in self.executed_modules.
obj._unload_dependencies()
obj.unload()
except KeyboardInterrupt as e:
# Tell the status server to shut down, and give it time to clean up.
if self.status.running:
self.status.shutdown = True
while not self.status.finished:
time.sleep(0.1)
raise e
return obj
......@@ -720,7 +772,7 @@ class Modules(object):
argv = self.argv(module, argv=self.arguments)
argv.update(kwargs)
argv.update(self.dependencies(module, argv['enabled']))
return module(**argv)
return module(self, **argv)
def dependencies(self, module, module_enabled):
import binwalk.modules
......@@ -859,6 +911,21 @@ class Modules(object):
else:
raise Exception("binwalk.core.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(obj))
def status_server(self, port):
'''
Starts the progress bar TCP service on the specified port.
This service will only be started once per instance, regardless of the
number of times this method is invoked.
Failure to start the status service is considered non-critical; that is,
a warning will be displayed to the user, but normal operation will proceed.
'''
if self.status_server_started == False:
self.status_server_started = True
try:
self.status_service = binwalk.core.statuserver.StatusServer(port, self)
except Exception as e:
binwalk.core.common.warning("Failed to start status server on port %d: %s" % (port, str(e)))
def process_kwargs(obj, kwargs):
'''
......@@ -869,7 +936,9 @@ def process_kwargs(obj, kwargs):
Returns None.
'''
return Modules().kwargs(obj, kwargs)
with Modules() as m:
kwargs = m.kwargs(obj, kwargs)
return kwargs
def show_help(fd=sys.stdout):
'''
......@@ -879,6 +948,7 @@ def show_help(fd=sys.stdout):
Returns None.
'''
fd.write(Modules().help())
with Modules() as m:
fd.write(m.help())
src/binwalk/core/settings.py
......@@ -17,7 +17,7 @@ class Settings:
o PLUGINS - Path to the plugins directory.
'''
# Release version
VERSION = "2.1.0"
VERSION = "2.1.2b"
# Sub directories
BINWALK_USER_DIR = ".binwalk"
......
src/binwalk/core/statuserver.py 0 → 100644
# Provides scan status information via a TCP socket service.
# Currently only works for signature scans.
import sys
import time
import errno
import threading
import binwalk.core.compat
# Python 2/3 compatibility
try:
import SocketServer
except ImportError:
import socketserver as SocketServer
class StatusRequestHandler(SocketServer.BaseRequestHandler):
def handle(self):
message_format = "%s %3d%% [ %d / %d ]"
last_status_message_len = 0
status_message = ''
message_sent = False
self.server.binwalk.status.running = True
while True:
time.sleep(0.1)
try:
self.request.send(binwalk.core.compat.str2bytes('\b' * last_status_message_len))
self.request.send(binwalk.core.compat.str2bytes(' ' * last_status_message_len))
self.request.send(binwalk.core.compat.str2bytes('\b' * last_status_message_len))
if self.server.binwalk.status.shutdown:
self.server.binwalk.status.finished = True
break
if self.server.binwalk.status.total != 0:
percentage = ((float(self.server.binwalk.status.completed) / float(self.server.binwalk.status.total)) * 100)
status_message = message_format % (self.server.binwalk.status.fp.path,
percentage,
self.server.binwalk.status.completed,
self.server.binwalk.status.total)
elif not message_sent:
status_message = "No status information available at this time!"
else:
continue
last_status_message_len = len(status_message)
self.request.send(binwalk.core.compat.str2bytes(status_message))
message_sent = True
except IOError as e:
if e.errno == errno.EPIPE:
break
except Exception as e:
binwalk.core.common.debug('StatusRequestHandler exception: ' + str(e) + '\n')
except KeyboardInterrupt as e:
raise e
self.server.binwalk.status.running = False
return
class ThreadedStatusServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
daemon_threads = True
allow_reuse_address = True
class StatusServer(object):
def __init__(self, port, binwalk):
self.server = ThreadedStatusServer(('127.0.0.1', port), StatusRequestHandler)
self.server.binwalk = binwalk
t = threading.Thread(target=self.server.serve_forever)
t.setDaemon(True)
t.start()
src/binwalk/magic/binarch
......@@ -17,15 +17,15 @@
#
# addiu $sp, XX
# jr $ra
0 belong 0x03e00008 MIPS instructions, function epilogue
>4 beshort !0x27BD {invalid}
0 beshort 0x27BD MIPS instructions, function epilogue
>2 belong !0x03e00008 {invalid}
0 ubelong 0x03e00008 MIPS instructions, function epilogue
>4 ubeshort !0x27BD {invalid}
0 ubeshort 0x27BD MIPS instructions, function epilogue
>2 ubelong !0x03e00008 {invalid}
0 lelong 0x03e00008 MIPSEL instructions, function epilogue
>6 leshort !0x27BD {invalid}
0 leshort 0x27BD MIPS instructions, function epilogue
>2 lelong !0x03e00008 {invalid}
0 ulelong 0x03e00008 MIPSEL instructions, function epilogue
>6 uleshort !0x27BD {invalid}
0 uleshort 0x27BD MIPS instructions, function epilogue
>2 ulelong !0x03e00008 {invalid}
# MIPS16e
# nop (x4)
......@@ -41,63 +41,63 @@
# move $sp, $s1
# restore XX, XX, XX
# jrc $ra
0 beshort 0x65B9 MIPS16e instructions, function epilogue
0 ubeshort 0x65B9 MIPS16e instructions, function epilogue
>3 byte !0x64 {invalid}
>4 beshort !0xE8A0 {invalid}
0 leshort 0x65B9 MIPSEL16e instructions, function epilogue
>4 ubeshort !0xE8A0 {invalid}
0 uleshort 0x65B9 MIPSEL16e instructions, function epilogue
>3 byte !0x64 {invalid}
>4 leshort !0xE8A0 {invalid}
>4 uleshort !0xE8A0 {invalid}
# jrc $ra
# nop
0 belong 0xe8a06500 MIPS16e instructions, function epilogue
0 lelong 0xe8a06500 MIPSEL16e instructions, function epilogue
0 ubelong 0xe8a06500 MIPS16e instructions, function epilogue
0 ulelong 0xe8a06500 MIPSEL16e instructions, function epilogue
# PowerPC prologue
# mflr r0
0 belong 0x7C0802A6 PowerPC big endian instructions, function prologue
0 lelong 0x7C0802A6 PowerPC little endian instructions, function prologue
0 ubelong 0x7C0802A6 PowerPC big endian instructions, function prologue
0 ulelong 0x7C0802A6 PowerPC little endian instructions, function prologue
# PowerPC epilogue
# blr
0 belong 0x4E800020 PowerPC big endian instructions, function epilogue
0 lelong 0x4E800020 PowerPC little endian instructions, function epilogue
0 ubelong 0x4E800020 PowerPC big endian instructions, function epilogue
0 ulelong 0x4E800020 PowerPC little endian instructions, function epilogue
# TODO: Add ARM Thumb dectection
# ARM prologue
# STMFD SP!, {XX}
# <any instruction whose opcode begins with 0xE>
0 beshort 0xE92D ARMEB instructions, function prologue
0 ubeshort 0xE92D ARMEB instructions, function prologue
>4 byte&0xF0 !0xE0 {invalid}
>8 byte&0xF0 !0xE0 {invalid}
0 leshort 0xE92D ARM instructions, function prologue{adjust:-2}
0 uleshort 0xE92D ARM instructions, function prologue{adjust:-2}
>5 byte&0xF0 !0xE0 {invalid}
>9 byte&0xF0 !0xE0 {invalid}
# ARM epilogue
# MOV R0, XX
# LDMFD SP!, {XX}
0 beshort 0xE1A0 ARMEB instructions, function epilogue
0 ubeshort 0xE1A0 ARMEB instructions, function epilogue
>4 beshort !0xE8BD {invalid}
0 leshort 0xE1A0 ARM instructions, function epilogue{adjust:-2}
0 uleshort 0xE1A0 ARM instructions, function epilogue{adjust:-2}
>4 leshort !0xE8BD {invalid}
# Ubicom32 prologue
# move.4 -4($sp)++, $ra
0 belong 0x02FF6125 Ubicom32 instructions, function prologue
0 ubelong 0x02FF6125 Ubicom32 instructions, function prologue
# Ubicom32 epilogues
# calli $ra, 0($ra)
# ret ($sp)4++
0 belong 0xF0A000A0 Ubicom32 instructions, function epilogue
0 belong 0x000022E1 Ubicom32 instructions, function epilogue
0 ubelong 0xF0A000A0 Ubicom32 instructions, function epilogue
0 ubelong 0x000022E1 Ubicom32 instructions, function epilogue
# AVR8 prologue
# push r28
# push r29
0 belong 0x93CF93DF AVR8 instructions, function prologue
0 belong 0x93DF93CF AVR8 instructions, function prologue
0 ubelong 0x93CF93DF AVR8 instructions, function prologue
0 ubelong 0x93DF93CF AVR8 instructions, function prologue
# AVR32 prologue
# pushm r7,lr
......
src/binwalk/magic/compressed
......@@ -115,6 +115,7 @@
>3 byte &0x02 \b, has header CRC
>3 byte&0x04 0x04
>>10 leshort x \b, has %d bytes of extra data
>>10 leshort <0 {invalid}(invalid extra data size)
>3 byte&0xC =0x08 \b, has original file name
>>10 string x \b: "%s"{name:%s}
>3 byte &0x10 \b, has comment
......@@ -137,14 +138,12 @@
>3 byte &0x20 \b, encrypted{invalid}
# Dates before 1992 are {invalid}, unless of course you're DD-WRT in which
# case you don't know how to set a date in your gzip files. Brilliant.
>4 lelong =0 \b, NULL date (1970-01-01 00:00:00)
>4 lelong <0 {invalid}
>4 lelong >0
>>4 lelong <694224000 {invalid}
>>4 lelong =694224000 {invalid}
>>4 lelong >694224000 \b, last modified:
>>>4 ledate x %s
>>>4 lelong x {epoch:%d}
>4 ledate x \b, last modified: %s
>4 lelong =0 (null date)
>4 lelong !0
>>4 lelong <694224000 (bogus date)
>>4 lelong =694224000 (bogus date)
# Supplementary magic data for the file(1) command to support
# rzip(1). The format is described in magic(5).
......
src/binwalk/magic/console
......@@ -75,7 +75,7 @@
>113 string x (%s)
#------------------------------------------------------------------------------
## Microsoft Xbox executables .xbe (Esa Hyytiä <ehyytia@cc.hut.fi>)
## Microsoft Xbox executables .xbe (Esa Hyyti <ehyytia@cc.hut.fi>)
0 string XBEH Microsoft Xbox executable (XBE),
## probabilistic checks whether signed or not
>0x0004 ulelong =0
......
src/binwalk/magic/crypto
......@@ -6,6 +6,7 @@
0 string -----BEGIN\x20CERTIFICATE\x20REQ PEM certificate request
0 string -----BEGIN\x20RSA\x20PRIVATE PEM RSA private key
0 string -----BEGIN\x20DSA\x20PRIVATE PEM DSA private key
0 string -----BEGIN\x20EC\x20PRIVATE PEM EC private key
# Type: OpenSSH key files
# From: Nicolas Collignon <tsointsoin@gmail.com>
......@@ -17,6 +18,9 @@
0 string ssh-dss\x20 OpenSSH DSA public key
0 string ssh-rsa\x20 OpenSSH RSA public key
0 string ecdsa-sha2-nistp256\x20 OpenSSH ECDSA (Curve P-256) public key
0 string ecdsa-sha2-nistp384\x20 OpenSSH ECDSA (Curve P-384) public key
0 string ecdsa-sha2-nistp521\x20 OpenSSH ECDSA (Curve P-521) public key
# Type: Certificates/key files in DER format
# From: Gert Hulselmans <hulselmansgert@gmail.com>
......
src/binwalk/magic/efi 0 → 100644
# http://blogs.phoenix.com/phoenix_technologies_bios/\
# 2007/02/uefi_pi_10_firm.html
40 string _FVH UEFI PI Firmware Volume
>32 ulequad >0xFFFFFFFF {invalid} unrealistic size
>32 ulequad x \b, volume size: %d
>52 uleshort x \b, header size: %d
>58 ubyte !0 {invalid} reserved byte set
>59 ubyte >1 {invalid} invalid revision
>59 ubyte x \b, revision: %d
# GUID: 7A9354D9-0468-444A-81CE-0BF617D890D
>16 string \xd9\x54\x93\x7a\x68\x04\x4a\x44\x81\xce\x0b\xf6\x17\xd8\x90\xdf \b, EFI Firmware File System
# GUID: 8C8CE578-8A3D-4F1C-9935-896185C32DD3
>16 string \x78\xe5\x8c\x8c\x3d\x8a\x1c\x4f\x99\x35\x89\x61\x85\xc3\x2d\xd3 \b, EFI Firmware File System v2
# GUID: 04ADEEAD-61FF-4D31-B6BA-64F8BF901F5A
>16 string \xad\xee\xad\x04\xff\x61\x31\x4d\xb6\xba\x64\xf8\xbf\x90\x1f\x5a \b, Apple Boot Volume
# GUID: 8C1B00BD-716A-7B48-A14F-0C2A2DCF7A5D
>16 string \x8c\x1b\x00\xbd\x71\x6a\x7b\x48\xa1\x4f\x0c\x2a\x2d\xcf\x7a\x5d \b, Apple Boot Volume v2
# GUID: AD3FFFFF-D28B-44C4-9F13-9EA98A97F9F0
>16 string \xff\xff\x3f\xad\x8b\xd2\xc4\x44\x9f\x13\x9e\xa9\x8a\x97\xf9\xf0 \b, Intel v1
# GUID: D6A1CD70-4B33-4994-A6EA-375F2CCC5437
>16 string \x70\xcd\xa1\xd6\x33\x4b\x94\x49\xa6\xea\x37\x5f\x2c\xcc\x54\x37 \b, Intel v2
# GUID: 4F494156-AED6-4D64-A537-B8A5557BCEEC
>16 string \x56\x41\x49\x4f\xd6\xae\x64\x4d\xa5\x37\xb8\xa5\x55\x7b\xce\xec \b, Sony v1
# GUID: FFF12B8D-7696-4C8B-85A9-2747075B4F50
>16 string \x8d\x2b\xf1\xff\x96\x76\x8b\x4c\xa9\x85\x27\x47\x07\x5b\x4f\x50 \b, Variable Storage
>16 ulelong x \b, GUID: %.8X-
>>20 uleshort x \b%.4X-
>>22 uleshort x \b%.4X-
>>24 uleshort x \b%.4X-
>>26 ubyte x \b%.2X
>>27 ubyte x \b%.2X
>>28 ubyte x \b%.2X
>>29 ubyte x \b%.2X
>>30 ubyte x \b%.2X
>>31 ubyte x \b%.2X
# http://www.intel.com/content/www/us/en/architecture-and-technology/\
# unified-extensible-firmware-interface/efi-capsule-specification.html
# GUID: 3B6686BD-0D76-4030-B70E-B5519E2FC5A0
0 string \xBD\x86\x66\x3B\x76\x0D\x30\x40\xB7\x0E\xB5\x51\x9E\x2F\xC5\xA0 EFI capsule v0.9
>16 lelong <0 {invalid}
>16 lelong x \b, header size: %d
>20 lelong x \b, flags: 0x%.8X
>24 lelong <0 {invalid}
>24 lelong x \b, capsule size: %d
# Intel/UEFI format
# http://www.uefi.org/sites/default/files/resources/UEFI%202_5.pdf
# GUID: 539182B9-ABB5-4391-B69A-E3A943F72FCC
0 string \xb9\x82\x91\x53\xb5\xab\x91\x43\xb6\x9a\xe3\xa9\x43\xf7\x2f\xcc UEFI capsule
>16 lelong <0 {invalid}
>16 lelong x \b, header size: %d
>20 lelong x \b, flags: 0x%.8X
>24 lelong <0 {invalid}
>24 lelong x \b, capsule size: %d
# GUID: 4A3CA68B-7723-48FB-803D-578CC1FEC44D
0 string \x8b\xa6\x3c\x4a\x23\x77\xfb\x48\x80\x3d\x57\x8c\xc1\xfe\xc4\x4d AMI Aptio extended EFI capsule
>16 lelong <0 {invalid}
>16 lelong x \b, header size: %d
>20 lelong x \b, flags: 0x%.8X
>24 lelong <0 {invalid}
>24 lelong x \b, capsule size: %d
# GUID: 14EEBB90-890A-43DB-AED1-5D3C4588A418
0 string \x90\xbb\xee\x14\x0a\x89\xdb\x43\xae\xd1\x5d\x3c\x45\x88\xa4\x18 AMI Aptio unsigned EFI capsule
>16 lelong <0 {invalid}
>16 lelong x \b, header size: %d
>20 lelong x \b, flags: 0x%.8X
>24 lelong <0 {invalid}
>24 lelong x \b, capsule size: %d
# GUID: 3BE07062-1D51-45D2-2B83-F093257ED461
0 string \x62\x70\xe0\x3b\x51\x1d\xd2\x45\x83\x2b\xf0\x93\x25\x7e\xd4\x61 Toshiba EFI capsule
>16 lelong <0 {invalid}
>16 lelong x \b, header size: %d
>20 lelong x \b, flags: 0x%.8X
>24 lelong <0 {invalid}
>24 lelong x \b, capsule size: %d
src/binwalk/magic/filesystems
......@@ -34,7 +34,11 @@
#>0x1e string minix \b, bootable
# YAFFS
0 string \x03\x00\x00\x00\x01\x00\x00\x00\xFF\xFF YAFFS filesystem
0 string \x03\x00\x00\x00\x01\x00\x00\x00\xFF\xFF\x00\x00 YAFFS filesystem, little endian
# The big endian signature has to be done a bit differently to prevent it from being self-overlapping
4 string \x00\x00\x00\x01\xFF\xFF YAFFS filesystem, big endian
>0 string !\x00\x00\x00\x03 {invalid}(first object is not a directory)
>10 string !\x00 {invalid}(unexpected name in the first object entry)
# EFS2 file system - jojo@utulsa.edu
0 lelong 0x53000000 EFS2 Qualcomm filesystem super block, little endian,
......@@ -68,8 +72,11 @@
# MPFS file system
0 string MPFS MPFS filesystem, Microchop,
>4 byte <0 {invalid}
>5 byte <0 {invalid}
>4 byte x version %d.
>5 byte x \b%d,
>6 leshort <0 {invalid}
>6 leshort x %d file entries
# cramfs filesystem - russell@coker.com.au
......@@ -116,15 +123,17 @@
>28 string !\x00*12 {invalid}
# http://lxr.free-electrons.com/source/fs/ubifs/ubifs-media.h
#0 string UBI\x23 UBI erase count header,
#>4 ubyte x version: %d,
#>5 string !\x00*3 {invalid}
#>8 ubequad x EC: 0x%lX,
#>16 ubelong x VID header offset: 0x%X,
#>20 ubelong x data offset: 0x%X
0 string UBI\x23 UBI erase count header,
>4 ubyte x version: %d,
>5 string !\x00*3 {invalid}
>8 ubequad x EC: 0x%lX,
>16 ubelong x VID header offset: 0x%X,
>20 ubelong x data offset: 0x%X
# dummy jump - actual jump value is determined in UBIValidPlugin
>20 ubyte x {jump:0}
# http://lxr.free-electrons.com/source/fs/ubifs/ubifs-media.h
0 lelong 0x06101831 UBIFS
0 lelong 0x06101831 UBIFS filesystem
>20 ubyte <6 {invalid}
>20 ubyte >7 {invalid} # Only look for superblock and master nodes
>22 leshort !0 {invalid} # 2 bytes of padding should be filled with NULLs
......
src/binwalk/magic/firmware
......@@ -67,6 +67,10 @@
>31 byte 3 compression type: lzma,
>32 string x image name: "%s"
# Hilink encrypted uImage firmware.
# Additional validation/processing is done by the hilink.py plugin.
0x23 string \x4A\x52\xCA\xDA Encrypted Hilink uImage firmware header
#IMG0 header, found in VxWorks-based Mercury router firmware
0 string IMG0 IMG0 (VxWorks) header,
>4 belong <1 {invalid}
......@@ -495,8 +499,6 @@
>35 byte x try decryption tool from:
>35 byte x http://download.modem-help.co.uk/mfcs-A/Alcatel/Modems/Misc/
16 string \xd9\x54\x93\x7a\x68\x04\x4a\x44\x81\xce\x0b\xf6\x17\xd8\x90\xdf UEFI PI firmware volume
# http://android.stackexchange.com/questions/23357/\
# is-there-a-way-to-look-inside-and-modify-an-adb-backup-created-file/\
# 23608#23608
......@@ -690,5 +692,13 @@
>44 lelong &0x4 SLT
>44 lelong &0xffffff00 {invalid}
# Android bootimg
# https://android.googlesource.com/platform/system/core.git/+/master/mkbootimg/bootimg.h
0 string ANDROID! Android bootimg
>8 ulelong x \b, kernel size: %d bytes
>12 ulelong x \b, kernel addr: 0x%X
>16 ulelong x \b, ramdisk size: %d bytes
>20 ulelong x \b, ramdisk addr: 0x%X
>48 string x \b, product name: "%s"
src/binwalk/magic/linux
......@@ -3,7 +3,7 @@
# Linux kernel boot images, from Albert Cahalan <acahalan@cs.uml.edu>
# and others such as Axel Kohlmeyer <akohlmey@rincewind.chemie.uni-ulm.de>
# and Nicolás Lichtmaier <nick@debian.org>
# and Nicolas Lichtmaier <nick@debian.org>
# All known start with: b8 c0 07 8e d8 b8 00 90 8e c0 b9 00 01 29 f6 29
0 string \xb8\xc0\x07\x8e\xd8\xb8\x00\x90\x8e\xc0\xb9\x00\x01\x29\xf6\x29 Linux kernel boot image
>514 string !HdrS {invalid}
......@@ -16,14 +16,23 @@
>>14 string x "%s"
# Linux ARM compressed kernel image
# See arch/arm/boot/compressed/head.S and arch/arm/boot/compressed/vmlinux.lds.S
0 ulelong 0x016f2818 Linux kernel ARM boot executable zImage (little-endian),
>4 ulelong x load address: "0x%.8X",
>8 ulelong x end address: "0x%.8X"
>12 ulelong !0x04030201 {invalid}
0 ubelong 0x016f2818 Linux kernel ARM boot executable zImage (big-endian),
>4 ubelong x load address: "0x%.8X",
>8 ubelong x end address: "0x%.8X"
>12 ubelong !0x04030201 {invalid}
# Starts with 8 NOPs, with 0x016F2818 at offset 0x24
36 ulelong 0x016F2818 Linux kernel ARM boot executable zImage (little-endian)
>0 ulelong !0xE1A00000 {invalid}(invalid)
>4 ulelong !0xE1A00000 {invalid}(invalid)
>8 ulelong !0xE1A00000 {invalid}(invalid)
>12 ulelong !0xE1A00000 {invalid}(invalid)
>16 ulelong !0xE1A00000 {invalid}(invalid)
>20 ulelong !0xE1A00000 {invalid}(invalid)
>24 ulelong !0xE1A00000 {invalid}(invalid)
>28 ulelong !0xE1A00000 {invalid}(invalid)
36 ubelong 0x016F2818 Linux kernel ARM boot executable zImage (big-endian)
>0 ubelong !0xE1A00000 {invalid}(invalid)
>4 ubelong !0xE1A00000 {invalid}(invalid)
>8 ubelong !0xE1A00000 {invalid}(invalid)
>12 ubelong !0xE1A00000 {invalid}(invalid)
>16 ubelong !0xE1A00000 {invalid}(invalid)
>20 ubelong !0xE1A00000 {invalid}(invalid)
>24 ubelong !0xE1A00000 {invalid}(invalid)
>28 ubelong !0xE1A00000 {invalid}(invalid)
src/binwalk/magic/misc
......@@ -41,8 +41,18 @@
# CodeGate 2011 http://nopsrus.blogspot.com/2013/05/codegate-ctf-2011-binary-100-points.html
0 string \x23\x40\x7e\x5e Windows Script Encoded Data (screnc.exe)
0 regex /[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_/].* Unix path:
0 regex /[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_]{1,25}/[a-zA-Z0-9\.\-_/].* Unix path:
>0 string x %s
>0 string !/home/
>>0 string !/bin/
>>>0 string !/sbin/
>>>>0 string !/usr/
>>>>>0 string !/sys/
>>>>>>0 string !/var/
>>>>>>>0 string !/opt/
>>>>>>>>0 string !/etc/
>>>>>>>>>0 string !/lib/
>>>>>>>>>>0 string !/dev/ {invalid}(likely false positive)
0 string neighbor Neighborly text,
>0 string x "%s
......
src/binwalk/magic/phones 0 → 100644
# From: http://arm.ninja/2016/03/04/reverse-engineering-samsung-s6-modem/
0 string TOC\x00\x00\x00\x00 Samsung modem TOC index,
>20 lelong x size: 0x%X bytes,
>24 lelong !0 invalid TOC CRC,{invalid}
>28 lelong !1 invalid TOC index{invalid}
>0x20 string x TOC entries: %s
>0x40 byte !0
>>0x40 string x \b, %s
>0x60 byte !0
>>0x60 string x \b, %s
>0x80 byte !0
>>0x80 string x \b, %s
>0xA0 byte !0
>>0xA0 string x \b, %s
>0xC0 byte !0
>>0xC0 string x \b, %s
>0xE0 byte !0
>>0xE0 string x \b, %s
>0x100 byte !0
>>0x100 string x \b, %s
src/binwalk/modules/compression.py
......@@ -2,11 +2,14 @@
import os
import zlib
import lzma
import struct
import binwalk.core.compat
import binwalk.core.common
from binwalk.core.module import Option, Kwarg, Module
try:
import lzma
except ImportError:
from backports import lzma
class LZMAHeader(object):
def __init__(self, **kwargs):
......@@ -159,18 +162,38 @@ class Deflate(object):
self.module.extractor.add_rule(regex='^%s' % self.DESCRIPTION.lower(), extension="deflate", cmd=self.extractor)
def extractor(self, file_name):
in_data = ""
out_data = ""
retval = False
out_file = os.path.splitext(file_name)[0]
with binwalk.core.common.BlockFile(file_name, 'r') as fp_in:
while True:
(data, dlen) = fp_in.read_block()
if not data or dlen == 0:
break
else:
in_data += data
try:
out_data = zlib.decompress(binwalk.core.compat.str2bytes(in_data), -15)
with binwalk.core.common.BlockFile(out_file, 'w') as fp_out:
fp_out.write(out_data)
retval = True
break
except zlib.error as e:
pass
return retval
def decompress(self, data):
valid = True
description = None
# Prepend data with a standard zlib header
data = "\x78\x9C" + data
# Looking for either a valid decompression, or an error indicating truncated input data
try:
zlib.decompress(binwalk.core.compat.str2bytes(data))
# Negative window size (e.g., -15) indicates that raw decompression should be performed
zlib.decompress(binwalk.core.compat.str2bytes(data), -15)
except zlib.error as e:
if not str(e).startswith("Error -5"):
# Bad data.
......
src/binwalk/modules/entropy.py
......@@ -3,7 +3,6 @@
import os
import math
import zlib
import multiprocessing
import binwalk.core.common
from binwalk.core.compat import *
from binwalk.core.module import Module, Option, Kwarg
......@@ -113,20 +112,13 @@ class Entropy(Module):
else:
self.block_size = None
def _entropy_sigterm_handler(self, *args):
print ("FUck it all.")
def run(self):
# Need to invoke the pyqtgraph stuff via a separate process, as calling pg.exit
# is pretty much required. pg.exit calls os._exit though, and we don't want to
# exit out of the main process (especially if being run via the API).
if not binwalk.core.common.MSWindows():
p = multiprocessing.Process(target=self._run)
p.start()
p.join()
else:
# There seem to be all kinds of issues using the multiprocessing module in
# Windows, as done above.
#
# This means that when run in Windows, pg.exit will cause binwalk
# to exit.
# If generating a graphical plot, this function will never return, as it invokes
# pg.exit. Calling pg.exit is pretty much required, but pg.exit calls os._exit in
# order to work around QT cleanup issues.
self._run()
def _run(self):
......@@ -135,7 +127,7 @@ class Entropy(Module):
try:
import pyqtgraph as pg
except ImportError as e:
binwalk.core.common.warning("pyqtgraph not found, visual entropy graphing will be disabled")
binwalk.core.common.warning("Failed to import pyqtgraph module, visual entropy graphing will be disabled")
self.do_plot = False
for fp in iter(self.next_file, None):
......
src/binwalk/modules/extractor.py
......@@ -72,6 +72,11 @@ class Extractor(Module):
type=int,
kwargs={'max_size' : 0},
description='Limit the size of each extracted file'),
Option(short='n',
long='count',
type=int,
kwargs={'max_count' : 0},
description='Limit the number of extracted files'),
Option(short='r',
long='rm',
kwargs={'remove_after_execute' : True},
......@@ -84,6 +89,7 @@ class Extractor(Module):
KWARGS = [
Kwarg(name='max_size', default=None),
Kwarg(name='max_count', default=None),
Kwarg(name='base_directory', default=None),
Kwarg(name='remove_after_execute', default=False),
Kwarg(name='load_default_rules', default=False),
......@@ -100,6 +106,10 @@ class Extractor(Module):
self.directory = None
# Key value pairs of input file path and output extraction path
self.output = {}
# Number of extracted files
self.extraction_count = 0
# Override the directory name used for extraction output directories
self.output_directory_override = None
if self.load_default_rules:
self.load_defaults()
......@@ -154,7 +164,7 @@ class Extractor(Module):
# Only extract valid results that have been marked for extraction and displayed to the user.
# Note that r.display is still True even if --quiet has been specified; it is False if the result has been
# explicitly excluded via the -y/-x options.
if r.valid and r.extract and r.display:
if r.valid and r.extract and r.display and (not self.max_count or self.extraction_count < self.max_count):
# Create some extract output for this file, it it doesn't already exist
if not binwalk.core.common.has_key(self.output, r.file.path):
self.output[r.file.path] = ExtractInfo()
......@@ -165,6 +175,9 @@ class Extractor(Module):
# If the extraction was successful, self.extract will have returned the output directory and name of the dd'd file
if extraction_directory and dd_file:
# Track the number of extracted files
self.extraction_count += 1
# Get the full path to the dd'd file and save it in the output info for this file
dd_file_path = os.path.join(extraction_directory, dd_file)
self.output[r.file.path].carved[r.offset] = dd_file_path
......@@ -190,7 +203,10 @@ class Extractor(Module):
self.output[r.file.path].extracted[r.offset].append(real_file_path)
# If recursion was specified, and the file is not the same one we just dd'd
if self.matryoshka and file_path != dd_file_path and scan_extracted_files:
if (self.matryoshka and
file_path != dd_file_path and
scan_extracted_files and
self.directory in real_file_path):
# If the recursion level of this file is less than or equal to our desired recursion level
if len(real_file_path.split(self.directory)[1].split(os.path.sep)) <= self.matryoshka:
# If this is a directory and we are supposed to process directories for this extractor,
......@@ -271,18 +287,19 @@ class Extractor(Module):
if match:
self.append_rule(r)
def remove_rule(self, text):
def remove_rules(self, description):
'''
Remove all rules that match a specified text.
Remove all rules that match a specified description.
@text - The text to match against.
@description - The description to match against.
Returns the number of rules removed.
'''
rm = []
description = description.lower()
for i in range(0, len(self.extract_rules)):
if self.extract_rules[i]['regex'].match(text):
if self.extract_rules[i]['regex'].search(description):
rm.append(i)
for i in rm:
......@@ -290,6 +307,27 @@ class Extractor(Module):
return len(rm)
def edit_rules(self, description, key, value):
'''
Edit all rules that match a specified description.
@description - The description to match against.
@key - The key to change for each matching rule.
@value - The new key value for each matching rule.
Returns the number of rules modified.
'''
count = 0
description = description.lower()
for i in range(0, len(self.extract_rules)):
if self.extract_rules[i]['regex'].search(description):
if has_key(self.extract_rules[i], key):
self.extract_rules[i][key] = value
count += 1
return count
def clear_rules(self):
'''
Deletes all extraction rules.
......@@ -298,11 +336,25 @@ class Extractor(Module):
'''
self.extract_rules = []
def get_rules(self):
def get_rules(self, description=None):
'''
Returns a list of all extraction rules.
Returns a list of extraction rules that match a given description.
@description - The description to match against.
Returns a list of extraction rules that match the given description.
If no description is provided, a list of all rules are returned.
'''
return self.extract_rules
if description:
rules = []
description = description.lower()
for i in range(0, len(self.extract_rules)):
if self.extract_rules[i]['regex'].search(description):
rules.append(self.extract_rules[i])
else:
rules = self.extract_rules
return rules
def load_from_file(self, fname):
'''
......@@ -344,6 +396,23 @@ class Extractor(Module):
if binwalk.core.common.DEBUG:
raise Exception("Extractor.load_defaults failed to load file '%s': %s" % (extract_file, str(e)))
def get_output_directory_override(self):
'''
Returns the current output directory basename override value.
'''
return self.output_directory_override
def override_output_directory_basename(self, dirname):
'''
Allows the overriding of the default extraction directory basename.
@dirname - The directory base name to use.
Returns the current output directory basename override value.
'''
self.output_directory_override = dirname
return self.output_directory_override
def build_output_directory(self, path):
'''
Set the output directory for extracted files.
......@@ -357,31 +426,42 @@ class Extractor(Module):
basedir = os.path.dirname(path)
basename = os.path.basename(path)
# Make sure we put the initial extracted file in the CWD
# Make sure we put the initial extraction directory in the CWD
if self.directory is None:
if self.base_directory is None:
basedir = os.getcwd()
self.directory = os.getcwd()
if basedir != self.directory:
# During recursive extraction, extracted files will be in subdirectories
# of the CWD. This allows us to figure out the subdirectory by simply
# splitting the target file's base directory on our known CWD.
#
# However, the very *first* file being scanned is not necessarily in the
# CWD, so this will raise an IndexError. This is easy to handle though,
# since the very first file being scanned needs to have its contents
# extracted to ${CWD}/_basename.extracted, so we just set the subdir
# variable to a blank string when an IndexError is encountered.
try:
subdir = basedir.split(self.directory)[1][1:]
except IndexError as e:
subdir = ""
else:
basedir = self.base_directory
if not os.path.exists(basedir):
os.mkdir(basedir)
subdir = ""
outdir = os.path.join(basedir, '_' + basename)
if self.output_directory_override:
output_directory = os.path.join(self.directory, subdir, self.output_directory_override)
else:
outdir = os.path.join(self.directory, subdir, '_' + basename)
output_directory = unique_file_name(outdir, extension='extracted')
if not os.path.exists(output_directory):
os.mkdir(output_directory)
self.extraction_directories[path] = output_directory
self.output[path].directory = os.path.realpath(output_directory) + os.path.sep
# Else, just use the already created directory
else:
output_directory = self.extraction_directories[path]
# Set the initial base extraction directory for later determining the level of recusion
if self.directory is None:
self.directory = os.path.realpath(output_directory) + os.path.sep
self.output[path].directory = self.directory
return output_directory
def cleanup_extracted_files(self, tf=None):
......@@ -413,6 +493,7 @@ class Extractor(Module):
Returns the name of the extracted file (blank string if nothing was extracted).
'''
fname = ''
recurse = False
original_dir = os.getcwd()
rules = self.match(description)
file_path = os.path.realpath(file_name)
......@@ -423,6 +504,7 @@ class Extractor(Module):
else:
binwalk.core.common.debug("Found %d matching extraction rules" % len(rules))
# Generate the output directory name where extracted files will be stored
output_directory = self.build_output_directory(file_name)
# Extract to end of file if no size was specified
......@@ -689,7 +771,7 @@ class Extractor(Module):
except KeyboardInterrupt as e:
raise e
except Exception as e:
binwalk.core.common.warning("Extractor.execute failed to run external extrator '%s': %s" % (str(cmd), str(e)))
binwalk.core.common.warning("Extractor.execute failed to run external extractor '%s': %s" % (str(cmd), str(e)))
retval = None
if tmp is not None:
......
src/binwalk/modules/general.py
......@@ -80,6 +80,11 @@ class General(Module):
type=str,
kwargs={'file_name_exclude_regex' : ""},
description='Do not scan files whose names match this regex'),
Option(short='s',
long='status',
type=int,
kwargs={'status_server_port' : 0},
description='Enable the status server on the specified port'),
Option(long=None,
short=None,
type=binwalk.core.common.BlockFile,
......@@ -96,6 +101,7 @@ class General(Module):
Kwarg(name='offset', default=0),
Kwarg(name='base', default=0),
Kwarg(name='block', default=0),
Kwarg(name='status_server_port', default=0),
Kwarg(name='swap_size', default=0),
Kwarg(name='log_file', default=None),
Kwarg(name='csv', default=False),
......@@ -113,6 +119,7 @@ class General(Module):
PRIMARY = False
def load(self):
self.threads_active = False
self.target_files = []
# A special case for when we're loaded into IDA
......@@ -141,6 +148,9 @@ class General(Module):
if not binwalk.core.idb.LOADED_IN_IDA:
sys.exit(0)
if self.status_server_port > 0:
self.parent.status_server(self.status_server_port)
def reset(self):
pass
......@@ -181,7 +191,13 @@ class General(Module):
if swap is None:
swap = self.swap_size
return binwalk.core.common.BlockFile(fname, subclass=self.subclass, length=length, offset=offset, swap=swap, block=block, peek=peek)
return binwalk.core.common.BlockFile(fname,
subclass=self.subclass,
length=length,
offset=offset,
swap=swap,
block=block,
peek=peek)
def _open_target_files(self):
'''
......
src/binwalk/modules/hexdiff.py
......@@ -118,6 +118,13 @@ class HexDiff(Module):
loop_count = 0
sep_count = 0
# Figure out the maximum diff size (largest file size)
self.status.total = 0
for i in range(0, len(target_files)):
if target_files[i].size > self.status.total:
self.status.total = target_files[i].size
self.status.fp = target_files[i]
while True:
line = ""
done_files = 0
......@@ -168,6 +175,7 @@ class HexDiff(Module):
last_line = line
loop_count += 1
self.status.completed += self.block
def init(self):
# To mimic expected behavior, if all options are False, we show everything
......
src/binwalk/plugins/compressd.py
import binwalk.core.C
#import binwalk.core.C
import binwalk.core.plugin
from binwalk.core.common import *
#from binwalk.core.common import *
class CompressdPlugin(binwalk.core.plugin.Plugin):
'''
Searches for and validates compress'd data.
'''
# '''
# Searches for and validates compress'd data.
# '''
MODULES = ['Signature']
READ_SIZE = 64
#READ_SIZE = 64
COMPRESS42 = "compress42"
#COMPRESS42 = "compress42"
#COMPRESS42_FUNCTIONS = [
# binwalk.core.C.Function(name="is_compressed", type=bool),
#]
comp = None
#comp = None
def init(self):
#def init(self):
#self.comp = binwalk.core.C.Library(self.COMPRESS42, self.COMPRESS42_FUNCTIONS)
# This plugin is currently disabled due to the need to move away from supporting C
# libraries and into a pure Python project, for cross-platform support and ease of
# installation / package maintenance. A Python implementation will likely need to
# be custom developed in the future, but for now, since this compression format is
# not very common, especially in firmware, simply disable it.
self.comp = None
#self.comp = None
def scan(self, result):
if self.comp and result.file and result.description.lower().startswith("compress'd data"):
fd = self.module.config.open_file(result.file.name, offset=result.offset, length=self.READ_SIZE)
compressed_data = fd.read(self.READ_SIZE)
fd.close()
#def scan(self, result):
# if self.comp and result.file and result.description.lower().startswith("compress'd data"):
# fd = self.module.config.open_file(result.file.name, offset=result.offset, length=self.READ_SIZE)
# compressed_data = fd.read(self.READ_SIZE)
# fd.close()
if not self.comp.is_compressed(compressed_data, len(compressed_data)):
result.valid = False
# if not self.comp.is_compressed(compressed_data, len(compressed_data)):
# result.valid = False
src/binwalk/plugins/hilink.py 0 → 100644
#!/usr/bin/env python
import struct
import string
import binwalk.core.plugin
import binwalk.core.compat
import binwalk.core.common
try:
# Requires the pycrypto library
from Crypto.Cipher import DES
except ImportError as e:
DES = None
class HilinkDecryptor(binwalk.core.plugin.Plugin):
'''
Plugin to decrypt, validate, and extract Hilink encrypted firmware.
'''
MODULES = ["Signature"]
DES_KEY = "H@L9K*(3"
SIGNATURE_DESCRIPTION = "Encrypted Hilink uImage firmware".lower()
def init(self):
if DES is None:
self.enabled = False
else:
self.enabled = True
if self.enabled is True and self.module.extractor.enabled is True:
# Add an extraction rule for encrypted Hilink firmware signature results
self.module.extractor.add_rule(regex="^%s" % self.SIGNATURE_DESCRIPTION,
extension="enc",
cmd=self._decrypt_and_extract)
def _decrypt_and_extract(self, fname):
'''
This does the extraction (e.g., it decrypts the image and writes it to a new file on disk).
'''
with open(fname, "r") as fp_in:
encrypted_data = fp_in.read()
decrypted_data = self._hilink_decrypt(encrypted_data)
with open(binwalk.core.common.unique_file_name(fname[:-4], "dec"), "w") as fp_out:
fp_out.write(decrypted_data)
def _hilink_decrypt(self, encrypted_firmware):
'''
This does the actual decryption.
'''
cipher = DES.new(self.DES_KEY, DES.MODE_ECB)
p1 = encrypted_firmware[0:3]
p2 = encrypted_firmware[3:]
p2 += b"\x00" * (8 - (len(p2) % 8))
d1 = p1 + cipher.decrypt(p2)
d1 += b"\x00" * (8 - (len(d1) % 8))
return cipher.decrypt(d1)
def scan(self, result):
'''
Validate signature results.
'''
if self.enabled is True:
if result.valid is True:
if result.description.lower().startswith(self.SIGNATURE_DESCRIPTION) is True:
# Read in the first 64 bytes of the suspected encrypted uImage header
fd = self.module.config.open_file(result.file.name, offset=result.offset)
encrypted_header_data = binwalk.core.compat.str2bytes(fd.read(64))
fd.close()
# Decrypt the header
decrypted_header_data = self._hilink_decrypt(encrypted_header_data)
# Pull out the image size and image name fields from the decrypted uImage header
# and add them to the printed description.
result.size = struct.unpack(b">L", decrypted_header_data[12:16])[0]
result.description += ", size: %d" % (result.size)
# NOTE: The description field should be 32 bytes? Hilink seems to use only 24 bytes for this field,
# even though the header size is still 64 bytes?
result.description += ', image name: "%s"' % binwalk.core.compat.bytes2str(decrypted_header_data[32:56]).strip("\x00")
# Do some basic validation on the decrypted size and image name fields
if result.size > (result.file.size - result.offset):
result.valid = False
if not all(c in string.printable for c in result.description):
result.valid = False
src/binwalk/plugins/lzmaextract.py
......@@ -9,7 +9,14 @@ class LZMAExtractPlugin(binwalk.core.plugin.Plugin):
def init(self):
try:
# lzma package in Python 2.0 decompress() does not handle multiple
# compressed streams, only first stream is extracted.
# backports.lzma package could be used to keep consistent behaviour.
try:
import lzma
except ImportError:
from backports import lzma
self.decompressor = lzma.decompress
# If the extractor is enabled for the module we're currently loaded
......@@ -19,6 +26,10 @@ class LZMAExtractPlugin(binwalk.core.plugin.Plugin):
regex="^lzma compressed data",
extension="7z",
cmd=self.extractor)
self.module.extractor.add_rule(txtrule=None,
regex="^xz compressed data",
extension="xz",
cmd=self.extractor)
except ImportError as e:
pass
......
src/binwalk/plugins/lzmavalid.py
......@@ -17,7 +17,10 @@ class LZMAPlugin(binwalk.core.plugin.Plugin):
def init(self):
try:
try:
import lzma
except ImportError:
from backports import lzma
self.decompressor = lzma.decompress
except ImportError as e:
self.decompressor = None
......
src/binwalk/plugins/ubivalid.py 0 → 100755
import struct
import binascii
import binwalk.core.plugin
import binwalk.core.compat
class UBIValidPlugin(binwalk.core.plugin.Plugin):
'''
Helps validate UBI erase count signature results.
Checks header CRC and calculates jump value
'''
MODULES = ['Signature']
current_file=None
last_ec_hdr_offset = None
peb_size = None
def _check_crc(self, ec_header):
# Get the header's reported CRC value
header_crc = struct.unpack(">I", ec_header[60:64])[0]
# Calculate the actual CRC
calculated_header_crc = ~binascii.crc32(ec_header[0:60]) & 0xffffffff
# Make sure they match
return header_crc == calculated_header_crc
def _process_result(self, result):
if self.current_file == result.file.name:
result.display=False
else:
# Reset everything in case new file is encountered
self.peb_size=None
self.last_ec_hdr_offset=None
self.peb_size=None
# Display result and trigger extraction
result.display=True
self.current_file = result.file.name
if not self.peb_size and self.last_ec_hdr_offset:
# Calculate PEB size by subtracting last EC block offset
self.peb_size = result.offset - self.last_ec_hdr_offset
else:
# First time plugin is called on file, save EC block offset
self.last_ec_hdr_offset = result.offset
if self.peb_size:
# If PEB size has been determined jump PEB size
result.jump = self.peb_size
else:
result.jump = 0
def scan(self, result):
if result.file and result.description.lower().startswith('ubi erase count header'):
# Seek to and read the suspected UBI erase count header
fd = self.module.config.open_file(result.file.name, offset=result.offset)
ec_header = binwalk.core.compat.str2bytes(fd.read(1024))
fd.close()
result.valid = self._check_crc(ec_header[0:64])
if result.valid:
self._process_result(result)
src/scripts/binwalk
......@@ -2,7 +2,6 @@
import os
import sys
from threading import Thread
# If installed to a custom prefix directory, binwalk may not be in
# the default module search path(s). Try to resolve the prefix module
......@@ -15,7 +14,8 @@ for _module_path in [
# from build dir: build/scripts-3.4/ -> build/lib/
os.path.join(_parent_dir, "lib"),
# installed in non-default path: bin/ -> lib/python3.4/site-packages/
os.path.join(_parent_dir, "lib",
os.path.join(_parent_dir,
"lib",
"python%d.%d" % (sys.version_info[0], sys.version_info[1]),
"site-packages")
]:
......@@ -24,37 +24,13 @@ for _module_path in [
import binwalk
import binwalk.modules
from binwalk.core.compat import user_input
def display_status(m):
# Display the current scan progress when the enter key is pressed.
while True:
try:
user_input()
percentage = ((float(m.status.completed) / float(m.status.total)) * 100)
sys.stderr.write("Progress: %.2f%% (%d / %d)\n\n" % (percentage,
m.status.completed,
m.status.total))
except KeyboardInterrupt as e:
raise e
except Exception:
pass
def usage(modules):
sys.stderr.write(modules.help())
sys.exit(1)
def main():
modules = binwalk.Modules()
# Start the display_status function as a daemon thread.
t = Thread(target=display_status, args=(modules,))
t.setDaemon(True)
t.start()
with binwalk.Modules() as modules:
try:
if len(sys.argv) == 1:
usage(modules)
sys.stderr.write(modules.help())
sys.exit(1)
# If no explicit module was enabled in the command line arguments,
# run again with the default signature scan explicitly enabled.
elif not modules.execute():
......@@ -66,9 +42,9 @@ def main():
else:
sys.stderr.write("Error: Signature scans not supported; ")
sys.stderr.write("make sure you have python-lzma installed and try again.\n")
sys.exit(1)
sys.exit(2)
except binwalk.ModuleException as e:
sys.exit(1)
sys.exit(3)
if __name__ == '__main__':
try:
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment