diff --git a/src/bin/binwalk b/src/bin/binwalk
index 3506ca9..206cc15 100755
--- a/src/bin/binwalk
+++ b/src/bin/binwalk
@@ -78,8 +78,9 @@ def main():
matryoshka = 1
block_size = 0
failed_open_count = 0
- weight = None
+ max_points = None
max_extract_size = None
+ do_2d = False
quiet = False
do_comp = False
do_files = False
@@ -157,6 +158,10 @@ def main():
show_grids = True
elif opt in ("-3", "--3D", "--3d"):
requested_scans.append(binwalk.Binwalk.BINVIS)
+ do_2d = False
+ elif opt in ("-2", "--2D", "--2d"):
+ requested_scans.append(binwalk.Binwalk.BINVIS)
+ do_2d = True
elif opt in ("-E", "--entropy"):
requested_scans.append(binwalk.Binwalk.ENTROPY)
elif opt in ("-W", "--diff"):
@@ -179,8 +184,8 @@ def main():
requested_scans.append(binwalk.Binwalk.STRINGS)
elif opt in ("-O", "--skip-unopened"):
ignore_failed_open = True
- elif opt in ("-Z", "--weight"):
- weight = binwalk.common.str2int(arg)
+ elif opt in ("-Z", "--max-points"):
+ max_points = binwalk.common.str2int(arg)
elif opt in ("-o", "--offset"):
offset = binwalk.common.str2int(arg)
elif opt in ("-l", "--length"):
@@ -468,8 +473,11 @@ def main():
elif scan_type == binwalk.Binwalk.BINVIS:
# Always enable verbose mode; generating the plot can take some time for large files,
- # and without verbose mode enabled it looks like binwalk is just sitting there doing nothing.
- bwalk.plot3d(target_files, offset=offset, length=length, weight=weight, show_grids=show_grids, verbose=True)
+ # and without verbose mode enabled it looks like binwalk is just sitting there doing nothing.
+ if do_2d:
+ bwalk.plot2d(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=True)
+ else:
+ bwalk.plot3d(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=True)
elif scan_type == binwalk.Binwalk.ENTROPY:
diff --git a/src/binwalk/__init__.py b/src/binwalk/__init__.py
index f15065e..7acb617 100644
--- a/src/binwalk/__init__.py
+++ b/src/binwalk/__init__.py
@@ -290,14 +290,14 @@ class Binwalk(object):
return data
- def plot3d(self, target_files, offset=0, length=0, weight=None, show_grids=False, verbose=False):
+ def plot3d(self, target_files, offset=0, length=0, max_points=None, show_grids=False, verbose=False):
'''
Generates a 3D data plot of the specified target files.
@target_files - File or list of files to scan.
@offset - Starting offset at which to start the scan.
@length - Number of bytes to scan. Specify 0 to scan the entire file(s).
- @weight - A data point must occur at least this many times before being plotted (default: auto-detect).
+ @max_points - Set the maximum number of data points to plot.
@show_grids - Set to True to show axis grids in the 3D plot.
@verbose - Set to True to enable verbose output.
@@ -306,7 +306,25 @@ class Binwalk(object):
if not isinstance(target_files, type([])):
target_files = [target_files]
- Plotter3D(target_files, offset=offset, length=length, weight=weight, show_grids=show_grids, verbose=verbose).plot()
+ Plotter3D(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=verbose).plot()
+
+ def plot2d(self, target_files, offset=0, length=0, max_points=None, show_grids=False, verbose=False):
+ '''
+ Generates a 2D data plot of the specified target files.
+
+ @target_files - File or list of files to scan.
+ @offset - Starting offset at which to start the scan.
+ @length - Number of bytes to scan. Specify 0 to scan the entire file(s).
+ @max_points - Set the maximum number of data points to plot.
+ @show_grids - Set to True to show axis grids in the 3D plot.
+ @verbose - Set to True to enable verbose output.
+
+ Returns None.
+ '''
+ if not isinstance(target_files, type([])):
+ target_files = [target_files]
+
+ Plotter2D(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=verbose).plot()
def scan(self, target_files, offset=0, length=0, show_invalid_results=False, callback=None, start_callback=None, end_callback=None, base_dir=None, matryoshka=1, plugins_whitelist=[], plugins_blacklist=[]):
'''
diff --git a/src/binwalk/cmdopts.py b/src/binwalk/cmdopts.py
index 628f1a2..a4644da 100644
--- a/src/binwalk/cmdopts.py
+++ b/src/binwalk/cmdopts.py
@@ -5,8 +5,9 @@ import os
import sys
import binwalk.config
-short_options = "3AaBbCcdEeGHhIiJkLMNnOPpQqrSTtUuVvWwz?D:F:f:g:j:K:o:l:m:R:s:X:x:Y:y:Z:"
+short_options = "23AaBbCcdEeGHhIiJkLMNnOPpQqrSTtUuVvWwz?D:F:f:g:j:K:o:l:m:R:s:X:x:Y:y:Z:"
long_options = [
+ "2D",
"3D",
"3d",
"rm",
@@ -45,7 +46,7 @@ long_options = [
"no-legend",
"strings",
"carve",
- "weight=",
+ "max-points=",
"matryoshka=",
"list-plugins",
"disable-plugins",
@@ -110,7 +111,8 @@ def usage(fd):
fd.write("Binary Visualization:\n")
fd.write("\t-3, --3D Generate a 3D binary visualization\n")
- fd.write("\t-Z, --weight Manually set the cutoff weight (lower weight, more data points)\n")
+ fd.write("\t-2, --2D Project data points onto 3D cube walls only\n")
+ fd.write("\t-Z, --max-points Set the maximum number of plotted data points (defulat: %d)\n" % binwalk.plotter.Plotter.MAX_PLOT_POINTS)
fd.write("\t-V, --show-grids Display the x-y-z grids in the resulting plot\n")
fd.write("\n")
diff --git a/src/binwalk/hashmatch.py b/src/binwalk/hashmatch.py
index 43e056f..99d9a50 100644
--- a/src/binwalk/hashmatch.py
+++ b/src/binwalk/hashmatch.py
@@ -183,7 +183,7 @@ class HashMatch(object):
else:
return self.lib.fuzzy_compare(hash1, hash2)
except Exception as e:
- print "WARNING: Exception while doing fuzzy hash:", e
+ print ("WARNING: Exception while doing fuzzy hash: %s" % e)
return None
@@ -348,7 +348,7 @@ if __name__ == '__main__':
import sys
hmatch = HashMatch(strings=True, name=False, types={True:"^elf"})
- print hmatch.file(sys.argv[1], sys.argv[2:])
+ print (hmatch.file(sys.argv[1], sys.argv[2:]))
#for (match, fname) in hmatch.directories(sys.argv[1], sys.argv[2]):
#for (match, fname) in hmatch.find_file(sys.argv[1], sys.argv[2:]):
# print match, fname
diff --git a/src/binwalk/plotter.py b/src/binwalk/plotter.py
index d57f8bc..8bdbbe7 100644
--- a/src/binwalk/plotter.py
+++ b/src/binwalk/plotter.py
@@ -3,20 +3,41 @@ from binwalk.compat import *
from binwalk.common import BlockFile
class Plotter(object):
+ '''
+ Base class for plotting binaries in Qt.
+ Other plotter classes are derived from this.
+ '''
DIMENSIONS = 3
VIEW_DISTANCE = 1024
+ MAX_PLOT_POINTS = 25000
+
+ def __init__(self, files, offset=0, length=0, max_points=MAX_PLOT_POINTS, show_grids=False, verbose=False):
+ '''
+ Class constructor.
- def __init__(self, files, offset=0, length=0, weight=None, show_grids=False, verbose=False):
+ @files - A list of files to plot in the graph.
+ @offset - The starting offset for each file.
+ @length - The number of bytes to analyze from each file.
+ @max_points - The maximum number of data points to display.
+ @show_grids - Set to True to display x-y-z grids.
+ @verbse - Set to False to disable verbose print statements.
+
+ Returns None.
+ '''
import pyqtgraph.opengl as gl
from pyqtgraph.Qt import QtGui
self.verbose = verbose
self.show_grids = show_grids
self.files = files
- self.weight = weight
self.offset = offset
self.length = length
+
+ if not max_points:
+ self.max_points = self.MAX_PLOT_POINTS
+ else:
+ self.max_points = max_points
self.app = QtGui.QApplication([])
self.window = gl.GLViewWidget()
@@ -26,55 +47,92 @@ class Plotter(object):
self.window.setWindowTitle(self.files[0])
def _print(self, message):
+ '''
+ Print console messages. For internal use only.
+ '''
if self.verbose:
print (message)
- def _generate_plot_points(self, data_points, data_weights):
- plot_points = set()
- max_plot_points = (24 * 1024)
+ def _generate_plot_points(self, data_points):
+ '''
+ Generates plot points from a list of data points.
+
+ @data_points - A dictionary containing each unique point and its frequency of occurance.
+
+ Returns a set of plot points.
+ '''
+ total = 0
+ min_weight = 0
+ weightings = {}
+ plot_points = {}
+
+ # If the number of data points exceeds the maximum number of allowed data points, use a
+ # weighting system to eliminate data points that occur less freqently.
+ if sum(data_points.itervalues()) > self.max_points:
+
+ # First, generate a set of weight values 1 - 10
+ for i in range(1, 11):
+ weightings[i] = 0
+
+ # Go through every data point and how many times that point occurs
+ for (point, count) in iterator(data_points):
+ # For each data point, compare it to each remaining weight value
+ for w in get_keys(weightings):
+
+ # If the number of times this data point occurred is >= the weight value,
+ # then increment the weight value. Since weight values are ordered lowest
+ # to highest, this means that more frequent data points also increment lower
+ # weight values. Thus, the more high-frequency data points there are, the
+ # more lower-frequency data points are eliminated.
+ if count >= w:
+ weightings[w] += 1
+ else:
+ break
- if self.weight:
- weight = self.weight
- else:
- self._print("Calculating weight...")
+ # Throw out weight values that exceed the maximum number of data points
+ if weightings[w] > self.max_points:
+ del weightings[w]
- weight = 1
+ # If there's only one weight value left, no sense in continuing the loop...
+ if len(weightings) == 1:
+ break
- if len(data_points) > max_plot_points:
- weightings = {}
+ # The least weighted value is our minimum weight
+ min_weight = min(weightings)
- for i in range(1, 11):
- weightings[i] = 0
+ # Get rid of all data points that occur less frequently than our minimum weight
+ for point in get_keys(data_points):
+ if data_points[point] < min_weight:
+ del data_points[point]
- for point in data_points:
- for w in get_keys(weightings):
- if data_weights[point] >= w:
- weightings[w] += 1
+ for point in sorted(data_points, key=data_points.get, reverse=True):
+ plot_points[point] = data_points[point]
+ total += 1
+ if total >= self.max_points:
+ break
- if weightings[w] > max_plot_points:
- del weightings[w]
-
- if len(weightings) <= 1:
- break
-
- if weightings:
- weight = min(weightings)
-
- self._print("Weight: %d" % weight)
-
- for point in data_points:
- if data_weights[point] >= weight:
- plot_points.add(point)
-
return plot_points
def _generate_data_point(self, data):
+ '''
+ Subclasses must override this to return the appropriate data point.
+
+ @data - A string of data self.DIMENSIONS in length.
+
+ Returns a data point tuple.
+ '''
return (0,0,0)
def _generate_data_points(self, file_name):
+ '''
+ Generates a dictionary of data points and their frequency of occurrance.
+
+ @file_name - The file to generate data points from.
+
+ Returns a dictionary.
+ '''
i = 0
- data_weights = {}
- data_points = set()
+ data_points = {}
self._print("Generating data points for %s" % file_name)
@@ -89,40 +147,44 @@ class Plotter(object):
i = 0
while (i+(self.DIMENSIONS-1)) < dlen:
point = self._generate_data_point(data[i:i+self.DIMENSIONS])
- if point in data_points:
- data_weights[point] += 1
+ if has_key(data_points, point):
+ data_points[point] += 1
else:
- data_points.add(point)
- data_weights[point] = 1
+ data_points[point] = 1
i += 3
- return (data_points, data_weights)
+ return data_points
- def _generate_plot(self, plot_points, point_weights):
+ def _generate_plot(self, plot_points):
import numpy as np
import pyqtgraph.opengl as gl
- nitems = len(plot_points)
+ nitems = float(len(plot_points))
pos = np.empty((nitems, 3))
size = np.empty((nitems))
color = np.empty((nitems, 4))
i = 0
- for point in plot_points:
+ for (point, weight) in iterator(plot_points):
r = 0.0
g = 0.0
b = 0.0
pos[i] = point
- size[i] = .05
+ frequency_percentage = (weight / nitems)
- if point_weights[point] > 15:
+ # Give points that occur more frequently a brighter color and larger point size.
+ # Frequency is determined as a percentage of total unique data points.
+ if frequency_percentage > .005:
+ size[i] = .20
r = 1.0
- elif point_weights[point] > 5:
+ elif frequency_percentage > .002:
+ size[i] = .10
g = 1.0
r = 1.0
else:
+ size[i] = .05
g = 1.0
color[i] = (r, g, b, 1.0)
@@ -158,15 +220,16 @@ class Plotter(object):
zgrid.scale(12.8, 12.8, 12.8)
for file_name in self.files:
- (data_points, data_weights) = self._generate_data_points(file_name)
+ data_points = self._generate_data_points(file_name)
self._print("Generating plot points from %d data points" % len(data_points))
- plot_points = self._generate_plot_points(data_points, data_weights)
+ plot_points = self._generate_plot_points(data_points)
+ del data_points
self._print("Generating graph from %d plot points" % len(plot_points))
- self.window.addItem(self._generate_plot(plot_points, data_weights))
+ self.window.addItem(self._generate_plot(plot_points))
if wait:
self.wait()
@@ -180,7 +243,9 @@ class Plotter(object):
class Plotter3D(Plotter):
-
+ '''
+ Plot data points within a 3D cube.
+ '''
DIMENSIONS = 3
def _generate_data_point(self, data):
@@ -188,15 +253,16 @@ class Plotter3D(Plotter):
class Plotter2D(Plotter):
'''
- This is of questionable use.
+ Plot data points projected on each cube face.
'''
DIMENSIONS = 2
+ MAX_PLOT_POINTS = 12500
plane_count = -1
def _generate_data_point(self, data):
self.plane_count += 1
- if self.plane_count > 2:
+ if self.plane_count > 5:
self.plane_count = 0
if self.plane_count == 0:
@@ -205,6 +271,12 @@ class Plotter2D(Plotter):
return (ord(data[0]), 0, ord(data[1]))
elif self.plane_count == 2:
return (ord(data[0]), ord(data[1]), 0)
+ elif self.plane_count == 3:
+ return (255, ord(data[0]), ord(data[1]))
+ elif self.plane_count == 4:
+ return (ord(data[0]), 255, ord(data[1]))
+ elif self.plane_count == 5:
+ return (ord(data[0]), ord(data[1]), 255)
if __name__ == '__main__':
import sys
@@ -214,5 +286,5 @@ if __name__ == '__main__':
except:
weight = None
- Plotter3D(sys.argv[1:], weight=weight, verbose=True).plot()
+ Plotter2D(sys.argv[1:], weight=weight, verbose=True).plot()
diff --git a/src/easy_install.sh b/src/easy_install.sh
index adc63eb..c0dea84 100755
--- a/src/easy_install.sh
+++ b/src/easy_install.sh
@@ -102,6 +102,10 @@ function debian
# Install binwalk/fmk pre-requisites and extraction tools
$SUDO apt-get -y install git build-essential mtd-utils zlib1g-dev liblzma-dev ncompress gzip bzip2 tar arj p7zip p7zip-full openjdk-6-jdk
$SUDO apt-get -y install python-opengl python-qt4 python-qt4-gl python-numpy python-scipy
+ if [ "$(which python3)" != "" ]
+ then
+ $SUDO apt-get -y install python3-pyqt4 python3-numpy python3-scipy
+ fi
}
function redhat
@@ -109,12 +113,16 @@ function redhat
$SUDO yum groupinstall -y "Development Tools"
$SUDO yum install -y git mtd-utils unrar zlib1g-dev liblzma-dev xz-devel compress gzip bzip2 tar arj p7zip p7zip-full openjdk-6-jdk
$SUDO yum install -y python-opengl python-qt4 python-qt4-gl python-numpy python-scipy
+ if [ "$(which python3)" != "" ]
+ then
+ $SUDO yum -y install python3-pyqt4 python3-numpy python3-scipy
+ fi
}
if [ "$1" == "" ] || [ "$1" == "--sumount" ]
then
- PLATFORM=$(python -c 'import platform; print platform.system().lower()')
- DISTRO=$(python -c 'import platform; print platform.linux_distribution()[0].lower()')
+ PLATFORM=$(python -c 'import platform; print (platform.system().lower())')
+ DISTRO=$(python -c 'import platform; print (platform.linux_distribution()[0].lower())')
else
DISTRO="$1"
fi
@@ -185,7 +193,7 @@ then
fi
# Get and build the firmware mod kit
-#fmk
+fmk
# Install binwalk
$SUDO python setup.py install