Fixed python3 bugs; updated Plotter3D weighting code; added --2D option.

src/bin/binwalk

@@ -78,8 +78,9 @@ def main():
 	matryoshka = 1
 	block_size = 0
 	failed_open_count = 0
-	weight = None
+	max_points = None
 	max_extract_size = None
+	do_2d = False
 	quiet = False
 	do_comp = False
 	do_files = False
@@ -157,6 +158,10 @@ def main():
 			show_grids = True
 		elif opt in ("-3", "--3D", "--3d"):
 			requested_scans.append(binwalk.Binwalk.BINVIS)
+			do_2d = False
+		elif opt in ("-2", "--2D", "--2d"):
+			requested_scans.append(binwalk.Binwalk.BINVIS)
+			do_2d = True
 		elif opt in ("-E", "--entropy"):
 			requested_scans.append(binwalk.Binwalk.ENTROPY)
 		elif opt in ("-W", "--diff"):
@@ -179,8 +184,8 @@ def main():
 			requested_scans.append(binwalk.Binwalk.STRINGS)
 		elif opt in ("-O", "--skip-unopened"):
 			ignore_failed_open = True
-		elif opt in ("-Z", "--weight"):
-			weight = binwalk.common.str2int(arg)
+		elif opt in ("-Z", "--max-points"):
+			max_points = binwalk.common.str2int(arg)
 		elif opt in ("-o", "--offset"):
 			offset = binwalk.common.str2int(arg)
 		elif opt in ("-l", "--length"):
@@ -468,8 +473,11 @@ def main():
 			elif scan_type == binwalk.Binwalk.BINVIS:
 			
 				# Always enable verbose mode; generating the plot can take some time for large files,
-				# and without verbose mode enabled it looks like binwalk is just sitting there doing nothing.	
-				bwalk.plot3d(target_files, offset=offset, length=length, weight=weight, show_grids=show_grids, verbose=True)
+				# and without verbose mode enabled it looks like binwalk is just sitting there doing nothing.
+				if do_2d:
+					bwalk.plot2d(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=True)
+				else:
+					bwalk.plot3d(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=True)
 
 			elif scan_type == binwalk.Binwalk.ENTROPY:
 

src/binwalk/__init__.py

@@ -290,14 +290,14 @@ class Binwalk(object):
 
 		return data
 
-	def plot3d(self, target_files, offset=0, length=0, weight=None, show_grids=False, verbose=False):
+	def plot3d(self, target_files, offset=0, length=0, max_points=None, show_grids=False, verbose=False):
 		'''
 		Generates a 3D data plot of the specified target files.
 
 		@target_files   - File or list of files to scan.
 		@offset         - Starting offset at which to start the scan.
                 @length         - Number of bytes to scan. Specify 0 to scan the entire file(s).
-		@weight         - A data point must occur at least this many times before being plotted (default: auto-detect).
+		@max_points     - Set the maximum number of data points to plot.
 		@show_grids     - Set to True to show axis grids in the 3D plot.
 		@verbose        - Set to True to enable verbose output.
 
@@ -306,7 +306,25 @@ class Binwalk(object):
 		if not isinstance(target_files, type([])):
 			target_files = [target_files]
 
-		Plotter3D(target_files, offset=offset, length=length, weight=weight, show_grids=show_grids, verbose=verbose).plot()
+		Plotter3D(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=verbose).plot()
+
+	def plot2d(self, target_files, offset=0, length=0, max_points=None, show_grids=False, verbose=False):
+		'''
+		Generates a 2D data plot of the specified target files.
+
+		@target_files   - File or list of files to scan.
+		@offset         - Starting offset at which to start the scan.
+                @length         - Number of bytes to scan. Specify 0 to scan the entire file(s).
+		@max_points     - Set the maximum number of data points to plot.
+		@show_grids     - Set to True to show axis grids in the 3D plot.
+		@verbose        - Set to True to enable verbose output.
+
+		Returns None.
+		'''
+		if not isinstance(target_files, type([])):
+			target_files = [target_files]
+
+		Plotter2D(target_files, offset=offset, length=length, max_points=max_points, show_grids=show_grids, verbose=verbose).plot()
 
 	def scan(self, target_files, offset=0, length=0, show_invalid_results=False, callback=None, start_callback=None, end_callback=None, base_dir=None, matryoshka=1, plugins_whitelist=[], plugins_blacklist=[]):
 		'''

src/binwalk/cmdopts.py

@@ -5,8 +5,9 @@ import os
 import sys
 import binwalk.config
 
-short_options = "3AaBbCcdEeGHhIiJkLMNnOPpQqrSTtUuVvWwz?D:F:f:g:j:K:o:l:m:R:s:X:x:Y:y:Z:"
+short_options = "23AaBbCcdEeGHhIiJkLMNnOPpQqrSTtUuVvWwz?D:F:f:g:j:K:o:l:m:R:s:X:x:Y:y:Z:"
 long_options = [
+		"2D",
 		"3D",
 		"3d",
 		"rm",
@@ -45,7 +46,7 @@ long_options = [
 		"no-legend", 
 		"strings",
 		"carve",
-		"weight=",
+		"max-points=",
 		"matryoshka=",
 		"list-plugins",
 		"disable-plugins",
@@ -110,7 +111,8 @@ def usage(fd):
 
 	fd.write("Binary Visualization:\n")
 	fd.write("\t-3, --3D                      Generate a 3D binary visualization\n")
-	fd.write("\t-Z, --weight                  Manually set the cutoff weight (lower weight, more data points)\n")
+	fd.write("\t-2, --2D                      Project data points onto 3D cube walls only\n")
+	fd.write("\t-Z, --max-points              Set the maximum number of plotted data points (defulat: %d)\n" % binwalk.plotter.Plotter.MAX_PLOT_POINTS)
 	fd.write("\t-V, --show-grids              Display the x-y-z grids in the resulting plot\n")
 	fd.write("\n")
 

src/binwalk/hashmatch.py

@@ -183,7 +183,7 @@ class HashMatch(object):
 						else:
 							return self.lib.fuzzy_compare(hash1, hash2)
 				except Exception as e:
-					print "WARNING: Exception while doing fuzzy hash:", e
+					print ("WARNING: Exception while doing fuzzy hash: %s" % e)
 
 		return None
 
@@ -348,7 +348,7 @@ if __name__ == '__main__':
 	import sys
 	
 	hmatch = HashMatch(strings=True, name=False, types={True:"^elf"})
-	print hmatch.file(sys.argv[1], sys.argv[2:])
+	print (hmatch.file(sys.argv[1], sys.argv[2:]))
 	#for (match, fname) in hmatch.directories(sys.argv[1], sys.argv[2]):
 	#for (match, fname) in hmatch.find_file(sys.argv[1], sys.argv[2:]):
 	#	print match, fname

src/binwalk/plotter.py

@@ -3,20 +3,41 @@ from binwalk.compat import *
 from binwalk.common import BlockFile
 
 class Plotter(object):
+	'''
+	Base class for plotting binaries in Qt.
+	Other plotter classes are derived from this.
+	'''
 
 	DIMENSIONS = 3
 	VIEW_DISTANCE = 1024
+	MAX_PLOT_POINTS = 25000
+
+	def __init__(self, files, offset=0, length=0, max_points=MAX_PLOT_POINTS, show_grids=False, verbose=False):
+		'''
+		Class constructor.
 
-	def __init__(self, files, offset=0, length=0, weight=None, show_grids=False, verbose=False):
+		@files      - A list of files to plot in the graph.
+		@offset     - The starting offset for each file.
+		@length     - The number of bytes to analyze from each file.
+		@max_points - The maximum number of data points to display.
+		@show_grids - Set to True to display x-y-z grids.
+		@verbse     - Set to False to disable verbose print statements.
+
+		Returns None.
+		'''
 		import pyqtgraph.opengl as gl
 		from pyqtgraph.Qt import QtGui
 
 		self.verbose = verbose
 		self.show_grids = show_grids
 		self.files = files
-		self.weight = weight
 		self.offset = offset
 		self.length = length
+	
+		if not max_points:
+			self.max_points = self.MAX_PLOT_POINTS
+		else:
+			self.max_points = max_points
 
 		self.app = QtGui.QApplication([])
 		self.window = gl.GLViewWidget()
@@ -26,55 +47,92 @@ class Plotter(object):
 			self.window.setWindowTitle(self.files[0])
 
 	def _print(self, message):
+		'''
+		Print console messages. For internal use only.
+		'''
 		if self.verbose:
 			print (message)
 
-	def _generate_plot_points(self, data_points, data_weights):
-		plot_points = set()
-		max_plot_points = (24 * 1024)
+	def _generate_plot_points(self, data_points):
+		'''
+		Generates plot points from a list of data points.
+		
+		@data_points - A dictionary containing each unique point and its frequency of occurance.
+
+		Returns a set of plot points.
+		'''
+		total = 0
+		min_weight = 0
+		weightings = {}
+		plot_points = {}
+
+		# If the number of data points exceeds the maximum number of allowed data points, use a
+		# weighting system to eliminate data points that occur less freqently.
+		if sum(data_points.itervalues()) > self.max_points:
+
+			# First, generate a set of weight values 1 - 10
+			for i in range(1, 11):
+				weightings[i] = 0
+
+			# Go through every data point and how many times that point occurs
+			for (point, count) in iterator(data_points):
+				# For each data point, compare it to each remaining weight value
+				for w in get_keys(weightings):
+
+					# If the number of times this data point occurred is >= the weight value,
+					# then increment the weight value. Since weight values are ordered lowest
+					# to highest, this means that more frequent data points also increment lower
+					# weight values. Thus, the more high-frequency data points there are, the
+					# more lower-frequency data points are eliminated.
+					if count >= w:
+						weightings[w] += 1
+					else:
+						break
 
-		if self.weight:
-			weight = self.weight
-		else:
-			self._print("Calculating weight...")
+					# Throw out weight values that exceed the maximum number of data points
+					if weightings[w] > self.max_points:
+						del weightings[w]
 
-			weight = 1
+				# If there's only one weight value left, no sense in continuing the loop...
+				if len(weightings) == 1:
+					break
 
-			if len(data_points) > max_plot_points:
-				weightings = {}
+			# The least weighted value is our minimum weight
+			min_weight = min(weightings)
 
-				for i in range(1, 11):
-					weightings[i] = 0
+			# Get rid of all data points that occur less frequently than our minimum weight
+			for point in get_keys(data_points):
+				if data_points[point] < min_weight:
+					del data_points[point]
 
-				for point in data_points:
-					for w in get_keys(weightings):
-						if data_weights[point] >= w:
-							weightings[w] += 1
+		for point in sorted(data_points, key=data_points.get, reverse=True):
+			plot_points[point] = data_points[point]
+			total += 1
+			if total >= self.max_points:
+				break
 					
-						if weightings[w] > max_plot_points:
-							del weightings[w]
-
-					if len(weightings) <= 1:
-						break
-
-				if weightings:
-					weight = min(weightings)
-
-		self._print("Weight: %d" % weight)
-
-		for point in data_points:
-			if data_weights[point] >= weight:
-				plot_points.add(point)
-			
 		return plot_points
 
 	def _generate_data_point(self, data):
+		'''
+		Subclasses must override this to return the appropriate data point.
+
+		@data - A string of data self.DIMENSIONS in length.
+
+		Returns a data point tuple.
+		'''
 		return (0,0,0)
 
 	def _generate_data_points(self, file_name):
+		'''
+		Generates a dictionary of data points and their frequency of occurrance.
+
+		@file_name - The file to generate data points from.
+
+		Returns a dictionary.
+		'''
 		i = 0
-		data_weights = {}
-		data_points = set()
+		data_points = {}
 
 		self._print("Generating data points for %s" % file_name)
 
@@ -89,40 +147,44 @@ class Plotter(object):
 				i = 0
 				while (i+(self.DIMENSIONS-1)) < dlen:
 					point = self._generate_data_point(data[i:i+self.DIMENSIONS])
-					if point in data_points:	
-						data_weights[point] += 1
+					if has_key(data_points, point):	
+						data_points[point] += 1
 					else:
-						data_points.add(point)
-						data_weights[point] = 1
+						data_points[point] = 1
 					i += 3
 
-		return (data_points, data_weights)
+		return data_points
 
-	def _generate_plot(self, plot_points, point_weights):
+	def _generate_plot(self, plot_points):
 		import numpy as np
 		import pyqtgraph.opengl as gl
 		
-		nitems = len(plot_points)
+		nitems = float(len(plot_points))
 
 		pos = np.empty((nitems, 3))
 		size = np.empty((nitems))
 		color = np.empty((nitems, 4))
 
 		i = 0
-		for point in plot_points:
+		for (point, weight) in iterator(plot_points):
 			r = 0.0
 			g = 0.0
 			b = 0.0
 
 			pos[i] = point
-			size[i] = .05
+			frequency_percentage = (weight / nitems)
 
-			if point_weights[point] > 15:
+			# Give points that occur more frequently a brighter color and larger point size.
+			# Frequency is determined as a percentage of total unique data points.
+			if frequency_percentage > .005:
+				size[i] = .20
 				r = 1.0
-			elif point_weights[point] > 5:
+			elif frequency_percentage > .002:
+				size[i] = .10
 				g = 1.0
 				r = 1.0
 			else:
+				size[i] = .05
 				g = 1.0
 
 			color[i] = (r, g, b, 1.0)
@@ -158,15 +220,16 @@ class Plotter(object):
 			zgrid.scale(12.8, 12.8, 12.8)
 
 		for file_name in self.files:
-			(data_points, data_weights) = self._generate_data_points(file_name)
+			data_points = self._generate_data_points(file_name)
 
 			self._print("Generating plot points from %d data points" % len(data_points))
 
-			plot_points = self._generate_plot_points(data_points, data_weights)
+			plot_points = self._generate_plot_points(data_points)
+			del data_points
 
 			self._print("Generating graph from %d plot points" % len(plot_points))
 
-			self.window.addItem(self._generate_plot(plot_points, data_weights))
+			self.window.addItem(self._generate_plot(plot_points))
 
 		if wait:
 			self.wait()
@@ -180,7 +243,9 @@ class Plotter(object):
 
 
 class Plotter3D(Plotter):
-
+	'''
+	Plot data points within a 3D cube.
+	'''
 	DIMENSIONS = 3
 
 	def _generate_data_point(self, data):
@@ -188,15 +253,16 @@ class Plotter3D(Plotter):
 	
 class Plotter2D(Plotter):
 	'''
-	This is of questionable use.
+	Plot data points projected on each cube face.
 	'''
 
 	DIMENSIONS = 2
+	MAX_PLOT_POINTS = 12500
 	plane_count = -1
 
 	def _generate_data_point(self, data):
 		self.plane_count += 1
-		if self.plane_count > 2:
+		if self.plane_count > 5:
 			self.plane_count = 0
 
 		if self.plane_count == 0:
@@ -205,6 +271,12 @@ class Plotter2D(Plotter):
 			return (ord(data[0]), 0, ord(data[1]))
 		elif self.plane_count == 2:
 			return (ord(data[0]), ord(data[1]), 0)
+		elif self.plane_count == 3:
+			return (255, ord(data[0]), ord(data[1]))
+		elif self.plane_count == 4:
+			return (ord(data[0]), 255, ord(data[1]))
+		elif self.plane_count == 5:
+			return (ord(data[0]), ord(data[1]), 255)
 		
 if __name__ == '__main__':
 	import sys
@@ -214,5 +286,5 @@ if __name__ == '__main__':
 	except:
 		weight = None
 
-	Plotter3D(sys.argv[1:], weight=weight, verbose=True).plot()
+	Plotter2D(sys.argv[1:], weight=weight, verbose=True).plot()
 

src/easy_install.sh

@@ -102,6 +102,10 @@ function debian
 	# Install binwalk/fmk pre-requisites and extraction tools
 	$SUDO apt-get -y install git build-essential mtd-utils zlib1g-dev liblzma-dev ncompress gzip bzip2 tar arj p7zip p7zip-full openjdk-6-jdk
 	$SUDO apt-get -y install python-opengl python-qt4 python-qt4-gl python-numpy python-scipy
+	if [ "$(which python3)" != "" ]
+	then
+		$SUDO apt-get -y install python3-pyqt4 python3-numpy python3-scipy
+	fi
 }
 
 function redhat
@@ -109,12 +113,16 @@ function redhat
 	$SUDO yum groupinstall -y "Development Tools"
 	$SUDO yum install -y git mtd-utils unrar zlib1g-dev liblzma-dev xz-devel compress gzip bzip2 tar arj p7zip p7zip-full openjdk-6-jdk
 	$SUDO yum install -y python-opengl python-qt4 python-qt4-gl python-numpy python-scipy
+	if [ "$(which python3)" != "" ]
+	then
+		$SUDO yum -y install python3-pyqt4 python3-numpy python3-scipy
+	fi
 }
 
 if [ "$1" == "" ] || [ "$1" == "--sumount" ]
 then
-	PLATFORM=$(python -c 'import platform; print platform.system().lower()')
-	DISTRO=$(python -c 'import platform; print platform.linux_distribution()[0].lower()')
+	PLATFORM=$(python -c 'import platform; print (platform.system().lower())')
+	DISTRO=$(python -c 'import platform; print (platform.linux_distribution()[0].lower())')
 else
 	DISTRO="$1"
 fi
@@ -185,7 +193,7 @@ then
 fi
 
 # Get and build the firmware mod kit
-#fmk
+fmk
 
 # Install binwalk
 $SUDO python setup.py install