Consolidated code into the binwalk.module.Module parent class.

src/binwalk/module.py

@@ -6,8 +6,11 @@ import binwalk.common
 from binwalk.compat import *
 
 class ModuleOption(object):
+	'''
+	A container class that allows modules to declare command line options.
+	'''
 
-	def __init__(self, kwargs={}, nargs=0, priority=0, description="", short="", long="", type=str):
+	def __init__(self, kwargs={}, nargs=0, priority=0, description="", short="", long="", type=str, dtype=""):
 		'''
 		Class constructor.
 
@@ -18,6 +21,9 @@ class ModuleOption(object):
 		@short       - The short option to use (optional).
 		@long        - The long option to use (if None, this option will not be displayed in help output).
 		@type        - The accepted data type (one of: io.FileIO/argparse.FileType/binwalk.common.BlockFile, list, str, int, float).
+		@dtype       - The accepted data type, as displayed in the help output.
+
+		Returns None.
 		'''
 		self.kwargs = kwargs
 		self.nargs = nargs
@@ -26,9 +32,16 @@ class ModuleOption(object):
 		self.short = short
 		self.long = long
 		self.type = type
+		self.dtype = str(dtype)
+
+		if not self.dtype and self.type:
+			self.dtype = str(self.type)
 
 class ModuleKwarg(object):
-		
+		'''
+		A container class allowing modules to specify their expected __init__ kwarg(s).
+		'''
+
 		def __init__(self, name="", default=None, description=""):
 			'''
 			Class constructor.
@@ -43,17 +56,203 @@ class ModuleKwarg(object):
 			self.default = default
 			self.description = description
 
+class Result(object):
+	'''
+	Generic class for storing and accessing scan results.
+	'''
 
-def process_kwargs(module, kwargs):
-	return Modules(dummy=True).kwargs(module, kwargs)
+	def __init__(self, **kwargs):
+		'''
+		Class constructor.
+
+		@offset      - The file offset of the result.
+		@description - The result description, as displayed to the user.
+		@file        - The file object of the scanned file.
+		@valid       - Set to True if the result if value, False if invalid.
+		@display     - Set to True to display the result to the user, False to hide it.
+
+		Provide additional kwargs as necessary.
+		Returns None.
+		'''
+		self.offset = 0
+		self.description = ''
+		self.file = None
+		self.valid = True
+		self.display = True
+
+		for (k, v) in iterator(kwargs):
+			setattr(self, k, v)
+
+class Error(Result):
+	'''
+	A subclass of binwalk.module.Result.
+	Accepts all the same kwargs as binwalk.module.Result, but the following are also suggested:
+
+	@exception - In case of an exception, this is the exception object.
+
+	__init__ returns None.
+	'''
+	pass
+
+class Module(object):
+	'''
+	All module classes must be subclassed from this.
+	'''
+
+	# The module name, as displayed in help output
+	NAME = ""
+
+	# A list of binwalk.module.ModuleOption command line options
+	CLI = []
+
+	# A list of binwalk.module.ModuleKwargs accepted by __init__
+	KWARGS = []
+
+	# Format string for printing the header during a scan
+	HEADER_FORMAT = "%s\n"
+
+	# Format string for printing each result during a scan 
+	RESULT_FORMAT = "%.8d      %s\n"
+
+	# The header to print during a scan.
+	# Set to None to not print a header.
+	# Note that this will be formatted per the HEADER_FORMAT format string.
+	HEADER = ["OFFSET      DESCRIPTION"]
+
+	# The attribute names to print during a scan, as provided to the self.results method.
+	# Set to None to not print any results.
+	# Note that these will be formatted per the RESULT_FORMAT format string.
+	RESULT = ['offset', 'description']
+
+	def __init__(self, **kwargs):
+		# TODO: Instantiate plugins object
+		# self.plugins = x
+		self.results = []
+		process_kwargs(self, kwargs)
+
+	def init(self):
+		'''
+		Invoked prior to self.run.
+		May be overridden by the module sub-class.
+
+		Returns None.
+		'''
+		return None
+
+	def run(self):
+		'''
+		Executes the main module routine.
+		Must be overridden by the module sub-class.
+
+		Returns True on success, False on failure.
+		'''
+		return False
+
+	def validate(self, r):
+		'''
+		Validates the result.
+		May be overridden by the module sub-class.
+
+		@r - The result, an instance of binwalk.module.Result.
+
+		Returns None.
+		'''
+		r.valid = True
+		return None
+
+	def _plugins_pre_scan(self):
+		# plugins(self)
+		return None
+
+	def _plugins_post_scan(self):
+		# plugins(self)
+		return None
+
+	def _plugins_callback(self, r):
+		return None
 
-def show_help():
-	print Modules(dummy=True).help()
+	def _build_display_args(self, r):
+		args = []
+
+		if self.RESULT:
+			if type(self.RESULT) != type([]):
+				result = [self.RESULT]
+			else:
+				result = self.RESULT
+	
+			for name in result:
+				args.append(getattr(r, name))
+		
+		return args
+
+	def result(self, **kwargs):
+		'''
+		Validates a result, stores it in self.results and prints it.
+
+		Accepts the same kwargs as the binwalk.module.Result class.
+
+		Returns None.
+		'''
+		r = Result(**kwargs)
+
+		self.validate(r)
+		self._plugins_callback(r)
+
+		if r.valid:
+			self.results.append(r)
+			if r.display:
+				display_args = self._build_display_args(r)
+				if display_args:
+					self.config.display.result(*display_args)
+
+	def error(self, **kwargs):
+		'''
+		Stores the specified error in self.errors.
+
+		Accepts the same kwargs as the binwalk.module.Error class.
+
+		Returns None.
+		'''
+		e = Error(**kwargs)
+		self.errors.append(e)
+
+	def main(self):
+		'''
+		Responsible for calling self.init, initializing self.config.display, printing the header and calling self.run.
+
+		Returns the value returned from self.run.
+		'''
+		self.init()
+
+		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
+		if type(self.HEADER) == type([]):
+			self.config.display.header(*self.HEADER)
+		elif self.HEADER:
+			self.config.display.header(self.HEADER)
+		
+		self._plugins_pre_scan()
+		retval = self.run()
+		self._plugins_post_scan()
+		
+		self.config.display.footer()
+		return retval
 
 class Modules(object):
+	'''
+	Main class used for running and managing modules.
+	'''
+
+	def __init__(self, argv=sys.argv[1:], dummy=False):
+		'''
+		Class constructor.
+
+		@argv  - List of command line options. Must not include the program name (sys.argv[0]).
+		@dummy - Set to True if you only need the class instance for interrogating modules (run, load, execute will not work).
 
-	def __init__(self, dummy=False):
+		Returns None.
+		'''
 		self.config = None
+		self.argv = argv
 		self.dependency_results = {}
 
 		if not dummy:
@@ -61,62 +260,70 @@ class Modules(object):
 			self.config = self.load(Configuration)
 
 	def list(self, attribute="run"):
+		'''
+		Finds all modules with the specified attribute.
+
+		@attribute - The desired module attribute.
+
+		Returns a list of modules that contain the specified attribute.
+		'''
 		import binwalk.modules
 
-		objects = []
+		modules = []
+
+		for (name, module) in inspect.getmembers(binwalk.modules):
+			if inspect.isclass(module) and hasattr(module, attribute):
+				modules.append(module)
 
-		for (name, obj) in inspect.getmembers(binwalk.modules):
-			if inspect.isclass(obj) and hasattr(obj, attribute):
-				objects.append(obj)
-		return objects
+		return modules
 
 	def help(self):
 		help_string = ""
 
 		for obj in self.list(attribute="CLI"):
-			help_string += "\n%s Options:\n" % obj.NAME
+			if obj.CLI:
+				help_string += "\n%s Options:\n" % obj.NAME
 
-			for module_option in obj.CLI:
-				if module_option.long:
-					long_opt = '--' + module_option.long
+				for module_option in obj.CLI:
+					if module_option.long:
+						long_opt = '--' + module_option.long
 					
-					if module_option.nargs > 0:
-						optargs = "=%s" % str(module_option.type)
-					else:
-						optargs = ""
+						if module_option.nargs > 0:
+							optargs = "=%s" % module_option.dtype
+						else:
+							optargs = ""
 
-					if module_option.short:
-						short_opt = "-" + module_option.short + ","
-					else:
-						short_opt = "   "
+						if module_option.short:
+							short_opt = "-" + module_option.short + ","
+						else:
+							short_opt = "   "
 
-					fmt = "    %%s %%s%%-%ds%%s\n" % (32-len(long_opt))
-					help_string += fmt % (short_opt, long_opt, optargs, module_option.description)
+						fmt = "    %%s %%s%%-%ds%%s\n" % (32-len(long_opt))
+						help_string += fmt % (short_opt, long_opt, optargs, module_option.description)
 
 		return help_string
 
 	def execute(self):
-		results = {}
+		run_modules = []
 		for module in self.list():
-			result = self.run(module)
-			if result is not None:
-				results[module] = result
-		return results
+			if self.run(module):
+				run_modules.append(module)
+		return run_modules
 
 	def run(self, module):
-		results = None
+		retval = False
 		obj = self.load(module)
 
 		if obj.enabled:
 			try:
-				results = obj.run()
+				retval = obj.main()
 			except AttributeError as e:
 				print("WARNING:", e)
 
-		return results
+		return retval
 			
 	def load(self, module):
-		kwargs = self.argv(module)
+		kwargs = self.argv(module, argv=self.argv)
 		kwargs.update(self.dependencies(module))
 		return module(**kwargs)
 
@@ -132,7 +339,8 @@ class Modules(object):
 
 			for (kwarg, mod) in iterator(module.DEPENDS):
 				if not has_key(self.dependency_results, mod):
-					self.dependency_results[mod] = self.run(mod)
+					self.dependency_results[mod] = self.load(mod)
+					self.dependency_results[mod].main()
 				kwargs[kwarg] = self.dependency_results[mod]
 	
 			self.config.display.log = orig_log	
@@ -211,7 +419,7 @@ class Modules(object):
 							else:
 								kwargs[name] = value
 		else:
-			raise Exception("binwalk.module.argv: %s has no attribute 'CLI'" % str(module))
+			raise Exception("binwalk.module.Modules.argv: %s has no attribute 'CLI'" % str(module))
 
 		if self.config is not None and not has_key(kwargs, 'config'):
 			kwargs['config'] = self.config
@@ -243,5 +451,28 @@ class Modules(object):
 			if not hasattr(module, 'enabled'):
 				setattr(module, 'enabled', False)
 		else:
-			raise Exception("binwalk.module.process_kwargs: %s has no attribute 'KWARGS'" % str(module))
+			raise Exception("binwalk.module.Modules.process_kwargs: %s has no attribute 'KWARGS'" % str(module))
+
+
+def process_kwargs(obj, kwargs):
+	'''
+	Convenience wrapper around binwalk.module.Modules.kwargs.
+
+	@obj    - The class object (an instance of a sub-class of binwalk.module.Module).
+	@kwargs - The kwargs provided to the object's __init__ method.
+
+	Returns None.
+	'''
+	return Modules(dummy=True).kwargs(module, kwargs)
+
+def show_help(fd=sys.stdout):
+	'''
+	Convenience wrapper around binwalk.module.Modules.help.
+
+	@fd - An object with a write method (e.g., sys.stdout, sys.stderr, etc).
+
+	Returns None.
+	'''
+	fd.write(Modules(dummy=True).help())
+
 

src/binwalk/modules/binvis.py

@@ -3,7 +3,7 @@ import binwalk.module
 from binwalk.compat import *
 from binwalk.common import BlockFile
 
-class Plotter(object):
+class Plotter(binwalk.module.Module):
 	'''
 	Base class for plotting binaries in Qt.
 	Other plotter classes are derived from this.
@@ -41,21 +41,13 @@ class Plotter(object):
 			binwalk.module.ModuleKwarg(name='show_grids', default=False),
 	]
 
-	def __init__(self, **kwargs):
-		'''
-		Class constructor.
-
-		@axis       - Set to 2 for 2D plotting, 3 for 3D plotting.
-		@max_points - The maximum number of data points to display.
-		@show_grids - Set to True to display x-y-z grids.
+	HEADER = None
+	RESULT = None
 
-		Returns None.
-		'''
+	def init(self):
 		import pyqtgraph.opengl as gl
 		from pyqtgraph.Qt import QtGui
 
-		binwalk.module.process_kwargs(self, kwargs)
-
 		self.verbose = self.config.verbose
 		self.offset = self.config.offset
 		self.length = self.config.length
@@ -142,6 +134,7 @@ class Plotter(object):
 
 		for point in sorted(data_points, key=data_points.get, reverse=True):
 			plot_points[point] = data_points[point]
+			self.result(point=point)
 			total += 1
 			if total >= self.max_points:
 				break
@@ -305,5 +298,5 @@ class Plotter(object):
 	
 	def run(self):
 		self.plot()
-		return self.plot_points
+		return True
 

src/binwalk/modules/configuration.py

@@ -6,7 +6,7 @@ import binwalk.display
 from binwalk.config import *
 from binwalk.compat import *
 
-class Configuration(object):
+class Configuration(binwalk.module.Module):
 
 	NAME = "General"
 	CLI = [
@@ -28,12 +28,6 @@ class Configuration(object):
 									type=int,
 									kwargs={'block' : 0},
 									description='Set file block size'),
-		binwalk.module.ModuleOption(long='grep',
-									short='g',
-									nargs=1,
-									kwargs={'grep' : []},
-									type=list,
-									description='Grep results for the specified text'),
 		binwalk.module.ModuleOption(long='log',
 									short='f',
 									nargs=1,
@@ -81,10 +75,8 @@ class Configuration(object):
 		binwalk.module.ModuleKwarg(name='log_file', default=None),
 		binwalk.module.ModuleKwarg(name='csv', default=False),
 		binwalk.module.ModuleKwarg(name='format_to_terminal', default=False),
-		binwalk.module.ModuleKwarg(name='grep', default=[]),
 		binwalk.module.ModuleKwarg(name='quiet', default=False),
 		binwalk.module.ModuleKwarg(name='verbose', default=[]),
-		binwalk.module.ModuleKwarg(name='debug_verbose', default=False),
 		binwalk.module.ModuleKwarg(name='skip_unopened', default=False),
 		binwalk.module.ModuleKwarg(name='files', default=[]),
 		binwalk.module.ModuleKwarg(name='show_help', default=False),

src/binwalk/modules/hashmatch.py

@@ -20,7 +20,7 @@ class HashResult(object):
 		self.hash = hash
 		self.strings = strings
 
-class HashMatch(object):
+class HashMatch(binwalk.module.Module):
 	'''
 	Class for fuzzy hash matching of files and directories.
 	'''
@@ -74,11 +74,12 @@ class HashMatch(object):
 	# Files smaller than this won't produce meaningful fuzzy results (from ssdeep.h)
 	FUZZY_MIN_FILE_SIZE = 4096
 
-	HEADER = ["SIMILARITY", "FILE NAME"]
 	HEADER_FORMAT = "\n%s" + " " * 11 + "%s\n" 
 	RESULT_FORMAT = "%4d%%" + " " * 16 + "%s\n"
+	HEADER = ["SIMILARITY", "FILE NAME"]
+	RESULT = ["percentage", "description"]
 
-	def __init__(self, **kwargs):
+	def init(self):
 		'''
 		Class constructor.
 
@@ -94,8 +95,6 @@ class HashMatch(object):
 
 		Returns None.
 		'''
-		binwalk.module.process_kwargs(self, kwargs)
-
 		self.total = 0
 		self.last_file1 = HashResult(None)
 		self.last_file2 = HashResult(None)
@@ -112,13 +111,10 @@ class HashMatch(object):
 	def _get_strings(self, fname):
 		return ''.join(list(binwalk.common.strings(fname, minimum=10)))
 
-	def _print(self, match, fname):
+	def _show_result(self, match, fname):
 		if self.abspath:
 			fname = os.path.abspath(fname)
-		self.config.display.result(match, fname)
-
-	def _print_footer(self):
-		self.config.display.footer()
+		self.result(percentage=match, description=fname)
 
 	def _compare_files(self, file1, file2):
 		'''
@@ -270,21 +266,17 @@ class HashMatch(object):
 		
 		Returns a list of tuple results.
 		'''
-		results = []
 		self.total = 0
 
 		for f in haystack:
 			m = self._compare_files(needle, f)
 			if m is not None and self.is_match(m):
-				self._print(m, f)
-				results.append((m, f))
+				self._show_result(m, f)
 					
 				self.total += 1
 				if self.max_results and self.total >= self.max_results:
 					break
 
-		return results
-
 	def hash_file(self, needle, haystack):
 		'''
 		Search for one file inside one or more directories.
@@ -300,7 +292,7 @@ class HashMatch(object):
 				f = os.path.join(directory, f)
 				m = self._compare_files(needle, f)
 				if m is not None and self.is_match(m):
-					self._print(m, f)
+					self._show_result(m, f)
 					matching_files.append((m, f))
 					
 					self.total += 1
@@ -319,7 +311,6 @@ class HashMatch(object):
 		Returns a list of tuple results.
 		'''
 		done = False
-		results = []
 		self.total = 0
 
 		source_files = self._get_file_list(needle)
@@ -334,8 +325,7 @@ class HashMatch(object):
 
 					m = self._compare_files(file1, file2)
 					if m is not None and self.is_match(m):
-						self._print(m, file2)
-						results.append((m, file2))
+						self._show_result(m, file2)
 
 						self.total += 1
 						if self.max_results and self.total >= self.max_results:
@@ -344,31 +334,22 @@ class HashMatch(object):
 			if done:
 				break
 
-		return results
-
 	def run(self):
 		'''
 		Main module method.
 		'''
-		results = None
 		needle = self.config.target_files[0].name
 		haystack = []
 		
 		for fp in self.config.target_files[1:]:
 			haystack.append(fp.name)
 
-		self.config.display.format_strings(self.HEADER_FORMAT, self.RESULT_FORMAT)
-		self.config.display.header(*self.HEADER)
-		
 		if os.path.isfile(needle):
 			if os.path.isfile(haystack[0]):
-				results = self.hash_files(needle, haystack)
+				self.hash_files(needle, haystack)
 			else:
-				results = self.hash_file(needle, haystack)
+				self.hash_file(needle, haystack)
 		else:
-			results = self.hash_directories(needle, haystack)
-
-		self.config.display.footer()
-
-		return results
+			self.hash_directories(needle, haystack)
 
+		return True

src/binwalk/modules/hexdiff.py

@@ -6,7 +6,8 @@ import binwalk.module
 import binwalk.common as common
 from binwalk.compat import *
 
-class HexDiff(object):
+# TODO: This code is an effing mess.
+class HexDiff(binwalk.module.Module):
 
 	ALL_SAME = 0
 	ALL_DIFF = 1
@@ -52,12 +53,10 @@ class HexDiff(object):
 			binwalk.module.ModuleKwarg(name='terse', default=False),
 	]
 
-	def __init__(self, **kwargs):
-		binwalk.module.process_kwargs(self, kwargs)
-
-		self.block_hex = ""
-		self.printed_alt_text = False
-
+	HEADER_FORMAT = "\n%s\n"
+	RESULT_FORMAT = "%s\n"
+	RESULT = ['description']
+	
 	def _no_colorize(self, c, color="red", bold=True):
 		return c
 
@@ -84,19 +83,16 @@ class HexDiff(object):
 		return False
 
 	def _print_block_hex(self, alt_text="*"):
-		printed = False
-
 		if self._color_filter(self.block_hex):
-			self.config.display.result(self.block_hex)
+			desc = self.block_hex
 			self.printed_alt_text = False
-			printed = True
 		elif not self.printed_alt_text:
-			self.config.display.result("%s" % alt_text)
+			desc = "%s" % alt_text
 			self.printed_alt_text = True
-			printed = True
 
+		self.result(description=desc)
 		self.block_hex = ""
-		return printed
+		return True
 
 	def _build_block(self, c, highlight=None):
 		if highlight == self.ALL_DIFF:
@@ -117,12 +113,33 @@ class HexDiff(object):
 
 		return header
 
+	def init(self):
+		block = self.config.block
+		if not block:
+			block = self.DEFAULT_BLOCK_SIZE
+
+		if self.terse:
+			header_files = self.config.target_files[:1]
+		else:
+			header_files = self.config.target_files
+
+		self.HEADER = self._build_header(header_files, block)
+
+		if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() and platform.system() != 'Windows':
+			curses.setupterm()
+			self.colorize = self._colorize
+		else:
+			self.colorize = self._no_colorize
+
 	def run(self):
 		i = 0
 		total = 0
 		data = {}
 		delim = '/'
 
+		self.block_hex = ""
+		self.printed_alt_text = False
+		
 		offset = self.config.offset
 		size = self.config.length
 		block = self.config.block
@@ -130,25 +147,10 @@ class HexDiff(object):
 		if not block:
 			block = self.DEFAULT_BLOCK_SIZE
 
-		self.config.display.format_strings("\n%s\n", "%s\n")
-
-		if hasattr(sys.stderr, 'isatty') and sys.stderr.isatty() and platform.system() != 'Windows':
-			curses.setupterm()
-			self.colorize = self._colorize
-		else:
-			self.colorize = self._no_colorize
-
 		# If negative offset, then we're going that far back from the end of the file
 		if offset < 0:
 			size = offset * -1
 
-		if self.terse:
-			header = self._build_header(self.config.target_files[:1], block)
-		else:
-			header = self._build_header(self.config.target_files, block)
-
-		self.config.display.header(header)
-
 		if common.BlockFile.READ_BLOCK_SIZE < block:
 			read_block_size = block
 		else:
@@ -242,7 +244,5 @@ class HexDiff(object):
 				i += block
 			total += read_block_size
 		
-		self.config.display.footer()
-
 		return True