Replaced binwalk script with one using the new API.

src/binwalk/filter.py

@@ -134,33 +134,37 @@ class MagicFilter:
 
 		return self.FILTER_INCLUDE
 
-	def invalid(self, data):
+	def valid_magic_result(self, data):
 		'''
 		Checks if the given string contains invalid data.
 		Called internally by Binwalk.scan().
 
 		@data - String to validate.
 
-		Returns True if data is invalid, False if valid.
+		Returns True if data is valid, False if invalid.
 		'''
 		# A result of 'data' is never ever valid.
 		if data == self.DATA_RESULT:
-			return True
+			return False
 
-		# If showing invalid results, just return False.
-		if self.show_invalid_results:
+		# Make sure this result wasn't filtered
+		if self.filter(data) == self.FILTER_EXCLUDE:
 			return False
 
+		# If showing invalid results, just return True without further checking.
+		if self.show_invalid_results:
+			return True
+
 		# Don't include quoted strings or keyword arguments in this search, as 
 		# strings from the target file may legitimately contain the INVALID_RESULT text.
 		if self.INVALID_RESULT in common.strip_quoted_strings(self.smart._strip_tags(data)):
-			return True
+			return False
 
 		# There should be no non-printable characters in any of the data
 		if self.NON_PRINTABLE_RESULT in data:
-			return True
+			return False
 
-		return False
+		return True
 
 	def grep(self, data=None, filters=[]):
 		'''

src/binwalk/module.py

@@ -104,9 +104,6 @@ class Module(object):
 	'''
 	All module classes must be subclassed from this.
 	'''
-	# The module name, automatically populated.
-	NAME = ""
-
 	# The module title, as displayed in help output
 	TITLE = ""
 
@@ -140,7 +137,8 @@ class Module(object):
 		# self.plugins = x
 		self.errors = []
 		self.results = []
-		self.NAME = self.__class__.__name__
+		self.status = None
+		self.name = self.__class__.__name__
 
 		process_kwargs(self, kwargs)
 
@@ -218,21 +216,29 @@ class Module(object):
 		
 		return args
 
-	def result(self, **kwargs):
+	def result(self, r=None, **kwargs):
 		'''
 		Validates a result, stores it in self.results and prints it.
-
 		Accepts the same kwargs as the binwalk.module.Result class.
 
+		@r - An existing instance of binwalk.module.Result.
+
 		Returns None.
 		'''
-		r = Result(**kwargs)
+		if r is None:
+			r = Result(**kwargs)
 
 		self.validate(r)
 		self._plugins_callback(r)
 
 		if r.valid:
 			self.results.append(r)
+
+			# Update the progress status automatically if it is not being done manually by the module
+			if r.file and not self.status.total:
+				self.status.total = r.file.length
+				self.status.completed = r.file.tell() - r.file.offset
+
 			if r.display:
 				display_args = self._build_display_args(r)
 				if display_args:
@@ -266,12 +272,14 @@ class Module(object):
 	def footer(self):
 		self.config.display.footer()
 			
-	def main(self):
+	def main(self, status):
 		'''
 		Responsible for calling self.init, initializing self.config.display, and calling self.run.
 
 		Returns the value returned from self.run.
 		'''
+		self.status = status
+
 		try:
 			self.init()
 		except KeyboardInterrupt as e:
@@ -302,6 +310,16 @@ class Module(object):
 
 		return retval
 
+class Status(object):
+	
+	def __init__(self, **kwargs):
+		self.kwargs = kwargs
+		self.clear()
+
+	def clear(self):
+		for (k,v) in iterator(self.kwargs):
+			setattr(self, k, v)
+
 class Modules(object):
 	'''
 	Main class used for running and managing modules.
@@ -316,21 +334,24 @@ class Modules(object):
 
 		Returns None.
 		'''
-		argv = list(argv)
+		self.arguments = []
+		self.loaded_modules = {}
+		self.status = Status(completed=0, total=0)
+
+		self._set_arguments(list(argv), kargv)
 
+	def _set_arguments(self, argv=[], kargv={}):
 		for (k,v) in iterator(kargv):
 			k = self._parse_api_opt(k)
-
 			if v not in [True, False, None]:
 				argv.append("%s %s" % (k, v))
 			else:
 				argv.append(k)
 
-		if not argv:
-			argv = sys.argv[1:]
-
-		self.arguments = argv
-		self.loaded_modules = {}
+		if not argv and not self.arguments:
+			self.arguments = sys.argv[1:]
+		elif argv:
+			self.arguments = argv
 
 	def _parse_api_opt(self, opt):
 		# If the argument already starts with a hyphen, don't add hyphens in front of it
@@ -384,18 +405,22 @@ class Modules(object):
 
 		return help_string + "\n"
 
-	def execute(self):
+	def execute(self, *args, **kwargs):
 		run_modules = []
+		self._set_arguments(list(args), kwargs)
+
 		for module in self.list():
-			if self.run(module):
-				run_modules.append(module)
+			obj = self.run(module)
+			if obj.enabled:
+				run_modules.append(obj)
 		return run_modules
 
 	def run(self, module):
 		obj = self.load(module)
 
 		if isinstance(obj, binwalk.module.Module) and obj.enabled:
-			obj.main()
+			obj.main(status=self.status)
+			self.status.clear()
 
 		# Add object to loaded_modules here, that way if a module has already been
 		# loaded directly and is subsequently also listed as a dependency we don't waste

src/binwalk/modules/configuration.py

@@ -2,6 +2,7 @@ import os
 import sys
 import binwalk.common
 import binwalk.module
+import binwalk.config
 import binwalk.display
 from binwalk.config import *
 from binwalk.compat import *
@@ -87,6 +88,7 @@ class Configuration(binwalk.module.Module):
 	def load(self):
 		self.target_files = []
 
+		self.settings = binwalk.config.Config()
 		self.display = binwalk.display.Display(log=self.log_file,
 											   csv=self.csv,
 											   quiet=self.quiet,

src/binwalk/modules/signature.py

 import magic
-import binwalk.config
 import binwalk.module
 import binwalk.parser
 import binwalk.filter
@@ -37,9 +36,6 @@ class Signature(binwalk.module.Module):
 	MAGIC_FLAGS = magic.MAGIC_NO_CHECK_TEXT | magic.MAGIC_NO_CHECK_ENCODING | magic.MAGIC_NO_CHECK_APPTYPE | magic.MAGIC_NO_CHECK_TOKENS
 
 	def init(self):
-		# Instantiate the config class so we can access file/directory paths
-		self.conf = binwalk.config.Config()
-
 		# Create SmartSignature and MagicParser class instances. These are mostly for internal use.
 		self.filter = binwalk.filter.MagicFilter()
 		self.smart = binwalk.smartsignature.SmartSignature(self.filter, ignore_smart_signatures=False)
@@ -49,8 +45,8 @@ class Signature(binwalk.module.Module):
 		if not self.magic_files:
 			# Append the user's magic file first so that those signatures take precedence
 			self.magic_files = [
-				self.conf.paths['user'][self.conf.BINWALK_MAGIC_FILE],
-				self.conf.paths['system'][self.conf.BINWALK_MAGIC_FILE],
+				self.config.settings.paths['user'][self.config.settings.BINWALK_MAGIC_FILE],
+				self.config.settings.paths['system'][self.config.settings.BINWALK_MAGIC_FILE],
 			]
 
 		# Parse the magic file(s) and initialize libmagic
@@ -61,15 +57,28 @@ class Signature(binwalk.module.Module):
 		# Once the temporary magic file is loaded into libmagic, we don't need it anymore; delete the temp file
 		self.parser.rm_magic_file()
 
+	def validate(self, r):
+		'''
+		Called automatically by self.result.
+		'''
+		if not r.description:
+			r.valid = False
+
+		if r.size and (r.size + r.offset) > r.file.size:
+			r.valid = False
+
+		if r.jump and (r.jump + r.offset) > r.file.size:
+			r.valid = False
+
 	def scan_file(self, fp):
 		while True:
-			current_block_offset = 0
-
 			(data, dlen) = fp.read_block()
 			if not data:
 				break
 
+			current_block_offset = 0
 			block_start = fp.tell() - dlen
+			self.status.completed = block_start - fp.offset
 
 			for candidate_offset in self.parser.find_signature_candidates(data, dlen):
 				if candidate_offset < current_block_offset:
@@ -81,19 +90,27 @@ class Signature(binwalk.module.Module):
 				# Pass the data to libmagic, and split out multiple results into a list
 				magic_result = self.magic.buffer(candidate_data)
 
-				# TODO: Should filter process other validations? Reported size, for example?
-				if not self.filter.invalid(magic_result):
+				if self.filter.valid_magic_result(magic_result):
 					# The smart filter parser returns a dictionary of keyword values and the signature description.
-					smart = self.smart.parse(magic_result)
-					self.result(description=smart['description'], offset=block_start+candidate_offset)
-						
-					if smart['jump'] > 0:
-						fp.seek(block_start + candidate_offset + smart['jump'])
-						current_block_offset = smart['jump']
+					r = self.smart.parse(magic_result)
+					r.offset = block_start + candidate_offset + r.adjust
+					r.file = fp
+
+					self.result(r=r)
+					
+					if r.valid and r.jump > 0:
+						fp.seek(r.offset + r.jump)
+						current_block_offset = r.jump
 
 	def run(self):
 		for fp in self.config.target_files:
 			self.header()
+			
+			self.status.clear()
+			self.status.total = fp.size
+			self.status.completed = 0
+
 			self.scan_file(fp)
+			
 			self.footer()
 

src/binwalk/parser.py

@@ -55,7 +55,9 @@ class MagicParser:
 	def __del__(self):
 		try:
 			self.cleanup()
-		except:
+		except KeyboardInterrupt as e:
+			raise e
+		except Exception:
 			pass
 
 	def rm_magic_file(self):
@@ -66,7 +68,9 @@ class MagicParser:
 		'''
 		try:
 			self.fd.close()
-		except:
+		except KeyboardInterrupt as e:
+			raise e
+		except Exception:
 			pass
 
 	def cleanup(self):
@@ -79,7 +83,9 @@ class MagicParser:
 
 		try:
 			self.raw_fd.close()
-		except:
+		except KeyboardInterrupt as e:
+			raise e
+		except Exception:
 			pass
 
 	def file_from_string(self, signature_string, offset=0, display_name=DEFAULT_DISPLAY_NAME):
@@ -170,6 +176,8 @@ class MagicParser:
 					self.fd.write(str2bytes(line))
 
 			self.build_signature_set()
+		except KeyboardInterrupt as e:
+			raise e
 		except Exception as e:
 			raise Exception("Error parsing magic file '%s' on line %d: %s" % (file_name, line_count, str(e)))
 		
@@ -207,6 +215,8 @@ class MagicParser:
 			# The condition line may contain escaped sequences, so be sure to decode it properly.
 			entry['condition'] = string_decode(line_parts[2])
 			entry['description'] = ' '.join(line_parts[3:])
+		except KeyboardInterrupt as e:
+			raise e
 		except Exception as e:
 			raise Exception("%s :: %s", (str(e), line))
 
@@ -214,6 +224,8 @@ class MagicParser:
 		# throw an exception, but let's catch it just in case...
 		try:
 			entry['offset'] = str2int(entry['offset'])
+		except KeyboardInterrupt as e:
+			raise e
 		except Exception as e:
 			raise Exception("%s :: %s", (str(e), line))
 
@@ -234,6 +246,8 @@ class MagicParser:
 			# but needing that is rare.
 			try:
 				intval = str2int(entry['condition'].strip('L'))
+			except KeyboardInterrupt as e:
+				raise e
 			except Exception as e:
 				raise Exception("Failed to evaluate condition for '%s' type: '%s', condition: '%s', error: %s" % (entry['description'], entry['type'], entry['condition'], str(e)))
 
@@ -322,6 +336,8 @@ class MagicParser:
 		'''
 		try:
 			return data.split(self.RESULT_SEPERATOR)
-		except:
+		except KeyboardInterrupt as e:
+			raise e
+		except Exception:
 			return []
 

src/binwalk/plugins.py

 import os
 import sys
 import imp
+import binwalk.config
 from binwalk.compat import *
 
-# Valid return values for plugins
-PLUGIN_CONTINUE     = 0x00
-PLUGIN_NO_EXTRACT   = 0x01
-PLUGIN_NO_DISPLAY   = 0x02
-PLUGIN_STOP_PLUGINS = 0x04
-PLUGIN_TERMINATE    = 0x08
-
 class Plugins:
 	'''
 	Class to load and call plugin callback functions, handled automatically by Binwalk.scan / Binwalk.single_scan.
@@ -66,7 +60,7 @@ class Plugins:
 			# Set to False to have this plugin disabled by default.
 			ENABLED = True
 
-			def __init__(self, binwalk):
+			def __init__(self):
 				self.binwalk = binwalk
 				print 'Scanning initialized!'
 
@@ -86,16 +80,15 @@ class Plugins:
 				return PLUGIN_CONTINUE
 	'''
 
-	CALLBACK = 'callback'
+	RESULT = 'result'
 	PRESCAN = 'pre_scan'
 	POSTSCAN = 'post_scan'
-	PREPARSER = 'pre_parser'
 	PLUGIN = 'Plugin'
 	MODULE_EXTENSION = '.py'
 
-	def __init__(self, binwalk, whitelist=[], blacklist=[]):
-		self.binwalk = binwalk
-		self.callback = []
+	def __init__(self, whitelist=[], blacklist=[]):
+		self.config = binwalk.config.Config()
+		self.result = []
 		self.pre_scan = []
 		self.pre_parser = []
 		self.post_scan = []
@@ -103,16 +96,13 @@ class Plugins:
 		self.blacklist = blacklist
 
 	def __del__(self):
-		self._cleanup()
+		pass
 
-	def __exit__(self, t, v, traceback):
-		self._cleanup()
+	def __enter__(self):
+		return self
 
-	def _cleanup(self):
-		try:
-			del self.binwalk
-		except:
-			pass
+	def __exit__(self, t, v, traceback):
+		pass
 
 	def _call_plugins(self, callback_list, arg):
 		retval = PLUGIN_CONTINUE
@@ -168,7 +158,7 @@ class Plugins:
 		}
 
 		for key in plugins.keys():
-			plugins[key]['path'] = self.binwalk.config.paths[key][self.binwalk.config.PLUGINS]
+			plugins[key]['path'] = self.config.paths[key][self.config.PLUGINS]
 
 			for file_name in os.listdir(plugins[key]['path']):
 				if file_name.endswith(self.MODULE_EXTENSION):
@@ -214,10 +204,10 @@ class Plugins:
 				except:
 					pass
 
-				class_instance = plugin_class(self.binwalk)
+				class_instance = plugin_class()
 
 				try:
-					self.callback.append(getattr(class_instance, self.CALLBACK))
+					self.result.append(getattr(class_instance, self.RESULT))
 				except:
 					pass
 
@@ -227,11 +217,6 @@ class Plugins:
 					pass
 
 				try:
-					self.pre_parser.append(getattr(class_instance, self.PREPARSER))
-				except:
-					pass
-					
-				try:
 					self.post_scan.append(getattr(class_instance, self.POSTSCAN))
 				except:
 					pass
@@ -239,15 +224,12 @@ class Plugins:
 			except Exception as e:
 				sys.stderr.write("WARNING: Failed to load plugin module '%s': %s\n" % (module, str(e)))
 
-	def _pre_scan_callbacks(self, fd):
-		return self._call_plugins(self.pre_scan, fd)
-
-	def _post_scan_callbacks(self, fd):
-		return self._call_plugins(self.post_scan, fd)
+	def _pre_scan_callbacks(self, obj):
+		return self._call_plugins(self.pre_scan, obj)
 
-	def _scan_callbacks(self, results):
-		return self._call_plugins(self.callback, results)
+	def _post_scan_callbacks(self, obj):
+		return self._call_plugins(self.post_scan, obj)
 
-	def _scan_pre_parser_callbacks(self, results):
-		return self._call_plugins(self.pre_parser, results)
+	def _result_callbacks(self, obj):
+		return self._call_plugins(self.result, obj)
 

src/binwalk/smartsignature.py

 import re
+import binwalk.module
 from binwalk.compat import *
 from binwalk.common import str2int, get_quoted_strings, MathExpression
 
@@ -18,20 +19,20 @@ class SmartSignature:
 	KEYWORD_DELIM_START = "{"
 	KEYWORD_DELIM_END = "}"
 	KEYWORDS = {
-		'jump'			: '%sjump-to-offset:' % KEYWORD_DELIM_START,
-		'filename'		: '%sfile-name:' % KEYWORD_DELIM_START,
-		'filesize'		: '%sfile-size:' % KEYWORD_DELIM_START,
-		'raw-string'		: '%sraw-string:' % KEYWORD_DELIM_START,	# This one is special and must come last in a signature block
-		'string-len'		: '%sstring-len:' % KEYWORD_DELIM_START,	# This one is special and must come last in a signature block
-		'raw-size'		: '%sraw-string-length:' % KEYWORD_DELIM_START,
-		'adjust'		: '%soffset-adjust:' % KEYWORD_DELIM_START,
-		'delay'			: '%sextract-delay:' % KEYWORD_DELIM_START,
-		'year'			: '%sfile-year:' % KEYWORD_DELIM_START,
-		'epoch'			: '%sfile-epoch:' % KEYWORD_DELIM_START,
-		'math'		: '%smath:' % KEYWORD_DELIM_START,
-
-		'raw-replace'		: '%sraw-replace%s' % (KEYWORD_DELIM_START, KEYWORD_DELIM_END),
-		'one-of-many'		: '%sone-of-many%s' % (KEYWORD_DELIM_START, KEYWORD_DELIM_END),
+		'jump'					: '%sjump-to-offset:' % KEYWORD_DELIM_START,
+		'filename'				: '%sfile-name:' % KEYWORD_DELIM_START,
+		'filesize'				: '%sfile-size:' % KEYWORD_DELIM_START,
+		'raw-string'			: '%sraw-string:' % KEYWORD_DELIM_START,	# This one is special and must come last in a signature block
+		'string-len'			: '%sstring-len:' % KEYWORD_DELIM_START,	# This one is special and must come last in a signature block
+		'raw-size'				: '%sraw-string-length:' % KEYWORD_DELIM_START,
+		'adjust'				: '%soffset-adjust:' % KEYWORD_DELIM_START,
+		'delay'					: '%sextract-delay:' % KEYWORD_DELIM_START,
+		'year'					: '%sfile-year:' % KEYWORD_DELIM_START,
+		'epoch'					: '%sfile-epoch:' % KEYWORD_DELIM_START,
+		'math'					: '%smath:' % KEYWORD_DELIM_START,
+
+		'raw-replace'			: '%sraw-replace%s' % (KEYWORD_DELIM_START, KEYWORD_DELIM_END),
+		'one-of-many'			: '%sone-of-many%s' % (KEYWORD_DELIM_START, KEYWORD_DELIM_END),
 		'string-len-replace'	: '%sstring-len%s' % (KEYWORD_DELIM_START, KEYWORD_DELIM_END),
 	}
 
@@ -45,7 +46,7 @@ class SmartSignature:
 		Returns None.
 		'''
 		self.filter = filter
-		self.invalid = False
+		self.valid = True
 		self.last_one_of_many = None
 		self.ignore_smart_signatures = ignore_smart_signatures
 
@@ -58,28 +59,30 @@ class SmartSignature:
 		Returns a dictionary of parsed values.
 		'''
 		results = {
-			'offset'	: '',		# Offset where the match was found, filled in by Binwalk.single_scan.
+			'offset'		: '',		# Offset where the match was found, filled in by Binwalk.single_scan.
 			'description'	: '',		# The libmagic data string, stripped of all keywords
-			'name'		: '',		# The original name of the file, if known
-			'delay'		: '',		# Extract delay description
-			'extract'	: '',		# Name of the extracted file, filled in by Binwalk.single_scan.
-			'jump'		: 0,		# The relative offset to resume the scan from
-			'size'		: 0,		# The size of the file, if known
-			'adjust'	: 0,		# The relative offset to add to the reported offset
-			'year'		: 0,		# The file's creation/modification year, if reported in the signature
-			'epoch'		: 0,		# The file's creation/modification epoch time, if reported in the signature
-			'invalid'	: False,	# Set to True if parsed numerical values appear invalid
+			'name'			: '',		# The original name of the file, if known
+			'delay'			: '',		# Extract delay description
+			'extract'		: '',		# Name of the extracted file, filled in by Binwalk.single_scan.
+			'jump'			: 0,		# The relative offset to resume the scan from
+			'size'			: 0,		# The size of the file, if known
+			'adjust'		: 0,		# The relative offset to add to the reported offset
+			'year'			: 0,		# The file's creation/modification year, if reported in the signature
+			'epoch'			: 0,		# The file's creation/modification epoch time, if reported in the signature
+			'valid'			: True,		# Set to False if parsed numerical values appear invalid
 		}
 
-		self.invalid = False
+		self.valid = True
 
 		# If smart signatures are disabled, or the result data is not valid (i.e., potentially malicious), 
 		# don't parse anything, just return the raw data as the description.
 		if self.ignore_smart_signatures or not self._is_valid(data):
 			results['description'] = data
 		else:
-			# Calculate and replace math keyword values
+			# Calculate and replace special keywords/values
 			data = self._replace_maths(data)
+			data = self._parse_raw_strings(data)
+			data = self._parse_string_len(data)
 
 			# Parse the offset-adjust value. This is used to adjust the reported offset at which 
 			# a signature was located due to the fact that MagicParser.match expects all signatures
@@ -91,17 +94,23 @@ class SmartSignature:
 			# extracted (see Binwalk.scan).
 			try:
 				results['size'] = str2int(self._get_math_arg(data, 'filesize'))
-			except:
+			except KeyboardInterrupt as e:
+				raise e
+			except Exception:
 				pass
 
 			try:
 				results['year'] = str2int(self._get_keyword_arg(data, 'year'))
-			except:
+			except KeyboardInterrupt as e:
+				raise e
+			except Exception:
 				pass
 			
 			try:
 				results['epoch'] = str2int(self._get_keyword_arg(data, 'epoch'))
-			except:
+			except KeyboardInterrupt as e:
+				raise e
+			except Exception:
 				pass
 
 			results['delay'] = self._get_keyword_arg(data, 'delay')
@@ -116,9 +125,9 @@ class SmartSignature:
 				results['name'] = self._get_keyword_arg(data, 'filename').strip('"')
 				results['description'] = self._strip_tags(data)
 
-		results['invalid'] = self.invalid
+		results['valid'] = self.valid
 
-		return results
+		return binwalk.module.Result(**results)
 
 	def _is_valid(self, data):
 		'''
@@ -152,7 +161,7 @@ class SmartSignature:
 		Returns True if the string result is one of many.
 		Returns False if the string result is not one of many.
 		'''
-		if not self.filter.invalid(data):
+		if self.filter.valid_magic_result(data):
 			if self.last_one_of_many is not None and data.startswith(self.last_one_of_many):
 				return True
 		
@@ -197,7 +206,7 @@ class SmartSignature:
 			value = MathExpression(arg).value
 			if value is None:
 				value = 0
-				self.invalid = True
+				self.valid = False
 
 		return value
 
@@ -215,7 +224,9 @@ class SmartSignature:
 		if offset_str:
 			try:
 				offset = str2int(offset_str)
-			except:
+			except KeyboardInterrupt as e:
+				raise e
+			except Exception:
 				pass
 
 		return offset
@@ -281,7 +292,9 @@ class SmartSignature:
 				# Convert the string to an integer as a sanity check
 				try:
 					string_length = '%d' % len(raw_string)
-				except:
+				except KeyboardInterrupt as e:
+					raise e
+				except Exception:
 					string_length = '0'
 
 				# Strip out *everything* after the string-len keyword, including the keyword itself.