Merge branch 'master' into master

.travis.yml

 language: python
 python:
  - "2.7"
- - "3.3"
  - "3.4"
  - "3.5"
  - "3.6"
+ - "3.7"
+ - "3.8"
  - "nightly"
  - "pypy"
  - "pypy3"

API.md

@@ -86,12 +86,12 @@ Each module object will also have an additional `extractor` attribute, which is
 ```python
 for module in binwalk.scan(sys.argv[1], signature=True, quiet=True, extract=True):
     for result in module.results:
-        if module.extractor.output.has_key(result.file.path):
+        if result.file.path in module.extractor.output:
             # These are files that binwalk carved out of the original firmware image, a la dd
-            if module.extractor.output[result.file.path].carved.has_key(result.offset):
+            if result.offset in module.extractor.output[result.file.path].carved:
                 print "Carved data from offset 0x%X to %s" % (result.offset, module.extractor.output[result.file.path].carved[result.offset])
             # These are files/directories created by extraction utilities (gunzip, tar, unsquashfs, etc)
-            if module.extractor.output[result.file.path].extracted.has_key(result.offset):
+            if result.offset in module.extractor.output[result.file.path].extracted:
                 print "Extracted %d files from offset 0x%X to '%s' using '%s'" % (len(module.extractor.output[result.file.path].extracted[result.offset].files),
                                                                                   result.offset,
                                                                                   module.extractor.output[result.file.path].extracted[result.offset].files[0],

deps.sh

@@ -13,9 +13,14 @@ set -o nounset
 if ! which lsb_release > /dev/null
 then
     function lsb_release {
-        if [ -f /etc/lsb-release ]
+        if [ -f /etc/os-release ]
         then
-            cat /etc/lsb-release | grep DISTRIB_ID | cut -d= -f 2
+            [[ "$1" = "-i" ]] && cat /etc/os-release | grep ^"ID" | cut -d= -f 2
+            [[ "$1" = "-r" ]] && cat /etc/os-release | grep "VERSION_ID" | cut -d= -d'"' -f 2
+        elif [ -f /etc/lsb-release ]
+        then
+            [[ "$1" = "-i" ]] && cat /etc/lsb-release | grep "DISTRIB_ID" | cut -d= -f 2
+            [[ "$1" = "-r" ]] && cat /etc/lsb-release | grep "DISTRIB_RELEASE" | cut -d= -f 2
         else
             echo Unknown
         fi
@@ -103,16 +108,14 @@ function install_cramfstools
   INSTALL_LOCATION=/usr/local/bin
 
   # https://github.com/torvalds/linux/blob/master/fs/cramfs/README#L106
-  wget  https://downloads.sourceforge.net/project/cramfs/cramfs/1.1/cramfs-1.1.tar.gz?ts=$TIME -O cramfs-1.1.tar.gz
-  tar xf cramfs-1.1.tar.gz
+  git clone https://github.com/npitre/cramfs-tools
   # There is no "make install"
-  (cd cramfs-1.1 \
+  (cd cramfs-tools \
   && make \
   && $SUDO install mkcramfs $INSTALL_LOCATION \
   && $SUDO install cramfsck $INSTALL_LOCATION)
 
-  rm cramfs-1.1.tar.gz
-  rm -rf cramfs-1.1
+  rm -rf cramfs-tools
 }
 
 

setup.py

@@ -329,7 +329,7 @@ for data_dir in ["magic", "config", "plugins", "modules", "core"]:
 # If doing a build or installation, then create a version.py file
 # which defines the current binwalk version. This file is excluded
 # from git in the .gitignore file.
-if 'install' in ' '.join(sys.argv) or 'build' in ' '.join(sys.argv):
+if 'install' in ' '.join(sys.argv) or 'build' in ' '.join(sys.argv) or 'sdist' in ' '.join(sys.argv):
     sys.stdout.write("creating %s\n" % (VERSION_FILE))
 
     try:

src/binwalk/config/extract.conf

@@ -69,10 +69,10 @@
 ^linux ext:ext:tsk_recover -i raw -f ext -a -v '%e' '%%ext-root%%':0:False
 
 # Try mounting the file system (this requires root privileges)
-^squashfs filesystem:squashfs:mkdir squashfs-root && mount -t squashfs '%e' squashfs-root:0:False
-^cramfs filesystem:cramfs:mkdir cramfs-root && mount -t cramfs '%e' cramfs-root:0:False
-^linux ext filesystem:ext2:mkdir ext-root && mount '%e' ext-root:0:False
-^romfs filesystem:romfs:mkdir romfs-root && mount -t romfs '%e' romfs-root:0:False
+^squashfs filesystem:squashfs:mkdir '%%squashfs-root%%' && mount -t squashfs '%e' '%%squashfs-root%%':0:False
+^cramfs filesystem:cramfs:mkdir '%%cramfs-root%%' && mount -t cramfs '%e' '%%cramfs-root%%':0:False
+^linux ext filesystem:ext2:mkdir '%%ext-root%%' && mount '%e' '%%ext-root%%':0:False
+^romfs filesystem:romfs:mkdir '%%romfs-root%%' && mount -t romfs '%e' '%%romfs-root%%':0:False
 
 # Use sviehb's jefferson.py tool for JFFS2 extraction
 ^jffs2 filesystem:jffs2:jefferson -d '%%jffs2-root%%' '%e':0:False

src/binwalk/core/module.py

@@ -415,7 +415,7 @@ class Module(object):
 
             # Values in self.target_file_list are either already open files (BlockFile instances), or paths
             # to files that need to be opened for scanning.
-            if isinstance(next_target_file, str):
+            if isinstance(next_target_file, str) or isinstance(next_target_file, unicode):
                 fp = self.config.open_file(next_target_file)
             else:
                 fp = next_target_file

src/binwalk/magic/binarch

@@ -133,3 +133,9 @@
 0   string  \xF6\x69\x0B\x00\xF6\x68            SuperH instructions, little endian, function epilogue (gcc)
 0   string  \x69\xF6\x00\x0B\x68\xF6            SuperH instructions, big endian, function epilogue (gcc)
 
+# AArch64
+# ret
+0 ulelong 0xd65f03c0 AArch64 instructions, function epilogue
+
+# nop
+0 ulelong 0xd503201f AArch64 instructions, nop

src/binwalk/magic/crypto

@@ -36,6 +36,14 @@
 >2      beshort         x               header length: 4, sequence length: %d
 >2      beshort+4       x               {size:%d}
 
+# Type: PKCS#7 signature DER format
+0       string          \x30\x82        Object signature in DER format (PKCS#7),
+# OBJECT IDENTIFIER 1.2.840.113549.1.7.2 signedData (PKCS #7)
+>4      string          !\x06\x09\x2a\x86\x48\x86\xF7\x0D\x01\x07\x02  {invalid}
+>2      beshort         <0              {invalid}
+>2      beshort         x               header length: 4, sequence length: %d
+>2      beshort+4       x               {size:%d}
+
 # GnuPG
 # The format is very similar to pgp
 0       string          \001gpg                 GPG key trust database
@@ -129,6 +137,7 @@
 >72     string          sha256                  %s
 >72     string          sha512                  %s
 >72     string          ripemd160               %s
+
 #-------------------------------------------------------------------------------
 # Nagra Stuff
 0      string          \x00\x00\x01\6c         Nagra PK
@@ -148,3 +157,116 @@
         
 0      string    \x52\x09\x6a\xd5   AES Inverse S-Box 
 >4     string   !\x30\x36\xA5\x38   {invalid}
+
+# magic signatures to detect PGP crypto material (from stef)
+# detects and extracts metadata from:
+#  - symmetric encrypted packet header
+#  - RSA (e=65537) secret (sub-)keys
+
+# 1024b RSA encrypted data
+
+0       string  \x84\x8c\x03        PGP RSA encrypted session key -
+>3      ubelong x                   keyid: %X
+>7      ubelong x                   %X
+>11     byte    <0x01               {invalid}
+>11     byte    >0x02               {invalid}
+>11     byte    0x01                RSA (Encrypt or Sign) 1024b
+>11     byte    0x02                RSA Encrypt-Only 1024b
+>12     string  \x04\x00
+>12     string  \x03\xff
+>12     string  \x03\xfe
+>12     string  \x03\xfd
+>12     string  \x03\xfc
+>12     string  \x03\xfb
+>12     string  \x03\xfa
+>12     string  \x03\xf9
+>142    byte    0xd2
+
+# 2048b RSA encrypted data
+
+0       string  \x85\x01\x0c\x03    PGP RSA encrypted session key -
+>4      ubelong x                   keyid: %X
+>8      ubelong x                   %X
+>12     byte    <0x01               {invalid}
+>12     byte    >0x02               {invalid}
+>12     byte    0x01                RSA (Encrypt or Sign) 2048b
+>12     byte    0x02                RSA Encrypt-Only 2048b
+>13     string  \x08\x00
+>13     string  \x07\xff
+>13     string  \x07\xfe
+>13     string  \x07\xfd
+>13     string  \x07\xfc
+>13     string  \x07\xfb
+>13     string  \x07\xfa
+>13     string  \x07\xf9
+>271    byte    0xd2
+
+# 3072b RSA encrypted data
+
+0       string  \x85\x01\x8c\x03    PGP RSA encrypted session key -
+>4      ubelong x                   keyid: %X
+>8      ubelong x                   %X
+>12     byte    <0x01               {invalid}
+>12     byte    >0x02               {invalid}
+>12     byte    0x01                RSA (Encrypt or Sign) 3072b
+>12     byte    0x02                RSA Encrypt-Only 3072b
+>13     string  \x0c\x00
+>13     string  \x0b\xff
+>13     string  \x0b\xfe
+>13     string  \x0b\xfd
+>13     string  \x0b\xfc
+>13     string  \x0b\xfb
+>13     string  \x0b\xfa
+>13     string  \x0b\xf9
+>399    byte    0xd2
+
+# 3072b RSA encrypted data
+
+0       string  \x85\x02\x0c\x03    PGP RSA encrypted session key -
+>4      ubelong x                   keyid: %X
+>8      ubelong x                   %X
+>12     byte    <0x01               {invalid}
+>12     byte    >0x02               {invalid}
+>12     byte    0x01                RSA (Encrypt or Sign) 4096b
+>12     byte    0x02                RSA Encrypt-Only 4096b
+>13     string  \x10\x00
+>13     string  \x0f\xff
+>13     string  \x0f\xfe
+>13     string  \x0f\xfd
+>13     string  \x0f\xfc
+>13     string  \x0f\xfb
+>13     string  \x0f\xfa
+>13     string  \x0f\xf9
+>527    byte    0xd2
+
+# 4096b RSA encrypted data
+
+0       string  \x85\x04\x0c\x03    PGP RSA encrypted session key -
+>4      ubelong x                   keyid: %X
+>8      ubelong x                   %X
+>12     byte    <0x01               {invalid}
+>12     byte    >0x02               {invalid}
+>12     byte    0x01                RSA (Encrypt or Sign) 8129b
+>12     byte    0x02                RSA Encrypt-Only 8129b
+>13     string  \x20\x00
+>13     string  \x1f\xff
+>13     string  \x1f\xfe
+>13     string  \x1f\xfd
+>13     string  \x1f\xfc
+>13     string  \x1f\xfb
+>13     string  \x1f\xfa
+>13     string  \x1f\xf9
+>1039   byte    0xd2
+
+
+# From bsdphk: These four signatures can be used to spot DES encryption implementations
+0   string \x38\x30\x28\x20\x18\x10\x08\x00\x39\x31\x29\x21\x19\x11\x09\x01\x3a\x32\x2a\x22\x1a\x12\x0a\x02\x3b\x33\x2b\x23\x3e\x36\x2e\x26\x1e\x16\x0e\x06\x3d\x35\x2d\x25\x1d\x15\x0d\x05\x3c\x34\x2c\x24\x1c\x14\x0c\x04\x1b\x13\x0b\x03 DES PC1 table
+
+0   string \x0d\x10\x0a\x17\x00\x04\x02\x1b\x0e\x05\x14\x09\x16\x12\x0b\x03\x19\x07\x0f\x06\x1a\x13\x0c\x01\x28\x33\x1e\x24\x2e\x36\x1d\x27\x32\x2c\x20\x2f\x2b\x30\x26\x37\x21\x34\x2d\x29\x31\x23\x1c\x1f DES PC2 table
+
+0   string \x01\x01\x04\x00\x00\x00\x00\x00\x00\x01\x00\x00\x01\x01\x04\x04\x01\x01\x00\x04\x00\x01\x04\x04\x00\x00\x00\x04\x00\x01\x00\x00 DES SP1, big endian
+0   string \x00\x04\x01\x01\x00\x00\x00\x00\x00\x00\x01\x00\x04\x04\x01\x01\x04\x00\x01\x01\x04\x04\x01\x00\x04\x00\x00\x00\x00\x00\x01\x00 DES SP1, little endian{overlap}
+
+0   string \x80\x10\x80\x20\x80\x00\x80\x00\x00\x00\x80\x00\x00\x10\x80\x20\x00\x10\x00\x00\x00\x00\x00\x20\x80\x10\x00\x20\x80\x00\x80\x20 DES SP2, big endian
+0   string \x20\x80\x10\x80\x00\x80\x00\x80\x00\x80\x00\x00\x20\x80\x10\x00\x00\x00\x10\x00\x20\x00\x00\x00\x20\x00\x10\x80\x20\x80\x00\x80 DES SP2, little endian
+

src/binwalk/magic/filesystems

@@ -216,8 +216,10 @@
 >>20    beshort 2       \blzma,
 >>20    beshort 3       \bgzip (non-standard type definition),
 >>20    beshort 4       \bxz,
+>>20    beshort 5       \blz4,
+>>20    beshort 6       \bzstd,
 >>20    beshort 0       \b{invalid},
->>20    beshort >4      \b{invalid},
+>>20    beshort >6      \b{invalid},
 >28     beshort <3
 >>8     belong  x       size: %d bytes,
 >>8     belong  x       \b{jump:%d}
@@ -256,8 +258,10 @@
 >>20    leshort 2       \blzma,
 >>20    leshort 3       \bgzip (non-standard type definition),
 >>20    leshort 4       \bxz,
+>>20    leshort 5       \blz4,
+>>20    leshort 6       \bzstd,
 >>20    leshort 0       \b{invalid},
->>20    leshort >4      \b{invalid},
+>>20    leshort >6      \b{invalid},
 >28     leshort <3
 >>8     lelong  x       size: %d bytes,
 >>8     lelong  x       {size:%d}
@@ -299,8 +303,10 @@
 >>20    beshort 2       \blzma,
 >>20    beshort 3       \bgzip (non-standard type definition),
 >>20    beshort 4       \blzma (non-standard type definition),
+>>20    beshort 5       \blz4,
+>>20    beshort 6       \bzstd,
 >>20    beshort 0       \b{invalid},
->>20    beshort >4      \b{invalid},
+>>20    beshort >6      \b{invalid},
 >28     beshort <3
 >>8     belong  x       size: %d bytes,
 >>8     belong  x       {size:%d}
@@ -342,8 +348,10 @@
 >>20    beshort 2       \blzma,
 >>20    beshort 3       \bgzip (non-standard type definition),
 >>20    beshort 4       \bxz,
+>>20    beshort 5       \blz4,
+>>20    beshort 6       \bzstd,
 >>20    beshort 0       \b{invalid},
->>20    beshort >4      \b{invalid},
+>>20    beshort >6      \b{invalid},
 >28     beshort <3
 >>8     belong  x       size: %d bytes,
 >>8     belong  x       {size:%d}
@@ -386,8 +394,10 @@
 >>20    beshort 2       \blzma,
 >>20    beshort 3       \bgzip (non-standard type definition),
 >>20    beshort 4       \bxz,
+>>20    beshort 5       \blz4,
+>>20    beshort 6       \bzstd,
 >>20    beshort 0       \b{invalid},
->>20    beshort >4      \b{invalid},
+>>20    beshort >6      \b{invalid},
 >28     beshort <3
 >>8     belong  x       size: %d bytes,
 >>8     belong  <1      {invalid}
@@ -432,8 +442,10 @@
 >>20    leshort 2       \blzma,
 >>20    leshort 3       \bgzip (non-standard type definition),
 >>20    leshort 4       \bxz,
+>>20    leshort 5       \blz4,
+>>20    leshort 6       \bzstd,
 >>20    leshort 0       \b{invalid},
->>20    leshort >4      \b{invalid},
+>>20    leshort >6      \b{invalid},
 >28     leshort <3
 >>8     lelong  x       size: %d bytes,
 >>8     lelong  x       {size:%d}
@@ -475,8 +487,10 @@
 >>20    leshort 2       \blzma,
 >>20    leshort 3       \bgzip (non-standard type definition),
 >>20    leshort 4       \bxz,
+>>20    leshort 5       \blz4,
+>>20    leshort 6       \bzstd,
 >>20    leshort 0       \b{invalid},
->>20    leshort >4      \b{invalid},
+>>20    leshort >6      \b{invalid},
 >28     leshort <3
 >>8     lelong  x       size: %d bytes,
 >>8     lelong  x       {size:%d}

src/binwalk/magic/firmware

@@ -467,6 +467,20 @@
 -4   string      \x00\x00\x00\x00ENDS   Signed Ubiquiti end header, RSA 2048 bit, header size: 264 bytes
 >260 ubelong     !0                     {invalid}
 
+# Ubiquiti additional data
+-4           string   \x00\x00\x00\x00EXEC  Ubiquiti firmware additional data,
+# Non-empty string (16 bytes max)
+>4           byte     0                     {invalid}
+>4           string   x                         name: %s,
+# Size of attached data
+>48          ubelong  x                         size: %d bytes,
+# Size, again
+>52          ubelong  x                         size2: %d bytes,
+# CRC32 of header + data
+>>(48.L+56)  ubelong  x                         CRC32: %x
+# Padding
+>>(48.L+60)  ubelong  !0                    {invalid}
+
 # Found in DIR-100 firmware
 0       string      AIH0        AIH0 firmware header, header size: 48,
 >12     ubelong     0           {invalid}
@@ -758,10 +772,6 @@
 >20   ulelong    x             \b, ramdisk addr: 0x%X
 >48   string     x             \b, product name: "%s"
 
-# DTB
-# http://elinux.org/images/c/cf/Power_ePAPR_APPROVED_v1.1.pdf
-0     string     \xd0\x0d\xfe\xed    device tree image (dtb)
-
 # QCDT
 # https://source.codeaurora.org/quic/la/device/qcom/common/tree/dtbtool?h=LA.BF64.1.2.2_rb4.42
 0     string     QCDT          Qualcomm device tree container
@@ -896,3 +906,7 @@
 >32         ubelong   0           {invalid}
 # Size of structure block, must be greater than 0
 >36         ubelong   0           {invalid}
+
+# Toshiba SSD Firmware Update
+# The version string seems to be at 0xe2f4, but I'm unsure if that offset is fixed
+0           string    ID\x01\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00 Toshiba SSD Firmware Update

src/binwalk/magic/lzma

@@ -64,7 +64,11 @@
 # A value of -1 IS valid!
 >5        lequad    !-1
 >>5        lequad    <32        {invalid}
->>5        lequad   >1000000000000 {invalid}
+# Some oddball LZMA implementations use the 0x5D properties byte,
+# but don't include the ucompressed size. For this specific LZMA
+# property, don't check the uncompressed size here (it is validated
+# in a plugin).
+#>>5        lequad   >1000000000000 {invalid}
 
 
 # ------------------------------------------------------------------

src/binwalk/modules/extractor.py

@@ -59,9 +59,10 @@ class Extractor(Module):
         Option(short='D',
                long='dd',
                type=list,
-               dtype='type:ext:cmd',
+               dtype='type[:ext[:cmd]]',
                kwargs={'manual_rules': [], 'enabled': True},
-               description='Extract <type> signatures, give the files an extension of <ext>, and execute <cmd>'),
+               description='Extract <type> signatures (regular expression), give the files an extension of <ext>, '
+                           'and execute <cmd>'),
         Option(short='M',
                long='matryoshka',
                kwargs={'matryoshka': 8},

src/binwalk/plugins/lzmamod.py

-import os
-import binwalk.core.plugin
-from binwalk.core.compat import *
-from binwalk.core.common import BlockFile
-
-
-class LZMAModPlugin(binwalk.core.plugin.Plugin):
-
-    '''
-    Finds and extracts modified LZMA files commonly found in cable modems.
-    Based on Bernardo Rodrigues' work: http://w00tsec.blogspot.com/2013/11/unpacking-firmware-images-from-cable.html
-    '''
-    MODULES = ['Signature']
-
-    FAKE_LZMA_SIZE = "\xFF\xFF\xFF\xFF\xFF\xFF\xFF\xFF"
-    SIGNATURE = "lzma compressed data"
-
-    def init(self):
-        pass
-        #if self.module.extractor.enabled:
-        #    self.module.extractor.add_rule(txtrule=None,
-        #                                   regex="^%s" % self.SIGNATURE,
-        #                                   extension="7z",
-        #                                   cmd=self.lzma_cable_extractor, prepend=True)
-
-    def lzma_cable_extractor(self, fname):
-        result = False
-
-        out_name = os.path.splitext(fname)[0] + '-patched' + os.path.splitext(fname)[1]
-        fp_out = BlockFile(out_name, 'w')
-        # Use self.module.config.open_file here to ensure that other config
-        # settings (such as byte-swapping) are honored
-        fp_in = self.module.config.open_file(fname, offset=0, length=0)
-        fp_in.set_block_size(peek=0)
-        i = 0
-
-        while i < fp_in.length:
-            (data, dlen) = fp_in.read_block()
-
-            if i == 0:
-                out_data = data[0:5] + self.FAKE_LZMA_SIZE + data[5:]
-            else:
-                out_data = data
-
-            fp_out.write(out_data)
-
-            i += dlen
-
-        fp_in.close()
-        fp_out.close()
-
-        for rule in self.module.extractor.get_rules(self.SIGNATURE):
-            if rule['cmd'] != self.lzma_cable_extractor:
-                result = self.module.extractor.execute(rule['cmd'], out_name)
-                if result == True:
-                    break
-
-        os.remove(out_name)
-        return result
-
-    def scan(self, result):
-        # The modified cable modem LZMA headers all have valid dictionary sizes
-        # and a properties byte of 0x5D.
-        if result.description.lower().startswith(self.SIGNATURE) and "invalid uncompressed size" in result.description:
-            if "properties: 0x5D" in result.description and "invalid dictionary size" not in result.description:
-                result.valid = True
-                result.description = result.description.split("invalid uncompressed size")[0] + "missing uncompressed size"

src/binwalk/plugins/lzmavalid.py

@@ -62,3 +62,5 @@ class LZMAPlugin(binwalk.core.plugin.Plugin):
                 data = data[:5] + self.FAKE_LZMA_SIZE + data[5:]
                 if not self.is_valid_lzma(data):
                     result.valid = False
+                else:
+                    result.description = ",".join(result.description.split(',')[:-1] + [" missing uncompressed size"])