deps: Fix install on newer ubuntu distros and drop python2 support.

Dependencies such as jefferson are now python3 only and python2 is no
longer supported, as it appears all our dependencies have python3
versions it should be safe to drop python2 support.

.travis.yml

+dist: xenial
+addons:
+  apt:
+    update: true
 language: python
 arch: 
  - ppc64le
@@ -9,13 +13,21 @@ python:
  - "3.6"
  - "3.7"
  - "3.8"
+ - "3.9"
  - "nightly"
 install:
+ - ./deps.sh --yes
  - pip install coverage
- - pip install backports.lzma
  - ./setup.py install
 script: ./setup.py test
 cache: pip
-matrix:
+jobs:
+  include:
+   - python: "3.6"
+     dist: bionic
+   - python: "3.8"
+     dist: focal
+   - python: "3.9"
+     dist: focal
   allow_failures:
    - python: nightly

deps.sh

@@ -41,24 +41,26 @@ APTGETCMD="apt-get"
 YUMCMD="yum"
 if [ $distro = "Kali" ]
 then
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract util-linux firmware-mod-kit cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop cpio"
-elif [ $distro_version = "17" ]
+    APT_CANDIDATES="git build-essential qt5base-dev mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract util-linux firmware-mod-kit cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop cpio"
+elif [ $distro_version = "14" ]
 then
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+elif [ $distro_version = "15" ]
+then
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+elif [ $distro_version = "16" ]
+then
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 elif [ $distro_version = "18" ]
 then
     APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 else
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+    APT_CANDIDATES="git build-essential qtbase5-dev mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 fi
-PYTHON2_APT_CANDIDATES="python-crypto python-lzo python-lzma python-pip python-tk"
-PYTHON3_APT_CANDIDATES="python3-crypto python3-pip python3-tk"
+PYTHON3_APT_CANDIDATES="python3-crypto python3-gnupg python3-pip python3-tk"
 PYTHON3_YUM_CANDIDATES=""
 YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma lzop srecord"
-PYTHON2_YUM_CANDIDATES="python-pip python-Bottleneck cpio"
-APT_CANDIDATES="$APT_CANDIDATES $PYTHON2_APT_CANDIDATES"
-YUM_CANDIDATES="$YUM_CANDIDATES $PYTHON2_YUM_CANDIDATES"
-PIP_COMMANDS="pip"
+PIP_COMMANDS="pip3"
 
 # Check for root privileges
 if [ $UID -eq 0 ]
@@ -72,7 +74,7 @@ fi
 function install_yaffshiv
 {
     git clone https://github.com/devttys0/yaffshiv
-    (cd yaffshiv && $SUDO python2 setup.py install)
+    (cd yaffshiv && $SUDO python3 setup.py install)
     $SUDO rm -rf yaffshiv
 }
 
@@ -122,10 +124,7 @@ function install_cramfstools
 function install_ubireader
 {
     git clone https://github.com/jrspruitt/ubi_reader
-    # Some UBIFS extraction breaks after this commit, due to "Added fatal error check if UBI block extends beyond file size"
-    # (see this commit: https://github.com/jrspruitt/ubi_reader/commit/af678a5234dc891e8721ec985b1a6e74c77620b6)
-    # Reset to a known working commit.
-    (cd ubi_reader && git reset --hard 0955e6b95f07d849a182125919a1f2b6790d5b51 && $SUDO python setup.py install)
+    (cd ubi_reader && $SUDO python3 setup.py install)
     $SUDO rm -rf ubi_reader
 }
 
@@ -257,6 +256,7 @@ if [ $? -ne 0 ]
     echo "Package installation failed: $PKG_CANDIDATES"
     exit 1
 fi
+install_pip_package setuptools
 install_pip_package matplotlib
 install_pip_package capstone
 install_sasquatch

setup.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
+import io
 import os
 import sys
 import glob
 import shutil
 import subprocess
-from distutils.core import setup, Command
+try:
+    from setuptools import setup, Command
+except ImportError:
+    from distutils.core import setup, Command
 from distutils.dir_util import remove_tree
 
 MODULE_NAME = "binwalk"
@@ -24,7 +28,7 @@ except ImportError:
 # by binwalk.
 try:
     label = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], stderr=DEVNULL).decode('utf-8')
-    MODULE_VERSION = "%s-%s" % (MODULE_VERSION, label.strip())
+    MODULE_VERSION = "%s+%s" % (MODULE_VERSION, label.strip())
 except KeyboardInterrupt as e:
     raise e
 except Exception:
@@ -318,14 +322,21 @@ install_data_files = []
 for data_dir in ["magic", "config", "plugins", "modules", "core"]:
     install_data_files.append("%s%s*" % (data_dir, os.path.sep))
 
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
 # Install the module, script, and support files
 setup(
     name=MODULE_NAME,
     version=MODULE_VERSION,
     description="Firmware analysis tool",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
     author="Craig Heffner",
     url="https://github.com/ReFirmLabs/%s" % MODULE_NAME,
     requires=[],
+    python_requires=">=3",
     package_dir={"": "src"},
     packages=[MODULE_NAME],
     package_data={MODULE_NAME: install_data_files},

src/binwalk/core/compat.py

@@ -81,7 +81,4 @@ def user_input(prompt=''):
     '''
     For getting raw user input in Python 2 and 3.
     '''
-    if PY_MAJOR_VERSION > 2:
-        return input(prompt)
-    else:
-        return raw_input(prompt)
+    return input(prompt)

src/binwalk/core/magic.py

@@ -625,7 +625,7 @@ class Magic(object):
                     try:
                         # If the operator value of this signature line is just
                         # an integer value, use it
-                        if isinstance(line.opvalue, int) or isinstance(line.opvalue, long):
+                        if isinstance(line.opvalue, int):
                             opval = line.opvalue
                         # Else, evaluate the complex expression
                         else:

src/binwalk/plugins/tar.py

@@ -33,7 +33,7 @@ class TarPlugin(binwalk.core.plugin.Plugin):
                 raise ValueError("invalid tar header")
         else:
             n = 0
-            for i in xrange(len(s) - 1):
+            for i in range(len(s) - 1):
                 n <<= 8
                 n += ord(s[i + 1])
         return n