Merge pull request #488 from jameshilliard/python3-only

deps: Fix install on newer ubuntu distros and drop python2 support.

.travis.yml

+os: linux
+dist: bionic
 language: python
-arch: 
- - ppc64le
- - amd64
-python:
- - "2.7"
- - "3.4"
- - "3.5"
- - "3.6"
- - "3.7"
- - "3.8"
- - "nightly"
+arch: amd64
+addons:
+  apt:
+    update: true
 install:
+  - ./deps.sh --yes
   - pip install coverage
- - pip install backports.lzma
   - ./setup.py install
 script: ./setup.py test
 cache: pip
-matrix:
-  allow_failures:
-   - python: nightly
+jobs:
+  include:
+   - dist: bionic
+     python: 3.6
+   - dist: bionic
+     python: 3.7
+   - dist: bionic
+     python: 3.8
+   - dist: bionic
+     python: 3.9
+   - dist: focal
+     python: 3.6
+   - dist: focal
+     python: 3.7
+   - dist: focal
+     python: 3.8
+   - dist: focal
+     python: 3.9

deps.sh

@@ -41,24 +41,26 @@ APTGETCMD="apt-get"
 YUMCMD="yum"
 if [ $distro = "Kali" ]
 then
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract util-linux firmware-mod-kit cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop cpio"
-elif [ $distro_version = "17" ]
+    APT_CANDIDATES="git build-essential qt5base-dev mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract util-linux firmware-mod-kit cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop cpio"
+elif [ $distro_version = "14" ]
 then
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+elif [ $distro_version = "15" ]
+then
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+elif [ $distro_version = "16" ]
+then
+    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 elif [ $distro_version = "18" ]
 then
     APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 else
-    APT_CANDIDATES="git build-essential libqt4-opengl mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsprogs cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
+    APT_CANDIDATES="git build-essential qtbase5-dev mtd-utils gzip bzip2 tar arj lhasa p7zip p7zip-full cabextract cramfsswap squashfs-tools zlib1g-dev liblzma-dev liblzo2-dev sleuthkit default-jdk lzop srecord cpio"
 fi
-PYTHON2_APT_CANDIDATES="python-crypto python-lzo python-lzma python-pip python-tk"
-PYTHON3_APT_CANDIDATES="python3-crypto python3-pip python3-tk"
+PYTHON3_APT_CANDIDATES="python3-crypto python3-gnupg python3-pip python3-tk"
 PYTHON3_YUM_CANDIDATES=""
 YUM_CANDIDATES="git gcc gcc-c++ make openssl-devel qtwebkit-devel qt-devel gzip bzip2 tar arj p7zip p7zip-plugins cabextract squashfs-tools zlib zlib-devel lzo lzo-devel xz xz-compat-libs xz-libs xz-devel xz-lzma-compat python-backports-lzma lzip pyliblzma perl-Compress-Raw-Lzma lzop srecord"
-PYTHON2_YUM_CANDIDATES="python-pip python-Bottleneck cpio"
-APT_CANDIDATES="$APT_CANDIDATES $PYTHON2_APT_CANDIDATES"
-YUM_CANDIDATES="$YUM_CANDIDATES $PYTHON2_YUM_CANDIDATES"
-PIP_COMMANDS="pip"
+PIP_COMMANDS="pip3"
 
 # Check for root privileges
 if [ $UID -eq 0 ]
@@ -72,7 +74,7 @@ fi
 function install_yaffshiv
 {
     git clone https://github.com/devttys0/yaffshiv
-    (cd yaffshiv && $SUDO python2 setup.py install)
+    (cd yaffshiv && $SUDO python3 setup.py install)
     $SUDO rm -rf yaffshiv
 }
 
@@ -122,10 +124,7 @@ function install_cramfstools
 function install_ubireader
 {
     git clone https://github.com/jrspruitt/ubi_reader
-    # Some UBIFS extraction breaks after this commit, due to "Added fatal error check if UBI block extends beyond file size"
-    # (see this commit: https://github.com/jrspruitt/ubi_reader/commit/af678a5234dc891e8721ec985b1a6e74c77620b6)
-    # Reset to a known working commit.
-    (cd ubi_reader && git reset --hard 0955e6b95f07d849a182125919a1f2b6790d5b51 && $SUDO python setup.py install)
+    (cd ubi_reader && $SUDO python3 setup.py install)
     $SUDO rm -rf ubi_reader
 }
 
@@ -257,6 +256,7 @@ if [ $? -ne 0 ]
     echo "Package installation failed: $PKG_CANDIDATES"
     exit 1
 fi
+install_pip_package setuptools
 install_pip_package matplotlib
 install_pip_package capstone
 install_sasquatch

setup.py

-#!/usr/bin/env python
+#!/usr/bin/env python3
+import io
 import os
 import sys
 import glob
 import shutil
 import subprocess
-from distutils.core import setup, Command
+try:
+    from setuptools import setup, Command
+except ImportError:
+    from distutils.core import setup, Command
 from distutils.dir_util import remove_tree
 
 MODULE_NAME = "binwalk"
@@ -24,7 +28,7 @@ except ImportError:
 # by binwalk.
 try:
     label = subprocess.check_output(["git", "rev-parse", "--short", "HEAD"], stderr=DEVNULL).decode('utf-8')
-    MODULE_VERSION = "%s-%s" % (MODULE_VERSION, label.strip())
+    MODULE_VERSION = "%s+%s" % (MODULE_VERSION, label.strip())
 except KeyboardInterrupt as e:
     raise e
 except Exception:
@@ -318,14 +322,21 @@ install_data_files = []
 for data_dir in ["magic", "config", "plugins", "modules", "core"]:
     install_data_files.append("%s%s*" % (data_dir, os.path.sep))
 
+this_directory = os.path.abspath(os.path.dirname(__file__))
+with io.open(os.path.join(this_directory, 'README.md'), encoding='utf-8') as f:
+    long_description = f.read()
+
 # Install the module, script, and support files
 setup(
     name=MODULE_NAME,
     version=MODULE_VERSION,
     description="Firmware analysis tool",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
     author="Craig Heffner",
     url="https://github.com/ReFirmLabs/%s" % MODULE_NAME,
     requires=[],
+    python_requires=">=3",
     package_dir={"": "src"},
     packages=[MODULE_NAME],
     package_data={MODULE_NAME: install_data_files},

src/binwalk/core/compat.py

@@ -81,7 +81,4 @@ def user_input(prompt=''):
     '''
     For getting raw user input in Python 2 and 3.
     '''
-    if PY_MAJOR_VERSION > 2:
     return input(prompt)
-    else:
-        return raw_input(prompt)

src/binwalk/core/magic.py

@@ -625,7 +625,7 @@ class Magic(object):
                     try:
                         # If the operator value of this signature line is just
                         # an integer value, use it
-                        if isinstance(line.opvalue, int) or isinstance(line.opvalue, long):
+                        if isinstance(line.opvalue, int):
                             opval = line.opvalue
                         # Else, evaluate the complex expression
                         else:

src/binwalk/plugins/tar.py

@@ -33,7 +33,7 @@ class TarPlugin(binwalk.core.plugin.Plugin):
                 raise ValueError("invalid tar header")
         else:
             n = 0
-            for i in xrange(len(s) - 1):
+            for i in range(len(s) - 1):
                 n <<= 8
                 n += ord(s[i + 1])
         return n