Python: Copy Python extractor to codeql repo

python/extractor/buildtools/version.py

@@ -0,0 +1,223 @@
+import sys
+import os
+import subprocess
+import tokenize
+import re
+
+from buildtools.helper import print_exception_indented
+
+
+TROVE = re.compile(r"Programming Language\s+::\s+Python\s+::\s+(\d)")
+
+if sys.version_info > (3,):
+    import collections.abc as collections
+    file_open = tokenize.open
+else:
+    import collections
+    file_open = open
+
+WIN = sys.platform == "win32"
+
+
+if WIN:
+    # installing `py` launcher is optional when installing Python on windows, so it's
+    # possible that the user did not install it, see
+    # https://github.com/github/codeql-cli-binaries/issues/125#issuecomment-1157429430
+    # so we check whether it has been installed. Newer versions have a `--list` option,
+    # but that has only been mentioned in the docs since 3.9, so to not risk it not
+    # working on potential older versions, we'll just use `py --version` which forwards
+    # the `--version` argument to the default python executable.
+
+    try:
+        subprocess.check_call(["py", "--version"])
+    except (subprocess.CalledProcessError, Exception):
+        sys.stderr.write("The `py` launcher is required for CodeQL to work on Windows.")
+        sys.stderr.write("Please include it when installing Python for Windows.")
+        sys.stderr.write("see https://docs.python.org/3/using/windows.html#python-launcher-for-windows")
+        sys.stderr.flush()
+        sys.exit(4) # 4 was a unique exit code at the time of writing
+
+AVAILABLE_VERSIONS = []
+
+def set_available_versions():
+    """Sets the global `AVAILABLE_VERSIONS` to a list of available (major) Python versions."""
+    global AVAILABLE_VERSIONS
+    if AVAILABLE_VERSIONS:
+        return # already set
+    for version in [3, 2]:
+        try:
+            subprocess.check_call(" ".join(executable_name(version) + ["-c", "pass"]), shell=True)
+            AVAILABLE_VERSIONS.append(version)
+        except Exception:
+            pass # If not available, we simply don't add it to the list
+    if not AVAILABLE_VERSIONS:
+        # If neither 'python3' nor 'python2' is available, we'll just try 'python' and hope for the best
+        AVAILABLE_VERSIONS = ['']
+
+def executable(version):
+    """Returns the executable to use for the given Python version."""
+    global AVAILABLE_VERSIONS
+    set_available_versions()
+    if version not in AVAILABLE_VERSIONS:
+        available_version = AVAILABLE_VERSIONS[0]
+        print("Wanted to run Python %s, but it is not available. Using Python %s instead" % (version, available_version))
+        version = available_version
+    return executable_name(version)
+
+
+def executable_name(version):
+    if WIN:
+        return ["py", "-%s" % version]
+    else:
+        return ["python%s" % version]
+
+PREFERRED_PYTHON_VERSION = None
+
+def extractor_executable():
+    '''
+    Returns the executable to use for the extractor.
+    If a Python executable name is specified using the extractor option, returns that name.
+    In the absence of a user-specified executable name, returns the executable name for
+    Python 3 if it is available, and Python 2 if not.
+    '''
+    executable_name = os.environ.get("CODEQL_EXTRACTOR_PYTHON_OPTION_PYTHON_EXECUTABLE_NAME", None)
+    if executable_name is not None:
+        print("Using Python executable name provided via the python_executable_name extractor option: {}"
+            .format(executable_name)
+        )
+        return [executable_name]
+    # Call machine_version() to ensure we've set PREFERRED_PYTHON_VERSION
+    if PREFERRED_PYTHON_VERSION is None:
+        machine_version()
+    return executable(PREFERRED_PYTHON_VERSION)
+
+def machine_version():
+    """If only Python 2 or Python 3 is installed, will return that version"""
+    global PREFERRED_PYTHON_VERSION
+    print("Trying to guess Python version based on installed versions")
+    if sys.version_info > (3,):
+        this, other = 3, 2
+    else:
+        this, other = 2, 3
+    try:
+        exe = executable(other)
+        # We need `shell=True` here in order for the test framework to function correctly. For
+        # whatever reason, the `PATH` variable is ignored if `shell=False`.
+        # Also, this in turn forces us to give the whole command as a string, rather than a list.
+        # Otherwise, the effect is that the Python interpreter is invoked _as a REPL_, rather than
+        # with the given piece of code.
+        subprocess.check_call(" ".join(exe + [ "-c", "pass" ]), shell=True)
+        print("This script is running Python {}, but Python {} is also available (as '{}')"
+            .format(this, other, ' '.join(exe))
+        )
+        # If both versions are available, our preferred version is Python 3
+        PREFERRED_PYTHON_VERSION = 3
+        return None
+    except Exception:
+        print("Only Python {} installed -- will use that version".format(this))
+        PREFERRED_PYTHON_VERSION = this
+        return this
+
+def trove_version(root):
+    print("Trying to guess Python version based on Trove classifiers in setup.py")
+    try:
+        full_path = os.path.join(root, "setup.py")
+        if not os.path.exists(full_path):
+            print("Did not find setup.py (expected it to be at {})".format(full_path))
+            return None
+
+        versions = set()
+        with file_open(full_path) as fd:
+            contents = fd.read()
+        for match in TROVE.finditer(contents):
+            versions.add(int(match.group(1)))
+
+        if 2 in versions and 3 in versions:
+            print("Found Trove classifiers for both Python 2 and Python 3 in setup.py -- will use Python 3")
+            return 3
+        elif len(versions) == 1:
+            result = versions.pop()
+            print("Found Trove classifier for Python {} in setup.py -- will use that version".format(result))
+            return result
+        else:
+            print("Found no Trove classifiers for Python in setup.py")
+    except Exception:
+        print("Skipping due to exception:")
+        print_exception_indented()
+    return None
+
+def wrap_with_list(x):
+    if isinstance(x, collections.Iterable) and not isinstance(x, str):
+        return x
+    else:
+        return [x]
+
+def travis_version(root):
+    print("Trying to guess Python version based on travis file")
+    try:
+        full_paths = [os.path.join(root, filename) for filename in [".travis.yml", "travis.yml"]]
+        travis_file_paths = [path for path in full_paths if os.path.exists(path)]
+        if not travis_file_paths:
+            print("Did not find any travis files (expected them at either {})".format(full_paths))
+            return None
+
+        try:
+            import yaml
+        except ImportError:
+            print("Found a travis file, but yaml library not available")
+            return None
+
+        with open(travis_file_paths[0]) as travis_file:
+            travis_yaml = yaml.safe_load(travis_file)
+        if "python" in travis_yaml:
+            versions = wrap_with_list(travis_yaml["python"])
+        else:
+            versions = []
+
+        # 'matrix' is an alias for 'jobs' now (https://github.com/travis-ci/docs-travis-ci-com/issues/1500)
+        # If both are defined, only the last defined will be used.
+        if "matrix" in travis_yaml and "jobs" in travis_yaml:
+            print("Ignoring 'matrix' and 'jobs' in Travis file, since they are both defined (only one of them should be).")
+        else:
+            matrix = travis_yaml.get("matrix") or travis_yaml.get("jobs") or dict()
+            includes = matrix.get("include") or []
+            for include in includes:
+                if "python" in include:
+                    versions.extend(wrap_with_list(include["python"]))
+
+        found = set()
+        for version in versions:
+            # Yaml may convert version strings to numbers, convert them back.
+            version = str(version)
+            if version.startswith("2"):
+                found.add(2)
+            if version.startswith("3"):
+                found.add(3)
+
+        if len(found) == 1:
+            result = found.pop()
+            print("Only found Python {} in travis file -- will use that version".format(result))
+            return result
+        elif len(found) == 2:
+            print("Found both Python 2 and Python 3 being used in travis file -- ignoring")
+        else:
+            print("Found no Python being used in travis file")
+    except Exception:
+        print("Skipping due to exception:")
+        print_exception_indented()
+    return None
+
+VERSION_TAG = "LGTM_PYTHON_SETUP_VERSION"
+
+def best_version(root, default):
+    if VERSION_TAG in os.environ:
+        try:
+            return int(os.environ[VERSION_TAG])
+        except ValueError:
+            raise SyntaxError("Illegal value for " + VERSION_TAG)
+    print("Will try to guess Python version, as it was not specified in `lgtm.yml`")
+    version = trove_version(root) or travis_version(root) or machine_version()
+    if version is None:
+        version = default
+        print("Could not guess Python version, will use default: Python {}".format(version))
+    return version