Python: Copy Python extractor to codeql repo

python/extractor/tests/buildtools/helper.py

@@ -0,0 +1,102 @@
+import os
+import stat
+import tempfile
+import shutil
+import time
+import sys
+import subprocess
+from contextlib import contextmanager
+from functools import wraps
+
+
+# Would have liked to use a decorator, but for Python 2 the functools.wraps is not good enough for
+# signature preservation that pytest can figure out what is going on. It would be possible to use
+# the decorator package, but that seemed like a bit too much of a hassle.
+@contextmanager
+def in_fresh_temp_dir():
+    old_cwd = os.getcwd()
+    with managed_temp_dir('extractor-python-buildtools-test-') as tmp:
+        os.chdir(tmp)
+        try:
+            yield tmp
+        finally:
+            os.chdir(old_cwd)
+
+
+@contextmanager
+def managed_temp_dir(prefix=None):
+    dir = tempfile.mkdtemp(prefix=prefix)
+    try:
+        yield dir
+    finally:
+        rmtree_robust(dir)
+
+
+def rmtree_robust(dir):
+    if is_windows():
+        # It's important that the path is a Unicode path on Windows, so
+        # that the right system calls get used.
+        dir = u'' + dir
+        if not dir.startswith("\\\\?\\"):
+            dir = "\\\\?\\" + os.path.abspath(dir)
+
+    # Emulate Python 3 "nonlocal" keyword
+    class state: pass
+    state.last_failed_delete = None
+
+
+    def _rmtree(path):
+        """wrapper of shutil.rmtree to handle Python 3.12 rename (onerror => onexc)"""
+        if sys.version_info >= (3, 12):
+            shutil.rmtree(path, onexc=remove_error)
+        else:
+            shutil.rmtree(path, onerror=remove_error)
+
+    def remove_error(func, path, excinfo):
+        # If we get an error twice in a row for the same path then just give up.
+        if state.last_failed_delete == path:
+            return
+        state.last_failed_delete = path
+
+        # The problem could be one of permissions, so setting path writable
+        # might fix it.
+        os.chmod(path, stat.S_IWRITE)
+
+        # On Windows, we sometimes get errors about directories not being
+        # empty, but immediately afterwards they are empty. Waiting a bit
+        # might therefore be sufficient.
+        t = 0.1
+        while (True):
+            try:
+                if os.path.isdir(path):
+                    _rmtree(path)
+                else:
+                    os.remove(path)
+            except OSError:
+                if (t > 1):
+                    return # Give up
+                time.sleep(t)
+                t *= 2
+    _rmtree(dir)
+    # On Windows, attempting to write immediately after deletion may result in
+    # an 'access denied' exception, so wait a bit.
+    if is_windows():
+        time.sleep(0.5)
+
+
+def is_windows():
+    return os.name == 'nt'
+
+
+@contextmanager
+def copy_repo_dir(repo_dir_in):
+    with managed_temp_dir(prefix="extractor-python-buildtools-test-") as tmp:
+        repo_dir = os.path.join(tmp, 'repo')
+        print('copying', repo_dir_in, 'to', repo_dir)
+        shutil.copytree(repo_dir_in, repo_dir, symlinks=True)
+        yield repo_dir
+
+################################################################################
+
+
+DEVNULL = subprocess.DEVNULL

python/extractor/tests/buildtools/test_index.py

@@ -0,0 +1,169 @@
+import os
+import pytest
+import shutil
+import glob
+
+import buildtools.index
+from tests.buildtools.helper import in_fresh_temp_dir
+
+# we use `monkeypatch.setenv` instead of setting `os.environ` directly, since that produces
+# cross-talk between tests. (using mock.patch.dict is only available for Python 3)
+
+
+class TestIncludeOptions:
+    @staticmethod
+    def test_LGTM_SRC(monkeypatch):
+        monkeypatch.setenv("LGTM_SRC", "path/src")
+        assert buildtools.index.get_include_options() == ["-R", "path/src"]
+
+    @staticmethod
+    def test_LGTM_INDEX_INCLUDE(monkeypatch):
+        monkeypatch.setenv("LGTM_INDEX_INCLUDE", "/foo\n/bar")
+        assert buildtools.index.get_include_options() == ["-R", "/foo", "-R", "/bar"]
+
+
+class TestPip21_3:
+    @staticmethod
+    def test_no_build_dir(monkeypatch):
+        with in_fresh_temp_dir() as path:
+            os.makedirs(os.path.join(path, "src"))
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_pip_21_3_build_dir_options() == []
+
+    @staticmethod
+    def test_faked_build_dir(monkeypatch):
+        # since I don't want to introduce specific pip version on our
+        # testing infrastructure, I'm just going to fake that `pip install .` had
+        # been called.
+        with in_fresh_temp_dir() as path:
+            os.makedirs(os.path.join(path, "build", "lib"))
+            monkeypatch.setenv("LGTM_SRC", path)
+            expected = ["-Y", os.path.join(path, "build")]
+            assert buildtools.index.exclude_pip_21_3_build_dir_options() == expected
+
+    @staticmethod
+    def test_disable_environment_variable(monkeypatch):
+        monkeypatch.setenv(
+            "CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_PIP_BUILD_DIR_EXCLUDE", "1"
+        )
+        with in_fresh_temp_dir() as path:
+            os.makedirs(os.path.join(path, "build", "lib"))
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_pip_21_3_build_dir_options() == []
+
+    @staticmethod
+    def test_code_build_dir(monkeypatch):
+        # simulating that you have the module `mypkg.build.lib.foo`
+        with in_fresh_temp_dir() as path:
+            os.makedirs(os.path.join(path, "mypkg", "build", "lib"))
+            open(os.path.join(path, "mypkg", "build", "lib", "foo.py"), "wt").write("print(42)")
+            open(os.path.join(path, "mypkg", "build", "lib", "__init__.py"), "wt").write("")
+            open(os.path.join(path, "mypkg", "build", "__init__.py"), "wt").write("")
+            open(os.path.join(path, "mypkg", "__init__.py"), "wt").write("")
+
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_pip_21_3_build_dir_options() == []
+
+
+def create_fake_venv(path, is_unix):
+    os.makedirs(path)
+    open(os.path.join(path, "pyvenv.cfg"), "wt").write("")
+    if is_unix:
+        os.mkdir(os.path.join(path, "bin"))
+        open(os.path.join(path, "bin", "activate"), "wt").write("")
+        os.makedirs(os.path.join(path, "lib", "python3.10", "site-packages"))
+    else:
+        os.mkdir(os.path.join(path, "Scripts"))
+        open(os.path.join(path, "Scripts", "activate.bat"), "wt").write("")
+        os.makedirs(os.path.join(path, "Lib", "site-packages"))
+
+class TestVenvIgnore:
+    @staticmethod
+    def test_no_venv(monkeypatch):
+        with in_fresh_temp_dir() as path:
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_venvs_options() == []
+
+    @staticmethod
+    @pytest.mark.parametrize("is_unix", [True,False])
+    def test_faked_venv_dir(monkeypatch, is_unix):
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=is_unix)
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_venvs_options() == ["-Y", os.path.join(path, "venv")]
+
+    @staticmethod
+    @pytest.mark.parametrize("is_unix", [True,False])
+    def test_multiple_faked_venv_dirs(monkeypatch, is_unix):
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=is_unix)
+            create_fake_venv(os.path.join(path, "venv2"), is_unix=is_unix)
+
+            monkeypatch.setenv("LGTM_SRC", path)
+
+            expected = [
+                "-Y", os.path.join(path, "venv"),
+                "-Y", os.path.join(path, "venv2"),
+            ]
+
+            actual = buildtools.index.exclude_venvs_options()
+            assert sorted(actual) == sorted(expected)
+
+    @staticmethod
+    def test_faked_venv_dir_no_pyvenv_cfg(monkeypatch):
+        """
+        Some times, the `pyvenv.cfg` file is not included when a virtual environment is
+        added to a git-repo, but we should be able to ignore the venv anyway.
+
+        See
+        - https://github.com/FiacreT/M-moire/tree/4089755191ffc848614247e98bbb641c1933450d/osintplatform/testNeo/venv
+        - https://github.com/Lynchie/KCM/tree/ea9eeed07e0c9eec41f9fc7480ce90390ee09876/VENV
+        """
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=True)
+            monkeypatch.setenv("LGTM_SRC", path)
+            os.remove(os.path.join(path, "venv", "pyvenv.cfg"))
+            assert buildtools.index.exclude_venvs_options() == ["-Y", os.path.join(path, "venv")]
+
+    @staticmethod
+    def test_faked_venv_no_bin_dir(monkeypatch):
+        """
+        Some times, the activate script is not included when a virtual environment is
+        added to a git-repo, but we should be able to ignore the venv anyway.
+        """
+
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=True)
+            monkeypatch.setenv("LGTM_SRC", path)
+            bin_dir = os.path.join(path, "venv", "bin")
+            assert os.path.isdir(bin_dir)
+            shutil.rmtree(bin_dir)
+            assert buildtools.index.exclude_venvs_options() == ["-Y", os.path.join(path, "venv")]
+
+    @staticmethod
+    def test_faked_venv_dir_no_lib_python(monkeypatch):
+        """
+        If there are no `lib/pyhton*` dirs within a unix venv, then it doesn't
+        constitute a functional virtual environment, and we don't exclude it. That's not
+        going to hurt, since it won't contain any installed packages.
+        """
+
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=True)
+            monkeypatch.setenv("LGTM_SRC", path)
+            glob_res = glob.glob(os.path.join(path, "venv", "lib", "python*"))
+            assert glob_res
+            for d in glob_res:
+                shutil.rmtree(d)
+            assert buildtools.index.exclude_venvs_options() == []
+
+    @staticmethod
+    @pytest.mark.parametrize("is_unix", [True,False])
+    def test_disable_environment_variable(monkeypatch, is_unix):
+        monkeypatch.setenv(
+            "CODEQL_EXTRACTOR_PYTHON_DISABLE_AUTOMATIC_VENV_EXCLUDE", "1"
+        )
+        with in_fresh_temp_dir() as path:
+            create_fake_venv(os.path.join(path, "venv"), is_unix=is_unix)
+            monkeypatch.setenv("LGTM_SRC", path)
+            assert buildtools.index.exclude_venvs_options() == []

python/extractor/tests/buildtools/test_install.py

@@ -0,0 +1,16 @@
+import pytest
+
+import buildtools.install
+from tests.buildtools.helper import in_fresh_temp_dir
+
+def test_basic(monkeypatch, mocker):
+    mocker.patch('subprocess.call')
+    mocker.patch('subprocess.check_call')
+
+    with in_fresh_temp_dir() as path:
+        monkeypatch.setenv('LGTM_WORKSPACE', path)
+        monkeypatch.setenv('SEMMLE_DIST', '<none>')
+
+        with pytest.raises(SystemExit) as exc_info:
+            buildtools.install.main(3, '.', [])
+        assert exc_info.value.code == 0

python/extractor/tests/buildtools/test_python_auto_install.py

@@ -0,0 +1,53 @@
+#!/usr/bin/env python
+
+import buildtools.semmle.requirements as requirements
+import unittest
+
+class RequirementsTests(unittest.TestCase):
+
+    def assertExpected(self, reqs, expected):
+        self.assertEqual(str(reqs), str(requirements.parse(expected.splitlines())))
+
+    _input = """\
+        SQLAlchemy<1.1.0,>=1.0.10 # MIT
+        sqlalchemy-migrate>=0.9.6 # Apache-2.0
+        stevedore>=1.10.0a4 # Apache-2.0
+        WebOb>1.2.3 # MIT
+        oslo.i18n!=2.1.0,==2.0.7 # Apache-2.0
+        foo>=0.9,<0.8 # Contradictory
+        bar>=1.3, <1.3 # Contradictory, but only just
+        baz>=3 # No dot in version number.
+        git+https://github.com/mozilla/elasticutils.git # Requirement in Git. Should be ignored.
+        -e git+https://github.com/Lasagne/Lasagne.git@8f4f9b2#egg=Lasagne==0.2.git # Another Git requirement.
+        """
+
+    def test_clean(self):
+        reqs = requirements.parse(self._input.splitlines())
+        expected = """\
+        SQLAlchemy<1.1.0,>=1.0.10
+        sqlalchemy-migrate>=0.9.6
+        stevedore>=1.10.0a4
+        WebOb>1.2.3
+        oslo.i18n!=2.1.0,==2.0.7
+        foo>=0.9
+        bar>=1.3
+        baz>=3
+        """
+        self.assertExpected(requirements.clean(reqs), expected)
+
+    def test_restricted(self):
+        reqs = requirements.parse(self._input.splitlines())
+        expected = """\
+        SQLAlchemy<1.1.0,>=1.0.10,==1.*
+        sqlalchemy-migrate>=0.9.6,==0.*
+        stevedore>=1.10.0a4,==1.*
+        WebOb>1.2.3,==1.*
+        oslo.i18n!=2.1.0,==2.0.7
+        foo>=0.9,==0.*
+        bar>=1.3,==1.*
+        baz==3.*,>=3
+        """
+        self.assertExpected(requirements.restrict(reqs), expected)
+
+if __name__ == "__main__":
+    unittest.main()

python/extractor/tests/buildtools/test_version.py

@@ -0,0 +1,244 @@
+import os
+import re
+from textwrap import dedent
+import itertools
+
+import pytest
+
+import buildtools.version as version
+from tests.buildtools.helper import in_fresh_temp_dir
+
+
+class TestTravisVersion:
+
+    # based on https://docs.travis-ci.com/user/customizing-the-build/#build-matrix
+    # and https://docs.travis-ci.com/user/languages/python/
+
+    def test_simple(self):
+        with in_fresh_temp_dir():
+            assert version.travis_version('.') is None
+
+
+    @pytest.mark.parametrize(
+        'name,expected,travis_file',[
+        ('empty', None, ''),
+        ('no_python', None, dedent("""\
+        language: ruby
+        rvm:
+        - 2.5
+        - 2.6
+        """)),
+
+        ('both', None, dedent("""\
+        language: python
+        python:
+        - "2.6"
+        - "2.7"
+        - "3.5"
+        - "3.6"
+        """)),
+
+        ('only_py2', 2, dedent("""\
+        language: python
+        python:
+        - "2.6"
+        - "2.7"
+        """)),
+
+        ('only_py3', 3, dedent("""\
+        language: python
+        python:
+        - "3.5"
+        - "3.6"
+        """)),
+
+        ('jobs_both', None, dedent("""\
+        language: python
+        jobs:
+            include:
+                - python: 2.6
+                - python: 2.7
+                - python: 3.5
+                - python: 3.6
+        """)),
+
+        ('jobs_only_py2', 2, dedent("""\
+        language: python
+        jobs:
+            include:
+                - python: 2.6
+                - python: 2.7
+        """)),
+
+        ('jobs_only_py3', 3, dedent("""\
+        language: python
+        jobs:
+            include:
+                - python: 3.5
+                - python: 3.6
+        """)),
+
+        ('top_level_and_jobs', None, dedent("""\
+        language: python
+        python:
+        - "2.6"
+        - "2.7"
+        jobs:
+            include:
+                - python: 3.5
+                - python: 3.6
+        """)),
+
+        ('jobs_unrelated', 2, dedent("""\
+        language: python
+        python:
+        - "2.6"
+        - "2.7"
+        jobs:
+            include:
+                - env: FOO=FOO
+                - env: FOO=BAR
+        """)),
+
+        ('jobs_no_python', None, dedent("""\
+        language: ruby
+        jobs:
+            include:
+                - rvm: 2.5
+                - rvm: 2.6
+        """)),
+
+        # matrix is the old name for jobs (still supported as of 2019-11)
+        ('matrix_only_py3', 3, dedent("""\
+        language: python
+        matrix:
+            include:
+                - python: 3.5
+                - python: 3.6
+        """)),
+
+        ('quoted_py2', 2, dedent("""\
+        language: python
+        python:
+        - "2.7"
+        """)),
+
+        ('unquoted_py2', 2, dedent("""\
+        language: python
+        python:
+        - 2.7
+        """)),
+    ])
+    def test_with_file(self, name, expected, travis_file):
+        with in_fresh_temp_dir():
+            with open('.travis.yml', 'w') as f:
+                f.write(travis_file)
+            assert version.travis_version('.') is expected, name
+
+    def test_filesnames(self):
+        """Should prefer .travis.yml over travis.yml (which we still support for some legacy reason)
+        """
+        with in_fresh_temp_dir():
+            with open('travis.yml', 'w') as f:
+                f.write(dedent("""\
+                    language: python
+                    python:
+                    - "2.6"
+                    - "2.7"
+                    """))
+            assert version.travis_version('.') is 2
+
+            with open('.travis.yml', 'w') as f:
+                f.write(dedent("""\
+                    language: python
+                    python:
+                    - "3.5"
+                    - "3.6"
+                    """))
+            assert version.travis_version('.') is 3
+class TestTroveVersion:
+
+    def test_empty(self):
+        with in_fresh_temp_dir():
+            assert version.trove_version('.') is None
+
+    def test_with_file(self):
+        def _to_file(classifiers):
+            with open('setup.py', 'wt') as f:
+                f.write(dedent("""\
+                setup(
+                    classifiers={!r}
+                )
+                """.format(classifiers)
+                ))
+
+        cases = [
+            (2, "Programming Language :: Python :: 2.7"),
+            (2, "Programming Language :: Python :: 2"),
+            (2, "Programming Language :: Python :: 2 :: Only"),
+            (3, "Programming Language :: Python :: 3.7"),
+            (3, "Programming Language :: Python :: 3"),
+            (3, "Programming Language :: Python :: 3 :: Only"),
+        ]
+
+        for expected, classifier in cases:
+            with in_fresh_temp_dir():
+                _to_file([classifier])
+                assert version.trove_version('.') == expected
+
+        for combination in itertools.combinations(cases, 2):
+            with in_fresh_temp_dir():
+                versions, classifiers = zip(*combination)
+                _to_file(classifiers)
+                expected = 3 if 3 in versions else 2
+                assert version.trove_version('.') == expected
+
+    @pytest.mark.xfail()
+    def test_tricked_regex_is_too_simple(self):
+        with in_fresh_temp_dir():
+            with open('setup.py', 'wt') as f:
+                f.write(dedent("""\
+                setup(
+                    name='Programming Language :: Python :: 2',
+                    classifiers=[],
+                )
+                """
+                ))
+            assert version.trove_version('.') is None
+
+    @pytest.mark.xfail()
+    def test_tricked_regex_is_too_simple2(self):
+        with in_fresh_temp_dir():
+            with open('setup.py', 'wt') as f:
+                f.write(dedent("""\
+                setup(
+                    # classifiers=['Programming Language :: Python :: 2'],
+                )
+                """
+                ))
+            assert version.trove_version('.') is None
+
+    @pytest.mark.xfail()
+    def test_tricked_not_running_as_code(self):
+        with in_fresh_temp_dir():
+            with open('setup.py', 'wt') as f:
+                f.write(dedent("""\
+                c = 'Programming Language :: ' + 'Python :: 2'
+                setup(
+                    classifiers=[c],
+                )
+                """
+                ))
+            assert version.trove_version('.') is 2
+
+    def test_constructing_other_place(self):
+        with in_fresh_temp_dir():
+            with open('setup.py', 'wt') as f:
+                f.write(dedent("""\
+                c = 'Programming Language :: Python :: 2'
+                setup(
+                    classifiers=[c],
+                )
+                """
+                ))
+            assert version.trove_version('.') is 2