Python: Copy Python extractor to codeql repo

python/extractor/buildtools/semmle/requirements.py

@@ -0,0 +1,136 @@
+import copy
+import tempfile
+import re
+from packaging.requirements import Requirement
+from packaging.version import Version
+from packaging.specifiers import SpecifierSet
+
+IGNORED_REQUIREMENTS = re.compile("^(-e\\s+)?(git|svn|hg)(?:\\+.*)?://.*$")
+
+def parse(lines):
+    'Parse a list of requirement strings into a list of `Requirement`s'
+    res = []
+    #Process
+    for line in lines:
+        if '#' in line:
+            line, _ = line.split('#', 1)
+        if not line:
+            continue
+        if IGNORED_REQUIREMENTS.match(line):
+            continue
+        try:
+            req = Requirement(line)
+        except:
+            print("Cannot parse requirements line '%s'" % line)
+        else:
+            res.append(req)
+    return res
+
+def parse_file(filename):
+    with open(filename, 'r') as fd:
+        return parse(fd.read().splitlines())
+
+def save_to_file(reqs):
+    'Takes a list of requirements, saves them to a temporary file and returns the filename'
+    with tempfile.NamedTemporaryFile(prefix="semmle-requirements", suffix=".txt", mode="w", delete=False) as fd:
+        for req in reqs:
+            if req.url is None:
+                fd.write(str(req))
+            else:
+                fd.write(req.url)
+            fd.write("\n")
+    return fd.name
+
+def clean(reqs):
+    'Look for self-contradictory specifier groups and remove the necessary specifier parts to make them consistent'
+    result = []
+    for req in reqs:
+        specs = req.specifier
+        cleaned_specs = _clean_specs(specs)
+        req.specifier = cleaned_specs
+        result.append(Requirement(str(req)))
+        req.specifier = specs
+    return result
+
+def _clean_specs(specs):
+    ok = SpecifierSet()
+    #Choose a deterministic order such that >= comes before <=.
+    for spec in sorted(iter(specs), key=str, reverse=True):
+        for ok_spec in ok:
+            if not _compatible_specifier(ok_spec, spec):
+                break
+        else:
+            ok &= SpecifierSet(str(spec))
+    return ok
+
+def restrict(reqs):
+    '''Restrict versions to "compatible" versions.
+    For example restrict >=1.2 to all versions >= 1.2 that have 1 as the major version number.
+    >=N... becomes >=N...,==N.* and >N... requirements becomes >N..,==N.*
+    '''
+    #First of all clean the requirements
+    reqs = clean(reqs)
+    result = []
+    for req in reqs:
+        specs = req.specifier
+        req.specifier = _restrict_specs(specs)
+        result.append(Requirement(str(req)))
+        req.specifier = specs
+    return result
+
+def _restrict_specs(specs):
+    restricted = copy.deepcopy(specs)
+    #Iteration order doesn't really matter here so we choose the
+    #same as for clean, just to be consistent
+    for spec in sorted(iter(specs), key=str, reverse=True):
+        if spec.operator in ('>', '>='):
+            base_version = spec.version.split(".", 1)[0]
+            restricted &= SpecifierSet('==' + base_version + '.*')
+    return restricted
+
+def _compatible_specifier(s1, s2):
+    overlaps = 0
+    overlaps += _min_version(s1) in s2
+    overlaps += _max_version(s1) in s2
+    overlaps += _min_version(s2) in s1
+    overlaps += _max_version(s2) in s1
+    if overlaps > 1:
+        return True
+    if overlaps == 1:
+        #One overlap -- Generally compatible, but not for <x, >=x
+        return not _is_strict(s1) and not _is_strict(s2)
+    #overlaps == 0:
+    return False
+
+MIN_VERSION = Version('0.0a0')
+MAX_VERSION = Version('1000000')
+
+def _min_version(s):
+    if s.operator in ('>', '>='):
+        return s.version
+    elif s.operator in ('<', '<=', '!='):
+        return MIN_VERSION
+    elif s.operator == '==':
+        v = s.version
+        if v[-1] == '*':
+            return v[:-1] + '0'
+        else:
+            return s.version
+    else:
+        # '~='
+        return s.version
+
+def _max_version(s):
+    if s.operator in ('<', '<='):
+        return s.version
+    elif s.operator in ('>', '>=', '!='):
+        return MAX_VERSION
+    elif s.operator in ('~=', '=='):
+        v = s.version
+        if v[-1] == '*' or s.operator == '~=':
+            return v[:-1] + '1000000'
+        else:
+            return s.version
+
+def _is_strict(s):
+    return s.operator in ('>', '<')