hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-05 13:45:19 +02:00
Code Issues Packages Projects Releases Wiki Activity

Codegen: mark generated checked in files as such

Browse Source
This commit is contained in:
Paolo Tranquilli
2023-05-05 09:15:19 +02:00
parent 2d5b35067e
commit 1155b97232
6 changed files with 1947 additions and 996 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
misc/codegen/codegen.py
View File

@@ -58,7 +58,9 @@ def _parse_args() -> argparse.Namespace:
help="output directory for generated C++ files, required if trap or cpp is provided to "
"--generate"),
p.add_argument("--generated-registry",
help="registry file containing information about checked-in generated code"),
help="registry file containing information about checked-in generated code. A .gitattributes"
"file is generated besides it to mark those files with linguist-generated=true. Must"
"be in a directory containing all generated code."),
]
p.add_argument("--script-name",
help="script name to put in header comments of generated files. By default, the path of this "
@@ -108,7 +110,7 @@ def run():
log_level = logging.INFO
logging.basicConfig(format="{levelname} {message}", style='{', level=log_level)
for target in opts.generate:
generate(target, opts, render.Renderer(opts.script_name, opts.root_dir))
generate(target, opts, render.Renderer(opts.script_name))
if __name__ == "__main__":

26
misc/codegen/lib/render.py
View File

@@ -25,14 +25,10 @@ class Error(Exception):
class Renderer:
""" Template renderer using mustache templates in the `templates` directory """
def __init__(self, generator: pathlib.Path, root_dir: pathlib.Path):
def __init__(self, generator: pathlib.Path):
self._r = pystache.Renderer(search_dirs=str(paths.templates_dir), escape=lambda u: u)
self._root_dir = root_dir
self._generator = generator
def _get_path(self, file: pathlib.Path):
return file.relative_to(self._root_dir)
def render(self, data: object, output: pathlib.Path):
""" Render `data` to `output`.
@@ -60,7 +56,7 @@ class Renderer:
def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path, force: bool = False) -> "RenderManager":
return RenderManager(self._generator, self._root_dir, generated, stubs, registry, force)
return RenderManager(self._generator, generated, stubs, registry, force)
class RenderManager(Renderer):
@@ -85,10 +81,10 @@ class RenderManager(Renderer):
pre: str
post: typing.Optional[str] = None
def __init__(self, generator: pathlib.Path, root_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
def __init__(self, generator: pathlib.Path, generated: typing.Iterable[pathlib.Path],
stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path, force: bool = False):
super().__init__(generator, root_dir)
super().__init__(generator)
self._registry_path = registry
self._force = force
self._hashes = {}
@@ -117,10 +113,13 @@ class RenderManager(Renderer):
self._hashes.pop(self._get_path(f), None)
# clean up the registry from files that do not exist any more
for f in list(self._hashes):
if not (self._root_dir / f).exists():
if not (self._registry_path.parent / f).exists():
self._hashes.pop(f)
self._dump_registry()
def _get_path(self, file: pathlib.Path):
return file.relative_to(self._registry_path.parent)
def _do_write(self, mnemonic: str, contents: str, output: pathlib.Path):
hash = self._hash_string(contents)
rel_output = self._get_path(output)
@@ -186,13 +185,16 @@ class RenderManager(Renderer):
try:
with open(self._registry_path) as reg:
for line in reg:
filename, prehash, posthash = line.split()
self._hashes[pathlib.Path(filename)] = self.Hashes(prehash, posthash)
if line.strip():
filename, prehash, posthash = line.split()
self._hashes[pathlib.Path(filename)] = self.Hashes(prehash, posthash)
except FileNotFoundError:
pass
def _dump_registry(self):
self._registry_path.parent.mkdir(parents=True, exist_ok=True)
with open(self._registry_path, 'w') as out:
with open(self._registry_path, 'w') as out, open(self._registry_path.parent / ".gitattributes", "w") as attrs:
print(self._registry_path.name, "linguist-generated", file=attrs)
for f, hashes in sorted(self._hashes.items()):
print(f, hashes.pre, hashes.post, file=out)
print(f, "linguist-generated", file=attrs)

2
misc/codegen/test/test_qlgen.py
View File

@@ -26,7 +26,7 @@ def ql_output_path(): return paths.root_dir / "ql/lib/other/path"
def ql_test_output_path(): return paths.root_dir / "ql/test/path"
def generated_registry_path(): return paths.root_dir / "registry.list"
def generated_registry_path(): return paths.root_dir / "ql/registry.list"
def import_file(): return stub_path().with_suffix(".qll")

133
misc/codegen/test/test_render.py
View File

@@ -24,14 +24,31 @@ def pystache_renderer(pystache_renderer_cls):
@pytest.fixture
def sut(pystache_renderer):
return render.Renderer(generator, paths.root_dir)
return render.Renderer(generator)
def assert_file(file, text):
assert file.is_file()
with open(file) as inp:
assert inp.read() == text
def create_registry(files_and_hashes):
if not files_and_hashes:
return ""
return "\n".join(" ".join(data) for data in files_and_hashes) + "\n"
def write_registry(file, *files_and_hashes):
write(file, create_registry(files_and_hashes))
def assert_registry(file, *files_and_hashes):
assert_file(file, create_registry(files_and_hashes))
files = [file.name] + [f for f, _, _ in files_and_hashes]
assert_file(file.parent / ".gitattributes", "\n".join(f"{f} linguist-generated" for f in files) + "\n")
def hash(text):
h = hashlib.sha256()
h.update(text.encode())
@@ -50,7 +67,7 @@ def test_render(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
sut.render(data, output)
assert_file(output, text)
@@ -63,16 +80,16 @@ def test_managed_render(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
registry = paths.root_dir / "a/registry.list"
write(registry)
write_registry(registry)
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
renderer.render(data, output)
assert renderer.written == {output}
assert_file(output, text)
assert_file(registry, f"some/output.txt {hash(text)} {hash(text)}\n")
assert_registry(registry, ("some/output.txt", hash(text), hash(text)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
@@ -82,7 +99,7 @@ def test_managed_render_with_no_registry(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
registry = paths.root_dir / "a/registry.list"
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
@@ -90,7 +107,7 @@ def test_managed_render_with_no_registry(pystache_renderer, sut):
assert renderer.written == {output}
assert_file(output, text)
assert_file(registry, f"some/output.txt {hash(text)} {hash(text)}\n")
assert_registry(registry, ("some/output.txt", hash(text), hash(text)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
@@ -101,9 +118,9 @@ def test_managed_render_with_post_processing(pystache_renderer, sut):
text = "some text"
postprocessed_text = "some other text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
registry = paths.root_dir / "a/registry.list"
write(registry)
write_registry(registry)
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
renderer.render(data, output)
@@ -111,36 +128,36 @@ def test_managed_render_with_post_processing(pystache_renderer, sut):
assert_file(output, text)
write(output, postprocessed_text)
assert_file(registry, f"some/output.txt {hash(text)} {hash(postprocessed_text)}\n")
assert_registry(registry, ("some/output.txt", hash(text), hash(postprocessed_text)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
def test_managed_render_with_erasing(pystache_renderer, sut):
output = paths.root_dir / "some/output.txt"
stub = paths.root_dir / "some/stub.txt"
output = paths.root_dir / "a/some/output.txt"
stub = paths.root_dir / "a/some/stub.txt"
registry = paths.root_dir / "a/registry.list"
write(output)
write(stub, "// generated bla bla")
write(registry)
write_registry(registry)
with sut.manage(generated=(output,), stubs=(stub,), registry=registry) as renderer:
pass
assert not output.is_file()
assert not stub.is_file()
assert_file(registry, "")
assert_registry(registry)
assert pystache_renderer.mock_calls == []
def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
some_output = "some output"
registry = paths.root_dir / "a/registry.list"
write(output, some_output)
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
write_registry(registry, ("some/output.txt", hash(some_output), hash(some_output)))
pystache_renderer.render_name.side_effect = (some_output,)
@@ -149,7 +166,7 @@ def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
assert renderer.written == set()
assert_file(output, some_output)
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
assert_registry(registry, ("some/output.txt", hash(some_output), hash(some_output)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
@@ -157,12 +174,12 @@ def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
def test_managed_render_with_skipping_of_stub_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
stub = paths.root_dir / "some/stub.txt"
stub = paths.root_dir / "a/some/stub.txt"
some_output = "// generated some output"
some_processed_output = "// generated some processed output"
registry = paths.root_dir / "a/registry.list"
write(stub, some_processed_output)
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
write_registry(registry, ("some/stub.txt", hash(some_output), hash(some_processed_output)))
pystache_renderer.render_name.side_effect = (some_output,)
@@ -171,45 +188,45 @@ def test_managed_render_with_skipping_of_stub_file(pystache_renderer, sut):
assert renderer.written == set()
assert_file(stub, some_processed_output)
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
assert_registry(registry, ("some/stub.txt", hash(some_output), hash(some_processed_output)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
def test_managed_render_with_modified_generated_file(pystache_renderer, sut):
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
some_processed_output = "// some processed output"
registry = paths.root_dir / "a/registry.list"
write(output, "// something else")
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
write_registry(registry, ("some/output.txt", "whatever", hash(some_processed_output)))
with pytest.raises(render.Error):
sut.manage(generated=(output,), stubs=(), registry=registry)
def test_managed_render_with_modified_stub_file_still_marked_as_generated(pystache_renderer, sut):
stub = paths.root_dir / "some/stub.txt"
stub = paths.root_dir / "a/some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.root_dir / "a/registry.list"
write(stub, "// generated something else")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
write_registry(registry, ("some/stub.txt", "whatever", hash(some_processed_output)))
with pytest.raises(render.Error):
sut.manage(generated=(), stubs=(stub,), registry=registry)
def test_managed_render_with_modified_stub_file_not_marked_as_generated(pystache_renderer, sut):
stub = paths.root_dir / "some/stub.txt"
stub = paths.root_dir / "a/some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.root_dir / "a/registry.list"
write(stub, "// no more generated")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
write_registry(registry, ("some/stub.txt", "whatever", hash(some_processed_output)))
with sut.manage(generated=(), stubs=(stub,), registry=registry) as renderer:
pass
assert_file(registry, "")
assert_registry(registry)
class MyError(Exception):
@@ -220,45 +237,49 @@ def test_managed_render_exception_drops_written_and_inexsistent_from_registry(py
data = mock.Mock(spec=("template",))
text = "some text"
pystache_renderer.render_name.side_effect = (text,)
output = paths.root_dir / "some/output.txt"
registry = paths.root_dir / "x/registry.list"
output = paths.root_dir / "a/some/output.txt"
registry = paths.root_dir / "a/registry.list"
write(output, text)
write(paths.root_dir / "a")
write(paths.root_dir / "c")
write(registry, "a a a\n"
f"some/output.txt whatever {hash(text)}\n"
"b b b\n"
"c c c")
write(paths.root_dir / "a/a")
write(paths.root_dir / "a/c")
write_registry(registry,
"aaa",
("some/output.txt", "whatever", hash(text)),
"bbb",
"ccc")
with pytest.raises(MyError):
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
renderer.render(data, output)
raise MyError
assert_file(registry, "a a a\nc c c\n")
assert_registry(registry, "aaa", "ccc")
def test_managed_render_drops_inexsistent_from_registry(pystache_renderer, sut):
registry = paths.root_dir / "x/registry.list"
write(paths.root_dir / "a")
write(paths.root_dir / "c")
write(registry, f"a {hash('')} {hash('')}\n"
"b b b\n"
f"c {hash('')} {hash('')}")
registry = paths.root_dir / "a/registry.list"
write(paths.root_dir / "a/a")
write(paths.root_dir / "a/c")
write_registry(registry,
("a", hash(''), hash('')),
"bbb",
("c", hash(''), hash('')))
with sut.manage(generated=(), stubs=(), registry=registry):
pass
assert_file(registry, f"a {hash('')} {hash('')}\nc {hash('')} {hash('')}\n")
assert_registry(registry,
("a", hash(''), hash('')),
("c", hash(''), hash('')))
def test_managed_render_exception_does_not_erase(pystache_renderer, sut):
output = paths.root_dir / "some/output.txt"
stub = paths.root_dir / "some/stub.txt"
output = paths.root_dir / "a/some/output.txt"
stub = paths.root_dir / "a/some/stub.txt"
registry = paths.root_dir / "a/registry.list"
write(output)
write(stub, "// generated bla bla")
write(registry)
write_registry(registry)
with pytest.raises(MyError):
with sut.manage(generated=(output,), stubs=(stub,), registry=registry) as renderer:
@@ -288,11 +309,11 @@ def test_render_with_extensions(pystache_renderer, sut):
def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
some_output = "some output"
registry = paths.root_dir / "a/registry.list"
write(output, some_output)
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
write_registry(registry, ("some/output.txt", hash(some_output), hash(some_output)))
pystache_renderer.render_name.side_effect = (some_output,)
@@ -301,7 +322,7 @@ def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer
assert renderer.written == {output}
assert_file(output, some_output)
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
assert_registry(registry, ("some/output.txt", hash(some_output), hash(some_output)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
@@ -309,12 +330,12 @@ def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer
def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
stub = paths.root_dir / "some/stub.txt"
stub = paths.root_dir / "a/some/stub.txt"
some_output = "// generated some output"
some_processed_output = "// generated some processed output"
registry = paths.root_dir / "a/registry.list"
write(stub, some_processed_output)
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
write_registry(registry, ("some/stub.txt", hash(some_output), hash(some_processed_output)))
pystache_renderer.render_name.side_effect = (some_output,)
@@ -323,29 +344,29 @@ def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut
assert renderer.written == {stub}
assert_file(stub, some_output)
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_output)}\n")
assert_registry(registry, ("some/stub.txt", hash(some_output), hash(some_output)))
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=generator),
]
def test_managed_render_with_force_ignores_modified_generated_file(sut):
output = paths.root_dir / "some/output.txt"
output = paths.root_dir / "a/some/output.txt"
some_processed_output = "// some processed output"
registry = paths.root_dir / "a/registry.list"
write(output, "// something else")
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
write_registry(registry, ("some/output.txt", "whatever", hash(some_processed_output)))
with sut.manage(generated=(output,), stubs=(), registry=registry, force=True):
pass
def test_managed_render_with_force_ignores_modified_stub_file_still_marked_as_generated(sut):
stub = paths.root_dir / "some/stub.txt"
stub = paths.root_dir / "a/some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.root_dir / "a/registry.list"
write(stub, "// generated something else")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
write_registry(registry, ("some/stub.txt", "whatever", hash(some_processed_output)))
with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True):
pass