mirror of
https://github.com/github/codeql.git
synced 2025-12-17 01:03:14 +01:00
2cd58817d7
This is a developer QoL improvement, where running codegen will skip writing (and especially formatting) any files that were not changed. **Why?** While code generation in itself was pretty much instant, QL formatting of generated code was starting to take a long time. This made unconditionally running codegen quite annoying, for example before each test run as part of an IDE workflow or as part of the pre-commit hook. **How?** This was not completely straightforward as we could not work with the contents of the file prior to code generation as that was already post-processed by the QL formatting, so we had no chance of comparing the output of template rendering with that. We therefore store the hashes of the files _prior_ to QL formatting in a checked-in file (`swift/ql/.generated.list`). We can therefore load those hashes at the beginning of code generation, use them to compare the template rendering output and update them in this special registry file. **What else?** We also extend this mechanism to detect accidental modification of generated files in a more robust way. Before this patch, we were doing it with a rough regexp based heuristic. Now, we just store the hashes of the files _after_ QL formatting in the same checked file, so we can check that and stop generation if a generated file was modified, or a stub was modified without removing the `// generated` header.
215 lines
7.6 KiB
Python
215 lines
7.6 KiB
Python
import sys
|
|
|
|
import pytest
|
|
|
|
from swift.codegen.test.utils import *
|
|
|
|
import hashlib
|
|
|
|
|
|
@pytest.fixture
|
|
def pystache_renderer_cls():
|
|
with mock.patch("pystache.Renderer") as ret:
|
|
yield ret
|
|
|
|
|
|
@pytest.fixture
|
|
def pystache_renderer(pystache_renderer_cls):
|
|
ret = mock.Mock()
|
|
pystache_renderer_cls.return_value = ret
|
|
return ret
|
|
|
|
|
|
@pytest.fixture
|
|
def sut(pystache_renderer):
|
|
return render.Renderer(paths.swift_dir)
|
|
|
|
|
|
def assert_file(file, text):
|
|
with open(file) as inp:
|
|
assert inp.read() == text
|
|
|
|
|
|
def hash(text):
|
|
h = hashlib.sha256()
|
|
h.update(text.encode())
|
|
return h.hexdigest()
|
|
|
|
|
|
def test_constructor(pystache_renderer_cls, sut):
|
|
pystache_init, = pystache_renderer_cls.mock_calls
|
|
assert set(pystache_init.kwargs) == {'search_dirs', 'escape'}
|
|
assert pystache_init.kwargs['search_dirs'] == str(paths.templates_dir)
|
|
an_object = object()
|
|
assert pystache_init.kwargs['escape'](an_object) is an_object
|
|
|
|
|
|
def test_render(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template",))
|
|
text = "some text"
|
|
pystache_renderer.render_name.side_effect = (text,)
|
|
output = paths.swift_dir / "some/output.txt"
|
|
sut.render(data, output)
|
|
|
|
assert_file(output, text)
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
|
|
]
|
|
|
|
|
|
def test_managed_render(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template",))
|
|
text = "some text"
|
|
pystache_renderer.render_name.side_effect = (text,)
|
|
output = paths.swift_dir / "some/output.txt"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(registry)
|
|
|
|
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
|
|
renderer.render(data, output)
|
|
assert renderer.written == {output}
|
|
assert_file(output, text)
|
|
|
|
assert_file(registry, f"some/output.txt {hash(text)} {hash(text)}\n")
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
|
|
]
|
|
|
|
|
|
def test_managed_render_with_post_processing(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template",))
|
|
text = "some text"
|
|
postprocessed_text = "some other text"
|
|
pystache_renderer.render_name.side_effect = (text,)
|
|
output = paths.swift_dir / "some/output.txt"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(registry)
|
|
|
|
with sut.manage(generated=(), stubs=(), registry=registry) as renderer:
|
|
renderer.render(data, output)
|
|
assert renderer.written == {output}
|
|
assert_file(output, text)
|
|
write(output, postprocessed_text)
|
|
|
|
assert_file(registry, f"some/output.txt {hash(text)} {hash(postprocessed_text)}\n")
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
|
|
]
|
|
|
|
|
|
def test_managed_render_with_erasing(pystache_renderer, sut):
|
|
output = paths.swift_dir / "some/output.txt"
|
|
stub = paths.swift_dir / "some/stub.txt"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(output)
|
|
write(stub, "// generated bla bla")
|
|
write(registry)
|
|
|
|
with sut.manage(generated=(output,), stubs=(stub,), registry=registry) as renderer:
|
|
pass
|
|
|
|
assert not output.is_file()
|
|
assert not stub.is_file()
|
|
assert_file(registry, "")
|
|
assert pystache_renderer.mock_calls == []
|
|
|
|
|
|
def test_managed_render_with_skipping_of_generated_file(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template",))
|
|
output = paths.swift_dir / "some/output.txt"
|
|
some_output = "some output"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(output, some_output)
|
|
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
|
|
|
|
pystache_renderer.render_name.side_effect = (some_output,)
|
|
|
|
with sut.manage(generated=(output,), stubs=(), registry=registry) as renderer:
|
|
renderer.render(data, output)
|
|
assert renderer.written == set()
|
|
assert_file(output, some_output)
|
|
|
|
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
|
|
]
|
|
|
|
|
|
def test_managed_render_with_skipping_of_stub_file(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template",))
|
|
stub = paths.swift_dir / "some/stub.txt"
|
|
some_output = "// generated some output"
|
|
some_processed_output = "// generated some processed output"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(stub, some_processed_output)
|
|
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
|
|
|
|
pystache_renderer.render_name.side_effect = (some_output,)
|
|
|
|
with sut.manage(generated=(), stubs=(stub,), registry=registry) as renderer:
|
|
renderer.render(data, stub)
|
|
assert renderer.written == set()
|
|
assert_file(stub, some_processed_output)
|
|
|
|
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
|
|
]
|
|
|
|
|
|
def test_managed_render_with_modified_generated_file(pystache_renderer, sut):
|
|
output = paths.swift_dir / "some/output.txt"
|
|
some_processed_output = "// some processed output"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(output, "// something else")
|
|
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
|
|
|
|
with pytest.raises(render.Error):
|
|
sut.manage(generated=(output,), stubs=(), registry=registry)
|
|
|
|
|
|
def test_managed_render_with_modified_stub_file_still_marked_as_generated(pystache_renderer, sut):
|
|
stub = paths.swift_dir / "some/stub.txt"
|
|
some_processed_output = "// generated some processed output"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(stub, "// generated something else")
|
|
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
|
|
|
|
with pytest.raises(render.Error):
|
|
sut.manage(generated=(), stubs=(stub,), registry=registry)
|
|
|
|
|
|
def test_managed_render_with_modified_stub_file_not_marked_as_generated(pystache_renderer, sut):
|
|
stub = paths.swift_dir / "some/stub.txt"
|
|
some_processed_output = "// generated some processed output"
|
|
registry = paths.swift_dir / "a/registry.list"
|
|
write(stub, "// no more generated")
|
|
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
|
|
|
|
with sut.manage(generated=(), stubs=(stub,), registry=registry) as renderer:
|
|
pass
|
|
|
|
assert_file(registry, "")
|
|
|
|
|
|
def test_render_with_extensions(pystache_renderer, sut):
|
|
data = mock.Mock(spec=("template", "extensions"))
|
|
data.template = "test_template"
|
|
data.extensions = ["foo", "bar", "baz"]
|
|
output = pathlib.Path("my", "test", "file")
|
|
expected_outputs = [pathlib.Path("my", "test", p) for p in ("file.foo", "file.bar", "file.baz")]
|
|
rendered = [f"text{i}" for i in range(len(expected_outputs))]
|
|
pystache_renderer.render_name.side_effect = rendered
|
|
sut.render(data, output)
|
|
expected_templates = ["test_template_foo", "test_template_bar", "test_template_baz"]
|
|
assert pystache_renderer.mock_calls == [
|
|
mock.call.render_name(t, data, generator=paths.exe_file.relative_to(paths.swift_dir))
|
|
for t in expected_templates
|
|
]
|
|
for expected_output, expected_contents in zip(expected_outputs, rendered):
|
|
assert_file(expected_output, expected_contents)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(pytest.main([__file__] + sys.argv[1:]))
|