diff --git a/swift/codegen/codegen.py b/swift/codegen/codegen.py index b2da3678b1c..58160e9b998 100755 --- a/swift/codegen/codegen.py +++ b/swift/codegen/codegen.py @@ -42,6 +42,8 @@ def _parse_args() -> argparse.Namespace: help="output directory for generated C++ files, required if trap or cpp is provided to --generate") p.add_argument("--generated-registry", type=_abspath, default=paths.swift_dir / "ql/.generated.list", help="registry file containing information about checked-in generated code") + p.add_argument("--force", "-f", action="store_true", + help="generate all files without skipping unchanged files and overwriting modified ones") return p.parse_args() diff --git a/swift/codegen/generators/qlgen.py b/swift/codegen/generators/qlgen.py index c1650974796..b6b50498028 100755 --- a/swift/codegen/generators/qlgen.py +++ b/swift/codegen/generators/qlgen.py @@ -302,7 +302,8 @@ def generate(opts, renderer): imports = {} - with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry) as renderer: + with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry, + force=opts.force) as renderer: db_classes = [cls for cls in classes.values() if not cls.ipa] renderer.render(ql.DbClasses(db_classes), out / "Raw.qll") diff --git a/swift/codegen/lib/render.py b/swift/codegen/lib/render.py index 8294ecfda2b..65014a42fa2 100644 --- a/swift/codegen/lib/render.py +++ b/swift/codegen/lib/render.py @@ -59,8 +59,8 @@ class Renderer: log.debug(f"{mnemonic}: generated {output.name}") def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path], - registry: pathlib.Path) -> "RenderManager": - return RenderManager(self._swift_dir, generated, stubs, registry) + registry: pathlib.Path, force: bool = False) -> "RenderManager": + return RenderManager(self._swift_dir, generated, stubs, registry, force) class RenderManager(Renderer): @@ -87,9 +87,10 @@ class RenderManager(Renderer): def __init__(self, swift_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path], - registry: pathlib.Path): + registry: pathlib.Path, force: bool = False): super().__init__(swift_dir) self._registry_path = registry + self._force = force self._hashes = {} self.written = set() self._existing = set() @@ -132,10 +133,13 @@ class RenderManager(Renderer): for f in generated: self._existing.add(f) rel_path = self._get_path(f) - if rel_path not in self._hashes: + if self._force: + pass + elif rel_path not in self._hashes: log.warning(f"{rel_path} marked as generated but absent from the registry") elif self._hashes[rel_path].post != self._hash_file(f): - raise Error(f"{rel_path} is generated but was modified, please revert the file") + raise Error(f"{rel_path} is generated but was modified, please revert the file " + "or pass --force to overwrite") def _process_stubs(self, stubs: typing.Iterable[pathlib.Path]): for f in stubs: @@ -144,10 +148,13 @@ class RenderManager(Renderer): self._hashes.pop(rel_path, None) continue self._existing.add(f) - if rel_path not in self._hashes: + if self._force: + pass + elif rel_path not in self._hashes: log.warning(f"{rel_path} marked as stub but absent from the registry") elif self._hashes[rel_path].post != self._hash_file(f): - raise Error(f"{rel_path} is a stub marked as generated, but it was modified") + raise Error(f"{rel_path} is a stub marked as generated, but it was modified, " + "please remove the `// generated` header, revert the file or pass --force to overwrite it") @staticmethod def is_customized_stub(file: pathlib.Path) -> bool: @@ -171,6 +178,8 @@ class RenderManager(Renderer): return h.hexdigest() def _load_registry(self): + if self._force: + return try: with open(self._registry_path) as reg: for line in reg: diff --git a/swift/codegen/test/test_qlgen.py b/swift/codegen/test/test_qlgen.py index 0db445f07b9..3b2598ed8fd 100644 --- a/swift/codegen/test/test_qlgen.py +++ b/swift/codegen/test/test_qlgen.py @@ -48,6 +48,7 @@ def qlgen_opts(opts): opts.generated_registry = generated_registry_path() opts.ql_format = True opts.swift_dir = paths.swift_dir + opts.force = False return opts @@ -430,7 +431,9 @@ def test_format_error(opts, generate, render_manager, run_mock): generate([schema.Class('A')]) -def test_manage_parameters(opts, generate, renderer): +@pytest.mark.parametrize("force", [False, True]) +def test_manage_parameters(opts, generate, renderer, force): + opts.force = force ql_a = opts.ql_output / "A.qll" ql_b = opts.ql_output / "B.qll" stub_a = opts.ql_stub_output / "A.qll" @@ -448,7 +451,7 @@ def test_manage_parameters(opts, generate, renderer): generate([schema.Class('A')]) assert renderer.mock_calls == [ mock.call.manage(generated={ql_a, ql_b, test_a, test_b, import_file()}, stubs={stub_a, stub_b}, - registry=opts.generated_registry) + registry=opts.generated_registry, force=force) ] diff --git a/swift/codegen/test/test_render.py b/swift/codegen/test/test_render.py index cb249046b7b..950ca385e2f 100644 --- a/swift/codegen/test/test_render.py +++ b/swift/codegen/test/test_render.py @@ -75,6 +75,7 @@ def test_managed_render(pystache_renderer, sut): mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)), ] + def test_managed_render_with_no_registry(pystache_renderer, sut): data = mock.Mock(spec=("template",)) text = "some text" @@ -266,5 +267,70 @@ def test_render_with_extensions(pystache_renderer, sut): assert_file(expected_output, expected_contents) +def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer, sut): + data = mock.Mock(spec=("template",)) + output = paths.swift_dir / "some/output.txt" + some_output = "some output" + registry = paths.swift_dir / "a/registry.list" + write(output, some_output) + write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n") + + pystache_renderer.render_name.side_effect = (some_output,) + + with sut.manage(generated=(output,), stubs=(), registry=registry, force=True) as renderer: + renderer.render(data, output) + assert renderer.written == {output} + assert_file(output, some_output) + + assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n") + assert pystache_renderer.mock_calls == [ + mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)), + ] + + +def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut): + data = mock.Mock(spec=("template",)) + stub = paths.swift_dir / "some/stub.txt" + some_output = "// generated some output" + some_processed_output = "// generated some processed output" + registry = paths.swift_dir / "a/registry.list" + write(stub, some_processed_output) + write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n") + + pystache_renderer.render_name.side_effect = (some_output,) + + with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True) as renderer: + renderer.render(data, stub) + assert renderer.written == {stub} + assert_file(stub, some_output) + + assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_output)}\n") + assert pystache_renderer.mock_calls == [ + mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)), + ] + + +def test_managed_render_with_force_ignores_modified_generated_file(sut): + output = paths.swift_dir / "some/output.txt" + some_processed_output = "// some processed output" + registry = paths.swift_dir / "a/registry.list" + write(output, "// something else") + write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n") + + with sut.manage(generated=(output,), stubs=(), registry=registry, force=True): + pass + + +def test_managed_render_with_force_ignores_modified_stub_file_still_marked_as_generated(sut): + stub = paths.swift_dir / "some/stub.txt" + some_processed_output = "// generated some processed output" + registry = paths.swift_dir / "a/registry.list" + write(stub, "// generated something else") + write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n") + + with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True): + pass + + if __name__ == '__main__': sys.exit(pytest.main([__file__] + sys.argv[1:])) diff --git a/swift/ql/.generated.list b/swift/ql/.generated.list index f0475a92903..597e435a0f7 100644 --- a/swift/ql/.generated.list +++ b/swift/ql/.generated.list @@ -387,7 +387,7 @@ ql/lib/codeql/swift/generated/Synth.qll cc7285c43e6c9f47ab67047916d232ad2078443b ql/lib/codeql/swift/generated/SynthConstructors.qll 0ff9cfcd64e7701003091f366ec903ec1bf82ec8385ee683b6e7b4e189033b11 0ff9cfcd64e7701003091f366ec903ec1bf82ec8385ee683b6e7b4e189033b11 ql/lib/codeql/swift/generated/UnknownFile.qll 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6 ql/lib/codeql/swift/generated/UnknownLocation.qll e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882 e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882 -ql/lib/codeql/swift/generated/UnspecifiedElement.qll a3a73f53c492adc6655fb88b40b1bd9b2c9d365fc5c019b3233c01d6110fb3f2 ba77cd5272cffd1d3aad8bea69786b97aec0c93a4b59a070d621fe2d21c2e90c +ql/lib/codeql/swift/generated/UnspecifiedElement.qll dbc6ca4018012977b26ca184a88044c55b0661e3998cd14d46295b62a8d69625 184c9a0ce18c2ac881943b0fb400613d1401ed1d5564f90716b6c310ba5afe71 ql/lib/codeql/swift/generated/decl/AbstractFunctionDecl.qll 8255b24dddda83e8a7dee9d69a4cf9883b5a7ae43676d7242b5aab5169f68982 407c7d63681fb03ad6cb4ea3c2b04be7ccb5ddbe655a8aec4219eb3799bc36e8 ql/lib/codeql/swift/generated/decl/AbstractStorageDecl.qll 66147ad36cefce974b4ae0f3e84569bd6742ea2f3e842c3c04e6e5cbd17e7928 ce7c2347e2dfe0b141db103ccb8e56a61d286476c201aebe6a275edd7fca2c0f ql/lib/codeql/swift/generated/decl/AbstractTypeParamDecl.qll 1e268b00d0f2dbbd85aa70ac206c5e4a4612f06ba0091e5253483635f486ccf9 5479e13e99f68f1f347283535f8098964f7fd4a34326ff36ad5711b2de1ab0d0 diff --git a/swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll b/swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll index 2658e63164a..c37d4d35515 100644 --- a/swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll +++ b/swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll @@ -5,9 +5,6 @@ import codeql.swift.elements.Element import codeql.swift.elements.ErrorElement module Generated { - /** - * bla - */ class UnspecifiedElement extends Synth::TUnspecifiedElement, ErrorElement { override string getAPrimaryQlClass() { result = "UnspecifiedElement" } diff --git a/swift/schema.py b/swift/schema.py index ffd9e15b58b..7654f9e2ec8 100644 --- a/swift/schema.py +++ b/swift/schema.py @@ -40,7 +40,6 @@ class ErrorElement(Locatable): @use_for_null class UnspecifiedElement(ErrorElement): - """bla""" parent: optional[Element] property: string index: optional[int]