hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-04-30 19:26:02 +02:00
Code Issues Packages Projects Releases Wiki Activity

Swift: add --force to codegen

Browse Source
This commit is contained in:
Paolo Tranquilli
2022-11-30 14:11:24 +01:00
parent d6aad13a98
commit bb3aa9e908
8 changed files with 92 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
swift/codegen/codegen.py
View File

@@ -42,6 +42,8 @@ def _parse_args() -> argparse.Namespace:
help="output directory for generated C++ files, required if trap or cpp is provided to --generate")
p.add_argument("--generated-registry", type=_abspath, default=paths.swift_dir / "ql/.generated.list",
help="registry file containing information about checked-in generated code")
p.add_argument("--force", "-f", action="store_true",
help="generate all files without skipping unchanged files and overwriting modified ones")
return p.parse_args()

3
swift/codegen/generators/qlgen.py
View File

@@ -302,7 +302,8 @@ def generate(opts, renderer):
imports = {}
with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry) as renderer:
with renderer.manage(generated=generated, stubs=stubs, registry=opts.generated_registry,
force=opts.force) as renderer:
db_classes = [cls for cls in classes.values() if not cls.ipa]
renderer.render(ql.DbClasses(db_classes), out / "Raw.qll")

23
swift/codegen/lib/render.py
View File

@@ -59,8 +59,8 @@ class Renderer:
log.debug(f"{mnemonic}: generated {output.name}")
def manage(self, generated: typing.Iterable[pathlib.Path], stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path) -> "RenderManager":
return RenderManager(self._swift_dir, generated, stubs, registry)
registry: pathlib.Path, force: bool = False) -> "RenderManager":
return RenderManager(self._swift_dir, generated, stubs, registry, force)
class RenderManager(Renderer):
@@ -87,9 +87,10 @@ class RenderManager(Renderer):
def __init__(self, swift_dir: pathlib.Path, generated: typing.Iterable[pathlib.Path],
stubs: typing.Iterable[pathlib.Path],
registry: pathlib.Path):
registry: pathlib.Path, force: bool = False):
super().__init__(swift_dir)
self._registry_path = registry
self._force = force
self._hashes = {}
self.written = set()
self._existing = set()
@@ -132,10 +133,13 @@ class RenderManager(Renderer):
for f in generated:
self._existing.add(f)
rel_path = self._get_path(f)
if rel_path not in self._hashes:
if self._force:
pass
elif rel_path not in self._hashes:
log.warning(f"{rel_path} marked as generated but absent from the registry")
elif self._hashes[rel_path].post != self._hash_file(f):
raise Error(f"{rel_path} is generated but was modified, please revert the file")
raise Error(f"{rel_path} is generated but was modified, please revert the file "
"or pass --force to overwrite")
def _process_stubs(self, stubs: typing.Iterable[pathlib.Path]):
for f in stubs:
@@ -144,10 +148,13 @@ class RenderManager(Renderer):
self._hashes.pop(rel_path, None)
continue
self._existing.add(f)
if rel_path not in self._hashes:
if self._force:
pass
elif rel_path not in self._hashes:
log.warning(f"{rel_path} marked as stub but absent from the registry")
elif self._hashes[rel_path].post != self._hash_file(f):
raise Error(f"{rel_path} is a stub marked as generated, but it was modified")
raise Error(f"{rel_path} is a stub marked as generated, but it was modified, "
"please remove the `// generated` header, revert the file or pass --force to overwrite it")
@staticmethod
def is_customized_stub(file: pathlib.Path) -> bool:
@@ -171,6 +178,8 @@ class RenderManager(Renderer):
return h.hexdigest()
def _load_registry(self):
if self._force:
return
try:
with open(self._registry_path) as reg:
for line in reg:

7
swift/codegen/test/test_qlgen.py
View File

@@ -48,6 +48,7 @@ def qlgen_opts(opts):
opts.generated_registry = generated_registry_path()
opts.ql_format = True
opts.swift_dir = paths.swift_dir
opts.force = False
return opts
@@ -430,7 +431,9 @@ def test_format_error(opts, generate, render_manager, run_mock):
generate([schema.Class('A')])
def test_manage_parameters(opts, generate, renderer):
@pytest.mark.parametrize("force", [False, True])
def test_manage_parameters(opts, generate, renderer, force):
opts.force = force
ql_a = opts.ql_output / "A.qll"
ql_b = opts.ql_output / "B.qll"
stub_a = opts.ql_stub_output / "A.qll"
@@ -448,7 +451,7 @@ def test_manage_parameters(opts, generate, renderer):
generate([schema.Class('A')])
assert renderer.mock_calls == [
mock.call.manage(generated={ql_a, ql_b, test_a, test_b, import_file()}, stubs={stub_a, stub_b},
registry=opts.generated_registry)
registry=opts.generated_registry, force=force)
]

66
swift/codegen/test/test_render.py
View File

@@ -75,6 +75,7 @@ def test_managed_render(pystache_renderer, sut):
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
]
def test_managed_render_with_no_registry(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
text = "some text"
@@ -266,5 +267,70 @@ def test_render_with_extensions(pystache_renderer, sut):
assert_file(expected_output, expected_contents)
def test_managed_render_with_force_not_skipping_generated_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
output = paths.swift_dir / "some/output.txt"
some_output = "some output"
registry = paths.swift_dir / "a/registry.list"
write(output, some_output)
write(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
pystache_renderer.render_name.side_effect = (some_output,)
with sut.manage(generated=(output,), stubs=(), registry=registry, force=True) as renderer:
renderer.render(data, output)
assert renderer.written == {output}
assert_file(output, some_output)
assert_file(registry, f"some/output.txt {hash(some_output)} {hash(some_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
]
def test_managed_render_with_force_not_skipping_stub_file(pystache_renderer, sut):
data = mock.Mock(spec=("template",))
stub = paths.swift_dir / "some/stub.txt"
some_output = "// generated some output"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
write(stub, some_processed_output)
write(registry, f"some/stub.txt {hash(some_output)} {hash(some_processed_output)}\n")
pystache_renderer.render_name.side_effect = (some_output,)
with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True) as renderer:
renderer.render(data, stub)
assert renderer.written == {stub}
assert_file(stub, some_output)
assert_file(registry, f"some/stub.txt {hash(some_output)} {hash(some_output)}\n")
assert pystache_renderer.mock_calls == [
mock.call.render_name(data.template, data, generator=paths.exe_file.relative_to(paths.swift_dir)),
]
def test_managed_render_with_force_ignores_modified_generated_file(sut):
output = paths.swift_dir / "some/output.txt"
some_processed_output = "// some processed output"
registry = paths.swift_dir / "a/registry.list"
write(output, "// something else")
write(registry, f"some/output.txt whatever {hash(some_processed_output)}\n")
with sut.manage(generated=(output,), stubs=(), registry=registry, force=True):
pass
def test_managed_render_with_force_ignores_modified_stub_file_still_marked_as_generated(sut):
stub = paths.swift_dir / "some/stub.txt"
some_processed_output = "// generated some processed output"
registry = paths.swift_dir / "a/registry.list"
write(stub, "// generated something else")
write(registry, f"some/stub.txt whatever {hash(some_processed_output)}\n")
with sut.manage(generated=(), stubs=(stub,), registry=registry, force=True):
pass
if __name__ == '__main__':
sys.exit(pytest.main([__file__] + sys.argv[1:]))

2
swift/ql/.generated.list
View File

@@ -385,7 +385,7 @@ ql/lib/codeql/swift/generated/Synth.qll 90df85be365c89c3c2e22041ee7dc9dd2ad9194b
ql/lib/codeql/swift/generated/SynthConstructors.qll 5c91f09bd82728651ed61f498704e0f62847788fa986dec5e674d81f294076c7 5c91f09bd82728651ed61f498704e0f62847788fa986dec5e674d81f294076c7
ql/lib/codeql/swift/generated/UnknownFile.qll 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6 0fcf9beb8de79440bcdfff4bb6ab3dd139bd273e6c32754e05e6a632651e85f6
ql/lib/codeql/swift/generated/UnknownLocation.qll e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882 e50efefa02a0ec1ff635a00951b5924602fc8cab57e5756e4a039382c69d3882
ql/lib/codeql/swift/generated/UnspecifiedElement.qll a3a73f53c492adc6655fb88b40b1bd9b2c9d365fc5c019b3233c01d6110fb3f2 ba77cd5272cffd1d3aad8bea69786b97aec0c93a4b59a070d621fe2d21c2e90c
ql/lib/codeql/swift/generated/UnspecifiedElement.qll dbc6ca4018012977b26ca184a88044c55b0661e3998cd14d46295b62a8d69625 184c9a0ce18c2ac881943b0fb400613d1401ed1d5564f90716b6c310ba5afe71
ql/lib/codeql/swift/generated/decl/AbstractFunctionDecl.qll 8255b24dddda83e8a7dee9d69a4cf9883b5a7ae43676d7242b5aab5169f68982 407c7d63681fb03ad6cb4ea3c2b04be7ccb5ddbe655a8aec4219eb3799bc36e8
ql/lib/codeql/swift/generated/decl/AbstractStorageDecl.qll 66147ad36cefce974b4ae0f3e84569bd6742ea2f3e842c3c04e6e5cbd17e7928 ce7c2347e2dfe0b141db103ccb8e56a61d286476c201aebe6a275edd7fca2c0f
ql/lib/codeql/swift/generated/decl/AbstractTypeParamDecl.qll 1e268b00d0f2dbbd85aa70ac206c5e4a4612f06ba0091e5253483635f486ccf9 5479e13e99f68f1f347283535f8098964f7fd4a34326ff36ad5711b2de1ab0d0

3
swift/ql/lib/codeql/swift/generated/UnspecifiedElement.qll
View File

@@ -5,9 +5,6 @@ import codeql.swift.elements.Element
import codeql.swift.elements.ErrorElement
module Generated {
/**
* bla
*/
class UnspecifiedElement extends Synth::TUnspecifiedElement, ErrorElement {
override string getAPrimaryQlClass() { result = "UnspecifiedElement" }

1
swift/schema.py
View File

@@ -40,7 +40,6 @@ class ErrorElement(Locatable):
@use_for_null
class UnspecifiedElement(ErrorElement):
"""bla"""
parent: optional[Element]
property: string
index: optional[int]