import sys
import os
import helpers
import json
import shutil
def write_csproj_prefix(ioWrapper):
ioWrapper.write('\n')
ioWrapper.write(' \n')
ioWrapper.write(' net8.0\n')
ioWrapper.write(' true\n')
ioWrapper.write(' bin\\n')
ioWrapper.write(
' false\n')
ioWrapper.write(' \n\n')
print('Script to generate stub file from a nuget package')
print(' Usage: python3 ' + sys.argv[0] +
' TEMPLATE NUGET_PACKAGE_NAME [VERSION=latest] [WORK_DIR=tempDir]')
print(' The script uses the dotnet cli, codeql cli, and dotnet format global tool')
print(' TEMPLATE should be either classlib or webapp, depending on the nuget package. For example, `Swashbuckle.AspNetCore.Swagger` should use `webapp` while `newtonsoft.json` should use `classlib`.')
if len(sys.argv) < 2:
print("\nPlease supply a template name.")
exit(1)
if len(sys.argv) < 3:
print("\nPlease supply a nuget package name.")
exit(1)
thisScript = sys.argv[0]
thisDir = os.path.abspath(os.path.dirname(thisScript))
template = sys.argv[1]
nuget = sys.argv[2]
# /input contains a dotnet project that's being extracted
workDir = os.path.abspath(helpers.get_argv(4, "tempDir"))
projectNameIn = "input"
projectDirIn = os.path.join(workDir, projectNameIn)
def run_cmd(cmd, msg="Failed to run command"):
helpers.run_cmd_cwd(cmd, workDir, msg)
# /output contains the output of the stub generation
outputDirName = "output"
outputDir = os.path.join(workDir, outputDirName)
# /output/raw contains the bqrs result from the query, the json equivalent
rawOutputDirName = "raw"
rawOutputDir = os.path.join(outputDir, rawOutputDirName)
os.makedirs(rawOutputDir)
# /output/output contains a dotnet project with the generated stubs
projectNameOut = "output"
projectDirOut = os.path.join(outputDir, projectNameOut)
# /db contains the extracted QL DB
dbName = 'db'
dbDir = os.path.join(workDir, dbName)
outputName = "stub"
outputFile = os.path.join(projectDirOut, outputName + '.cs')
bqrsFile = os.path.join(rawOutputDir, outputName + '.bqrs')
jsonFile = os.path.join(rawOutputDir, outputName + '.json')
version = helpers.get_argv(3, "latest")
print("\n* Creating new input project")
run_cmd(['dotnet', 'new', template, "-f", "net8.0", "--language", "C#", '--name',
projectNameIn, '--output', projectDirIn])
helpers.remove_files(projectDirIn, '.cs')
print("\n* Adding reference to package: " + nuget)
cmd = ['dotnet', 'add', projectDirIn, 'package', nuget]
if (version != "latest"):
cmd.append('--version')
cmd.append(version)
run_cmd(cmd)
sdk_version = '8.0.100'
print("\n* Creating new global.json file and setting SDK to " + sdk_version)
run_cmd(['dotnet', 'new', 'globaljson', '--force', '--sdk-version', sdk_version, '--output', workDir])
print("\n* Running stub generator")
helpers.run_cmd_cwd(['dotnet', 'run', '--project', thisDir + '/../../extractor/Semmle.Extraction.CSharp.DependencyStubGenerator/Semmle.Extraction.CSharp.DependencyStubGenerator.csproj'], projectDirIn)
print("\n* Creating new raw output project")
rawSrcOutputDirName = 'src'
rawSrcOutputDir = os.path.join(rawOutputDir, rawSrcOutputDirName)
run_cmd(['dotnet', 'new', template, "--language", "C#",
'--name', rawSrcOutputDirName, '--output', rawSrcOutputDir])
helpers.remove_files(rawSrcOutputDir, '.cs')
# copy each file from projectDirIn to rawSrcOutputDir
pathInfos = {}
codeqlStubsDir = os.path.join(projectDirIn, 'codeql_csharp_stubs')
for root, dirs, files in os.walk(codeqlStubsDir):
for file in files:
if file.endswith('.cs'):
path = os.path.join(root, file)
relPath, _ = os.path.splitext(os.path.relpath(path, codeqlStubsDir))
origDllPath = "/" + relPath + ".dll"
pathInfos[origDllPath] = os.path.join(rawSrcOutputDir, file)
shutil.copy2(path, rawSrcOutputDir)
print("\n --> Generated stub files: " + rawSrcOutputDir)
print("\n* Formatting files")
run_cmd(['dotnet', 'format', 'whitespace', rawSrcOutputDir])
print("\n --> Generated (formatted) stub files: " + rawSrcOutputDir)
print("\n* Processing project.assets.json to generate folder structure")
stubsDirName = 'stubs'
stubsDir = os.path.join(outputDir, stubsDirName)
os.makedirs(stubsDir)
frameworksDirName = '_frameworks'
frameworksDir = os.path.join(stubsDir, frameworksDirName)
frameworks = set()
copiedFiles = set()
assetsJsonFile = os.path.join(projectDirIn, 'obj', 'project.assets.json')
with open(assetsJsonFile) as json_data:
data = json.load(json_data)
if len(data['targets']) > 1:
print("ERROR: More than one target found in " + assetsJsonFile)
exit(1)
target = list(data['targets'].keys())[0]
print("Found target: " + target)
for package in data['targets'][target].keys():
parts = package.split('/')
name = parts[0]
version = parts[1]
packageDir = os.path.join(stubsDir, name, version)
if not os.path.exists(packageDir):
os.makedirs(packageDir)
print(' * Processing package: ' + name + '/' + version)
with open(os.path.join(packageDir, name + '.csproj'), 'a') as pf:
write_csproj_prefix(pf)
pf.write(' \n')
dlls = set()
if 'compile' in data['targets'][target][package]:
for dll in data['targets'][target][package]['compile']:
dlls.add(
(name + '/' + version + '/' + dll).lower())
if 'runtime' in data['targets'][target][package]:
for dll in data['targets'][target][package]['runtime']:
dlls.add((name + '/' + version + '/' + dll).lower())
for pathInfo in pathInfos:
for dll in dlls:
if pathInfo.lower().endswith(dll):
copiedFiles.add(pathInfo)
shutil.copy2(pathInfos[pathInfo], packageDir)
if 'dependencies' in data['targets'][target][package]:
for dependency in data['targets'][target][package]['dependencies'].keys():
depVersion = data['targets'][target][package]['dependencies'][dependency]
pf.write(' \n')
if 'frameworkReferences' in data['targets'][target][package]:
if not os.path.exists(frameworksDir):
os.makedirs(frameworksDir)
for framework in data['targets'][target][package]['frameworkReferences']:
frameworks.add(framework)
frameworkDir = os.path.join(
frameworksDir, framework)
if not os.path.exists(frameworkDir):
os.makedirs(frameworkDir)
pf.write(' \n')
pf.write(' \n')
pf.write(' \n')
pf.write('\n')
# Processing references frameworks
for framework in frameworks:
with open(os.path.join(frameworksDir, framework, framework + '.csproj'), 'a') as pf:
write_csproj_prefix(pf)
pf.write(' \n')
pf.write(
' \n')
pf.write(' \n')
pf.write('\n')
for pathInfo in pathInfos:
if framework.lower() + '.ref' in pathInfo.lower():
copiedFiles.add(pathInfo)
shutil.copy2(pathInfos[pathInfo], os.path.join(
frameworksDir, framework))
# Processing assemblies in Microsoft.NETCore.App.Ref
frameworkDir = os.path.join(frameworksDir, 'Microsoft.NETCore.App')
if not os.path.exists(frameworkDir):
os.makedirs(frameworkDir)
with open(os.path.join(frameworksDir, 'Microsoft.NETCore.App', 'Microsoft.NETCore.App.csproj'), 'a') as pf:
write_csproj_prefix(pf)
pf.write('\n')
for pathInfo in pathInfos:
if 'microsoft.netcore.app.ref/' in pathInfo.lower():
copiedFiles.add(pathInfo)
shutil.copy2(pathInfos[pathInfo], frameworkDir)
for pathInfo in pathInfos:
if pathInfo not in copiedFiles:
print('Not copied to nuget or framework folder: ' + pathInfo)
othersDir = os.path.join(stubsDir, 'others')
if not os.path.exists(othersDir):
os.makedirs(othersDir)
shutil.copy2(pathInfos[pathInfo], othersDir)
print("\n --> Generated structured stub files: " + stubsDir)
exit(0)