mirror of
https://github.com/hohn/codeql-info.git
synced 2025-12-16 20:53:04 +01:00
648 lines
44 KiB
HTML
648 lines
44 KiB
HTML
<!DOCTYPE html>
|
||
|
||
<html lang="en" data-content_root="../">
|
||
<head>
|
||
<meta charset="utf-8" />
|
||
<meta name="viewport" content="width=device-width, initial-scale=1.0" /><meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
|
||
<title>Creating CodeQL databases — CodeQL</title>
|
||
<link rel="stylesheet" type="text/css" href="../_static/pygments.css?v=fa44fd50" />
|
||
<link rel="stylesheet" type="text/css" href="../_static/alabaster.css?v=93459777" />
|
||
<script src="../_static/documentation_options.js?v=5929fcd5"></script>
|
||
<script src="../_static/doctools.js?v=888ff710"></script>
|
||
<script src="../_static/sphinx_highlight.js?v=dc90522c"></script>
|
||
<link rel="icon" href="../_static/favicon.ico"/>
|
||
<link rel="index" title="Index" href="../genindex.html" />
|
||
<link rel="search" title="Search" href="../search.html" />
|
||
<link rel="next" title="Extractor options" href="extractor-options.html" />
|
||
<link rel="prev" title="Getting started with the CodeQL CLI" href="getting-started-with-the-codeql-cli.html" />
|
||
|
||
<title>CodeQL docs</title>
|
||
<meta name="viewport" content="width=device-width, initial-scale=1" />
|
||
<link rel="stylesheet" href="../_static/custom.css" type="text/css" />
|
||
<link rel="stylesheet" href="../_static/primer.css" type="text/css" />
|
||
|
||
|
||
</head><body>
|
||
<header class="Header">
|
||
<div class="Header-item--full">
|
||
<a href="https://codeql.github.com/docs" class="Header-link f2 d-flex flex-items-center">
|
||
<!-- <%= octicon "mark-github", class: "mr-2", height: 32 %> -->
|
||
<svg height="32" class="octicon octicon-mark-github mr-2" viewBox="0 0 16 16" version="1.1" width="32"
|
||
aria-hidden="true">
|
||
<path fill-rule="evenodd"
|
||
d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0 0 16 8c0-4.42-3.58-8-8-8z">
|
||
</path>
|
||
</svg>
|
||
<span class="hide-sm">CodeQL documentation</span>
|
||
</a>
|
||
</div>
|
||
<div class="Header-item hide-sm hide-md">
|
||
<script src="https://addsearch.com/js/?key=93b4d287e2fc079a4089412b669785d5&categories=!0xhelp.semmle.com,0xcodeql.github.com,1xdocs,1xcodeql-standard-libraries,1xcodeql-query-help"></script>
|
||
</div>
|
||
<div class="Header-item">
|
||
|
||
<details class="dropdown details-reset details-overlay d-inline-block">
|
||
<summary class="btn bg-gray-dark text-white border" aria-haspopup="true">
|
||
CodeQL resources
|
||
<div class="dropdown-caret"></div>
|
||
</summary>
|
||
|
||
<ul class="dropdown-menu dropdown-menu-se dropdown-menu-dark">
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/codeql-overview">CodeQL overview</a></li>
|
||
<li class="dropdown-divider" role="separator"></li>
|
||
<div class="dropdown-header">
|
||
CodeQL tools
|
||
</div>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/codeql-for-visual-studio-code">CodeQL for VS Code</a>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/codeql-cli">CodeQL CLI</a>
|
||
</li>
|
||
<li class="dropdown-divider" role="separator"></li>
|
||
<div class="dropdown-header">
|
||
CodeQL guides
|
||
</div>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/writing-codeql-queries">Writing CodeQL queries</a></li>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/codeql-language-guides">CodeQL language guides</a>
|
||
<li class="dropdown-divider" role="separator"></li>
|
||
<div class="dropdown-header">
|
||
Reference docs
|
||
</div>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/docs/ql-language-reference/">QL language
|
||
reference</a>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/codeql-standard-libraries">CodeQL
|
||
standard-libraries</a>
|
||
<li><a class="dropdown-item" href="https://codeql.github.com/codeql-query-help">CodeQL
|
||
query help</a>
|
||
<li class="dropdown-divider" role="separator"></li>
|
||
<div class="dropdown-header">
|
||
Source files
|
||
</div>
|
||
<li><a class="dropdown-item" href="https://github.com/github/codeql">CodeQL repository</a>
|
||
</ul>
|
||
</details>
|
||
|
||
</div>
|
||
|
||
</header>
|
||
<main class="bg-gray-light clearfix">
|
||
<nav class="SideNav position-sticky top-0 col-lg-3 col-md-3 float-left p-4 hide-sm hide-md overflow-y-auto">
|
||
|
||
<ul class="current">
|
||
<li class="toctree-l1"><a class="reference internal" href="../codeql-overview/index.html">CodeQL overview</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../codeql-for-visual-studio-code/index.html">CodeQL for Visual Studio Code</a></li>
|
||
<li class="toctree-l1 current"><a class="reference internal" href="index.html">CodeQL CLI</a><ul class="current">
|
||
<li class="toctree-l2 current"><a class="reference internal" href="using-the-codeql-cli.html">Using the CodeQL CLI</a><ul class="current">
|
||
<li class="toctree-l3"><a class="reference internal" href="about-the-codeql-cli.html">About the CodeQL CLI</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="getting-started-with-the-codeql-cli.html">Getting started with the CodeQL CLI</a></li>
|
||
<li class="toctree-l3 current"><a class="current reference internal" href="#">Creating CodeQL databases</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="extractor-options.html">Extractor options</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="analyzing-databases-with-the-codeql-cli.html">Analyzing databases with the CodeQL CLI</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="upgrading-codeql-databases.html">Upgrading CodeQL databases</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="using-custom-queries-with-the-codeql-cli.html">Using custom queries with the CodeQL CLI</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="creating-codeql-query-suites.html">Creating CodeQL query suites</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="testing-custom-queries.html">Testing custom queries</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="testing-query-help-files.html">Testing query help files</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="creating-and-working-with-codeql-packs.html">Creating and working with CodeQL packs</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="publishing-and-using-codeql-packs.html">Publishing and using CodeQL packs</a></li>
|
||
<li class="toctree-l3"><a class="reference internal" href="specifying-command-options-in-a-codeql-configuration-file.html">Specifying command options</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l2"><a class="reference internal" href="codeql-cli-reference.html">CodeQL CLI reference</a></li>
|
||
<li class="toctree-l2"><a class="reference external" href="https://codeql.github.com/docs/codeql-cli/manual">CodeQL CLI manual</a></li>
|
||
</ul>
|
||
</li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../writing-codeql-queries/index.html">Writing CodeQL queries</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../codeql-language-guides/index.html">CodeQL language guides</a></li>
|
||
<li class="toctree-l1"><a class="reference internal" href="../ql-language-reference/index.html">QL language reference</a></li>
|
||
</ul>
|
||
|
||
|
||
</nav>
|
||
|
||
|
||
<div class="body col-sm-12 col-md-9 col-lg-9 float-left border-left">
|
||
|
||
<div class="hide-lg hide-xl px-4 pt-4">
|
||
|
||
<div class="related" role="navigation" aria-label="related navigation">
|
||
<ul>
|
||
<li class="nav-item nav-item-0"><a href="../contents.html">CodeQL</a> »</li>
|
||
<li class="nav-item nav-item-1"><a href="index.html"
|
||
>CodeQL CLI</a> »</li>
|
||
<li class="nav-item nav-item-2"><a href="using-the-codeql-cli.html"
|
||
accesskey="U">Using the CodeQL CLI</a> »</li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
|
||
<article class="p-4 col-lg-10 col-md-10 col-sm-12">
|
||
|
||
<section id="creating-codeql-databases">
|
||
<span id="id1"></span><h1>Creating CodeQL databases<a class="headerlink" href="#creating-codeql-databases" title="Link to this heading">¶</a></h1>
|
||
<p>Before you analyze your code using CodeQL, you need to create a CodeQL
|
||
database containing all the data required to run queries on your code.</p>
|
||
<p>CodeQL analysis relies on extracting relational data from your code, and
|
||
using it to build a <a class="reference internal" href="../codeql-overview/codeql-glossary.html#codeql-database"><span class="std std-ref">CodeQL database</span></a>. CodeQL
|
||
databases contain all of the important information about a codebase, which can
|
||
be analyzed by executing CodeQL queries against it.
|
||
Before you generate a CodeQL database, you need to:</p>
|
||
<ul class="simple">
|
||
<li><p>Install and set up the CodeQL CLI. For more information, see
|
||
“<a class="reference internal" href="getting-started-with-the-codeql-cli.html"><span class="doc">Getting started with the CodeQL CLI</span></a>.”</p></li>
|
||
<li><p>Check out the version of your codebase you want to analyze. The directory
|
||
should be ready to build, with all dependencies already installed.</p></li>
|
||
</ul>
|
||
<p>For information about using the CodeQL CLI in a third-party CI system to create results
|
||
to display in GitHub as code scanning alerts, see <a class="reference external" href="https://docs.github.com/en/code-security/secure-coding/using-codeql-code-scanning-with-your-existing-ci-system/configuring-codeql-cli-in-your-ci-system">Configuring CodeQL CLI in your CI system</a>
|
||
in the GitHub documentation. For information about enabling CodeQL code scanning using GitHub Actions,
|
||
see <a class="reference external" href="https://docs.github.com/en/code-security/secure-coding/automatically-scanning-your-code-for-vulnerabilities-and-errors/setting-up-code-scanning-for-a-repository">Setting up code scanning for a repository</a>
|
||
in the GitHub documentation.</p>
|
||
<section id="running-codeql-database-create">
|
||
<h2>Running <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">create</span></code><a class="headerlink" href="#running-codeql-database-create" title="Link to this heading">¶</a></h2>
|
||
<p>CodeQL databases are created by running the following command from the checkout root
|
||
of your project:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create <database> --language=<language-identifier>
|
||
</pre></div>
|
||
</div>
|
||
<p>You must specify:</p>
|
||
<ul>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre"><database></span></code>: a path to the new database to be created. This directory will
|
||
be created when you execute the command—you cannot specify an existing
|
||
directory.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--language</span></code>: the identifier for the language to create a database for.
|
||
When used with <code class="docutils literal notranslate"><span class="pre">--db-cluster</span></code>, the option accepts a comma-separated list,
|
||
or can be specified more than once.
|
||
CodeQL supports creating databases for the following languages:</p>
|
||
<table class="docutils align-default">
|
||
<colgroup>
|
||
<col style="width: 50.0%" />
|
||
<col style="width: 50.0%" />
|
||
</colgroup>
|
||
<thead>
|
||
<tr class="row-odd"><th class="head"><p>Language</p></th>
|
||
<th class="head"><p>Identifier</p></th>
|
||
</tr>
|
||
</thead>
|
||
<tbody>
|
||
<tr class="row-even"><td><p>C/C++</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">cpp</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>C#</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">csharp</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>Go</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">go</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>Java</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">java</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>JavaScript/TypeScript</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">javascript</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-odd"><td><p>Python</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">python</span></code></p></td>
|
||
</tr>
|
||
<tr class="row-even"><td><p>Ruby</p></td>
|
||
<td><p><code class="docutils literal notranslate"><span class="pre">ruby</span></code></p></td>
|
||
</tr>
|
||
</tbody>
|
||
</table>
|
||
</li>
|
||
</ul>
|
||
<p>You can specify additional options depending on the location of your source file,
|
||
if the code needs to be compiled, and if you want to create CodeQL databases for
|
||
more than one language:</p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--source-root</span></code>: the root folder for the primary source files used in
|
||
database creation. By default, the command assumes that the current
|
||
directory is the source root—use this option to specify a different location.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--db-cluster</span></code>: use for multi-language codebases when you want to create
|
||
databases for more than one language.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--command</span></code>: used when you create a database for one or more compiled languages,
|
||
omit if the only languages requested are Python and JavaScript.
|
||
This specifies the build commands needed to invoke the compiler.
|
||
Commands are run from the current folder, or <code class="docutils literal notranslate"><span class="pre">--source-root</span></code>
|
||
if specified. If you don’t include a <code class="docutils literal notranslate"><span class="pre">--command</span></code>, CodeQL will attempt to
|
||
detect the build system automatically, using a built-in autobuilder.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--no-run-unnecessary-builds</span></code>: used with <code class="docutils literal notranslate"><span class="pre">--db-cluster</span></code> to suppress the build
|
||
command for languages where the CodeQL CLI does not need to monitor the build
|
||
(for example, Python and JavaScript/TypeScript).</p></li>
|
||
</ul>
|
||
<p>You can specify extractor options to customize the behavior of extractors that create CodeQL databases. For more information, see
|
||
“<a class="reference internal" href="extractor-options.html"><span class="doc">Extractor options</span></a>.”</p>
|
||
<p>For full details of all the options you can use when creating databases,
|
||
see the <a class="reference external" href="../manual/database-create">database create reference documentation</a>.</p>
|
||
</section>
|
||
<section id="progress-and-results">
|
||
<h2>Progress and results<a class="headerlink" href="#progress-and-results" title="Link to this heading">¶</a></h2>
|
||
<p>Errors are reported if there are any problems with the options you have
|
||
specified. For interpreted languages, the extraction progress is displayed in
|
||
the console—for each source file, it reports if extraction was successful or if
|
||
it failed. For compiled languages, the console will display the output of the
|
||
build system.</p>
|
||
<p>When the database is successfully created, you’ll find a new directory at the
|
||
path specified in the command. If you used the <code class="docutils literal notranslate"><span class="pre">--db-cluster</span></code> option to create
|
||
more than one database, a subdirectory is created for each language.
|
||
Each CodeQL database directory contains a number of
|
||
subdirectories, including the relational data (required for analysis) and a
|
||
source archive—a copy of the source files made at the time the database was
|
||
created—which is used for displaying analysis results.</p>
|
||
</section>
|
||
<section id="creating-databases-for-non-compiled-languages">
|
||
<h2>Creating databases for non-compiled languages<a class="headerlink" href="#creating-databases-for-non-compiled-languages" title="Link to this heading">¶</a></h2>
|
||
<p>The CodeQL CLI includes extractors to create databases for non-compiled
|
||
languages—specifically, JavaScript (and TypeScript), Python, and Ruby. These
|
||
extractors are automatically invoked when you specify JavaScript, Python, or Ruby as
|
||
the <code class="docutils literal notranslate"><span class="pre">--language</span></code> option when executing <code class="docutils literal notranslate"><span class="pre">database</span> <span class="pre">create</span></code>. When creating
|
||
databases for these languages you must ensure that all additional dependencies
|
||
are available.</p>
|
||
<blockquote class="pull-quote">
|
||
<div><p>Important</p>
|
||
<p>When you run <code class="docutils literal notranslate"><span class="pre">database</span> <span class="pre">create</span></code> for JavaScript, TypeScript, Python, and Ruby, you should not
|
||
specify a <code class="docutils literal notranslate"><span class="pre">--command</span></code> option. Otherwise this overrides the normal
|
||
extractor invocation, which will create an empty database. If you create
|
||
databases for multiple languages and one of them is a compiled language,
|
||
use the <code class="docutils literal notranslate"><span class="pre">--no-run-unnecessary-builds</span></code> option to skip the command for the languages that don’t need to be compiled.</p>
|
||
</div></blockquote>
|
||
<section id="javascript-and-typescript">
|
||
<h3>JavaScript and TypeScript<a class="headerlink" href="#javascript-and-typescript" title="Link to this heading">¶</a></h3>
|
||
<p>Creating databases for JavaScript requires no additional dependencies, but if
|
||
the project includes TypeScript files, you must install Node.js 6.x
|
||
or later. In the command line you can specify <code class="docutils literal notranslate"><span class="pre">--language=javascript</span></code> to
|
||
extract both JavaScript and TypeScript files:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create --language=javascript --source-root <folder-to-extract> <output-folder>/javascript-database
|
||
</pre></div>
|
||
</div>
|
||
<p>Here, we have specified a <code class="docutils literal notranslate"><span class="pre">--source-root</span></code> path, which is the location where
|
||
database creation is executed, but is not necessarily the checkout root of the
|
||
codebase.</p>
|
||
<p>By default, files in <code class="docutils literal notranslate"><span class="pre">node_modules</span></code> and <code class="docutils literal notranslate"><span class="pre">bower_components</span></code> directories are not extracted.</p>
|
||
</section>
|
||
<section id="python">
|
||
<h3>Python<a class="headerlink" href="#python" title="Link to this heading">¶</a></h3>
|
||
<p>When creating databases for Python you must ensure:</p>
|
||
<ul class="simple">
|
||
<li><p>You have the all of the required versions of Python installed.</p></li>
|
||
<li><p>You have access to the <a class="reference external" href="https://pypi.org/project/pip/">pip</a>
|
||
packaging management system and can install any
|
||
packages that the codebase depends on.</p></li>
|
||
<li><p>You have installed the <a class="reference external" href="https://pypi.org/project/virtualenv/">virtualenv</a> pip module.</p></li>
|
||
</ul>
|
||
<p>In the command line you must specify <code class="docutils literal notranslate"><span class="pre">--language=python</span></code>. For example::</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create --language=python <output-folder>/python-database
|
||
</pre></div>
|
||
</div>
|
||
<p>This executes the <code class="docutils literal notranslate"><span class="pre">database</span> <span class="pre">create</span></code> subcommand from the code’s checkout root,
|
||
generating a new Python database at <code class="docutils literal notranslate"><span class="pre"><output-folder>/python-database</span></code>.</p>
|
||
</section>
|
||
<section id="ruby">
|
||
<h3>Ruby<a class="headerlink" href="#ruby" title="Link to this heading">¶</a></h3>
|
||
<p>Creating databases for Ruby requires no additional dependencies.
|
||
In the command line you must specify <code class="docutils literal notranslate"><span class="pre">--language=ruby</span></code>. For example:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create --language=ruby --source-root <folder-to-extract> <output-folder>/ruby-database
|
||
</pre></div>
|
||
</div>
|
||
<p>Here, we have specified a <code class="docutils literal notranslate"><span class="pre">--source-root</span></code> path, which is the location where
|
||
database creation is executed, but is not necessarily the checkout root of the
|
||
codebase.</p>
|
||
</section>
|
||
</section>
|
||
<section id="creating-databases-for-compiled-languages">
|
||
<h2>Creating databases for compiled languages<a class="headerlink" href="#creating-databases-for-compiled-languages" title="Link to this heading">¶</a></h2>
|
||
<p>For compiled languages, CodeQL needs to invoke the required build system to
|
||
generate a database, therefore the build method must be available to the CLI.</p>
|
||
<section id="detecting-the-build-system">
|
||
<h3>Detecting the build system<a class="headerlink" href="#detecting-the-build-system" title="Link to this heading">¶</a></h3>
|
||
<p>The CodeQL CLI includes autobuilders for C/C++, C#, Go, and Java code. CodeQL
|
||
autobuilders allow you to build projects for compiled languages without
|
||
specifying any build commands. When an autobuilder is invoked, CodeQL examines
|
||
the source for evidence of a build system and attempts to run the optimal set of
|
||
commands required to extract a database.</p>
|
||
<p>An autobuilder is invoked automatically when you execute <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span>
|
||
<span class="pre">create</span></code> for a compiled <code class="docutils literal notranslate"><span class="pre">--language</span></code> if don’t include a
|
||
<code class="docutils literal notranslate"><span class="pre">--command</span></code> option. For example, for a Java codebase, you would simply run:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create --language=java <output-folder>/java-database
|
||
</pre></div>
|
||
</div>
|
||
<p>If a codebase uses a standard build system, relying on an autobuilder is often
|
||
the simplest way to create a database. For sources that require non-standard
|
||
build steps, you may need to explicitly define each step in the command line.</p>
|
||
<blockquote class="pull-quote">
|
||
<div><p>Creating databases for Go</p>
|
||
<p>For Go, install the Go toolchain (version 1.11 or later) and, if there
|
||
are dependencies, the appropriate dependency manager (such as <a class="reference external" href="https://golang.github.io/dep/">dep</a>).</p>
|
||
<p>The Go autobuilder attempts to automatically detect code written in Go in a repository,
|
||
and only runs build scripts in an attempt to fetch dependencies. To force
|
||
CodeQL to limit extraction to the files compiled by your build script, set the environment variable
|
||
<cite>CODEQL_EXTRACTOR_GO_BUILD_TRACING=on</cite> or use the <code class="docutils literal notranslate"><span class="pre">--command</span></code> option to specify a
|
||
build command.</p>
|
||
</div></blockquote>
|
||
</section>
|
||
<section id="specifying-build-commands">
|
||
<h3>Specifying build commands<a class="headerlink" href="#specifying-build-commands" title="Link to this heading">¶</a></h3>
|
||
<p>The following examples are designed to give you an idea of some of the build
|
||
commands that you can specify for compiled languages.</p>
|
||
<blockquote class="pull-quote">
|
||
<div><p>Important</p>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">--command</span></code> option accepts a single argument—if you need to
|
||
use more than one command, specify <code class="docutils literal notranslate"><span class="pre">--command</span></code> multiple times.</p>
|
||
<p>If you need to pass subcommands and options, the whole argument needs to be
|
||
quoted to be interpreted correctly.</p>
|
||
</div></blockquote>
|
||
<ul>
|
||
<li><p>C/C++ project built using <code class="docutils literal notranslate"><span class="pre">make</span></code>:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create cpp-database --language=cpp --command=make
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>C# project built using <code class="docutils literal notranslate"><span class="pre">dotnet</span> <span class="pre">build</span></code>:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>For C# projects using either `dotnet build` or `msbuild`, you should specify `/p:UseSharedCompilation=false`
|
||
in the build command. It is also a good idea to add `/t:rebuild` to ensure that all code will be built (code
|
||
that is not built will not be included in the CodeQL database):
|
||
|
||
codeql database create csharp-database --language=csharp --command='dotnet build /p:UseSharedCompilation=false /t:rebuild'
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Go project built using the <code class="docutils literal notranslate"><span class="pre">CODEQL_EXTRACTOR_GO_BUILD_TRACING=on</span></code> environment variable:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>CODEQL_EXTRACTOR_GO_BUILD_TRACING=on codeql database create go-database --language=go
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Go project built using a custom build script:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create go-database --language=go --command='./scripts/build.sh'
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Java project built using Gradle:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create java-database --language=java --command='gradle clean test'
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Java project built using Maven:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create java-database --language=java --command='mvn clean install'
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Java project built using Ant:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create java-database --language=java --command='ant -f build.xml'
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Project built using Bazel:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span># Navigate to the Bazel workspace.
|
||
|
||
# Before building, remove cached objects
|
||
# and stop all running Bazel server processes.
|
||
bazel clean --expunge
|
||
|
||
# Build using the following Bazel flags, to help CodeQL detect the build:
|
||
# `--spawn_strategy=local`: build locally, instead of using a distributed build
|
||
# `--nouse_action_cache`: turn off build caching, which might prevent recompilation of source code
|
||
# `--noremote_accept_cached`, `--noremote_upload_local_results`: avoid using a remote cache
|
||
codeql database create new-database --language=<language> \
|
||
--command='bazel build --spawn_strategy=local --nouse_action_cache --noremote_accept_cached --noremote_upload_local_results //path/to/package:target'
|
||
|
||
# After building, stop all running Bazel server processes.
|
||
# This ensures future build commands start in a clean Bazel server process
|
||
# without CodeQL attached.
|
||
bazel shutdown
|
||
</pre></div>
|
||
</div>
|
||
</li>
|
||
<li><p>Project built using a custom build script:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database create new-database --language=<language> --command='./scripts/build.sh'
|
||
</pre></div>
|
||
</div>
|
||
<p>This command runs a custom script that contains all of the commands required
|
||
to build the project.</p>
|
||
</li>
|
||
</ul>
|
||
</section>
|
||
<section id="using-indirect-build-tracing">
|
||
<h3>Using indirect build tracing<a class="headerlink" href="#using-indirect-build-tracing" title="Link to this heading">¶</a></h3>
|
||
<p>If the CodeQL CLI autobuilders for compiled languages do not work with your CI workflow and you cannot wrap invocations of build commands with <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">trace-command</span></code>, you can use indirect build tracing to create a CodeQL database. To use indirect build tracing, your CI system must be able to set custom environment variables for each build action.</p>
|
||
<p>To create a CodeQL database with indirect build tracing, run the following command from the checkout root of your project:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>codeql database init ... --begin-tracing <database>
|
||
</pre></div>
|
||
</div>
|
||
<p>You must specify:</p>
|
||
<ul class="simple">
|
||
<li><p><code class="docutils literal notranslate"><span class="pre"><database></span></code>: a path to the new database to be created. This directory will
|
||
be created when you execute the command—you cannot specify an existing
|
||
directory.</p></li>
|
||
<li><p><code class="docutils literal notranslate"><span class="pre">--begin-tracing</span></code>: creates scripts that can be used to set up an environment in which build commands will be traced.</p></li>
|
||
</ul>
|
||
<p>You may specify other options for the <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">init</span></code> command as normal.</p>
|
||
<blockquote class="pull-quote">
|
||
<div><p>Note</p>
|
||
<p>If the build runs on Windows, you must set either <code class="docutils literal notranslate"><span class="pre">--trace-process-level</span> <span class="pre"><number></span></code> or <code class="docutils literal notranslate"><span class="pre">--trace-process-name</span> <span class="pre"><parent</span> <span class="pre">process</span> <span class="pre">name></span></code> so that the option points to a parent CI process that will observe all build steps for the code being analyzed.</p>
|
||
</div></blockquote>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">init</span></code> command will output a message:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>Created skeleton <database>. This in-progress database is ready to be populated by an extractor.
|
||
In order to initialise tracing, some environment variables need to be set in the shell your build will run in.
|
||
A number of scripts to do this have been created in <database>/temp/tracingEnvironment.
|
||
Please run one of these scripts before invoking your build command.
|
||
|
||
Based on your operating system, we recommend you run: ...
|
||
</pre></div>
|
||
</div>
|
||
<p>The <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">init</span></code> command creates <code class="docutils literal notranslate"><span class="pre"><database>/temp/tracingEnvironment</span></code> with files that contain environment variables and values that will enable CodeQL to trace a sequence of build steps. These files are named <code class="docutils literal notranslate"><span class="pre">start-tracing.{json,sh,bat,ps1}</span></code>. Use one of these files with your CI system’s mechanism for setting environment variables for future steps. You can:</p>
|
||
<ul class="simple">
|
||
<li><p>Read the JSON file, process it, and print out environment variables in the format expected by your CI system. For example, Azure DevOps expects <code class="docutils literal notranslate"><span class="pre">echo</span> <span class="pre">"##vso[task.setvariable</span> <span class="pre">variable=NAME]VALUE"</span></code>.</p></li>
|
||
<li><p>Or, if your CI system persists the environment, source the appropriate <code class="docutils literal notranslate"><span class="pre">start-tracing</span></code> script to set the CodeQL variables in the shell environment of the CI system.</p></li>
|
||
</ul>
|
||
<p>Build your code; optionally, unset the environment variables using an <code class="docutils literal notranslate"><span class="pre">end-tracing.{json,sh,bat,ps1}</span></code> script from the directory where the <code class="docutils literal notranslate"><span class="pre">start-tracing</span></code> scripts are stored; and then run the command <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">finalize</span> <span class="pre"><database></span></code>.</p>
|
||
<p>Once you have created a CodeQL database using indirect build tracing, you can work with it like any other CodeQL database. For example, analyze the database, and upload the results to GitHub if you use code scanning.</p>
|
||
</section>
|
||
<section id="example-of-creating-a-codeql-database-using-indirect-build-tracing">
|
||
<h3>Example of creating a CodeQL database using indirect build tracing<a class="headerlink" href="#example-of-creating-a-codeql-database-using-indirect-build-tracing" title="Link to this heading">¶</a></h3>
|
||
<p>The following example shows how you could use indirect build tracing in an Azure DevOps pipeline to create a CodeQL database:</p>
|
||
<div class="highlight-none notranslate"><div class="highlight"><pre><span></span>steps:
|
||
# Download the CodeQL CLI and query packs...
|
||
# Check out the repository ...
|
||
|
||
# Run any pre-build tasks, for example, restore NuGet dependencies...
|
||
|
||
# Initialize the CodeQL database.
|
||
# In this example, the CodeQL CLI has been downloaded and placed on the PATH.
|
||
- task: CmdLine@1
|
||
displayName: Initialize CodeQL database
|
||
inputs:
|
||
# Assumes the source code is checked out to the current working directory.
|
||
# Creates a database at `<current working directory>/db`.
|
||
# Running on Windows, so specifies a trace process level.
|
||
script: "codeql database init --language csharp --trace-process-name Agent.Worker.exe --source-root . --begin-tracing db"
|
||
|
||
# Read the generated environment variables and values,
|
||
# and set them so they are available for subsequent commands
|
||
# in the build pipeline. This is done in PowerShell in this example.
|
||
- task: PowerShell@1
|
||
displayName: Set CodeQL environment variables
|
||
inputs:
|
||
targetType: inline
|
||
script: >
|
||
$json = Get-Content $(System.DefaultWorkingDirectory)/db/temp/tracingEnvironment/start-tracing.json | ConvertFrom-Json
|
||
$json.PSObject.Properties | ForEach-Object {
|
||
$template = "##vso[task.setvariable variable="
|
||
$template += $_.Name
|
||
$template += "]"
|
||
$template += $_.Value
|
||
echo "$template"
|
||
}
|
||
|
||
# Execute the pre-defined build step. Note the `msbuildArgs` variable.
|
||
- task: VSBuild@1
|
||
inputs:
|
||
solution: '**/*.sln'
|
||
# Disable MSBuild shared compilation for C# builds.
|
||
msbuildArgs: /p:OutDir=$(Build.ArtifactStagingDirectory) /p:UseSharedCompilation=false
|
||
platform: Any CPU
|
||
configuration: Release
|
||
# Execute a clean build, in order to remove any existing build artifacts prior to the build.
|
||
clean: True
|
||
displayName: Visual Studio Build
|
||
|
||
# Read and set the generated environment variables to end build tracing. This is done in PowerShell in this example.
|
||
- task: PowerShell@1
|
||
displayName: Clear CodeQL environment variables
|
||
inputs:
|
||
targetType: inline
|
||
script: >
|
||
$json = Get-Content $(System.DefaultWorkingDirectory)/db/temp/tracingEnvironment/end-tracing.json | ConvertFrom-Json
|
||
$json.PSObject.Properties | ForEach-Object {
|
||
$template = "##vso[task.setvariable variable="
|
||
$template += $_.Name
|
||
$template += "]"
|
||
$template += $_.Value
|
||
echo "$template"
|
||
}
|
||
|
||
- task: CmdLine@2
|
||
displayName: Finalize CodeQL database
|
||
inputs:
|
||
script: 'codeql database finalize db'
|
||
|
||
# Other tasks go here, for example:
|
||
# `codeql database analyze`
|
||
# then `codeql github upload-results` ...
|
||
</pre></div>
|
||
</div>
|
||
</section>
|
||
</section>
|
||
<section id="obtaining-databases-from-lgtm-com">
|
||
<h2>Obtaining databases from LGTM.com<a class="headerlink" href="#obtaining-databases-from-lgtm-com" title="Link to this heading">¶</a></h2>
|
||
<p><a class="reference external" href="https://lgtm.com">LGTM.com</a> analyzes thousands of open-source projects using
|
||
CodeQL. For each project on LGTM.com, you can download an archived CodeQL
|
||
database corresponding to the most recently analyzed revision of the code. These
|
||
databases can also be analyzed using the CodeQL CLI or used with the CodeQL
|
||
extension for Visual Studio Code.</p>
|
||
<p>To download a database from LGTM.com:</p>
|
||
<ol class="arabic simple">
|
||
<li><p>Log in to <a class="reference external" href="https://lgtm.com/">LGTM.com</a>.</p></li>
|
||
<li><p>Find a project you’re interested in and display the Integrations tab (for example, <a class="reference external" href="https://lgtm.com/projects/g/apache/kafka/ci/">Apache Kafka</a>).</p></li>
|
||
<li><p>Scroll to the <strong>CodeQL databases for local analysis</strong> section at the bottom of the page.</p></li>
|
||
<li><p>Download databases for the languages that you want to explore.</p></li>
|
||
</ol>
|
||
<p>Before running an analysis, unzip the databases and try <a class="reference internal" href="upgrading-codeql-databases.html"><span class="doc">upgrading</span></a> the
|
||
unzipped databases to ensure they are compatible with your local copy of the
|
||
CodeQL queries and libraries.</p>
|
||
<blockquote class="pull-quote">
|
||
<div><p>Note</p>
|
||
<p>The CodeQL CLI currently extracts data from additional, external files in a
|
||
different way to the legacy QL tools. For example, when you run <code class="docutils literal notranslate"><span class="pre">codeql</span> <span class="pre">database</span> <span class="pre">create</span></code>
|
||
the CodeQL CLI extracts data from some relevant XML files for Java and C#, but not
|
||
for the other supported languages, such as JavaScript. This means that CodeQL databases
|
||
created using the CodeQL CLI may be slightly different from those obtained from LGTM.com or
|
||
created using the legacy QL command-line tools. As such, analysis results generated from
|
||
databases created using the CodeQL CLI may also differ from those generated from
|
||
databases obtained from elsewhere.</p>
|
||
</div></blockquote>
|
||
</section>
|
||
<section id="further-reading">
|
||
<h2>Further reading<a class="headerlink" href="#further-reading" title="Link to this heading">¶</a></h2>
|
||
<ul class="simple">
|
||
<li><p>“<a class="reference internal" href="../codeql-for-visual-studio-code/analyzing-your-projects.html#analyzing-your-projects"><span class="std std-ref">Analyzing your projects in CodeQL for VS Code</span></a>”</p></li>
|
||
</ul>
|
||
</section>
|
||
</section>
|
||
|
||
|
||
</article>
|
||
|
||
<!-- GitHub footer, with links to terms and privacy statement -->
|
||
<div class="px-3 px-md-6 f6 py-4 d-sm-flex flex-justify-between flex-row-reverse flex-items-center border-top">
|
||
<ul class="list-style-none d-flex flex-items-center mb-3 mb-sm-0 lh-condensed-ultra">
|
||
<li class="mr-3">
|
||
<a href="https://twitter.com/github" title="GitHub on Twitter" style="color: #959da5;">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 273.5 222.3" class="d-block" height="18">
|
||
<path
|
||
d="M273.5 26.3a109.77 109.77 0 0 1-32.2 8.8 56.07 56.07 0 0 0 24.7-31 113.39 113.39 0 0 1-35.7 13.6 56.1 56.1 0 0 0-97 38.4 54 54 0 0 0 1.5 12.8A159.68 159.68 0 0 1 19.1 10.3a56.12 56.12 0 0 0 17.4 74.9 56.06 56.06 0 0 1-25.4-7v.7a56.11 56.11 0 0 0 45 55 55.65 55.65 0 0 1-14.8 2 62.39 62.39 0 0 1-10.6-1 56.24 56.24 0 0 0 52.4 39 112.87 112.87 0 0 1-69.7 24 119 119 0 0 1-13.4-.8 158.83 158.83 0 0 0 86 25.2c103.2 0 159.6-85.5 159.6-159.6 0-2.4-.1-4.9-.2-7.3a114.25 114.25 0 0 0 28.1-29.1"
|
||
fill="currentColor"></path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
<li class="mr-3">
|
||
<a href="https://www.facebook.com/GitHub" title="GitHub on Facebook" style="color: #959da5;">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 15.3 15.4" class="d-block" height="18">
|
||
<path
|
||
d="M14.5 0H.8a.88.88 0 0 0-.8.9v13.6a.88.88 0 0 0 .8.9h7.3v-6h-2V7.1h2V5.4a2.87 2.87 0 0 1 2.5-3.1h.5a10.87 10.87 0 0 1 1.8.1v2.1h-1.3c-1 0-1.1.5-1.1 1.1v1.5h2.3l-.3 2.3h-2v5.9h3.9a.88.88 0 0 0 .9-.8V.8a.86.86 0 0 0-.8-.8z"
|
||
fill="currentColor"></path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
<li class="mr-3">
|
||
<a href="https://www.youtube.com/github" title="GitHub on YouTube" style="color: #959da5;">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19.17 13.6" class="d-block" height="16">
|
||
<path
|
||
d="M18.77 2.13A2.4 2.4 0 0 0 17.09.42C15.59 0 9.58 0 9.58 0a57.55 57.55 0 0 0-7.5.4A2.49 2.49 0 0 0 .39 2.13 26.27 26.27 0 0 0 0 6.8a26.15 26.15 0 0 0 .39 4.67 2.43 2.43 0 0 0 1.69 1.71c1.52.42 7.5.42 7.5.42a57.69 57.69 0 0 0 7.51-.4 2.4 2.4 0 0 0 1.68-1.71 25.63 25.63 0 0 0 .4-4.67 24 24 0 0 0-.4-4.69zM7.67 9.71V3.89l5 2.91z"
|
||
fill="currentColor"></path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
<li class="mr-3 flex-self-start">
|
||
<a href="https://www.linkedin.com/company/github" title="GitHub on Linkedin" style="color: #959da5;">
|
||
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 19 18" class="d-block" height="18">
|
||
<path
|
||
d="M3.94 2A2 2 0 1 1 2 0a2 2 0 0 1 1.94 2zM4 5.48H0V18h4zm6.32 0H6.34V18h3.94v-6.57c0-3.66 4.77-4 4.77 0V18H19v-7.93c0-6.17-7.06-5.94-8.72-2.91z"
|
||
fill="currentColor"></path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
<li>
|
||
<a href="https://github.com/github" title="GitHub's organization" style="color: #959da5;">
|
||
<svg version="1.1" width="20" height="20" viewBox="0 0 16 16" class="octicon octicon-mark-github"
|
||
aria-hidden="true">
|
||
<path fill-rule="evenodd"
|
||
d="M8 0C3.58 0 0 3.58 0 8c0 3.54 2.29 6.53 5.47 7.59.4.07.55-.17.55-.38 0-.19-.01-.82-.01-1.49-2.01.37-2.53-.49-2.69-.94-.09-.23-.48-.94-.82-1.13-.28-.15-.68-.52-.01-.53.63-.01 1.08.58 1.23.82.72 1.21 1.87.87 2.33.66.07-.52.28-.87.51-1.07-1.78-.2-3.64-.89-3.64-3.95 0-.87.31-1.59.82-2.15-.08-.2-.36-1.02.08-2.12 0 0 .67-.21 2.2.82.64-.18 1.32-.27 2-.27.68 0 1.36.09 2 .27 1.53-1.04 2.2-.82 2.2-.82.44 1.1.16 1.92.08 2.12.51.56.82 1.27.82 2.15 0 3.07-1.87 3.75-3.65 3.95.29.25.54.73.54 1.48 0 1.07-.01 1.93-.01 2.2 0 .21.15.46.55.38A8.013 8.013 0 0016 8c0-4.42-3.58-8-8-8z">
|
||
</path>
|
||
</svg>
|
||
</a>
|
||
</li>
|
||
</ul>
|
||
<ul class="list-style-none d-flex text-gray">
|
||
<li class="mr-3">©
|
||
<script type="text/javascript">document.write(new Date().getFullYear());</script> GitHub, Inc.</li>
|
||
<li class="mr-3"><a
|
||
href="https://docs.github.com/github/site-policy/github-terms-of-service"
|
||
class="link-gray">Terms </a></li>
|
||
<li><a href="https://docs.github.com/github/site-policy/github-privacy-statement"
|
||
class="link-gray">Privacy </a></li>
|
||
</ul>
|
||
</div>
|
||
</div>
|
||
</main>
|
||
|
||
<script type="text/javascript">
|
||
$(document).ready(function () {
|
||
$(".toggle > *").hide();
|
||
$(".toggle .name").show();
|
||
$(".toggle .name").click(function () {
|
||
$(this).parent().children().not(".name").toggle(400);
|
||
$(this).parent().children(".name").toggleClass("open");
|
||
})
|
||
});
|
||
</script>
|
||
|
||
</body>
|
||
</html> |