mirror of
https://github.com/github/codeql.git
synced 2026-06-13 08:51:20 +02:00
move system prompt injection to non-experimental
This commit is contained in:
@@ -0,0 +1,31 @@
|
||||
<!DOCTYPE qhelp PUBLIC
|
||||
"-//Semmle//qhelp//EN"
|
||||
"qhelp.dtd">
|
||||
<qhelp>
|
||||
|
||||
<overview>
|
||||
<p>If user-controlled data is included in a system prompt, an attacker can manipulate the instructions
|
||||
that govern the AI model's behavior, bypassing intended restrictions and potentially causing sensitive
|
||||
data leaks or unintended operations.</p>
|
||||
</overview>
|
||||
|
||||
<recommendation>
|
||||
<p>Do not include user input in system-level or developer-level prompts. If user input must influence
|
||||
the system prompt, validate it against a fixed allowlist of permitted values.</p>
|
||||
</recommendation>
|
||||
|
||||
<example>
|
||||
<p>In the following example, a user-controlled value is inserted directly into a system-level prompt
|
||||
without validation, allowing an attacker to manipulate the AI's behavior.</p>
|
||||
<sample src="examples/prompt-injection.js" />
|
||||
<p>The fix validates the user input against a fixed allowlist of permitted values before
|
||||
including it in the prompt.</p>
|
||||
<sample src="examples/prompt-injection_fixed.js" />
|
||||
</example>
|
||||
|
||||
<references>
|
||||
<li>OWASP: <a href="https://genai.owasp.org/llmrisk/llm01-prompt-injection/">LLM01: Prompt Injection</a>.</li>
|
||||
<li>MITRE CWE: <a href="https://cwe.mitre.org/data/definitions/1427.html">CWE-1427: Improper Neutralization of Input Used for LLM Prompting</a>.</li>
|
||||
</references>
|
||||
|
||||
</qhelp>
|
||||
19
javascript/ql/src/Security/CWE-1427/SystemPromptInjection.ql
Normal file
19
javascript/ql/src/Security/CWE-1427/SystemPromptInjection.ql
Normal file
@@ -0,0 +1,19 @@
|
||||
/**
|
||||
* @name Prompt injection
|
||||
* @kind path-problem
|
||||
* @problem.severity error
|
||||
* @security-severity 5.0
|
||||
* @precision high
|
||||
* @id js/system-prompt-injection
|
||||
* @tags security
|
||||
* external/cwe/cwe-1427
|
||||
*/
|
||||
|
||||
import javascript
|
||||
import semmle.javascript.security.dataflow.SystemPromptInjectionQuery
|
||||
import SystemPromptInjectionFlow::PathGraph
|
||||
|
||||
from SystemPromptInjectionFlow::PathNode source, SystemPromptInjectionFlow::PathNode sink
|
||||
where SystemPromptInjectionFlow::flowPath(source, sink)
|
||||
select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(),
|
||||
"user-provided value"
|
||||
@@ -0,0 +1,26 @@
|
||||
const express = require("express");
|
||||
const OpenAI = require("openai");
|
||||
|
||||
const app = express();
|
||||
const client = new OpenAI();
|
||||
|
||||
app.get("/chat", async (req, res) => {
|
||||
let persona = req.query.persona;
|
||||
|
||||
// BAD: user input is used directly in a system-level prompt
|
||||
const response = await client.chat.completions.create({
|
||||
model: "gpt-4.1",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful assistant. Act as a " + persona,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: req.query.message,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
res.json(response);
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
const express = require("express");
|
||||
const OpenAI = require("openai");
|
||||
|
||||
const app = express();
|
||||
const client = new OpenAI();
|
||||
|
||||
const ALLOWED_PERSONAS = ["pirate", "teacher", "poet"];
|
||||
|
||||
app.get("/chat", async (req, res) => {
|
||||
let persona = req.query.persona;
|
||||
|
||||
// GOOD: user input is validated against a fixed allowlist before use in a prompt
|
||||
if (!ALLOWED_PERSONAS.includes(persona)) {
|
||||
return res.status(400).json({ error: "Invalid persona" });
|
||||
}
|
||||
|
||||
const response = await client.chat.completions.create({
|
||||
model: "gpt-4.1",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful assistant. Act as a " + persona,
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: req.query.message,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
res.json(response);
|
||||
});
|
||||
Reference in New Issue
Block a user