mirror of
https://github.com/github/codeql.git
synced 2026-05-21 06:37:10 +02:00
move system prompt injection to non-experimental
This commit is contained in:
@@ -11,9 +11,9 @@ private import semmle.javascript.Concepts
|
||||
private import semmle.javascript.security.dataflow.RemoteFlowSources
|
||||
private import semmle.javascript.dataflow.internal.BarrierGuards
|
||||
private import semmle.javascript.frameworks.data.ModelsAsData
|
||||
private import experimental.semmle.javascript.frameworks.OpenAI
|
||||
private import experimental.semmle.javascript.frameworks.Anthropic
|
||||
private import experimental.semmle.javascript.frameworks.GoogleGenAI
|
||||
private import semmle.javascript.frameworks.OpenAI
|
||||
private import semmle.javascript.frameworks.Anthropic
|
||||
private import semmle.javascript.frameworks.GoogleGenAI
|
||||
|
||||
/**
|
||||
* Provides default sources, sinks and sanitizers for detecting
|
||||
@@ -4,14 +4,13 @@
|
||||
* @problem.severity error
|
||||
* @security-severity 5.0
|
||||
* @precision high
|
||||
* @id js/prompt-injection
|
||||
* @id js/system-prompt-injection
|
||||
* @tags security
|
||||
* experimental
|
||||
* external/cwe/cwe-1427
|
||||
*/
|
||||
|
||||
import javascript
|
||||
import experimental.semmle.javascript.security.PromptInjection.SystemPromptInjectionQuery
|
||||
import semmle.javascript.security.dataflow.SystemPromptInjectionQuery
|
||||
import SystemPromptInjectionFlow::PathGraph
|
||||
|
||||
from SystemPromptInjectionFlow::PathNode source, SystemPromptInjectionFlow::PathNode sink
|
||||
@@ -23,4 +23,4 @@ app.get("/chat", async (req, res) => {
|
||||
});
|
||||
|
||||
res.json(response);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
<!DOCTYPE qhelp PUBLIC
|
||||
"-//Semmle//qhelp//EN"
|
||||
"qhelp.dtd">
|
||||
<qhelp>
|
||||
|
||||
<overview>
|
||||
<p>If untrusted input is included in a user-role prompt sent to an AI model, an attacker can inject
|
||||
instructions that manipulate the model's behavior. This is known as <i>indirect prompt injection</i>
|
||||
when the malicious content arrives through data the model processes, or <i>direct prompt injection</i>
|
||||
when the attacker controls the prompt directly.</p>
|
||||
|
||||
<p>Unlike system prompt injection, user prompt injection targets the user-role messages. Although
|
||||
user messages are expected to carry user input, passing unsanitized data directly into structured
|
||||
prompt templates can still allow an attacker to override intended instructions, extract sensitive
|
||||
context, or trigger unintended tool calls.</p>
|
||||
</overview>
|
||||
|
||||
<recommendation>
|
||||
<p>To mitigate user prompt injection:</p>
|
||||
<ul>
|
||||
<li>Validate user input against a fixed allowlist of permitted values before including it in a prompt.</li>
|
||||
<li>Use parameterized prompt templates that clearly separate instructions from user data.</li>
|
||||
<li>Apply output filtering to detect and block responses that indicate prompt injection attempts.</li>
|
||||
</ul>
|
||||
</recommendation>
|
||||
|
||||
<example>
|
||||
<p>In the following example, user-controlled data is inserted directly into a user-role prompt
|
||||
without any validation, allowing an attacker to inject arbitrary instructions.</p>
|
||||
<sample src="examples/user-prompt-injection.js" />
|
||||
<p>The fix validates the user input against a fixed allowlist of permitted values before
|
||||
including it in the prompt.</p>
|
||||
<sample src="examples/user-prompt-injection_fixed.js" />
|
||||
</example>
|
||||
|
||||
<references>
|
||||
<li>OWASP: <a href="https://genai.owasp.org/llmrisk/llm01-prompt-injection/">LLM01: Prompt Injection</a>.</li>
|
||||
<li>MITRE CWE: <a href="https://cwe.mitre.org/data/definitions/1427.html">CWE-1427: Improper Neutralization of Input Used for LLM Prompting</a>.</li>
|
||||
</references>
|
||||
|
||||
</qhelp>
|
||||
@@ -3,9 +3,9 @@
|
||||
* @description Untrusted input flowing into a user-role prompt of an AI model
|
||||
* may allow an attacker to manipulate the model's behavior.
|
||||
* @kind path-problem
|
||||
* @problem.severity error
|
||||
* @problem.severity warning
|
||||
* @security-severity 5.0
|
||||
* @precision high
|
||||
* @precision medium
|
||||
* @id js/user-prompt-injection
|
||||
* @tags security
|
||||
* experimental
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
const express = require("express");
|
||||
const OpenAI = require("openai");
|
||||
|
||||
const app = express();
|
||||
const client = new OpenAI();
|
||||
|
||||
app.get("/chat", async (req, res) => {
|
||||
let topic = req.query.topic;
|
||||
|
||||
// BAD: user input is used directly in a user-role prompt
|
||||
const response = await client.chat.completions.create({
|
||||
model: "gpt-4.1",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful assistant that summarizes topics.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "Summarize the following topic: " + topic,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
res.json(response);
|
||||
});
|
||||
@@ -0,0 +1,32 @@
|
||||
const express = require("express");
|
||||
const OpenAI = require("openai");
|
||||
|
||||
const app = express();
|
||||
const client = new OpenAI();
|
||||
|
||||
const ALLOWED_TOPICS = ["science", "history", "technology"];
|
||||
|
||||
app.get("/chat", async (req, res) => {
|
||||
let topic = req.query.topic;
|
||||
|
||||
// GOOD: user input is validated against a fixed allowlist before use in a prompt
|
||||
if (!ALLOWED_TOPICS.includes(topic)) {
|
||||
return res.status(400).json({ error: "Invalid topic" });
|
||||
}
|
||||
|
||||
const response = await client.chat.completions.create({
|
||||
model: "gpt-4.1",
|
||||
messages: [
|
||||
{
|
||||
role: "system",
|
||||
content: "You are a helpful assistant that summarizes topics.",
|
||||
},
|
||||
{
|
||||
role: "user",
|
||||
content: "Summarize the following topic: " + topic,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
res.json(response);
|
||||
});
|
||||
@@ -11,9 +11,9 @@ private import semmle.javascript.Concepts
|
||||
private import semmle.javascript.security.dataflow.RemoteFlowSources
|
||||
private import semmle.javascript.dataflow.internal.BarrierGuards
|
||||
private import semmle.javascript.frameworks.data.ModelsAsData
|
||||
private import experimental.semmle.javascript.frameworks.OpenAI
|
||||
private import experimental.semmle.javascript.frameworks.Anthropic
|
||||
private import experimental.semmle.javascript.frameworks.GoogleGenAI
|
||||
private import semmle.javascript.frameworks.OpenAI
|
||||
private import semmle.javascript.frameworks.Anthropic
|
||||
private import semmle.javascript.frameworks.GoogleGenAI
|
||||
|
||||
/**
|
||||
* Provides default sources, sinks and sanitizers for detecting
|
||||
|
||||
@@ -1 +1 @@
|
||||
experimental/Security/CWE-1427/SystemPromptInjection.ql
|
||||
Security/CWE-1427/SystemPromptInjection.ql
|
||||
|
||||
Reference in New Issue
Block a user