move system prompt injection to non-experimental

This commit is contained in:
BazookaMusic
2026-05-20 10:48:07 +02:00
parent 5ef09a102c
commit 6c5c8e1c9b
15 changed files with 111 additions and 13 deletions

View File

@@ -11,9 +11,9 @@ private import semmle.javascript.Concepts
private import semmle.javascript.security.dataflow.RemoteFlowSources
private import semmle.javascript.dataflow.internal.BarrierGuards
private import semmle.javascript.frameworks.data.ModelsAsData
private import experimental.semmle.javascript.frameworks.OpenAI
private import experimental.semmle.javascript.frameworks.Anthropic
private import experimental.semmle.javascript.frameworks.GoogleGenAI
private import semmle.javascript.frameworks.OpenAI
private import semmle.javascript.frameworks.Anthropic
private import semmle.javascript.frameworks.GoogleGenAI
/**
* Provides default sources, sinks and sanitizers for detecting

View File

@@ -4,14 +4,13 @@
* @problem.severity error
* @security-severity 5.0
* @precision high
* @id js/prompt-injection
* @id js/system-prompt-injection
* @tags security
* experimental
* external/cwe/cwe-1427
*/
import javascript
import experimental.semmle.javascript.security.PromptInjection.SystemPromptInjectionQuery
import semmle.javascript.security.dataflow.SystemPromptInjectionQuery
import SystemPromptInjectionFlow::PathGraph
from SystemPromptInjectionFlow::PathNode source, SystemPromptInjectionFlow::PathNode sink

View File

@@ -23,4 +23,4 @@ app.get("/chat", async (req, res) => {
});
res.json(response);
});
});

View File

@@ -0,0 +1,41 @@
<!DOCTYPE qhelp PUBLIC
"-//Semmle//qhelp//EN"
"qhelp.dtd">
<qhelp>
<overview>
<p>If untrusted input is included in a user-role prompt sent to an AI model, an attacker can inject
instructions that manipulate the model's behavior. This is known as <i>indirect prompt injection</i>
when the malicious content arrives through data the model processes, or <i>direct prompt injection</i>
when the attacker controls the prompt directly.</p>
<p>Unlike system prompt injection, user prompt injection targets the user-role messages. Although
user messages are expected to carry user input, passing unsanitized data directly into structured
prompt templates can still allow an attacker to override intended instructions, extract sensitive
context, or trigger unintended tool calls.</p>
</overview>
<recommendation>
<p>To mitigate user prompt injection:</p>
<ul>
<li>Validate user input against a fixed allowlist of permitted values before including it in a prompt.</li>
<li>Use parameterized prompt templates that clearly separate instructions from user data.</li>
<li>Apply output filtering to detect and block responses that indicate prompt injection attempts.</li>
</ul>
</recommendation>
<example>
<p>In the following example, user-controlled data is inserted directly into a user-role prompt
without any validation, allowing an attacker to inject arbitrary instructions.</p>
<sample src="examples/user-prompt-injection.js" />
<p>The fix validates the user input against a fixed allowlist of permitted values before
including it in the prompt.</p>
<sample src="examples/user-prompt-injection_fixed.js" />
</example>
<references>
<li>OWASP: <a href="https://genai.owasp.org/llmrisk/llm01-prompt-injection/">LLM01: Prompt Injection</a>.</li>
<li>MITRE CWE: <a href="https://cwe.mitre.org/data/definitions/1427.html">CWE-1427: Improper Neutralization of Input Used for LLM Prompting</a>.</li>
</references>
</qhelp>

View File

@@ -3,9 +3,9 @@
* @description Untrusted input flowing into a user-role prompt of an AI model
* may allow an attacker to manipulate the model's behavior.
* @kind path-problem
* @problem.severity error
* @problem.severity warning
* @security-severity 5.0
* @precision high
* @precision medium
* @id js/user-prompt-injection
* @tags security
* experimental

View File

@@ -0,0 +1,26 @@
const express = require("express");
const OpenAI = require("openai");
const app = express();
const client = new OpenAI();
app.get("/chat", async (req, res) => {
let topic = req.query.topic;
// BAD: user input is used directly in a user-role prompt
const response = await client.chat.completions.create({
model: "gpt-4.1",
messages: [
{
role: "system",
content: "You are a helpful assistant that summarizes topics.",
},
{
role: "user",
content: "Summarize the following topic: " + topic,
},
],
});
res.json(response);
});

View File

@@ -0,0 +1,32 @@
const express = require("express");
const OpenAI = require("openai");
const app = express();
const client = new OpenAI();
const ALLOWED_TOPICS = ["science", "history", "technology"];
app.get("/chat", async (req, res) => {
let topic = req.query.topic;
// GOOD: user input is validated against a fixed allowlist before use in a prompt
if (!ALLOWED_TOPICS.includes(topic)) {
return res.status(400).json({ error: "Invalid topic" });
}
const response = await client.chat.completions.create({
model: "gpt-4.1",
messages: [
{
role: "system",
content: "You are a helpful assistant that summarizes topics.",
},
{
role: "user",
content: "Summarize the following topic: " + topic,
},
],
});
res.json(response);
});

View File

@@ -11,9 +11,9 @@ private import semmle.javascript.Concepts
private import semmle.javascript.security.dataflow.RemoteFlowSources
private import semmle.javascript.dataflow.internal.BarrierGuards
private import semmle.javascript.frameworks.data.ModelsAsData
private import experimental.semmle.javascript.frameworks.OpenAI
private import experimental.semmle.javascript.frameworks.Anthropic
private import experimental.semmle.javascript.frameworks.GoogleGenAI
private import semmle.javascript.frameworks.OpenAI
private import semmle.javascript.frameworks.Anthropic
private import semmle.javascript.frameworks.GoogleGenAI
/**
* Provides default sources, sinks and sanitizers for detecting

View File

@@ -1 +1 @@
experimental/Security/CWE-1427/SystemPromptInjection.ql
Security/CWE-1427/SystemPromptInjection.ql