mirror of
https://github.com/github/codeql.git
synced 2026-04-27 17:55:19 +02:00
python: Add query for prompt injection
This pull request introduces a new CodeQL query for detecting prompt injection vulnerabilities in Python code targeting AI prompting APIs such as agents and openai. The changes includes a new experimental query, new taint flow and type models, a customizable dataflow configuration, documentation, and comprehensive test coverage.
This commit is contained in:
@@ -0,0 +1,24 @@
|
||||
<!DOCTYPE qhelp PUBLIC
|
||||
"-//Semmle//qhelp//EN"
|
||||
"qhelp.dtd">
|
||||
<qhelp>
|
||||
|
||||
<overview>
|
||||
<p>Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or
|
||||
operations that were not intended.</p>
|
||||
</overview>
|
||||
|
||||
<recommendation>
|
||||
<p>Sanitize user input and also avoid using user input in developer or system level prompts.</p>
|
||||
</recommendation>
|
||||
|
||||
<example>
|
||||
<p>In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.</p>
|
||||
<sample src="examples/example.py" />
|
||||
</example>
|
||||
|
||||
<references>
|
||||
<li>OpenAI: <a href="https://openai.github.io/openai-guardrails-python">Guardrails</a>.</li>
|
||||
</references>
|
||||
|
||||
</qhelp>
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* @name Prompt injection
|
||||
* @kind path-problem
|
||||
* @problem.severity error
|
||||
* @security-severity 5.0
|
||||
* @precision high
|
||||
* @id py/prompt-injection
|
||||
* @tags security
|
||||
* experimental
|
||||
* external/cwe/cwe-1427
|
||||
*/
|
||||
|
||||
import python
|
||||
import experimental.semmle.python.security.dataflow.PromptInjectionQuery
|
||||
import PromptInjectionFlow::PathGraph
|
||||
|
||||
from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink
|
||||
where PromptInjectionFlow::flowPath(source, sink)
|
||||
select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(),
|
||||
"user-provided value"
|
||||
@@ -0,0 +1,17 @@
|
||||
from flask import Flask, request
|
||||
from agents import Agent
|
||||
from guardrails import GuardrailAgent
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input():
|
||||
input = request.args.get("input")
|
||||
|
||||
goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured.
|
||||
config=Path("guardrails_config.json"),
|
||||
name="Assistant",
|
||||
instructions="This prompt is customized for " + input)
|
||||
|
||||
badAgent = Agent(
|
||||
name="Assistant",
|
||||
instructions="This prompt is customized for " + input # BAD: user input in agent instruction.
|
||||
)
|
||||
Reference in New Issue
Block a user