mirror of
https://github.com/github/codeql.git
synced 2026-02-12 05:01:06 +01:00
python: Add query for prompt injection
This pull request introduces a new CodeQL query for detecting prompt injection vulnerabilities in Python code targeting AI prompting APIs such as agents and openai. The changes includes a new experimental query, new taint flow and type models, a customizable dataflow configuration, documentation, and comprehensive test coverage.
This commit is contained in:
yoff
@@ -87,6 +87,7 @@ ql/python/ql/src/experimental/Security/CWE-079/EmailXss.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-094/Js2Py.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-176/UnicodeBypassValidation.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/PossibleTimingAttackAgainstHash.ql
|
||||
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/TimingAttackAgainstHash.ql
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
---
|
||||
category: minorAnalysis
|
||||
---
|
||||
* Added experimental query `py/prompt-injection` to detect potential prompt injection vulnerabilities in code using LLMs.
|
||||
* Added taint flow model and type model for `agents` and `openai` modules.
|
||||
6
python/ql/lib/semmle/python/frameworks/agent.model.yml
Normal file
6
python/ql/lib/semmle/python/frameworks/agent.model.yml
Normal file
@@ -0,0 +1,6 @@
|
||||
extensions:
|
||||
- addsTo:
|
||||
pack: codeql/python-all
|
||||
extensible: sinkModel
|
||||
data:
|
||||
- ['agents', 'Member[Agent].Argument[instructions:]', 'prompt-injection']
|
||||
12
python/ql/lib/semmle/python/frameworks/openai.model.yml
Normal file
12
python/ql/lib/semmle/python/frameworks/openai.model.yml
Normal file
@@ -0,0 +1,12 @@
|
||||
extensions:
|
||||
- addsTo:
|
||||
pack: codeql/python-all
|
||||
extensible: sinkModel
|
||||
data:
|
||||
- ['OpenAI', 'Member[beta].Member[assistants].Member[create].Argument[instructions:]', 'prompt-injection']
|
||||
|
||||
- addsTo:
|
||||
pack: codeql/python-all
|
||||
extensible: typeModel
|
||||
data:
|
||||
- ['OpenAI', 'openai', 'Member[OpenAI,AsyncOpenAI,AzureOpenAI].ReturnValue']
|
||||
@@ -0,0 +1,24 @@
|
||||
<!DOCTYPE qhelp PUBLIC
|
||||
"-//Semmle//qhelp//EN"
|
||||
"qhelp.dtd">
|
||||
<qhelp>
|
||||
|
||||
<overview>
|
||||
<p>Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or
|
||||
operations that were not intended.</p>
|
||||
</overview>
|
||||
|
||||
<recommendation>
|
||||
<p>Sanitize user input and also avoid using user input in developer or system level prompts.</p>
|
||||
</recommendation>
|
||||
|
||||
<example>
|
||||
<p>In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.</p>
|
||||
<sample src="examples/example.py" />
|
||||
</example>
|
||||
|
||||
<references>
|
||||
<li>OpenAI: <a href="https://openai.github.io/openai-guardrails-python">Guardrails</a>.</li>
|
||||
</references>
|
||||
|
||||
</qhelp>
|
||||
@@ -0,0 +1,20 @@
|
||||
/**
|
||||
* @name Prompt injection
|
||||
* @kind path-problem
|
||||
* @problem.severity error
|
||||
* @security-severity 5.0
|
||||
* @precision high
|
||||
* @id py/prompt-injection
|
||||
* @tags security
|
||||
* experimental
|
||||
* external/cwe/cwe-1427
|
||||
*/
|
||||
|
||||
import python
|
||||
import experimental.semmle.python.security.dataflow.PromptInjectionQuery
|
||||
import PromptInjectionFlow::PathGraph
|
||||
|
||||
from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink
|
||||
where PromptInjectionFlow::flowPath(source, sink)
|
||||
select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(),
|
||||
"user-provided value"
|
||||
@@ -0,0 +1,17 @@
|
||||
from flask import Flask, request
|
||||
from agents import Agent
|
||||
from guardrails import GuardrailAgent
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input():
|
||||
input = request.args.get("input")
|
||||
|
||||
goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured.
|
||||
config=Path("guardrails_config.json"),
|
||||
name="Assistant",
|
||||
instructions="This prompt is customized for " + input)
|
||||
|
||||
badAgent = Agent(
|
||||
name="Assistant",
|
||||
instructions="This prompt is customized for " + input # BAD: user input in agent instruction.
|
||||
)
|
||||
@@ -483,3 +483,28 @@ class EmailSender extends DataFlow::Node instanceof EmailSender::Range {
|
||||
*/
|
||||
DataFlow::Node getABody() { result in [super.getPlainTextBody(), super.getHtmlBody()] }
|
||||
}
|
||||
|
||||
/**
|
||||
* A data-flow node that prompts an AI model.
|
||||
*
|
||||
* Extend this class to refine existing API models. If you want to model new APIs,
|
||||
* extend `AIPrompt::Range` instead.
|
||||
*/
|
||||
class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range {
|
||||
/** Gets an input that is used as AI prompt. */
|
||||
DataFlow::Node getAPrompt() { result = super.getAPrompt() }
|
||||
}
|
||||
|
||||
/** Provides a class for modeling new AI prompting mechanisms. */
|
||||
module AIPrompt {
|
||||
/**
|
||||
* A data-flow node that prompts an AI model.
|
||||
*
|
||||
* Extend this class to model new APIs. If you want to refine existing API models,
|
||||
* extend `AIPrompt` instead.
|
||||
*/
|
||||
abstract class Range extends DataFlow::Node {
|
||||
/** Gets an input that is used as AI prompt. */
|
||||
abstract DataFlow::Node getAPrompt();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -13,6 +13,7 @@ private import experimental.semmle.python.frameworks.Scrapli
|
||||
private import experimental.semmle.python.frameworks.Twisted
|
||||
private import experimental.semmle.python.frameworks.JWT
|
||||
private import experimental.semmle.python.frameworks.Csv
|
||||
private import experimental.semmle.python.frameworks.OpenAI
|
||||
private import experimental.semmle.python.libraries.PyJWT
|
||||
private import experimental.semmle.python.libraries.Python_JWT
|
||||
private import experimental.semmle.python.libraries.Authlib
|
||||
|
||||
@@ -0,0 +1,88 @@
|
||||
/**
|
||||
* Provides classes modeling security-relevant aspects of the `openAI` Agents SDK package.
|
||||
* See https://github.com/openai/openai-agents-python.
|
||||
* As well as the regular openai python interface.
|
||||
* See https://github.com/openai/openai-python.
|
||||
*/
|
||||
|
||||
private import python
|
||||
private import semmle.python.ApiGraphs
|
||||
|
||||
/**
|
||||
* Provides models for agents SDK (instances of the `agents.Runner` class etc).
|
||||
*
|
||||
* See https://github.com/openai/openai-agents-python.
|
||||
*/
|
||||
module AgentSDK {
|
||||
/** Gets a reference to the `agents.Runner` class. */
|
||||
API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") }
|
||||
|
||||
/** Gets a reference to the `run` members. */
|
||||
API::Node runMembers() { result = classRef().getMember(["run", "run_sync", "run_streamed"]) }
|
||||
|
||||
/** Gets a reference to a potential property of `agents.Runner` called input which can refer to a system prompt depending on the role specified. */
|
||||
API::Node getContentNode() {
|
||||
result = runMembers().getKeywordParameter("input").getASubscript().getSubscript("content")
|
||||
or
|
||||
result = runMembers().getParameter(_).getASubscript().getSubscript("content")
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Provides models for Agent (instances of the `openai.OpenAI` class).
|
||||
*
|
||||
* See https://github.com/openai/openai-python.
|
||||
*/
|
||||
module OpenAI {
|
||||
/** Gets a reference to the `openai.OpenAI` class. */
|
||||
API::Node classRef() {
|
||||
result =
|
||||
API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]).getReturn()
|
||||
}
|
||||
|
||||
/** Gets a reference to a potential property of `openai.OpenAI` called instructions which refers to the system prompt. */
|
||||
API::Node getContentNode() {
|
||||
exists(API::Node content |
|
||||
content =
|
||||
classRef()
|
||||
.getMember("responses")
|
||||
.getMember("create")
|
||||
.getKeywordParameter(["input", "instructions"])
|
||||
or
|
||||
content =
|
||||
classRef()
|
||||
.getMember("responses")
|
||||
.getMember("create")
|
||||
.getKeywordParameter(["input", "instructions"])
|
||||
.getASubscript()
|
||||
.getSubscript("content")
|
||||
or
|
||||
content =
|
||||
classRef()
|
||||
.getMember("realtime")
|
||||
.getMember("connect")
|
||||
.getReturn()
|
||||
.getMember("conversation")
|
||||
.getMember("item")
|
||||
.getMember("create")
|
||||
.getKeywordParameter("item")
|
||||
.getSubscript("content")
|
||||
or
|
||||
content =
|
||||
classRef()
|
||||
.getMember("chat")
|
||||
.getMember("completions")
|
||||
.getMember("create")
|
||||
.getKeywordParameter("messages")
|
||||
.getASubscript()
|
||||
.getSubscript("content")
|
||||
|
|
||||
// content
|
||||
if not exists(content.getASubscript())
|
||||
then result = content
|
||||
else
|
||||
// content.text
|
||||
result = content.getASubscript().getSubscript("text")
|
||||
)
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
/**
|
||||
* Provides default sources, sinks and sanitizers for detecting
|
||||
* "prompt injection"
|
||||
* vulnerabilities, as well as extension points for adding your own.
|
||||
*/
|
||||
|
||||
import python
|
||||
private import semmle.python.dataflow.new.DataFlow
|
||||
private import semmle.python.Concepts
|
||||
private import experimental.semmle.python.Concepts
|
||||
private import semmle.python.dataflow.new.RemoteFlowSources
|
||||
private import semmle.python.dataflow.new.BarrierGuards
|
||||
private import semmle.python.frameworks.data.ModelsAsData
|
||||
private import experimental.semmle.python.frameworks.OpenAI
|
||||
|
||||
/**
|
||||
* Provides default sources, sinks and sanitizers for detecting
|
||||
* "prompt injection"
|
||||
* vulnerabilities, as well as extension points for adding your own.
|
||||
*/
|
||||
module PromptInjection {
|
||||
/**
|
||||
* A data flow source for "prompt injection" vulnerabilities.
|
||||
*/
|
||||
abstract class Source extends DataFlow::Node { }
|
||||
|
||||
/**
|
||||
* A data flow sink for "prompt injection" vulnerabilities.
|
||||
*/
|
||||
abstract class Sink extends DataFlow::Node { }
|
||||
|
||||
/**
|
||||
* A sanitizer for "prompt injection" vulnerabilities.
|
||||
*/
|
||||
abstract class Sanitizer extends DataFlow::Node { }
|
||||
|
||||
/**
|
||||
* An active threat-model source, considered as a flow source.
|
||||
*/
|
||||
private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
|
||||
|
||||
/**
|
||||
* A prompt to an AI model, considered as a flow sink.
|
||||
*/
|
||||
class AIPromptAsSink extends Sink {
|
||||
AIPromptAsSink() { this = any(AIPrompt p).getAPrompt() }
|
||||
}
|
||||
|
||||
private class SinkFromModel extends Sink {
|
||||
SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() }
|
||||
}
|
||||
|
||||
private class PromptContentSink extends Sink {
|
||||
PromptContentSink() {
|
||||
this = OpenAI::getContentNode().asSink()
|
||||
or
|
||||
this = AgentSDK::getContentNode().asSink()
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* A comparison with a constant, considered as a sanitizer-guard.
|
||||
*/
|
||||
class ConstCompareAsSanitizerGuard extends Sanitizer, ConstCompareBarrier { }
|
||||
}
|
||||
@@ -0,0 +1,25 @@
|
||||
/**
|
||||
* Provides a taint-tracking configuration for detecting "prompt injection" vulnerabilities.
|
||||
*
|
||||
* Note, for performance reasons: only import this file if
|
||||
* `PromptInjection::Configuration` is needed, otherwise
|
||||
* `PromptInjectionCustomizations` should be imported instead.
|
||||
*/
|
||||
|
||||
private import python
|
||||
import semmle.python.dataflow.new.DataFlow
|
||||
import semmle.python.dataflow.new.TaintTracking
|
||||
import PromptInjectionCustomizations::PromptInjection
|
||||
|
||||
private module PromptInjectionConfig implements DataFlow::ConfigSig {
|
||||
predicate isSource(DataFlow::Node node) { node instanceof Source }
|
||||
|
||||
predicate isSink(DataFlow::Node node) { node instanceof Sink }
|
||||
|
||||
predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
|
||||
|
||||
predicate observeDiffInformedIncrementalMode() { any() }
|
||||
}
|
||||
|
||||
/** Global taint-tracking for detecting "prompt injection" vulnerabilities. */
|
||||
module PromptInjectionFlow = TaintTracking::Global<PromptInjectionConfig>;
|
||||
@@ -0,0 +1,94 @@
|
||||
#select
|
||||
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:33:33:33:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:33:33:33:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:53:33:53:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:53:33:53:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:67:28:67:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:67:28:67:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:71:28:71:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:71:28:71:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:84:28:84:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:84:28:84:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
edges
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:17:13:17:19 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:94 |
|
||||
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | agent_instructions.py:17:5:17:9 | ControlFlowNode for input | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58614 |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:33:33:33:37 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:53:33:53:37 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:67:28:67:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:71:28:71:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:84:28:84:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | |
|
||||
nodes
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:33:33:33:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:53:33:53:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:67:28:67:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:71:28:71:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:84:28:84:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
subpaths
|
||||
@@ -0,0 +1,2 @@
|
||||
query: experimental/Security/CWE-1427/PromptInjection.ql
|
||||
postprocess: utils/test/InlineExpectationsTestQuery.ql
|
||||
@@ -0,0 +1,38 @@
|
||||
from agents import Agent, Runner
|
||||
from flask import Flask, request # $ Source
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input1():
|
||||
input = request.args.get("input")
|
||||
|
||||
agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection]
|
||||
|
||||
result = Runner.run_sync(agent, "This is a user message.")
|
||||
print(result.final_output)
|
||||
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input2():
|
||||
input = request.args.get("input")
|
||||
|
||||
agent = Agent(name="Assistant", instructions="This prompt is not customized.")
|
||||
result = Runner.run_sync(
|
||||
agent=agent,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": input, # $Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result2 = Runner.run_sync(
|
||||
agent,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": input, # $Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
from openai import OpenAI, AsyncOpenAI, AzureOpenAI
|
||||
from flask import Flask, request # $ Source
|
||||
app = Flask(__name__)
|
||||
|
||||
client = OpenAI()
|
||||
async_client = AsyncOpenAI()
|
||||
azure_client = AzureOpenAI()
|
||||
|
||||
|
||||
@app.route("/openai")
|
||||
async def get_input_openai():
|
||||
persona = request.args.get("persona")
|
||||
query = request.args.get("query")
|
||||
role = request.args.get("role")
|
||||
|
||||
response1 = client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=query, # $ Alert[py/prompt-injection]
|
||||
)
|
||||
|
||||
response2 = client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": query # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response3 = await async_client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=query, # $ Alert[py/prompt-injection]
|
||||
)
|
||||
|
||||
async with client.realtime.connect(model="gpt-realtime") as connection:
|
||||
await connection.conversation.item.create(
|
||||
item={
|
||||
"type": "message",
|
||||
"role": role,
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": query # $ Alert[py/prompt-injection]
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
completion1 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": role,
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
completion2 = azure_client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Test Agent",
|
||||
model="gpt-4.1",
|
||||
instructions="Talks like a " + persona # $ Alert[py/prompt-injection]
|
||||
)
|
||||
Reference in New Issue
Block a user