mirror of
https://github.com/github/codeql.git
synced 2026-05-05 21:55:19 +02:00
python: Add query for prompt injection
This pull request introduces a new CodeQL query for detecting prompt injection vulnerabilities in Python code targeting AI prompting APIs such as agents and openai. The changes includes a new experimental query, new taint flow and type models, a customizable dataflow configuration, documentation, and comprehensive test coverage.
This commit is contained in:
@@ -87,6 +87,7 @@ ql/python/ql/src/experimental/Security/CWE-079/EmailXss.ql
|
|||||||
ql/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql
|
ql/python/ql/src/experimental/Security/CWE-091/XsltInjection.ql
|
||||||
ql/python/ql/src/experimental/Security/CWE-094/Js2Py.ql
|
ql/python/ql/src/experimental/Security/CWE-094/Js2Py.ql
|
||||||
ql/python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql
|
ql/python/ql/src/experimental/Security/CWE-1236/CsvInjection.ql
|
||||||
|
ql/python/ql/src/experimental/Security/CWE-1427/PromptInjection.ql
|
||||||
ql/python/ql/src/experimental/Security/CWE-176/UnicodeBypassValidation.ql
|
ql/python/ql/src/experimental/Security/CWE-176/UnicodeBypassValidation.ql
|
||||||
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/PossibleTimingAttackAgainstHash.ql
|
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/PossibleTimingAttackAgainstHash.ql
|
||||||
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/TimingAttackAgainstHash.ql
|
ql/python/ql/src/experimental/Security/CWE-208/TimingAttackAgainstHash/TimingAttackAgainstHash.ql
|
||||||
|
|||||||
@@ -0,0 +1,5 @@
|
|||||||
|
---
|
||||||
|
category: minorAnalysis
|
||||||
|
---
|
||||||
|
* Added experimental query `py/prompt-injection` to detect potential prompt injection vulnerabilities in code using LLMs.
|
||||||
|
* Added taint flow model and type model for `agents` and `openai` modules.
|
||||||
6
python/ql/lib/semmle/python/frameworks/agent.model.yml
Normal file
6
python/ql/lib/semmle/python/frameworks/agent.model.yml
Normal file
@@ -0,0 +1,6 @@
|
|||||||
|
extensions:
|
||||||
|
- addsTo:
|
||||||
|
pack: codeql/python-all
|
||||||
|
extensible: sinkModel
|
||||||
|
data:
|
||||||
|
- ['agents', 'Member[Agent].Argument[instructions:]', 'prompt-injection']
|
||||||
12
python/ql/lib/semmle/python/frameworks/openai.model.yml
Normal file
12
python/ql/lib/semmle/python/frameworks/openai.model.yml
Normal file
@@ -0,0 +1,12 @@
|
|||||||
|
extensions:
|
||||||
|
- addsTo:
|
||||||
|
pack: codeql/python-all
|
||||||
|
extensible: sinkModel
|
||||||
|
data:
|
||||||
|
- ['OpenAI', 'Member[beta].Member[assistants].Member[create].Argument[instructions:]', 'prompt-injection']
|
||||||
|
|
||||||
|
- addsTo:
|
||||||
|
pack: codeql/python-all
|
||||||
|
extensible: typeModel
|
||||||
|
data:
|
||||||
|
- ['OpenAI', 'openai', 'Member[OpenAI,AsyncOpenAI,AzureOpenAI].ReturnValue']
|
||||||
@@ -0,0 +1,24 @@
|
|||||||
|
<!DOCTYPE qhelp PUBLIC
|
||||||
|
"-//Semmle//qhelp//EN"
|
||||||
|
"qhelp.dtd">
|
||||||
|
<qhelp>
|
||||||
|
|
||||||
|
<overview>
|
||||||
|
<p>Prompts can be constructed to bypass the original purposes of an agent and lead to sensitive data leak or
|
||||||
|
operations that were not intended.</p>
|
||||||
|
</overview>
|
||||||
|
|
||||||
|
<recommendation>
|
||||||
|
<p>Sanitize user input and also avoid using user input in developer or system level prompts.</p>
|
||||||
|
</recommendation>
|
||||||
|
|
||||||
|
<example>
|
||||||
|
<p>In the following examples, the cases marked GOOD show secure prompt construction; whereas in the case marked BAD they may be susceptible to prompt injection.</p>
|
||||||
|
<sample src="examples/example.py" />
|
||||||
|
</example>
|
||||||
|
|
||||||
|
<references>
|
||||||
|
<li>OpenAI: <a href="https://openai.github.io/openai-guardrails-python">Guardrails</a>.</li>
|
||||||
|
</references>
|
||||||
|
|
||||||
|
</qhelp>
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
/**
|
||||||
|
* @name Prompt injection
|
||||||
|
* @kind path-problem
|
||||||
|
* @problem.severity error
|
||||||
|
* @security-severity 5.0
|
||||||
|
* @precision high
|
||||||
|
* @id py/prompt-injection
|
||||||
|
* @tags security
|
||||||
|
* experimental
|
||||||
|
* external/cwe/cwe-1427
|
||||||
|
*/
|
||||||
|
|
||||||
|
import python
|
||||||
|
import experimental.semmle.python.security.dataflow.PromptInjectionQuery
|
||||||
|
import PromptInjectionFlow::PathGraph
|
||||||
|
|
||||||
|
from PromptInjectionFlow::PathNode source, PromptInjectionFlow::PathNode sink
|
||||||
|
where PromptInjectionFlow::flowPath(source, sink)
|
||||||
|
select sink.getNode(), source, sink, "This prompt construction depends on a $@.", source.getNode(),
|
||||||
|
"user-provided value"
|
||||||
@@ -0,0 +1,17 @@
|
|||||||
|
from flask import Flask, request
|
||||||
|
from agents import Agent
|
||||||
|
from guardrails import GuardrailAgent
|
||||||
|
|
||||||
|
@app.route("/parameter-route")
|
||||||
|
def get_input():
|
||||||
|
input = request.args.get("input")
|
||||||
|
|
||||||
|
goodAgent = GuardrailAgent( # GOOD: Agent created with guardrails automatically configured.
|
||||||
|
config=Path("guardrails_config.json"),
|
||||||
|
name="Assistant",
|
||||||
|
instructions="This prompt is customized for " + input)
|
||||||
|
|
||||||
|
badAgent = Agent(
|
||||||
|
name="Assistant",
|
||||||
|
instructions="This prompt is customized for " + input # BAD: user input in agent instruction.
|
||||||
|
)
|
||||||
@@ -483,3 +483,28 @@ class EmailSender extends DataFlow::Node instanceof EmailSender::Range {
|
|||||||
*/
|
*/
|
||||||
DataFlow::Node getABody() { result in [super.getPlainTextBody(), super.getHtmlBody()] }
|
DataFlow::Node getABody() { result in [super.getPlainTextBody(), super.getHtmlBody()] }
|
||||||
}
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A data-flow node that prompts an AI model.
|
||||||
|
*
|
||||||
|
* Extend this class to refine existing API models. If you want to model new APIs,
|
||||||
|
* extend `AIPrompt::Range` instead.
|
||||||
|
*/
|
||||||
|
class AIPrompt extends DataFlow::Node instanceof AIPrompt::Range {
|
||||||
|
/** Gets an input that is used as AI prompt. */
|
||||||
|
DataFlow::Node getAPrompt() { result = super.getAPrompt() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Provides a class for modeling new AI prompting mechanisms. */
|
||||||
|
module AIPrompt {
|
||||||
|
/**
|
||||||
|
* A data-flow node that prompts an AI model.
|
||||||
|
*
|
||||||
|
* Extend this class to model new APIs. If you want to refine existing API models,
|
||||||
|
* extend `AIPrompt` instead.
|
||||||
|
*/
|
||||||
|
abstract class Range extends DataFlow::Node {
|
||||||
|
/** Gets an input that is used as AI prompt. */
|
||||||
|
abstract DataFlow::Node getAPrompt();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|||||||
@@ -13,6 +13,7 @@ private import experimental.semmle.python.frameworks.Scrapli
|
|||||||
private import experimental.semmle.python.frameworks.Twisted
|
private import experimental.semmle.python.frameworks.Twisted
|
||||||
private import experimental.semmle.python.frameworks.JWT
|
private import experimental.semmle.python.frameworks.JWT
|
||||||
private import experimental.semmle.python.frameworks.Csv
|
private import experimental.semmle.python.frameworks.Csv
|
||||||
|
private import experimental.semmle.python.frameworks.OpenAI
|
||||||
private import experimental.semmle.python.libraries.PyJWT
|
private import experimental.semmle.python.libraries.PyJWT
|
||||||
private import experimental.semmle.python.libraries.Python_JWT
|
private import experimental.semmle.python.libraries.Python_JWT
|
||||||
private import experimental.semmle.python.libraries.Authlib
|
private import experimental.semmle.python.libraries.Authlib
|
||||||
|
|||||||
@@ -0,0 +1,88 @@
|
|||||||
|
/**
|
||||||
|
* Provides classes modeling security-relevant aspects of the `openAI` Agents SDK package.
|
||||||
|
* See https://github.com/openai/openai-agents-python.
|
||||||
|
* As well as the regular openai python interface.
|
||||||
|
* See https://github.com/openai/openai-python.
|
||||||
|
*/
|
||||||
|
|
||||||
|
private import python
|
||||||
|
private import semmle.python.ApiGraphs
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides models for agents SDK (instances of the `agents.Runner` class etc).
|
||||||
|
*
|
||||||
|
* See https://github.com/openai/openai-agents-python.
|
||||||
|
*/
|
||||||
|
module AgentSDK {
|
||||||
|
/** Gets a reference to the `agents.Runner` class. */
|
||||||
|
API::Node classRef() { result = API::moduleImport("agents").getMember("Runner") }
|
||||||
|
|
||||||
|
/** Gets a reference to the `run` members. */
|
||||||
|
API::Node runMembers() { result = classRef().getMember(["run", "run_sync", "run_streamed"]) }
|
||||||
|
|
||||||
|
/** Gets a reference to a potential property of `agents.Runner` called input which can refer to a system prompt depending on the role specified. */
|
||||||
|
API::Node getContentNode() {
|
||||||
|
result = runMembers().getKeywordParameter("input").getASubscript().getSubscript("content")
|
||||||
|
or
|
||||||
|
result = runMembers().getParameter(_).getASubscript().getSubscript("content")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides models for Agent (instances of the `openai.OpenAI` class).
|
||||||
|
*
|
||||||
|
* See https://github.com/openai/openai-python.
|
||||||
|
*/
|
||||||
|
module OpenAI {
|
||||||
|
/** Gets a reference to the `openai.OpenAI` class. */
|
||||||
|
API::Node classRef() {
|
||||||
|
result =
|
||||||
|
API::moduleImport("openai").getMember(["OpenAI", "AsyncOpenAI", "AzureOpenAI"]).getReturn()
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Gets a reference to a potential property of `openai.OpenAI` called instructions which refers to the system prompt. */
|
||||||
|
API::Node getContentNode() {
|
||||||
|
exists(API::Node content |
|
||||||
|
content =
|
||||||
|
classRef()
|
||||||
|
.getMember("responses")
|
||||||
|
.getMember("create")
|
||||||
|
.getKeywordParameter(["input", "instructions"])
|
||||||
|
or
|
||||||
|
content =
|
||||||
|
classRef()
|
||||||
|
.getMember("responses")
|
||||||
|
.getMember("create")
|
||||||
|
.getKeywordParameter(["input", "instructions"])
|
||||||
|
.getASubscript()
|
||||||
|
.getSubscript("content")
|
||||||
|
or
|
||||||
|
content =
|
||||||
|
classRef()
|
||||||
|
.getMember("realtime")
|
||||||
|
.getMember("connect")
|
||||||
|
.getReturn()
|
||||||
|
.getMember("conversation")
|
||||||
|
.getMember("item")
|
||||||
|
.getMember("create")
|
||||||
|
.getKeywordParameter("item")
|
||||||
|
.getSubscript("content")
|
||||||
|
or
|
||||||
|
content =
|
||||||
|
classRef()
|
||||||
|
.getMember("chat")
|
||||||
|
.getMember("completions")
|
||||||
|
.getMember("create")
|
||||||
|
.getKeywordParameter("messages")
|
||||||
|
.getASubscript()
|
||||||
|
.getSubscript("content")
|
||||||
|
|
|
||||||
|
// content
|
||||||
|
if not exists(content.getASubscript())
|
||||||
|
then result = content
|
||||||
|
else
|
||||||
|
// content.text
|
||||||
|
result = content.getASubscript().getSubscript("text")
|
||||||
|
)
|
||||||
|
}
|
||||||
|
}
|
||||||
@@ -0,0 +1,65 @@
|
|||||||
|
/**
|
||||||
|
* Provides default sources, sinks and sanitizers for detecting
|
||||||
|
* "prompt injection"
|
||||||
|
* vulnerabilities, as well as extension points for adding your own.
|
||||||
|
*/
|
||||||
|
|
||||||
|
import python
|
||||||
|
private import semmle.python.dataflow.new.DataFlow
|
||||||
|
private import semmle.python.Concepts
|
||||||
|
private import experimental.semmle.python.Concepts
|
||||||
|
private import semmle.python.dataflow.new.RemoteFlowSources
|
||||||
|
private import semmle.python.dataflow.new.BarrierGuards
|
||||||
|
private import semmle.python.frameworks.data.ModelsAsData
|
||||||
|
private import experimental.semmle.python.frameworks.OpenAI
|
||||||
|
|
||||||
|
/**
|
||||||
|
* Provides default sources, sinks and sanitizers for detecting
|
||||||
|
* "prompt injection"
|
||||||
|
* vulnerabilities, as well as extension points for adding your own.
|
||||||
|
*/
|
||||||
|
module PromptInjection {
|
||||||
|
/**
|
||||||
|
* A data flow source for "prompt injection" vulnerabilities.
|
||||||
|
*/
|
||||||
|
abstract class Source extends DataFlow::Node { }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A data flow sink for "prompt injection" vulnerabilities.
|
||||||
|
*/
|
||||||
|
abstract class Sink extends DataFlow::Node { }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A sanitizer for "prompt injection" vulnerabilities.
|
||||||
|
*/
|
||||||
|
abstract class Sanitizer extends DataFlow::Node { }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* An active threat-model source, considered as a flow source.
|
||||||
|
*/
|
||||||
|
private class ActiveThreatModelSourceAsSource extends Source, ActiveThreatModelSource { }
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A prompt to an AI model, considered as a flow sink.
|
||||||
|
*/
|
||||||
|
class AIPromptAsSink extends Sink {
|
||||||
|
AIPromptAsSink() { this = any(AIPrompt p).getAPrompt() }
|
||||||
|
}
|
||||||
|
|
||||||
|
private class SinkFromModel extends Sink {
|
||||||
|
SinkFromModel() { this = ModelOutput::getASinkNode("prompt-injection").asSink() }
|
||||||
|
}
|
||||||
|
|
||||||
|
private class PromptContentSink extends Sink {
|
||||||
|
PromptContentSink() {
|
||||||
|
this = OpenAI::getContentNode().asSink()
|
||||||
|
or
|
||||||
|
this = AgentSDK::getContentNode().asSink()
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
/**
|
||||||
|
* A comparison with a constant, considered as a sanitizer-guard.
|
||||||
|
*/
|
||||||
|
class ConstCompareAsSanitizerGuard extends Sanitizer, ConstCompareBarrier { }
|
||||||
|
}
|
||||||
@@ -0,0 +1,25 @@
|
|||||||
|
/**
|
||||||
|
* Provides a taint-tracking configuration for detecting "prompt injection" vulnerabilities.
|
||||||
|
*
|
||||||
|
* Note, for performance reasons: only import this file if
|
||||||
|
* `PromptInjection::Configuration` is needed, otherwise
|
||||||
|
* `PromptInjectionCustomizations` should be imported instead.
|
||||||
|
*/
|
||||||
|
|
||||||
|
private import python
|
||||||
|
import semmle.python.dataflow.new.DataFlow
|
||||||
|
import semmle.python.dataflow.new.TaintTracking
|
||||||
|
import PromptInjectionCustomizations::PromptInjection
|
||||||
|
|
||||||
|
private module PromptInjectionConfig implements DataFlow::ConfigSig {
|
||||||
|
predicate isSource(DataFlow::Node node) { node instanceof Source }
|
||||||
|
|
||||||
|
predicate isSink(DataFlow::Node node) { node instanceof Sink }
|
||||||
|
|
||||||
|
predicate isBarrier(DataFlow::Node node) { node instanceof Sanitizer }
|
||||||
|
|
||||||
|
predicate observeDiffInformedIncrementalMode() { any() }
|
||||||
|
}
|
||||||
|
|
||||||
|
/** Global taint-tracking for detecting "prompt injection" vulnerabilities. */
|
||||||
|
module PromptInjectionFlow = TaintTracking::Global<PromptInjectionConfig>;
|
||||||
@@ -0,0 +1,94 @@
|
|||||||
|
#select
|
||||||
|
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:33:33:33:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:33:33:33:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:53:33:53:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:53:33:53:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:67:28:67:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:67:28:67:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:71:28:71:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:71:28:71:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:84:28:84:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:84:28:84:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||||
|
edges
|
||||||
|
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||||
|
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | |
|
||||||
|
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:17:13:17:19 | ControlFlowNode for request | provenance | |
|
||||||
|
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:94 |
|
||||||
|
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||||
|
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||||
|
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | |
|
||||||
|
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | provenance | |
|
||||||
|
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | provenance | |
|
||||||
|
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||||
|
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||||
|
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | agent_instructions.py:17:5:17:9 | ControlFlowNode for input | provenance | |
|
||||||
|
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||||
|
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | |
|
||||||
|
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58614 |
|
||||||
|
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||||
|
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||||
|
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||||
|
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:33:33:33:37 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:53:33:53:37 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:67:28:67:32 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:71:28:71:32 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:84:28:84:32 | ControlFlowNode for query | provenance | |
|
||||||
|
| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||||
|
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||||
|
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | |
|
||||||
|
nodes
|
||||||
|
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||||
|
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||||
|
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||||
|
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||||
|
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||||
|
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||||
|
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||||
|
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||||
|
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||||
|
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||||
|
| openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona |
|
||||||
|
| openai_test.py:12:15:12:21 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||||
|
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||||
|
| openai_test.py:13:5:13:9 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||||
|
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||||
|
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||||
|
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:33:33:33:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:53:33:53:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:67:28:67:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:71:28:71:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
| openai_test.py:84:28:84:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||||
|
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||||
|
subpaths
|
||||||
@@ -0,0 +1,2 @@
|
|||||||
|
query: experimental/Security/CWE-1427/PromptInjection.ql
|
||||||
|
postprocess: utils/test/InlineExpectationsTestQuery.ql
|
||||||
@@ -0,0 +1,38 @@
|
|||||||
|
from agents import Agent, Runner
|
||||||
|
from flask import Flask, request # $ Source
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
@app.route("/parameter-route")
|
||||||
|
def get_input1():
|
||||||
|
input = request.args.get("input")
|
||||||
|
|
||||||
|
agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection]
|
||||||
|
|
||||||
|
result = Runner.run_sync(agent, "This is a user message.")
|
||||||
|
print(result.final_output)
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/parameter-route")
|
||||||
|
def get_input2():
|
||||||
|
input = request.args.get("input")
|
||||||
|
|
||||||
|
agent = Agent(name="Assistant", instructions="This prompt is not customized.")
|
||||||
|
result = Runner.run_sync(
|
||||||
|
agent=agent,
|
||||||
|
input=[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": input, # $Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
result2 = Runner.run_sync(
|
||||||
|
agent,
|
||||||
|
[
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": input, # $Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
@@ -0,0 +1,93 @@
|
|||||||
|
from openai import OpenAI, AsyncOpenAI, AzureOpenAI
|
||||||
|
from flask import Flask, request # $ Source
|
||||||
|
app = Flask(__name__)
|
||||||
|
|
||||||
|
client = OpenAI()
|
||||||
|
async_client = AsyncOpenAI()
|
||||||
|
azure_client = AzureOpenAI()
|
||||||
|
|
||||||
|
|
||||||
|
@app.route("/openai")
|
||||||
|
async def get_input_openai():
|
||||||
|
persona = request.args.get("persona")
|
||||||
|
query = request.args.get("query")
|
||||||
|
role = request.args.get("role")
|
||||||
|
|
||||||
|
response1 = client.responses.create(
|
||||||
|
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||||
|
input=query, # $ Alert[py/prompt-injection]
|
||||||
|
)
|
||||||
|
|
||||||
|
response2 = client.responses.create(
|
||||||
|
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||||
|
input=[
|
||||||
|
{
|
||||||
|
"role": "developer",
|
||||||
|
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "input_text",
|
||||||
|
"text": query # $ Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
response3 = await async_client.responses.create(
|
||||||
|
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||||
|
input=query, # $ Alert[py/prompt-injection]
|
||||||
|
)
|
||||||
|
|
||||||
|
async with client.realtime.connect(model="gpt-realtime") as connection:
|
||||||
|
await connection.conversation.item.create(
|
||||||
|
item={
|
||||||
|
"type": "message",
|
||||||
|
"role": role,
|
||||||
|
"content": [
|
||||||
|
{
|
||||||
|
"type": "input_text",
|
||||||
|
"text": query # $ Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
],
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
completion1 = client.chat.completions.create(
|
||||||
|
messages=[
|
||||||
|
{
|
||||||
|
"role": "developer",
|
||||||
|
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": query, # $ Alert[py/prompt-injection]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": role,
|
||||||
|
"content": query, # $ Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
completion2 = azure_client.chat.completions.create(
|
||||||
|
messages=[
|
||||||
|
{
|
||||||
|
"role": "developer",
|
||||||
|
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"role": "user",
|
||||||
|
"content": query, # $ Alert[py/prompt-injection]
|
||||||
|
}
|
||||||
|
]
|
||||||
|
)
|
||||||
|
|
||||||
|
assistant = client.beta.assistants.create(
|
||||||
|
name="Test Agent",
|
||||||
|
model="gpt-4.1",
|
||||||
|
instructions="Talks like a " + persona # $ Alert[py/prompt-injection]
|
||||||
|
)
|
||||||
@@ -46,7 +46,9 @@ module KindValidation<KindValidationConfigSig Config> {
|
|||||||
// Go-only currently, but may be shared in the future
|
// Go-only currently, but may be shared in the future
|
||||||
"jwt",
|
"jwt",
|
||||||
// CPP-only currently
|
// CPP-only currently
|
||||||
"remote-sink"
|
"remote-sink",
|
||||||
|
// Python-only currently, but may be shared in the future
|
||||||
|
"prompt-injection"
|
||||||
]
|
]
|
||||||
or
|
or
|
||||||
this.matches([
|
this.matches([
|
||||||
|
|||||||
Reference in New Issue
Block a user