mirror of
https://github.com/github/codeql.git
synced 2026-02-12 05:01:06 +01:00
python: Add query for prompt injection
This pull request introduces a new CodeQL query for detecting prompt injection vulnerabilities in Python code targeting AI prompting APIs such as agents and openai. The changes includes a new experimental query, new taint flow and type models, a customizable dataflow configuration, documentation, and comprehensive test coverage.
This commit is contained in:
yoff
@@ -0,0 +1,94 @@
|
||||
#select
|
||||
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | This prompt construction depends on a $@. | agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:18:15:18:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:18:15:18:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:33:33:33:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:33:33:33:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:42:15:42:19 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:42:15:42:19 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:53:33:53:37 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:53:33:53:37 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:67:28:67:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:67:28:67:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:71:28:71:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:71:28:71:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:84:28:84:32 | ControlFlowNode for query | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:84:28:84:32 | ControlFlowNode for query | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | This prompt construction depends on a $@. | openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | user-provided value |
|
||||
edges
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | agent_instructions.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:7:13:7:19 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | agent_instructions.py:17:13:17:19 | ControlFlowNode for request | provenance | |
|
||||
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:94 |
|
||||
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | agent_instructions.py:7:5:7:9 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:25:28:25:32 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | agent_instructions.py:35:28:35:32 | ControlFlowNode for input | provenance | |
|
||||
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | agent_instructions.py:17:5:17:9 | ControlFlowNode for input | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | openai_test.py:2:26:2:32 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:12:15:12:21 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | openai_test.py:13:13:13:19 | ControlFlowNode for request | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | provenance | |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | provenance | Sink:MaD:58614 |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | openai_test.py:12:5:12:11 | ControlFlowNode for persona | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:18:15:18:19 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:33:33:33:37 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:42:15:42:19 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:53:33:53:37 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:67:28:67:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:71:28:71:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | openai_test.py:84:28:84:32 | ControlFlowNode for query | provenance | |
|
||||
| openai_test.py:13:13:13:19 | ControlFlowNode for request | openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | provenance | AdditionalTaintStep |
|
||||
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | provenance | dict.get |
|
||||
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | openai_test.py:13:5:13:9 | ControlFlowNode for query | provenance | |
|
||||
nodes
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||
| agent_instructions.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:7:5:7:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:7:13:7:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:7:13:7:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| agent_instructions.py:7:13:7:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| agent_instructions.py:9:50:9:89 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| agent_instructions.py:17:5:17:9 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:17:13:17:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| agent_instructions.py:17:13:17:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| agent_instructions.py:17:13:17:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| agent_instructions.py:25:28:25:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| agent_instructions.py:35:28:35:32 | ControlFlowNode for input | semmle.label | ControlFlowNode for input |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for ImportMember | semmle.label | ControlFlowNode for ImportMember |
|
||||
| openai_test.py:2:26:2:32 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:12:5:12:11 | ControlFlowNode for persona | semmle.label | ControlFlowNode for persona |
|
||||
| openai_test.py:12:15:12:21 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:12:15:12:26 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| openai_test.py:12:15:12:41 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| openai_test.py:13:5:13:9 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:13:13:13:19 | ControlFlowNode for request | semmle.label | ControlFlowNode for request |
|
||||
| openai_test.py:13:13:13:24 | ControlFlowNode for Attribute | semmle.label | ControlFlowNode for Attribute |
|
||||
| openai_test.py:13:13:13:37 | ControlFlowNode for Attribute() | semmle.label | ControlFlowNode for Attribute() |
|
||||
| openai_test.py:17:22:17:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:18:15:18:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:22:22:22:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:26:28:26:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:33:33:33:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:41:22:41:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:42:15:42:19 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:53:33:53:37 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:63:28:63:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:67:28:67:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:71:28:71:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:80:28:80:51 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
| openai_test.py:84:28:84:32 | ControlFlowNode for query | semmle.label | ControlFlowNode for query |
|
||||
| openai_test.py:92:22:92:46 | ControlFlowNode for BinaryExpr | semmle.label | ControlFlowNode for BinaryExpr |
|
||||
subpaths
|
||||
@@ -0,0 +1,2 @@
|
||||
query: experimental/Security/CWE-1427/PromptInjection.ql
|
||||
postprocess: utils/test/InlineExpectationsTestQuery.ql
|
||||
@@ -0,0 +1,38 @@
|
||||
from agents import Agent, Runner
|
||||
from flask import Flask, request # $ Source
|
||||
app = Flask(__name__)
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input1():
|
||||
input = request.args.get("input")
|
||||
|
||||
agent = Agent(name="Assistant", instructions="This prompt is customized for " + input) # $Alert[py/prompt-injection]
|
||||
|
||||
result = Runner.run_sync(agent, "This is a user message.")
|
||||
print(result.final_output)
|
||||
|
||||
|
||||
@app.route("/parameter-route")
|
||||
def get_input2():
|
||||
input = request.args.get("input")
|
||||
|
||||
agent = Agent(name="Assistant", instructions="This prompt is not customized.")
|
||||
result = Runner.run_sync(
|
||||
agent=agent,
|
||||
input=[
|
||||
{
|
||||
"role": "user",
|
||||
"content": input, # $Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
result2 = Runner.run_sync(
|
||||
agent,
|
||||
[
|
||||
{
|
||||
"role": "user",
|
||||
"content": input, # $Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
@@ -0,0 +1,93 @@
|
||||
from openai import OpenAI, AsyncOpenAI, AzureOpenAI
|
||||
from flask import Flask, request # $ Source
|
||||
app = Flask(__name__)
|
||||
|
||||
client = OpenAI()
|
||||
async_client = AsyncOpenAI()
|
||||
azure_client = AzureOpenAI()
|
||||
|
||||
|
||||
@app.route("/openai")
|
||||
async def get_input_openai():
|
||||
persona = request.args.get("persona")
|
||||
query = request.args.get("query")
|
||||
role = request.args.get("role")
|
||||
|
||||
response1 = client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=query, # $ Alert[py/prompt-injection]
|
||||
)
|
||||
|
||||
response2 = client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": query # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
response3 = await async_client.responses.create(
|
||||
instructions="Talks like a " + persona, # $ Alert[py/prompt-injection]
|
||||
input=query, # $ Alert[py/prompt-injection]
|
||||
)
|
||||
|
||||
async with client.realtime.connect(model="gpt-realtime") as connection:
|
||||
await connection.conversation.item.create(
|
||||
item={
|
||||
"type": "message",
|
||||
"role": role,
|
||||
"content": [
|
||||
{
|
||||
"type": "input_text",
|
||||
"text": query # $ Alert[py/prompt-injection]
|
||||
}
|
||||
],
|
||||
}
|
||||
)
|
||||
|
||||
completion1 = client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": role,
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
completion2 = azure_client.chat.completions.create(
|
||||
messages=[
|
||||
{
|
||||
"role": "developer",
|
||||
"content": "Talk like a " + persona # $ Alert[py/prompt-injection]
|
||||
},
|
||||
{
|
||||
"role": "user",
|
||||
"content": query, # $ Alert[py/prompt-injection]
|
||||
}
|
||||
]
|
||||
)
|
||||
|
||||
assistant = client.beta.assistants.create(
|
||||
name="Test Agent",
|
||||
model="gpt-4.1",
|
||||
instructions="Talks like a " + persona # $ Alert[py/prompt-injection]
|
||||
)
|
||||
Reference in New Issue
Block a user