hohn/codeql
1
0
Fork 0
mirror of https://github.com/github/codeql.git synced 2026-05-14 19:29:28 +02:00
Code Issues Packages Projects Releases Wiki Activity

remove guardrails sanitizer for now

Browse Source
This commit is contained in:
BazookaMusic
2026-05-13 13:37:44 +02:00
parent 34da804aee
commit 9c136264de
4 changed files with 113 additions and 84 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
javascript/ql/lib/ext/openai.model.yml
View File

@@ -21,8 +21,3 @@ extensions:
- ["@openai/agents", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"]
- ["@openai/guardrails", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"]
- ["@openai/guardrails", "Member[GuardrailAgent].Member[create].Argument[2]", "system-prompt-injection"]
- ["openai.Client", "Member[responses].Member[create].Argument[0].Member[input]", "user-prompt-injection"]
- ["openai.Client", "Member[completions].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"]
- ["openai.Client", "Member[images].Member[generate,edit].Argument[0].Member[prompt]", "user-prompt-injection"]
- ["openai.Client", "Member[embeddings].Member[create].Argument[0].Member[input]", "user-prompt-injection"]
- ["openai.Client", "Member[audio].Member[transcriptions,translations].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"]

86
javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll
View File

@@ -16,18 +16,6 @@ private predicate isSystemOrDevMessage(API::Node msg) {
msg.getMember("role").asSink().mayHaveStringValue(["system", "developer", "assistant"])
}
module OpenAIGuardrails {
/** Gets a reference to the `GuardrailsOpenAI` class. */
API::Node classRef() {
result = API::moduleImport("@openai/guardrails")
}
API::Node getSanitizerNode() {
// checkPlainText(userInput, bundle) or runGuardrails(userInput, bundle)
result = classRef().getMember(["checkPlainText", "runGuardrails"])
}
}
module OpenAI {
/** Gets a reference to all OpenAI client instances. */
private API::Node allClients() {
@@ -43,6 +31,33 @@ module OpenAI {
.getPromised()
}
/** Gets a guarded client that is clearly configured without input guardrails. */
private API::Node unprotectedGuardedClient() {
exists(API::Node createCall |
createCall =
API::moduleImport("@openai/guardrails")
.getMember(["GuardrailsOpenAI", "GuardrailsAzureOpenAI"])
.getMember("create") and
result = createCall.getReturn().getPromised() and
exists(createCall.getParameter(0).getMember("version")) and
not exists(
createCall.getParameter(0).getMember("input").getMember("guardrails").getArrayElement()
) and
not exists(
createCall.getParameter(0).getMember("pre_flight").getMember("guardrails").getArrayElement()
)
)
}
/** Gets a reference to all clients without input guardrails. */
private API::Node clientsNoGuardrails() {
result = API::moduleImport("openai").getInstance()
or
result = API::moduleImport("openai").getMember(["OpenAI", "AzureOpenAI"]).getInstance()
or
result = unprotectedGuardedClient()
}
/**
* Gets role-filtered system/developer/assistant message sinks.
* These require checking a sibling `role` property and cannot be expressed in MaD.
@@ -100,10 +115,18 @@ module OpenAI {
* These require checking a sibling `role` property and cannot be expressed in MaD.
*/
API::Node getUserPromptNode() {
// responses.create({ input: "string" })
result =
clientsNoGuardrails()
.getMember("responses")
.getMember("create")
.getParameter(0)
.getMember("input")
or
// responses.create({ input: [{ role: "user", content: ... }] })
exists(API::Node msg |
msg =
allClients()
clientsNoGuardrails()
.getMember("responses")
.getMember("create")
.getParameter(0)
@@ -117,7 +140,7 @@ module OpenAI {
// chat.completions.create({ messages: [{ role: "user", content: ... }] })
exists(API::Node msg, API::Node content |
msg =
allClients()
clientsNoGuardrails()
.getMember("chat")
.getMember("completions")
.getMember("create")
@@ -132,10 +155,34 @@ module OpenAI {
result = content.getArrayElement().getMember("text")
)
or
// Legacy completions API: completions.create({ prompt: ... })
result =
clientsNoGuardrails()
.getMember("completions")
.getMember("create")
.getParameter(0)
.getMember("prompt")
or
// images.generate({ prompt: ... }) and images.edit({ prompt: ... })
result =
clientsNoGuardrails()
.getMember("images")
.getMember(["generate", "edit"])
.getParameter(0)
.getMember("prompt")
or
// embeddings.create({ input: ... })
result =
clientsNoGuardrails()
.getMember("embeddings")
.getMember("create")
.getParameter(0)
.getMember("input")
or
// beta.threads.messages.create(threadId, { role: "user", content: ... })
exists(API::Node msg |
msg =
allClients()
clientsNoGuardrails()
.getMember("beta")
.getMember("threads")
.getMember("messages")
@@ -145,6 +192,15 @@ module OpenAI {
|
result = msg.getMember("content")
)
or
// audio.transcriptions/translations.create({ prompt: ... })
result =
clientsNoGuardrails()
.getMember("audio")
.getMember(["transcriptions", "translations"])
.getMember("create")
.getParameter(0)
.getMember("prompt")
}
}

84
javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected
View File

@@ -9,20 +9,20 @@ edges
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:51:13:51:21 | userInput | provenance | |
| gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:58:13:58:21 | userInput | provenance | |
| gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:8:9:8:17 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:24:12:24:20 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:33:18:33:26 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:44:18:44:26 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:58:19:58:27 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:68:13:68:21 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:73:13:73:21 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:77:13:77:21 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:83:12:83:20 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:90:13:90:21 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:96:13:96:21 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:102:14:102:22 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:108:12:108:20 | userInput | provenance | |
| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:155:12:155:20 | userInput | provenance | |
| openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:16:9:16:17 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:22:12:22:20 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:31:18:31:26 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:42:18:42:26 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:56:19:56:27 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:66:13:66:21 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:71:13:71:21 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:75:13:75:21 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:81:12:81:20 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:88:13:88:21 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:13:94:21 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:14:100:22 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:106:12:106:20 | userInput | provenance | |
| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:153:12:153:20 | userInput | provenance | |
| openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:14:9:14:17 | userInput | provenance | |
nodes
| anthropic_user_test.js:8:9:8:17 | userInput | semmle.label | userInput |
| anthropic_user_test.js:8:21:8:39 | req.query.userInput | semmle.label | req.query.userInput |
@@ -36,21 +36,21 @@ nodes
| gemini_user_test.js:44:13:44:21 | userInput | semmle.label | userInput |
| gemini_user_test.js:51:13:51:21 | userInput | semmle.label | userInput |
| gemini_user_test.js:58:13:58:21 | userInput | semmle.label | userInput |
| openai_user_test.js:16:9:16:17 | userInput | semmle.label | userInput |
| openai_user_test.js:16:21:16:39 | req.query.userInput | semmle.label | req.query.userInput |
| openai_user_test.js:24:12:24:20 | userInput | semmle.label | userInput |
| openai_user_test.js:33:18:33:26 | userInput | semmle.label | userInput |
| openai_user_test.js:44:18:44:26 | userInput | semmle.label | userInput |
| openai_user_test.js:58:19:58:27 | userInput | semmle.label | userInput |
| openai_user_test.js:68:13:68:21 | userInput | semmle.label | userInput |
| openai_user_test.js:73:13:73:21 | userInput | semmle.label | userInput |
| openai_user_test.js:77:13:77:21 | userInput | semmle.label | userInput |
| openai_user_test.js:83:12:83:20 | userInput | semmle.label | userInput |
| openai_user_test.js:90:13:90:21 | userInput | semmle.label | userInput |
| openai_user_test.js:96:13:96:21 | userInput | semmle.label | userInput |
| openai_user_test.js:102:14:102:22 | userInput | semmle.label | userInput |
| openai_user_test.js:108:12:108:20 | userInput | semmle.label | userInput |
| openai_user_test.js:155:12:155:20 | userInput | semmle.label | userInput |
| openai_user_test.js:14:9:14:17 | userInput | semmle.label | userInput |
| openai_user_test.js:14:21:14:39 | req.query.userInput | semmle.label | req.query.userInput |
| openai_user_test.js:22:12:22:20 | userInput | semmle.label | userInput |
| openai_user_test.js:31:18:31:26 | userInput | semmle.label | userInput |
| openai_user_test.js:42:18:42:26 | userInput | semmle.label | userInput |
| openai_user_test.js:56:19:56:27 | userInput | semmle.label | userInput |
| openai_user_test.js:66:13:66:21 | userInput | semmle.label | userInput |
| openai_user_test.js:71:13:71:21 | userInput | semmle.label | userInput |
| openai_user_test.js:75:13:75:21 | userInput | semmle.label | userInput |
| openai_user_test.js:81:12:81:20 | userInput | semmle.label | userInput |
| openai_user_test.js:88:13:88:21 | userInput | semmle.label | userInput |
| openai_user_test.js:94:13:94:21 | userInput | semmle.label | userInput |
| openai_user_test.js:100:14:100:22 | userInput | semmle.label | userInput |
| openai_user_test.js:106:12:106:20 | userInput | semmle.label | userInput |
| openai_user_test.js:153:12:153:20 | userInput | semmle.label | userInput |
subpaths
#select
| anthropic_user_test.js:18:18:18:26 | userInput | anthropic_user_test.js:8:21:8:39 | req.query.userInput | anthropic_user_test.js:18:18:18:26 | userInput | This prompt construction depends on a $@. | anthropic_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
@@ -61,16 +61,16 @@ subpaths
| gemini_user_test.js:44:13:44:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:44:13:44:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
| gemini_user_test.js:51:13:51:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:51:13:51:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
| gemini_user_test.js:58:13:58:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:58:13:58:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value |
| openai_user_test.js:24:12:24:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:24:12:24:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:33:18:33:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:33:18:33:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:44:18:44:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:44:18:44:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:58:19:58:27 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:58:19:58:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:68:13:68:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:68:13:68:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:73:13:73:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:73:13:73:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:77:13:77:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:77:13:77:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:83:12:83:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:83:12:83:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:90:13:90:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:90:13:90:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:96:13:96:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:96:13:96:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:102:14:102:22 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:102:14:102:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:108:12:108:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:108:12:108:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:155:12:155:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:155:12:155:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value |
| openai_user_test.js:22:12:22:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:22:12:22:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:31:18:31:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:31:18:31:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:42:18:42:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:42:18:42:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:56:19:56:27 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:56:19:56:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:66:13:66:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:66:13:66:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:71:13:71:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:71:13:71:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:75:13:75:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:75:13:75:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:81:12:81:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:81:12:81:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:88:13:88:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:88:13:88:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:94:13:94:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:13:94:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:100:14:100:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:14:100:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:106:12:106:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:106:12:106:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |
| openai_user_test.js:153:12:153:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:153:12:153:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value |

22
javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js
View File

@@ -4,8 +4,6 @@ const { AzureOpenAI } = require("openai");
const {
GuardrailsOpenAI,
GuardrailsAzureOpenAI,
checkPlainText,
runGuardrails,
} = require("@openai/guardrails");
const app = express();
@@ -155,26 +153,6 @@ app.get("/test", async (req, res) => {
input: userInput, // $ Alert[js/user-prompt-injection]
});
// === checkPlainText sanitizer (SHOULD NOT ALERT) ===
await checkPlainText(userInput, configBundle);
// After checkPlainText, the input is safe because it would have thrown
await client.responses.create({
model: "gpt-4.1",
input: userInput, // OK - sanitized by checkPlainText
});
// === runGuardrails sanitizer (SHOULD NOT ALERT) ===
const userInput2 = req.query.userInput2;
await runGuardrails(userInput2, configBundle);
await client.responses.create({
model: "gpt-4.1",
input: userInput2, // OK - sanitized by runGuardrails
});
// === Constant comparison sanitizer (SHOULD NOT ALERT) ===
const userInput3 = req.query.userInput3;