diff --git a/javascript/ql/lib/ext/openai.model.yml b/javascript/ql/lib/ext/openai.model.yml index 055b37a5e8e..0c610d00977 100644 --- a/javascript/ql/lib/ext/openai.model.yml +++ b/javascript/ql/lib/ext/openai.model.yml @@ -21,8 +21,3 @@ extensions: - ["@openai/agents", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"] - ["@openai/guardrails", "Member[tool].Argument[0].Member[description]", "system-prompt-injection"] - ["@openai/guardrails", "Member[GuardrailAgent].Member[create].Argument[2]", "system-prompt-injection"] - - ["openai.Client", "Member[responses].Member[create].Argument[0].Member[input]", "user-prompt-injection"] - - ["openai.Client", "Member[completions].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"] - - ["openai.Client", "Member[images].Member[generate,edit].Argument[0].Member[prompt]", "user-prompt-injection"] - - ["openai.Client", "Member[embeddings].Member[create].Argument[0].Member[input]", "user-prompt-injection"] - - ["openai.Client", "Member[audio].Member[transcriptions,translations].Member[create].Argument[0].Member[prompt]", "user-prompt-injection"] diff --git a/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll b/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll index 33c54c02006..3e970b92a35 100644 --- a/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll +++ b/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll @@ -16,18 +16,6 @@ private predicate isSystemOrDevMessage(API::Node msg) { msg.getMember("role").asSink().mayHaveStringValue(["system", "developer", "assistant"]) } -module OpenAIGuardrails { - /** Gets a reference to the `GuardrailsOpenAI` class. */ - API::Node classRef() { - result = API::moduleImport("@openai/guardrails") - } - - API::Node getSanitizerNode() { - // checkPlainText(userInput, bundle) or runGuardrails(userInput, bundle) - result = classRef().getMember(["checkPlainText", "runGuardrails"]) - } -} - module OpenAI { /** Gets a reference to all OpenAI client instances. */ private API::Node allClients() { @@ -43,6 +31,33 @@ module OpenAI { .getPromised() } + /** Gets a guarded client that is clearly configured without input guardrails. */ + private API::Node unprotectedGuardedClient() { + exists(API::Node createCall | + createCall = + API::moduleImport("@openai/guardrails") + .getMember(["GuardrailsOpenAI", "GuardrailsAzureOpenAI"]) + .getMember("create") and + result = createCall.getReturn().getPromised() and + exists(createCall.getParameter(0).getMember("version")) and + not exists( + createCall.getParameter(0).getMember("input").getMember("guardrails").getArrayElement() + ) and + not exists( + createCall.getParameter(0).getMember("pre_flight").getMember("guardrails").getArrayElement() + ) + ) + } + + /** Gets a reference to all clients without input guardrails. */ + private API::Node clientsNoGuardrails() { + result = API::moduleImport("openai").getInstance() + or + result = API::moduleImport("openai").getMember(["OpenAI", "AzureOpenAI"]).getInstance() + or + result = unprotectedGuardedClient() + } + /** * Gets role-filtered system/developer/assistant message sinks. * These require checking a sibling `role` property and cannot be expressed in MaD. @@ -100,10 +115,18 @@ module OpenAI { * These require checking a sibling `role` property and cannot be expressed in MaD. */ API::Node getUserPromptNode() { + // responses.create({ input: "string" }) + result = + clientsNoGuardrails() + .getMember("responses") + .getMember("create") + .getParameter(0) + .getMember("input") + or // responses.create({ input: [{ role: "user", content: ... }] }) exists(API::Node msg | msg = - allClients() + clientsNoGuardrails() .getMember("responses") .getMember("create") .getParameter(0) @@ -117,7 +140,7 @@ module OpenAI { // chat.completions.create({ messages: [{ role: "user", content: ... }] }) exists(API::Node msg, API::Node content | msg = - allClients() + clientsNoGuardrails() .getMember("chat") .getMember("completions") .getMember("create") @@ -132,10 +155,34 @@ module OpenAI { result = content.getArrayElement().getMember("text") ) or + // Legacy completions API: completions.create({ prompt: ... }) + result = + clientsNoGuardrails() + .getMember("completions") + .getMember("create") + .getParameter(0) + .getMember("prompt") + or + // images.generate({ prompt: ... }) and images.edit({ prompt: ... }) + result = + clientsNoGuardrails() + .getMember("images") + .getMember(["generate", "edit"]) + .getParameter(0) + .getMember("prompt") + or + // embeddings.create({ input: ... }) + result = + clientsNoGuardrails() + .getMember("embeddings") + .getMember("create") + .getParameter(0) + .getMember("input") + or // beta.threads.messages.create(threadId, { role: "user", content: ... }) exists(API::Node msg | msg = - allClients() + clientsNoGuardrails() .getMember("beta") .getMember("threads") .getMember("messages") @@ -145,6 +192,15 @@ module OpenAI { | result = msg.getMember("content") ) + or + // audio.transcriptions/translations.create({ prompt: ... }) + result = + clientsNoGuardrails() + .getMember("audio") + .getMember(["transcriptions", "translations"]) + .getMember("create") + .getParameter(0) + .getMember("prompt") } } diff --git a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected index 5faf0a318ae..f0f2db6a40f 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected +++ b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected @@ -9,20 +9,20 @@ edges | gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:51:13:51:21 | userInput | provenance | | | gemini_user_test.js:8:9:8:17 | userInput | gemini_user_test.js:58:13:58:21 | userInput | provenance | | | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:8:9:8:17 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:24:12:24:20 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:33:18:33:26 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:44:18:44:26 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:58:19:58:27 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:68:13:68:21 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:73:13:73:21 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:77:13:77:21 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:83:12:83:20 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:90:13:90:21 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:96:13:96:21 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:102:14:102:22 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:108:12:108:20 | userInput | provenance | | -| openai_user_test.js:16:9:16:17 | userInput | openai_user_test.js:155:12:155:20 | userInput | provenance | | -| openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:16:9:16:17 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:22:12:22:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:31:18:31:26 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:42:18:42:26 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:56:19:56:27 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:66:13:66:21 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:71:13:71:21 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:75:13:75:21 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:81:12:81:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:88:13:88:21 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:13:94:21 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:14:100:22 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:106:12:106:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:153:12:153:20 | userInput | provenance | | +| openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:14:9:14:17 | userInput | provenance | | nodes | anthropic_user_test.js:8:9:8:17 | userInput | semmle.label | userInput | | anthropic_user_test.js:8:21:8:39 | req.query.userInput | semmle.label | req.query.userInput | @@ -36,21 +36,21 @@ nodes | gemini_user_test.js:44:13:44:21 | userInput | semmle.label | userInput | | gemini_user_test.js:51:13:51:21 | userInput | semmle.label | userInput | | gemini_user_test.js:58:13:58:21 | userInput | semmle.label | userInput | -| openai_user_test.js:16:9:16:17 | userInput | semmle.label | userInput | -| openai_user_test.js:16:21:16:39 | req.query.userInput | semmle.label | req.query.userInput | -| openai_user_test.js:24:12:24:20 | userInput | semmle.label | userInput | -| openai_user_test.js:33:18:33:26 | userInput | semmle.label | userInput | -| openai_user_test.js:44:18:44:26 | userInput | semmle.label | userInput | -| openai_user_test.js:58:19:58:27 | userInput | semmle.label | userInput | -| openai_user_test.js:68:13:68:21 | userInput | semmle.label | userInput | -| openai_user_test.js:73:13:73:21 | userInput | semmle.label | userInput | -| openai_user_test.js:77:13:77:21 | userInput | semmle.label | userInput | -| openai_user_test.js:83:12:83:20 | userInput | semmle.label | userInput | -| openai_user_test.js:90:13:90:21 | userInput | semmle.label | userInput | -| openai_user_test.js:96:13:96:21 | userInput | semmle.label | userInput | -| openai_user_test.js:102:14:102:22 | userInput | semmle.label | userInput | -| openai_user_test.js:108:12:108:20 | userInput | semmle.label | userInput | -| openai_user_test.js:155:12:155:20 | userInput | semmle.label | userInput | +| openai_user_test.js:14:9:14:17 | userInput | semmle.label | userInput | +| openai_user_test.js:14:21:14:39 | req.query.userInput | semmle.label | req.query.userInput | +| openai_user_test.js:22:12:22:20 | userInput | semmle.label | userInput | +| openai_user_test.js:31:18:31:26 | userInput | semmle.label | userInput | +| openai_user_test.js:42:18:42:26 | userInput | semmle.label | userInput | +| openai_user_test.js:56:19:56:27 | userInput | semmle.label | userInput | +| openai_user_test.js:66:13:66:21 | userInput | semmle.label | userInput | +| openai_user_test.js:71:13:71:21 | userInput | semmle.label | userInput | +| openai_user_test.js:75:13:75:21 | userInput | semmle.label | userInput | +| openai_user_test.js:81:12:81:20 | userInput | semmle.label | userInput | +| openai_user_test.js:88:13:88:21 | userInput | semmle.label | userInput | +| openai_user_test.js:94:13:94:21 | userInput | semmle.label | userInput | +| openai_user_test.js:100:14:100:22 | userInput | semmle.label | userInput | +| openai_user_test.js:106:12:106:20 | userInput | semmle.label | userInput | +| openai_user_test.js:153:12:153:20 | userInput | semmle.label | userInput | subpaths #select | anthropic_user_test.js:18:18:18:26 | userInput | anthropic_user_test.js:8:21:8:39 | req.query.userInput | anthropic_user_test.js:18:18:18:26 | userInput | This prompt construction depends on a $@. | anthropic_user_test.js:8:21:8:39 | req.query.userInput | user-provided value | @@ -61,16 +61,16 @@ subpaths | gemini_user_test.js:44:13:44:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:44:13:44:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value | | gemini_user_test.js:51:13:51:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:51:13:51:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value | | gemini_user_test.js:58:13:58:21 | userInput | gemini_user_test.js:8:21:8:39 | req.query.userInput | gemini_user_test.js:58:13:58:21 | userInput | This prompt construction depends on a $@. | gemini_user_test.js:8:21:8:39 | req.query.userInput | user-provided value | -| openai_user_test.js:24:12:24:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:24:12:24:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:33:18:33:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:33:18:33:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:44:18:44:26 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:44:18:44:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:58:19:58:27 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:58:19:58:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:68:13:68:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:68:13:68:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:73:13:73:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:73:13:73:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:77:13:77:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:77:13:77:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:83:12:83:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:83:12:83:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:90:13:90:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:90:13:90:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:96:13:96:21 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:96:13:96:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:102:14:102:22 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:102:14:102:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:108:12:108:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:108:12:108:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | -| openai_user_test.js:155:12:155:20 | userInput | openai_user_test.js:16:21:16:39 | req.query.userInput | openai_user_test.js:155:12:155:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:16:21:16:39 | req.query.userInput | user-provided value | +| openai_user_test.js:22:12:22:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:22:12:22:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:31:18:31:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:31:18:31:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:42:18:42:26 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:42:18:42:26 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:56:19:56:27 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:56:19:56:27 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:66:13:66:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:66:13:66:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:71:13:71:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:71:13:71:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:75:13:75:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:75:13:75:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:81:12:81:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:81:12:81:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:88:13:88:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:88:13:88:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:94:13:94:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:13:94:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:100:14:100:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:14:100:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:106:12:106:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:106:12:106:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:153:12:153:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:153:12:153:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | diff --git a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js index fc67e3961f4..d8ecdf71c96 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js +++ b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js @@ -4,8 +4,6 @@ const { AzureOpenAI } = require("openai"); const { GuardrailsOpenAI, GuardrailsAzureOpenAI, - checkPlainText, - runGuardrails, } = require("@openai/guardrails"); const app = express(); @@ -155,26 +153,6 @@ app.get("/test", async (req, res) => { input: userInput, // $ Alert[js/user-prompt-injection] }); - // === checkPlainText sanitizer (SHOULD NOT ALERT) === - - await checkPlainText(userInput, configBundle); - - // After checkPlainText, the input is safe because it would have thrown - await client.responses.create({ - model: "gpt-4.1", - input: userInput, // OK - sanitized by checkPlainText - }); - - // === runGuardrails sanitizer (SHOULD NOT ALERT) === - - const userInput2 = req.query.userInput2; - await runGuardrails(userInput2, configBundle); - - await client.responses.create({ - model: "gpt-4.1", - input: userInput2, // OK - sanitized by runGuardrails - }); - // === Constant comparison sanitizer (SHOULD NOT ALERT) === const userInput3 = req.query.userInput3;