From 535adc7a31355b91c9e69dcc1f83826aa80fbcc2 Mon Sep 17 00:00:00 2001 From: BazookaMusic Date: Fri, 15 May 2026 12:14:14 +0200 Subject: [PATCH] add barrier when data flows into user messages for system prompt detection, remove embeddings from user prompt injection query --- javascript/ql/lib/ext/google-genai.model.yml | 1 - .../javascript/frameworks/Anthropic.qll | 4 +- .../javascript/frameworks/GoogleGenAI.qll | 4 +- .../semmle/javascript/frameworks/OpenAI.qll | 8 -- .../SystemPromptInjectionCustomizations.qll | 18 +++ .../SystemPromptInjection.expected | 71 +++++++----- .../SystemPromptInjection/anthropic_test.js | 32 ++++++ .../SystemPromptInjection/openai_test.js | 8 -- .../UserPromptInjection.expected | 27 ++--- .../UserPromptInjection/openai_user_test.js | 6 - prompt-injection-detection-report.md | 106 ++++++++++++++++++ 11 files changed, 218 insertions(+), 67 deletions(-) create mode 100644 prompt-injection-detection-report.md diff --git a/javascript/ql/lib/ext/google-genai.model.yml b/javascript/ql/lib/ext/google-genai.model.yml index 1aa871f2a09..9ff8fd44e4b 100644 --- a/javascript/ql/lib/ext/google-genai.model.yml +++ b/javascript/ql/lib/ext/google-genai.model.yml @@ -19,5 +19,4 @@ extensions: - ["google-genai.Client", "Member[models].Member[generateVideos].Argument[0].Member[prompt]", "user-prompt-injection"] - ["google-genai.Client", "Member[chats].Member[create].ReturnValue.Member[sendMessage,sendMessageStream].Argument[0].Member[message]", "user-prompt-injection"] - ["google-genai.Client", "Member[chats].Member[create].ReturnValue.Member[sendMessage,sendMessageStream].Argument[0].Member[content]", "user-prompt-injection"] - - ["google-genai.Client", "Member[models].Member[embedContent].Argument[0].Member[content]", "user-prompt-injection"] - ["google-genai.Client", "Member[interactions].Member[create].Argument[0].Member[input]", "user-prompt-injection"] diff --git a/javascript/ql/src/experimental/semmle/javascript/frameworks/Anthropic.qll b/javascript/ql/src/experimental/semmle/javascript/frameworks/Anthropic.qll index cabd3c2b8b3..30e5f2e91b1 100644 --- a/javascript/ql/src/experimental/semmle/javascript/frameworks/Anthropic.qll +++ b/javascript/ql/src/experimental/semmle/javascript/frameworks/Anthropic.qll @@ -33,7 +33,7 @@ module Anthropic { // messages: [{ role: "assistant", content: "..." }] exists(API::Node msg | msg = messagesCreateParams().getMember("messages").getArrayElement() and - msg.getMember("role").asSink().mayHaveStringValue("assistant") + msg.getMember("role").asSink().mayHaveStringValue(["system", "assistant"]) | result = msg.getMember("content") ) @@ -47,7 +47,7 @@ module Anthropic { // messages: [{ role: "user", content: "..." }] exists(API::Node msg | msg = messagesCreateParams().getMember("messages").getArrayElement() and - not msg.getMember("role").asSink().mayHaveStringValue("assistant") + not msg.getMember("role").asSink().mayHaveStringValue(["system", "assistant"]) | result = msg.getMember("content") ) diff --git a/javascript/ql/src/experimental/semmle/javascript/frameworks/GoogleGenAI.qll b/javascript/ql/src/experimental/semmle/javascript/frameworks/GoogleGenAI.qll index ff4615bfe5d..83f470f2e23 100644 --- a/javascript/ql/src/experimental/semmle/javascript/frameworks/GoogleGenAI.qll +++ b/javascript/ql/src/experimental/semmle/javascript/frameworks/GoogleGenAI.qll @@ -33,7 +33,7 @@ module GoogleGenAI { .getParameter(0) .getMember("contents") .getArrayElement() and - msg.getMember("role").asSink().mayHaveStringValue("model") + msg.getMember("role").asSink().mayHaveStringValue(["system", "model"]) | result = msg.getMember("parts").getArrayElement().getMember("text") ) @@ -53,7 +53,7 @@ module GoogleGenAI { .getParameter(0) .getMember("contents") .getArrayElement() and - not msg.getMember("role").asSink().mayHaveStringValue("model") + not msg.getMember("role").asSink().mayHaveStringValue(["system", "model"]) | result = msg.getMember("parts").getArrayElement().getMember("text") ) diff --git a/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll b/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll index 3e970b92a35..17bd260a776 100644 --- a/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll +++ b/javascript/ql/src/experimental/semmle/javascript/frameworks/OpenAI.qll @@ -171,14 +171,6 @@ module OpenAI { .getParameter(0) .getMember("prompt") or - // embeddings.create({ input: ... }) - result = - clientsNoGuardrails() - .getMember("embeddings") - .getMember("create") - .getParameter(0) - .getMember("input") - or // beta.threads.messages.create(threadId, { role: "user", content: ... }) exists(API::Node msg | msg = diff --git a/javascript/ql/src/experimental/semmle/javascript/security/PromptInjection/SystemPromptInjectionCustomizations.qll b/javascript/ql/src/experimental/semmle/javascript/security/PromptInjection/SystemPromptInjectionCustomizations.qll index ec34b27712d..a367eea8b83 100644 --- a/javascript/ql/src/experimental/semmle/javascript/security/PromptInjection/SystemPromptInjectionCustomizations.qll +++ b/javascript/ql/src/experimental/semmle/javascript/security/PromptInjection/SystemPromptInjectionCustomizations.qll @@ -74,6 +74,24 @@ module SystemPromptInjection { } } + /** + * Content placed in a message with `role: "user"` is not a system prompt + * injection vector; it is intended user-role content. + * + * This prevents false positives when user input and system prompts are + * combined in the same message array (e.g. `[{role:"system", content: ...}, + * {role:"user", content: tainted}]`) and taint would otherwise propagate + * through array operations to the system message. + */ + private class UserRoleMessageContentBarrier extends Sanitizer { + UserRoleMessageContentBarrier() { + exists(DataFlow::SourceNode obj | + obj.getAPropertySource("role").mayHaveStringValue("user") and + this = obj.getAPropertyWrite("content").getRhs() + ) + } + } + /** * A comparison with a constant, considered as a sanitizer-guard. */ diff --git a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/SystemPromptInjection.expected b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/SystemPromptInjection.expected index ccf446609ad..514798e13c0 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/SystemPromptInjection.expected +++ b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/SystemPromptInjection.expected @@ -33,6 +33,7 @@ edges | anthropic_test.js:8:9:8:15 | persona | anthropic_test.js:99:35:99:41 | persona | provenance | | | anthropic_test.js:8:9:8:15 | persona | anthropic_test.js:110:30:110:36 | persona | provenance | | | anthropic_test.js:8:9:8:15 | persona | anthropic_test.js:117:30:117:36 | persona | provenance | | +| anthropic_test.js:8:9:8:15 | persona | anthropic_test.js:141:49:141:55 | persona | provenance | | | anthropic_test.js:8:19:8:35 | req.query.persona | anthropic_test.js:8:9:8:15 | persona | provenance | | | anthropic_test.js:17:30:17:36 | persona | anthropic_test.js:17:13:17:36 | "Talk l ... persona | provenance | | | anthropic_test.js:30:32:30:38 | persona | anthropic_test.js:30:15:30:38 | "Talk l ... persona | provenance | | @@ -42,6 +43,15 @@ edges | anthropic_test.js:99:35:99:41 | persona | anthropic_test.js:99:18:99:41 | "Talk l ... persona | provenance | | | anthropic_test.js:110:30:110:36 | persona | anthropic_test.js:110:13:110:36 | "Talk l ... persona | provenance | | | anthropic_test.js:117:30:117:36 | persona | anthropic_test.js:117:13:117:36 | "Talk l ... persona | provenance | | +| anthropic_test.js:140:9:140:17 | messages2 [0, content] | anthropic_test.js:144:22:144:30 | messages2 [0, content] | provenance | | +| anthropic_test.js:140:21:143:3 | [\\n { ... },\\n ] [0, content] | anthropic_test.js:140:9:140:17 | messages2 [0, content] | provenance | | +| anthropic_test.js:141:5:141:57 | { role: ... rsona } [content] | anthropic_test.js:140:21:143:3 | [\\n { ... },\\n ] [0, content] | provenance | | +| anthropic_test.js:141:32:141:55 | "Talk l ... persona | anthropic_test.js:141:5:141:57 | { role: ... rsona } [content] | provenance | | +| anthropic_test.js:141:49:141:55 | persona | anthropic_test.js:141:32:141:55 | "Talk l ... persona | provenance | | +| anthropic_test.js:144:9:144:18 | systemMsg2 [content] | anthropic_test.js:148:13:148:22 | systemMsg2 [content] | provenance | | +| anthropic_test.js:144:22:144:30 | messages2 [0, content] | anthropic_test.js:144:22:144:63 | message ... ystem") [content] | provenance | | +| anthropic_test.js:144:22:144:63 | message ... ystem") [content] | anthropic_test.js:144:9:144:18 | systemMsg2 [content] | provenance | | +| anthropic_test.js:148:13:148:22 | systemMsg2 [content] | anthropic_test.js:148:13:148:30 | systemMsg2.content | provenance | | | gemini_test.js:8:9:8:15 | persona | gemini_test.js:18:43:18:49 | persona | provenance | | | gemini_test.js:8:9:8:15 | persona | gemini_test.js:30:42:30:48 | persona | provenance | | | gemini_test.js:8:9:8:15 | persona | gemini_test.js:59:43:59:49 | persona | provenance | | @@ -62,11 +72,11 @@ edges | openai_test.js:11:9:11:15 | persona | openai_test.js:83:35:83:41 | persona | provenance | | | openai_test.js:11:9:11:15 | persona | openai_test.js:97:36:97:42 | persona | provenance | | | openai_test.js:11:9:11:15 | persona | openai_test.js:110:35:110:41 | persona | provenance | | -| openai_test.js:11:9:11:15 | persona | openai_test.js:149:36:149:42 | persona | provenance | | -| openai_test.js:11:9:11:15 | persona | openai_test.js:160:36:160:42 | persona | provenance | | -| openai_test.js:11:9:11:15 | persona | openai_test.js:166:52:166:58 | persona | provenance | | -| openai_test.js:11:9:11:15 | persona | openai_test.js:172:31:172:37 | persona | provenance | | -| openai_test.js:11:9:11:15 | persona | openai_test.js:200:49:200:55 | persona | provenance | | +| openai_test.js:11:9:11:15 | persona | openai_test.js:141:36:141:42 | persona | provenance | | +| openai_test.js:11:9:11:15 | persona | openai_test.js:152:36:152:42 | persona | provenance | | +| openai_test.js:11:9:11:15 | persona | openai_test.js:158:52:158:58 | persona | provenance | | +| openai_test.js:11:9:11:15 | persona | openai_test.js:164:31:164:37 | persona | provenance | | +| openai_test.js:11:9:11:15 | persona | openai_test.js:192:49:192:55 | persona | provenance | | | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:11:9:11:15 | persona | provenance | | | openai_test.js:19:36:19:42 | persona | openai_test.js:19:19:19:42 | "Talk l ... persona | provenance | | | openai_test.js:29:35:29:41 | persona | openai_test.js:29:18:29:41 | "Talk l ... persona | provenance | | @@ -75,11 +85,11 @@ edges | openai_test.js:83:35:83:41 | persona | openai_test.js:83:18:83:41 | "Talk l ... persona | provenance | | | openai_test.js:97:36:97:42 | persona | openai_test.js:97:19:97:42 | "Talk l ... persona | provenance | | | openai_test.js:110:35:110:41 | persona | openai_test.js:110:18:110:41 | "Talk l ... persona | provenance | | -| openai_test.js:149:36:149:42 | persona | openai_test.js:149:19:149:42 | "Talk l ... persona | provenance | | -| openai_test.js:160:36:160:42 | persona | openai_test.js:160:19:160:42 | "Talk l ... persona | provenance | | -| openai_test.js:166:52:166:58 | persona | openai_test.js:166:30:166:58 | "Also t ... persona | provenance | | -| openai_test.js:172:31:172:37 | persona | openai_test.js:172:14:172:37 | "Talk l ... persona | provenance | | -| openai_test.js:200:49:200:55 | persona | openai_test.js:200:32:200:55 | "Talk l ... persona | provenance | | +| openai_test.js:141:36:141:42 | persona | openai_test.js:141:19:141:42 | "Talk l ... persona | provenance | | +| openai_test.js:152:36:152:42 | persona | openai_test.js:152:19:152:42 | "Talk l ... persona | provenance | | +| openai_test.js:158:52:158:58 | persona | openai_test.js:158:30:158:58 | "Also t ... persona | provenance | | +| openai_test.js:164:31:164:37 | persona | openai_test.js:164:14:164:37 | "Talk l ... persona | provenance | | +| openai_test.js:192:49:192:55 | persona | openai_test.js:192:32:192:55 | "Talk l ... persona | provenance | | nodes | agents_test.js:8:9:8:15 | persona | semmle.label | persona | | agents_test.js:8:19:8:35 | req.query.persona | semmle.label | req.query.persona | @@ -120,6 +130,16 @@ nodes | anthropic_test.js:110:30:110:36 | persona | semmle.label | persona | | anthropic_test.js:117:13:117:36 | "Talk l ... persona | semmle.label | "Talk l ... persona | | anthropic_test.js:117:30:117:36 | persona | semmle.label | persona | +| anthropic_test.js:140:9:140:17 | messages2 [0, content] | semmle.label | messages2 [0, content] | +| anthropic_test.js:140:21:143:3 | [\\n { ... },\\n ] [0, content] | semmle.label | [\\n { ... },\\n ] [0, content] | +| anthropic_test.js:141:5:141:57 | { role: ... rsona } [content] | semmle.label | { role: ... rsona } [content] | +| anthropic_test.js:141:32:141:55 | "Talk l ... persona | semmle.label | "Talk l ... persona | +| anthropic_test.js:141:49:141:55 | persona | semmle.label | persona | +| anthropic_test.js:144:9:144:18 | systemMsg2 [content] | semmle.label | systemMsg2 [content] | +| anthropic_test.js:144:22:144:30 | messages2 [0, content] | semmle.label | messages2 [0, content] | +| anthropic_test.js:144:22:144:63 | message ... ystem") [content] | semmle.label | message ... ystem") [content] | +| anthropic_test.js:148:13:148:22 | systemMsg2 [content] | semmle.label | systemMsg2 [content] | +| anthropic_test.js:148:13:148:30 | systemMsg2.content | semmle.label | systemMsg2.content | | gemini_test.js:8:9:8:15 | persona | semmle.label | persona | | gemini_test.js:8:19:8:35 | req.query.persona | semmle.label | req.query.persona | | gemini_test.js:18:26:18:49 | "Talk l ... persona | semmle.label | "Talk l ... persona | @@ -150,16 +170,16 @@ nodes | openai_test.js:97:36:97:42 | persona | semmle.label | persona | | openai_test.js:110:18:110:41 | "Talk l ... persona | semmle.label | "Talk l ... persona | | openai_test.js:110:35:110:41 | persona | semmle.label | persona | -| openai_test.js:149:19:149:42 | "Talk l ... persona | semmle.label | "Talk l ... persona | -| openai_test.js:149:36:149:42 | persona | semmle.label | persona | -| openai_test.js:160:19:160:42 | "Talk l ... persona | semmle.label | "Talk l ... persona | -| openai_test.js:160:36:160:42 | persona | semmle.label | persona | -| openai_test.js:166:30:166:58 | "Also t ... persona | semmle.label | "Also t ... persona | -| openai_test.js:166:52:166:58 | persona | semmle.label | persona | -| openai_test.js:172:14:172:37 | "Talk l ... persona | semmle.label | "Talk l ... persona | -| openai_test.js:172:31:172:37 | persona | semmle.label | persona | -| openai_test.js:200:32:200:55 | "Talk l ... persona | semmle.label | "Talk l ... persona | -| openai_test.js:200:49:200:55 | persona | semmle.label | persona | +| openai_test.js:141:19:141:42 | "Talk l ... persona | semmle.label | "Talk l ... persona | +| openai_test.js:141:36:141:42 | persona | semmle.label | persona | +| openai_test.js:152:19:152:42 | "Talk l ... persona | semmle.label | "Talk l ... persona | +| openai_test.js:152:36:152:42 | persona | semmle.label | persona | +| openai_test.js:158:30:158:58 | "Also t ... persona | semmle.label | "Also t ... persona | +| openai_test.js:158:52:158:58 | persona | semmle.label | persona | +| openai_test.js:164:14:164:37 | "Talk l ... persona | semmle.label | "Talk l ... persona | +| openai_test.js:164:31:164:37 | persona | semmle.label | persona | +| openai_test.js:192:32:192:55 | "Talk l ... persona | semmle.label | "Talk l ... persona | +| openai_test.js:192:49:192:55 | persona | semmle.label | persona | subpaths #select | agents_test.js:16:19:16:42 | "Talk l ... persona | agents_test.js:8:19:8:35 | req.query.persona | agents_test.js:16:19:16:42 | "Talk l ... persona | This prompt construction depends on a $@. | agents_test.js:8:19:8:35 | req.query.persona | user-provided value | @@ -179,6 +199,7 @@ subpaths | anthropic_test.js:99:18:99:41 | "Talk l ... persona | anthropic_test.js:8:19:8:35 | req.query.persona | anthropic_test.js:99:18:99:41 | "Talk l ... persona | This prompt construction depends on a $@. | anthropic_test.js:8:19:8:35 | req.query.persona | user-provided value | | anthropic_test.js:110:13:110:36 | "Talk l ... persona | anthropic_test.js:8:19:8:35 | req.query.persona | anthropic_test.js:110:13:110:36 | "Talk l ... persona | This prompt construction depends on a $@. | anthropic_test.js:8:19:8:35 | req.query.persona | user-provided value | | anthropic_test.js:117:13:117:36 | "Talk l ... persona | anthropic_test.js:8:19:8:35 | req.query.persona | anthropic_test.js:117:13:117:36 | "Talk l ... persona | This prompt construction depends on a $@. | anthropic_test.js:8:19:8:35 | req.query.persona | user-provided value | +| anthropic_test.js:148:13:148:30 | systemMsg2.content | anthropic_test.js:8:19:8:35 | req.query.persona | anthropic_test.js:148:13:148:30 | systemMsg2.content | This prompt construction depends on a $@. | anthropic_test.js:8:19:8:35 | req.query.persona | user-provided value | | gemini_test.js:18:26:18:49 | "Talk l ... persona | gemini_test.js:8:19:8:35 | req.query.persona | gemini_test.js:18:26:18:49 | "Talk l ... persona | This prompt construction depends on a $@. | gemini_test.js:8:19:8:35 | req.query.persona | user-provided value | | gemini_test.js:30:25:30:48 | "Talk l ... persona | gemini_test.js:8:19:8:35 | req.query.persona | gemini_test.js:30:25:30:48 | "Talk l ... persona | This prompt construction depends on a $@. | gemini_test.js:8:19:8:35 | req.query.persona | user-provided value | | gemini_test.js:59:26:59:49 | "Talk l ... persona | gemini_test.js:8:19:8:35 | req.query.persona | gemini_test.js:59:26:59:49 | "Talk l ... persona | This prompt construction depends on a $@. | gemini_test.js:8:19:8:35 | req.query.persona | user-provided value | @@ -192,8 +213,8 @@ subpaths | openai_test.js:83:18:83:41 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:83:18:83:41 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | | openai_test.js:97:19:97:42 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:97:19:97:42 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | | openai_test.js:110:18:110:41 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:110:18:110:41 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | -| openai_test.js:149:19:149:42 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:149:19:149:42 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | -| openai_test.js:160:19:160:42 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:160:19:160:42 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | -| openai_test.js:166:30:166:58 | "Also t ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:166:30:166:58 | "Also t ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | -| openai_test.js:172:14:172:37 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:172:14:172:37 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | -| openai_test.js:200:32:200:55 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:200:32:200:55 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | +| openai_test.js:141:19:141:42 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:141:19:141:42 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | +| openai_test.js:152:19:152:42 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:152:19:152:42 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | +| openai_test.js:158:30:158:58 | "Also t ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:158:30:158:58 | "Also t ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | +| openai_test.js:164:14:164:37 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:164:14:164:37 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | +| openai_test.js:192:32:192:55 | "Talk l ... persona | openai_test.js:11:19:11:35 | req.query.persona | openai_test.js:192:32:192:55 | "Talk l ... persona | This prompt construction depends on a $@. | openai_test.js:11:19:11:35 | req.query.persona | user-provided value | diff --git a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/anthropic_test.js b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/anthropic_test.js index 656179601f8..a622617c9a2 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/anthropic_test.js +++ b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/anthropic_test.js @@ -117,6 +117,38 @@ app.get("/test", async (req, res) => { system: "Talk like a " + persona, // $ Alert[js/prompt-injection] }); + // === Barrier: user-role content in shared message array === + + // SHOULD NOT ALERT — user input placed in { role: "user" } should not + // taint system messages extracted from the same array. + const messages = [ + { role: "system", content: "You are a helpful assistant" }, + { role: "user", content: query }, // OK - user role barrier + ]; + const systemMsg = messages.find((m) => m.role === "system"); + const m6 = await client.messages.create({ + model: "claude-sonnet-4-20250514", + max_tokens: 1024, + system: systemMsg.content, + messages: [{ role: "user", content: query }], + }); + + // === Barrier does NOT suppress: tainted value in system role === + + // SHOULD ALERT — tainted data goes into system role; barrier on user role + // must not suppress the system-role taint path. + const messages2 = [ + { role: "system", content: "Talk like a " + persona }, // $ Alert[js/prompt-injection] + { role: "user", content: query }, + ]; + const systemMsg2 = messages2.find((m) => m.role === "system"); + const m7 = await client.messages.create({ + model: "claude-sonnet-4-20250514", + max_tokens: 1024, + system: systemMsg2.content, + messages: [{ role: "user", content: query }], + }); + // === Sanitizer: constant comparison === // SHOULD NOT ALERT diff --git a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/openai_test.js b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/openai_test.js index fcf7096b075..2a7fbf49233 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/openai_test.js +++ b/javascript/ql/test/experimental/Security/CWE-1427/SystemPromptInjection/openai_test.js @@ -132,14 +132,6 @@ app.get("/test", async (req, res) => { prompt: "Edit to look like " + persona, // $ Alert[js/prompt-injection] }); - // === Embeddings API === - - // embeddings.create (SHOULD ALERT) - const e1 = await client.embeddings.create({ - model: "text-embedding-3-small", - input: "Embed this: " + persona, // $ Alert[js/prompt-injection] - }); - // === Assistants API (beta) === // assistants.create (SHOULD ALERT) diff --git a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected index f0f2db6a40f..91f8df25fd8 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected +++ b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/UserPromptInjection.expected @@ -16,12 +16,11 @@ edges | openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:66:13:66:21 | userInput | provenance | | | openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:71:13:71:21 | userInput | provenance | | | openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:75:13:75:21 | userInput | provenance | | -| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:81:12:81:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:82:13:82:21 | userInput | provenance | | | openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:88:13:88:21 | userInput | provenance | | -| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:13:94:21 | userInput | provenance | | -| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:14:100:22 | userInput | provenance | | -| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:106:12:106:20 | userInput | provenance | | -| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:153:12:153:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:94:14:94:22 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:100:12:100:20 | userInput | provenance | | +| openai_user_test.js:14:9:14:17 | userInput | openai_user_test.js:147:12:147:20 | userInput | provenance | | | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:14:9:14:17 | userInput | provenance | | nodes | anthropic_user_test.js:8:9:8:17 | userInput | semmle.label | userInput | @@ -45,12 +44,11 @@ nodes | openai_user_test.js:66:13:66:21 | userInput | semmle.label | userInput | | openai_user_test.js:71:13:71:21 | userInput | semmle.label | userInput | | openai_user_test.js:75:13:75:21 | userInput | semmle.label | userInput | -| openai_user_test.js:81:12:81:20 | userInput | semmle.label | userInput | +| openai_user_test.js:82:13:82:21 | userInput | semmle.label | userInput | | openai_user_test.js:88:13:88:21 | userInput | semmle.label | userInput | -| openai_user_test.js:94:13:94:21 | userInput | semmle.label | userInput | -| openai_user_test.js:100:14:100:22 | userInput | semmle.label | userInput | -| openai_user_test.js:106:12:106:20 | userInput | semmle.label | userInput | -| openai_user_test.js:153:12:153:20 | userInput | semmle.label | userInput | +| openai_user_test.js:94:14:94:22 | userInput | semmle.label | userInput | +| openai_user_test.js:100:12:100:20 | userInput | semmle.label | userInput | +| openai_user_test.js:147:12:147:20 | userInput | semmle.label | userInput | subpaths #select | anthropic_user_test.js:18:18:18:26 | userInput | anthropic_user_test.js:8:21:8:39 | req.query.userInput | anthropic_user_test.js:18:18:18:26 | userInput | This prompt construction depends on a $@. | anthropic_user_test.js:8:21:8:39 | req.query.userInput | user-provided value | @@ -68,9 +66,8 @@ subpaths | openai_user_test.js:66:13:66:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:66:13:66:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | | openai_user_test.js:71:13:71:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:71:13:71:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | | openai_user_test.js:75:13:75:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:75:13:75:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | -| openai_user_test.js:81:12:81:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:81:12:81:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:82:13:82:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:82:13:82:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | | openai_user_test.js:88:13:88:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:88:13:88:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | -| openai_user_test.js:94:13:94:21 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:13:94:21 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | -| openai_user_test.js:100:14:100:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:14:100:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | -| openai_user_test.js:106:12:106:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:106:12:106:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | -| openai_user_test.js:153:12:153:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:153:12:153:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:94:14:94:22 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:94:14:94:22 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:100:12:100:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:100:12:100:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | +| openai_user_test.js:147:12:147:20 | userInput | openai_user_test.js:14:21:14:39 | req.query.userInput | openai_user_test.js:147:12:147:20 | userInput | This prompt construction depends on a $@. | openai_user_test.js:14:21:14:39 | req.query.userInput | user-provided value | diff --git a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js index d8ecdf71c96..9a28b74f361 100644 --- a/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js +++ b/javascript/ql/test/experimental/Security/CWE-1427/UserPromptInjection/openai_user_test.js @@ -75,12 +75,6 @@ app.get("/test", async (req, res) => { prompt: userInput, // $ Alert[js/user-prompt-injection] }); - // Embeddings API - await client.embeddings.create({ - model: "text-embedding-3-small", - input: userInput, // $ Alert[js/user-prompt-injection] - }); - // Audio API await client.audio.transcriptions.create({ file: "audio.mp3", diff --git a/prompt-injection-detection-report.md b/prompt-injection-detection-report.md new file mode 100644 index 00000000000..3a4355c613b --- /dev/null +++ b/prompt-injection-detection-report.md @@ -0,0 +1,106 @@ +# `js/prompt-injection` Detection Report + +**Date:** May 15, 2026 +**Branch:** `bazookamusic/cwe-1427` +**Queries:** `SystemPromptInjection.ql`, `UserPromptInjection.ql` + +## Summary + +Evaluated 11 repositories with `js/prompt-injection` findings. **9 True Positives, 2 False Positives.** + +## Detections + +### 1. Harsh5225/CodeBuddy — **TP** + +**Finding:** System prompt injection +**Description:** Direct system prompt injection. User-controlled input flows into the system prompt of an LLM call without sanitization. + +--- + +### 2. barnesy/momentum (×6 findings) — **TP** + +**Finding:** System prompt injection (6 paths) +**Description:** Multiple system prompt injection paths. User input is concatenated or interpolated into system-level prompts across several endpoints. + +--- + +### 3. shane-reaume/TalkToDev (×3 findings) — **TP** + +**Finding:** System prompt injection (3 paths) +**Description:** Multiple system prompt injection paths. User-controlled data flows into system prompts for LLM calls. + +--- + +### 4. huggingface/responses.js — **TP** + +**Finding:** `responses.ts:271` +**Description:** An open API endpoint populates the system prompt directly from request data. There is no authentication guarding the endpoint, meaning any caller can control the system-level instructions sent to the model. + +--- + +### 5. FlowiseAI/Flowise — **TP** + +**Finding:** `assistants/index.ts:107` +**Description:** User input flows into the OpenAI Assistants API `instructions` field. The `instructions` field is a developer-level system prompt — it defines the assistant's behavior and is not designed for end-user content. Even though Flowise has RBAC, authenticated users can craft `instructions` that affect other users' conversations with the created assistant. Exposing this field to user input is a prompt injection vector regardless of authentication. + +--- + +### 6. sjinnovation/CollabAI (×2 findings) — **TP** + +**Finding:** `openai.js` (2 paths) +**Description:** The POST route for creating OpenAI assistants does **not** have `authenticateUser` middleware applied. Unauthenticated users can create OpenAI assistants with arbitrary `instructions`, directly controlling the system prompt. The missing auth middleware is visible in the route definition — other routes in the same file do use `authenticateUser`. + +--- + +### 7. theodi/chat2db — **TP** + +**Finding:** `openaiClient.js:49` +**Description:** No authentication on the `/v1/chat/completions` route. The route accepts a `messages` array from the client, which can include `role: "system"` messages. An unauthenticated caller can fully override the system prompt. + +--- + +### 8. torarnehave1/mystmkra.io — **TP** + +**Finding:** `assistants.js:58` +**Description:** No authentication on `/assistants/*` routes. An `isAuthenticated` middleware exists in the codebase but is **not applied** to the assistant routes. Unauthenticated users can create or modify assistants with arbitrary instructions, controlling the system prompt. + +--- + +### 9. kvadou/franchise-manager — **TP** + +**Finding:** `generation.ts:449` +**Description:** User-controlled `moduleContext.title` and `moduleContext.description` (from `request.json()`) are concatenated directly into the system prompt. Even with authentication, this is a prompt injection vector: a user can embed instructions like "Ignore all previous instructions" in the title/description fields, overriding the developer's intended system prompt behavior. + +--- + +### 10. armando3069/AI-Inbox — **FP** + +**Finding:** `ai-assistant.service.ts:121` +**Description:** The system prompt tone is selected from a hardcoded `TONE_PROMPTS` map. User input selects which tone to use (e.g., "professional", "casual"), but the actual prompt text is developer-controlled. The false positive arose from CodeQL's array taint propagation — user-tainted content in a `{role:"user"}` message caused the entire messages array to appear tainted, including the `{role:"system"}` message with the hardcoded tone. **The `UserRoleMessageContentBarrier` now correctly blocks this.** + +--- + +### 11. mckaywrigley/chatbot-ui — **FP** + +**Finding:** `anthropic/route.ts:67` +**Description:** Users authenticate via Supabase and provide their own Anthropic API key. The "system prompt" is a personal configuration set by the user for their own chatbot instance. The user is effectively the developer in this context — they are configuring their own model's behavior using their own API key. There is no multi-tenant risk; the system prompt only affects the user who set it. + +--- + +## Verdict Summary + +| # | Repository | Finding Location | Verdict | Key Factor | +|---|-----------|-----------------|---------|------------| +| 1 | Harsh5225/CodeBuddy | system prompt | **TP** | Direct injection | +| 2 | barnesy/momentum | ×6 locations | **TP** | Multiple injection paths | +| 3 | shane-reaume/TalkToDev | ×3 locations | **TP** | Multiple injection paths | +| 4 | huggingface/responses.js | `responses.ts:271` | **TP** | Open API, no auth | +| 5 | FlowiseAI/Flowise | `assistants/index.ts:107` | **TP** | `instructions` is developer API, not user API | +| 6 | sjinnovation/CollabAI | `openai.js` ×2 | **TP** | Missing `authenticateUser` middleware | +| 7 | theodi/chat2db | `openaiClient.js:49` | **TP** | No auth, accepts `role:"system"` | +| 8 | torarnehave1/mystmkra.io | `assistants.js:58` | **TP** | Auth exists but not applied to routes | +| 9 | kvadou/franchise-manager | `generation.ts:449` | **TP** | User content in system prompt position | +| 10 | armando3069/AI-Inbox | `ai-assistant.service.ts:121` | **FP** | Hardcoded prompts, array taint propagation | +| 11 | mckaywrigley/chatbot-ui | `anthropic/route.ts:67` | **FP** | User's own API key, self-configured | + +**Precision: 9/11 (81.8%)**