fix(security): treat an unavailable prompt classifier as no signal (#12452)
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli-linux (push) Blocked by required conditions
Canary / Upload Install Script (push) Waiting to run
Canary / bundle-macos-arm64 (push) Blocked by required conditions
Canary / bundle-macos-x64 (push) Blocked by required conditions
Canary / bundle-windows (push) Blocked by required conditions
Canary / bundle-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / Check GDK API Reference is Up-to-Date (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Check GDK Release Configuration (push) Waiting to run
CI / changes (push) Waiting to run
CI / goose ACP npm packages (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Build Lean ACP Binary (push) Blocked by required conditions
CI / Check goose-sdk UniFFI (push) Blocked by required conditions
CI / Test Kotlin GDK package (push) Blocked by required conditions
CI / Build and Test TLS Backend (native-tls) (push) Blocked by required conditions
CI / Build and Test TLS Backend (rustls-tls) (push) Blocked by required conditions
CI / Build and Test Roaming Feature (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
CI / Test Desktop Updater Install (ubuntu-latest) (push) Blocked by required conditions
CI / Test Desktop Updater Install (windows-latest) (push) Blocked by required conditions
MCP Conformance / Build Conformance Binaries (push) Waiting to run
MCP Conformance / Conformance 2025-11-25 / 0.1.16 (push) Blocked by required conditions
MCP Conformance / Conformance 2025-11-25 / 0.2.0-alpha.10 (push) Blocked by required conditions
MCP Conformance / Conformance 2026-07-28 / 0.2.0-alpha.10 (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run

Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
dorien-koelemeijer 2026-09-23 21:26:44 +00:00 • committed by GitHub
parent 6a2c3dadf6
commit 80c1197583
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -241,14 +241,17 @@ impl PromptInjectionScanner {
});
}
let max_confidence = stream::iter(user_messages)
let (max_confidence, any_scanned) = stream::iter(user_messages)
.map(|msg| async move {
self.scan_with_classifier(&msg, classifier, ClassifierType::Prompt)
.await
})
.buffer_unordered(ML_SCAN_CONCURRENCY)
.fold(0.0_f32, |acc, result| async move {
result.unwrap_or(0.0).max(acc)
.fold((0.0_f32, false), |(acc, scanned), result| async move {
match result {
Some(confidence) => (confidence.max(acc), true),
None => (acc, scanned),
}
})
.await;
@ -256,7 +259,7 @@ impl PromptInjectionScanner {
confidence: max_confidence,
pattern_confidence: 0.0,
pattern_matches: Vec::new(),
ml_confidence: Some(max_confidence),
ml_confidence: any_scanned.then_some(max_confidence),
used_pattern_detection: false,
})
}
@ -432,7 +435,7 @@ impl Default for PromptInjectionScanner {
mod tests {
use super::*;
use rmcp::object;
use wiremock::matchers::method;
use wiremock::matchers::{body_string_contains, method};
use wiremock::{Mock, MockServer, ResponseTemplate};
async fn classifier_with_confidence(
@ -715,4 +718,75 @@ mod tests {
vec!["newest prompt", "middle prompt", "oldest prompt"]
);
}
async fn failing_prompt_classifier() -> (MockServer, ClassificationClient) {
let mock_server = MockServer::start().await;
Mock::given(method("POST"))
.respond_with(ResponseTemplate::new(500))
.mount(&mock_server)
.await;
let classifier =
ClassificationClient::from_endpoint(mock_server.uri(), None, None).unwrap();
(mock_server, classifier)
}
#[tokio::test]
async fn failed_prompt_scan_does_not_lower_command_confidence() {
let (_command_server, command_classifier) = classifier_with_confidence(0.85).await;
let (_prompt_server, prompt_classifier) = failing_prompt_classifier().await;
let scanner = PromptInjectionScanner {
pattern_matcher: PatternMatcher::new(),
command_classifier: Some(command_classifier),
prompt_classifier: Some(prompt_classifier),
};
let tool_call =
CallToolRequestParams::new("shell").with_arguments(object!({"command": "echo hello"}));
let messages = vec![Message::user().with_text("deploy the service")];
let result = scanner
.analyze_tool_call_with_context(&tool_call, &messages)
.await
.unwrap();
assert!((result.confidence - 0.85).abs() < 1e-6);
}
#[tokio::test]
async fn partially_failed_prompt_scan_keeps_successful_scores() {
let mock_server = MockServer::start().await;
Mock::given(method("POST"))
.and(body_string_contains("unscannable"))
.respond_with(ResponseTemplate::new(500))
.mount(&mock_server)
.await;
Mock::given(method("POST"))
.respond_with(
ResponseTemplate::new(200).set_body_json(serde_json::json!([[{
"label": "INJECTION",
"score": 0.6
}, {
"label": "SAFE",
"score": 0.4
}]])),
)
.mount(&mock_server)
.await;
let classifier =
ClassificationClient::from_endpoint(mock_server.uri(), None, None).unwrap();
let scanner = PromptInjectionScanner {
pattern_matcher: PatternMatcher::new(),
command_classifier: None,
prompt_classifier: Some(classifier),
};
let messages = vec![
Message::user().with_text("unscannable message"),
Message::user().with_text("ordinary message"),
];
let result = scanner.scan_conversation(&messages).await.unwrap();
let confidence = result.ml_confidence.expect("surviving scan should report");
assert!((confidence - 0.6).abs() < 1e-6);
}
}