mirror of
https://github.com/block/goose.git
synced 2026-10-18 09:30:25 +00:00
fix(security): treat an unavailable prompt classifier as no signal (#12452)
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli-linux (push) Blocked by required conditions
Canary / Upload Install Script (push) Waiting to run
Canary / bundle-macos-arm64 (push) Blocked by required conditions
Canary / bundle-macos-x64 (push) Blocked by required conditions
Canary / bundle-windows (push) Blocked by required conditions
Canary / bundle-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / Check GDK API Reference is Up-to-Date (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Check GDK Release Configuration (push) Waiting to run
CI / changes (push) Waiting to run
CI / goose ACP npm packages (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Build Lean ACP Binary (push) Blocked by required conditions
CI / Check goose-sdk UniFFI (push) Blocked by required conditions
CI / Test Kotlin GDK package (push) Blocked by required conditions
CI / Build and Test TLS Backend (native-tls) (push) Blocked by required conditions
CI / Build and Test TLS Backend (rustls-tls) (push) Blocked by required conditions
CI / Build and Test Roaming Feature (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
CI / Test Desktop Updater Install (ubuntu-latest) (push) Blocked by required conditions
CI / Test Desktop Updater Install (windows-latest) (push) Blocked by required conditions
MCP Conformance / Build Conformance Binaries (push) Waiting to run
MCP Conformance / Conformance 2025-11-25 / 0.1.16 (push) Blocked by required conditions
MCP Conformance / Conformance 2025-11-25 / 0.2.0-alpha.10 (push) Blocked by required conditions
MCP Conformance / Conformance 2026-07-28 / 0.2.0-alpha.10 (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Some checks are pending
Canary / Prepare Version (push) Waiting to run
Canary / build-cli-linux (push) Blocked by required conditions
Canary / Upload Install Script (push) Waiting to run
Canary / bundle-macos-arm64 (push) Blocked by required conditions
Canary / bundle-macos-x64 (push) Blocked by required conditions
Canary / bundle-windows (push) Blocked by required conditions
Canary / bundle-windows-cuda (push) Blocked by required conditions
Canary / Release (push) Blocked by required conditions
Unused Dependencies / machete (push) Waiting to run
CI / Check GDK API Reference is Up-to-Date (push) Waiting to run
CI / Check Rust Code Format (push) Blocked by required conditions
CI / Check GDK Release Configuration (push) Waiting to run
CI / changes (push) Waiting to run
CI / goose ACP npm packages (push) Blocked by required conditions
CI / Build and Test Rust Project (push) Blocked by required conditions
CI / Build Lean ACP Binary (push) Blocked by required conditions
CI / Check goose-sdk UniFFI (push) Blocked by required conditions
CI / Test Kotlin GDK package (push) Blocked by required conditions
CI / Build and Test TLS Backend (native-tls) (push) Blocked by required conditions
CI / Build and Test TLS Backend (rustls-tls) (push) Blocked by required conditions
CI / Build and Test Roaming Feature (push) Blocked by required conditions
CI / Build Rust Project on Windows (push) Waiting to run
CI / Check MSRV (push) Blocked by required conditions
CI / Lint Rust Code (push) Blocked by required conditions
CI / Check Generated Schemas are Up-to-Date (push) Blocked by required conditions
CI / Test and Lint Electron Desktop App (push) Blocked by required conditions
CI / Test Desktop Updater Install (ubuntu-latest) (push) Blocked by required conditions
CI / Test Desktop Updater Install (windows-latest) (push) Blocked by required conditions
MCP Conformance / Build Conformance Binaries (push) Waiting to run
MCP Conformance / Conformance 2025-11-25 / 0.1.16 (push) Blocked by required conditions
MCP Conformance / Conformance 2025-11-25 / 0.2.0-alpha.10 (push) Blocked by required conditions
MCP Conformance / Conformance 2026-07-28 / 0.2.0-alpha.10 (push) Blocked by required conditions
Create Minor Release PR / check-version-bump-pr (push) Waiting to run
Create Minor Release PR / release (push) Blocked by required conditions
Live Provider Tests / check-fork (push) Waiting to run
Live Provider Tests / changes (push) Blocked by required conditions
Live Provider Tests / Build Binary (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (push) Blocked by required conditions
Live Provider Tests / Smoke Tests (Code Execution) (push) Blocked by required conditions
Live Provider Tests / Compaction Tests (push) Blocked by required conditions
Publish Docker Image / docker (push) Waiting to run
Scorecard supply-chain security / Scorecard analysis (push) Waiting to run
Co-authored-by: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
parent
6a2c3dadf6
commit
80c1197583
1 changed files with 79 additions and 5 deletions
|
|
@ -241,14 +241,17 @@ impl PromptInjectionScanner {
|
|||
});
|
||||
}
|
||||
|
||||
let max_confidence = stream::iter(user_messages)
|
||||
let (max_confidence, any_scanned) = stream::iter(user_messages)
|
||||
.map(|msg| async move {
|
||||
self.scan_with_classifier(&msg, classifier, ClassifierType::Prompt)
|
||||
.await
|
||||
})
|
||||
.buffer_unordered(ML_SCAN_CONCURRENCY)
|
||||
.fold(0.0_f32, |acc, result| async move {
|
||||
result.unwrap_or(0.0).max(acc)
|
||||
.fold((0.0_f32, false), |(acc, scanned), result| async move {
|
||||
match result {
|
||||
Some(confidence) => (confidence.max(acc), true),
|
||||
None => (acc, scanned),
|
||||
}
|
||||
})
|
||||
.await;
|
||||
|
||||
|
|
@ -256,7 +259,7 @@ impl PromptInjectionScanner {
|
|||
confidence: max_confidence,
|
||||
pattern_confidence: 0.0,
|
||||
pattern_matches: Vec::new(),
|
||||
ml_confidence: Some(max_confidence),
|
||||
ml_confidence: any_scanned.then_some(max_confidence),
|
||||
used_pattern_detection: false,
|
||||
})
|
||||
}
|
||||
|
|
@ -432,7 +435,7 @@ impl Default for PromptInjectionScanner {
|
|||
mod tests {
|
||||
use super::*;
|
||||
use rmcp::object;
|
||||
use wiremock::matchers::method;
|
||||
use wiremock::matchers::{body_string_contains, method};
|
||||
use wiremock::{Mock, MockServer, ResponseTemplate};
|
||||
|
||||
async fn classifier_with_confidence(
|
||||
|
|
@ -715,4 +718,75 @@ mod tests {
|
|||
vec!["newest prompt", "middle prompt", "oldest prompt"]
|
||||
);
|
||||
}
|
||||
|
||||
async fn failing_prompt_classifier() -> (MockServer, ClassificationClient) {
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
let classifier =
|
||||
ClassificationClient::from_endpoint(mock_server.uri(), None, None).unwrap();
|
||||
(mock_server, classifier)
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn failed_prompt_scan_does_not_lower_command_confidence() {
|
||||
let (_command_server, command_classifier) = classifier_with_confidence(0.85).await;
|
||||
let (_prompt_server, prompt_classifier) = failing_prompt_classifier().await;
|
||||
let scanner = PromptInjectionScanner {
|
||||
pattern_matcher: PatternMatcher::new(),
|
||||
command_classifier: Some(command_classifier),
|
||||
prompt_classifier: Some(prompt_classifier),
|
||||
};
|
||||
|
||||
let tool_call =
|
||||
CallToolRequestParams::new("shell").with_arguments(object!({"command": "echo hello"}));
|
||||
let messages = vec![Message::user().with_text("deploy the service")];
|
||||
|
||||
let result = scanner
|
||||
.analyze_tool_call_with_context(&tool_call, &messages)
|
||||
.await
|
||||
.unwrap();
|
||||
|
||||
assert!((result.confidence - 0.85).abs() < 1e-6);
|
||||
}
|
||||
|
||||
#[tokio::test]
|
||||
async fn partially_failed_prompt_scan_keeps_successful_scores() {
|
||||
let mock_server = MockServer::start().await;
|
||||
Mock::given(method("POST"))
|
||||
.and(body_string_contains("unscannable"))
|
||||
.respond_with(ResponseTemplate::new(500))
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
Mock::given(method("POST"))
|
||||
.respond_with(
|
||||
ResponseTemplate::new(200).set_body_json(serde_json::json!([[{
|
||||
"label": "INJECTION",
|
||||
"score": 0.6
|
||||
}, {
|
||||
"label": "SAFE",
|
||||
"score": 0.4
|
||||
}]])),
|
||||
)
|
||||
.mount(&mock_server)
|
||||
.await;
|
||||
let classifier =
|
||||
ClassificationClient::from_endpoint(mock_server.uri(), None, None).unwrap();
|
||||
let scanner = PromptInjectionScanner {
|
||||
pattern_matcher: PatternMatcher::new(),
|
||||
command_classifier: None,
|
||||
prompt_classifier: Some(classifier),
|
||||
};
|
||||
let messages = vec![
|
||||
Message::user().with_text("unscannable message"),
|
||||
Message::user().with_text("ordinary message"),
|
||||
];
|
||||
|
||||
let result = scanner.scan_conversation(&messages).await.unwrap();
|
||||
|
||||
let confidence = result.ml_confidence.expect("surviving scan should report");
|
||||
assert!((confidence - 0.6).abs() < 1e-6);
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue