mirror of
https://github.com/block/buzz.git
synced 2026-11-08 18:31:56 +00:00
test(agent): cover unsupported image recovery
Co-authored-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz> Signed-off-by: Wren <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@buzz.block.builderlab.xyz>
This commit is contained in:
@@ -1711,10 +1711,6 @@ fn is_unsupported_image_input_error(body: &str) -> bool {
|
||||
.contains("no endpoints found that support image input")
|
||||
}
|
||||
|
||||
fn unsupported_image_input_error(error_body: String) -> AgentError {
|
||||
AgentError::UnsupportedImageInput(error_body)
|
||||
}
|
||||
|
||||
/// Build the terminal `AgentError::Llm` for a `post()` exit that has given up
|
||||
/// retrying — persistent retryable status, transport failure, or a body-read
|
||||
/// break. `detail` carries the specific cause (status/body, or the transport
|
||||
@@ -1875,7 +1871,9 @@ where
|
||||
if status == 404 {
|
||||
let error_body = read_error_body(resp).await;
|
||||
if is_unsupported_image_input_error(&error_body) {
|
||||
return Err(PostError::Agent(unsupported_image_input_error(error_body)));
|
||||
return Err(PostError::Agent(AgentError::UnsupportedImageInput(
|
||||
error_body,
|
||||
)));
|
||||
}
|
||||
return Err(PostError::Agent(AgentError::LlmModelNotFound(format!(
|
||||
"{status}: {error_body}"
|
||||
@@ -2130,7 +2128,7 @@ async fn openrouter_post(
|
||||
// `LlmModelNotFound` (or vice versa) sends the user to the wrong fix.
|
||||
let error_body = read_error_body(resp).await;
|
||||
if is_unsupported_image_input_error(&error_body) {
|
||||
return Err(unsupported_image_input_error(error_body));
|
||||
return Err(AgentError::UnsupportedImageInput(error_body));
|
||||
}
|
||||
if error_body.contains("No endpoints found that can handle the requested parameters") {
|
||||
return Err(openrouter_parameter_routing_error(&error_body));
|
||||
|
||||
@@ -12,6 +12,7 @@
|
||||
//! (use a large value, e.g. 999, to simulate hang)
|
||||
//! FAKE_MCP_RESULT_SIZE=N — `tools/call` returns an N-byte text result
|
||||
//! (default: the literal "ok"); grows history
|
||||
//! FAKE_MCP_IMAGE_RESULT=1 — `tools/call` returns text plus a PNG image block
|
||||
//! FAKE_MCP_PID_FILE=path — write the child PID to `path` on startup
|
||||
//! (for tests that want to verify the child died)
|
||||
//! FAKE_MCP_SPAWN_GRANDCHILD=1
|
||||
@@ -300,10 +301,18 @@ fn main() {
|
||||
} else {
|
||||
"ok".to_owned()
|
||||
};
|
||||
let content = if env_flag("FAKE_MCP_IMAGE_RESULT") {
|
||||
json!([
|
||||
{ "type": "text", "text": result_text },
|
||||
{ "type": "image", "data": "aW1n", "mimeType": "image/png" },
|
||||
])
|
||||
} else {
|
||||
json!([{ "type": "text", "text": result_text }])
|
||||
};
|
||||
write_response(
|
||||
id,
|
||||
json!({
|
||||
"content": [{ "type": "text", "text": result_text }],
|
||||
"content": content,
|
||||
"isError": false,
|
||||
}),
|
||||
);
|
||||
|
||||
@@ -57,9 +57,26 @@ async fn spawn_fake_llm(responses: Vec<Value>) -> String {
|
||||
url
|
||||
}
|
||||
|
||||
struct CannedResponse {
|
||||
status: u16,
|
||||
body: Value,
|
||||
}
|
||||
|
||||
/// Like `spawn_fake_llm` but also captures the full JSON request body from each
|
||||
/// incoming HTTP request. Returns (url, captured_requests).
|
||||
async fn spawn_capturing_fake_llm(responses: Vec<Value>) -> (String, Arc<Mutex<Vec<Value>>>) {
|
||||
spawn_capturing_fake_llm_with_statuses(
|
||||
responses
|
||||
.into_iter()
|
||||
.map(|body| CannedResponse { status: 200, body })
|
||||
.collect(),
|
||||
)
|
||||
.await
|
||||
}
|
||||
|
||||
async fn spawn_capturing_fake_llm_with_statuses(
|
||||
responses: Vec<CannedResponse>,
|
||||
) -> (String, Arc<Mutex<Vec<Value>>>) {
|
||||
let listener = TcpListener::bind("127.0.0.1:0").await.unwrap();
|
||||
let url = format!("http://{}", listener.local_addr().unwrap());
|
||||
let queue = Arc::new(Mutex::new(VecDeque::from(responses)));
|
||||
@@ -122,15 +139,22 @@ async fn spawn_capturing_fake_llm(responses: Vec<Value>) -> (String, Arc<Mutex<V
|
||||
}
|
||||
|
||||
// Send canned response.
|
||||
let body = queue
|
||||
.lock()
|
||||
.await
|
||||
.pop_front()
|
||||
.unwrap_or_else(|| json!({ "error": "no canned response" }));
|
||||
let body_s = serde_json::to_string(&body).unwrap();
|
||||
let response = queue.lock().await.pop_front().unwrap_or(CannedResponse {
|
||||
status: 500,
|
||||
body: json!({ "error": "no canned response" }),
|
||||
});
|
||||
let body_s = serde_json::to_string(&response.body).unwrap();
|
||||
let reason = if response.status == 200 {
|
||||
"OK"
|
||||
} else {
|
||||
"Error"
|
||||
};
|
||||
let resp = format!(
|
||||
"HTTP/1.1 200 OK\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
body_s.len(), body_s,
|
||||
"HTTP/1.1 {} {}\r\nContent-Type: application/json\r\nContent-Length: {}\r\nConnection: close\r\n\r\n{}",
|
||||
response.status,
|
||||
reason,
|
||||
body_s.len(),
|
||||
body_s,
|
||||
);
|
||||
let _ = sock.write_all(resp.as_bytes()).await;
|
||||
let _ = sock.shutdown().await;
|
||||
@@ -316,6 +340,103 @@ async fn tool_call_then_end_turn() {
|
||||
h.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn unsupported_image_response_recovers_without_replaying_image() {
|
||||
let responses = vec![
|
||||
CannedResponse {
|
||||
status: 200,
|
||||
body: openai_tool_call("call_image", "fake__tool_0", json!({})),
|
||||
},
|
||||
CannedResponse {
|
||||
status: 404,
|
||||
body: json!({
|
||||
"error": { "message": "No endpoints found that support image input" }
|
||||
}),
|
||||
},
|
||||
CannedResponse {
|
||||
status: 200,
|
||||
body: openai_text("recovered"),
|
||||
},
|
||||
];
|
||||
let (url, captures) = spawn_capturing_fake_llm_with_statuses(responses).await;
|
||||
let mut h = Harness::spawn(&url).await;
|
||||
|
||||
h.send(
|
||||
"initialize",
|
||||
json!({"protocolVersion":2,"clientCapabilities":{}}),
|
||||
)
|
||||
.await;
|
||||
let _ = h.recv().await;
|
||||
let session_id = h
|
||||
.send(
|
||||
"session/new",
|
||||
json!({
|
||||
"cwd": "/tmp",
|
||||
"mcpServers": [{
|
||||
"name": "fake",
|
||||
"command": env!("CARGO_BIN_EXE_fake-mcp"),
|
||||
"args": [],
|
||||
"env": [{ "name": "FAKE_MCP_IMAGE_RESULT", "value": "1" }],
|
||||
}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
let session = h.recv_until(|v| v["id"] == json!(session_id)).await;
|
||||
let sid = session["result"]["sessionId"].as_str().unwrap();
|
||||
|
||||
let prompt_id = h
|
||||
.send(
|
||||
"session/prompt",
|
||||
json!({
|
||||
"sessionId": sid,
|
||||
"prompt": [{"type":"text","text":"inspect the image"}],
|
||||
}),
|
||||
)
|
||||
.await;
|
||||
loop {
|
||||
let message = h.recv().await;
|
||||
if message.get("method") == Some(&json!("session/request_permission")) {
|
||||
h.write(json!({
|
||||
"jsonrpc": "2.0",
|
||||
"id": message["id"],
|
||||
"result": { "outcome": { "outcome": "selected", "optionId": "allow" } },
|
||||
}))
|
||||
.await;
|
||||
} else if message["id"] == json!(prompt_id) {
|
||||
assert_eq!(message["result"]["stopReason"], "end_turn");
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
let requests = captures.lock().await;
|
||||
assert_eq!(
|
||||
requests.len(),
|
||||
3,
|
||||
"expected tool, rejection, recovery requests"
|
||||
);
|
||||
let rejected = requests[1].to_string();
|
||||
assert!(
|
||||
rejected.contains("data:image/png;base64,aW1n"),
|
||||
"second request must contain the MCP image: {rejected}"
|
||||
);
|
||||
let recovered = requests[2].to_string();
|
||||
assert!(
|
||||
!recovered.contains("image_url") && !recovered.contains("data:image"),
|
||||
"recovery request must not replay image input: {recovered}"
|
||||
);
|
||||
assert!(
|
||||
recovered.contains("does not support image input")
|
||||
&& recovered.contains("text-based inspection"),
|
||||
"recovery request must give the model actionable guidance: {recovered}"
|
||||
);
|
||||
assert!(
|
||||
recovered.contains("call_image") && recovered.contains("tool_call_id"),
|
||||
"recovery must preserve tool-call/result pairing: {recovered}"
|
||||
);
|
||||
drop(requests);
|
||||
h.shutdown().await;
|
||||
}
|
||||
|
||||
#[tokio::test(flavor = "multi_thread", worker_threads = 2)]
|
||||
async fn rejects_concurrent_prompts() {
|
||||
// Slow first response so the second prompt arrives mid-flight.
|
||||
|
||||
Reference in New Issue
Block a user