mirror of
https://github.com/block/buzz.git
synced 2026-10-27 04:30:52 +00:00
Add Buzz-native collaboration benchmarks (#6264)
## Why Terminal-Bench can show task correctness while missing Buzz-specific collaboration failures, such as posting a final answer outside the user's thread. Add a small Buzz-native suite that scores relay behavior directly. ## What - Add Harbor tasks for direct thread replies, event-level callback mentions to a three-word user identity, targeted reads of explicitly named paths outside the workspace, and exact temporary-channel membership against a seeded directory of 50 users and 10 bots - Export sanitized post-agent relay evidence and extend solo runtime/provisioning for deterministic verification - Expose channel TTL in CLI search output and cover positive and negative transcript/state fixtures ## Risk Assessment Low to medium — this primarily adds benchmark and testbed machinery. The CLI change is an additive `ttl_seconds` field in channel-search JSON; relay behavior is unchanged. ## References - Motivated by the Buzz-specific behavior observed while benchmarking https://github.com/block/buzz/pull/6128 - The named-path regression case covers the issue being fixed by https://github.com/block/buzz/pull/6261 - Local live Sonnet runs: `reply-to-thread`, `user-mention`, and `create-channel-invite-users` each scored reward 1.0 with zero exceptions - The current branch prompt scored 3/5 on a concurrent `read-named-path-outside-workspace` batch: one explicit workspace-boundary refusal and one missing final response Generated with Codex --------- Signed-off-by: Salman Mohammed <smohammed@squareup.com>
This commit is contained in:
@@ -161,6 +161,7 @@ struct ChannelSummary {
|
||||
about: Option<String>,
|
||||
topic: Option<String>,
|
||||
purpose: Option<String>,
|
||||
ttl_seconds: Option<i64>,
|
||||
}
|
||||
|
||||
impl ChannelSummary {
|
||||
@@ -176,6 +177,7 @@ impl ChannelSummary {
|
||||
let mut about: Option<String> = None;
|
||||
let mut topic: Option<String> = None;
|
||||
let mut purpose: Option<String> = None;
|
||||
let mut ttl_seconds: Option<i64> = None;
|
||||
|
||||
for tag in tags {
|
||||
let Some(tag_arr) = tag.as_array() else {
|
||||
@@ -194,6 +196,7 @@ impl ChannelSummary {
|
||||
"about" => about = val.map(str::to_string),
|
||||
"topic" => topic = val.map(str::to_string),
|
||||
"purpose" => purpose = val.map(str::to_string),
|
||||
"ttl" => ttl_seconds = val.and_then(|value| value.parse().ok()),
|
||||
"archived" => archived = val == Some("true"),
|
||||
_ => {}
|
||||
}
|
||||
@@ -208,6 +211,7 @@ impl ChannelSummary {
|
||||
about,
|
||||
topic,
|
||||
purpose,
|
||||
ttl_seconds,
|
||||
})
|
||||
}
|
||||
}
|
||||
@@ -1215,6 +1219,7 @@ mod tests {
|
||||
["about", "About text"],
|
||||
["topic", "Composer work"],
|
||||
["purpose", "Track UI for the composer"],
|
||||
["ttl", "3600"],
|
||||
]));
|
||||
let s = ChannelSummary::from_event(&ev).expect("parse");
|
||||
assert_eq!(s.channel_id, "11111111-1111-1111-1111-111111111111");
|
||||
@@ -1225,6 +1230,7 @@ mod tests {
|
||||
assert_eq!(s.about.as_deref(), Some("About text"));
|
||||
assert_eq!(s.topic.as_deref(), Some("Composer work"));
|
||||
assert_eq!(s.purpose.as_deref(), Some("Track UI for the composer"));
|
||||
assert_eq!(s.ttl_seconds, Some(3600));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
||||
Reference in New Issue
Block a user