mirror of
https://github.com/block/buzz.git
synced 2026-10-29 21:41:33 +00:00
a9640c7cc4
## Why Terminal-Bench can show task correctness while missing Buzz-specific collaboration failures, such as posting a final answer outside the user's thread. Add a small Buzz-native suite that scores relay behavior directly. ## What - Add Harbor tasks for direct thread replies, event-level callback mentions to a three-word user identity, targeted reads of explicitly named paths outside the workspace, and exact temporary-channel membership against a seeded directory of 50 users and 10 bots - Export sanitized post-agent relay evidence and extend solo runtime/provisioning for deterministic verification - Expose channel TTL in CLI search output and cover positive and negative transcript/state fixtures ## Risk Assessment Low to medium — this primarily adds benchmark and testbed machinery. The CLI change is an additive `ttl_seconds` field in channel-search JSON; relay behavior is unchanged. ## References - Motivated by the Buzz-specific behavior observed while benchmarking https://github.com/block/buzz/pull/6128 - The named-path regression case covers the issue being fixed by https://github.com/block/buzz/pull/6261 - Local live Sonnet runs: `reply-to-thread`, `user-mention`, and `create-channel-invite-users` each scored reward 1.0 with zero exceptions - The current branch prompt scored 3/5 on a concurrent `read-named-path-outside-workspace` batch: one explicit workspace-boundary refusal and one missing final response Generated with Codex --------- Signed-off-by: Salman Mohammed <smohammed@squareup.com>
30 lines
564 B
TOML
30 lines
564 B
TOML
[project]
|
|
name = "harbor-buzz-orchestra"
|
|
version = "0.1.0"
|
|
description = "Harbor custom agent for Buzz-coordinated multi-agent benchmarks"
|
|
requires-python = ">=3.12"
|
|
dependencies = [
|
|
"harbor>=0.16.1,<0.18",
|
|
"pydantic>=2.11",
|
|
"pyyaml>=6.0",
|
|
]
|
|
|
|
[build-system]
|
|
requires = ["hatchling"]
|
|
build-backend = "hatchling.build"
|
|
|
|
[project.optional-dependencies]
|
|
dev = [
|
|
"pytest>=8.4",
|
|
"pytest-asyncio>=1.2",
|
|
"ruff==0.16.3",
|
|
]
|
|
|
|
[tool.pytest.ini_options]
|
|
testpaths = ["tests"]
|
|
asyncio_mode = "auto"
|
|
|
|
[tool.ruff]
|
|
target-version = "py312"
|
|
line-length = 88
|