mirror of
https://github.com/block/buzz.git
synced 2026-10-27 04:30:52 +00:00
f437e43d09
Signed-off-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co> Signed-off-by: Tyler Longwell <tlongwell@block.xyz> Co-authored-by: npub12gtutshhh76rx0jx697f32f9tffd4hhp3hx58fp4x6u4uemkm7sqf8f757 <5217c5c2f7bfb4333e46d17c98a9255a52dadee18dcd43a43536b95e6776dfa0@sprout-oss.stage.blox.sqprod.co> Co-authored-by: npub1qyvc0c5kl4gqv2fd97fsk46tu378sqgy35vc83rvgfwne90sel7s0ed67d <011987e296fd5006292d2f930b574be47c7801048d1983c46c425d3c95f0cffd@sprout-oss.stage.blox.sqprod.co> Co-authored-by: Tyler Longwell <tlongwell@block.xyz>
124 lines
4.5 KiB
Python
124 lines
4.5 KiB
Python
"""Live provisioning tests against a running benchmark stack.
|
|
|
|
Gated by BUZZ_TESTBED_LIVE=1 with stack coordinates in the environment:
|
|
BUZZ_TESTBED_RELAY_HTTP (default http://localhost:3000)
|
|
BUZZ_TESTBED_RELAY_WS (default ws://host.docker.internal:3000)
|
|
BUZZ_TESTBED_OWNER_KEY relay owner secret key (hex)
|
|
BUZZ_TESTBED_PG_DSN benchmark Postgres DSN
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import uuid
|
|
|
|
import psycopg
|
|
import pytest
|
|
from harbor_buzz_testbed.buzz_cli import BuzzCli, BuzzCliError
|
|
from harbor_buzz_testbed.provisioner import (
|
|
BuzzTrialProvisioner,
|
|
ProvisioningError,
|
|
TestbedConfig,
|
|
)
|
|
|
|
pytestmark = pytest.mark.skipif(
|
|
os.environ.get("BUZZ_TESTBED_LIVE") != "1",
|
|
reason="live testbed suite; set BUZZ_TESTBED_LIVE=1 with a running stack",
|
|
)
|
|
|
|
|
|
@pytest.fixture()
|
|
def provisioner() -> BuzzTrialProvisioner:
|
|
owner_key = os.environ.get("BUZZ_TESTBED_OWNER_KEY")
|
|
dsn = os.environ.get("BUZZ_TESTBED_PG_DSN")
|
|
if not owner_key or not dsn:
|
|
pytest.fail("BUZZ_TESTBED_OWNER_KEY and BUZZ_TESTBED_PG_DSN are required")
|
|
return BuzzTrialProvisioner(
|
|
TestbedConfig(
|
|
relay_http_url=os.environ.get(
|
|
"BUZZ_TESTBED_RELAY_HTTP", "http://localhost:3000"
|
|
),
|
|
relay_ws_url=os.environ.get(
|
|
"BUZZ_TESTBED_RELAY_WS", "ws://host.docker.internal:3000"
|
|
),
|
|
owner_secret_key=owner_key,
|
|
postgres_dsn=dsn,
|
|
llm_api_keys={
|
|
"databricks/glm": "test-glm-key",
|
|
"databricks/opus": "test-opus-key",
|
|
},
|
|
)
|
|
)
|
|
|
|
|
|
def cli_for(provisioner: BuzzTrialProvisioner, credential) -> BuzzCli:
|
|
return provisioner._cli_for(credential)
|
|
|
|
|
|
def test_create_is_idempotent_and_isolated(provisioner, manifest):
|
|
run_id = f"live-{uuid.uuid4().hex[:8]}"
|
|
trial_a = str(uuid.uuid4())
|
|
trial_b = str(uuid.uuid4())
|
|
|
|
provisioner.healthcheck()
|
|
handle_a = provisioner.create_trial(run_id, trial_a, manifest)
|
|
handle_b = provisioner.create_trial(run_id, trial_b, manifest)
|
|
try:
|
|
# Shape: one credential per roster slot, orchestrator first.
|
|
assert len(handle_a.credentials) == 3
|
|
assert handle_a.credentials[0].role == "orchestrator"
|
|
assert handle_a.manifest_hash == manifest.sha256
|
|
|
|
# Idempotency: same key returns the stored handle, keys included.
|
|
again = provisioner.create_trial(run_id, trial_a, manifest)
|
|
assert again == handle_a
|
|
|
|
# Isolation: trials get distinct channels and fresh keys...
|
|
assert handle_a.channel_id != handle_b.channel_id
|
|
keys_a = {c.nostr_secret_key for c in handle_a.credentials}
|
|
keys_b = {c.nostr_secret_key for c in handle_b.credentials}
|
|
assert not keys_a & keys_b
|
|
|
|
# ...and trial B's orchestrator cannot see trial A's channel.
|
|
cli_a = cli_for(provisioner, handle_a.credentials[0])
|
|
cli_b = cli_for(provisioner, handle_b.credentials[0])
|
|
cli_a.run(
|
|
"messages",
|
|
"send",
|
|
"--channel",
|
|
handle_a.channel_id,
|
|
"--content",
|
|
"trial A secret",
|
|
)
|
|
foreign_read = cli_b.run(
|
|
"messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
|
|
)
|
|
assert foreign_read == [], "cross-trial read must return nothing"
|
|
with pytest.raises(BuzzCliError, match="private"):
|
|
cli_b.run("channels", "join", "--channel", handle_a.channel_id)
|
|
|
|
# Members can read their own channel.
|
|
own_read = cli_a.run(
|
|
"messages", "get", "--channel", handle_a.channel_id, "--limit", "10"
|
|
)
|
|
assert [m["content"] for m in own_read] == ["trial A secret"]
|
|
|
|
# Same trial key + different manifest must be rejected, not silently
|
|
# reprovisioned.
|
|
altered = manifest.model_copy(update={"condition": "O-altered"})
|
|
with pytest.raises(ProvisioningError, match="already provisioned"):
|
|
provisioner.create_trial(run_id, trial_a, altered)
|
|
finally:
|
|
provisioner.teardown(handle_a)
|
|
provisioner.teardown(handle_b)
|
|
|
|
# Teardown is idempotent and stamps archived_at.
|
|
provisioner.teardown(handle_a)
|
|
with psycopg.connect(os.environ["BUZZ_TESTBED_PG_DSN"]) as conn:
|
|
row = conn.execute(
|
|
"SELECT archived_at FROM benchmark.trial_manifest"
|
|
" WHERE run_id = %s AND trial_id = %s",
|
|
(run_id, trial_a),
|
|
).fetchone()
|
|
assert row is not None and row[0] is not None
|