mirror of
https://github.com/block/buzz.git
synced 2026-10-27 12:40:30 +00:00
fix(channels): return complete member rosters (#5765)
## Summary - return complete channel rosters instead of truncating at 1,000 members - chunk `event_mentions` inserts inside one transaction so large kind `39002` snapshots remain discoverable by every `p` tag - add a targeted `buzz-admin reconcile-channels --channel <uuid>` force-republish path for stale discovery snapshots - cover a 1,501-member roster, 11,000-tag mention index, and kind `39002` tag construction past member 1,000 ## Why The relay builds NIP-29 discovery and several authorization decisions from `get_members()`, but that helper silently returned only the first 1,000 active members. Desktop then counted the truncated kind `39002` event, while late members could be rejected by roster-scanning member actions. Removing the roster cap exposes PostgreSQL's 65,535 bind-parameter ceiling in mention indexing, so the insert is chunked transactionally to preserve all-or-nothing indexing. The existing reconcilers only fill missing discovery events. The targeted admin option bypasses the separately known 1,000-channel reconciliation-list ceiling and replaces an existing channel snapshot using the configured production relay key. ## Attribution This supersedes and builds on #3166 by @LordMelkor. Thank you for identifying the roster boundary and contributing the original complete-roster and mention-index patch. The production roster/query changes and the two PostgreSQL regressions retain that work's shape; this PR rebases it onto current `main`, adds relay coverage, and adds the targeted repair operation requested for rollout. ## Validation Exact pushed head: `24d02e4f3824150ed84913c9d230e675502e5b12` - `cargo check -p buzz-db -p buzz-admin` - `cargo test -p buzz-db channel::tests::get_members_returns_full_roster_beyond_1000 -- --ignored --exact --nocapture` - `cargo test -p buzz-db feed::tests::insert_mentions_indexes_rosters_past_bind_parameter_cap -- --ignored --exact --nocapture` - `cargo test -p buzz-relay --lib handlers::side_effects::tests::group_members_snapshot_keeps_members_past_one_thousand -- --exact` - `cargo run -q -p buzz-admin -- reconcile-channels --help` - mandatory pre-push hook: branch-skew, desktop checks/typecheck/tests, mobile tests, Rust tests, and desktop Tauri checks all passed on the pushed head ## Rollout 1. Deploy the relay/backend build. 2. Run `buzz-admin reconcile-channels --channel <general-channel-uuid>` with `BUZZ_RELAY_PRIVATE_KEY` configured. 3. Verify the replacement kind `39002` roster count matches the active database membership count. No schema migration or desktop release is required. Fixes #3156 Supersedes #3166 --------- Signed-off-by: Wes <wesbillman@users.noreply.github.com> Co-authored-by: Carl <c7ebe626f000404285d3686e1dc74cc07cc60a9754a150041ba132e14bd3e2ec@buzz.block.builderlab.xyz>
This commit is contained in:
Wes
127
scripts/e2e-large-channel-roster.sh
Executable file
127
scripts/e2e-large-channel-roster.sh
Executable file
@@ -0,0 +1,127 @@
|
||||
#!/usr/bin/env bash
|
||||
# Prove a channel member past the historical 1,000-row boundary is present in
|
||||
# relay-served discovery, can write, and survives an authoritative republish.
|
||||
set -euo pipefail
|
||||
|
||||
REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
|
||||
cd "$REPO_ROOT"
|
||||
|
||||
: "${DATABASE_URL:?set DATABASE_URL to the isolated relay database}"
|
||||
: "${BUZZ_RELAY_URL:=http://localhost:3030}"
|
||||
: "${RELAY_URL:=ws://localhost:3030}"
|
||||
: "${BUZZ_RELAY_PRIVATE_KEY:=0000000000000000000000000000000000000000000000000000000000000001}"
|
||||
export BUZZ_RELAY_URL RELAY_URL BUZZ_RELAY_PRIVATE_KEY
|
||||
unset BUZZ_AUTH_TAG
|
||||
|
||||
for binary in buzz buzz-admin; do
|
||||
resolved="$(command -v "$binary" || true)"
|
||||
[[ "$resolved" == "$REPO_ROOT/target/release/$binary" ]] || {
|
||||
echo "error: $binary must resolve to $REPO_ROOT/target/release/$binary (got ${resolved:-not found})" >&2
|
||||
exit 1
|
||||
}
|
||||
done
|
||||
command -v jq >/dev/null || { echo "error: jq is required" >&2; exit 1; }
|
||||
command -v psql >/dev/null || { echo "error: psql is required" >&2; exit 1; }
|
||||
|
||||
key_field() {
|
||||
awk -v label="$1" '$1 == label && $2 == "key:" { print $3 }'
|
||||
}
|
||||
|
||||
OWNER_GEN="$(buzz-admin generate-key)"
|
||||
OWNER_SK="$(printf '%s\n' "$OWNER_GEN" | key_field Secret)"
|
||||
export BUZZ_PRIVATE_KEY="$OWNER_SK"
|
||||
|
||||
CHANNEL="$(buzz channels create \
|
||||
--name "roster-boundary-$$" --type stream --visibility open | jq -er '.channel_id')"
|
||||
|
||||
LATE_GEN="$(buzz-admin generate-key)"
|
||||
LATE_SK="$(printf '%s\n' "$LATE_GEN" | key_field Secret)"
|
||||
LATE_PUBKEY="$(printf '%s\n' "$LATE_GEN" | key_field Public)"
|
||||
|
||||
# The creator is roster position 1. Add 1,499 fixtures followed by the real
|
||||
# late identity at position 1,501. A final API-added member forces the relay to
|
||||
# emit a fresh discovery snapshot through the normal membership-change path.
|
||||
psql "$DATABASE_URL" -v ON_ERROR_STOP=1 \
|
||||
--set=channel="$CHANNEL" --set=late_pubkey="$LATE_PUBKEY" <<'SQL'
|
||||
WITH target AS (
|
||||
SELECT community_id, id
|
||||
FROM channels
|
||||
WHERE id = :'channel'::uuid
|
||||
), fixtures AS (
|
||||
SELECT n, decode(lpad(to_hex(n + 65536), 64, '0'), 'hex') AS pubkey
|
||||
FROM generate_series(1, 1499) AS n
|
||||
)
|
||||
INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at)
|
||||
SELECT target.community_id, target.id, fixtures.pubkey, 'member',
|
||||
NOW() + fixtures.n * interval '1 millisecond'
|
||||
FROM target CROSS JOIN fixtures;
|
||||
|
||||
INSERT INTO channel_members (community_id, channel_id, pubkey, role, joined_at)
|
||||
SELECT community_id, id, decode(:'late_pubkey', 'hex'), 'member',
|
||||
NOW() + interval '2 seconds'
|
||||
FROM channels
|
||||
WHERE id = :'channel'::uuid;
|
||||
SQL
|
||||
|
||||
TRIGGER_GEN="$(buzz-admin generate-key)"
|
||||
TRIGGER_PUBKEY="$(printf '%s\n' "$TRIGGER_GEN" | key_field Public)"
|
||||
buzz channels add-member --channel "$CHANNEL" --pubkey "$TRIGGER_PUBKEY" --role member >/dev/null
|
||||
|
||||
BEFORE="$(buzz channels members --channel "$CHANNEL")"
|
||||
BEFORE_COUNT="$(jq 'length' <<<"$BEFORE")"
|
||||
jq -e --arg pk "$LATE_PUBKEY" 'any(.[]; .pubkey == $pk and .role == "member")' \
|
||||
<<<"$BEFORE" >/dev/null
|
||||
(( BEFORE_COUNT > 1000 ))
|
||||
printf 'PASS discovery-before-republish channel=%s members=%s late_pubkey=%s\n' \
|
||||
"$CHANNEL" "$BEFORE_COUNT" "$LATE_PUBKEY"
|
||||
|
||||
export BUZZ_PRIVATE_KEY="$LATE_SK"
|
||||
ACTION="$(buzz messages send --channel "$CHANNEL" --content "member-1501-action")"
|
||||
jq -e '.accepted == true' <<<"$ACTION" >/dev/null
|
||||
ACTION_ID="$(jq -er '.event_id' <<<"$ACTION")"
|
||||
buzz messages get --channel "$CHANNEL" --limit 10 \
|
||||
| jq -e --arg id "$ACTION_ID" 'any(.[]; .id == $id and .content == "member-1501-action")' >/dev/null
|
||||
printf 'PASS late-member-action event_id=%s\n' "$ACTION_ID"
|
||||
|
||||
# Targeted roster repair must not replace canonical metadata or admin events.
|
||||
# Preserve both the event ID and complete tags, including fields this backfill
|
||||
# command does not know how to rebuild (DM participants, topic, TTL, etc.).
|
||||
DISCOVERY_BEFORE="$(psql "$DATABASE_URL" -AtX -v ON_ERROR_STOP=1 \
|
||||
--set=channel="$CHANNEL" <<'SQL'
|
||||
SELECT jsonb_object_agg(kind::text, jsonb_build_object(
|
||||
'id', encode(id, 'hex'),
|
||||
'tags', tags
|
||||
) ORDER BY kind)::text
|
||||
FROM events
|
||||
WHERE channel_id = :'channel'::uuid
|
||||
AND kind IN (39000, 39001)
|
||||
AND deleted_at IS NULL;
|
||||
SQL
|
||||
)"
|
||||
jq -e 'has("39000") and has("39001")' <<<"$DISCOVERY_BEFORE" >/dev/null
|
||||
|
||||
DATABASE_URL="$DATABASE_URL" RELAY_URL="$RELAY_URL" \
|
||||
buzz-admin reconcile-channels --channel "$CHANNEL" >/dev/null
|
||||
|
||||
DISCOVERY_AFTER="$(psql "$DATABASE_URL" -AtX -v ON_ERROR_STOP=1 \
|
||||
--set=channel="$CHANNEL" <<'SQL'
|
||||
SELECT jsonb_object_agg(kind::text, jsonb_build_object(
|
||||
'id', encode(id, 'hex'),
|
||||
'tags', tags
|
||||
) ORDER BY kind)::text
|
||||
FROM events
|
||||
WHERE channel_id = :'channel'::uuid
|
||||
AND kind IN (39000, 39001)
|
||||
AND deleted_at IS NULL;
|
||||
SQL
|
||||
)"
|
||||
[[ "$DISCOVERY_AFTER" == "$DISCOVERY_BEFORE" ]]
|
||||
printf 'PASS targeted-repair-preserves-metadata-and-admin-events channel=%s\n' "$CHANNEL"
|
||||
|
||||
AFTER="$(buzz channels members --channel "$CHANNEL")"
|
||||
AFTER_COUNT="$(jq 'length' <<<"$AFTER")"
|
||||
jq -e --arg pk "$LATE_PUBKEY" 'any(.[]; .pubkey == $pk and .role == "member")' \
|
||||
<<<"$AFTER" >/dev/null
|
||||
(( AFTER_COUNT == BEFORE_COUNT ))
|
||||
printf 'PASS discovery-after-republish channel=%s members=%s late_pubkey=%s\n' \
|
||||
"$CHANNEL" "$AFTER_COUNT" "$LATE_PUBKEY"
|
||||
Reference in New Issue
Block a user