diff --git a/.config/nextest.toml b/.config/nextest.toml index 4481ebc6ba..1f9f3d156f 100644 --- a/.config/nextest.toml +++ b/.config/nextest.toml @@ -32,6 +32,10 @@ max-threads = 1 [[profile.postgres-ci.overrides]] filter = "test(/cluster_global_/)" test-group = "postgres-cluster-global" +# The group only serializes these tests with each other. They read unfiltered, +# cluster-wide `pg_stat_activity`, so any overlapping test's session can mask +# activity; claim every worker so they run alone. +threads-required = "num-test-threads" [[profile.postgres-ci.scripts]] # Script filters are separate from default-filter: they attach the setup and diff --git a/.env.example b/.env.example index 8e1fe15939..73d4d9f6ea 100644 --- a/.env.example +++ b/.env.example @@ -52,25 +52,30 @@ REDIS_URL=redis://localhost:6379 # Emergency kill switch for all automatic partition DDL, including catch-all # advancement; audits remain enabled. # BUZZ_PARTITION_MANAGER_CREATE_ENABLED=true -# Opt in to replacing an empty partition catch-all with six months of dedicated -# monthlies, at startup and on every audit interval (default false). Has no -# effect while the kill switch above is off. -# BUZZ_PARTITION_MANAGER_ADVANCE_ENABLED=false +# Replace an empty partition catch-all with six months of dedicated monthlies, +# at startup and on every audit interval (default true). Has no effect while the +# kill switch above is off. +# BUZZ_PARTITION_MANAGER_ADVANCE_ENABLED=true # Writer-session Postgres timeouts for buzz-db-backed pools and the relay audit -# pool, all in milliseconds; 0 disables. The separately deployed push gateway -# owns its own database and session policy and does not consume these knobs. +# pool, all in milliseconds; 0 disables the lock and idle timeouts. The +# separately deployed push gateway owns its own database and session policy +# and does not consume these knobs. # lock_timeout: fail a statement that waits this long on any lock instead of # parking behind a wedged holder (default 5000). # BUZZ_DB_LOCK_TIMEOUT_MS=5000 # idle_in_transaction_session_timeout: reap sessions idle inside an open # transaction — bounds how long a wedged client can hold locks (default 60000). # BUZZ_DB_IDLE_TXN_TIMEOUT_MS=60000 -# statement_timeout: cap any single statement's runtime. Off by default — -# startup migrations/backfills legitimately run long statements. Warning: a -# pathologically low value (e.g. 1) also times out connection setup and can -# prevent any DB connection from establishing. -# BUZZ_DB_STATEMENT_TIMEOUT_MS=0 +# statement_timeout: cap any single statement's runtime (default 120000). A +# session cannot disable it: 0 or anything above 300000 installs 300000 (5 +# minutes) and logs a warning at startup. Migrations, schema destruction, and +# community-deletion inventory and purge transactions lift it for themselves. +# Generic event reads (REQ history, /query, /count) and read-replica sessions +# are further capped at 20s. Warning: a pathologically low value (e.g. 1) also +# times out connection setup and can prevent any DB connection from +# establishing. +# BUZZ_DB_STATEMENT_TIMEOUT_MS=120000 # ----------------------------------------------------------------------------- # Typesense (search) @@ -150,6 +155,9 @@ BUZZ_PUSH_GATEWAY_DELIVERY_URL=https://push.buzz.xyz/v1/deliveries/apns # Deployment-global operator-listener routes. Each configured Nostr identity # may register target pubkeys and receives mention notifications at its URL. +# URLs may use HTTP or HTTPS. Use HTTP only for operator-controlled internal +# connections: NIP-98 signs requests but does not encrypt them, and +# notifications carry full message content. # Format: pubkey:deliveryUrl;pubkey:deliveryUrl # BUZZ_OPERATOR_LISTENER_TIMEOUT_MS=5000 # BUZZ_OPERATOR_LISTENERS=<64-char hex pubkey>:https://listener.example/mentions @@ -369,3 +377,18 @@ RUST_LOG=buzz_relay=debug,buzz_datastore=info,buzz_db=debug,buzz_auth=debug,buzz # BUZZ_TERMS_OF_SERVICE_MARKDOWN="# Terms of Service\n\nFull terms here." # BUZZ_PRIVACY_POLICY_MARKDOWN="# Privacy Policy\n\nFull policy here." # BUZZ_AGE_ATTESTATION_REQUIRED=true + +# ── NIP-FI database community source ───────────────────────────────────────── +# Where NIP-FI communities come from: "env" (BUZZ_NIP_FI_COMMUNITIES, default) +# or "database" (bindings created by provisioning; needs +# BUZZ_NIP_FI_PROVISION_BINDING and must not be combined with +# BUZZ_NIP_FI_COMMUNITIES). See docs/operator-nip-fi-bindings.md. +# BUZZ_NIP_FI_COMMUNITY_SOURCE=env +# Most community-issuer rows one snapshot may hold (1..=10000000). +# BUZZ_NIP_FI_SNAPSHOT_MAX_ROWS=1000000 +# Snapshot load timeout in ms; must be under the 30s refresh interval. +# BUZZ_NIP_FI_SNAPSHOT_REFRESH_TIMEOUT_MS=10000 +# Hosts whose "not found" is cached for 5s (1..=1000000). +# BUZZ_NIP_FI_RESOLVER_NEGATIVE_CACHE_MAX=100000 +# Concurrent live lookups; clamped below the database pool size. +# BUZZ_NIP_FI_RESOLVER_MAX_INFLIGHT=8 diff --git a/.github/workflows/_ci-clients.yml b/.github/workflows/_ci-clients.yml index 310ab676b7..8c9d080430 100644 --- a/.github/workflows/_ci-clients.yml +++ b/.github/workflows/_ci-clients.yml @@ -57,11 +57,30 @@ jobs: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} + # Mobile lanes run in parallel; the required check reads the combined + # matrix result. Test shards split files, so each compiles only its own. mobile: - name: Mobile + name: Mobile (${{ matrix.name }}) runs-on: ubuntu-latest timeout-minutes: 30 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.mobile) + strategy: + fail-fast: false + matrix: + include: + - name: Checks + lane: checks + - name: Tests 1/3 + lane: test + shard: 0 + - name: Tests 2/3 + lane: test + shard: 1 + - name: Tests 3/3 + lane: test + shard: 2 + - name: Android + lane: android permissions: contents: read steps: @@ -107,47 +126,127 @@ jobs: path: ~/.pub-cache key: pub-${{ runner.os }}-${{ hashFiles('mobile/pubspec.lock') }} - name: Format check + if: matrix.lane == 'checks' run: cd mobile && dart format --output=none --set-exit-if-changed . - name: Analyze + if: matrix.lane == 'checks' run: cd mobile && flutter analyze - name: Test - run: cd mobile && flutter test --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + if: matrix.lane == 'test' + run: /bin/bash scripts/mobile-test-shard.sh ${{ matrix.shard }} 3 --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + - name: Test configured authentication domains + if: matrix.lane == 'checks' + run: >- + cd mobile && flutter test + --dart-define=BUZZ_PUSH_GATEWAY_URL=https://push.example + --dart-define=BUZZ_AUTH_DOMAINS=buzz.xyz,builderlab.xyz,builderlabstaging.xyz + test/shared/auth/community_entry_requirements_test.dart + test/features/invites/invite_join_provider_test.dart + test/features/onboarding/onboarding_entry_routes_test.dart + test/shared/hosted/hosted_environment_test.dart + test/shared/hosted/self_hosted_entry_test.dart + test/features/onboarding/hosted_app_gate_environment_test.dart + test/features/onboarding/hosted_journeys_test.dart + test/features/onboarding/hosted_communities_test.dart + test/features/pairing/pairing_hosted_handoff_test.dart - name: Test mobile without push configuration + if: matrix.lane == 'checks' run: cd mobile && flutter test test/shared/push/push_unconfigured_build_test.dart - name: Test mobile recipe argument forwarding + if: matrix.lane == 'checks' run: /bin/bash scripts/test-mobile-gateway-recipes.sh + - name: Restore Gradle cache + if: matrix.lane == 'android' + uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('mobile/android/**/*.gradle*', 'mobile/android/gradle/wrapper/gradle-wrapper.properties', 'mobile/pubspec.lock') }} + restore-keys: gradle-${{ runner.os }}- + # SDK component and plugin downloads intermittently fail on hosted + # runners; retry once only for those, so real build failures still fail. - name: Build Android debug APK - run: just mobile-build-android + if: matrix.lane == 'android' + run: | + set -o pipefail + log="$RUNNER_TEMP/android-build.log" + if ! just mobile-build-android 2>&1 | tee "$log"; then + grep -qE 'InstallFailedException|Could not (resolve|GET|download)' "$log" + just mobile-build-android + fi + - name: Test Android JVM units + if: matrix.lane == 'android' + run: cd mobile/android && ./gradlew :app:testDebugUnitTest --console=plain + - name: Save Gradle cache + if: matrix.lane == 'android' && github.event_name == 'push' + uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 + continue-on-error: true + with: + path: | + ~/.gradle/caches + ~/.gradle/wrapper + key: gradle-${{ runner.os }}-${{ hashFiles('mobile/android/**/*.gradle*', 'mobile/android/gradle/wrapper/gradle-wrapper.properties', 'mobile/pubspec.lock') }} + # The unsigned release build needs nothing from the native lane, so it runs + # beside it instead of after the simulator tests. mobile-swift: - name: Mobile Swift + name: Mobile Swift (${{ matrix.name }}) runs-on: macos-latest - timeout-minutes: 30 + timeout-minutes: 45 if: inputs.lane == 'mobile-swift' && inputs.mobile + strategy: + fail-fast: false + matrix: + include: + - name: Native + lane: native + - name: Release + lane: release steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 - name: Install Flutter dependencies run: cd mobile && flutter pub get - - name: Build - run: swift build --package-path mobile/ios/BuzzPushKit - - name: Build release - run: swift build -c release --package-path mobile/ios/BuzzPushKit + # `swift test` builds BuzzPushKit; the Release lane covers its release build. - name: Test + if: matrix.lane == 'native' run: swift test --package-path mobile/ios/BuzzPushKit - name: Test native pagination geometry + if: matrix.lane == 'native' run: /bin/bash scripts/test-mobile-pagination.sh - - name: Build complete iOS simulator app - run: cd mobile && flutter build ios --simulator --debug --no-pub - - name: Test native pagination accessibility + - name: Generate iOS build configuration + if: matrix.lane == 'native' + run: cd mobile && flutter build ios --simulator --debug --no-pub --config-only + # Build the complete simulator app and RunnerTests once, in the + # background, while the simulator boots and runs the native-control + # scripts. The build also guards Apple Silicon simulator linking (#7646). + - name: Test native controls and Runner unit tests + if: matrix.lane == 'native' run: | - device=$(xcrun simctl create "Pagination accessibility" "iPhone 16") + device=$(xcrun simctl create "Native tests" "iPhone 16") trap 'xcrun simctl shutdown "$device"; xcrun simctl delete "$device"' EXIT + xcode=(xcodebuild -workspace mobile/ios/Runner.xcworkspace -scheme Runner + -configuration Debug -destination "platform=iOS Simulator,id=$device" + CODE_SIGNING_ALLOWED=NO) + build_log="$RUNNER_TEMP/runner-build.log" + "${xcode[@]}" build-for-testing > "$build_log" 2>&1 & + build=$! xcrun simctl boot "$device" xcrun simctl bootstatus "$device" -b /bin/bash scripts/test-mobile-pagination-accessibility.sh "$device" + /bin/bash scripts/test-mobile-carousel.sh "$device" /bin/bash scripts/test-mobile-member-picker.sh "$device" + /bin/bash scripts/test-mobile-me-agent-accessibility.sh "$device" + if ! wait "$build"; then + cat "$build_log" + exit 1 + fi + tail -n 5 "$build_log" + "${xcode[@]}" -parallel-testing-enabled NO -only-testing:RunnerTests \ + test-without-building - name: Build complete unsigned iOS release + if: matrix.lane == 'release' run: | /bin/bash scripts/test-mobile-gateway-recipes.sh cd mobile && flutter build ios --release --no-codesign --no-pub diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index 28fb80f9c5..4c5e19fe9c 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -110,6 +110,8 @@ jobs: -p buzz-test-client \ --lib \ --test e2e_event_reminder \ + --test e2e_author_only_deletion \ + --test e2e_existence_oracle \ --archive-file target/ci/backend-integration-tests.tar.zst - name: Build PostgreSQL test archive if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'postgres-tests' @@ -584,10 +586,12 @@ jobs: run: | cargo nextest run \ --archive-file target/ci/backend-integration-tests.tar.zst \ - -E 'binary(e2e_event_reminder)' \ + -E 'binary(e2e_event_reminder) | binary(e2e_author_only_deletion) | binary(e2e_existence_oracle)' \ --run-ignored ignored-only env: RELAY_URL: ws://localhost:3000 + # schema/schema.sql (applied above) indexes kind:5 content. + BUZZ_TEST_SEARCH_INDEXES_DELETIONS: "1" - name: Upload relay log if: failure() uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 @@ -631,6 +635,13 @@ jobs: cargo test -p buzz-test-client --test e2e_relay invite -- --ignored --nocapture cargo test -p buzz-test-client --test e2e_relay nip43_membership_snapshots_are_rejected -- --ignored --nocapture cargo test -p buzz-test-client --test e2e_relay nip29_departure_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_batch_put_user_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_announce_channel_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_unlisted_channel_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_solo_share_and_convert_wire -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_member_raises_guest_agent -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_guest_is_read_only_in_normal_channel -- --ignored --nocapture + cargo test -p buzz-test-client --test e2e_relay nip29_relay_rejects_role_change_by_unprivileged_actor -- --ignored --nocapture env: RELAY_URL: ws://localhost:3000 RELAY_HTTP_URL: http://localhost:3000 diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a1ff1c47fe..403c809604 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -17,7 +17,9 @@ jobs: changes: name: Detect Changed Paths runs-on: ubuntu-latest - timeout-minutes: 2 + # The steps take about 75 seconds, but GitHub can finish the job minutes + # after the last step. A 2-minute limit cancelled finished runs. + timeout-minutes: 10 permissions: contents: read pull-requests: read @@ -51,6 +53,8 @@ jobs: - '.config/nextest.toml' - 'scripts/postgres-test-*.sh' - 'scripts/reconcile-schema-after-pgschema.sql' + - 'scripts/repair-events-pkey.sql' + - 'scripts/start-relay-for-tests.sh' - 'bin/pgschema' - 'bin/.pgschema-*.pkg' - 'scripts/check-postgres-test-discovery.py' @@ -86,7 +90,10 @@ jobs: - 'pnpm-lock.yaml' mobile: - 'mobile/**' + - 'Justfile' + - 'scripts/test-mobile-gateway-recipes.sh' - 'scripts/mobile-release.sh' + - 'scripts/mobile-release-source.sh' - 'scripts/mobile-worktree-overrides.sh' - 'scripts/mobile-worktree-clean.sh' - 'scripts/publish-mobile-release-candidate.sh' diff --git a/.github/workflows/docker.yml b/.github/workflows/docker.yml index 3d824bfc56..233943268c 100644 --- a/.github/workflows/docker.yml +++ b/.github/workflows/docker.yml @@ -16,8 +16,8 @@ name: Docker image # the relay image version tracks crates/buzz-relay/Cargo.toml, never desktop. # # Triggers: -# - push to main → :main + :sha-<7> -# + :debug-main + :debug-sha-<7> +# - push to main → :main + :sha-<7> + :main--<7> +# + the matching :debug-* tags # - push tags relay-v*.*.* → :{version} + :{major}.{minor} + :{major} # + matching :debug-* tags # (+ :latest/:debug-latest for stable releases) @@ -34,6 +34,16 @@ name: Docker image # intact) and the bare version flows straight to the semver parser. On a # real push event value= is empty and the match strips relay-v from the ref. # +# ghcr.io/block/buzz-push-gateway carries the same :main, :sha-<7> and +# :main--<7> tags on main pushes, and publishes only after the +# relay's `qualify` job accepts the same source SHA. +# +# :main--<7> names the same main build as :sha-<7>, prefixed +# with its UTC commit time so the tag sorts by build order. Kargo's Lexical +# strategy picks the newest from the tag list alone; :sha-<7> cannot be +# ordered without fetching every image's metadata. It uses the commit date, +# not the run date, so a rerun of the same commit re-pushes the same tag. +# # The :latest tag tracks the latest STABLE relay release: metadata-action's # `flavor.latest=auto` (its default) emits :latest only for non-prerelease # semver, so relay-v0.3.0-rc.1 publishes :0.3.0-rc.1 without moving :latest, @@ -162,6 +172,7 @@ jobs: type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} labels: | org.opencontainers.image.title=Buzz org.opencontainers.image.description=WebSocket relay server for the Buzz communications platform @@ -371,6 +382,7 @@ jobs: type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Create and push manifest list id: manifest @@ -606,7 +618,9 @@ jobs: name: Publish public push gateway image if: github.event_name != 'pull_request' runs-on: ubuntu-24.04 - needs: push-gateway-build + # Staging promotes every published main gateway image automatically, so the + # gateway publishes only from a source the relay's same-SHA CI qualified. + needs: [push-gateway-build, qualify] timeout-minutes: 15 permissions: contents: read @@ -638,6 +652,7 @@ jobs: type=sha,prefix=sha-,format=short,enable=${{ github.event_name != 'workflow_dispatch' || inputs.version == '' }} type=semver,pattern={{version}},match=^relay-v(.*)$,value=${{ inputs.version }} type=semver,pattern={{major}}.{{minor}},match=^relay-v(.*)$,value=${{ inputs.version }} + type=raw,value=main-{{commit_date 'YYYYMMDDHHmmss'}}-{{sha}},enable=${{ github.event_name == 'push' && github.ref == 'refs/heads/main' }} - name: Merge and publish manifest id: manifest working-directory: /tmp/gateway-digests diff --git a/.github/workflows/mobile-release-candidate.yml b/.github/workflows/mobile-release-candidate.yml index e1f1c1c918..131e8aab0b 100644 --- a/.github/workflows/mobile-release-candidate.yml +++ b/.github/workflows/mobile-release-candidate.yml @@ -12,8 +12,13 @@ on: description: Expected next release-candidate number required: true type: string + source_branch: + description: Source branch (main or mobile-vMAJOR.MINOR matching version) + required: true + default: main + type: string target_sha: - description: Exact current block/buzz main commit + description: Pinned commit in the selected source branch history required: true type: string @@ -65,5 +70,6 @@ jobs: GH_TOKEN: ${{ steps.release-tagger.outputs.token }} MOBILE_VERSION: ${{ inputs.version }} CANDIDATE_NUMBER: ${{ inputs.candidate_number }} + SOURCE_BRANCH: ${{ inputs.source_branch }} TARGET_SHA: ${{ inputs.target_sha }} - run: scripts/publish-mobile-release-candidate.sh "$MOBILE_VERSION" "$CANDIDATE_NUMBER" "$TARGET_SHA" + run: scripts/publish-mobile-release-candidate.sh "$MOBILE_VERSION" "$CANDIDATE_NUMBER" "$TARGET_SHA" "$SOURCE_BRANCH" diff --git a/.intersect/sadscan.yaml b/.intersect/sadscan.yaml index 77710ae0d9..7f6c5e57a2 100644 --- a/.intersect/sadscan.yaml +++ b/.intersect/sadscan.yaml @@ -4,11 +4,15 @@ exclude_rules_for_files: - Cargo.lock # Self-signed test fixture generated solely to exercise reqwest identity parsing. kingfisher.privkey.2: + # Published upstream synthetic CA key, compiled only under cfg(test). + # The gateway pins Apple's root and cannot trust this fixture CA. + - "*appattest-test-ca-key.pem" - "*apns-test-identity.pem" - "*apns-test-key-only.pem" - "*apns-test-encrypted-identity.pem" - "*apns-test-mismatched-identity.pem" np.pem.1: + - "*appattest-test-ca-key.pem" - "*apns-test-identity.pem" - "*apns-test-key-only.pem" - "*apns-test-encrypted-identity.pem" diff --git a/.pgschemaignore b/.pgschemaignore new file mode 100644 index 0000000000..059350e198 --- /dev/null +++ b/.pgschemaignore @@ -0,0 +1,25 @@ +# pgschema reads this file from its working directory; every caller runs it +# from the repository root. +# +# The relay's partition manager creates dated `events` / `delivery_log` +# monthlies at runtime (see crates/buzz-db/src/store/partition/). They are not +# declared in schema/partitions/, so without this file re-applying pgschema to a +# database a relay has used drops them, and their rows, with +# `DROP TABLE IF EXISTS ... CASCADE`. Only the monthlies schema/partitions/ +# declares stay managed by pgschema; keep the `!` exemptions below in sync with +# them. +[tables] +patterns = [ + "events_p2*", + "!events_p2026_01", + "!events_p2026_02", + "!events_p2026_03", + "!events_p2026_04", + "!events_p2026_05", + "!events_p2026_06", + "delivery_log_p2*", + "!delivery_log_p2026_03", + "!delivery_log_p2026_04", + "!delivery_log_p2026_05", + "!delivery_log_p2026_06", +] diff --git a/AGENTS.md b/AGENTS.md index 5932bcd9e1..665ac30312 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -14,7 +14,12 @@ Before planning or reviewing a non-trivial change: 2. Read the `VISION_*.md` documents relevant to the affected product surface. 3. Read the applicable guidance in [TESTING.md](TESTING.md) and any package-local `TESTING.md`. -4. Check that the proposed design advances, or at least does not contradict, +4. For any change to `migrations/`, `schema/`, or schema-dependent `buzz-db` + queries, read [docs/database.md](docs/database.md): no new triggers, + functions, or foreign keys; no data backfills in migrations; update every + schema location; and keep every migration compatible with the relay that + is currently deployed. +5. Check that the proposed design advances, or at least does not contradict, that product intent. Call out any intentional tension explicitly. Implementation describes the product today; the vision documents describe the @@ -128,7 +133,8 @@ crates/ desktop/ # Tauri 2 + React 19 desktop app web/ # Browser web client (repo browser, served by the relay) mobile/ # Flutter mobile app -migrations/ # SQL migrations (auto-applied on relay startup) +migrations/ # SQL migrations (run at relay startup when BUZZ_AUTO_MIGRATE is on: binary default off, Helm default on; deployments at scale apply them out of band — see docs/database.md) +schema/ # Desired-state schema (schema/tables/public/ is the declarative schema source) scripts/ # Dev tooling .env.example # Config template — copy to .env before running ``` @@ -313,7 +319,9 @@ Filters and queries must scope to `h` tags when operating within a channel. This applies to events *inside* a channel. Addressable events that describe a channel carry its id in their `d` tag instead: kind:39000 (metadata), kind:39001, kind:39002 (membership). `get_channels` resolves a user's channels -from the `d` tag of their kind:39002 events, not from `h`. +from the `d` tag of their kind:39002 events, not from `h`. The relay leaves +system channels out of these reads unless the filter asks for them (see +`docs/nips/NIP-CL.md`). **Agent-facing operations go in `buzz-cli`**: New agent-facing features belong in `buzz-cli` — add a subcommand there first, then wire the REST/WebSocket call in `client.rs`. `buzz-dev-mcp` (shell + file tools for `buzz-agent`) is separate. @@ -363,8 +371,13 @@ canonical signed Nostr event fields (`id`, `pubkey`, `kind`, `content`, `{event_id, accepted, message}`; creates add the entity ID. Exit codes: 0=ok, 1=input error, 2=network/relay, 3=auth, 4=other, 5=write conflict (NIP-33 LWW). -`--format compact` is a **global** flag — it goes before the subcommand: +`--format` is a **global** flag — it goes before the subcommand: `buzz --format compact channels list`, NOT `buzz channels list --format compact`. +Without the flag it falls back to `BUZZ_OUTPUT_FORMAT` (unknown values mean +`json`); buzz-acp sets that to `agent` for agent +processes, so their event reads drop `sig`, trim the NIP-OA `auth` tag to +`["auth", ]`, and fold the NIP-10 `root`/`reply` tags into `reply_to` +(plus `thread_root` on nested replies); other tags remain. See `crates/buzz-cli/TESTING.md` for the full live-testing runbook. @@ -600,7 +613,7 @@ description. See [PR #803](https://github.com/block/buzz/pull/803). 4. **Worktrees: `cd` in the same command** — shell CWD doesn't persist between tool calls. Use `cd /path && cargo build` as one command. 5. **Desktop crate excluded from root workspace** — `cargo test` at repo root does NOT run desktop tests. Use `cargo test --manifest-path desktop/src-tauri/Cargo.toml` explicitly. 6. **React render perf: `React.memo` is all-or-nothing** — it only skips a re-render when *every* prop is reference-stable; one unstable prop (inline arrow/JSX, or a hook returning a fresh `{}`/`[]`/`Map` each render) defeats it. Two repeat offenders: (a) React Query results (`useMutation`/`useQuery`) are a **new object each render** — depend on the stable method (`mutation.mutateAsync`), not the object; (b) derived `Map`/array state that recomputes on a version bump — wrap in a content-equality ref cache (`shared/hooks/useStableReference.ts`). When chasing interaction lag, **measure with DevTools closed and no perf probes** (an open Web Inspector + per-keystroke `console.log` inflate the numbers), and isolate by removing one suspect at a time rather than guessing. -7. **`pgschema` omits seed DML and some storage parameters** — `schema/schema.sql` is an ordered `\i` manifest: `schema/tables/public/.sql` holds only that table's `CREATE TABLE` and `CREATE INDEX` statements (one table per file, the shape SchemaBot reads), and types, functions, partitions, triggers, and seed rows live in the sibling `schema/` directories. Fresh desired-state bootstraps use `./bin/pgschema apply`, which does not execute `INSERT` statements or preserve every table storage parameter from `schema/schema.sql`. Put each unsupported invariant in `scripts/reconcile-schema-after-pgschema.sql` as an idempotent convergence statement plus a live catalog or data assertion. Every `pgschema apply` caller must run that script. A string assertion against `schema.sql` alone does not prove the pgschema-created database has the intended state. +7. **`pgschema` omits seed DML and some storage parameters** — `schema/schema.sql` is an ordered `\i` manifest: `schema/tables/public/
.sql` holds only that table's `CREATE TABLE` and `CREATE INDEX` statements (one table per file, the shape out-of-band schema tools read), and types, functions, partitions, triggers, and seed rows live in the sibling `schema/` directories. Fresh desired-state bootstraps use `./bin/pgschema apply`, which does not execute `INSERT` statements or preserve every table storage parameter from `schema/schema.sql`. Put each unsupported invariant in `scripts/reconcile-schema-after-pgschema.sql` as an idempotent convergence statement plus a live catalog or data assertion. Every `pgschema apply` caller must run that script, and must run pgschema from the repository root so it reads `.pgschemaignore`; from any other directory a re-apply plans to drop the partitions the relay created at runtime. A string assertion against `schema.sql` alone does not prove the pgschema-created database has the intended state. --- @@ -763,12 +776,37 @@ usage. ### Testing Conventions -- Prefer **widget tests** over unit tests for UI components — test the - whole widget tree, not individual methods. -- Use `ProviderScope(overrides: [...])` to inject fake notifiers. -- Fake notifiers should extend the real notifier class and override `build()`. -- Use the `WidgetHelpers.testable()` wrapper for simple widget tests or - build a custom `ProviderScope` + `MaterialApp` when you need specific overrides. +Favor a few functional tests over many small ones. Every test is code we +maintain, so each one must protect a distinct user outcome or invariant. + +- **Start with journeys.** For each user-visible flow, write one widget test + that drives it through real widgets, notifiers and coordinators. Assert what + the user sees and what is durably saved, not internal stages or call counts. + Where Flutter can observe it, include semantics and non-primary input in the + journey: screen-reader labels and actions, keyboard, alternate gestures, text + scale and reduced motion. +- **Put failure paths inside the journey.** Retry, cancel, resume and a second + request arriving midway are steps of a journey, not separate tests. +- **Fake only external boundaries:** network, relay, OS, browser and storage. + Keep the flow's own notifiers and coordinators real. Override boundary + providers with `ProviderScope(overrides: [...])`, using + `WidgetHelpers.testable()` or a custom `ProviderScope` + `MaterialApp`. +- **Use focused tests only for what a journey can't enumerate or observe.** + That means security and data-loss invariants, protocol or crypto vectors, + pure functions with many input combinations, and accessibility or platform + behavior owned by native code that a widget journey can't observe, such as + VoiceOver stops and traits, Dynamic Type layout and reduced motion in native + views. Make them table-driven where cases repeat. +- **Extend before adding.** When a review finds a defect, add a step to an + existing journey or a row to an existing table. Don't add a new test per + finding. When a Review-Proven Rule says to test each path, cover the paths + as journey steps or table rows. +- **Delete what a journey covers.** Remove a lower-level test once a journey + demonstrably catches the same failures; passing through the same path is not + enough. Keep distinct safety, interop, accessibility and native-platform + invariants. +- **Prove it can fail.** Break the production code a test guards once and + confirm the test fails. A test that only exercises a fake protects nothing. --- diff --git a/ARCHITECTURE.md b/ARCHITECTURE.md index 531b1ceee7..7ff26b4ef7 100644 --- a/ARCHITECTURE.md +++ b/ARCHITECTURE.md @@ -145,7 +145,7 @@ buzz-admin (operator CLI: relay membership + key generation) buzz-test-client (integration test harness + manual CLI) ``` -**Key architectural principle:** The relay is the single source of truth. `buzz-relay` orchestrates all subsystems by calling them directly — it imports `buzz-db`, `buzz-auth`, `buzz-pubsub`, `buzz-search`, `buzz-audit`, and `buzz-workflow`. However, those subsystems are isolated from each other: `buzz-workflow` never calls `buzz-pubsub`, `buzz-search` never calls `buzz-db`, etc. Cross-subsystem coordination happens only through the relay. In multi-community mode, the relay also owns propagation of `TenantContext`; service crates should receive community-scoped inputs rather than independently deriving tenancy from client-controlled event tags. +**Key architectural principle:** The relay is the single source of truth. `buzz-relay` orchestrates all subsystems by calling them directly — it imports `buzz-db`, `buzz-auth`, `buzz-pubsub`, `buzz-search`, `buzz-audit`, and `buzz-workflow`. However, those subsystems are isolated from each other: `buzz-workflow` never calls `buzz-pubsub`, `buzz-search` never calls `buzz-db`, etc. The one sanctioned exception is `buzz-audit`, which calls `buzz_db::deletion::guard_community_write` to admit each audit append into its community before taking the chain lock. Cross-subsystem coordination happens only through the relay. In multi-community mode, the relay also owns propagation of `TenantContext`; service crates should receive community-scoped inputs rather than independently deriving tenancy from client-controlled event tags. --- @@ -189,6 +189,7 @@ The `kind` integer is the only dispatch switch. The relay routes, stores, and fa | 45003 | KIND_FORUM_COMMENT | Forum thread reply | | 46001–46012 | KIND_WORKFLOW_* | Workflow execution events | | 20001 | KIND_PRESENCE_UPDATE | Ephemeral presence heartbeat | +| 20003 | KIND_AGENT_PRESENCE | Ephemeral presence heartbeat an owner sends for its agents | `buzz-core` defines each event kind as a `pub const u32` and exports the full registry as `ALL_KINDS: &[u32]` (127 kinds at the time of writing); `crates/buzz-core/src/kind.rs` is the source of truth for the current list. Kinds are `u32` (NIP-01 specifies unsigned integer; `u32` covers the full range). Buzz uses both standard Nostr kinds (e.g., kind 7 for reactions) and custom ranges (40000+). @@ -305,6 +306,8 @@ Ephemeral events bypass DB storage, audit, and search. Two sub-paths: ``` Presence events skip membership checks and use local-only fan-out. Multi-node presence fan-out would require Redis pub/sub (documented as future work). +**Agent presence (kind 20003):** an owner's one heartbeat for all its agents, so an app that runs agents without a socket per agent can still show them online. Content is the status (`online`, `away` or `offline`) and each `p` tag names an agent, at most 256. WebSocket only. The relay keeps the agents the signer owns (`users.agent_owner_pubkey`, through the owner cache), skips the rest, and sets or clears their presence exactly as their own kind 20001 would, so the presence snapshot reports them unchanged. It is not fanned out, since live presence is matched by author; viewers see it on their next snapshot read. + **Other ephemeral events (e.g., typing indicators):** ``` 1. VERIFY — spawn_blocking(verify_event) @@ -471,7 +474,7 @@ All database access. Uses `sqlx::query()` (runtime, not compile-time macros) — | `user.rs` | User profile storage | | `error.rs` | Database error types | -**Channel types:** `Stream`, `Forum`, `Dm`, `Workflow` +**Channel types:** `Stream`, `Forum`, `Dm`, `Workflow`, `System` **Member roles:** `Owner`, `Admin`, `Member`, `Guest`, `Bot` **Workflow statuses:** `Active`, `Disabled`, `Archived` **Run statuses:** `Pending`, `Running`, `WaitingApproval`, `Completed`, `Failed`, `Cancelled` @@ -498,8 +501,9 @@ shape is an explicit transition, not a flag day: - **Community fence (transition foundation):** runtime admission APIs now offer one shared community deletion lock per transaction, while deletion lifecycle transitions take the matching exclusive lock. A supported serving transaction - never spans communities; unmigrated paths still rely on trigger/function - backstops. + never spans communities. Serving leases use the same application admission. + Startup and deletion catalog validation tolerate the universal fence objects + being absent; inventories still record live fences for older executors. - **Replica floor (transition foundation):** runtime now provides a shared replica-floor lock helper for compliant channel-event writers, and the writer probe handshake takes the exclusive counterpart before sampling `S`, scanning @@ -510,7 +514,8 @@ shape is an explicit transition, not a flag day: - **Dual enforcement (current):** application-owned lock+precheck paths run in front of the existing trigger/function enforcement; commit-time trigger checks remain authoritative during this phase. Removing those backstops is a separate - gated migration, not part of the lock foundation. + gated migration, after every pod tolerates their absence and no rollback below + that version is expected. - **Role separation:** relay/runtime code owns admission and lock protocols; operator maintenance/backfill workflows must either use those protocols or run under explicit reviewed procedures that keep routing fences closed. @@ -520,6 +525,11 @@ shape is an explicit transition, not a flag day: behavior, pgschema bootstraps run reconciliation, and lock/pool metrics remain the operational evidence path for rollout safety. +The database end state (a declarative schema source, applied out of band at +scale; no triggers, functions, or foreign keys; backfills outside migrations) +and the expand → deploy → backfill → contract rules every schema change follows +are in [docs/database.md](docs/database.md). + **Does NOT:** cache queries, implement connection pooling logic (delegated to sqlx), or make network calls outside Postgres. --- @@ -593,7 +603,7 @@ Tamper-evident append-only log with SHA-256 hash chaining. **Hash covers:** seq (big-endian bytes), timestamp (RFC3339), event_id, event_kind (big-endian), actor_pubkey, action string, channel_id (16 bytes or 16 zero bytes if None), canonical metadata JSON (BTreeMap for deterministic key ordering), prev_hash. -**Single-writer guarantee:** `pg_advisory_lock` before each transaction. Lock released in all branches including panic (`catch_unwind`). +**Single-writer guarantee:** each append runs in one transaction that first passes community write admission (the shared community deletion lock plus the active/undeleted check, via `buzz-db`'s `DeletionStore::guard_transaction`), then takes a per-community `pg_advisory_xact_lock`. Both locks release on commit or rollback, including a dropped future. **10 audit actions:** `EventCreated`, `EventDeleted`, `ChannelCreated`, `ChannelUpdated`, `ChannelDeleted`, `MemberAdded`, `MemberRemoved`, `AuthSuccess`, `AuthFailure`, `RateLimitExceeded`. @@ -870,6 +880,18 @@ Every security-sensitive operation uses an explicit, verified pattern. No implic Applied in: `buzz-auth` (JWKS boundary), `buzz-workflow` (CallWebhook action), desktop `link_preview` (SSRF check). +JWKS uses `public_https` by default and applies this public-address check to +every resolved answer before pinning the first permitted address. Explicit +`mesh` mode accepts HTTP only for a DNS hostname (not an IP literal) without +userinfo, query, fragment, or backslash. It checks every DNS answer against the +positive address allow-list in the +[NIP-FI JWKS snapshot contract](docs/nips/NIP-FI.md#jwks-snapshot), rejects +empty or mixed answers, and pins the first checked address. Mesh selection is +an operator assertion, not a transport property the Rust relay verifies. +Operators must enforce mutual TLS peer authentication and integrity +protection, authorize the JWKS endpoint, and restrict egress. Unauthenticated +HTTP permits key substitution and issuer impersonation. + ### Audit Integrity - Hash chain: each entry's SHA-256 covers all fields including `prev_hash` — tampering any entry breaks all subsequent hashes diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index dbe4ba5dd2..2beb40bd5a 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -83,7 +83,8 @@ We review as capacity allows — focused PRs that follow this guide move fastest | Rust | 1.88+ | Install via [rustup](https://rustup.rs/) | | Node.js | 24+ | Required for desktop app commands and `just ci` | | pnpm | 10+ | Required for desktop app commands and `just ci` | -| Flutter | 3.41+ | Required for mobile app — install via [flutter.dev](https://docs.flutter.dev/get-started/install) | +| Flutter | 3.47.7+ | Required for mobile app — install via [flutter.dev](https://docs.flutter.dev/get-started/install) | +| Xcode | 27.0 recommended; 26 supported | Required for iOS builds; see [iOS toolchain setup](mobile/README.md#ios-toolchain) | | Docker | 24+ | For Postgres, Redis, MinIO | | `just` | latest | Task runner — `cargo install just` | | `lefthook` | 2.1.3 (Hermit-pinned) | Auto-installed by `just hooks` — no manual install needed | diff --git a/Cargo.lock b/Cargo.lock index b4986b67b4..1cb97df5cd 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -158,24 +158,6 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" -[[package]] -name = "appattest" -version = "0.1.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d0a66767aaf7d483c556386fb68ca2fba9347684d8bb17a4bd8b755851870f7" -dependencies = [ - "arrayvec", - "aws-lc-rs", - "base64 0.22.1", - "byteorder", - "ciborium", - "minicbor", - "openssl", - "p256", - "rustls-pki-types", - "rustls-webpki", -] - [[package]] name = "apple-native-keyring-store" version = "1.0.2" @@ -507,7 +489,7 @@ dependencies = [ "serde", "serde_json", "url", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -1040,14 +1022,26 @@ dependencies = [ "webbrowser", ] +[[package]] +name = "buzz-audio-client" +version = "0.1.0" +dependencies = [ + "futures-util", + "neteq", + "opus", + "serde_json", + "tokio", + "tokio-tungstenite 0.29.0", +] + [[package]] name = "buzz-audit" version = "0.1.0" dependencies = [ "buzz-core", "buzz-datastore-tracing", + "buzz-db", "chrono", - "futures-util", "hex", "metrics", "serde", @@ -1220,6 +1214,7 @@ dependencies = [ "clap", "deadpool-redis", "hex", + "nostr 0.44.7", "redis", "serde", "serde_json", @@ -1391,21 +1386,25 @@ name = "buzz-push-gateway" version = "0.1.0" dependencies = [ "aes-gcm", - "appattest", "async-trait", + "aws-lc-rs", "axum", "base64 0.22.1", - "byteorder", "chrono", + "ciborium", "getrandom 0.4.3", "hex", "metrics", "metrics-exporter-prometheus", "minicbor", "nostr 0.44.7", + "openssl", + "p256", "proptest", "rand 0.10.1", "reqwest 0.13.4", + "rustls-pki-types", + "rustls-webpki", "serde", "serde_json", "sha2 0.11.0", @@ -1418,6 +1417,7 @@ dependencies = [ "tracing-subscriber", "url", "uuid", + "x509-parser", ] [[package]] @@ -1425,6 +1425,7 @@ name = "buzz-relay" version = "0.2.1" dependencies = [ "anyhow", + "arc-swap", "async-compression", "async-trait", "axum", @@ -1455,6 +1456,7 @@ dependencies = [ "http-body-util", "infer", "jsonwebtoken", + "lru 0.16.4", "metrics", "metrics-exporter-prometheus", "metrics-util", @@ -1488,6 +1490,7 @@ dependencies = [ "tracing", "tracing-opentelemetry", "tracing-subscriber", + "tungstenite 0.29.0", "url", "uuid", ] @@ -1502,6 +1505,8 @@ dependencies = [ "hex", "hmac 0.13.0", "iroh", + "metrics", + "metrics-util", "nostr 0.44.7", "postcard", "proptest", @@ -2036,12 +2041,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "const-oid" -version = "0.9.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" - [[package]] name = "const-oid" version = "0.10.2" @@ -2301,12 +2300,16 @@ checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-bigint" -version = "0.5.5" +version = "0.7.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0dc92fb57ca44df6db8059111ab3af99a63d5d0f8375d9972e319a379c6bab76" +checksum = "d811ea23fdfbcce2e2d5132344d471abdb8f901ac496a0893a3df92e70525e52" dependencies = [ - "generic-array", - "rand_core 0.6.4", + "cpubits", + "ctutils", + "getrandom 0.4.3", + "hybrid-array", + "num-traits", + "rand_core 0.10.1", "subtle", "zeroize", ] @@ -2393,11 +2396,12 @@ dependencies = [ [[package]] name = "ctutils" -version = "0.4.2" +version = "0.4.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d5515a3834141de9eafb9717ad39eea8247b5674e6066c404e8c4b365d2a29e" +checksum = "38d28fb6aa7dd680b3ffd1a3e64e115c177bdba63c0ae7f9cd48129ff05c4fa0" dependencies = [ "cmov", + "subtle", ] [[package]] @@ -2663,25 +2667,14 @@ version = "0.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5729f5117e208430e437df2f4843f5e5952997175992d1414f94c57d61e270b4" -[[package]] -name = "der" -version = "0.7.10" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" -dependencies = [ - "const-oid 0.9.6", - "pem-rfc7468 0.7.0", - "zeroize", -] - [[package]] name = "der" version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ - "const-oid 0.10.2", - "pem-rfc7468 1.0.0", + "const-oid", + "pem-rfc7468", "zeroize", ] @@ -2785,7 +2778,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ "block-buffer 0.10.4", - "const-oid 0.9.6", "crypto-common 0.1.7", "subtle", ] @@ -2797,7 +2789,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f1dd6dbb5841937940781866fa1281a1ff7bd3bf827091440879f9994983d5c2" dependencies = [ "block-buffer 0.12.0", - "const-oid 0.10.2", + "const-oid", "crypto-common 0.2.2", "ctutils", ] @@ -2895,16 +2887,17 @@ checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "ecdsa" -version = "0.16.9" +version = "0.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ee27f32b5c5292967d2d4a9d7f1e0b0aed2c15daded5a60300e4abb9d8020bca" +checksum = "c0681a4fc24c767085329728d8dfba959af91228aa4610cca4f8ce317ba46ae0" dependencies = [ - "der 0.7.10", - "digest 0.10.7", + "der", + "digest 0.11.3", "elliptic-curve", "rfc6979", - "signature 2.2.0", - "spki 0.7.3", + "signature 3.0.0", + "spki", + "zeroize", ] [[package]] @@ -2913,7 +2906,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "29fcf32e6c73d1079f83ab4d782de2d81620346a5f38c6237a86a22f8368980a" dependencies = [ - "pkcs8 0.11.0", + "pkcs8", "serdect", "signature 3.0.0", ] @@ -2945,19 +2938,20 @@ dependencies = [ [[package]] name = "elliptic-curve" -version = "0.13.8" +version = "0.14.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b5e6043086bf7973472e0c7dff2142ea0b680d30e18d9cc40f267efbf222bd47" +checksum = "900f37e7e18d74a2347389af6cd69e3a30d5cc9e697f40004865679aa75e9c7c" dependencies = [ - "base16ct 0.2.0", + "base16ct 1.0.0", "crypto-bigint", - "digest 0.10.7", + "crypto-common 0.2.2", + "digest 0.11.3", "ff", - "generic-array", "group", - "pem-rfc7468 0.7.0", - "pkcs8 0.10.2", - "rand_core 0.6.4", + "hybrid-array", + "pem-rfc7468", + "pkcs8", + "rand_core 0.10.1", "sec1", "subtle", "zeroize", @@ -3196,11 +3190,11 @@ dependencies = [ [[package]] name = "ff" -version = "0.13.1" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0b50bfb653653f9ca9095b427bed08ab8d75a137839d9ad64eb11810d5b6393" +checksum = "a1f686ab92a9fb0eaf188f6c6c87b89490baa6fdb0db4544ba4dc47f7942489f" dependencies = [ - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -3630,12 +3624,12 @@ dependencies = [ [[package]] name = "group" -version = "0.13.0" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0f9ef7462f7c099f518d754361858f86d8a07af53ba9af0fe635bbccb151a63" +checksum = "7fd1a1c7a5206c5b7a3f5a0d7ccd3ff85d0c8f5133d62a02680255b0004af5f4" dependencies = [ "ff", - "rand_core 0.6.4", + "rand_core 0.10.1", "subtle", ] @@ -4015,7 +4009,9 @@ version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "818356c5132c1fede50f837ca96afbe78ff42413047f4abb886217845e1b6c8c" dependencies = [ + "subtle", "typenum", + "zeroize", ] [[package]] @@ -4074,7 +4070,7 @@ dependencies = [ "tokio", "tokio-rustls", "tower-service", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -4611,7 +4607,7 @@ dependencies = [ "tokio-websockets", "tracing", "url", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", "ws_stream_wasm", ] @@ -5183,6 +5179,9 @@ name = "lru" version = "0.16.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7f66e8d5d03f609abc3a39e6f08e4164ebf1447a732906d39eb9b99b7919ef39" +dependencies = [ + "hashbrown 0.16.1", +] [[package]] name = "lru" @@ -5951,6 +5950,21 @@ name = "minicbor" version = "0.25.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0452a60c1863c1f50b5f77cd295e8d2786849f35883f0b9e18e7e6e1b5691b0" +dependencies = [ + "half", + "minicbor-derive", +] + +[[package]] +name = "minicbor-derive" +version = "0.15.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bd2209fff77f705b00c737016a48e73733d7fbccb8b007194db148f03561fb70" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.117", +] [[package]] name = "minidom" @@ -6213,9 +6227,9 @@ checksum = "27b02d87554356db9e9a873add8782d4ea6e3e58ea071a9adb9a2e8ddb884a8b" [[package]] name = "negentropy" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f0efe882e02d206d8d279c20eb40e03baf7cb5136a1476dc084a324fbc3ec42d" +checksum = "a3d66662cb5ca384d4991509755ed5a42dd8e31c4282bf881938e506fc3146ec" [[package]] name = "netdev" @@ -6244,6 +6258,19 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "neteq" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57fbe9601485d95750187c92344fa056e75055027b00e375c53d9f4b8b883112" +dependencies = [ + "log", + "ringbuf", + "serde", + "thiserror 1.0.69", + "web-time", +] + [[package]] name = "netlink-packet-core" version = "0.8.1" @@ -6902,6 +6929,15 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7c87def4c32ab89d880effc9e097653c8da5d6ef28e6b539d313baaacfbafcbe" +[[package]] +name = "openssl-src" +version = "300.6.1+3.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46eb8fb9fb3b61ce1c0f8a026c4c1a0714d3a9e138e7fbde78753ce2babc3846" +dependencies = [ + "cc", +] + [[package]] name = "openssl-sys" version = "0.9.116" @@ -6910,6 +6946,7 @@ checksum = "f28a22dc7140cda5f096e5e7724a6962ca81a7f8bfd2979f9b18c11af56318c4" dependencies = [ "cc", "libc", + "openssl-src", "pkg-config", "vcpkg", ] @@ -7000,6 +7037,24 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "opus" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "33718946cc77d4032911d4efe03a66dbcbfbd2bb16c3da06aaeadcc637c32216" +dependencies = [ + "opusic-sys", +] + +[[package]] +name = "opusic-sys" +version = "0.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c9d1ecdf206421bc74343ab3bb2f30ad2abbfee41fa341f7181fecbaf957769a" +dependencies = [ + "cmake", +] + [[package]] name = "ordered-float" version = "2.10.1" @@ -7076,14 +7131,15 @@ dependencies = [ [[package]] name = "p256" -version = "0.13.2" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c9863ad85fa8f4460f9c48cb909d38a0d689dba1f6f6988a5e3e0d31071bcd4b" +checksum = "d2c9239b2dbc807adbbe147e8cf72ea7450c3a0aabe62cb8e75ff4ec22e1f72a" dependencies = [ "ecdsa", "elliptic-curve", + "primefield", "primeorder", - "sha2 0.10.9", + "sha2 0.11.0", ] [[package]] @@ -7208,15 +7264,6 @@ dependencies = [ "serde_core", ] -[[package]] -name = "pem-rfc7468" -version = "0.7.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" -dependencies = [ - "base64ct", -] - [[package]] name = "pem-rfc7468" version = "1.0.0" @@ -7405,24 +7452,14 @@ dependencies = [ "futures-io", ] -[[package]] -name = "pkcs8" -version = "0.10.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" -dependencies = [ - "der 0.7.10", - "spki 0.7.3", -] - [[package]] name = "pkcs8" version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.2", - "spki 0.8.0", + "der", + "spki", ] [[package]] @@ -7621,12 +7658,30 @@ dependencies = [ ] [[package]] -name = "primeorder" -version = "0.13.6" +name = "primefield" +version = "0.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "353e1ca18966c16d9deb1c69278edbc5f194139612772bd9537af60ac231e1e6" +checksum = "c555a6e4eb7d4e158fcb028c835c3b8642206ddc279b5c6b202ef9a8bdb592f4" +dependencies = [ + "crypto-bigint", + "crypto-common 0.2.2", + "ff", + "rand_core 0.10.1", + "subtle", + "zeroize", +] + +[[package]] +name = "primeorder" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c9f42978c78a00e3d68f69fc03e57a234debae69da4020a4fb588fcdcd07b06" dependencies = [ "elliptic-curve", + "once_cell", + "primefield", + "serdect", + "wnaf", ] [[package]] @@ -8484,7 +8539,7 @@ dependencies = [ "wasm-bindgen-futures", "wasm-streams 0.4.2", "web-sys", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -8555,12 +8610,12 @@ checksum = "1e061d1b48cb8d38042de4ae0a7a6401009d6143dc80d2e2d6f31f0bdd6470c7" [[package]] name = "rfc6979" -version = "0.4.0" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f8dd2a808d456c4a54e300a23e9f5a67e122c3024119acbfd73e3bf664491cb2" +checksum = "b4a459cddafb3fe76b31fd8f1108007566c40301feb64dc7b54656eb7388172b" dependencies = [ - "hmac 0.12.1", - "subtle", + "crypto-bigint", + "hmac 0.13.0", ] [[package]] @@ -8577,6 +8632,15 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ringbuf" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79abed428d1fd2a128201cec72c5f6938e2da607c6f3745f769fabea399d950a" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "rmcp" version = "1.8.0" @@ -8762,9 +8826,9 @@ dependencies = [ [[package]] name = "rustls" -version = "0.23.45" +version = "0.23.46" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d41d731c7d2f962d1ccc364cec258de3c0e93b38c2fb3ba97ac74513048d634" +checksum = "48e13bd8c0e9365c43cfa5c9e8f9ad49d3c8444926c9aac819e0e4dc503c8fdf" dependencies = [ "aws-lc-rs", "log", @@ -8827,9 +8891,9 @@ checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" [[package]] name = "rustls-webpki" -version = "0.103.15" +version = "0.103.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c3cf1d8b1e7d4927e2d154c3fcb02979afb9939629c62cd9048d4f07b60ac2" +checksum = "4443b1b6d39d7c663139942cc4558262562d2c84123bb6282ef177009ca8b281" dependencies = [ "aws-lc-rs", "ring", @@ -8990,14 +9054,14 @@ dependencies = [ [[package]] name = "sec1" -version = "0.7.3" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d3e97a565f76233a6003f9f5c54be1d9c5bdfa3eccfb189469f11ec4901c47dc" +checksum = "d56d437c2f19203ce5f7122e507831de96f3d2d4d3be5af44a0b0a09d8a80e4d" dependencies = [ - "base16ct 0.2.0", - "der 0.7.10", - "generic-array", - "pkcs8 0.10.2", + "base16ct 1.0.0", + "ctutils", + "der", + "hybrid-array", "subtle", "zeroize", ] @@ -9446,7 +9510,6 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest 0.10.7", "rand_core 0.6.4", ] @@ -9456,6 +9519,7 @@ version = "3.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d567dcbaf0049cb8ac2608a76cd95ff9e4412e1899d389ee400918ca7537f5" dependencies = [ + "digest 0.11.3", "rand_core 0.10.1", ] @@ -9737,16 +9801,6 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "023a211cb3138dbc438680b32560ad89f699977624c9f8dbb95a47d5b4c07dd3" -[[package]] -name = "spki" -version = "0.7.3" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" -dependencies = [ - "base64ct", - "der 0.7.10", -] - [[package]] name = "spki" version = "0.8.0" @@ -9754,7 +9808,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.2", + "der", ] [[package]] @@ -9826,7 +9880,7 @@ dependencies = [ "tracing", "url", "uuid", - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] @@ -10708,6 +10762,7 @@ dependencies = [ "futures-util", "log", "rustls", + "rustls-native-certs", "rustls-pki-types", "tokio", "tokio-rustls", @@ -11557,9 +11612,9 @@ dependencies = [ [[package]] name = "webpki-root-certs" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f31141ce3fc3e300ae89b78c0dd67f9708061d1d2eda54b8209346fd6be9a92c" +checksum = "58ec80e73fd8baf57bf0d9615554c75348bda990d6d5bbeab964dd068319fecb" dependencies = [ "rustls-pki-types", ] @@ -11570,14 +11625,14 @@ version = "0.26.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "521bc38abb08001b01866da9f51eb7c5d647a19260e00054a8c7fd5f9e57f7a9" dependencies = [ - "webpki-roots 1.0.7", + "webpki-roots 1.0.10", ] [[package]] name = "webpki-roots" -version = "1.0.7" +version = "1.0.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "52f5ee44c96cf55f1b349600768e3ece3a8f26010c05265ab73f945bb1a2eb9d" +checksum = "1c1c8e20c8a38fc7d65abfe74eb7280e4aa667e8eec8bc84996d2874810748d9" dependencies = [ "rustls-pki-types", ] @@ -12194,6 +12249,18 @@ dependencies = [ "windows-core 0.62.2", ] +[[package]] +name = "wnaf" +version = "0.14.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "795ca18b3fdb5e62bf982199278341ddcf7ebf7d32e25e212ad05d496e95f6fa" +dependencies = [ + "ff", + "group", + "hybrid-array", + "primefield", +] + [[package]] name = "writeable" version = "0.6.3" diff --git a/Cargo.toml b/Cargo.toml index d27c63d234..6207d5a8c0 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -31,6 +31,7 @@ members = [ "crates/buzz-mesh-smoke", "crates/buzz-dev-mcp", "crates/buzz-voice", + "crates/buzz-audio-client", "crates/buzz-backend-kubernetes", "crates/buzz-datastore-tracing", "examples/countdown-bot", @@ -123,6 +124,8 @@ zeroize = "1.8" # Concurrent data structures dashmap = "6" moka = { version = "0.12", features = ["sync"] } +arc-swap = "1" +lru = "0.16" # Async stream utilities futures-util = "0.3" diff --git a/Dockerfile b/Dockerfile index 2c1e8adbb5..1f10b0f260 100644 --- a/Dockerfile +++ b/Dockerfile @@ -54,6 +54,7 @@ FROM chef AS builder RUN apt-get update \ && apt-get install -y --no-install-recommends \ build-essential \ + cmake \ pkg-config \ libssl-dev \ ca-certificates \ diff --git a/Dockerfile.push-gateway b/Dockerfile.push-gateway index 5b0aca8bd3..2b899c89e5 100644 --- a/Dockerfile.push-gateway +++ b/Dockerfile.push-gateway @@ -13,7 +13,7 @@ RUN cargo chef prepare --recipe-path recipe.json FROM chef AS builder ARG BUZZ_PUSH_CARGO_FEATURES="" RUN apt-get update \ - && apt-get install -y --no-install-recommends build-essential pkg-config libssl-dev ca-certificates \ + && apt-get install -y --no-install-recommends build-essential cmake pkg-config libssl-dev ca-certificates \ && rm -rf /var/lib/apt/lists/* COPY --from=planner /build/recipe.json recipe.json RUN cargo chef cook --release --recipe-path recipe.json --features "$BUZZ_PUSH_CARGO_FEATURES" diff --git a/Justfile b/Justfile index 2ff716a36b..bcbe142304 100644 --- a/Justfile +++ b/Justfile @@ -398,8 +398,14 @@ test-unit: set -euo pipefail ./scripts/test-ensure-local-relay-key.sh python3 scripts/check-community-ban-route-inventory.py + # Shared Huddle transport, codec, and loopback authentication regressions. + cargo test -p buzz-audio-client if command -v cargo-nextest &>/dev/null; then cargo nextest run -p buzz-core -p buzz-auth --lib + # Mesh JWKS proxy-bypass regression: infra-free loopback I/O with + # child-process environment isolation. Its integration-test binary + # is not selected by --lib above. Keep the Cargo fallback in step. + cargo nextest run -p buzz-auth --test mesh_jwks_proxy cargo nextest run -p buzz-audit --lib # Cross-pod disconnect payload tests (NIP-FI and community archive # fence); all infra-free. @@ -412,6 +418,10 @@ test-unit: # already runs it. cargo test -p buzz-auth --doc cargo nextest run -p buzz-voice --lib + # Inter-relay mesh: membership, gossip, registry, and loopback-UDP + # runtime tests. Infra-free (Redis-backed paths are unit-faked or + # never dialed). Keep the Cargo fallback in step. + cargo nextest run -p buzz-relay-mesh cargo nextest run -p buzz-cli # buzz-sdk builder/validation unit tests: pure event-builder and input # validation (e.g. the canvas writer-discipline/skew guard and the @@ -447,10 +457,11 @@ test-unit: # suites are otherwise absent from the infra-free unit lane. cargo nextest run -p buzz-media --lib \ -E 'test(=bucket_index::tests::bucket_snapshot_json_round_trip_preserves_community_keys)' - # buzz-admin: storage-snapshot worker plus the community archive - # command parser, validation, and propagation-evidence tests. + # buzz-admin: storage-snapshot worker, the community archive command + # parser, validation, and propagation-evidence tests, and the partition + # audit outcome contract, NIP-FI policy/backfill, and user-deletion intake. cargo nextest run -p buzz-admin \ - -E 'test(storage_snapshot) + test(/^communities::tests::/) + test(/^tests::communities_/)' + -E 'test(storage_snapshot) + test(/^communities::tests::/) + test(/^tests::communities_/) + test(=tests::partition_audit_outcomes_keep_their_json_names_and_exit_codes) + test(/^nip_fi::tests::/) + test(/^user_deletions::tests::/)' # Multi-tenant conformance gate (buzz-conformance): the independent # replay checker + golden fixtures. No infra — pure in-process trace # replay — so it belongs in the unit job. Run all targets (lib + the @@ -516,20 +527,24 @@ test-unit: # unit job either. The REQ subscription-lifecycle tests are picked by # exact name for the same reason: the rest of handlers::req needs a DB. # The third clause adds the NIP-FI HTTP ingress and its router/config - # neighbours: nip_fi_http, nip_fi_config, router, api::parse_query_tests, - # and the Git transport off_mode_precedence_tests. All are infra-free + # neighbours: nip_fi_http, nip_fi_config, nip_fi_resolver (snapshot + # resolver and its WS/HTTP/audio router tests), router, api::parse_query_tests, + # the profile search request bounds (api::profiles), and the Git + # transport off_mode_precedence_tests. All are infra-free # and finish in well under a second with no DATABASE_URL, so none wait # out the sqlx acquire timeout. `--bin buzz-relay` adds main.rs's # `tests::` module (JWKS refresh cadence) and `composition_tests::` # (JWKS source + refresh-loop composition), which `--lib` cannot reach # because they live in the binary target; the nested # `tests::postgres_tests::` stays in the PostgreSQL lane. - # The fourth clause adds the startup step timers (startup_steps) and the - # env- and log-capture tests in telemetry and config. All three modules - # are infra-free; config parses env into a struct and never connects. + # The fourth clause adds the startup step timers (startup_steps), the + # mesh boot wiring (mesh_boot; Redis and Postgres point at refused + # ports), and the env- and log-capture tests in telemetry and config. + # All four modules are infra-free; config parses env into a struct and + # never connects. # `^config::` is anchored so it does not also select nip_fi_config. cargo nextest run -p buzz-relay --lib --bin buzz-relay \ - -E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' + -E 'test(/^api::admin::/) + test(/^push_runtime::/) + test(/^operator_listener::tests::/) + test(/^handlers::channel_authz::/) + test(/^handlers::moderation_authz::/) + test(/^handlers::side_effects::tests::/) + test(/^handlers::system_channels::tests::/) + test(/^nip11::tests::/) + test(/^storage_sweep::tests::/) + test(/^nip_fi_core::tests::/) + test(/^nip_fi_http::tests::/) + test(/^nip_fi_config::tests::/) + test(/^nip_fi_resolver::tests::/) + test(/^readiness::tests::/) + test(/^router::tests::/) + test(/^api::parse_query_tests::/) + test(/^api::profiles::/) + test(/^api::git::transport::off_mode_precedence_tests::/) + test(/^audio::join::tests::/) + test(/^audio::handler::tests::/) + test(/^nip_fi_gate::tests::/) + test(/^nip_fi_session::tests::/) + test(/^nip_fi_shadow::tests::/) + test(/^nip_fi_shadow_session::tests::/) + test(/^startup_steps::tests::/) + test(/^mesh_boot::tests::/) + test(/^telemetry::tests::/) + test(/^config::tests::/) + test(=state::tests::neither_a_confirmed_inactive_community_nor_a_failed_lookup_admits_the_socket) + test(=state::tests::periodic_revalidation_disconnects_inside_the_fenced_callback) + test(=state::tests::bare_community_disconnect_fails_closed_when_the_fence_is_unavailable) + test(=connection::tests::send_loop_sends_policy_close_when_community_is_archived) + test(=state::tests::community_disconnect_then_nip_fi_keeps_community_deleted_reason) + test(=state::tests::disconnect_community_wins_reason_losing_nip_fi_does_not_enqueue_frame) + test(=handlers::req::tests::timed_out_historical_read_deregisters_before_closed) + test(=handlers::req::tests::superseded_timeout_leaves_replacement_intact) + test(=handlers::req::tests::search_claim_retires_live_and_yields_to_replacement) + test(=handlers::req::tests::concurrent_claims_and_stale_teardowns_keep_the_last_owner) + test(=handlers::req::tests::timeout_closed_is_emitted_before_a_replacement_can_claim) + test(=handlers::req::tests::revoke_then_replacement_keeps_replacement_whole) + test(=handlers::req::tests::claims_after_connection_cleanup_are_refused) + test(=handlers::req::tests::dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::revoke_dropped_terminal_frame_cancels_connection) + test(=handlers::req::tests::count_fallback_covers_filters_that_can_match_private_deletions) + test(=handlers::req::tests::group_state_t_and_creator_tags_reach_sql_only_for_group_state_kinds) + test(=api::bridge::postgres_tests::bridge_search_mode_extension_accepts_prefix_snake_or_camel_case) + test(=api::bridge::postgres_tests::bridge_search_sort_extension_selects_newest_only_when_named) + (kind(bin) & (test(/^tests::/) + test(/^composition_tests::/) + test(/^env_filter_tests::/)) - test(/^tests::postgres_tests::/))' # Note on audio::join::tests scope: the full suite is infra-free (no DB, # no Redis). The infra-free audio/FI regression witnesses — bootstrap # ordering barrier, CommitConfirmed arm, pending-close invisibility, @@ -538,7 +553,9 @@ test-unit: # DB-backed audio join tests use #[ignore] and run in the postgres lane. # NIP-FI (S3/S4) relay witnesses: the wholly-new nip_fi_upgrade and # api::nip_fi modules, - # the auth metrics contract module, plus the exact NIP-FI tests added, + # the auth metrics contract module, the DB-free partition alert series + # tests (their Postgres case is #[ignore]d and runs in the PostgreSQL + # lane), plus the exact NIP-FI tests added, # or whose assertions changed, in mixed modules (audio::room, # connection, handlers::*, state). nip_fi_config and router are # selected whole by the command above. Mixed modules are listed by exact @@ -549,6 +566,7 @@ test-unit: cargo nextest run -p buzz-relay --lib -E ' test(/^nip_fi_upgrade::/) + test(/^metrics::contract_tests::/) + + test(/^metrics::partition_alert_tests::/) + test(=audio::room::tests::roster_revisions_are_ordered_and_snapshot_is_authoritative) + test(=connection::tests::auth_lifecycle_reconciles_every_terminal_and_never_leaks_gauge) + test(=audio::room::tests::b1_pending_peer_removed_before_commit_emits_no_delta) @@ -566,12 +584,26 @@ test-unit: + test(=connection::tests::expiry_notice_queued_on_ctrl_before_cancel) + test(=connection::tests::f3_root_outer_wrapper_delivers_denial_on_bootstrap_cancellation) + test(=connection::tests::f3_root_pre_built_expired_gate_terminates_connection) + + test(=connection::terminal_close_delivery_tests::key_pairing_mismatch_closes_with_1008_on_real_socket) + + test(=connection::terminal_close_delivery_tests::session_expiry_denial_closes_with_1008_on_real_socket) + + test(=connection::terminal_close_delivery_tests::server_holds_socket_for_client_close_reply) + + test(=connection::terminal_close_delivery_tests::server_finishes_promptly_when_client_answers_close) + + test(=connection::terminal_close_delivery_tests::stalled_cleanup_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::bootstrap_drain_holds_socket_for_client_close_reply) + + test(=connection::terminal_close_delivery_tests::stalled_expiry_barrier_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::blocked_frame_handler_does_not_hold_socket_open) + + test(=connection::terminal_close_delivery_tests::refused_bootstrap_holds_connection_slot_until_socket_ends) + + test(=connection::terminal_close_delivery_tests::stalled_bootstrap_holds_connection_slot) + + test(=connection::terminal_close_delivery_tests::peer_wait_replies_to_client_initiated_close) + test(=handlers::auth::tests::b2_pre_cancelled_connection_never_becomes_authenticated) + test(=handlers::auth::tests::fi_ban_check_error_emits_terminal_authorization_unavailable) + test(=handlers::auth::tests::fi_root_authorization_denied_rows_emit_identical_frames) + test(=handlers::auth::tests::fi_invalid_nip42_proof_emits_terminal_evidence_rejected) + test(=handlers::auth::tests::handle_auth_pairing_mismatch_runs_full_root_denial_path) + test(=handlers::auth::tests::shadow_root_auth_records_pairing_denial) + + test(=handlers::auth::tests::shadow_root_auth_records_unavailable_owner_read) + + test(=handlers::ingest::postgres_tests::relay_admin_ban_maps_to_blocked_auth_failure) + + test(=handlers::ingest::postgres_tests::relay_admin_internal_maps_to_error_not_client_fault) + test(=handlers::auth::tests::shadow_root_invalid_nip42_retires_without_record) + test(=handlers::auth::tests::nip42_denial_class_separates_internal_failure_from_bad_evidence) + test(=handlers::event::tests::fanout_access::owner_only_kinds_keep_only_the_owner) @@ -584,6 +616,7 @@ test-unit: + test(=state::tests::conn_manager_disconnect_nip_fi_is_issuer_scoped) + test(=state::tests::conn_manager_disconnect_nip_fi_sets_authorization_denied_reason) + test(=state::tests::nip_fi_disconnect_audio_is_issuer_scoped) + + test(=state::tests::nip_fi_owner_disconnect_closes_owned_agent_sockets_on_root_and_audio) + test(=state::tests::nip_fi_disconnect_closes_proven_audio_socket_and_sends_policy_close_reason) + test(=state::tests::nip_fi_disconnect_closes_target_audio_only_and_preserves_collocated_peer) + test(=state::tests::nip_fi_disconnect_does_not_close_different_pubkey_audio_socket) @@ -1018,10 +1051,14 @@ mobile-build-android: mobile-dev: #!/usr/bin/env bash set -euo pipefail - if ! pgrep -x Simulator &>/dev/null; then - open -a Simulator - sleep 3 + developer_dir="$(xcode-select -p)" + simulator_app="$developer_dir/../Applications/DeviceHub.app" + if [[ ! -d "$simulator_app" ]]; then + simulator_app="$developer_dir/Applications/Simulator.app" fi + # Use the selected Xcode even when another version's app is already running. + open -a "$simulator_app" + sleep 3 ./scripts/mobile-worktree-overrides.sh gateway_url="${BUZZ_PUSH_GATEWAY_URL:-}" overrides_file="{{mobile_dir}}/ios/Flutter/AppOverrides.xcconfig" diff --git a/NOSTR.md b/NOSTR.md index cce70f2f77..507923d70c 100644 --- a/NOSTR.md +++ b/NOSTR.md @@ -50,27 +50,29 @@ PGPASSWORD=buzz_dev psql -h localhost -U buzz -d buzz -c \ | **Reactions (kind:7)** | ✅ | Standard NIP-25; channel derived from target event's `#e` tag (client `#h` ignored) | | **Deletions (kind:5)** | ✅ | Standard NIP-09; self-authored only. `#h` optional, `#e` required | | **User profiles (kind:0)** | ✅ | NIP-01 metadata; synced to users table (display_name, avatar, about, NIP-05). NIP-05 handles must canonicalize to this relay's domain — off-domain or invalid handles are silently cleared. If a NIP-05 handle collides with another user's (UNIQUE constraint), the handle is skipped but other profile fields (display_name, avatar, about) are still synced. | -| **Group creation (kind:9007)** | ✅ | NIP-29; include `name` tag, optional `visibility` and `channel_type` | -| **Add user (kind:9000)** | ✅ | Open: any user, subject to target's `channel_add_policy` (`owner_only`/`nobody` can block). Private: owner/admin only. Self-add bypasses agent policy but not private-channel auth. | +| **Group creation (kind:9007)** | ✅ | NIP-29; include `name` tag, optional `visibility`, `channel_type`, label `t` and `unlisted` tags | +| **Add user (kind:9000)** | ✅ | Open: any user, subject to target's `channel_add_policy` (`owner_only`/`nobody` can block). Private: owner/admin only. Self-add bypasses agent policy but not private-channel auth. A guest cannot add anyone. A new member with no role tag gets the channel's `join_role`; a non-member adding themselves may ask for `join_role` or a lower role. One event may name up to 100 people: one `p` tag each, with an optional role as the `p` tag's third element (NIP-29), falling back to the event's `role` tag. Each person is checked and added on their own. When the event names several people, the OK message is `members: ` followed by JSON listing the non-empty `added`, `updated` (role changed), `already_member` and `rejected` (`[{"reason", "pubkeys"}]`) results. The event is refused only when it is malformed or nobody it names may be added; then one person keeps the single-add error, and several get `invalid: members: {…}`. A resent event returns `duplicate:` without results; to retry an `error:` rejection, send a new event. Re-adding someone at their current role posts no second `member_joined` notice. | | **Remove user (kind:9001)** | ✅ | Self-remove allowed (with last-owner guard). Removing others: owner/admin only. | -| **Edit group metadata (kind:9002)** | ✅ | `name`/`about` tags: owner/admin. `topic`/`purpose` tags: any member. | +| **Edit group metadata (kind:9002)** | ✅ | `name`/`about`/label `t`/`visibility`/`ttl`/`archived`/`join_role`/`unlisted` tags: owner/admin. `topic`/`purpose` tags: any member except a guest. Buzz applies 9002 as a partial update: an absent tag leaves that setting unchanged. | | **Admin delete event (kind:9005)** | ✅ | Event author can always delete own. Otherwise owner/admin required. Target must be in same channel. | | **Group deletion (kind:9008)** | ✅ | Owner only. | | **Leave group (kind:9022)** | ✅ | Any member. Last-owner guard prevents orphaned groups. | -| **Group metadata (kind:39000)** | ✅ | Relay-signed; always `d`, `name`, `closed` tags; `about` only if description non-empty; `private` if applicable; `hidden` for DM channels | -| **Group admins (kind:39001)** | ✅ | Relay-signed; `d` tag + `p` tags with roles (`owner`, `admin`) | -| **Group members (kind:39002)** | ✅ | Relay-signed; `d` tag + `p` tags for all members | +| **Group metadata (kind:39000)** | ✅ | Relay-signed; always `d`, `name`, `t`, `P`, `created_at`, `join_role`; `public` or `private`; `restricted` and `closed` per the access flags below; `unlisted` when the channel is unlisted; `about` only if description non-empty; `hidden` for DM channels and private system channels | +| **Unlisted channels** | ✅ | kind:9002 `["unlisted", "true"\|"false"]`, owner/admin, or at creation on kind:9007. An unlisted open channel is left out of every read that does not name it, for non-members: channel lists, kind:39000 without `#d`, messages, COUNT, NIP-50 search and artifact queries. Naming it by `#h` (or `#d` on 39000–39003) reads it, and kind:9021 joins it, as for any open channel. See [Unlisted channels](#unlisted-channels). | +| **Group admins (kind:39001)** | ✅ | Relay-signed; `d`, channel type `t` and creator `P` tags + `p` tags with roles (`owner`, `admin`) | +| **Group members (kind:39002)** | ✅ | Relay-signed; `d`, channel type `t` and creator `P` tags + `p` tags for all members | | **Membership notifications** | ✅ | kind:44100 (added) / kind:44101 (removed); relay-signed, community-global scope (`channel_id=None` inside the connected community) | | **Presence (kind:20001)** | ✅ | Ephemeral; arbitrary status string (truncated to 128 chars); writes to Redis (`set_presence`/`clear_presence` on `"offline"`), then fan-out to local subscribers. In multi-community mode presence is scoped to the connected community. | +| **Agent presence (kind:20003)** | ✅ | Ephemeral; an owner's presence for the agents it owns, named by `p` tags (at most 256); `online`/`away` set and `offline` clears each agent's presence as its own kind:20001 would. Not fanned out. | | **Typing indicators (kind:20002)** | ✅ | Ephemeral, not stored; published via Redis pub/sub (multi-node capable unlike presence fan-out) | | **NIP-42 authentication** | ✅ | Proactive challenge; optional pubkey allowlist | | **NIP-11 relay info** | ✅ | `GET /` with `Accept: application/nostr+json` | | **Blossom media** | ✅ | `PUT /media/upload` (BUD-02), `GET /media/{sha256}.{ext}` (BUD-01) | -| **NIP-50 search** | ✅ | One-shot search REQs: `{"search":"query","kinds":[9],"#h":[""]}` → relevance-sorted results → EOSE. Not registered as persistent subscriptions. | +| **NIP-50 search** | ✅ | One-shot search REQs: `{"search":"query","kinds":[9],"#h":[""]}` → relevance-sorted results → EOSE. Not registered as persistent subscriptions. HTTP `POST /query` also accepts `search_sort: "newest"` (advertised as `buzz-search-sort` in NIP-11 `supported_extensions`) for newest-first results, paged with `page`. | | **NIP-10 threads** | ✅ | WS-submitted replies with `["e","","","reply"]` tags create `thread_metadata` atomically. Visible in REST thread queries. Unknown parents rejected. | | **NIP-17 DMs (gift wrap)** | ✅ | kind:1059 accepted with ephemeral signing keys. Stored community-globally (`channel_id=None` inside the connected community). Delivered via `#p`-filtered subscriptions. Not indexed in search. | | **DM discovery** | ✅ | DM creation emits kind:39000 (with `hidden` tag) + kind:44100 membership notifications. NIP-29 clients discover DMs via standard group discovery flow. | -| **Join request (kind:9021)** | ✅ | Open channels only. Adds member, emits system message + group discovery events + kind:44100 membership notification. Private channels rejected at ingest. | +| **Join request (kind:9021)** | ✅ | Open channels only. Adds the joiner with the channel's `join_role` (`member` by default), emits system message + group discovery events + kind:44100 membership notification. Private channels rejected at ingest. | | **Edits (kind:40003)** | ⚠️ | Works on the wire but Buzz-only — no standard NIP-29 client renders these | | **Rich content (kind:40002)** | ⚠️ | Works on the wire but Buzz-only — no standard NIP-29 client renders these | @@ -99,16 +101,158 @@ relay to specific external Nostr identities without granting full access. SELECT encode(pubkey, 'hex'), added_at, note FROM pubkey_allowlist; ``` +### Announce channels + +Roles have the same permissions in every channel. A channel's settings +decide only who can join and which role a new person gets: + +- `visibility` (`open` | `private`) decides who can find, join and read. +- `join_role` (`member` | `guest`) is the role a person gets when nobody + picks one: a self-join (kind:9021, or a kind:9000 self-add), or a kind:9000 + add with no `role` tag. The default, `member`, keeps today's behavior. + +An open channel with `join_role = guest` is an **announce channel**: anyone +can find, read and join it, but people who join are guests and cannot write. + +What each role can do: + +| Action | Guest | Member or bot | Admin / owner | +|--------|-------|---------------|---------------| +| Read, leave | Yes | Yes | Yes | +| Post, reply, edit, delete own, react, artifacts, canvas, forum, pins, typing, workflow messages, huddles | No | Yes | Yes | +| Topic, purpose (kind:9002) | No | Yes | Yes | +| Add people (kind:9000) | No | Yes, as member, guest or bot | Yes, any role | +| Push to the channel's git repositories | No | Yes | Yes | +| Name, about, visibility, TTL, `join_role`, `unlisted` | No | No | Yes | + +A non-member may write only where joining would give a role that can write: +an open channel with `join_role = member`. A refused write gets +`restricted: only members can post in this channel`. + +Other rules: + +- A non-member adding themselves with kind:9000 may ask for `join_role` or a + lower role. A higher role is refused. +- Any member who can write (member, bot, owner or admin) can make a guest a + member with kind:9000 (`role` `member`). This is the same for a person and + an agent. Only owners and admins can make any other role change. +- The human owner of an agent can edit (kind:40003) and delete (kind:9005) + that agent's messages, also as a guest or a non-member. This is existing + behavior. It lets a person correct or remove what their agent posted; it + does not let them post. +- Owners and admins change the setting with kind:9002 + `["join_role", "guest"]` or `["join_role", "member"]`. A missing tag means + "no change". Changing it does not change anyone's role: existing members + can still write after a change to `guest`, and existing guests stay + read-only after a change to `member`. A stored value the relay does not + know is treated as `guest`. + +kind:39000 publishes the NIP-29 access flags from these settings, and an +explicit `["join_role", ...]` tag so clients can tell what joining gives: + +| `visibility` | `join_role` | `private` | `public` | `restricted` | `closed` | +|--------------|-------------|-----------|----------|--------------|----------| +| open | `member` | | ✓ | | | +| open | `guest` | | ✓ | ✓ | | +| private | either | ✓ | | ✓ | ✓ | + +### Unlisted channels + +An **unlisted** open channel works like a link-shared document: anyone in +the community who has its id (for example from a shared link) can read it +and join it, but it does not show up for people who are not members. + +- Owners and admins set it with kind:9002 `["unlisted", "true"]` and clear + it with `["unlisted", "false"]`. A missing tag means "no change". It can be + sent in the same event as `["visibility", "open"]`, so a private channel + becomes open and unlisted in one step. +- kind:9007 also takes `["unlisted", "true"|"false"]`, so a channel can + start unlisted; a missing tag means listed. Any other value is refused + before the event is stored, as on kind:9002. +- For a non-member, an unlisted open channel, of any type, is left out of + every read that does not name it: a filter without `#h` (including + kind:39000 discovery without `#d`, and the system-channel opt-in by `#P` + or `#t:["system"]`), COUNT, NIP-50 search, artifact queries and the + channel list. A filter that names it by `#h`, or by `#d` + on a filter whose kinds are all 39000–39003, reads it under the ordinary + open-channel rules. This is the same "named-only" rule that applies to + system channels ([NIP-CL](docs/nips/NIP-CL.md)). +- Joining (kind:9021) and posting work as on any open channel; `join_role` + still decides what a joiner gets. +- Members see it everywhere, as before. +- The flag is stored on its own and matters only while the channel is open. + Private channels are already invisible to non-members. A private channel + can be marked unlisted ahead of time, so opening it later never lists it. +- kind:39000 carries a bare `["unlisted"]` flag whenever the setting is on, + whatever the visibility. It does not add NIP-29 `hidden`: the metadata + stays readable by id. +- Relays that support it advertise `buzz-unlisted-channels` in NIP-11 + `supported_extensions`. An older relay applies `["visibility", "open"]` + and ignores `unlisted`, so a client must check for it before opening a + channel as unlisted. +- Unlisted is not an access control. Anyone with the id can read the channel. + ### Group Discovery The relay emits NIP-29 group state events when channels are created, updated, or membership changes. +[NIP-CL](docs/nips/NIP-CL.md) defines the channel identity tags on these events and how clients +use them to find channels. All discovery events include a `d` tag set to the channel UUID (NIP-29 addressable event convention): | Kind | Tags | Content | |------|------|---------| -| **39000** | `d=`, `name`, `closed` (always); `about` (if description non-empty); `private` (if applicable); `hidden` (DM channels only) | Group metadata. **Note:** `closed` is always emitted per NIP-29 convention (Buzz channels require explicit membership), but open channels are still readable/writable by non-members at runtime. The tag reflects the membership model, not access enforcement. | -| **39001** | `d=`, `p` tags with role label (`owner`, `admin`) | Admin list | -| **39002** | `d=`, `p` tags for all members | Member list | +| **39000** | `d=`, `name`, `t=`, `P=`, `created_at=`, `join_role` (always); `public` or `private`; `restricted` (private channels, and `join_role=guest`); `closed` (private channels only); `unlisted` (unlisted channels); `about` (if description non-empty); `hidden` (DM channels and private system channels) | Group metadata. The access flags follow the channel settings (see [Announce channels](#announce-channels)). | +| **39001** | `d=`, `t=`, `P=`, `p` tags with role label (`owner`, `admin`) | Admin list | +| **39002** | `d=`, `t=`, `P=`, `p` tags for all members | Member list | + +On all three kinds the channel type `t` tag comes first, then one `t` tag +per channel label, then `P`. Clients read the first `t` value as the +channel type. `P` is +the public key that signed the channel's kind:9007. The relay writes it +from the database, and it does not change when ownership moves. Uppercase +`P` follows NIP-22, NIP-34 and NIP-72, where it names the author of the root +object; lowercase `p` already lists members and DM participants on these +kinds. + +kind:39000 also carries `["created_at", ]`: the time the +channel was created. The event's own `created_at` is the time of the last +edit, so use the tag to find the older of two channels. + +**Channel labels.** A label groups channels across channel types, for +example `["t", "workspace"]`. Set labels with `["t",
.sql` | The table's final `CREATE TABLE` and its `CREATE INDEX` statements, one table per file. This is the declarative schema source. | +| `schema/schema.sql` | An `\i` line for each new file, after the tables it depends on. | +| `schema/types/`, `schema/partitions/`, `schema/seeds/` | The matching non-table objects. The layout test permits nothing else here. Rule 1 forbids any addition to `functions/` and `triggers/`. | +| `scripts/reconcile-schema-after-pgschema.sql` | Idempotent convergence, plus a live assertion, for anything `pgschema` does not preserve: seed rows, some storage parameters, partition attachment, and the shapes listed in AGENTS.md *Common Gotchas* #7. | +| `crates/buzz-push-gateway/migrations/` **and** the relay copies | The six `push_gateway_*` tables have two homes. The gateway applies its own migration history to its own database, through a pre-install/pre-upgrade Helm Job ([push-gateway-deployment.md](push-gateway-deployment.md)). The relay schema also defines them: `migrations/` (`0015`, `0043`, `0045`, `0060`), `schema/tables/public/push_gateway_*.sql`, and `schema/seeds/push_gateway_*.sql`. Until the duplication is removed, a gateway schema change goes in both, as relay `0060` mirrors gateway `0006`. Every rule here applies to both. The target is for the gateway to own its tables alone, in its own migrations and declarative schema; do not add new gateway tables to the relay schema. | + +A fresh desired-state database and a migrated database MUST converge on the +same live catalog. The desired-state layout test in +`crates/buzz-db/src/runtime/migration.rs` checks file structure only. Parity +tests cover selected surfaces, and the full drift gate does not exist yet. For +tables and indexes you touch, add or extend a parity assertion that compares +the live catalog from both bootstrap paths, following +`admin_schema_parity_between_desired_state_and_migrations`. A string match +against `schema.sql` does not count. + +The existing migration lints still apply. Every tenant-scoped table has +`community_id NOT NULL`, every key on a scoped table leads with `community_id`, +and operator-global tables are registered in all three places listed in the +provenance-only checklist under Rule 1. + +### 5. Every schema change is backward compatible with the running app + +Buzz supports two ways of applying migrations, and every change MUST be safe +under both: + +- **Auto-migration.** The relay binary defaults `BUZZ_AUTO_MIGRATE` to off, + but the Helm chart defaults `migrate.autoMigrate` to `true`. With it on, the + first new pod migrates at boot while old pods are still serving. +- **Out of band.** An operator applies the migrations while the current relay + keeps serving traffic, then deploys the new relay. Block's hosted + deployment works this way at scale, with `migrate.autoMigrate=false`. + +Either way, for a period the **old relay runs against the new schema**. If the +new relay is rolled back, the old relay runs against the new schema again. +Every migration MUST be safe for the currently deployed release to read and +write against. + +The relay has no general schema-version compatibility check. It verifies only +specific catalog surfaces, such as the deletion serving fence and the replica +floor guard, so the ordering below is the main protection. When +`BUZZ_AUTO_MIGRATE` is on, an older binary refuses a database that has +migrations it does not know (sqlx `VersionMissing`), so `helm rollback` with +auto-migration on fails at startup, and so does any old pod that restarts +mid-rollout after a new pod has migrated. A rollback therefore depends on the +older binary not running the migrator. Making rollback work under +auto-migration is part of supporting both modes. + +One feature therefore usually ships as several PRs, each deployed before the +next starts: + +1. **Expand (schema).** Add only things the old relay ignores: nullable columns, + columns with a constant default, new tables, new indexes. Add nothing the old + relay could violate. +2. **Deploy (code).** Ship the relay that writes the new shape on every write + path and reads both old and new rows correctly. +3. **Backfill (data).** After the new code is fully rolled out and old writers + are drained, repair historical rows with a backfill (Rule 3). Readers stay + correct for both repaired and unrepaired rows throughout. +4. **Contract (schema, then code).** After the backfill completes and + validates, add the constraints you wanted from the start: `NOT NULL`, + `CHECK`, `UNIQUE`. When nothing deployed still reads them, drop the old + columns, indexes, triggers, or functions. Each tightening is its own + migration, applied after the release that stops depending on the old shape. + +Skip a step only when it has nothing to do. For example, a new table with no +historical data needs no backfill. + +**Under auto-migration the steps can collapse.** The migrator applies every +pending migration at one boot, before the new code serves and before any +backfill or repair script runs. An operator who upgrades across several +releases therefore applies expand and contract together. How auto-migration +should order such an upgrade is unresolved. Until it is, a contract depends on +one or both of these, and each is handled differently: + +- **Data (a backfill or repair has run).** The contract migration MUST fail + closed when it has not. `VALIDATE CONSTRAINT` and `SET NOT NULL` already + fail on unrepaired rows. A contract that drops a column or table, or + otherwise loses data, MUST first check that the backfill or repair has + completed, and `RAISE` from a top-level `DO` block when unrepaired rows + remain. The PR MUST add a Postgres-gated test that applies the expand and + contract migrations in one run against unrepaired rows and asserts the + migration errors without dropping data. +- **Writers (a release that changes how the app writes is fully deployed).** + Dropping a trigger, function, or foreign key whose work moved into the app, + and adding a constraint that old writers could violate, depend on a release, + not on rows. No row check can prove it: the data may already be valid, or + there may be nothing to backfill, while old writers still serve. + Auto-migration does not protect this case. + +For either kind, the release notes MUST name the release to stop at (the one +that ships the writer) and any repair to run there, before upgrading past the +contract release. A failed data check stops the relay from starting. That is +an outage, not data loss, but recovery is not clean: the expand migrations +before the contract have already committed, so an old pod that restarts hits +`VersionMissing`. + +#### What breaks the old relay + +| Change | Why it breaks | Safe form | +|---|---|---| +| Add a `NOT NULL` column without a default | Old `INSERT`s omit it and fail | Add it nullable, deploy writers, backfill, then add the constraint (see below) | +| Rename a column or table | Old queries reference the old name | Add the new name, dual-write, backfill, move readers, then drop the old name | +| Change a column type | Old binds and decodes mismatch, and the change may rewrite the table | Add a new column and migrate through the expand/contract sequence | +| Add `CHECK`, `UNIQUE`, or a narrower index | Old writes violate it, or an old `ON CONFLICT` target no longer matches an index | Deploy writers that satisfy it, backfill, then add it. Add a `CHECK` as `NOT VALID`, then `VALIDATE CONSTRAINT` in a later step. Keep every index an old `ON CONFLICT` names until no deployed code uses it | +| Add an enum value that new code writes | Old code fails to decode the unknown variant | Deploy readers that accept the value, then deploy writers that produce it | +| Drop a column, table, index, trigger, function, or FK | Old code reads it or relies on its side effect | Contract step only, after the release that stops relying on it is fully deployed | +| Add a table with a `community_id` column | The old relay's exact-set deletion catalog check (`EXPECTED_SCOPED_TABLES`) sees an unknown table, so community deletion fails closed until the new release deploys. Serving is unaffected, apart from the auto-migration restart case in Rule 5 | Allowed. For a tenant-owned table, only after the desired-state change in Rule 1. Keep the gap between expand and deploy short, and do not run community deletion during it | +| `SELECT *` or `RETURNING *` decoded by position | Column order differs between desired-state and migrated databases and changes when columns are added | Select named columns | + +#### Adding `NOT NULL` to a populated table + +`ALTER TABLE ... SET NOT NULL` scans the whole table under `ACCESS EXCLUSIVE`. +Instead, ship three migration files, each its own step. sqlx runs each file in +a transaction, which `SET LOCAL` needs; pasted into autocommit `psql`, +`SET LOCAL` only warns and has no effect. + +```sql +-- contract migration A +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t ADD CONSTRAINT t_col_not_null CHECK (col IS NOT NULL) NOT VALID; + +-- contract migration B (separate step; scans but does not block writes) +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t VALIDATE CONSTRAINT t_col_not_null; + +-- contract migration C (PostgreSQL 12+ uses the validated CHECK and skips the scan) +SET LOCAL lock_timeout = '5s'; +ALTER TABLE t ALTER COLUMN col SET NOT NULL; +ALTER TABLE t DROP CONSTRAINT t_col_not_null; +``` + +The desired-state file declares only the final `col ... NOT NULL`, and that +change lands in the same PR as migration C. Until then, the desired state keeps +the column nullable. Between A and C, migrated databases carry +`t_col_not_null` and fresh ones do not, so exclude that constraint from any +parity assertion on the table for that window. + +#### Retiring a trigger, function, or foreign key + +1. Ship application code that does the trigger's work, or the foreign key's + check and cascade, while the database object is still in place. The two + paths MUST be idempotent with each other, so the duplicate work is harmless. +2. After that release is fully deployed, drop the database object in a + separate migration. Remove it from `schema/` in the same PR. The release + notes name the step-1 release as a required stop (Rule 5). +3. Removing a guard requires a falsifiable test showing that the application + path enforces what the guard enforced. + +### 6. Bound every lock and statement + +- Every migration MUST start with `SET LOCAL lock_timeout`, and SHOULD set + `SET LOCAL statement_timeout` when a statement can scan. A migration that + cannot get its lock fails and is retried. It MUST NOT queue behind + traffic on a hot table. +- In `migrations/`, an index on an existing populated table MUST be built + with `CREATE INDEX CONCURRENTLY`, because auto-migrating deployments have no + online schema change tool to build it for them. CONCURRENTLY cannot run + inside the migration transaction, so follow the prebuild-then-verify + procedure in [thread-window-deployment.md](thread-window-deployment.md). + Partitioned parents need per-partition concurrent builds attached to an + `ON ONLY` parent index. +- The declarative schema source declares indexes with plain `CREATE INDEX`. + The tool that applies it out of band decides how to build them online, so + `schema/` files MUST NOT add `CONCURRENTLY`. The desired-state layout test + rejects it. +- A change MUST NOT rewrite or fully scan a large table under a + write-blocking lock. That includes column type changes, volatile defaults, + `SET NOT NULL` without a validated check, and changes to the generated + `events.search_tsv` column. Use the expand/contract forms above. +- Write changes that out-of-band apply can make online: `CREATE TABLE`, + `CREATE INDEX`, and bounded `ALTER TABLE` on tables below a size ceiling, + with short lock and statement budgets. Do not rely on a multi-statement + change applying atomically. A change outside that envelope needs a + separately reviewed operational procedure, and the PR MUST say so. + +### 7. Old and new relays coexist during rollout + +Rolling deploys run two relay versions against one database. Durable records +written by the new release, such as backfill rows, outbox items, and new enum +values, MUST be handled safely by the old one. This means ignoring them, +leaving them for the new release, or failing closed. The old release MUST NOT +misinterpret them. + +## Review checklist + +A PR that touches the schema or a schema-dependent query states in its +description: + +- [ ] Which phase this is (expand, deploy, backfill, or contract), and which + PRs or releases come before and after it. +- [ ] That no trigger, function, procedure, rule, or foreign key is added. +- [ ] That no migration mutates existing rows, and the backfill ID (or, + until the framework ships, the `scripts/maintenance/` script and how it + takes the community deletion lock) for any repair. +- [ ] For a contract migration, how it fails closed when its backfill has not + run, and which writer release the release notes name as a required stop. +- [ ] That every location in Rule 4 is updated, with a catalog-level parity + test for each changed table and index. +- [ ] That the currently deployed relay works against the new schema, and how + that was checked. +- [ ] Lock and statement timeouts, concurrent index builds in `migrations/`, + and any operator prebuild step. + +## Related documents + +- [ARCHITECTURE.md](../ARCHITECTURE.md): *Supported Writer Contract* and the + `buzz-db` crate. +- [data-backfills.md](data-backfills.md): the backfill specification. +- [multi-tenant-conformance.md](multi-tenant-conformance.md): tenant-scoping + obligations. +- [thread-window-deployment.md](thread-window-deployment.md): the brownfield + concurrent-index procedure. +- [crates/buzz-db/TESTING.md](../crates/buzz-db/TESTING.md): desired-state + and migration-mode tests. +- `schema/schema.sql` header: the desired-state layout. diff --git a/docs/formal/nip-pl/delivery.py b/docs/formal/nip-pl/delivery.py index 2c8c5ac350..39a1ca9d34 100644 --- a/docs/formal/nip-pl/delivery.py +++ b/docs/formal/nip-pl/delivery.py @@ -7,7 +7,7 @@ transient release, revocation ordering, custody, and the constant APNs body. """ from itertools import permutations, product -FIXED_BODY = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"mutable-content":1}}' +FIXED_BODY = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"sound":"default","mutable-content":1}}' class Gateway: def __init__(self): diff --git a/docs/formal/nip-pl/fixed_payload.py b/docs/formal/nip-pl/fixed_payload.py index e69af9c434..5b424cd21f 100644 --- a/docs/formal/nip-pl/fixed_payload.py +++ b/docs/formal/nip-pl/fixed_payload.py @@ -5,7 +5,7 @@ caller-controlled or capability-derived category; none is an argument to body(). """ from itertools import product -C = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"mutable-content":1}}' +C = b'{"aps":{"alert":{"body":"Reconnect to your relay now"},"sound":"default","mutable-content":1}}' DOMAINS = [ (b"request-a", b"request-b"), # exact signed body (b"auth-a", b"auth-b"), # NIP-98 event/header diff --git a/docs/native-notification-enrichment.md b/docs/native-notification-enrichment.md new file mode 100644 index 0000000000..f4e6c71d56 --- /dev/null +++ b/docs/native-notification-enrichment.md @@ -0,0 +1,75 @@ +# Sender information in iOS notifications + +Buzz must show a sender's name and avatar even when the main app has never +loaded that sender's profile. The notification service extension fetches and +caches the information it needs. The main app supplies signed-in accounts, +relay configuration and credentials in the shared Keychain. It does not export +profiles, channel details or avatar images for notifications. + +## Display behavior + +The extension uses one freshness threshold of one hour for profiles, channel +details, and avatar refresh checks. The period starts at the last successful +check. Reading a record does not extend it. When a record is older, the +extension tries to refresh it and can use the previous record if the refresh +fails. This setting does not extend authorization. Image requests also follow +HTTP caching rules, which can avoid downloading unchanged image data again. + +After it resolves the message, the extension allows up to 1.5 seconds for the +sender and channel details. It then delivers the best available notification. +If no sender name or avatar is available, it omits those values where iOS +supports this. It must not use a public key as the displayed name. + +When a sender changes their avatar, the previous image remains usable until +the replacement arrives. When a verified profile removes the avatar, the old +image is no longer shown. A late response must not restore a removed avatar. + +The extension supports the same avatar sources as the app: Buzz uploads, +external HTTP(S) images and supported inline images. It uses existing server +thumbnails for eligible Buzz uploads. A separate resized-image cache is not +required. Animated avatars use the sender-selected still image. Inline SVGs +are rendered natively. Image decoding has size limits; SVG decoding also limits +nesting and repeated references. Embedded fonts, external SVG resources and +recursive references are rejected. An image that cannot be decoded follows the +same stale-image or missing-image behavior as a failed download. +An empty profile query is not a verified removal and does not clear stale data. + +## Fetching and storage + +The extension uses native networking and its own persistent storage. Image +responses use URLSession and URLCache; verified profiles and channel details +have a separate bounded store. Each account and relay has separate cache data. +Credentials are not stored in these caches. Profile and channel requests run +independently, so a large or slow channel roster cannot block a sender profile +or avatar that is available before the delivery deadline. +When only channel metadata or membership refreshes successfully, the extension +keeps that record and removes its old counterpart in the same write. It does +not derive a group identity or recipient count from a mixed pair. If neither +request succeeds, the previous pair remains available. + +Delivery happens only once. Ordinary requests can continue after delivery if +iOS keeps the process alive. A late profile response can start an avatar fetch, +and completed results can be saved for the next notification. These requests +use normal network timeouts. There are no background sessions, and successful +notification delivery does not depend on work continuing after delivery. + +A cache miss is normal. Interrupted writes must leave complete old or new +records, not partial records. Invalid cache entries are reported and treated +as misses. The extension must work after iOS removes cache files or starts a +new process. + +Removing an account immediately prevents access to its cached information. +The extension deletes the account's cache files when it next runs. Results +from requests started before removal cannot become usable in a new sign-in. + +## Validation + +Tests cover a sender never loaded by the app, cache hits and misses, stale +records, avatar replacement and removal, account removal, late responses and +exactly-once delivery. Storage tests cover concurrent access and interruption. +Device validation checks actual notification appearance, authenticated image +cache reuse, process restarts and notification bursts. In particular, accepting +nil name and image values in an API does not by itself prove how iOS renders +those values. + +Persistent image caching for the main app is separate from this change. diff --git a/docs/nips/NIP-AE.md b/docs/nips/NIP-AE.md index c24078d8a4..906c30e02e 100644 --- a/docs/nips/NIP-AE.md +++ b/docs/nips/NIP-AE.md @@ -149,6 +149,23 @@ A body MAY reference other slugs using wiki-link syntax: `[[]]`, where `", - "created_at": , - "tags": [ - ["p", ""] // agent subject, or: - // ["t", ""] // team subject - ], - "content": "" -} -``` +A client stores config on the relay only when the relay's [NIP-11](11.md) document shows that it supports both of these: -- The owner authors every version. Agents do not author instructions versions. -- An event MUST carry exactly one **subject tag**: either one `p` tag whose value is the agent's 64-character lowercase hex pubkey, or one `t` tag whose value is a team id (see "Team identity"). A subject tag MUST have exactly two elements. Tags are counted by their first element, so an event with both a `p` and a `t` tag, two `p` tags, two `t` tags, or a valueless `["p"]` or `["t"]` is invalid. -- `t` values are case-sensitive. Clients and relays MUST NOT lowercase them, unlike [NIP-24](24.md) hashtags. -- The event MUST NOT carry an `h` tag; it belongs to no channel. +- NIP-CL system channels and labels; +- NIP-AR artifacts. -Implementations MAY include a [NIP-31](31.md) `["alt", "agent instructions version"]` tag. Other tags are not defined by this NIP and, apart from the prohibited `h` tag, have no effect on validity. +How a relay advertises each one is defined by the specification of that feature. Otherwise the client MUST keep config local, and MUST NOT create config channels. -### Content +### Config channels -`content` MUST be [NIP-44](44.md) v2 ciphertext encrypted with the conversation key of `(seckey_o, pubkey_o)`: the owner encrypts to themselves. +| Channel | One per | Created by | Family label | Subject label | +| --- | --- | --- | --- | --- | +| Agent config channel | agent | the agent's owner | `agent-config` | the agent's 64-character lowercase hex pubkey | +| Team config channel | team | the team's owner | `team-config` | `team-` followed by the first 59 lowercase hex characters of the SHA-256 hash of the team id's UTF-8 bytes | -The plaintext is the three ASCII bytes `v1:` followed by the instructions text in UTF-8. The prefix lets an empty text be encrypted; NIP-44 cannot encrypt an empty plaintext. +The owner creates a config channel with a kind:9007 that carries `["channel_type", "system"]`, both labels as `t` tags, and no `visibility` or `ttl` tag. A config channel MUST be private and MUST NOT have a TTL. The owner MUST NOT change a config channel's labels. -- The text MAY be empty. An empty version means "no instructions" for its subject. -- The text MUST be valid UTF-8, MUST NOT contain U+0000, and MUST NOT exceed 131,072 bytes. -- A plaintext of 65,536 bytes or more uses NIP-44 v2's extended length prefix: two zero bytes followed by the plaintext length as a 32-bit big-endian integer. An implementation that supports only the two-byte prefix cannot decrypt such a version and MUST treat it as invalid, not absent. -- With these bounds, `content` is at most 218,548 bytes. +Both subject labels are 64 characters. A team id can contain characters a label cannot, so the team label is derived from a hash; the team id itself is stored on the `buzz.team` artifact (see "Config artifacts"). -Writers MUST enforce the text rules before encrypting. Readers MUST check, in this order: +An agent config channel's members are the owner, the agent, and any others the owner adds. A team config channel holds the owner and the team's members, and can hold others who are not members (see "Membership"). -1. `content` is at most 218,548 bytes, before decoding or decrypting it; -2. it decrypts under NIP-44 v2; -3. the plaintext starts with `v1:`; -4. the text after the prefix is valid UTF-8, contains no U+0000, and is at most 131,072 bytes. +### Discovery -A version that fails any check is **invalid**. Ciphertext length cannot enforce the text bound by itself: a maximum-length text and a text one byte longer produce `content` of the same length. - -### Current version and history - -A subject's **current version** is its event with the greatest `created_at`; ties are broken by lowest event `id`, as in [NIP-01](01.md). +To find a config channel, a client queries by creator and subject label: ``` -Agent: {kinds: [44300], authors: [pubkey_o], "#p": []} -Team: {kinds: [44300], authors: [pubkey_o], "#t": []} +Agent: {kinds: [39000], "#P": [pubkey_o], "#t": []} +Team: {kinds: [39000], "#P": [pubkey_o], "#t": []} ``` -With `limit: 1`, these filters return the current version. Clients query one subject per filter. +To find the teams an agent is on, the agent's client queries the channels the agent belongs to, opting in to system channels: -History is the same filter, ordered by `created_at` descending and then by `id` ascending, and paged with a composite cursor: the `created_at` and `id` of the last event on the previous page, sent as `until` and `before_id`. The next page holds events with `created_at < until`, or with `created_at = until` and `id > before_id`. `before_id` is an extension to the NIP-01 filter, not a standard field. A timestamp-only cursor is not sufficient: concurrent saves on different devices can give several versions of one subject the same `created_at`, and paging by `until` alone either repeats or skips them. A relay MAY return fewer events than `limit`, so a client has read a subject's complete history only when a page returns no events. +``` +{kinds: [39002], "#p": [], "#t": ["system"]} +``` -If a subject's current version is invalid, clients MUST show the error and MUST NOT substitute an older version, a definition's prompt, or a legacy copy. +then reads those channels' kind:39000 by `#d`. -### Writing +In every case the client keeps only channels whose type is `system`, whose labels include the family label, and whose `P` is the expected owner. It MUST NOT put the family label in the same `#t` filter, because values in one filter combine with OR. When more than one channel remains for one agent or team, the client uses the oldest, by NIP-CL's "Choosing one channel" rule. A client creates a config channel only when this query returns none. A client MUST NOT delete or archive a config channel because it was not selected. -To save text `x` for a subject: +### Config artifacts -1. Validate `x` against the text rules. Reject if invalid. -2. Fetch the subject's current version and let `T` be its `created_at` (or 0 if none). Set `created_at := max(now, T + 1)`. -3. Encrypt `v1:` followed by `x` (see "Content"). Tag the event with the subject. -4. Sign with `seckey_o` and publish. +| Type | Channel | `title` | Extra tags | Content | +| --- | --- | --- | --- | --- | +| `buzz.agent.settings` | agent config | `Agent settings` | | `{"version": 1, "settings": { ... }}` | +| `buzz.agent.instructions` | agent config | `Agent instructions` | | `{"version": 1, "text": ""}` | +| `buzz.team` | team config | the team name | `["team", ""]` | `{"version": 1, "name": "", "instructions": ""}` | -There is no compare-and-swap. If two devices save concurrently, the relay stores both, the newer becomes current, and the other remains in history. A client SHOULD fetch the current version again after publishing and tell the user when its save was stored but is not current. +A config channel holds at most one artifact of each of its types. Its `d` is the UUIDv5 of the type name in the namespace of the channel's UUID, so two clients creating the same artifact collide on `d` and one create fails. The client whose create fails fetches the current revision and edits it; it MUST NOT choose a different `d`. A client accepts a config artifact only when its `d`, `type`, and `h` match the config channel it was read from. A `buzz.team` artifact has no member list; members come from the channel (see "Membership"). -Before a subject's first version, a client SHOULD publish the text the subject currently resolves to (see "Launch composition" and "Team instructions"), so the text from before the first edit can be restored. +**Team id.** A `buzz.team` artifact carries exactly one `team` tag, whose value is the team id. A client MUST check that the team id's subject label equals the channel's subject label, and MUST treat the artifact as invalid otherwise. -- **Restore** publishes an older version's text as a new version. -- **Reset to template**, for an agent, publishes its linked definition's current `system_prompt` as a new version, or empty text when it has none. +**Team name.** `name` is also the artifact's `title`, so it MUST be nonblank and at most 512 UTF-8 bytes, and MUST equal the `title` tag. -Clients MUST NOT delete versions to roll back. Deleting the current version makes the previous version current; it is not a restore. +**Instructions.** `text` and `instructions` MAY be empty, which means "no instructions". They MUST be valid UTF-8 and MUST NOT contain U+0000. + +**Settings.** `settings` is a JSON object of the agent's launch configuration. Its fields are defined by the client and harness that launch the agent, not by this NIP. It MUST NOT contain credentials, secrets, machine-local paths, or device state. A client MUST NOT launch an agent with a setting it does not support silently replaced by a default; it shows the error instead. + +**Size.** A config artifact's `content` MUST NOT exceed 131,072 bytes after JSON encoding, and `buzz.agent.settings` content MUST NOT exceed 65,535 bytes. The signed event MUST also fit within the relay's advertised event-size limits. Writers check these limits on the encoded content and the signed event, reject an edit that exceeds them, and MUST NOT truncate. + +An artifact whose content does not parse, has an unknown `version`, or breaks these rules is **invalid**. + +### Artifact state + +When a client reads a config artifact, the artifact is in one of these states: + +- **present:** its current revision was read and is valid; +- **invalid:** its current revision was read and is invalid; +- **absent:** the relay confirmed that the artifact's address has never been used; +- **deleted:** a lookup by `d` returned a deletion; +- **unavailable:** anything else, including a failed read, an unreadable channel, an artifact moved to another channel, a redacted current revision, or a lookup that returns nothing without that confirmation. + +Only an absent artifact lets a client fall back to another source (see "Launch composition"). An invalid, deleted, or unavailable artifact blocks the start: the client MUST show the error and MUST NOT substitute an older revision, another source, or local config, except as "Offline start" below allows for an unavailable artifact. A deleted artifact is recovered with NIP-AR `restore`. + +**Offline start.** When the relay cannot be reached, because the read failed in transport rather than returning an answer, a client MAY start the agent from the last revision it confirmed as present and kept locally. It MUST NOT do so if the most recent answer it read from the relay showed that the artifact is invalid or deleted, that its current revision was redacted, that the config channel is archived, or that it has lost access to the channel, and it MUST NOT use any other source. + +### Editing + +Every channel member who is not a guest can edit the channel's artifacts. This includes members who are not on the team, such as humans and agents with another owner. On a relay whose guest roles are read-only, guests can only read them, and the owner can make an agent read-only for a config channel by giving it the guest role there. + +Every edit is a new NIP-AR revision and names the current revision in `prev`. On conflict, the client reconciles with the current revision as NIP-AR requires. An edit MUST keep any JSON fields and tags the client does not recognise. + +### History + +Every revision of a config artifact has a stable ID, its event ID. A config artifact's revisions are its history. History is kept until the owner deliberately purges it or a moderator removes a revision. Config channels MUST be exempt from age-based community retention, and a client MUST NOT configure a config artifact to keep only its latest revision. A client shows a removed revision as a gap in history, not as the absence of history. + +**Revert** publishes an older revision's content as an `op=update` revision carrying a complete, valid snapshot. Reverting a team's instructions keeps the team's current name. Clients MUST NOT redact revisions to roll back. ### Launch composition An agent's own instructions layer is the first of these that exists: -1. its current agent-subject version. An empty version yields no instructions and stops resolution; an invalid version blocks the start; -2. the `system_prompt` of its linked `kind:30175` definition; -3. for a definition-less instance, the `system_prompt` in its legacy `kind:30177` event. +1. the `text` of its current `buzz.agent.instructions` artifact. Empty text yields no instructions and stops resolution. An invalid, deleted, or unavailable artifact blocks the start (see "Artifact state"); +2. while that artifact is absent, instructions held in local configuration; +3. the `system_prompt` of its linked `kind:30175` definition; +4. for a definition-less instance, the `system_prompt` in its legacy `kind:30177` event. -The team layer is resolved independently (see "Launch with teams"). The client delivers both layers to the agent's harness as separate layers, the agent's own first. +A client moving local instructions to the relay MUST create the `buzz.agent.instructions` artifact and confirm that the relay accepted it before it stops using the local copy. -A version takes effect at the agent's next start; a running agent is unchanged. A client MAY restart an agent to apply a new version. A client that lets local configuration override the resolved instructions SHOULD show the owner that an override is in effect. +Team sections are resolved independently (see "Launch with teams"). The client delivers the agent's own layer and each team section to the agent's harness separately, the agent's own first. -This NIP does not define how `kind:44300` relates to the private managed-agent aggregate reserved by [NIP-PMA](NIP-PMA.md). That integration is left to NIP-PMA. +A change to the agent's instructions or to any of its teams MUST take effect no later than the agent's next session start. A client MAY apply it sooner, including to an active session between turns, and MAY start a new session to apply it. An active session need not be interrupted. A client that lets local configuration override the resolved instructions SHOULD show the owner that an override is in effect. -### Launch records +### Deleting an agent -A client SHOULD record, for each agent start, which instructions it used: - -- the agent-subject version id, or that the layer came from a definition, came from a legacy event, or was empty; -- the team id and version id of every team that contributed; -- a SHA-256 hash of each layer's text as delivered, after any local override, and the source of any override; -- a run identifier and the start time. - -A client that records launches SHOULD write the record once the agent has started and retry until the relay stores it. The record MUST be encrypted to the owner, for example as a [NIP-AE](NIP-AE.md) engram. Version ids and hashes MUST NOT appear in plaintext tags. A launch record shows what the client handed to the harness, not that the harness used it. - -### Privacy - -Instructions versions are readable only by their author. The agent's key has no access to them; the client decrypts the agent's instructions and hands them to the agent at start. The `p` tag names the subject, not a reader, and grants the agent nothing. - -The relay still sees each event's subject tag, timestamp, and padded size, so it can observe how often each agent's or team's instructions change and roughly how long they are. - -Clients MUST publish instructions versions, and deletion requests for them, only to the relay community that holds the subject. - -### Deletion: kind:44300 - -Owners MAY publish [NIP-09](09.md) deletion requests for instructions versions, one `e` tag per version. A deletion request for a `kind:44300` event MUST carry `["k", "44300"]`; the relay classifies it by that tag, so it stays private after its target is removed (see "Access control: kind:44300 author-only"). A relay MAY accept only one target per request. Clients SHOULD publish one request per version, retry until every version is deleted, and MUST NOT report a subject's history deleted after deleting only part of it. - -- A client that deletes an agent SHOULD delete all of that agent's versions. It MUST NOT delete any team's versions. -- A client that deletes a team follows "Deleting a team". - -Deletion withholds versions from readers. It does not erase copies held by the relay operator, and it does not reach copies in exports, `kind:30178` shares, or launch records. +A client deletes an agent by archiving its agent config channel. A client MUST NOT start a new session for an agent whose config channel is archived. An active session need not be interrupted. The archived channel stays discoverable, so the agent can be recovered by unarchiving it. Purging the channel is a separate action. Deleting an agent never purges a team's config or history. ### Older clients -Clients that do not implement `kind:44300` ignore it. They resolve an agent's instructions from `kind:30175` and `kind:30177` as before, so they launch agents with the definition's prompt rather than the agent's current version, and they do not see team versions. +Clients that do not implement config channels do not see them, because NIP-CL leaves system channels out of reads that do not name them. They resolve an agent's instructions from `kind:30175` and `kind:30177` as before, so they launch agents with the definition's prompt rather than the agent's current instructions, and they do not see teams stored on the relay. ## Relay behavior @@ -454,26 +453,6 @@ Kind `30178` is stored globally and its content is unvalidated, exactly as for ` - The relay MUST enforce **exactly one** `d` tag whose value is non-empty, at most 64 characters, and free of Unicode control characters and whitespace. Tags are counted by their first element, so a valueless `["d"]` counts toward the total and fails the value check on its own — otherwise `["d"]` alongside `["d",""]` would pass, and a consumer that reads `["d"]` as an empty-valued first `d` tag would address the event at `""` while this relay addresses it at ``. Without the non-empty check, generic NIP-33 storage maps a missing or empty `d` to the empty coordinate, collapsing every team into the single `(pubkey_o, 30178, "")` slot — last-write-wins data loss. The character bound keeps the value usable as a NIP-33 coordinate and as a log field. - The relay MUST NOT apply the persona slug grammar to a `30178` `d` tag; team ids legitimately contain characters (notably `:`) that the slug grammar forbids. -### Ingest validation: kind:44300 - -- The relay MUST reject, with `invalid:`, a `kind:44300` event that does not carry exactly one subject tag as defined in "Event envelope", whose `p` value is not 64-character lowercase hex, or whose `t` value does not match the team id grammar. -- The relay MUST reject a `kind:44300` event with an empty `content` or a `content` longer than 218,548 bytes. It cannot validate the plaintext; that is a client responsibility. -- The relay MUST reject a `kind:44300` event that carries an `h` tag. -- The relay MUST reject a `kind:5` deletion request that targets a stored `kind:44300` event and does not carry `["k", "44300"]`. -- The relay stores `kind:44300` events globally, outside any channel. - -### Access control: kind:44300 author-only - -The relay MUST withhold `kind:44300` events, and their existence, from every reader except their authenticated author, on every read surface: historical REQ delivery, NIP-01 `ids` lookup, live fan-out, COUNT, and the HTTP bridge's query and count endpoints. - -The relay MUST apply the same rule, on the same read surfaces and in search results, to every `kind:5` deletion request that carries `["k", "44300"]`, whether or not its target is still stored. A deletion request reveals a version's id and the time it was deleted. - -The relay MUST exclude `kind:44300` from full-text search results for every reader, including the author. - -The relay MUST apply a `#p` or `#t` filter on `kind:44300` before `ORDER BY … LIMIT`, so that `limit: 1` returns a subject's current version even when the owner has newer versions for other subjects. - -The relay MUST support the composite history cursor (see "Current version and history") for `kind:44300`. A filter carrying `before_id` without `until`, or a `before_id` that is not 64 hexadecimal characters, MUST be rejected rather than ignored. - ### Access control: author-only-unless-shared Kind `30175` uses **shared-tag-gated read semantics** to protect system prompts and `respond_to_allowlist` from being visible to all community members as a side-effect of device sync. @@ -507,13 +486,13 @@ These rules are enforced at the following relay read surfaces (content and event ## Security considerations -- **No encryption for definitions.** Persona and team-catalog events store system prompts, model names, runtime identifiers, and all configuration unencrypted. `kind:44300` instructions versions are the exception: they are encrypted to the owner. Shared persona events are readable community-wide. Operators MUST NOT store secrets in persona event content. +- **No encryption for definitions.** Persona and team-catalog events store system prompts, model names, runtime identifiers, and all configuration unencrypted. Shared persona events are readable community-wide. Operators MUST NOT store secrets in persona event content. - **System prompt protection.** System prompts and `respond_to_allowlist` pubkeys are sensitive. The relay's author-only-unless-shared gate ensures they are not visible to other community members unless the owner explicitly opts in by publishing a `["shared", "true"]` head. Shared persona events are readable community-wide; operators who need additional confidentiality should use relay-level access controls or choose not to share. - **Write authority.** Only the holder of `seckey_o` can publish or replace persona events. NIP-33 replacement is scoped by pubkey — no spoofing risk from other relay members. - **Slug collision across pubkeys.** Two different owners can publish personas with the same slug. Clients MUST always scope queries by author pubkey, not just slug. - **Metadata exposure.** The `(pubkey, kind:30175, slug)` triple reveals persona existence. Event timestamps reveal edit history. - **No owner write authority over agents.** Persona events define *what* an agent should be; they do not grant runtime control over a running agent. The agent consumes the persona at spawn time. Updates to the persona event do not automatically propagate to running agents. -- **Instructions history is owner-private, not secret from the owner's clients.** `kind:44300` versions are encrypted to the owner and withheld from other readers. Any client holding `seckey_o` can read every version; deleting versions does not revoke copies already exported, shared, or recorded at launch. +- **Config is readable by every config channel member.** Agent settings, agent instructions, and team instructions are not encrypted. Every member and guest of a config channel can read its artifacts and their history, so adding someone to a config channel shares that config with them. Every member who is not a guest can also edit it, including humans and agents that are not on the team. The relay operator can read them too. Removing someone, archiving the channel, or redacting a revision does not revoke copies already read, exported, or shared. - **Sharing a team shares every member's instructions.** A `kind:30178` head carrying `["shared","true"]` exposes the team's own fields *and* the embedded projection of every member — including members whose own `kind:30175` heads are unshared and therefore still private. Clients MUST make this explicit at the point of sharing; the relay cannot infer it. ## Reference test vectors diff --git a/docs/nips/NIP-CL.md b/docs/nips/NIP-CL.md new file mode 100644 index 0000000000..9b98829280 --- /dev/null +++ b/docs/nips/NIP-CL.md @@ -0,0 +1,305 @@ +NIP-CL +====== + +Channel Identity and Discovery +------------------------------ + +`draft` `optional` `relay` + +This NIP adds identity facts to the relay-signed NIP-29 channel-state events +(kinds 39000–39002). With these facts, a client can find channels with +ordinary NIP-01 filters. It needs no new filter key and no relay-specific API. + +## Motivation + +Plugins and agents often create a channel for their own use and must find it +again later, for example "the config channel that this agent created". With +plain NIP-29, a client cannot: + +- tell who created a channel; +- read the channel type from any event except kind:39000; or +- tell which of two channels is older. The `created_at` of a kind:39000 event + is the time of the last edit, not the time of creation. + +A client also cannot mark its own channels so that it can find them again. +Channel labels (below) do this. + +Also, every channel that a person belongs to shows in their client as a chat. +So a group that exists only to control who can read some data appears in +sidebars, search results and unread counts. System channels (below) fix this. + +## Channel identity tags + +On every channel-state event that the relay signs, the relay MUST put these +tags in this order, before any other `t` or `P` tag: + +1. `["t", ]`, for example `stream`, `forum`, `dm`, `workflow`, `system`. +2. `["t",