diff --git a/.github/workflows/_ci-desktop.yml b/.github/workflows/_ci-desktop.yml index cd1532aab9..a0b5e7e628 100644 --- a/.github/workflows/_ci-desktop.yml +++ b/.github/workflows/_ci-desktop.yml @@ -21,14 +21,15 @@ env: PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright jobs: - desktop-core: - name: Desktop Core + # Biome, node:test, and the Vite build do not compile Rust, but node:test + # also pins desktop constants to relay/DB crate sources (for example + # relayReconnectReplay.test.mjs), so Rust changes select this job too. + desktop-js: + name: Desktop JS runs-on: ubuntu-latest - # The compiled-flag verification step rebuilds the workspace for each - # BUZZ_BUILD_* state and runs the full suite under all three compile - # states; the complete recipe needs ~46m, so budget 60m. - timeout-minutes: 60 - if: github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust + timeout-minutes: 20 + # Keep in sync with JS_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs). + if: github.event_name == 'push' || inputs.desktop || inputs.rust permissions: contents: read steps: @@ -36,36 +37,6 @@ jobs: with: fetch-depth: 2 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 - - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 - - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 - with: - workspaces: desktop/src-tauri - save-if: ${{ github.event_name != 'pull_request' }} - - name: Install Tauri dependencies (Linux) - env: - DEBIAN_FRONTEND: noninteractive - run: | - sudo apt-get update \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=30 \ - -o Acquire::https::Timeout=30 - sudo apt-get install -y --no-install-recommends \ - -o Acquire::Retries=3 \ - -o Acquire::http::Timeout=30 \ - -o Acquire::https::Timeout=30 \ - -o DPkg::Lock::Timeout=120 \ - build-essential \ - curl \ - file \ - libasound2-dev \ - libayatana-appindicator3-dev \ - libgtk-3-dev \ - librsvg2-dev \ - libssl-dev \ - libwebkit2gtk-4.1-dev \ - libxdo-dev \ - patchelf \ - wget - name: Get pnpm store directory id: pnpm-cache run: echo "STORE_PATH=$(pnpm store path --silent)" >> "$GITHUB_OUTPUT" @@ -83,35 +54,6 @@ jobs: run: just desktop-test - name: Desktop build run: just desktop-build - - name: Desktop Tauri clippy - run: just desktop-tauri-clippy - env: - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - - name: Desktop Tauri check - run: just desktop-tauri-check - env: - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - - name: Desktop Tauri tests - run: just desktop-tauri-test - env: - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - - name: Desktop Mesh feature tests - run: cargo test --manifest-path desktop/src-tauri/Cargo.toml --features mesh-llm --lib - env: - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - - name: Desktop Tauri compiled-flag verification - run: just desktop-tauri-test-compiled-flags - env: - CMAKE_POLICY_VERSION_MINIMUM: "3.5" - - name: Upload desktop e2e artifacts - if: failure() - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 - with: - name: desktop-e2e-artifacts - path: | - desktop/playwright-report - desktop/test-results - if-no-files-found: ignore - name: Save pnpm store cache if: github.event_name == 'push' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 @@ -119,17 +61,85 @@ jobs: path: ${{ steps.pnpm-cache.outputs.STORE_PATH }} key: pnpm-${{ runner.os }}-${{ hashFiles('**/pnpm-lock.yaml') }} - desktop-smoke-e2e: - name: Desktop Smoke E2E (${{ matrix.shard }}) + # Tauri Rust lanes run in parallel. The compiled-flags lane only rebuilds + # buzz_lib under other BUZZ_BUILD_* values, so it restores the core lane's + # cache read-only; the mesh-llm feature pulls its own dependency graph. + desktop-tauri: + name: Desktop Tauri (${{ matrix.lane }}) runs-on: ubuntu-latest - # Chromium dependency setup can take 10m; a serial smoke shard needs - # ~25m. Leave time for the full suite and its reports to finish. - timeout-minutes: 45 - if: github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust + # A lane needs up to ~15m. ci-apt-retry.sh bounds the apt step at ~34m worst + # case (update ~8.5m + install ~25m); 55m fits both, so a degraded mirror + # fails with the wrapper's error instead of the job timeout. + timeout-minutes: 55 + # Keep in sync with RUST_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs). + if: github.event_name == 'push' || inputs.desktop_rust || inputs.rust strategy: fail-fast: false matrix: - shard: [1, 2, 3, 4] + include: + - lane: core + cache: desktop-core + save-cache: true + run: just desktop-tauri-clippy && just desktop-tauri-check && just desktop-tauri-test + - lane: mesh + cache: desktop-mesh + save-cache: true + run: just desktop-tauri-test-mesh + - lane: compiled-flags + cache: desktop-core + save-cache: false + run: just desktop-tauri-test-compiled-flags + permissions: + contents: read + steps: + - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + with: + fetch-depth: 2 + - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 + - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 + - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 + with: + workspaces: desktop/src-tauri + shared-key: ${{ matrix.cache }} + save-if: ${{ matrix.save-cache && github.event_name != 'pull_request' }} + - name: Install Tauri dependencies (Linux) + env: + DEBIAN_FRONTEND: noninteractive + run: | + CI_APT_ATTEMPT_SECONDS=90 scripts/ci-apt-retry.sh sudo apt-get update + scripts/ci-apt-retry.sh sudo apt-get install -y --no-install-recommends \ + build-essential \ + curl \ + file \ + libasound2-dev \ + libayatana-appindicator3-dev \ + libgtk-3-dev \ + librsvg2-dev \ + libssl-dev \ + libwebkit2gtk-4.1-dev \ + libxdo-dev \ + patchelf \ + wget + - name: Desktop Tauri ${{ matrix.lane }} + run: ${{ matrix.run }} + env: + CMAKE_POLICY_VERSION_MINIMUM: "3.5" + + desktop-smoke-e2e: + name: Desktop Smoke E2E (${{ matrix.shard }}) + runs-on: ubuntu-latest + # A smoke shard needs ~12m, and Chromium dependency setup is bounded by + # ci-apt-retry.sh (~25m worst case). Leave time for the full suite + # and its reports to finish. + timeout-minutes: 45 + # Keep in sync with FRONTEND_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs). + if: github.event_name == 'push' || inputs.desktop + strategy: + fail-fast: false + matrix: + # Playwright shards smoke specs by test count, not duration; ten + # shards keep the slowest one near 11 minutes of test time. + shard: [1, 2, 3, 4, 5, 6, 7, 8, 9, 10] permissions: contents: read steps: @@ -159,7 +169,7 @@ jobs: if: steps.playwright-cache.outputs.cache-hit != 'true' run: cd desktop && pnpm exec playwright install chromium - name: Install Playwright system dependencies - run: cd desktop && pnpm exec playwright install-deps chromium + run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium - name: Save Playwright browser cache if: steps.playwright-cache.outputs.cache-hit != 'true' && github.event_name == 'push' && matrix.shard == 1 uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 @@ -169,7 +179,7 @@ jobs: - name: Desktop E2E build run: pnpm -C desktop build:e2e - name: Desktop smoke e2e - run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/4 + run: cd desktop && pnpm exec playwright test --project=smoke --shard=${{ matrix.shard }}/${{ strategy.job-total }} - name: Summarize flaky tests if: ${{ !cancelled() }} run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop Smoke E2E (${{ matrix.shard }})" @@ -190,32 +200,44 @@ jobs: name: Desktop runs-on: ubuntu-latest timeout-minutes: 5 - needs: [desktop-core, desktop-smoke-e2e, desktop-windows-build] + needs: [desktop-js, desktop-tauri, desktop-smoke-e2e, desktop-windows-build] if: always() && (github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust) permissions: contents: read steps: - name: Check desktop jobs + env: + JS_SELECTED: ${{ github.event_name == 'push' || inputs.desktop || inputs.rust }} + RUST_SELECTED: ${{ github.event_name == 'push' || inputs.desktop_rust || inputs.rust }} + FRONTEND_SELECTED: ${{ github.event_name == 'push' || inputs.desktop }} + JS: ${{ needs.desktop-js.result }} + TAURI: ${{ needs.desktop-tauri.result }} + SMOKE: ${{ needs.desktop-smoke-e2e.result }} + WINDOWS: ${{ needs.desktop-windows-build.result }} run: | - if [ "${{ needs.desktop-core.result }}" != "success" ]; then - echo "Desktop Core finished with: ${{ needs.desktop-core.result }}" - exit 1 - fi - if [ "${{ needs.desktop-smoke-e2e.result }}" != "success" ]; then - echo "Desktop Smoke E2E shards finished with: ${{ needs.desktop-smoke-e2e.result }}" - exit 1 - fi - if [ "${{ needs.desktop-windows-build.result }}" != "success" ]; then - echo "Desktop Windows Build finished with: ${{ needs.desktop-windows-build.result }}" - exit 1 - fi - echo "Desktop jobs passed" + # A selected job must succeed; an unselected job must be skipped. + status=0 + check() { + local name="$1" selected="$2" result="$3" want=skipped + [ "$selected" = true ] && want=success + if [ "$result" != "$want" ]; then + echo "$name finished with: $result (expected $want)" + status=1 + fi + } + check "Desktop JS" "$JS_SELECTED" "$JS" + check "Desktop Tauri lanes" "$RUST_SELECTED" "$TAURI" + check "Desktop Smoke E2E shards" "$FRONTEND_SELECTED" "$SMOKE" + check "Desktop Windows Build" "$FRONTEND_SELECTED" "$WINDOWS" + [ "$status" = 0 ] && echo "Desktop jobs passed" + exit "$status" desktop-windows-build: name: Desktop Windows Build runs-on: windows-latest timeout-minutes: 20 - if: github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust + # Keep in sync with FRONTEND_SELECTED in the Desktop gate (pinned by ci-required-gates.test.mjs). + if: github.event_name == 'push' || inputs.desktop permissions: contents: read steps: diff --git a/.github/workflows/_ci-relay.yml b/.github/workflows/_ci-relay.yml index 335ae00fba..b6cab48aaf 100644 --- a/.github/workflows/_ci-relay.yml +++ b/.github/workflows/_ci-relay.yml @@ -32,11 +32,26 @@ env: PLAYWRIGHT_BROWSERS_PATH: ${{ github.workspace }}/.cache/ms-playwright jobs: + # The relay binaries and the two nextest archives compile independently, so + # they build in parallel; consumers download every desktop-e2e-relay-* + # artifact into target/ci as before. desktop-e2e-relay: - name: Desktop E2E Relay + name: Desktop E2E Relay (${{ matrix.artifact }}) runs-on: ubuntu-latest timeout-minutes: 45 if: inputs.lane == 'artifacts' && (github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust) + strategy: + fail-fast: false + matrix: + include: + - artifact: relay-binaries + paths: | + target/ci/buzz-relay + target/ci/git-credential-nostr + - artifact: backend-integration-tests + paths: target/ci/backend-integration-tests.tar.zst + - artifact: postgres-tests + paths: target/ci/postgres-tests.tar.zst permissions: contents: read env: @@ -45,28 +60,27 @@ jobs: steps: - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 - uses: cashapp/activate-hermit@cea9af7913204a965fd488637a8d1811bba2e616 # v1 - # Reuse the relay binaries and backend test archive when none of their - # inputs changed (desktop-only PRs hit this every time). The key covers - # everything they embed, including migrations via sqlx migrate!. + # Reuse each artifact when none of its inputs changed (desktop-only PRs + # hit this every time). The key covers everything they embed, including + # migrations via sqlx migrate!. - name: Restore relay artifacts cache id: relay-artifacts-cache uses: actions/cache/restore@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | - target/ci/buzz-relay - target/ci/git-credential-nostr - target/ci/backend-integration-tests.tar.zst - target/ci/postgres-tests.tar.zst - key: relay-artifacts-${{ runner.os }}-${{ hashFiles('crates/**', 'migrations/**', 'Dockerfile', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/config.toml', '.config/nextest.toml', 'scripts/postgres-test-*.sh', 'scripts/check-postgres-test-discovery.py', '.github/workflows/ci.yml', '.github/workflows/_ci-*.yml') }} + path: ${{ matrix.paths }} + key: relay-artifacts-${{ matrix.artifact }}-${{ runner.os }}-${{ hashFiles('crates/**', 'migrations/**', 'Dockerfile', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/config.toml', '.config/nextest.toml', 'scripts/postgres-test-*.sh', 'scripts/check-postgres-test-discovery.py', '.github/workflows/ci.yml', '.github/workflows/_ci-*.yml') }} - uses: rui314/setup-mold@7e4f20ad28a2e8ca6fd0892ccf72e2abb706b9c3 # v1 if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' + # All lanes restore the shared Rust cache; only the postgres-tests lane, + # which compiles the widest test graph, writes it so a faster lane + # cannot save a narrower target dir under the same key. - uses: Swatinem/rust-cache@e18b497796c12c097a38f9edb9d0641fb99eee32 if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' with: workspaces: | . desktop/src-tauri - save-if: ${{ github.event_name != 'pull_request' }} + save-if: ${{ github.event_name != 'pull_request' && matrix.artifact == 'postgres-tests' }} # Cache rustc outputs for unchanged workspace crates. Trusted pushes write; # the bounded PR 5224 trial writes only to its isolated merge-ref scope. - name: Set up sccache @@ -75,16 +89,20 @@ jobs: with: version: v0.16.0 - name: Install cargo-nextest - if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' + if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact != 'relay-binaries' uses: taiki-e/install-action@0fd46367812ee04360509b4169d9f659d6892bb2 # v2.79.15 with: tool: cargo-nextest@0.9.136 - - name: Build relay artifacts - if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' + - name: Build relay binaries + if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'relay-binaries' + env: + RUSTC_WRAPPER: sccache + run: cargo build --profile ci -p buzz-relay -p git-credential-nostr + - name: Build backend integration test archive + if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'backend-integration-tests' env: RUSTC_WRAPPER: sccache run: | - cargo build --profile ci -p buzz-relay -p git-credential-nostr cargo nextest archive \ --cargo-profile ci \ -p buzz-db \ @@ -93,6 +111,11 @@ jobs: --lib \ --test e2e_event_reminder \ --archive-file target/ci/backend-integration-tests.tar.zst + - name: Build PostgreSQL test archive + if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && matrix.artifact == 'postgres-tests' + env: + RUSTC_WRAPPER: sccache + run: | postgres_package_args=() while IFS= read -r package; do postgres_package_args+=(-p "$package") @@ -113,21 +136,13 @@ jobs: if: steps.relay-artifacts-cache.outputs.cache-hit != 'true' && github.event_name == 'push' uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 with: - path: | - target/ci/buzz-relay - target/ci/git-credential-nostr - target/ci/backend-integration-tests.tar.zst - target/ci/postgres-tests.tar.zst - key: relay-artifacts-${{ runner.os }}-${{ hashFiles('crates/**', 'migrations/**', 'Dockerfile', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/config.toml', '.config/nextest.toml', 'scripts/postgres-test-*.sh', 'scripts/check-postgres-test-discovery.py', '.github/workflows/ci.yml', '.github/workflows/_ci-*.yml') }} + path: ${{ matrix.paths }} + key: relay-artifacts-${{ matrix.artifact }}-${{ runner.os }}-${{ hashFiles('crates/**', 'migrations/**', 'Dockerfile', 'Cargo.toml', 'Cargo.lock', 'rust-toolchain.toml', '.cargo/config.toml', '.config/nextest.toml', 'scripts/postgres-test-*.sh', 'scripts/check-postgres-test-discovery.py', '.github/workflows/ci.yml', '.github/workflows/_ci-*.yml') }} - name: Upload relay artifacts uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7 with: - name: desktop-e2e-relay - path: | - target/ci/buzz-relay - target/ci/git-credential-nostr - target/ci/backend-integration-tests.tar.zst - target/ci/postgres-tests.tar.zst + name: desktop-e2e-relay-${{ matrix.artifact }} + path: ${{ matrix.paths }} if-no-files-found: error retention-days: 1 @@ -196,7 +211,8 @@ jobs: - name: Download backend test archive uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: desktop-e2e-relay + pattern: desktop-e2e-relay-* + merge-multiple: true path: target/ci - name: Restore relay executable permission run: chmod +x ./target/ci/buzz-relay @@ -209,14 +225,17 @@ jobs: --archive-file target/ci/postgres-tests.tar.zst desktop-e2e-integration-shard: - name: Desktop E2E Integration (${{ matrix.shard }}/2) + name: Desktop E2E Integration (${{ matrix.shard }}/${{ strategy.job-total }}) runs-on: ubuntu-latest - timeout-minutes: 20 + # A shard needs ~10m; Playwright install-deps is bounded by + # ci-apt-retry.sh (~25m worst case), so a degraded mirror fails with its + # own error instead of the job timeout. + timeout-minutes: 40 if: inputs.lane == 'required' && (github.event_name == 'push' || inputs.desktop || inputs.desktop_rust || inputs.rust) strategy: fail-fast: false matrix: - shard: [1, 2] + shard: [1, 2, 3] permissions: contents: read steps: @@ -259,7 +278,7 @@ jobs: if: steps.playwright-cache.outputs.cache-hit != 'true' run: cd desktop && pnpm exec playwright install chromium - name: Install Playwright system dependencies - run: cd desktop && pnpm exec playwright install-deps chromium + run: cd desktop && ../scripts/ci-apt-retry.sh pnpm exec playwright install-deps chromium - name: Save Playwright browser cache if: steps.playwright-cache.outputs.cache-hit != 'true' && github.event_name == 'push' && matrix.shard == 1 uses: actions/cache/save@caa296126883cff596d87d8935842f9db880ef25 # v5 @@ -290,7 +309,8 @@ jobs: - name: Download relay binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: desktop-e2e-relay + pattern: desktop-e2e-relay-* + merge-multiple: true path: target/ci - name: Apply schema and seed deployment community # MT: the relay resolves each request's tenant from the communities host @@ -360,10 +380,10 @@ jobs: - name: Seed desktop e2e data run: bash scripts/setup-desktop-test-data.sh - name: Desktop relay-backed e2e - run: cd desktop && pnpm exec playwright test --project=integration --shard=${{ matrix.shard }}/2 + run: cd desktop && pnpm exec playwright test --project=integration --shard=${{ matrix.shard }}/${{ strategy.job-total }} - name: Summarize flaky tests if: ${{ !cancelled() }} - run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop E2E Integration (${{ matrix.shard }}/2)" + run: node scripts/summarize-flaky-tests.mjs playwright-report.json "Desktop E2E Integration (${{ matrix.shard }}/${{ strategy.job-total }})" working-directory: desktop - name: Upload desktop integration artifacts if: ${{ !cancelled() }} @@ -450,7 +470,8 @@ jobs: - name: Download relay artifacts uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: desktop-e2e-relay + pattern: desktop-e2e-relay-* + merge-multiple: true path: target/ci - name: Apply schema and seed deployment community # MT: the relay resolves each request's tenant from the communities host @@ -589,7 +610,8 @@ jobs: - name: Download relay binary uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: - name: desktop-e2e-relay + pattern: desktop-e2e-relay-* + merge-multiple: true path: target/ci - name: Start relay run: | diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e0e9268ce9..a1ff1c47fe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -64,8 +64,13 @@ jobs: - 'scripts/run-tests.sh' - 'scripts/model-capabilities.json' - 'scripts/normative-corpus.json' - - 'justfile' + - 'scripts/ci-apt-retry.sh' + - 'Justfile' desktop: + - '.github/workflows/ci.yml' + - '.github/workflows/_ci-desktop.yml' + - 'scripts/ci-apt-retry.sh' + - 'Justfile' - 'scripts/generate-macos-icon.py' - 'scripts/macos-icon.sh' - 'bin/uv' @@ -129,7 +134,7 @@ jobs: - name: CI required-context isolation contract run: scripts/test-ci-required-context-isolation.sh - name: CI path selection regression tests - run: node --test scripts/ci-selection.test.mjs scripts/ci-required-gates.test.mjs + run: node --test scripts/ci-selection.test.mjs scripts/ci-required-gates.test.mjs scripts/ci-apt-retry.test.mjs - name: File size policy run: just file-size-check diff --git a/Justfile b/Justfile index c1a68e2f3a..c85b620fd8 100644 --- a/Justfile +++ b/Justfile @@ -175,8 +175,12 @@ _ensure-sidecar-stubs: if [[ "$TARGET" != *windows* ]]; then SIDECARS+=(buzz-backend-kubernetes) fi + # Create missing stubs only: tauri-build reruns build.rs (and recompiles + # buzz-desktop) whenever an externalBin's mtime changes, so touching + # existing files would invalidate every cached desktop test build. for bin in "${SIDECARS[@]}"; do - touch "desktop/src-tauri/binaries/${bin}-${TARGET}" + stub="desktop/src-tauri/binaries/${bin}-${TARGET}" + [[ -e "$stub" ]] || : > "$stub" done # Ensure Docker dev services (Postgres, Redis, etc.) are running and healthy @@ -227,52 +231,82 @@ desktop-tauri-check: _ensure-sidecar-stubs desktop-tauri-test: _ensure-sidecar-stubs cd desktop/src-tauri && cargo test --workspace +# Run the desktop Tauri lib tests with the mesh-llm feature graph +desktop-tauri-test-mesh: _ensure-sidecar-stubs + cargo test --manifest-path {{desktop_tauri_manifest}} --features mesh-llm --lib + # Run the native terminal latency gate explicitly on a known-idle host. # This is intentionally excluded from shared CI: scheduler contention makes a # wall-clock assertion flaky, and the release profile is the shipped shape. desktop-terminal-performance-test: cargo test --manifest-path desktop/src-tauri/crates/buzz-terminal/Cargo.toml --release --test latency g3_renderer_acquire_stays_within_frame_budget -- --ignored --exact --nocapture -# Verify compiled-flag behavior under both compile states (clean + capability set). -# Runs the auto-connect and owner-only access focused tests twice with -# independently supplied expected values; build.rs rerun-if-env-changed -# triggers recompilation. +# Verify compiled-flag behavior in each compile state. +# The clean (OSS) state's full suite already runs in desktop-tauri-test, so +# only its flag assertions run here. The internal state sets both release +# capabilities together, as internal release packaging does, and reruns the +# buzz_lib suite. An auto-connect-only state asserts the two capabilities stay +# independent (a swapped build.rs variable fails it). The demo state sets only +# the demo slug, asserts neither capability leaks on, and reruns the suite. +# Every invocation uses --lib so each state compiles a single test harness; +# build.rs rerun-if-env-changed triggers the recompile between states. The +# workspace member crates and integration tests do not read these build +# variables, so they are not rerun. desktop-tauri-test-compiled-flags: _ensure-sidecar-stubs #!/usr/bin/env bash set -euo pipefail cd desktop/src-tauri + unset BUZZ_BUILD_AUTO_CONNECT_DEFAULT_RELAY BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY BUZZ_BUILD_DEMO_SLUG + unset BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY BUZZ_TEST_EXPECTED_DEMO_SLUG + # Run ignored assertions by name; libtest passes a filter that matches + # nothing, so require exactly one passing test per filter. + run_ignored() { + local out + out="$(cargo test --lib -- --ignored --nocapture "$@" 2>&1 | tee >(cat >&2))" + if ! grep -qE "^test result: ok\. $# passed;" <<<"$out"; then + echo "expected exactly $# ignored compiled-flag assertions to pass: $*" >&2 + exit 1 + fi + } + flag_tests=(compiled_flag_matches_expected compiled_policy_matches_expected) echo "=== Clean build (no flag) → expect false ===" - env -u BUZZ_BUILD_AUTO_CONNECT_DEFAULT_RELAY \ - BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=false \ - cargo test compiled_flag_matches_expected -- --ignored --nocapture - env -u BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY \ - BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=false \ - cargo test --lib - env -u BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY \ - BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=false \ - cargo test compiled_policy_matches_expected -- --ignored --nocapture + ( + export BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=false + export BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=false + run_ignored "${flag_tests[@]}" + ) echo "=== Internal build (flags set) → expect true ===" - BUZZ_BUILD_AUTO_CONNECT_DEFAULT_RELAY=1 \ - BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=true \ - cargo test compiled_flag_matches_expected -- --ignored --nocapture - BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY=1 \ - BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=true \ + ( + export BUZZ_BUILD_AUTO_CONNECT_DEFAULT_RELAY=1 + export BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY=1 + export BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=true + export BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=true + run_ignored "${flag_tests[@]}" cargo test --lib - BUZZ_BUILD_AGENT_ACCESS_OWNER_ONLY=1 \ - BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=true \ - cargo test compiled_policy_matches_expected -- --ignored --nocapture + ) + echo "=== Auto-connect only → capabilities stay independent ===" + ( + export BUZZ_BUILD_AUTO_CONNECT_DEFAULT_RELAY=1 + export BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=true + export BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=false + run_ignored "${flag_tests[@]}" + ) echo "=== Maximum accepted demo name reaches Rust build validation ===" DEMO_CONFIG="$(node ../scripts/demo-build-config.mjs "$(printf 'x%.0s' {1..31})" /dev/null 1234567812345678)" DEMO_SLUG="$(node -e 'console.log(JSON.parse(process.argv[1]).slug)' "$DEMO_CONFIG")" - BUZZ_BUILD_DEMO_SLUG="$DEMO_SLUG" \ - BUZZ_TEST_EXPECTED_DEMO_SLUG="$DEMO_SLUG" \ - cargo test compiled_demo_slug_matches_expected -- --ignored --nocapture - BUZZ_BUILD_DEMO_SLUG="$DEMO_SLUG" cargo test --workspace + ( + export BUZZ_BUILD_DEMO_SLUG="$DEMO_SLUG" + export BUZZ_TEST_EXPECTED_DEMO_SLUG="$DEMO_SLUG" + export BUZZ_TEST_EXPECTED_AUTO_CONNECT_DEFAULT_RELAY=false + export BUZZ_TEST_EXPECTED_AGENT_ACCESS_OWNER_ONLY=false + run_ignored compiled_demo_slug_matches_expected "${flag_tests[@]}" + cargo test --lib + ) if node ../scripts/demo-build-config.mjs "$(printf 'x%.0s' {1..32})" /dev/null 1234567812345678; then echo "A 32-character demo name unexpectedly passed JavaScript validation" >&2 exit 1 fi - echo "Both compiled states and the accepted/rejected demo-name boundary verified." + echo "All compiled states and the accepted/rejected demo-name boundary verified." # Build the full desktop Tauri app locally (unsigned, for testing) # Sidecar binary list must stay in sync with _ensure-sidecar-stubs above. diff --git a/lefthook.yml b/lefthook.yml index 2bc5d2c409..7b75acb715 100644 --- a/lefthook.yml +++ b/lefthook.yml @@ -13,12 +13,13 @@ # tree, not the pushed commit — the always-on `push-head-scope` lane warns when # they diverge and CI remains the authoritative gate for that ref. # Deliberate deviations: -# - The `.github/workflows/ci.yml` path CI adds to its `rust`/`mobile` filters -# is omitted; a CI-workflow-only edit doesn't need a local test run. -# - `desktop-check`/`desktop-typecheck`/`desktop-test` don't trigger on `rust` -# changes, though CI's Desktop Core job does. Those commands are pure TS -# (biome + tsc + node:test) with no Rust dependency, so the extra trigger -# would be spurious locally. +# - The `.github/workflows/ci.yml` path CI adds to its `rust`/`desktop`/`mobile` +# filters, and `scripts/ci-apt-retry.sh` in `rust`/`desktop`, are omitted; +# a CI-plumbing-only edit doesn't need a local test run. +# - `desktop-test` doesn't trigger on `crates/**` changes, though CI's Desktop +# JS job does: a few node:test cases read relay/DB crate sources to pin shared +# constants. Running the full desktop suite on every crate push would be +# expensive locally, so CI is the authoritative check for that coupling. # - The repository-wide `file-size-check` is deliberately unfiltered. Its own # merge-base diff is the path filter; duplicating its governed roots here is # the coverage drift this gate is intended to prevent. @@ -108,7 +109,7 @@ pre-push: run: ./scripts/without-git-env.sh just test-unit desktop-check: files: git diff --name-only origin/main...HEAD - glob: ["desktop/**", "pnpm-lock.yaml"] + glob: ["desktop/**", "pnpm-lock.yaml", "Justfile"] exclude: ["desktop/src-tauri/**"] env: &pnpm-verify-deps # Warn when node_modules looks stale against the workspace (pnpm uses @@ -120,18 +121,18 @@ pre-push: run: ./scripts/without-git-env.sh just desktop-check desktop-typecheck: files: git diff --name-only origin/main...HEAD - glob: ["desktop/**", "pnpm-lock.yaml"] + glob: ["desktop/**", "pnpm-lock.yaml", "Justfile"] exclude: ["desktop/src-tauri/**"] env: *pnpm-verify-deps run: ./scripts/without-git-env.sh just desktop-typecheck desktop-test: files: git diff --name-only origin/main...HEAD - glob: ["desktop/**", "pnpm-lock.yaml"] + glob: ["desktop/**", "pnpm-lock.yaml", "Justfile"] exclude: ["desktop/src-tauri/**"] env: *pnpm-verify-deps run: ./scripts/without-git-env.sh just desktop-test desktop-tauri-checks: - # Keep local lint parity with Desktop Core CI for every path that can + # Keep local lint parity with Desktop Tauri CI for every path that can # affect the Tauri crate or its path dependencies. Run clippy and tests # serially so parallel pre-push hooks do not contend for Cargo's lock. files: git diff --name-only origin/main...HEAD diff --git a/scripts/ci-apt-retry.sh b/scripts/ci-apt-retry.sh new file mode 100755 index 0000000000..2ed72eaf9a --- /dev/null +++ b/scripts/ci-apt-retry.sh @@ -0,0 +1,83 @@ +#!/usr/bin/env bash +# ============================================================================= +# ci-apt-retry.sh — run an apt-backed CI command with a per-attempt deadline +# ============================================================================= +# Ubuntu mirrors on hosted runners are sometimes degraded: most requests hit +# apt's Acquire timeouts and apt keeps re-queuing them, so one apt step can run +# past the job timeout without failing. This kills each attempt after a +# deadline, stops any apt-get/dpkg the attempt left behind, and retries. +# Downloaded .debs stay in /var/cache/apt/archives, so a retry resumes where +# the last attempt stopped. It retries the same mirror, so +# against a mirror that stays degraded this bounds each invocation (about 25m +# with the defaults) rather than recovering it. Size job timeouts to fit the +# bound of every invocation in the job plus the job's real work. +# +# Ephemeral hosted runners only: between attempts it stops every apt-get/dpkg +# on the machine, not just the ones it started. Don't use it on a self-hosted +# runner or a developer machine. +# +# It also writes the apt retry/timeout options to apt.conf.d, so commands that +# call apt-get themselves (e.g. `playwright install-deps`) pick them up too. +# +# Usage: +# scripts/ci-apt-retry.sh [args...] +# +# Env: +# CI_APT_ATTEMPTS Attempts before failing (default: 3). +# CI_APT_ATTEMPT_SECONDS Deadline per attempt (default: 420). Healthy apt +# steps finish in under 4 minutes; degraded ones run +# past 13. +# CI_APT_REPAIR_SECONDS Deadline for the dpkg repair between attempts +# (default: 60). +# ============================================================================= +set -euo pipefail + +attempts="${CI_APT_ATTEMPTS:-3}" +deadline="${CI_APT_ATTEMPT_SECONDS:-420}" +repair_deadline="${CI_APT_REPAIR_SECONDS:-60}" + +if [ "$#" -eq 0 ]; then + echo "usage: $0 [args...]" >&2 + exit 2 +fi + +printf '%s\n' \ + 'Acquire::Retries "3";' \ + 'Acquire::http::Timeout "30";' \ + 'Acquire::https::Timeout "30";' \ + 'DPkg::Lock::Timeout "120";' | + sudo tee /etc/apt/apt.conf.d/80-ci-timeouts >/dev/null + +# Intentionally machine-wide: any other apt-get/dpkg would hold the same lock. +reap_package_managers() { + sudo pkill -TERM -x 'apt-get|dpkg' || true + for _ in $(seq 1 15); do + pgrep -x 'apt-get|dpkg' >/dev/null || return 0 + sleep 1 + done + sudo pkill -KILL -x 'apt-get|dpkg' || true +} + +for attempt in $(seq 1 "$attempts"); do + rc=0 + timeout --kill-after=15s "$deadline" "$@" || rc=$? + [ "$rc" -eq 0 ] && exit 0 + if [ "$rc" -eq 124 ] || [ "$rc" -eq 137 ]; then + echo "::warning::attempt $attempt/$attempts timed out after ${deadline}s: $*" + else + echo "::warning::attempt $attempt/$attempts failed (exit $rc): $*" + fi + # timeout(1) signals only its own process group, but sudo runs its command + # in a new one. When apt-get is not sudo's direct command (playwright + # install-deps runs `sudo sh -c "apt-get update && apt-get install ..."`), + # the forwarded signal stops sh and apt-get survives, holding the dpkg lock + # so every retry is a lock wait. Stop any survivor before retrying, and + # before giving up so no apt-get outlives the step. + reap_package_managers + [ "$attempt" -eq "$attempts" ] && break + # A killed apt/dpkg run can leave dpkg half-configured; repair before + # retrying, under its own deadline so a stuck repair can't eat the job. + timeout --kill-after=15s "$repair_deadline" sudo dpkg --configure -a || true +done +echo "::error::all $attempts attempts failed: $*" +exit 1 diff --git a/scripts/ci-apt-retry.test.mjs b/scripts/ci-apt-retry.test.mjs new file mode 100644 index 0000000000..e1d40fba88 --- /dev/null +++ b/scripts/ci-apt-retry.test.mjs @@ -0,0 +1,161 @@ +import assert from "node:assert/strict"; +import { spawnSync } from "node:child_process"; +import { chmodSync, mkdtempSync, readFileSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; +import { test } from "node:test"; +import { fileURLToPath } from "node:url"; + +const script = fileURLToPath(new URL("./ci-apt-retry.sh", import.meta.url)); + +// The wrapper and these fakes rely on GNU timeout, util-linux flock/setsid and +// procps pkill/pgrep. CI runs this suite on Ubuntu; skip it elsewhere. +const skip = + process.platform !== "linux" && + "needs Linux (GNU timeout, flock, setsid, procps)"; + +// Run the wrapper with a fake `sudo` on PATH that records each call instead +// of touching the system's apt or dpkg state. +function run(args, { PATH: extraPath, ...env } = {}) { + const dir = mkdtempSync(join(tmpdir(), "ci-apt-retry-")); + const calls = join(dir, "calls"); + writeFileSync(calls, ""); + writeFileSync( + join(dir, "sudo"), + `#!/usr/bin/env bash +case "$1" in + tee) cat > "${dir}/conf" ;; + pkill) "$@" ;; + dpkg) echo repair >> "${calls}"; [ -n "\${STALL_REPAIR:-}" ] && sleep 30; true ;; + *) echo cmd >> "${calls}"; "$@" ;; +esac +`, + ); + chmodSync(join(dir, "sudo"), 0o755); + const started = Date.now(); + const result = spawnSync("bash", [script, ...args], { + encoding: "utf8", + timeout: 20_000, + env: { + ...process.env, + PATH: [dir, extraPath, process.env.PATH].filter(Boolean).join(":"), + CI_APT_ATTEMPT_SECONDS: "1", + CI_APT_REPAIR_SECONDS: "1", + ...env, + }, + }); + return { + status: result.status, + seconds: (Date.now() - started) / 1000, + calls: readFileSync(calls, "utf8").split("\n").filter(Boolean), + conf: (() => { + try { + return readFileSync(join(dir, "conf"), "utf8"); + } catch { + return ""; + } + })(), + stdout: result.stdout, + }; +} + +test("success runs the command once and writes apt options", { skip }, () => { + const r = run(["sudo", "true"]); + assert.equal(r.status, 0); + assert.deepEqual(r.calls, ["cmd"]); + assert.match(r.conf, /Acquire::https::Timeout "30";/); +}); + +test("a failed attempt is repaired and retried", { skip }, () => { + const dir = mkdtempSync(join(tmpdir(), "ci-apt-retry-flaky-")); + const marker = join(dir, "seen"); + const r = run([ + "sudo", + "bash", + "-c", + `[ -e ${marker} ] || { touch ${marker}; exit 1; }`, + ]); + assert.equal(r.status, 0); + assert.deepEqual(r.calls, ["cmd", "repair", "cmd"]); +}); + +test("persistent failure stops after the attempt budget, without a final repair", { skip }, () => { + const r = run(["sudo", "false"]); + assert.equal(r.status, 1); + assert.deepEqual(r.calls, ["cmd", "repair", "cmd", "repair", "cmd"]); + assert.match(r.stdout, /::error::all 3 attempts failed/); +}); + +test("a stalled attempt is killed at its deadline and retried", { skip }, () => { + const r = run(["sudo", "sleep", "30"], { CI_APT_ATTEMPTS: "2" }); + assert.equal(r.status, 1); + assert.deepEqual(r.calls, ["cmd", "repair", "cmd"]); + assert.match(r.stdout, /timed out after 1s/); + assert.ok(r.seconds < 10, `took ${r.seconds}s`); +}); + +test("a stalled dpkg repair is bounded and the next attempt still runs", { skip }, () => { + const r = run(["sudo", "bash", "-c", "exit 23"], { + CI_APT_ATTEMPTS: "2", + STALL_REPAIR: "1", + }); + assert.equal(r.status, 1); + assert.deepEqual(r.calls, ["cmd", "repair", "cmd"]); + assert.ok(r.seconds < 10, `took ${r.seconds}s`); +}); + +test("no command is a usage error", { skip }, () => { + assert.equal(run([]).status, 2); +}); + +// playwright install-deps reaches apt-get through `sudo sh -c "..."`, and sudo +// runs that in a new process group, so timeout(1)'s group signal misses it. +// Model that: the first attempt starts an `apt-get` in its own session that +// holds a lock and blocks; the attempt's own process exits on TERM. The retry +// must find the lock free, so the survivor has to have been stopped. +// Writes a fake `apt-get` that takes a lock: the first call holds it and +// blocks, later calls succeed only if the lock is free. +function escapingAptGet() { + const dir = mkdtempSync(join(tmpdir(), "ci-apt-retry-escape-")); + const lock = join(dir, "lock"); + writeFileSync( + join(dir, "apt-get"), + // #!/bin/bash, not env, so the process is named apt-get. + `#!/bin/bash +exec 9>"${lock}" +flock -n 9 || { echo "lock held"; exit 1; } +[ -e "${dir}/seen" ] && exit 0 +touch "${dir}/seen" +mkfifo "${dir}/fifo"; exec 8<>"${dir}/fifo" +read -r -t 30 -u 8 || true +`, + ); + chmodSync(join(dir, "apt-get"), 0o755); + const lockIsFree = () => + spawnSync("flock", ["-n", lock, "true"]).status === 0; + return { dir, lockIsFree }; +} + +test("a package manager that escapes the attempt's process group is stopped before the retry", { skip }, () => { + const { dir } = escapingAptGet(); + const r = run(["bash", "-c", "setsid apt-get & wait $!"], { + CI_APT_ATTEMPTS: "2", + PATH: dir, + }); + assert.equal(r.status, 0, r.stdout); + assert.match(r.stdout, /attempt 1\/2 timed out/); + assert.doesNotMatch(r.stdout, /lock held/); + assert.ok(r.seconds < 10, `took ${r.seconds}s`); +}); + +test("a package manager that escapes the final attempt is stopped before the wrapper exits", { skip }, () => { + const { dir, lockIsFree } = escapingAptGet(); + const r = run(["bash", "-c", "setsid apt-get & wait $!"], { + CI_APT_ATTEMPTS: "1", + PATH: dir, + }); + assert.equal(r.status, 1, r.stdout); + assert.deepEqual(r.calls, []); + assert.ok(lockIsFree(), "apt-get outlived the wrapper"); + assert.ok(r.seconds < 10, `took ${r.seconds}s`); +}); diff --git a/scripts/ci-required-gates.test.mjs b/scripts/ci-required-gates.test.mjs index e495a452ca..db0bb38480 100644 --- a/scripts/ci-required-gates.test.mjs +++ b/scripts/ci-required-gates.test.mjs @@ -88,3 +88,61 @@ for (const gate of gates) { } }); } + +// The Desktop gate re-derives which of its needs were selected. Each job's +// `if:` must match the gate's expression, or a selected job could be skipped +// (or an unselected one run) with the gate's expectation out of step. +const desktopWorkflow = readFileSync( + new URL("../.github/workflows/_ci-desktop.yml", import.meta.url), + "utf8", +); +function desktopJob(name) { + return desktopWorkflow.match( + new RegExp(`^ ${name}:\\n([\\s\\S]*?)(?=^ [\\w-]+:|$(?![\\s\\S]))`, "m"), + )[1]; +} +const desktopGate = desktopJob("desktop"); +const gateSelection = (key) => + desktopGate.match(new RegExp(`^ {10}${key}: \\$\\{\\{ (.+) \\}\\}$`, "m"))[1]; +const desktopJobs = { + "desktop-js": "JS_SELECTED", + "desktop-tauri": "RUST_SELECTED", + "desktop-smoke-e2e": "FRONTEND_SELECTED", + "desktop-windows-build": "FRONTEND_SELECTED", +}; +function evaluate(expression, event, inputs) { + const substituted = expression.replace( + /github\.event_name|inputs\.([\w_]+)/g, + (key, input) => + input === undefined ? JSON.stringify(event) : String(inputs[input] === true), + ); + return runInNewContext(substituted, {}, { timeout: 100 }); +} +test("desktop gate selection matches each job's if: condition", () => { + for (const [job, key] of Object.entries(desktopJobs)) { + const condition = desktopJob(job).match(/^ {4}if: (.+)$/m)[1]; + assert.equal(condition, gateSelection(key), `${job} vs ${key}`); + } +}); +test("desktop gate selects jobs per changed-path input", () => { + const selected = (event, inputs) => + Object.keys(desktopJobs) + .filter((job) => evaluate(gateSelection(desktopJobs[job]), event, inputs)) + .sort(); + // Rust-only: Desktop JS still runs, because node:test pins relay/DB constants. + assert.deepEqual(selected("pull_request", { rust: true }), [ + "desktop-js", + "desktop-tauri", + ]); + assert.deepEqual(selected("pull_request", { desktop: true }), [ + "desktop-js", + "desktop-smoke-e2e", + "desktop-windows-build", + ]); + assert.deepEqual( + selected("pull_request", { desktop: true, desktop_rust: true }), + Object.keys(desktopJobs).sort(), + ); + assert.deepEqual(selected("push", {}), Object.keys(desktopJobs).sort()); + assert.deepEqual(selected("pull_request", {}), []); +}); diff --git a/scripts/ci-selection.test.mjs b/scripts/ci-selection.test.mjs index 6639063384..0ce88ef067 100644 --- a/scripts/ci-selection.test.mjs +++ b/scripts/ci-selection.test.mjs @@ -222,7 +222,18 @@ const scenarios = [ ["Tauri", ["desktop/src-tauri/src/main.rs"], ["desktop", "desktop-rust"]], ["relay", ["crates/buzz-relay/src/main.rs"], ["rust"]], ["migration", ["migrations/123.sql"], ["rust"]], - ["shared workflow", [".github/workflows/ci.yml"], ["rust", "mobile"]], + [ + "shared workflow", + [".github/workflows/ci.yml"], + ["desktop", "mobile", "rust"], + ], + [ + "desktop workflow", + [".github/workflows/_ci-desktop.yml"], + ["desktop", "mobile", "rust"], + ], + ["Justfile", ["Justfile"], ["desktop", "rust"]], + ["CI apt retry", ["scripts/ci-apt-retry.sh"], ["desktop", "rust"]], ["integration services", ["docker-compose.yml"], ["rust"]], ["CI integration images", ["docker-compose.ci.yml"], ["rust"]], [