diff --git a/.github/workflows/deploy-mainnet.yml b/.github/workflows/deploy-mainnet.yml new file mode 100644 index 0000000..6bd3d3f --- /dev/null +++ b/.github/workflows/deploy-mainnet.yml @@ -0,0 +1,63 @@ +name: Deploy mainnet + +# CD for the vector-marketplace host (91.98.147.172). +# +# Trigger: automatically after the "CI" workflow succeeds on main, or manually +# via workflow_dispatch (Actions tab -> Deploy mainnet -> Run workflow). +# +# The runner only SSHes in and executes deploy/mainnet/deploy.sh AT THE NEW +# COMMIT (fetched server-side via `git show FETCH_HEAD:`), so deploy logic is +# versioned with the code it deploys. The script itself decides which compose +# projects to rebuild based on the old..new diff, drains busy suppliers, and +# health-gates every restart — see deploy/mainnet/deploy.sh. +# +# Secrets: DEPLOY_SSH_KEY — ed25519 private key whose public half is in +# root@vector-marketplace:~/.ssh/authorized_keys. + +on: + workflow_run: + workflows: ["CI"] + types: [completed] + branches: [main] + workflow_dispatch: + inputs: + force: + description: "Redeploy even if the server is already at origin/main" + type: boolean + default: false + +concurrency: + group: deploy-mainnet + cancel-in-progress: false # never kill a deploy mid-rollout + +env: + DEPLOY_HOST: 91.98.147.172 + DEPLOY_USER: root + # Pinned host key (ssh-keyscan -t ed25519 91.98.147.172) + DEPLOY_HOSTKEY: "91.98.147.172 ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIJvjM8D+pNUb2Qr4yJbqVcRrUzbpbfyv0/tupUgD5Th0" + +jobs: + deploy: + name: deploy to vector-marketplace + runs-on: ubuntu-latest + # workflow_run fires on failure too — only deploy green main. + if: github.event_name == 'workflow_dispatch' || github.event.workflow_run.conclusion == 'success' + steps: + - name: Set up SSH + run: | + install -m 700 -d ~/.ssh + printf '%s\n' "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + printf '%s\n' "$DEPLOY_HOSTKEY" > ~/.ssh/known_hosts + + - name: Run deploy script on server + run: | + FORCE=0 + [ "${{ github.event.inputs.force }}" = "true" ] && FORCE=1 + ssh -o BatchMode=yes "$DEPLOY_USER@$DEPLOY_HOST" " + set -euo pipefail + cd /root/agents-marketplace + git fetch origin main + git show FETCH_HEAD:deploy/mainnet/deploy.sh > /run/marketplace-deploy.sh + FORCE=$FORCE bash /run/marketplace-deploy.sh + " diff --git a/deploy/README.md b/deploy/README.md index cb18e58..644d7c6 100644 --- a/deploy/README.md +++ b/deploy/README.md @@ -289,3 +289,34 @@ For the M1-F-2 (LiveOgmiosProvider) handoff: the supplier compose already joins `apex-dashboard_apex-net` and overrides `OGMIOS_URL=ws://ogmios:1337`, so once the supplier code stops booting `ReadOnlyOgmiosProvider` and starts talking to a real Ogmios, no compose changes are needed. + +## 10. Continuous deployment (mainnet, vector-marketplace host) + +Merges to `main` deploy automatically to the mainnet host once CI is green: + +1. `.github/workflows/ci.yml` runs typecheck + vitest on the merge commit. +2. On success, `.github/workflows/deploy-mainnet.yml` SSHes into the host + (secret `DEPLOY_SSH_KEY`, host key pinned in the workflow) and executes + `deploy/mainnet/deploy.sh` **at the new commit** (`git show FETCH_HEAD:...`), + so deploy logic always matches the code being deployed. +3. The script hard-resets `/root/agents-marketplace` to `origin/main`, maps the + old..new diff to affected compose projects (changes under `packages/`, + `patches/`, `contracts/` or root manifests rebuild everything), builds all + affected images up front, then does a rolling `up -d` with a health gate per + project. Suppliers are drained first: `/status` is polled until not + `working` (up to `DRAIN_TIMEOUT_SECS`, default 600) so an in-flight job + isn't killed and its escrow bond forfeited. + +Manual controls: + +- **Manual deploy / redeploy**: Actions → "Deploy mainnet" → Run workflow + (check *force* to redeploy the same sha), or on the host: + `bash /root/agents-marketplace/deploy/mainnet/deploy.sh` (`FORCE=1` to + redeploy, `DRY_RUN=1` to preview which projects would restart). +- **Rollback**: `cd /root/agents-marketplace && git reset --hard && + FORCE=1 bash deploy/mainnet/deploy.sh`. Every deploy appends + `old -> new (projects)` to `/var/log/marketplace-deploy.log`. + +Projects with no running containers on the host (e.g. wallet-monitor) are +skipped; image-only projects (ollama, chatmock, tts-piper) redeploy only when +their own compose file changes. diff --git a/deploy/mainnet/deploy.sh b/deploy/mainnet/deploy.sh new file mode 100755 index 0000000..3ddf5b7 --- /dev/null +++ b/deploy/mainnet/deploy.sh @@ -0,0 +1,178 @@ +#!/usr/bin/env bash +# deploy/mainnet/deploy.sh — pull-based CD for the vector-marketplace host. +# +# Runs ON the server (root@vector-marketplace, repo at /root/agents-marketplace). +# Invoked by .github/workflows/deploy-mainnet.yml over SSH, but safe to run by +# hand: `bash deploy/mainnet/deploy.sh`. +# +# What it does: +# 1. Fetches origin/main and hard-resets the checkout to it. +# 2. Diffs old..new to work out which compose projects actually need a +# rebuild (gateway vs suppliers vs buyer vs indexer ...). Changes under +# packages/, patches/, contracts/ or the root manifests rebuild everything, +# because every Dockerfile COPYs those into its build context. +# 3. Builds affected images first (all 17 supplier projects share one +# Dockerfile/context, so build #2..#17 are layer-cache hits), THEN does a +# rolling `up -d` so the downtime window is restart-only, not build+restart. +# 4. Suppliers are drained before recreate: /status is polled until the +# supplier is not "working" (a busy supplier killed mid-job forfeits its +# escrow bond). After DRAIN_TIMEOUT_SECS we recreate anyway, so a wedged +# supplier cannot block the deploy forever. +# 5. Every restarted container must report healthy (or plain running when it +# has no healthcheck) within HEALTH_TIMEOUT_SECS or the deploy fails. +# +# Env knobs: +# FORCE=1 redeploy even if already at origin/main +# DRY_RUN=1 print what would be rebuilt/restarted, change nothing +# DRAIN_TIMEOUT_SECS=N max wait per busy supplier (default 600, 0 = skip) +# HEALTH_TIMEOUT_SECS=N max wait for healthy (default 180) +# +# Rollback: git reset --hard && FORCE=1 bash deploy/mainnet/deploy.sh +# (old sha is printed below and appended to /var/log/marketplace-deploy.log). + +set -euo pipefail + +REPO=/root/agents-marketplace +COMPOSE_DIR="$REPO/deploy/mainnet" +LOCK=/run/marketplace-deploy.lock +LOG=/var/log/marketplace-deploy.log +DRAIN_TIMEOUT_SECS="${DRAIN_TIMEOUT_SECS:-600}" +HEALTH_TIMEOUT_SECS="${HEALTH_TIMEOUT_SECS:-180}" +FORCE="${FORCE:-0}" +DRY_RUN="${DRY_RUN:-0}" + +exec 9>"$LOCK" +flock -n 9 || { echo "another deploy is already running (lock: $LOCK)"; exit 1; } + +cd "$REPO" + +OLD=$(git rev-parse HEAD) +git fetch origin main +NEW=$(git rev-parse FETCH_HEAD) + +if [ "$OLD" = "$NEW" ] && [ "$FORCE" != "1" ]; then + echo "already at origin/main ($NEW) — nothing to deploy (FORCE=1 to override)" + exit 0 +fi + +echo "deploying $OLD -> $NEW" +if [ "$DRY_RUN" = "1" ]; then + echo "(dry run — checkout not reset, nothing rebuilt or restarted)" +else + git reset --hard FETCH_HEAD +fi + +CHANGED=$(git diff --name-only "$OLD" "$NEW" || true) +[ "$FORCE" = "1" ] && [ -z "$CHANGED" ] && CHANGED="(forced)" + +# Paths COPY'd into every image's build context (see */Dockerfile). +SHARED_RE='^(packages/|patches/|contracts/|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|tsconfig\.base\.json$|\.npmrc$)' + +changed_matches() { # changed_matches -> 0 if any changed path matches + [ "$FORCE" = "1" ] && return 0 + echo "$CHANGED" | grep -qE "$1" +} + +# Compose project -> the source prefixes that require rebuilding its image. +# Image-only projects (ollama, chatmock, tts-piper) redeploy only when their +# own compose file changes. +project_re() { + case "$1" in + docker-compose.indexer.yml) echo "^indexer/|^indexer-ui/|$SHARED_RE" ;; + docker-compose.buyer.yml) echo "^buyer/|$SHARED_RE" ;; + docker-compose.gateway.yml) echo "^gateway/|$SHARED_RE" ;; + docker-compose.wallet-monitor.yml) echo "^wallet-monitor/|$SHARED_RE" ;; + docker-compose.supplier*.yml) echo "^supplier/|$SHARED_RE" ;; + *) echo "" ;; + esac +} + +# Restart order: infra first, suppliers last (each supplier drains first). +ORDERED_NON_SUPPLIER="docker-compose.indexer.yml docker-compose.buyer.yml docker-compose.gateway.yml docker-compose.wallet-monitor.yml docker-compose.ollama.yml docker-compose.chatmock.yml docker-compose.tts-piper.yml" +SUPPLIERS=$(cd "$COMPOSE_DIR" && ls docker-compose.supplier*.yml) + +AFFECTED=() +for f in $ORDERED_NON_SUPPLIER $SUPPLIERS; do + [ -f "$COMPOSE_DIR/$f" ] || continue + re=$(project_re "$f") + hit=0 + [ -n "$re" ] && changed_matches "$re" && hit=1 + changed_matches "^deploy/mainnet/$f\$" && hit=1 + [ "$hit" = "1" ] || continue + if [ -z "$(docker compose -f "$COMPOSE_DIR/$f" ps -q 2>/dev/null)" ]; then + echo "skip $f — affected but no running containers on this host" + continue + fi + AFFECTED+=("$f") +done + +if [ "${#AFFECTED[@]}" -eq 0 ]; then + echo "no running compose project is affected by $OLD..$NEW — done" + echo "$(date -Is) $OLD -> $NEW (no services affected)" >> "$LOG" + exit 0 +fi + +echo "affected projects: ${AFFECTED[*]}" + +if [ "$DRY_RUN" = "1" ]; then + echo "(dry run — stopping before build/rollout)" + exit 0 +fi + +echo "== build phase ==" +for f in "${AFFECTED[@]}"; do + echo "-- building $f" + docker compose -f "$COMPOSE_DIR/$f" build +done + +drain_supplier() { # drain_supplier + [ "$DRAIN_TIMEOUT_SECS" = "0" ] && return 0 + local cid deadline status + cid=$(docker compose -f "$COMPOSE_DIR/$1" ps -q | head -1) + [ -n "$cid" ] || return 0 + deadline=$(( $(date +%s) + DRAIN_TIMEOUT_SECS )) + while [ "$(date +%s)" -lt "$deadline" ]; do + status=$(docker exec "$cid" wget -qO- http://localhost:8080/status 2>/dev/null \ + | grep -o '"status":"[^"]*"' | head -1 | cut -d'"' -f4 || true) + if [ "$status" != "working" ]; then + return 0 + fi + echo " supplier busy (status=working) — waiting for job to finish..." + sleep 15 + done + echo " WARNING: still busy after ${DRAIN_TIMEOUT_SECS}s — recreating anyway" +} + +wait_healthy() { # wait_healthy + local deadline cid state + deadline=$(( $(date +%s) + HEALTH_TIMEOUT_SECS )) + for cid in $(docker compose -f "$COMPOSE_DIR/$1" ps -q); do + while :; do + state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid") + case "$state" in + healthy|running) break ;; + starting|unhealthy|created|restarting) + if [ "$(date +%s)" -ge "$deadline" ]; then + echo " FAILED: $(docker inspect -f '{{.Name}}' "$cid") is '$state' after ${HEALTH_TIMEOUT_SECS}s; last logs:" + docker logs --tail 50 "$cid" 2>&1 | sed 's/^/ | /' + return 1 + fi + sleep 5 ;; + *) echo " FAILED: container state '$state'"; docker logs --tail 50 "$cid" 2>&1 | sed 's/^/ | /'; return 1 ;; + esac + done + done +} + +echo "== rollout phase ==" +for f in "${AFFECTED[@]}"; do + echo "-- rolling out $f" + case "$f" in docker-compose.supplier*) drain_supplier "$f" ;; esac + docker compose -f "$COMPOSE_DIR/$f" up -d + wait_healthy "$f" + echo " ok" +done + +echo "$(date -Is) $OLD -> $NEW (${AFFECTED[*]})" >> "$LOG" +echo "== deploy complete: $NEW ==" +echo "rollback: cd $REPO && git reset --hard $OLD && FORCE=1 bash deploy/mainnet/deploy.sh"