feat(deploy): CI-gated continuous deployment to the vector-marketplace host
Merges to main now deploy automatically once CI passes: - .github/workflows/deploy-mainnet.yml: fires on successful CI runs of main (or manually via workflow_dispatch with an optional force input), SSHes into the mainnet host with the DEPLOY_SSH_KEY secret (host key pinned) and runs deploy/mainnet/deploy.sh at the new commit via git show FETCH_HEAD:, so deploy logic ships with the code it deploys. - deploy/mainnet/deploy.sh: hard-resets /root/agents-marketplace to origin/main, maps the old..new diff to affected compose projects (packages/, patches/, contracts/ and root manifests rebuild all image builds; the 17 supplier projects share one Dockerfile so builds after the first are cache hits), builds everything up front, then does a rolling up -d with a per-project health gate. Suppliers drain first: /status is polled until not "working" (DRAIN_TIMEOUT_SECS, default 600s) so an in-flight job is not killed and its bond forfeited. FORCE=1 redeploys the current sha, DRY_RUN=1 previews the plan, and each deploy logs old -> new to /var/log/marketplace-deploy.log with a printed rollback one-liner. Verified: DRY_RUN=1 on the host against the pending 678ee86..6156900 delta selects indexer, buyer, gateway and all 17 suppliers, skips the non-running wallet-monitor, and leaves image-only projects untouched. Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
david
178
deploy/mainnet/deploy.sh
Executable file
178
deploy/mainnet/deploy.sh
Executable file
@@ -0,0 +1,178 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy/mainnet/deploy.sh — pull-based CD for the vector-marketplace host.
|
||||
#
|
||||
# Runs ON the server (root@vector-marketplace, repo at /root/agents-marketplace).
|
||||
# Invoked by .github/workflows/deploy-mainnet.yml over SSH, but safe to run by
|
||||
# hand: `bash deploy/mainnet/deploy.sh`.
|
||||
#
|
||||
# What it does:
|
||||
# 1. Fetches origin/main and hard-resets the checkout to it.
|
||||
# 2. Diffs old..new to work out which compose projects actually need a
|
||||
# rebuild (gateway vs suppliers vs buyer vs indexer ...). Changes under
|
||||
# packages/, patches/, contracts/ or the root manifests rebuild everything,
|
||||
# because every Dockerfile COPYs those into its build context.
|
||||
# 3. Builds affected images first (all 17 supplier projects share one
|
||||
# Dockerfile/context, so build #2..#17 are layer-cache hits), THEN does a
|
||||
# rolling `up -d` so the downtime window is restart-only, not build+restart.
|
||||
# 4. Suppliers are drained before recreate: /status is polled until the
|
||||
# supplier is not "working" (a busy supplier killed mid-job forfeits its
|
||||
# escrow bond). After DRAIN_TIMEOUT_SECS we recreate anyway, so a wedged
|
||||
# supplier cannot block the deploy forever.
|
||||
# 5. Every restarted container must report healthy (or plain running when it
|
||||
# has no healthcheck) within HEALTH_TIMEOUT_SECS or the deploy fails.
|
||||
#
|
||||
# Env knobs:
|
||||
# FORCE=1 redeploy even if already at origin/main
|
||||
# DRY_RUN=1 print what would be rebuilt/restarted, change nothing
|
||||
# DRAIN_TIMEOUT_SECS=N max wait per busy supplier (default 600, 0 = skip)
|
||||
# HEALTH_TIMEOUT_SECS=N max wait for healthy (default 180)
|
||||
#
|
||||
# Rollback: git reset --hard <old-sha> && FORCE=1 bash deploy/mainnet/deploy.sh
|
||||
# (old sha is printed below and appended to /var/log/marketplace-deploy.log).
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
REPO=/root/agents-marketplace
|
||||
COMPOSE_DIR="$REPO/deploy/mainnet"
|
||||
LOCK=/run/marketplace-deploy.lock
|
||||
LOG=/var/log/marketplace-deploy.log
|
||||
DRAIN_TIMEOUT_SECS="${DRAIN_TIMEOUT_SECS:-600}"
|
||||
HEALTH_TIMEOUT_SECS="${HEALTH_TIMEOUT_SECS:-180}"
|
||||
FORCE="${FORCE:-0}"
|
||||
DRY_RUN="${DRY_RUN:-0}"
|
||||
|
||||
exec 9>"$LOCK"
|
||||
flock -n 9 || { echo "another deploy is already running (lock: $LOCK)"; exit 1; }
|
||||
|
||||
cd "$REPO"
|
||||
|
||||
OLD=$(git rev-parse HEAD)
|
||||
git fetch origin main
|
||||
NEW=$(git rev-parse FETCH_HEAD)
|
||||
|
||||
if [ "$OLD" = "$NEW" ] && [ "$FORCE" != "1" ]; then
|
||||
echo "already at origin/main ($NEW) — nothing to deploy (FORCE=1 to override)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "deploying $OLD -> $NEW"
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "(dry run — checkout not reset, nothing rebuilt or restarted)"
|
||||
else
|
||||
git reset --hard FETCH_HEAD
|
||||
fi
|
||||
|
||||
CHANGED=$(git diff --name-only "$OLD" "$NEW" || true)
|
||||
[ "$FORCE" = "1" ] && [ -z "$CHANGED" ] && CHANGED="(forced)"
|
||||
|
||||
# Paths COPY'd into every image's build context (see */Dockerfile).
|
||||
SHARED_RE='^(packages/|patches/|contracts/|package\.json$|pnpm-lock\.yaml$|pnpm-workspace\.yaml$|tsconfig\.base\.json$|\.npmrc$)'
|
||||
|
||||
changed_matches() { # changed_matches <ere> -> 0 if any changed path matches
|
||||
[ "$FORCE" = "1" ] && return 0
|
||||
echo "$CHANGED" | grep -qE "$1"
|
||||
}
|
||||
|
||||
# Compose project -> the source prefixes that require rebuilding its image.
|
||||
# Image-only projects (ollama, chatmock, tts-piper) redeploy only when their
|
||||
# own compose file changes.
|
||||
project_re() {
|
||||
case "$1" in
|
||||
docker-compose.indexer.yml) echo "^indexer/|^indexer-ui/|$SHARED_RE" ;;
|
||||
docker-compose.buyer.yml) echo "^buyer/|$SHARED_RE" ;;
|
||||
docker-compose.gateway.yml) echo "^gateway/|$SHARED_RE" ;;
|
||||
docker-compose.wallet-monitor.yml) echo "^wallet-monitor/|$SHARED_RE" ;;
|
||||
docker-compose.supplier*.yml) echo "^supplier/|$SHARED_RE" ;;
|
||||
*) echo "" ;;
|
||||
esac
|
||||
}
|
||||
|
||||
# Restart order: infra first, suppliers last (each supplier drains first).
|
||||
ORDERED_NON_SUPPLIER="docker-compose.indexer.yml docker-compose.buyer.yml docker-compose.gateway.yml docker-compose.wallet-monitor.yml docker-compose.ollama.yml docker-compose.chatmock.yml docker-compose.tts-piper.yml"
|
||||
SUPPLIERS=$(cd "$COMPOSE_DIR" && ls docker-compose.supplier*.yml)
|
||||
|
||||
AFFECTED=()
|
||||
for f in $ORDERED_NON_SUPPLIER $SUPPLIERS; do
|
||||
[ -f "$COMPOSE_DIR/$f" ] || continue
|
||||
re=$(project_re "$f")
|
||||
hit=0
|
||||
[ -n "$re" ] && changed_matches "$re" && hit=1
|
||||
changed_matches "^deploy/mainnet/$f\$" && hit=1
|
||||
[ "$hit" = "1" ] || continue
|
||||
if [ -z "$(docker compose -f "$COMPOSE_DIR/$f" ps -q 2>/dev/null)" ]; then
|
||||
echo "skip $f — affected but no running containers on this host"
|
||||
continue
|
||||
fi
|
||||
AFFECTED+=("$f")
|
||||
done
|
||||
|
||||
if [ "${#AFFECTED[@]}" -eq 0 ]; then
|
||||
echo "no running compose project is affected by $OLD..$NEW — done"
|
||||
echo "$(date -Is) $OLD -> $NEW (no services affected)" >> "$LOG"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "affected projects: ${AFFECTED[*]}"
|
||||
|
||||
if [ "$DRY_RUN" = "1" ]; then
|
||||
echo "(dry run — stopping before build/rollout)"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
echo "== build phase =="
|
||||
for f in "${AFFECTED[@]}"; do
|
||||
echo "-- building $f"
|
||||
docker compose -f "$COMPOSE_DIR/$f" build
|
||||
done
|
||||
|
||||
drain_supplier() { # drain_supplier <compose-file>
|
||||
[ "$DRAIN_TIMEOUT_SECS" = "0" ] && return 0
|
||||
local cid deadline status
|
||||
cid=$(docker compose -f "$COMPOSE_DIR/$1" ps -q | head -1)
|
||||
[ -n "$cid" ] || return 0
|
||||
deadline=$(( $(date +%s) + DRAIN_TIMEOUT_SECS ))
|
||||
while [ "$(date +%s)" -lt "$deadline" ]; do
|
||||
status=$(docker exec "$cid" wget -qO- http://localhost:8080/status 2>/dev/null \
|
||||
| grep -o '"status":"[^"]*"' | head -1 | cut -d'"' -f4 || true)
|
||||
if [ "$status" != "working" ]; then
|
||||
return 0
|
||||
fi
|
||||
echo " supplier busy (status=working) — waiting for job to finish..."
|
||||
sleep 15
|
||||
done
|
||||
echo " WARNING: still busy after ${DRAIN_TIMEOUT_SECS}s — recreating anyway"
|
||||
}
|
||||
|
||||
wait_healthy() { # wait_healthy <compose-file>
|
||||
local deadline cid state
|
||||
deadline=$(( $(date +%s) + HEALTH_TIMEOUT_SECS ))
|
||||
for cid in $(docker compose -f "$COMPOSE_DIR/$1" ps -q); do
|
||||
while :; do
|
||||
state=$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}{{.State.Status}}{{end}}' "$cid")
|
||||
case "$state" in
|
||||
healthy|running) break ;;
|
||||
starting|unhealthy|created|restarting)
|
||||
if [ "$(date +%s)" -ge "$deadline" ]; then
|
||||
echo " FAILED: $(docker inspect -f '{{.Name}}' "$cid") is '$state' after ${HEALTH_TIMEOUT_SECS}s; last logs:"
|
||||
docker logs --tail 50 "$cid" 2>&1 | sed 's/^/ | /'
|
||||
return 1
|
||||
fi
|
||||
sleep 5 ;;
|
||||
*) echo " FAILED: container state '$state'"; docker logs --tail 50 "$cid" 2>&1 | sed 's/^/ | /'; return 1 ;;
|
||||
esac
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
echo "== rollout phase =="
|
||||
for f in "${AFFECTED[@]}"; do
|
||||
echo "-- rolling out $f"
|
||||
case "$f" in docker-compose.supplier*) drain_supplier "$f" ;; esac
|
||||
docker compose -f "$COMPOSE_DIR/$f" up -d
|
||||
wait_healthy "$f"
|
||||
echo " ok"
|
||||
done
|
||||
|
||||
echo "$(date -Is) $OLD -> $NEW (${AFFECTED[*]})" >> "$LOG"
|
||||
echo "== deploy complete: $NEW =="
|
||||
echo "rollback: cd $REPO && git reset --hard $OLD && FORCE=1 bash deploy/mainnet/deploy.sh"
|
||||
Reference in New Issue
Block a user