deploy: automatically test and release pushed main commits
This commit is contained in:
syntaxbullet
106
deploy/deploy.sh
Normal file
106
deploy/deploy.sh
Normal file
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Installed on the server as /usr/local/bin/minabot-deploy.
|
||||
set -Eeuo pipefail
|
||||
umask 077
|
||||
root=${MINABOT_DEPLOY_ROOT:-/opt/minabot}
|
||||
cd "$root"
|
||||
exec 9>deploy.lock
|
||||
flock -n 9 || exit 0
|
||||
export GIT_SSH_COMMAND="ssh -i $root/git-auth/id_ed25519 -o IdentitiesOnly=yes -o BatchMode=yes -o StrictHostKeyChecking=yes -o UserKnownHostsFile=$root/git-auth/known_hosts -o ConnectTimeout=10 -o ServerAliveInterval=15 -o ServerAliveCountMax=2"
|
||||
repo="$root/git-sync.git"
|
||||
target=''
|
||||
switching=false
|
||||
schema_before=''
|
||||
previous_image=''
|
||||
docker_cmd() { sudo -n docker "$@" </dev/null; }
|
||||
compose() { docker_cmd compose --project-directory "$root" -f "$root/compose.yaml" --env-file "$root/.env" "$@"; }
|
||||
schema_fingerprint() {
|
||||
docker_cmd run --rm --network none -v /srv/minabot/data:/data:ro "$1" bun -e '
|
||||
import {Database} from "bun:sqlite";
|
||||
const db=new Database("/data/minabot.sqlite",{readonly:true});
|
||||
const schema=db.query("SELECT type,name,sql FROM sqlite_master WHERE sql IS NOT NULL ORDER BY type,name").all();
|
||||
const migrations=db.query("SELECT * FROM __drizzle_migrations ORDER BY id").all();
|
||||
console.log(new Bun.CryptoHasher("sha256").update(JSON.stringify({schema,migrations})).digest("hex"));
|
||||
db.close();'
|
||||
}
|
||||
failed() {
|
||||
local result=$?
|
||||
trap - ERR
|
||||
set +e
|
||||
printf 'Deployment failed: %s (exit %s)\n' "${target:-fetch}" "$result" >&2
|
||||
printf 'Build and test logs: %s\n' "${release:-$root}" >&2
|
||||
if [[ -n "$target" ]]; then printf '%s\n' "$target" > failed-sha; fi
|
||||
if $switching; then
|
||||
compose stop caddy app monitor
|
||||
local schema_after
|
||||
schema_after=$(schema_fingerprint "$previous_image")
|
||||
if [[ -n "$schema_before" && "$schema_before" == "$schema_after" ]]; then
|
||||
cp "$rollback/.env" .env
|
||||
cp "$rollback/compose.yaml" compose.yaml
|
||||
cp "$rollback/Caddyfile" Caddyfile
|
||||
if compose up -d --wait --wait-timeout 120; then
|
||||
echo 'Previous release restored; database was not replaced.' >&2
|
||||
else
|
||||
echo 'Rollback failed; inspect the journal and saved release configuration.' >&2
|
||||
fi
|
||||
else
|
||||
echo "Database migrations changed or could not be verified. Services remain stopped; inspect $rollback and /srv/minabot/data/backups before recovery. No database was overwritten." >&2
|
||||
fi
|
||||
fi
|
||||
exit "$result"
|
||||
}
|
||||
trap failed ERR
|
||||
|
||||
if [[ ! -d "$repo" ]]; then
|
||||
git init --bare "$repo"
|
||||
git --git-dir="$repo" remote add origin git@git.ayau.me:syntaxbullet/minabot.git
|
||||
fi
|
||||
timeout 90 git --git-dir="$repo" fetch --quiet origin +refs/heads/main:refs/remotes/origin/main
|
||||
target=$(git --git-dir="$repo" rev-parse refs/remotes/origin/main)
|
||||
[[ "$target" =~ ^[0-9a-f]{40}$ ]]
|
||||
[[ "$target" != "$(cat deployed-sha 2>/dev/null || true)" ]] || exit 0
|
||||
if [[ "${1:-}" != '--retry' && "$target" == "$(cat failed-sha 2>/dev/null || true)" ]]; then exit 0; fi
|
||||
|
||||
echo "Testing main at $target"
|
||||
available_kb=$(df -Pk "$root" | awk 'NR==2 {print $4}')
|
||||
if (( available_kb < 2097152 )); then echo 'Less than 2 GiB free; refusing to build.' >&2; false; fi
|
||||
release="$root/releases/$target"
|
||||
mkdir -p "$release"
|
||||
git --git-dir="$repo" archive "$target" | tar -x -C "$release"
|
||||
image="minabot:git-${target:0:12}"
|
||||
checks="$image-checks"
|
||||
docker_cmd build --target build -t "$checks" "$release" > "$release/build-checks.log" 2>&1
|
||||
docker_cmd run --rm "$checks" sh -c 'bun run typecheck && bun test' > "$release/tests.log" 2>&1
|
||||
docker_cmd build -t "$image" "$release" > "$release/build-runtime.log" 2>&1
|
||||
docker_cmd run --rm "$image" bun scripts/api-smoke.ts > "$release/smoke.log" 2>&1
|
||||
docker_cmd image rm "$checks" > /dev/null
|
||||
|
||||
previous_image=$(sed -n 's/^APP_IMAGE=//p' .env)
|
||||
[[ -n "$previous_image" ]]
|
||||
awk -v image="$image" '/^APP_IMAGE=/ {$0="APP_IMAGE=" image} {print}' .env > next.env
|
||||
docker_cmd compose --project-directory "$root" -f "$release/compose.yaml" --env-file "$root/next.env" config --quiet
|
||||
caddy_image=$(sed -n 's/^CADDY_IMAGE=//p' .env)
|
||||
docker_cmd run --rm -v "$release/Caddyfile:/etc/caddy/Caddyfile:ro" "$caddy_image" caddy validate --config /etc/caddy/Caddyfile > "$release/caddy-check.log" 2>&1
|
||||
|
||||
rollback="$release/previous"
|
||||
mkdir -p "$rollback"
|
||||
cp .env compose.yaml Caddyfile "$rollback/"
|
||||
echo "Switching to $target"
|
||||
# Prevent user writes during migration and never overlap Discord workers.
|
||||
# Obtain the fingerprint before stopping so failure here leaves production running.
|
||||
schema_before=$(schema_fingerprint "$previous_image")
|
||||
switching=true
|
||||
compose stop caddy app
|
||||
compose run --rm --no-deps --interactive=false --entrypoint bun app scripts/backup.ts > "$rollback/backup-path.txt"
|
||||
cp "$release/compose.yaml" compose.yaml
|
||||
cp "$release/Caddyfile" Caddyfile
|
||||
mv next.env .env
|
||||
compose up -d --wait --wait-timeout 120 app monitor
|
||||
compose up -d --wait --wait-timeout 120 caddy
|
||||
curl --fail --silent --show-error --retry 5 --retry-all-errors --retry-delay 2 --max-time 15 https://mina.teppelinlabs.com/api/health > "$release/public-health.json"
|
||||
printf '%s\n' "$target" > deployed-sha.tmp
|
||||
mv deployed-sha.tmp deployed-sha
|
||||
rm -f failed-sha
|
||||
docker_cmd image inspect "$image" --format '{{.Id}}' > release-image-id
|
||||
switching=false
|
||||
echo "Deployed $target"
|
||||
Reference in New Issue
Block a user