wealth/tools/deploy.sh

119 lines
6.2 KiB
Bash
Executable File

#!/usr/bin/env bash
# Deploys one app on this host: builds its image with apps/<app>/.env.production, checks the new release's
# health check (/healthz) in a container off to the side, then swaps the containers: stops the old release
# and starts the new one on HTTP_PORT. The old release stays as a stopped container, the one to roll back
# to; older ones are removed. A failed build or health check leaves the old release serving. The swap
# refuses connections for a few seconds: the old release finishes its in-flight requests (30 s at most),
# then the new one starts. Whatever sits in front (the edge that routes between bull and bear, 0001; this
# repo sets none up, 0067) points at HTTP_PORT.
#
# bash tools/deploy.sh <app> [KEY=VALUE]...
# HTTP_PORT=7001 bash tools/deploy.sh bear
# bash tools/deploy.sh bull NEXT_PUBLIC_SERVICE_WORKER_ENABLED=false
#
# HTTP_PORT ([host:]port) is where the app answers on this host; inside its container it listens on 3000.
# Default: the port the serving release publishes, or 8080 on the first deploy.
#
# apps/<app>/.env.production lives on this host only (git keeps .env.sample) and holds every value, secrets
# included. A KEY=VALUE argument overrides the file's value for this deploy, at build time and at runtime.
#
# Each deploy's container is <app>-<UTC time of the deploy>, such as bull-20261016143005: the one that
# serves and the previous one, stopped, which tools/rollback.sh <app> starts in its place.
# State in .deploy/<app>/, ignored by git and Docker, and readable by this user only: one folder per
# container, named after it, holds its .env.production (the app's file plus the overrides), mounted into it.
set -euo pipefail
cd "$(dirname "${BASH_SOURCE[0]}")/.."
app=${1:?Usage: tools/deploy.sh <app> [KEY=VALUE]...}
shift
[ -f "apps/$app/next.config.ts" ] || { echo "apps/$app/next.config.ts is missing" >&2; exit 1; }
[ -f "apps/$app/.env.production" ] \
|| { echo "apps/$app/.env.production is missing: copy apps/$app/.env.sample and fill it in" >&2; exit 1; }
for override in "$@"; do
[[ $override =~ ^[A-Za-z_][A-Za-z0-9_]*= ]] || { echo "Overrides are KEY=VALUE, got: $override" >&2; exit 1; }
done
state="$PWD/.deploy/$app"
mkdir -p "$state"
chmod 700 .deploy
exec 9>"$state/.lock"
flock --nonblock 9 || { echo "A deploy or rollback of $app is running" >&2; exit 1; }
# 1. The release that serves now, the new one's name, and the port. The port is checked before the build,
# unless the serving release already holds it.
containers() { docker ps "$@" --filter "name=^$app-[0-9]{14}\$" --format '{{.Names}}'; }
current=$(containers | head -n 1)
next=$app-$(date -u +%Y%m%d%H%M%S)
published=
if [ -n "$current" ]; then
published=$(docker port "$current" 3000/tcp 2>/dev/null | head -n 1 || true)
published=${published#0.0.0.0:}
published=${published#\[::\]:}
fi
http_port=${HTTP_PORT:-${published:-8080}}
port_host=127.0.0.1
[[ $http_port == *:* ]] && port_host=${http_port%:*}
if [ "$http_port" != "$published" ] && (exec 3<>"/dev/tcp/$port_host/${http_port##*:}") 2>/dev/null; then
echo "Port $http_port is taken; pass a free one: HTTP_PORT=8081 bash tools/deploy.sh $app" >&2
exit 1
fi
# 2. The release's env, in its own folder: the app's file, then the overrides; for a repeated key, the last
# line wins. The container's node user reads it, so it's 644 inside .deploy/, which only this user can
# open. Until the swap, any failure removes the new release and its folder.
mkdir -p "$state/$next"
{ cat "apps/$app/.env.production"; echo; printf '%s\n' "$@"; } > "$state/$next/.env.production"
chmod 644 "$state/$next/.env.production"
trap 'docker rm --force "$next" >/dev/null 2>&1 || true; rm -rf "${state:?}/$next"' EXIT
# 3. Build. The commit names the release and becomes NEXT_DEPLOYMENT_ID; uncommitted changes get their own.
release=$(git rev-parse --short=12 HEAD)
git diff --quiet HEAD || release="$release-dirty-$(date +%s)"
docker build --build-context appenv="$state/$next" \
--build-arg APP="$app" --build-arg NEXT_DEPLOYMENT_ID="$release" --tag "$app:$release" .
start_next() {
docker rm --force "$next" >/dev/null 2>&1 || true
docker run --detach --name "$next" --restart unless-stopped \
--cap-drop ALL --security-opt no-new-privileges --stop-timeout 30 \
--volume "$state/$next/.env.production:/app/apps/$app/.env.production:ro" "$@" "$app:$release" >/dev/null
}
# Docker turns the image's HEALTHCHECK unhealthy when /healthz keeps failing.
wait_healthy() {
local status
while status=$(docker inspect --format '{{.State.Status}} {{.State.Health.Status}}' "$next") \
&& [ "$status" = 'running starting' ]; do
sleep 2
done
[ "$status" = 'running healthy' ] && return 0
docker logs --tail 50 "$next" >&2
echo "$next ($app:$release) is $status" >&2
return 1
}
# 4. Check the new release off to the side, with no published port, while the old one keeps serving.
echo "Checking $app:$release against /healthz"
start_next
wait_healthy || { echo "${current:-No release} keeps serving" >&2; exit 1; }
docker rm --force "$next" >/dev/null
# 5. Swap. Stopping the old release frees the port; SIGTERM lets Next.js finish what it's serving, for up
# to 30 s (--stop-timeout). If the new release fails now, the old one starts again.
trap 'docker rm --force "$next" >/dev/null 2>&1 || true; rm -rf "${state:?}/$next"
if [ -n "$current" ]; then docker start "$current" >/dev/null && echo "$current serves again" >&2; fi' EXIT
if [ -n "$current" ]; then docker stop "$current" >/dev/null; fi
start_next --publish "$http_port:3000"
wait_healthy
trap - EXIT
# 6. Keep one previous release: the old one, stopped, with its env folder. Remove every older container and
# folder, and keep the app's three newest images (Docker never removes an image a container uses).
for old in $(containers --all); do
[ "$old" = "$next" ] || [ "$old" = "$current" ] || docker rm --force "$old" >/dev/null
done
for dir in "$state"/*/; do
dir=${dir%/}
[ "${dir##*/}" = "$next" ] || [ "${dir##*/}" = "$current" ] || rm -rf "$dir"
done
docker image ls "$app" --format '{{.Repository}}:{{.Tag}}' | tail -n +4 | xargs --no-run-if-empty docker image rm >/dev/null 2>&1 || true
echo "$app $release serves at $(docker port "$next" 3000/tcp | head -n 1) as $next${current:+; $current is stopped, kept for a rollback}"