From 1cda5779a47c62b0ac1774985f4074becfc2bf8d Mon Sep 17 00:00:00 2001 From: Aaron Po Date: Sun, 4 Oct 2026 20:29:37 -0400 Subject: [PATCH] add restart tolerance --- docs/web/docker.md | 46 ++++++++++++++++++++++++++++++++++-- web/docker-compose.prod.yaml | 36 ++++++++++++++++++++++++++-- web/frontend/server.js | 6 +++++ 3 files changed, 84 insertions(+), 4 deletions(-) diff --git a/docs/web/docker.md b/docs/web/docker.md index c19d8e5d..d675e7da 100644 --- a/docs/web/docker.md +++ b/docs/web/docker.md @@ -134,8 +134,9 @@ containers, which would end the run prematurely. See - Production logging levels - No database clearing - Optimized build configurations -- Health checks enabled -- Restart policies (unless-stopped) +- Health checks on every long-running service +- Restart policies that survive a reboot (see + [Restart tolerance](#restart-tolerance)) - Security hardening - Only `frontend` publishes a port to the host, bound to `127.0.0.1` so a reverse proxy can sit in front of it; `sqlserver`, `api.core` and `seaweedfs` are @@ -170,6 +171,41 @@ Docker seeds each volume with the ownership the image expects and labels it for SELinux, so the host needs no `chown` and no `:z`/`:Z` mount options on enforcing distributions such as RHEL, AlmaLinux and Fedora. +#### Restart tolerance + +`sqlserver`, `api.core`, `seaweedfs` and `frontend` run under +`restart: unless-stopped`, so the stack returns on its own after a container +crash, a Docker daemon restart or a host reboot, provided Docker itself starts at +boot (`systemctl enable docker`). `database.migrations` keeps `restart: "no"`: it +is a one-shot job that runs once per deploy and stays exited across a reboot, so +the schema is migrated on deploy rather than on every boot. + +The daemon replays restart policies without replaying the `depends_on` graph, so +after a reboot the containers come back in arbitrary order and each one tolerates +a dependency that arrives late. `api.core` opens its database and S3 connections +per request rather than at startup, and `frontend` answers with its own error +states while `api.core` is still coming up. + +Health checks report when the stack has converged: + +| Service | Probe | +| --- | --- | +| `sqlserver` | `sqlcmd -Q 'SELECT 1'`, with a 30s start period and 12 retries | +| `api.core` | `GET /health` spoken over bash's `/dev/tcp` — the .NET runtime image carries bash and no HTTP client | +| `seaweedfs` | `curl http://localhost:8333/status`, the S3 gateway, which serves once the master, volume and filer servers are up | +| `frontend` | `wget --spider http://127.0.0.1:5656/healthz`, answered by Express alone and so independent of the API | + +`sqlserver` also carries `stop_grace_period: 60s`, giving a checkpoint time to +finish on shutdown instead of being recovered on the next start. + +Deploy and wait for the stack to report healthy, or check convergence after a +reboot: + +```bash +docker compose --env-file web/.env.prod -f web/docker-compose.prod.yaml up -d --wait +docker compose --env-file web/.env.prod -f web/docker-compose.prod.yaml ps +``` + #### Seeding production (`docker-compose.prod.seed.yaml`) `database.seed` lives in its own compose file, run on demand once the prod stack @@ -266,6 +302,12 @@ graph TD In development, `api.core` also depends on `seaweedfs` (`condition: service_started`, not a health check) alongside `database.seed`. +In production the chain is gated on health instead: `api.core` waits for +`seaweedfs` to pass its health check and for `database.migrations` to exit +successfully, and `frontend` waits for `api.core` to report healthy. This ordering +applies to `docker compose up`; see +[Restart tolerance](#restart-tolerance) for what happens on a host reboot. + **Health Check Example** (SQL Server): ```yaml diff --git a/web/docker-compose.prod.yaml b/web/docker-compose.prod.yaml index 286f8960..1bea3e84 100644 --- a/web/docker-compose.prod.yaml +++ b/web/docker-compose.prod.yaml @@ -20,6 +20,8 @@ services: timeout: 5s retries: 12 start_period: 30s + restart: unless-stopped + stop_grace_period: 60s networks: - prodnet @@ -54,7 +56,7 @@ services: database.migrations: condition: service_completed_successfully seaweedfs: - condition: service_started + condition: service_healthy build: context: ./backend dockerfile: API/API.Core/Dockerfile @@ -79,6 +81,20 @@ services: SEAWEEDFS_ACCESS_KEY_ID: "${SEAWEEDFS_ACCESS_KEY_ID}" SEAWEEDFS_SECRET_ACCESS_KEY: "${SEAWEEDFS_SECRET_ACCESS_KEY}" SEAWEEDFS_BUCKET: "${SEAWEEDFS_BUCKET}" + healthcheck: + # The .NET runtime image carries bash but no HTTP client, so the probe speaks + # HTTP to /health over bash's /dev/tcp rather than adding curl to the image. + test: + [ + "CMD", + "bash", + "-c", + 'exec 3<>/dev/tcp/127.0.0.1/8080; printf "GET /health HTTP/1.1\r\nHost: localhost\r\nConnection: close\r\n\r\n" >&3; head -1 <&3 | grep -q " 200 "', + ] + interval: 15s + timeout: 5s + retries: 5 + start_period: 30s restart: unless-stopped networks: - prodnet @@ -97,6 +113,14 @@ services: S3_BUCKET: "${SEAWEEDFS_BUCKET}" volumes: - seaweedfsdata-prod:/data + healthcheck: + # Probes the S3 gateway, the one component api.core talks to; it comes up + # last, after the master, volume and filer servers are serving. + test: ["CMD", "curl", "-fsS", "-o", "/dev/null", "http://localhost:8333/status"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 20s restart: unless-stopped networks: - prodnet @@ -107,7 +131,7 @@ services: container_name: prod-env-frontend depends_on: api.core: - condition: service_started + condition: service_healthy build: context: ./frontend dockerfile: Dockerfile @@ -118,6 +142,14 @@ services: PORT: "5656" API_BASE_URL: "http://api.core:8080" SESSION_SECRET: "${SESSION_SECRET}" + healthcheck: + # /healthz answers from the Express server alone, so the frontend stays + # healthy while api.core is still starting and serves its own error states. + test: ["CMD", "wget", "--quiet", "--spider", "--tries=1", "http://127.0.0.1:5656/healthz"] + interval: 15s + timeout: 5s + retries: 5 + start_period: 15s restart: unless-stopped networks: - prodnet diff --git a/web/frontend/server.js b/web/frontend/server.js index 0fb51811..ef77795d 100644 --- a/web/frontend/server.js +++ b/web/frontend/server.js @@ -6,6 +6,12 @@ const app = express(); app.use('/assets', express.static('build/client/assets', { immutable: true, maxAge: '1y' })); app.use(express.static('build/client', { maxAge: '1h' })); +// Answers from Express alone so the container health check reports on this +// process only, independent of whether the API is reachable yet. +app.get('/healthz', (_req, res) => { + res.status(200).type('text/plain').send('ok'); +}); + const requestHandler = createRequestHandler({ build: () => import('./build/server/index.js'), });