Intermediate

# File: Dockerfile
 
# Build stage (includes build tools)
FROM node:18-alpine AS builder
# => Stage name: "builder"
# => Includes npm, node-gyp, build tools
 
WORKDIR /app
 
# Copy and install all dependencies (including devDependencies)
COPY package*.json ./
RUN npm ci
# => Installs all dependencies for building
 
# Copy source code
COPY .
 
# Build production bundle
RUN npm run build
# => Creates optimized production build in /app/dist/
# => Includes transpilation, minification, bundling
 
# Production stage (minimal runtime)
FROM node:18-alpine
# => Fresh FROM instruction starts new stage
# => Previous stage (builder) layers are discarded
 
WORKDIR /app
 
# Copy only production package files
COPY package*.json ./
 
# Install only production dependencies
RUN npm ci --only=production
# => Excludes devDependencies (webpack, babel, etc.)
# => Smaller node_modules
 
# Copy built artifacts from builder stage
COPY --from=builder /app/dist ./dist
# => COPY --from=<stage-name> copies files from previous stage
# => Only production bundle, not source code
 
# Non-root user for security
RUN addgroup -g 1001 -S nodejs && \
 adduser -S nodejs -u 1001 && \
 chown -R nodejs:nodejs /app
# => Creates non-privileged user
# => Changes ownership of /app
 
USER nodejs
# => Runs container as nodejs user (not root)
 
EXPOSE 3000
CMD ["node", "dist/main.js"]
# => Starts production server

# Build multi-stage image
docker build -t my-app:multi-stage .
# => [builder 1/5] FROM node:18-alpine (build stage — all dev tools)
# => [builder 4/5] RUN npm ci (installs devDependencies for building)
# => [builder 5/5] RUN npm run build (creates /app/dist/)
# => [stage-1 4/4] COPY --from=builder /app/dist ./dist (only artifacts)
# => Successfully tagged my-app:multi-stage
 
# Compare with single-stage image size
docker images my-app
# => my-app single-stage 450MB (source + devDependencies + build tools)
# => my-app multi-stage  120MB (runtime + production bundle only)
# => 73% size reduction
 
# Verify production image contents
docker run --rm my-app:multi-stage ls -lh /app
# => drwxr-xr-x nodejs nodejs 4.0K dist (production bundle only)
# => drwxr-xr-x nodejs nodejs  12K node_modules (production deps only)
 
# Source code NOT in production image (security)
docker run --rm my-app:multi-stage ls /app/src
# => ls: /app/src: No such file or directory (source stays in builder stage)
 
# Check user (runs as nodejs, not root)
docker run --rm my-app:multi-stage whoami
# => nodejs

# File: Dockerfile
 
# Build stage
FROM golang:1.21-alpine AS builder
 
# Build arguments for version information
ARG VERSION=dev
ARG BUILD_DATE
ARG GIT_COMMIT=unknown
 
WORKDIR /app
 
# Copy dependency files
COPY go.mod go.sum ./
RUN go mod download
# => Downloads Go dependencies
 
# Copy source code
COPY .
 
# Build binary with version information
RUN CGO_ENABLED=0 GOOS=linux go build \
 -ldflags="-X main.Version=${VERSION} \
 -X main.BuildDate=${BUILD_DATE} \
 -X main.GitCommit=${GIT_COMMIT} \
 -w -s" \
 -o /app/server ./cmd/server
# => Compiles Go binary with embedded version metadata
# => -ldflags=-w -s strips debug symbols (smaller binary)
# => CGO_ENABLED=0 creates fully static binary
 
# Production stage
FROM alpine:3.19
# => Minimal base image (~5MB)
# => golang:1.21 image not needed at runtime
 
# Security: Install CA certificates for HTTPS
RUN apk --no-cache add ca-certificates
# => Required for HTTPS requests
# => --no-cache prevents storing package index
 
# Create non-root user
RUN addgroup -g 1001 app && \
 adduser -D -u 1001 -G app app
 
WORKDIR /app
 
# Copy binary from builder stage
COPY --from=builder /app/server .
# => Only the compiled binary, no Go toolchain
 
# Change ownership
RUN chown -R app:app /app
 
USER app
 
EXPOSE 8080
 
# Pass build args as labels
ARG VERSION
ARG BUILD_DATE
ARG GIT_COMMIT
LABEL version="${VERSION}" \
 build_date="${BUILD_DATE}" \
 git_commit="${GIT_COMMIT}"
# => Metadata queryable via docker inspect
 
CMD ["./server"]

# Build with version information
docker build \
 --build-arg VERSION=1.2.3 \
 --build-arg BUILD_DATE=$(date -u +"%Y-%m-%dT%H:%M:%SZ") \
 --build-arg GIT_COMMIT=$(git rev-parse --short HEAD) \
 -t my-go-app:1.2.3 \
 .
# => Embeds version metadata in binary and image labels
 
# Verify version embedded in binary
docker run --rm my-go-app:1.2.3 ./server --version
# => my-go-app version 1.2.3
# => Built: 2025-12-29T10:50:00Z
# => Commit: abc1234
 
# Check image labels
docker inspect my-go-app:1.2.3 --format='{{json .Config.Labels}}' | jq
# => {
# => "version": "1.2.3",
# => "build_date": "2025-12-29T10:50:00Z",
# => "git_commit": "abc1234"
# => }
 
# Compare image sizes
docker images | grep my-go-app
# => my-go-app 1.2.3 15MB (alpine + binary)
# => If built from golang base: ~350MB
# => 96% size reduction!
 
# Verify binary is static (no dynamic linking)
docker run --rm my-go-app:1.2.3 ldd /app/server
# => not a dynamic executable
# => Fully static binary (portable across Linux distros)

# File: Dockerfile
 
# Stage 1: Build frontend (Node.js)
FROM node:18-alpine AS frontend-builder
# => Node.js 18 Alpine build environment for frontend
 
WORKDIR /app/frontend
# => Set frontend working directory
 
COPY frontend/package*.json ./
# => Copy package files for dependency caching
RUN npm ci
# => Install frontend dependencies
 
COPY frontend/ ./
# => Copy all frontend source files
RUN npm run build
# => Creates static assets in /app/frontend/dist
 
# Stage 2: Build backend (Go)
FROM golang:1.21-alpine AS backend-builder
# => Go 1.21 Alpine build environment
 
WORKDIR /app/backend
# => Set backend working directory
 
COPY backend/go.mod backend/go.sum ./
# => Copy module files for dependency caching
RUN go mod download
# => Download all Go module dependencies
 
COPY backend/ ./
# => Copy all backend source files
RUN CGO_ENABLED=0 go build -o /app/backend/server ./cmd/server
# => Compiles Go backend server
# => CGO_ENABLED=0: creates static binary
 
# Stage 3: Generate API documentation (Python)
FROM python:3.11-slim AS docs-builder
# => Python 3.11 slim environment for documentation
 
WORKDIR /app
# => Set working directory
 
RUN pip install --no-cache-dir mkdocs mkdocs-material
# => Install documentation generator
# => --no-cache-dir: smaller image
 
COPY docs/ ./docs/
# => Copy documentation source files
RUN mkdocs build -d /app/docs-output
# => Generates static documentation site
 
# Stage 4: Production runtime (Nginx + backend)
FROM nginx:alpine
# => Final minimal production image
 
# Copy frontend static assets
COPY --from=frontend-builder /app/frontend/dist /usr/share/nginx/html/
# => Frontend served by Nginx
 
# Copy API documentation
COPY --from=docs-builder /app/docs-output /usr/share/nginx/html/docs/
# => Documentation at /docs path
 
# Copy Go backend binary
COPY --from=backend-builder /app/backend/server /usr/local/bin/
# => Backend binary in PATH
 
# Nginx configuration for SPA + API
COPY nginx.conf /etc/nginx/conf.d/default.conf
# => Routes /api/* to backend, /* to frontend
 
# Startup script to run both Nginx and backend
COPY start.sh /usr/local/bin/
# => Copy startup script
RUN chmod +x /usr/local/bin/start.sh
# => Make startup script executable
 
EXPOSE 80
# => Document HTTP port
 
CMD ["/usr/local/bin/start.sh"]
# => Run startup script (manages both nginx and backend)

# File: start.sh
#!/bin/sh
 
# Start backend in background
/usr/local/bin/server &
# => Start Go backend in background
BACKEND_PID=$!
# => Go backend starts on port 8080
 
# Start Nginx in foreground
nginx -g 'daemon off;' &
# => Start Nginx in background
NGINX_PID=$!
# => Nginx starts on port 80, routes to backend
 
# Wait for either process to exit
wait -n
# => Returns when first process exits (crash detection)
 
# Kill both processes
kill $BACKEND_PID $NGINX_PID
# => Ensures clean shutdown of both services

# File: nginx.conf
server {
# => Nginx virtual server block
 listen 80;
 # => Nginx listens on port 80
 # => HTTP only (add SSL for production)
 
 # Frontend SPA
 location / {
# => Route all unmatched requests to frontend
 root /usr/share/nginx/html;
 # => Frontend static files served from here
 try_files $uri $uri/ /index.html;
 # => Falls back to index.html (SPA routing)
 # => React/Vue/Angular router handles URL
 }
 
 # API proxy to backend
 location /api/ {
# => Route /api/* to Go backend
 proxy_pass http://localhost:8080/;
 # => Forwards /api/* requests to Go backend on port 8080
 proxy_http_version 1.1;
# => Use HTTP/1.1 for keep-alive connections
 proxy_set_header Host $host;
# => Pass original Host header
 proxy_set_header X-Real-IP $remote_addr;
 # => Preserves original client IP for logging
 # => Backend can log real client IPs
 }
# => End of /api/ location block
 
 # API documentation
 location /docs/ {
# => Route /docs/* to MkDocs static site
 alias /usr/share/nginx/html/docs/;
 # => MkDocs static site at /docs path
 try_files $uri $uri/ /docs/index.html;
 # => Falls back to docs index for SPA-style routing
 }
# => End of /docs/ location block
}
# => End of server block

# Build multi-runtime image
docker build -t fullstack-app .
# => [frontend-builder] Builds React/Vue/Angular app
# => [backend-builder] Compiles Go API server
# => [docs-builder] Generates MkDocs site
# => [final] Combines all artifacts in Nginx image
 
# Check final image size
docker images fullstack-app
# => REPOSITORY TAG SIZE
# => fullstack-app latest 45MB
# => Nginx base: 23MB, frontend assets: 15MB, backend binary: 7MB
 
# Run full-stack container
docker run -d -p 8080:80 --name fullstack fullstack-app
# => Frontend at http://localhost:8080/
# => API at http://localhost:8080/api/
# => Docs at http://localhost:8080/docs/
 
# Test all components
curl http://localhost:8080/
# => <!DOCTYPE html>..(frontend SPA)
 
curl http://localhost:8080/api/health
# => {"status":"ok","version":"1.0.0"} (backend API)
 
curl http://localhost:8080/docs/
# => <!DOCTYPE html>..(MkDocs documentation)
 
# Verify no build tools in final image
docker run --rm fullstack-app which node
# => (no output - Node.js not in final image)
 
docker run --rm fullstack-app which go
# => (no output - Go not in final image)
 
docker run --rm fullstack-app which python
# => (no output - Python not in final image)

# File: Dockerfile
 
# syntax=docker/dockerfile:1.4
# => Enable BuildKit features (required for --mount=type=secret)
 
FROM node:18-alpine AS builder
# => Node.js 18 Alpine build stage
 
WORKDIR /app
# => Set working directory
 
# Install dependencies from private registry using secret
COPY package*.json ./
# => Copy package files for dependency caching
 
# Mount secret during npm install (doesn't persist in layer)
RUN --mount=type=secret,id=npmrc,target=/root/.npmrc \
# => Mount .npmrc secret file temporarily
 npm ci
# => Mounts secret file at /root/.npmrc during RUN only
# => Secret is NOT stored in image layer
# => Secret file is removed after RUN completes
 
COPY .
# => Copy all source files
 
# Build with API key from secret
RUN --mount=type=secret,id=build_api_key \
# => Mount build API key secret
 export BUILD_API_KEY=$(cat /run/secrets/build_api_key) && \
# => Read secret value into environment variable
 npm run build
# => Reads secret from /run/secrets/<id>
# => Secret available only during this RUN command
 
# Production stage (no secrets)
FROM node:18-alpine
# => Final production image (clean, no secrets)
 
WORKDIR /app
# => Set working directory
 
COPY package*.json ./
# => Copy package files
RUN npm ci --only=production
# => Install only production dependencies
 
COPY --from=builder /app/dist ./dist
# => Only built artifacts, no secrets
 
EXPOSE 3000
# => Document application port
CMD ["node", "dist/main.js"]
# => Start production server

# Create secret files (DO NOT commit to git)
echo "//registry.npmjs.org/:_authToken=npm_secret_token" > .npmrc
# => Creates npm auth token file for private registry access
echo "build-api-key-12345" > build_api_key.txt
# => Creates API key file used during build
 
# Add secrets to .gitignore
cat >> .gitignore << 'EOF'
# => Append to .gitignore to protect secret files
.npmrc
# => Exclude .npmrc from git
build_api_key.txt
# => Exclude API key file from git
EOF
# => Prevents accidental commit of secret files
 
# Build with secrets (BuildKit required)
DOCKER_BUILDKIT=1 docker build \
# => Enable BuildKit for secret mounting
 --secret id=npmrc,src=.npmrc \
# => Pass .npmrc as a build secret
 --secret id=build_api_key,src=build_api_key.txt \
# => Pass API key file as a build secret
 -t secure-app .
# => Mounts secrets during build without persisting in layers
 
# Verify secrets are NOT in image layers
docker history secure-app --no-trunc | grep -i "secret\|npmrc\|api_key"
# => (no matches - secrets not visible in history)
 
# Alternative: Use environment secrets
DOCKER_BUILDKIT=1 docker build \
# => Enable BuildKit for env var secret
 --secret id=build_api_key,env=BUILD_API_KEY \
# => Use environment variable as secret source
 -t secure-app .
# => Reads secret from BUILD_API_KEY environment variable on host
 
# Environment variable secret (cleaner for CI/CD)
BUILD_API_KEY=secret-value DOCKER_BUILDKIT=1 docker build \
# => Set env var inline for this command only
 --secret id=build_api_key,env=BUILD_API_KEY \
# => Reference env var as secret
 -t secure-app .
# => Inline env var assignment scoped to single command
 
# Inspect final image for leaked secrets (security check)
docker save secure-app | tar -xOf - | grep -a "secret_token"
# => (no matches - secret not in any layer)
 
# Bad example (INSECURE - secrets in layers)
cat > Dockerfile.insecure << 'EOF'
# => Create insecure Dockerfile for comparison (DO NOT USE)
FROM node:18-alpine
# => Base image
WORKDIR /app
# => Working directory
COPY .npmrc /root/.npmrc # BAD! Secret persists in layer
# => WRONG: secret baked into image layer permanently
RUN npm ci
# => npm install with secret in layer
RUN rm /root/.npmrc # Still in previous layer!
# => Removing file doesn't remove from previous layer
EOF
# => Insecure Dockerfile written (illustrative only)
 
docker build -f Dockerfile.insecure -t insecure-app .
# => Builds insecure image to demonstrate the problem
docker history insecure-app --no-trunc | grep -i "npmrc"
# => Shows "COPY .npmrc" in layer history
# => Secret is exposed in image layer (security risk!)

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version 3.8
 
services:
 frontend:
 # => Frontend service definition
 build:
 # => Build configuration for frontend
 context: ./frontend
 # => Build context: all files in ./frontend directory
 dockerfile: Dockerfile
 # => Uses default Dockerfile in context directory
 cache_from:
 # => List of images to use as cache sources
 - myregistry/frontend:latest
 # => Latest tag used as primary cache layer
 - myregistry/frontend:${GIT_BRANCH:-main}
 # => Uses remote images as cache sources
 # => Speeds up builds by reusing layers
 args:
 # => Build arguments passed to Dockerfile
 NODE_ENV: ${NODE_ENV:-production}
 # => Defaults to production if not set
 target: ${BUILD_TARGET:-production}
 # => Allows switching between dev/prod stages
 image: myregistry/frontend:${GIT_COMMIT:-latest}
 # => Tags with git commit for traceability
 
 backend:
 # => Go backend service definition
 build:
 context: ./backend
 # => Build context for Go backend
 cache_from:
 - myregistry/backend:latest
 # => Reuses cached Go modules and build layers
 args:
 GO_VERSION: 1.21
 # => Passes Go version as build argument
 image: myregistry/backend:${GIT_COMMIT:-latest}
 # => Same versioning pattern as frontend
 
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => No build needed - uses pre-built image

# Enable BuildKit for better caching
export DOCKER_BUILDKIT=1
# => Enables BuildKit build engine
export COMPOSE_DOCKER_CLI_BUILD=1
# => Tells Compose to use Docker CLI with BuildKit
 
# Build all services in parallel
docker compose build --parallel
# => Builds frontend and backend simultaneously
# => Utilizes multi-core CPUs efficiently
 
# Build with specific target (development)
BUILD_TARGET=development docker compose build frontend
# => Builds development stage with hot reloading support
 
# Pull cache images before building (CI/CD optimization)
docker compose pull frontend backend
docker compose build --pull frontend backend
# => Pulls latest images to use as cache
# => --pull ensures base images are up-to-date
 
# Build without cache (force rebuild)
docker compose build --no-cache
# => Rebuilds all layers from scratch
# => Useful when dependencies are corrupted
 
# Build with progress output
docker compose build --progress=plain frontend
# => Shows detailed build output
# => Useful for debugging build failures
 
# Build and push to registry (CI/CD)
GIT_COMMIT=$(git rev-parse --short HEAD) \
GIT_BRANCH=$(git branch --show-current) \
docker compose build --push
# => Builds and pushes images to registry
# => Uses GIT_COMMIT and GIT_BRANCH variables for tagging
 
# Build-time performance: Use .dockerignore
cat > .dockerignore << 'EOF'
node_modules
# => Excludes large dependency directory
.git
# => Excludes version control metadata
.env*
# => Excludes all environment variable files
*.md
# => Excludes markdown documentation files
docs/
# => Excludes documentation directory
tests/
# => Excludes test files not needed in image
.vscode/
# => Excludes editor configuration
.idea/
# => Excludes JetBrains IDE files
EOF
# => Excludes files from build context
# => Reduces context size sent to Docker daemon
# => Significantly speeds up builds
 
# Measure build time
time docker compose build frontend
# => `time` command reports elapsed wall time
# => real 0m15.234s (with cache)
# => real 2m30.123s (without cache)
 
# Use BuildKit cache mounts for dependencies
cat >> frontend/Dockerfile << 'EOF'
RUN --mount=type=cache,target=/root/.npm \
 npm ci
EOF
# => Caches npm packages across builds
# => Dramatically speeds up dependency installation

# File: docker-compose.yml
 
version: "3.8"
# => Docker Compose file version
 
services:
 database:
 image: postgres:15-alpine
 # => PostgreSQL 15 on Alpine Linux base
 environment:
 POSTGRES_PASSWORD: secret
 # => Database superuser password
 healthcheck:
 test: ["CMD-SHELL", "pg_isready -U postgres"]
 # => Executes pg_isready command to check PostgreSQL status
 # => Returns 0 if accepting connections, non-zero if not ready
 interval: 10s
 # => Run health check every 10 seconds
 timeout: 5s
 # => Mark check as failed if it takes longer than 5 seconds
 retries: 5
 # => Mark service unhealthy after 5 consecutive check failures
 start_period: 30s
 # => Grace period allowing database initialization before health checks count
 # => Failed checks during this period don't count toward retries
 
 redis:
 image: redis:7-alpine
 # => Redis 7 key-value cache on Alpine
 healthcheck:
 test: ["CMD", "redis-cli", "ping"]
 # => Executes redis-cli ping command
 # => Returns PONG if Redis is responsive
 interval: 5s
 # => Check every 5 seconds (faster than database)
 timeout: 3s
 # => Redis should respond quickly
 retries: 3
 # => Mark unhealthy after 3 failures
 start_period: 10s
 # => Redis starts faster than PostgreSQL
 
 api:
 build: ./api
 # => Builds API service from ./api directory
 depends_on:
 database:
 condition: service_healthy
 # => API waits for database health check to pass
 # => Prevents connection errors during startup
 redis:
 condition: service_healthy
 # => API waits for Redis to be ready
 healthcheck:
 test: ["CMD", "curl", "-f", "http://localhost:3000/health"]
 # => HTTP GET request to application health endpoint
 # => -f flag makes curl fail on HTTP errors (4xx, 5xx)
 interval: 15s
 # => Check every 15 seconds
 timeout: 10s
 # => Allow up to 10 seconds for health endpoint response
 retries: 3
 # => Mark unhealthy after 3 consecutive failures
 start_period: 40s
 # => API needs time to connect to database and initialize
 environment:
 DATABASE_URL: postgresql://postgres:secret@database:5432/mydb
 # => Database connection string using service name "database"
 REDIS_URL: redis://redis:6379
 # => Redis connection using service name "redis"
 
 web:
 image: nginx:alpine
 # => Nginx web server on Alpine
 depends_on:
 api:
 condition: service_healthy
 # => Nginx starts only after API is healthy
 healthcheck:
 test: ["CMD", "wget", "--quiet", "--tries=1", "--spider", "http://localhost/health"]
 # => wget downloads health endpoint without saving (--spider)
 # => --quiet suppresses output, --tries=1 attempts once
 interval: 10s
 # => Check every 10 seconds
 timeout: 5s
 # => Nginx should respond quickly
 retries: 3
 # => Unhealthy after 3 failures
 start_period: 20s
 # => Nginx startup time
 ports:
 - "8080:80"
 # => Expose Nginx on host port 8080

// File: api/health.js (API health endpoint implementation)
const express = require("express");
const { Pool } = require("pg");
const Redis = require("ioredis");
 
const app = express();
const db = new Pool({ connectionString: process.env.DATABASE_URL });
const redis = new Redis(process.env.REDIS_URL);
 
app.get("/health", async (req, res) => {
  try {
    // Check database connection
    await db.query("SELECT 1");
 
    // Check Redis connection
    await redis.ping();
 
    // All dependencies healthy
    res.status(200).json({
      status: "healthy",
      timestamp: new Date().toISOString(),
      dependencies: {
        database: "up",
        redis: "up",
      },
    });
  } catch (error) {
    // Dependency failure - unhealthy
    res.status(503).json({
      status: "unhealthy",
      error: error.message,
      timestamp: new Date().toISOString(),
    });
  }
});
 
app.listen(3000);

# Start services (observe startup coordination)
docker compose up -d
# => database and redis start first (no dependencies)
# => Health checks run during start_period
# => After 30s, database marked healthy
# => After 10s, redis marked healthy
# => api starts after both dependencies healthy
# => After 40s, api marked healthy
# => web starts after api healthy
 
# Monitor health status in real-time
watch -n 1 'docker compose ps'
# => NAME STATUS HEALTH
# => database Up 35 seconds healthy (5/5)
# => redis Up 35 seconds healthy (7/7)
# => api Up 10 seconds starting (health: starting)
# => web Created (waiting for api)
 
# Check health check logs
docker inspect myproject-database-1 --format='{{json .State.Health}}' | jq
# => {
# => "Status": "healthy",
# => "FailingStreak": 0,
# => "Log": [
# => {
# => "Start": "2025-12-29T11:00:00Z",
# => "End": "2025-12-29T11:00:00Z",
# => "ExitCode": 0,
# => "Output": "accepting connections\n"
# => }
# => ]
# => }
 
# Test health endpoint manually
curl http://localhost:8080/health
# => {
# => "status": "healthy",
# => "timestamp": "2025-12-29T11:00:00Z",
# => "dependencies": {
# => "database": "up",
# => "redis": "up"
# => }
# => }
 
# Simulate database failure
docker compose pause database
# => Database pauses (stops accepting connections)
 
# Wait for health checks to detect failure
sleep 30
 
docker compose ps
# => NAME STATUS HEALTH
# => database Up (Paused) unhealthy
# => api Up unhealthy (database check fails)
# => web Up healthy (Nginx still serves)
 
# Resume database
docker compose unpause database
 
# Health checks recover automatically
sleep 30
docker compose ps
# => All services return to healthy state

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # => Multi-service application stack
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => Uses Alpine-based PostgreSQL 15 image
 restart: always
 # => Restarts on any failure or Docker restart
 environment:
 # => Environment variables for PostgreSQL
 POSTGRES_PASSWORD: secret  # => PostgreSQL superuser password
 healthcheck:
 # => Health check configuration
 test: ["CMD-SHELL", "pg_isready"]  # => Checks if PostgreSQL accepts connections
 interval: 10s    # => Check every 10 seconds
 timeout: 5s      # => Fail if check takes longer than 5s
 retries: 5       # => Unhealthy after 5 consecutive failures
 start_period: 30s  # => Grace period before counting failures
 volumes:
 # => Volume mounts for data persistence
 - db-data:/var/lib/postgresql/data
 # => Maps named volume to PostgreSQL data directory
 # => Data persists across restarts
 
 message-queue:
 # => RabbitMQ message broker service
 image: rabbitmq:3-management-alpine  # => RabbitMQ with management web UI
 restart: unless-stopped
 # => Restarts on failure but not if manually stopped
 healthcheck:
 # => Health check for message queue
 test: ["CMD", "rabbitmq-diagnostics", "ping"]  # => Verifies RabbitMQ responds
 interval: 15s    # => Check every 15s (slower start than postgres)
 timeout: 10s     # => RabbitMQ may be slow to respond
 retries: 3
 # => Unhealthy after 3 consecutive failures
 start_period: 40s
 # => RabbitMQ takes 40s before health checks count  # => RabbitMQ takes longer to initialize
 
 worker:
 build: ./worker  # => Builds from ./worker/Dockerfile
 restart: on-failure:3
 # => Restarts up to 3 times on failure
 # => Gives up after 3 consecutive failures (avoids infinite restart loops)
 depends_on:
 # => Wait for dependencies to be healthy before starting
 database:
 condition: service_healthy  # => Worker waits for DB health check pass
 message-queue:
 condition: service_healthy  # => Worker waits for RabbitMQ health check pass
 environment:
 # => Worker connection configuration
 DATABASE_URL: postgresql://postgres:secret@database:5432/jobs  # => "database" is Docker DNS name
 RABBITMQ_URL: amqp://guest:guest@message-queue:5672  # => "message-queue" is Docker DNS name
 healthcheck:
 # => Worker process health check
 test: ["CMD", "pgrep", "-f", "worker"]
 # => Checks if worker process is running (by name)
 interval: 20s    # => Check every 20 seconds
 timeout: 5s
 # => Fail if check takes longer than 5s
 retries: 2
 # => Unhealthy after 2 consecutive failures
 start_period: 60s
 # => Worker initialization grace period  # => Worker needs time to connect and initialize
 
 api:
 # => REST API service
 build: ./api
 # => Builds API from ./api/Dockerfile
 # => Runs on port 3000
 restart: unless-stopped  # => Restarts unless explicitly stopped
 depends_on:
 # => API requires healthy database and queue
 database:
 condition: service_healthy  # => API waits for healthy database
 message-queue:
 condition: service_healthy  # => API waits for healthy RabbitMQ
 healthcheck:
 # => HTTP health check for API
 test: ["CMD", "curl", "-f", "http://localhost:3000/health"]  # => HTTP health endpoint
 interval: 15s
 # => Check every 15 seconds
 timeout: 5s
 # => Fail if check takes longer than 5s
 retries: 3
 # => Unhealthy after 3 consecutive failures
 start_period: 30s  # => API needs 30s to initialize after dependencies ready
 ports:
 # => Port mapping for external access
 - "3000:3000"  # => API accessible at http://localhost:3000
 # => Maps container port 3000 to host port 3000
 
volumes:
 # => Named volumes defined at top level
 db-data:  # => Named volume — survives docker-compose down
 # => Managed by Docker, persists across container restarts

# Start all services
docker compose up -d
# => Startup order: database → message-queue → worker, api (health-check gated)
# => Services with condition: service_healthy wait for health checks to pass
 
# Simulate database crash
docker compose exec database pkill postgres
# => Database main process killed
# => Docker detects exit code != 0 and triggers restart policy
 
# Observe automatic recovery
docker compose ps -a
# => database: Restarting (restart policy: always)
# => worker: Up (waiting for database to be healthy)
# => api: Up (health check will fail until database recovers)
# => Worker and api remain Up but degrade gracefully until db recovers
 
# Wait for database to recover
sleep 40
 
docker compose ps
# => database: Up, healthy (auto-restarted)
# => worker: Up, healthy (reconnected after database recovery)
# => api: Up, healthy (health check passes after database recovery)
 
# Simulate repeated worker failures
docker compose exec worker sh -c 'exit 1'
# => Worker exits with error code 1
# => restart: on-failure:3 counts this as failure 1 of 3
 
# First restart attempt
sleep 5
docker compose ps worker
# => STATUS: Restarting (1/3 attempts)
# => Docker waits briefly before restart (exponential backoff)
 
# Simulate second failure
docker compose exec worker sh -c 'exit 1'
sleep 5
docker compose ps worker
# => STATUS: Restarting (2/3 attempts)
# => Backoff interval increases between attempts
 
# Simulate third failure
docker compose exec worker sh -c 'exit 1'
sleep 5
docker compose ps worker
# => STATUS: Exited (1) (max restarts reached)
# => Worker stops trying after 3 failures
# => Prevents infinite restart loop from crashing the host
 
# Manually restart worker after fixing issue
docker compose up -d worker
# => Restart count resets
# => Worker starts fresh
# => Use this after deploying a fix to the worker code
 
# Test API resilience during message queue restart
docker compose restart message-queue
# => Message queue restarts
 
docker compose ps
# => message-queue: Up (restarting)
# => api: Up (health check may fail temporarily)
# => worker: Up (will reconnect when queue is healthy)
 
# Wait for queue to become healthy
sleep 40
 
# All services recover automatically
docker compose ps
# => All services: Up, healthy

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # => Service definitions for CPU limit demo
 # CPU-intensive task (limited)
 video-encoder:
 # => Video encoding service
 image: my-encoder
 # => Uses custom encoder image
 deploy:
 # => Deployment configuration
 resources:
 # => Resource limits and reservations
 limits:
 # => Hard upper limit for CPU usage
 cpus: "2.0"
 # => Maximum 2 CPU cores
 # => Can burst to 2 cores max
 reservations:
 # => Guaranteed minimum resources
 cpus: "0.5"
 # => Guaranteed 0.5 CPU cores minimum
 # => Scheduler ensures this baseline
 command: encode --input video.mp4 --output compressed.mp4
 # => Encoding command with input/output paths
 # => --input and --output specify file paths
 
 # Web API (moderate CPU)
 api:
 # => REST API service
 image: my-api  # => Pre-built API image
 deploy:
 # => Deployment resource configuration
 resources:
 # => Resource constraints for API
 limits:
 cpus: "1.0"
 # => Maximum 1 CPU core — bursts capped here
 reservations:
 cpus: "0.25"
 # => Guaranteed 0.25 CPU cores minimum
 ports:
 # => Port mappings
 - "3000:3000"  # => API accessible at localhost:3000
 
 # Background worker (low priority — runs when API is idle)
 worker:
 # => Background processing service
 image: my-worker
 # => Uses custom worker image
 deploy:
 # => Worker deployment configuration
 resources:
 # => Resource constraints for worker
 limits:
 cpus: "0.5"
 # => Maximum 0.5 CPU cores (50% of 1 core)
 reservations:
 cpus: "0.1"
 # => Guaranteed 0.1 CPU cores minimum
 # CPU shares for relative priority under contention
 cpu_shares: 512  # => Default is 1024
 # => Worker gets half CPU time of default containers when all compete

# Run containers with CPU limits (requires Docker Swarm mode or docker run)
docker run -d --name encoder \
# => Create encoder container in detached mode
 --cpus="2.0" \
# => Hard limit: maximum 2 CPU cores
 --cpu-shares=1024 \
# => CPU priority weight (default=1024)
 my-encoder
# => Limited to 2 CPU cores maximum
# => --cpu-shares sets relative CPU priority
 
# Check CPU usage in real-time
docker stats encoder
# => CONTAINER CPU % MEM USAGE / LIMIT
# => encoder 200.00% 1.5GiB / 8GiB
# => CPU % capped at 200% (2 cores)
 
# Run CPU stress test inside container
docker exec encoder sh -c 'for i in $(seq 1 8); do
 sh -c "while true; do :; done" &
done'
# => Spawns 8 infinite loops (tries to use all CPUs)
 
docker stats encoder
# => encoder 200.00% (capped at 2 cores even with 8 busy loops)
 
# Without limits (comparison)
docker run -d --name encoder-unlimited my-encoder
docker exec encoder-unlimited sh -c 'for i in $(seq 1 8); do
 sh -c "while true; do :; done" &
done'
 
docker stats encoder-unlimited
# => encoder-unlimited 800.00% (uses all 8 cores)
 
# CPU shares test: Relative priority
docker run -d --name high-priority --cpu-shares=2048 alpine sh -c 'while true; do :; done'
# => Runs with high CPU priority (2048 shares = 2x default)
docker run -d --name low-priority --cpu-shares=512 alpine sh -c 'while true; do :; done'
# => Runs with low CPU priority (512 shares = half default)'
# => high-priority gets 4x more CPU time than low-priority when both compete
 
docker stats high-priority low-priority
# => high-priority 400.00% (4 cores)
# => low-priority 100.00% (1 core)
# => Ratio matches cpu-shares ratio (2048:512 = 4:1)
 
# Inspect CPU settings
docker inspect encoder --format='{{.HostConfig.NanoCpus}}'
# => 2000000000 (2.0 CPUs in nanoseconds)
 
docker inspect encoder --format='{{.HostConfig.CpuShares}}'
# => 1024 (default CPU shares)
 
# Clean up
docker rm -f encoder encoder-unlimited high-priority low-priority
# => Removes all test containers forcefully

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # => Service definitions with memory constraints
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => Alpine-based PostgreSQL 15
 deploy:
 # => Deployment resource configuration
 resources:
 # => Resource limits and reservations
 limits:
 # => Hard memory limit
 memory: 2G
 # => Hard limit: 2 gigabytes
 # => Container killed if exceeds (OOM kill)
 reservations:
 # => Guaranteed memory reservation
 memory: 1G
 # => Guaranteed minimum: 1 gigabyte
 environment:
 # => PostgreSQL environment variables
 POSTGRES_PASSWORD: secret
 # => Database superuser password
 
 redis:
 # => Redis in-memory cache service
 image: redis:7-alpine
 # => Alpine-based Redis 7
 deploy:
 # => Redis deployment configuration
 resources:
 # => Redis resource constraints
 limits:
 memory: 512M
 # => 512 megabytes max
 command: redis-server --maxmemory 400mb --maxmemory-policy allkeys-lru
 # => Redis internal limit (400MB) below Docker limit (512MB)
 # => LRU eviction when Redis reaches 400MB
 
 api:
 # => REST API service
 image: my-api
 # => Pre-built API image
 deploy:
 # => API deployment configuration
 resources:
 # => API resource constraints
 limits:
 memory: 1G   # => Hard limit: 1GB — OOM kill if exceeded
 reservations:
 memory: 256M  # => Guaranteed 256MB minimum
 environment:
 # => Node.js memory configuration
 NODE_OPTIONS: "--max-old-space-size=896"
 # => Node.js heap limit (896MB) below Docker limit (1GB)
 # => Prevents Node from triggering OOM kill before graceful degradation
 
 worker:
 # => Background worker service
 image: my-worker
 # => Pre-built worker image
 deploy:
 # => Worker deployment configuration
 resources:
 # => Worker resource constraints
 limits:
 memory: 512M  # => Hard limit: 512MB for worker process
 # Memory swap disabled for predictable performance
 mem_swappiness: 0
 # => Prevents swapping (keeps everything in RAM, no I/O latency)

# Run container with memory limit
docker run -d --name mem-limited \
# => Create memory-limited container
 --memory="512m" \
# => Hard memory limit: 512MB
 --memory-reservation="256m" \
# => Soft memory limit: 256MB
 my-app
# => Hard limit: 512MB (OOM kill if exceeded)
# => Soft limit: 256MB (preferred maximum)
 
# Monitor memory usage
docker stats mem-limited
# => CONTAINER MEM USAGE / LIMIT
# => mem-limited 312MiB / 512MiB
 
# Test OOM behavior (exceeds limit)
docker exec mem-limited sh -c '
 # Allocate 600MB (exceeds 512MB limit)
 python3 -c "s = \" \" * (600 * 1024 * 1024); import time; time.sleep(60)"
'
# => Container killed by OOM killer after a few seconds
 
docker ps -a --filter name=mem-limited
# => STATUS: Exited (137)
# => Exit code 137 = SIGKILL from OOM killer
 
# Check OOM kill event
docker inspect mem-limited --format='{{.State.OOMKilled}}'
# => true
 
# Memory swap control
docker run -d --name no-swap \
# => Container with no swap allowed
 --memory="1g" \
# => Hard memory limit: 1GB
 --memory-swap="1g" \
# => memory-swap=memory means NO swap
 my-app
# => Uses custom application image
# => memory-swap = memory means NO swap
# => All memory must be RAM (no swapping to disk)
 
docker run -d --name with-swap \
 --memory="1g" \
 --memory-swap="2g" \
 my-app
# => Can use 1GB RAM + 1GB swap (2GB total virtual memory)
 
# Memory reservation (soft limit)
docker run -d --name soft-limit \
 --memory-reservation="256m" \
 my-app
# => No hard limit, but tries to stay under 256MB
# => Can exceed 256MB if host has available memory
# => Reclaimed to 256MB when host is under pressure
 
docker stats soft-limit
# => MEM USAGE / LIMIT
# => 512MiB / unlimited
# => Can grow beyond 256MB reservation
 
# Inspect memory settings
docker inspect mem-limited --format='{{.HostConfig.Memory}}'
# => 536870912 (512MB in bytes)
 
docker inspect mem-limited --format='{{.HostConfig.MemoryReservation}}'
# => 268435456 (256MB in bytes)
 
# Clean up
docker rm -f mem-limited no-swap with-swap soft-limit

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # => Multi-service stack with resource limits
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => Alpine-based PostgreSQL 15 image
 deploy:
 # => Database deployment configuration
 resources:
 # => CPU and memory constraints
 limits:
 cpus: "2.0"   # => Max 2 CPU cores for database
 memory: 4G    # => Hard memory cap at 4GB
 reservations:
 # => Guaranteed minimum resources
 cpus: "1.0"   # => Guaranteed 1 CPU core
 memory: 2G   # => Guaranteed 2GB minimum
 environment:
 # => PostgreSQL configuration
 POSTGRES_PASSWORD: secret
 # => Superuser password
 # PostgreSQL configuration tuned to resource limits
 POSTGRES_SHARED_BUFFERS: 1GB
 # => 25% of memory reservation — shared memory for caching
 POSTGRES_EFFECTIVE_CACHE_SIZE: 3GB
 # => 75% of memory limit — query planner cache size hint
 volumes:
 # => Volume mounts
 - db-data:/var/lib/postgresql/data  # => Named volume for data persistence
 
 redis:
 # => Redis cache service
 image: redis:7-alpine
 # => Alpine-based Redis 7
 deploy:
 # => Redis deployment configuration
 resources:
 # => Redis CPU and memory limits
 limits:
 cpus: "1.0"   # => Redis rarely needs more than 1 CPU
 memory: 1G   # => Hard limit for cache service
 reservations:
 # => Redis minimum resource guarantee
 cpus: "0.25"  # => Guaranteed minimum CPU
 memory: 512M  # => Guaranteed minimum memory
 command: >
 redis-server
 --maxmemory 900mb
 --maxmemory-policy allkeys-lru
 --save ""
 # => maxmemory 900MB (90% of Docker limit — headroom for Redis overhead)
 # => LRU eviction evicts least-recently-used keys when full
 # => --save "" disables RDB persistence to reduce I/O
 
 api:
 build: ./api  # => Builds from ./api/Dockerfile
 deploy:
 # => API deployment with replicas
 resources:
 # => Per-replica resource limits
 limits:
 cpus: "1.5"   # => Each API replica capped at 1.5 cores
 memory: 2G   # => Each API replica capped at 2GB
 reservations:
 # => Per-replica minimum resources
 cpus: "0.5"   # => Each replica guaranteed 0.5 core
 memory: 512M  # => Each replica guaranteed 512MB
 replicas: 3
 # => 3 replicas for horizontal scaling and redundancy
 environment:
 # => API runtime configuration
 NODE_OPTIONS: "--max-old-space-size=1792"
 # => Node heap: 1792MB (90% of 2GB Docker limit — graceful OOM before kill)
 ports:
 # => Port mapping for API access
 - "3000-3002:3000"  # => Port range mapped (3000, 3001, 3002 → container:3000)
 
 worker:
 # => Background job worker service
 build: ./worker
 # => Builds from ./worker/Dockerfile
 deploy:
 # => Worker deployment configuration
 resources:
 # => Per-worker resource limits
 limits:
 cpus: "2.0"   # => Workers can use up to 2 cores for job processing
 memory: 1G   # => Memory limit per worker replica
 reservations:
 # => Per-worker minimum resources
 cpus: "0.5"   # => Guaranteed 0.5 core per worker
 memory: 256M  # => Guaranteed 256MB per worker
 replicas: 2   # => 2 worker replicas for parallel job processing
 environment:
 # => Worker job concurrency configuration
 MAX_JOBS: 4
 # => Limit concurrent jobs to match CPU limit (avoid CPU oversubscription)
 
volumes:
 # => Named volumes defined at stack level
 db-data:  # => Named volume — persists database data across container restarts

# Deploy stack with resource limits (Swarm mode)
docker stack deploy -c docker-compose.yml myapp
# => Creates services with specified resource constraints
 
# Monitor resource usage across all services
docker stats $(docker ps --format '{{.Names}}')
# => CONTAINER CPU % MEM USAGE / LIMIT
# => db-1 45.2% 1.8GiB / 4GiB
# => redis-1 12.3% 800MiB / 1GiB
# => api-1 28.1% 1.2GiB / 2GiB
# => api-2 31.4% 1.4GiB / 2GiB
# => api-3 25.7% 1.1GiB / 2GiB
# => worker-1 98.5% 700MiB / 1GiB
# => worker-2 102.3% 750MiB / 1GiB
 
# Total resource allocation
# => CPU: 2 + 1 + (1.5 * 3) + (2 * 2) = 11.5 cores reserved
# => Memory: 4 + 1 + (2 * 3) + (1 * 2) = 13GB reserved
 
# Load test to verify limits
# Install apache bench
docker run --rm --network host jordi/ab \
 -n 10000 -c 100 http://localhost:3000/api/test
# => 10000 requests, 100 concurrent
 
# During load test, verify API doesn't exceed limits
watch -n 1 'docker stats --no-stream $(docker ps --filter name=api --format "{{.Names}}")'
# => Each API instance stays under 1.5 CPUs and 2GB RAM
 
# Check for OOM kills during load
docker ps -a --filter name=api --format '{{.Names}}: {{.Status}}'
# => api-1: Up 5 minutes (healthy)
# => api-2: Up 5 minutes (healthy)
# => api-3: Up 5 minutes (healthy)
# => No OOM kills occurred
 
# Verify resource reservations (Swarm mode)
docker service inspect myapp_api --format='{{json .Spec.TaskTemplate.Resources}}' | jq
# => {
# => "Limits": {
# => "NanoCPUs": 1500000000,
# => "MemoryBytes": 2147483648
# => },
# => "Reservations": {
# => "NanoCPUs": 500000000,
# => "MemoryBytes": 536870912
# => }
# => }

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # Default JSON file logging (local)
 app-default:
 # => Application with local JSON file logging
 image: my-app
 # => Application image
 logging:
 # => Logging driver configuration
 driver: json-file
 # => Stores logs as JSON on local disk
 options:
 # => JSON file logging options
 max-size: "10m"
 # => Rotate when log file reaches 10MB
 max-file: "3"
 # => Keep 3 rotated files
 # => Total max: 30MB per container
 labels: "service,environment"
 # => Include service and environment labels in logs
 
 # Syslog driver (centralized)
 app-syslog:
 # => Application with syslog forwarding
 image: my-app
 # => Application image
 logging:
 # => Syslog driver configuration
 driver: syslog
 # => Forwards logs to syslog server
 options:
 # => Syslog connection options
 syslog-address: "tcp://192.168.1.100:514"
 # => Remote syslog server
 syslog-format: "rfc5424"
 # => RFC5424 format (structured)
 tag: "{{.Name}}/{{.ID}}"
 # => Log tag with container name and ID
 
 # Fluentd driver (EFK stack)
 app-fluentd:
 # => Application with Fluentd forwarding
 image: my-app
 # => Application image
 logging:
 # => Fluentd driver configuration
 driver: fluentd
 # => Forwards logs to Fluentd aggregator
 options:
 # => Fluentd connection options
 fluentd-address: "localhost:24224"
 # => Fluentd forwarder address
 fluentd-async: "true"
 # => Non-blocking async mode
 tag: "docker.{{.Name}}"
 # => Tag for fluentd routing
 depends_on:
 # => Requires Fluentd to be running
 - fluentd
 # => Wait for fluentd service
 
 # GELF driver (Graylog/ELK)
 app-gelf:
 # => Application with GELF logging (Graylog Extended Log Format)
 image: my-app
 # => Application image
 logging:
 # => GELF driver configuration
 driver: gelf
 # => Forwards logs in GELF format
 options:
 # => GELF connection options
 gelf-address: "udp://graylog.example.com:12201"
 # => Graylog server
 tag: "{{.Name}}"
 # => Container name as log tag
 gelf-compression-type: "gzip"
 # => Compress logs before sending
 
 # Splunk driver
 app-splunk:
 # => Application with Splunk HEC logging
 image: my-app
 # => Application image
 logging:
 # => Splunk driver configuration
 driver: splunk
 # => Forwards logs to Splunk HEC
 options:
 # => Splunk HTTP Event Collector options
 splunk-token: "B5A79AAD-D822-46CC-80D1-819F80D7BFB0"
 # => Splunk HEC authentication token
 splunk-url: "https://splunk.example.com:8088"
 # => Splunk HEC endpoint URL
 splunk-insecureskipverify: "false"
 # => Verify SSL certificate
 
 # AWS CloudWatch driver (sends logs to AWS CloudWatch Logs)
 app-cloudwatch:
 # => Application with CloudWatch logging
 image: my-app
 # => Application image
 logging:
 # => CloudWatch driver configuration
 driver: awslogs  # => Requires AWS credentials on host (IAM role or env vars)
 options:
 # => CloudWatch Logs options
 awslogs-region: "us-east-1"   # => AWS region for CloudWatch Logs
 awslogs-group: "myapp"         # => Log group name in CloudWatch
 awslogs-stream: "{{.Name}}"   # => Log stream per container (by name)
 awslogs-create-group: "true"  # => Auto-create log group if it doesn't exist
 
 # Fluentd aggregator service
 fluentd:
 # => Fluentd aggregator service
 image: fluent/fluentd:v1.16-1  # => Official Fluentd forwarder image
 ports:
 # => Fluentd input port
 - "24224:24224"  # => Fluentd forward input port (TCP/UDP)
 volumes:
 # => Fluentd configuration file
 - ./fluentd.conf:/fluentd/etc/fluent.conf  # => Bind-mount custom config

# File: fluentd.conf
<source>
# => Input plugin configuration
 @type forward
# => Accept logs via Fluentd forward protocol
 port 24224
# => Listen on port 24224 (TCP/UDP)
 bind 0.0.0.0
# => Accept connections from any interface
</source>
 
<match docker.**>
# => Match all docker.* tagged logs
 @type file
# => Write matched logs to files
 path /fluentd/log/docker
# => Output file path prefix
 append true
# => Append to existing files
 <format>
# => Output format configuration
 @type json
# => Write logs as JSON
 </format>
 <buffer>
# => Buffering configuration
 flush_interval 10s
# => Flush buffer to disk every 10 seconds
 </buffer>
# => Buffer block end
</match>
# => Elasticsearch match block end

# Start services with different logging drivers
docker compose up -d
# => Starts all logging driver configurations
 
# View logs from json-file driver (works with docker logs)
docker logs app-default
# => Shows logs via docker logs command
# => Works only with json-file driver
# => Stored in /var/lib/docker/containers/<id>/<id>-json.log
 
# Check log file size and rotation
docker inspect app-default --format='{{.LogPath}}'
# => /var/lib/docker/containers/../..-json.log
 
ls -lh $(docker inspect app-default --format='{{.LogPath}}')
# => -rw-r----- 1 root root 8.5M Dec 29 11:00 ..-json.log
# => File rotates at 10MB
 
# Syslog driver: docker logs doesn't work (logs sent remotely)
docker logs app-syslog
# => Error: configured logging driver does not support reading
# => Syslog driver does not buffer logs locally
 
# Check syslog server instead
ssh syslog-server "tail -f /var/log/syslog | grep app-syslog"
# => Access logs directly on the syslog server
# => Dec 29 11:00:00 app-syslog[12345]: Application started
 
# Fluentd driver: Logs sent to fluentd
docker logs app-fluentd
# => Error: configured logging driver does not support reading
# => Fluentd driver does not buffer logs locally
 
# Check fluentd logs
docker exec fluentd cat /fluentd/log/docker.log
# => Reads aggregated logs from fluentd container
# => {"container_name":"app-fluentd","message":"Application log"}
 
# GELF driver: Logs sent to Graylog
# Access Graylog web UI: http://graylog.example.com:9000
# Search for logs: source:app-gelf
 
# Inspect logging configuration
docker inspect app-default --format='{{json .HostConfig.LogConfig}}' | jq
# => {
# => "Type": "json-file",
# => "Config": {
# => "max-size": "10m",
# => "max-file": "3"
# => }
# => }
 
# Change logging driver for running container (requires restart)
docker update --log-driver=syslog app-default
# => Note: Requires container restart to take effect
 
# Set default logging driver globally (daemon.json)
cat > /etc/docker/daemon.json << 'EOF'
{
 "log-driver": "json-file",
 "log-opts": {
 "max-size": "10m",
 "max-file": "3"
 }
}
EOF
 
sudo systemctl restart docker
# => All new containers use this driver by default

// File: logger.js (Winston structured logger)
const winston = require("winston");
 
const logger = winston.createLogger({
  level: process.env.LOG_LEVEL || "info",
  format: winston.format.combine(
    winston.format.timestamp({
      format: "YYYY-MM-DDTHH:mm:ss.SSSZ",
    }),
    winston.format.errors({ stack: true }),
    winston.format.json(),
    // => JSON format for structured logging
  ),
  defaultMeta: {
    service: "api",
    environment: process.env.NODE_ENV,
    container_id: process.env.HOSTNAME,
    // => HOSTNAME env var = container ID in Docker
  },
  transports: [
    new winston.transports.Console(),
    // => Logs to stdout (captured by Docker)
  ],
});
 
module.exports = logger;

// File: app.js (Example usage)
const express = require("express");
const logger = require("./logger");
 
const app = express();
 
// Request logging middleware
app.use((req, res, next) => {
  logger.info("HTTP request", {
    method: req.method,
    path: req.path,
    ip: req.ip,
    user_agent: req.get("user-agent"),
    request_id: req.id,
  });
  next();
});
 
app.get("/api/users/:id", async (req, res) => {
  const userId = req.params.id;
 
  logger.debug("Fetching user", { user_id: userId });
  // => {
  // => "timestamp": "2025-12-29T11:10:00.123Z",
  // => "level": "debug",
  // => "message": "Fetching user",
  // => "user_id": "12345",
  // => "service": "api",
  // => "environment": "production"
  // => }
 
  try {
    const user = await db.getUser(userId);
 
    logger.info("User retrieved successfully", {
      user_id: userId,
      duration_ms: 45,
    });
 
    res.json(user);
  } catch (error) {
    logger.error("Failed to fetch user", {
      user_id: userId,
      error: error.message,
      stack: error.stack,
    });
    // => {
    // => "timestamp": "2025-12-29T11:10:00.500Z",
    // => "level": "error",
    // => "message": "Failed to fetch user",
    // => "user_id": "12345",
    // => "error": "Database connection timeout",
    // => "stack": "Error: Database connection timeout\n at ..",
    // => "service": "api",
    // => "container_id": "abc123def456"
    // => }
 
    res.status(500).json({ error: "Internal server error" });
  }
});
 
app.listen(3000, () => {
  logger.info("Server started", { port: 3000 });
});

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # => Service configuration for structured logging
 api:
 # => API service with JSON structured logging
 build: .
 # => Builds from Dockerfile in current directory
 environment:
 # => Runtime environment variables
 NODE_ENV: production
 # => Activates production logging (no pretty-print)
 LOG_LEVEL: info
 # => Controls Winston log level (info/warn/error in production)
 logging:
 # => Logging driver configuration
 driver: json-file
 # => Docker's built-in JSON log driver
 options:
 # => JSON file rotation options
 max-size: "10m"
 # => Rotate log file when it reaches 10MB
 max-file: "3"
 # => Keep 3 rotated files (30MB max per container)
 labels: "service,environment"
 # => Include these labels in log metadata
 labels:
 # => Container labels for log routing
 service: "api"
 # => Added to all log lines for multi-service filtering
 environment: "production"
 # => Enables environment-based log routing

# View structured logs
docker logs api --tail 20
# => Shows last 20 log lines from api container
# => Each line is a JSON object with all structured fields
# => {"timestamp":"2025-12-29T11:10:00.123Z","level":"info","message":"Server started","port":3000,"service":"api","environment":"production","container_id":"abc123"}
# => Fields: timestamp, level, message, service, environment, container_id
# => {"timestamp":"2025-12-29T11:10:01.456Z","level":"info","message":"HTTP request","method":"GET","path":"/api/users/12345","ip":"::ffff:172.18.0.1","service":"api"}
# => HTTP requests logged with method, path, ip automatically
# => {"timestamp":"2025-12-29T11:10:01.500Z","level":"debug","message":"Fetching user","user_id":"12345","service":"api"}
# => Debug logs include user_id for full request traceability
 
# Filter logs with jq (extract specific fields)
docker logs api --tail 100 2>&1 | grep '^{' | jq 'select(.level=="error")'
# => Pipes last 100 log lines through grep to filter JSON-only lines
# => jq selects only entries where level=="error"
# => {
# => "timestamp": "2025-12-29T11:10:00.500Z",
# => "level": "error",
# => "message": "Failed to fetch user",
# => "user_id": "12345",
# => "error": "Database connection timeout"
# => }
# => Only error-level log entries are returned
 
# Extract all error messages from last 1000 lines
docker logs api --tail 1000 2>&1 | \
# => Fetch last 1000 lines including stderr (2>&1)
 grep '^{' | \
# => Filter for lines starting with { (valid JSON objects only)
 jq -r 'select(.level=="error") | "\(.timestamp) \(.message) [\(.error)]"'
# => Formats as "timestamp message [error details]" per line
# => 2025-12-29T11:10:00.500Z Failed to fetch user [Database connection timeout]
# => -r flag outputs raw strings without JSON quoting
# => 2025-12-29T11:12:30.789Z Payment processing failed [Stripe API error]
 
# Count log levels
docker logs api --tail 1000 2>&1 | \
# => Fetch last 1000 log entries including stderr
 grep '^{' | \
# => Filter valid JSON lines
 jq -r '.level' | \
# => Extract just the level field value from each entry
 sort | uniq -c
# => sort groups same values, uniq -c counts occurrences of each level
# => 850 info
# => 120 debug
# => 25 warn
# => 5 error
 
# Find slowest API requests (using duration_ms field)
docker logs api --tail 1000 2>&1 | \
# => Scan last 1000 log entries for request performance analysis
 grep '^{' | \
# => Filter to valid JSON lines only
 jq -r 'select(.duration_ms != null) | "\(.duration_ms) \(.method) \(.path)"' | \
# => Extract entries with duration_ms, format as "ms METHOD /path"
 sort -n | tail -10
# => sort -n sorts numerically by duration, tail -10 shows 10 slowest
# => 1250 GET /api/reports/monthly
# => 1450 POST /api/exports/csv
# => 2100 GET /api/analytics/dashboard
 
# Export logs to file for analysis
docker logs api --since 24h > api-logs-$(date +%Y%m%d).json
# => --since 24h: retrieves logs from last 24 hours only
# => date +%Y%m%d creates filename like api-logs-20251229.json
# => Creates daily log file for archival or offline analysis
# => Structured log field reference:
# => timestamp: ISO8601 timestamp of log event
# => level: Log severity (error/warn/info/debug)
# => message: Human-readable event description
# => service: Service name for multi-service filtering
# => environment: Deployment environment (production/staging)
# => container_id: Docker container ID for correlation
# => request_id: Unique ID to trace requests across services
# => user_id: User identifier for per-user log filtering
# => duration_ms: Request duration in milliseconds
# => method: HTTP method (GET/POST/PUT/DELETE)
# => path: URL path of the HTTP request
# => ip: Client IP address
# => stack: Stack trace for error entries
# => error: Error message string
# => Log analysis tips:
# => Use jq .level to extract all log levels
# => Use jq .service to filter by service name
# => Use jq select() to filter by field values
# => Use jq -r for raw string output
# => Pipe through sort | uniq -c for frequency counts
# => Use --since flag to limit time range of logs
# => Use --tail to limit number of lines
# => Combine 2>&1 to include stderr in pipe
# => grep '^{' filters only valid JSON lines
# => sort -n sorts numerically for duration analysis
# => tail -N shows last N lines (slowest/most recent)

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
# => EFK stack: 5 services — api, web, fluentd, elasticsearch, kibana
# => All api/web logs flow: container stdout → fluentd → elasticsearch → kibana
 
services:
 # Application services using fluentd driver
 api:
 # => Backend API service
 build: ./api  # => Build from ./api/Dockerfile
 logging:
 # => Logging driver for API
 # => fluentd driver replaces default json-file driver for this service
 driver: fluentd  # => Send logs to Fluentd forwarder
 options:
 # => Fluentd connection options
 fluentd-address: localhost:24224  # => Fluentd listening address
 tag: "docker.api"  # => Tag identifies log source as "api"
 depends_on:
 # => Service startup ordering
 - fluentd  # => Fluentd must start before api (logs have nowhere to go otherwise)
 
 web:
 # => Frontend web service
 build: ./web
 # => Build from ./web/Dockerfile
 logging:
 # => Web service logging configuration
 driver: fluentd
 # => Send web logs to Fluentd
 options:
 # => Fluentd connection for web
 fluentd-address: localhost:24224
 # => Same Fluentd instance as API
 tag: "docker.web"  # => Tag distinguishes web logs from api logs in Elasticsearch
 depends_on:
 # => Web depends on Fluentd
 - fluentd
 # => Fluentd collects web logs
 
 # Fluentd log forwarder — collects from all containers, routes to Elasticsearch
 fluentd:
 # => Fluentd log aggregation service
 image: fluent/fluentd:v1.16-debian-1  # => Debian variant supports Elasticsearch plugin
 ports:
 # => Fluentd input ports
 - "24224:24224"      # => TCP forward input
 - "24224:24224/udp"  # => UDP forward input (high-volume logging)
 volumes:
 # => Fluentd configuration mounts
 - ./fluentd/fluent.conf:/fluentd/etc/fluent.conf  # => Custom routing config
 - ./fluentd/plugins:/fluentd/plugins               # => Custom Fluentd plugins
 environment:
 # => Elasticsearch connection for Fluentd
 FLUENT_ELASTICSEARCH_HOST: elasticsearch  # => Docker DNS name of ES container
 FLUENT_ELASTICSEARCH_PORT: 9200           # => Elasticsearch REST API port
 depends_on:
 # => Fluentd must wait for Elasticsearch
 - elasticsearch  # => Elasticsearch must be running before Fluentd starts routing
 
 # Elasticsearch — search engine and log storage
 elasticsearch:
 # => Elasticsearch search and storage engine
 image: docker.elastic.co/elasticsearch/elasticsearch:8.11.0
 # => Version 8.11.0 with full-text search capabilities
 environment:
 # => Elasticsearch startup configuration
 - discovery.type=single-node  # => Single-node mode (no cluster needed for dev)
 - xpack.security.enabled=false  # => Disable auth for development simplicity
 - "ES_JAVA_OPTS=-Xms512m -Xmx512m"
 # => Heap size: 512MB min/max (prevents Java GC pauses from heap growth)
 ports:
 # => Elasticsearch API ports
 - "9200:9200"  # => Elasticsearch REST API
 volumes:
 # => Persistent data storage
 - es-data:/usr/share/elasticsearch/data  # => Named volume for index persistence
 
 # Kibana — log visualization and search UI
 kibana:
 # => Kibana visualization dashboard
 image: docker.elastic.co/kibana/kibana:8.11.0
 # => Kibana version matches Elasticsearch 8.11.0
 ports:
 # => Kibana web interface
 - "5601:5601"  # => Kibana web UI at http://localhost:5601
 environment:
 # => Kibana Elasticsearch connection
 ELASTICSEARCH_HOSTS: http://elasticsearch:9200  # => ES endpoint via Docker DNS
 depends_on:
 # => Kibana requires Elasticsearch
 - elasticsearch  # => Kibana requires running Elasticsearch
 
volumes:
 # => Named volumes for persistence
 es-data:  # => Named volume — ES indices survive docker-compose down
 # => Stores Elasticsearch index data persistently

# File: fluentd/fluent.conf
 
<source>
# => Input source configuration
 @type forward
# => Accept logs via Fluentd forward protocol
 port 24224
# => Listen on port 24224
 bind 0.0.0.0
# => Accept from all network interfaces
</source>
# => Input source block end
 
# Parse JSON logs
<filter docker.**>
# => Filter plugin for all docker.* tagged logs
 @type parser
# => Parse log field content as structured data
 key_name log
# => Parse the "log" field from container stdout
 <parse>
# => Parser configuration
 @type json
# => Parse log field as JSON
 </parse>
# => Parse block end
</filter>
# => JSON parser filter block end
 
# Add metadata
<filter docker.**>
# => Add metadata to all docker.* logs
 @type record_transformer
# => Transform records by adding fields
 <record>
# => Fields to add to every log entry
 hostname "#{Socket.gethostname}"
# => Add Fluentd host name to identify log source host
 fluentd_timestamp ${time}
# => Add Fluentd processing timestamp
 </record>
# => Record block end
</filter>
# => Record transformer filter block end
 
# Send to Elasticsearch
<match docker.**>
# => Output plugin for all docker.* tagged logs
 @type elasticsearch
# => Use Elasticsearch output plugin
 host "#{ENV['FLUENT_ELASTICSEARCH_HOST']}"
# => ES host from environment variable
 port "#{ENV['FLUENT_ELASTICSEARCH_PORT']}"
# => ES port from environment variable
 logstash_format true
# => Use Logstash-compatible index naming
 logstash_prefix docker
# => Index prefix: docker-YYYY.MM.DD
# => Creates time-series indices for easy retention management
 include_tag_key true
# => Include Fluentd tag in the document
 tag_key @log_name
# => Field name for the tag value
 <buffer>
# => Buffering for reliable delivery
 @type file
# => Store buffer on disk (survives Fluentd restarts)
 path /fluentd/log/buffer
# => Buffer file location
 flush_interval 10s
# => Send buffered logs every 10 seconds
 retry_max_interval 30s
# => Max backoff interval between retries
 retry_forever true
# => Keep retrying on ES failures (no data loss)
 </buffer>
</match>

# Start EFK stack
docker compose up -d
# => Starts elasticsearch, fluentd, kibana, api, web
# => Startup order: elasticsearch first (fluentd depends on it)
# => Elasticsearch takes 30-60 seconds before accepting connections
 
# Wait for Elasticsearch to be ready
until curl -s http://localhost:9200/_cluster/health | grep -q '"status":"green\|yellow"'; do
# => Poll cluster health until green or yellow status
# => green = all shards allocated, yellow = some replica shards missing (normal for single-node)
 echo "Waiting for Elasticsearch.."
# => Print status message while waiting
 sleep 5
# => Wait 5 seconds between health checks
done
# => Elasticsearch is now ready for indexing
 
# Generate some application logs
curl http://localhost:3000/api/test
# => Logs sent to fluentd → elasticsearch
 
# Query Elasticsearch directly
curl -X GET "localhost:9200/docker-*/_search?pretty" -H 'Content-Type: application/json' -d'
# => GET request to search all docker-* indices
{
 "query": {
 "match": {
 "level": "error"
 }
 },
 "size": 10,
 "sort": [
 { "@timestamp": "desc" }
 ]
}'
# => Returns last 10 error logs as JSON
# => sorted by timestamp descending
 
# Count logs by level
curl -X GET "localhost:9200/docker-*/_search?pretty" -H 'Content-Type: application/json' -d'
# => Aggregation query to count log levels
{
 "size": 0,
 "aggs": {
 "levels": {
 "terms": {
 "field": "level.keyword"
 }
 }
 }
}'
# => size: 0 means return only aggregation results
# => terms aggregation groups by level.keyword field
# => {
# => "aggregations": {
# => "levels": {
# => "buckets": [
# => { "key": "info", "doc_count": 1250 },
# => { "key": "debug", "doc_count": 340 },
# => { "key": "error", "doc_count": 15 },
# => { "key": "warn", "doc_count": 8 }
# => ]
# => }
# => }
# => }
 
# Access Kibana web UI
# => Open http://localhost:5601
# => Create index pattern: docker-*
# => Discover tab: View real-time logs
# => Visualize tab: Create dashboards
 
# Example Kibana query (KQL - Kibana Query Language)
# => level: "error" AND service: "api"
# => Shows only API error logs
 
# Create Kibana dashboard with visualizations:
# => 1. Error rate over time (line chart)
# => 2. Log level distribution (pie chart)
# => 3. Top error messages (data table)
# => 4. Request duration percentiles (histogram)
 
# Backup Elasticsearch data
docker exec elasticsearch curl -X PUT "localhost:9200/_snapshot/backup" -H 'Content-Type: application/json' -d'
# => Create snapshot repository via Elasticsearch API
{
 "type": "fs",
 "settings": {
 "location": "/usr/share/elasticsearch/backup"
 }
}'
# => Creates backup repository pointing to filesystem path
 
docker exec elasticsearch curl -X PUT "localhost:9200/_snapshot/backup/snapshot_1?wait_for_completion=true"
# => Creates snapshot named snapshot_1 from all indices
# => wait_for_completion=true: blocks until snapshot completes

# File: docker-compose.yml
 
version: "3.8"
# => Three services: frontend (web tier), api (app tier), database (data tier)
# => Network segmentation enforces security boundaries between tiers
 
services:
 frontend:
 image: nginx:alpine
 networks:
 - frontend-net
 # => Isolated network for frontend services
 # => frontend cannot reach database — not on backend-net
 ports:
 - "8080:80"
 # => Only frontend is exposed to host — single entry point
 
 api:
 image: my-api
 networks:
 - frontend-net
 - backend-net
 # => Connects to both networks (bridge between frontend and backend)
 # => api is the only service with access to both tiers
 environment:
 DATABASE_URL: postgresql://postgres:secret@database:5432/mydb
 # => Uses service name "database" for DNS resolution
 # => DNS works because api and database share backend-net
 
 database:
 image: postgres:15-alpine
 networks:
 - backend-net
 # => Only accessible from backend network (isolated from frontend)
 # => No ports exposed — database unreachable from outside backend-net
 environment:
 POSTGRES_PASSWORD: secret
 volumes:
 - db-data:/var/lib/postgresql/data
 # => Named volume: data persists across container restarts
 
networks:
 frontend-net:
 driver: bridge
 # => Custom bridge network with automatic DNS
 # => Containers on this network resolve each other by service name
 ipam:
 config:
 - subnet: 172.20.0.0/16
 # => Custom subnet (avoids conflicts)
 # => Explicit subnet prevents overlap with host network ranges
 backend-net:
 driver: bridge
 internal: true
 # => Internal-only network (no external access)
 # => database cannot make outbound internet requests (data exfiltration prevention)
 ipam:
 config:
 - subnet: 172.21.0.0/16
 # => Different subnet range from frontend-net
 
volumes:
 db-data:
 # => Named volume managed by Docker (persists across restarts)

# Start services with custom networks
docker compose up -d
 
# List networks
docker network ls
# => NETWORK ID NAME DRIVER SCOPE
# => abc123def456 myproject_frontend-net bridge local
# => def456ghi789 myproject_backend-net bridge local
 
# Inspect network
docker network inspect myproject_frontend-net
# => {
# => "Name": "myproject_frontend-net",
# => "Driver": "bridge",
# => "IPAM": {
# => "Config": [{ "Subnet": "172.20.0.0/16" }]
# => },
# => "Containers": {
# => "abc123": { "Name": "frontend", "IPv4Address": "172.20.0.2/16" },
# => "def456": { "Name": "api", "IPv4Address": "172.20.0.3/16" }
# => }
# => }
 
# Test DNS resolution (automatic service discovery)
docker exec frontend ping -c 1 api
# => PING api (172.20.0.3): 56 data bytes
# => 64 bytes from 172.20.0.3: seq=0 ttl=64 time=0.123 ms
# => DNS resolution works (service name → IP)
 
# Test network isolation (frontend CANNOT reach database directly)
docker exec frontend ping -c 1 database
# => ping: bad address 'database'
# => Database not reachable from frontend network (security isolation)
 
# API CAN reach database (connected to both networks)
docker exec api ping -c 1 database
# => PING database (172.21.0.2): 56 data bytes
# => 64 bytes from 172.21.0.2: seq=0 ttl=64 time=0.098 ms
 
# Create standalone network manually
docker network create \
 --driver bridge \
 --subnet 172.22.0.0/16 \
 --gateway 172.22.0.1 \
 custom-network
# => Network created with custom configuration
 
# Connect running container to network
docker network connect custom-network frontend
# => frontend now connected to custom-network
 
# Disconnect from network
docker network disconnect custom-network frontend
# => frontend disconnected from custom-network
 
# Network aliases (multiple DNS names)
docker run -d --name web \
 --network frontend-net \
 --network-alias webapp \
 --network-alias www \
 nginx:alpine
# => Container reachable via: web, webapp, www
 
docker exec api ping -c 1 webapp
# => Resolves to web container
 
# Clean up unused networks
docker network prune
# => WARNING! This will remove all custom networks not used by at least one container.

# Install networking tools in Alpine-based container
docker run -it --rm alpine sh
# Inside container:
apk add --no-cache curl wget netcat-openbsd bind-tools iproute2
# => curl: HTTP requests | wget: Downloads | netcat: Port testing
# => bind-tools: DNS queries (dig, nslookup) | iproute2: ip, ss commands
 
# Test HTTP connectivity
curl -v http://api:3000/health
# => Shows full HTTP request/response with timing
 
# Test port connectivity
nc -zv database 5432
# => Connection to database 5432 port [tcp/postgresql] succeeded!
 
# DNS troubleshooting
nslookup api
# => Server: 127.0.0.11 (Docker embedded DNS)
# => Name: api, Address: 172.20.0.3
 
dig api
# => Shows DNS query details and TTL
 
# Check network interfaces
ip addr show
# => eth0@if10: inet 172.20.0.2/16 — container's IP on Docker network
 
# Show routing table
ip route show
# => default via 172.20.0.1 dev eth0 — gateway is the bridge network
 
# Check listening ports
ss -tuln
# => tcp LISTEN 0.0.0.0:80 — process is bound and accepting connections
 
# Network performance testing with iperf3
docker run --rm -it --name iperf-server --network frontend-net \
 networkstatic/iperf3 -s
# => Server listening on 5201
docker run --rm -it --network frontend-net \
 networkstatic/iperf3 -c iperf-server
# => [ 5] 0.00-10.00 sec 10.0 GBytes 8.59 Gbits/sec (inter-container bandwidth)
 
# Packet capture with tcpdump
docker run --rm --net=host \
 --cap-add=NET_ADMIN \
 nicolaka/netshoot \
 tcpdump -i any port 80 -n
# => Captures HTTP traffic on all interfaces (requires NET_ADMIN cap)
 
# All-in-one network troubleshooting container
docker run --rm -it --network myproject_frontend-net \
 nicolaka/netshoot
# => Includes curl, dig, ping, traceroute, tcpdump, iperf3 — no installation needed
 
# Check container network configuration from host
docker inspect api --format='{{json .NetworkSettings.Networks}}' | jq
# => "frontend-net": {"IPAddress": "172.20.0.3", "Gateway": "172.20.0.1"}
# => "backend-net": {"IPAddress": "172.21.0.2"} — multi-network container
 
# Test connectivity from host to container
docker port api
# => 3000/tcp -> 0.0.0.0:3000
 
# Network traffic inspection
docker stats --no-stream --format "table {{.Container}}\t{{.NetIO}}"
# => api 1.2MB / 850kB | frontend 450kB / 120kB | database 200kB / 180kB
 
# Logs for network-related errors
docker logs api 2>&1 | grep -i "connection\|network\|timeout"
# => Filters logs for connection/network/timeout messages

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # Database (must be ready before API)
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => Alpine-based PostgreSQL 15
 environment:
 # => Database initialization settings
 POSTGRES_PASSWORD: secret   # => PostgreSQL superuser password
 POSTGRES_DB: myapp           # => Creates "myapp" database on first start
 healthcheck:
 # => Health check configuration
 test: ["CMD-SHELL", "pg_isready -U postgres"]  # => Checks if DB accepts connections
 interval: 5s   # => Check every 5 seconds (frequent for fast init detection)
 timeout: 3s
 # => Fail if check takes longer than 3s
 retries: 10    # => Up to 10 retries (50s total before unhealthy)
 volumes:
 # => Volume mounts for persistence
 - db-data:/var/lib/postgresql/data  # => Named volume for data persistence
 
 # Init container: Wait for database and run migrations
 db-migrate:
 # => Database migration init container
 image: my-api:latest
 # => Reuses app image for migration scripts
 command: npm run migrate
 # => Runs database migrations
 depends_on:
 # => Migration waits for healthy database
 database:
 # => Depends on database service
 condition: service_healthy
 # => Waits for database to be healthy
 environment:
 # => Database connection for migrations
 DATABASE_URL: postgresql://postgres:secret@database:5432/myapp
 # => Full connection string with credentials
 restart: "no"
 # => Runs once, doesn't restart on failure
 
 # Init container: Seed initial data
 db-seed:
 # => Database seeding init container
 image: my-api:latest
 # => Reuses app image for seed scripts
 command: npm run seed
 # => Seeds initial data (admin user, default settings)
 depends_on:
 # => Seed waits for migrations to complete
 db-migrate:
 # => Depends on migration service
 condition: service_completed_successfully
 # => Waits for migrations to complete successfully
 environment:
 # => Database connection for seeding
 DATABASE_URL: postgresql://postgres:secret@database:5432/myapp
 # => Same connection string as migration
 restart: "no"
 # => Runs once, doesn't restart
 
 # Main application (starts after init containers)
 api:
 # => Main API service
 image: my-api:latest
 # => Production API image
 depends_on:
 # => API waits for all init tasks
 database:
 # => Wait for healthy database
 condition: service_healthy
 # => Database must pass health check
 db-migrate:
 # => Wait for migrations
 condition: service_completed_successfully
 # => Migrations must complete successfully
 db-seed:
 # => Wait for data seeding
 condition: service_completed_successfully
 # => Starts only after all init tasks complete
 environment:
 # => API database configuration
 DATABASE_URL: postgresql://postgres:secret@database:5432/myapp
 # => Full database connection URL
 ports:
 # => API port mapping
 - "3000:3000"
 # => API accessible at localhost:3000
 restart: unless-stopped
 # => Restart on failure but not manual stop
 
volumes:
 db-data:
 # => Named volume for database persistence
 # => Persists data across container restarts

# Start services (observe init container execution order)
docker compose up -d
# => Starts all services in dependency order
# => [+] Running 4/4
# => ⠿ Container database Started (0.5s)
# => ⠿ Container db-migrate Started (10.2s) ← Waits for database health
# => ⠿ Container db-seed Started (15.8s) ← Waits for migration completion
# => ⠿ Container api Started (16.1s) ← Starts after all init tasks
 
# Check init container logs
docker compose logs db-migrate
# => Shows stdout from db-migrate container
# => Connecting to database..
# => Running migration 001_create_users_table.. OK
# => Running migration 002_create_posts_table.. OK
# => Migrations completed successfully
 
docker compose logs db-seed
# => Shows stdout from db-seed container
# => Seeding admin user..
# => Seeding default settings..
# => Seed completed successfully
 
# Init container exits after completion
docker compose ps
# => Lists all service statuses
# => NAME STATUS PORTS
# => database Up (healthy) 5432/tcp
# => db-migrate Exited (0) 30 seconds ago
# => db-seed Exited (0) 25 seconds ago
# => api Up 0.0.0.0:3000->3000/tcp
 
# Restart scenario: Init containers DON'T run again
docker compose restart api
# => Restarts only the api service
# => db-migrate and db-seed don't restart (one-time tasks)
 
# Full recreate: Init containers run again
docker compose down
# => Stops and removes all containers, networks, and volumes
# => Stops and removes all containers
docker compose up -d
# => Re-creates all containers from scratch
# => Init containers execute again on fresh deployment
 
# Alternative: Shell script as init container
cat > init.sh << 'EOF'
# => Create init.sh script file
#!/bin/sh
# => Shell shebang - uses /bin/sh
set -e
# => Exit on any error
 
echo "Waiting for database.."
# => Print status before polling
until nc -z database 5432; do
# => nc -z tests TCP connectivity to database:5432
 echo "Database not ready, retrying in 2s.."
# => Print retry message
 sleep 2
# => Wait 2 seconds before next check
done
echo "Database ready!"
# => Database is accepting connections
 
echo "Running migrations.."
# => Start migration phase
npm run migrate
# => Execute migration script from package.json
 
echo "Seeding data.."
# => Start data seeding phase
npm run seed
# => Execute seed script from package.json
 
echo "Init completed successfully"
# => All init tasks done
EOF
 
chmod +x init.sh
# => Make script executable
# => Required for Docker to execute the script
 
# Use script in Compose
cat >> docker-compose.yml << 'EOF'
# => Append init service definition to compose file
 init:
# => Init service definition
 image: my-api:latest
# => Same image as main app
 command: ["/init.sh"]
# => Run the init script
 volumes:
# => Mount the init script
 - ./init.sh:/init.sh:ro
# => Read-only mount of init.sh
 depends_on:
# => Wait for database
 - database
# => Init runs after database starts
 restart: "no"
# => Run once and exit
EOF
# => Appends init service to docker-compose.yml
 
# Multiple parallel init tasks
cat > docker-compose-parallel-init.yml << 'EOF'
# => Create parallel init containers compose file
version: '3.8'
# => Compose file version
 
services:
 # Init task 1: Download config from S3
 config-downloader:
 # => S3 config download init container
 image: amazon/aws-cli
# => AWS CLI image for S3 access
 command: s3 cp s3://my-bucket/config.json /config/
# => Download config file from S3 bucket
 volumes:
 - config-data:/config
# => Store downloaded config in named volume
 restart: "no"
# => Run once and exit
 
 # Init task 2: Generate SSL certificates
 cert-generator:
 # => SSL certificate generation init container
 image: alpine/openssl
# => Minimal OpenSSL image
 command: sh -c "openssl req -x509 -newkey rsa:4096 -keyout /certs/key.pem -out /certs/cert.pem -days 365 -nodes -subj '/CN=localhost'"
# => Generate self-signed certificate valid for 365 days
 volumes:
 - cert-data:/certs
# => Store generated certs in named volume
 restart: "no"
# => Run once and exit
 
 # Main app uses outputs from both init containers
 app:
 image: my-app
# => Main application image
 depends_on:
# => Wait for both init containers
 config-downloader:
 condition: service_completed_successfully
# => Config must be downloaded
 cert-generator:
 condition: service_completed_successfully
# => Certs must be generated
 volumes:
 - config-data:/app/config:ro
# => Mount downloaded config as read-only
 - cert-data:/app/certs:ro
# => Mount generated certs as read-only
 
volumes:
 config-data:
# => Named volume for S3 config
 cert-data:
# => Named volume for SSL certificates
EOF
# => Creates parallel init containers compose file
# => config-downloader and cert-generator run in parallel
# => Main app waits for both to complete successfully

# File: docker-compose.yml
 
version: "3.8"
# => Compose file format version
 
services:
 # Producer: Generates data files
 data-generator:
 # => Data producer service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do date > /data/timestamp.txt; sleep 10; done"
 # => Writes current timestamp to shared volume every 10 seconds
 volumes:
 # => Volume mounts for data sharing
 - shared-data:/data
 # => Mounts shared volume at /data
 
 # Consumer 1: Processes data files
 processor:
 # => Data processor service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do cat /data/timestamp.txt 2>/dev/null || echo 'No data yet'; sleep 5; done"
 # => Reads timestamp from shared volume every 5 seconds
 volumes:
 # => Read-only access to shared data
 - shared-data:/data:ro
 # => Read-only mount (cannot modify shared data)
 depends_on:
 # => Processor waits for data-generator
 - data-generator
 # => Ensures producer starts first
 
 # Consumer 2: Backs up data
 backup:
 # => Data backup service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do cp /data/timestamp.txt /backup/backup-$(date +%s).txt 2>/dev/null || true; sleep 30; done"
 # => Creates backup every 30 seconds
 volumes:
 # => Volume mounts for backup
 - shared-data:/data:ro
 # => Read-only access to shared data
 - backup-data:/backup
 # => Separate volume for backups
 
 # Multi-container write example (needs coordination)
 writer1:
 # => First log writer service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do echo 'Writer1: $(date)' >> /data/log.txt; sleep 5; done"
 # => Appends to shared log file every 5 seconds
 volumes:
 # => Shared log volume (read-write)
 - log-data:/data
 # => Mounts log volume for writing
 
 writer2:
 # => Second log writer service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do echo 'Writer2: $(date)' >> /data/log.txt; sleep 7; done"
 # => Appends to shared log file every 7 seconds
 volumes:
 - log-data:/data
 # => Both writers append to same file (potential race condition)
 
 # Reader: Monitors aggregated logs
 log-reader:
 # => Log reader/monitor service
 image: alpine
 # => Minimal Alpine image
 command: sh -c "while true; do tail -f /data/log.txt 2>/dev/null || sleep 1; done"
 # => Continuously follows the shared log file
 volumes:
 # => Read-only log volume access
 - log-data:/data:ro
 # => Read-only access to prevent modifications
 
volumes:
 # => Named volumes for data sharing
 shared-data:
 # => Named volume managed by Docker
 backup-data:
 # => Named volume for backup files
 log-data:
 # => Named volume for aggregated logs

# Start services
docker compose up -d
# => Starts all services and creates named volumes
# => Creates shared-data, backup-data, log-data volumes automatically
 
# Verify volume creation
docker volume ls | grep shared
# => Lists volumes with 'shared' in name
# => local myproject_shared-data
# => local myproject_backup-data
# => local myproject_log-data
 
# Check shared data from host
docker volume inspect myproject_shared-data
# => {
# => "Mountpoint": "/var/lib/docker/volumes/myproject_shared-data/_data",
# => "Driver": "local"
# => }
# => Data stored at Mountpoint path on host filesystem
 
# Read shared data from volume
docker run --rm -v myproject_shared-data:/data alpine cat /data/timestamp.txt
# => Temporary container reads directly from volume
# => Sun Dec 29 12:00:00 UTC 2025
# => Temporary container reads directly from volume
 
# Monitor logs (multiple writers)
docker compose logs -f log-reader
# => Follow log output from log-reader service
# => Writer1: Sun Dec 29 12:00:05 UTC 2025
# => Writer2: Sun Dec 29 12:00:07 UTC 2025
# => Writer1: Sun Dec 29 12:00:10 UTC 2025
# => Writer2: Sun Dec 29 12:00:14 UTC 2025
# => (interleaved writes from both containers)
 
# Backup volume data
docker run --rm \
# => Temporary container for backup operation
 -v myproject_shared-data:/source:ro \
# => Mount source volume as read-only
 -v $(pwd)/backup:/backup \
# => Mount local backup directory
 alpine tar czf /backup/data-$(date +%Y%m%d).tar.gz -C /source .
# => Creates compressed backup of volume contents
 
# Restore volume data
docker run --rm \
 -v myproject_shared-data:/target \
 -v $(pwd)/backup:/backup \
 alpine tar xzf /backup/data-20251229.tar.gz -C /target
# => Restores data to volume
 
# Inspect volume usage
docker system df -v | grep myproject_shared-data
# => Check volume size in disk usage report
# => myproject_shared-data 1 1 0B 0B
 
# Volume with bind mount for development
cat > docker-compose.dev.yml << 'EOF'
version: '3.8'
 
services:
 app:
 image: my-app
 volumes:
 - ./src:/app/src:cached
 # => Bind mount source code for live reloading
 # => :cached optimizes performance on macOS/Windows
 - ./config:/app/config:ro
 # => Read-only configuration
 - node_modules:/app/node_modules
 # => Named volume for dependencies (not bind mount)
 
volumes:
 node_modules:
EOF
 
# tmpfs volume (in-memory, not persisted)
docker run --rm \
 --tmpfs /tmp:size=100M,mode=1777 \
 alpine sh -c "echo 'test' > /tmp/file.txt && cat /tmp/file.txt"
# => Data stored in RAM, lost on container stop
 
# Clean up unused volumes
docker volume prune
# => Removes all unused volumes not attached to containers
# => WARNING! This will remove all local volumes not used by at least one container.

# Create external network manually (outside Compose)
docker network create \
# => Create Docker network before starting Compose
 --driver bridge \
 # => Bridge driver for container-to-container communication
 --subnet 172.25.0.0/16 \
 # => Custom subnet avoids conflicts with other networks
 --gateway 172.25.0.1 \
 # => Gateway IP for routing
 --attachable \
# => Allows non-Compose containers to join
 external-network
# => Network for cross-project communication
# => --attachable allows containers to join dynamically
 
# Create second external network (simulating different zones)
docker network create \
 --driver bridge \
 --subnet 172.26.0.0/16 \
 # => Separate subnet range from external-network
 dmz-network
# => DMZ network for public-facing services
# => Separates public traffic from internal network

# File: docker-compose.yml (Project A - Backend Services)
 
version: "3.8"
# => Project A backend services compose file
 
services:
 database:
 # => PostgreSQL database service
 image: postgres:15-alpine
 # => Alpine-based PostgreSQL 15
 environment:
 # => Database initialization settings
 POSTGRES_PASSWORD: secret
 # => Database superuser password
 networks:
 # => Network memberships
 - external-network
 # => Connects to pre-existing network
 volumes:
 # => Data persistence volume
 - db-data:/var/lib/postgresql/data
 # => Named volume for PostgreSQL data
 
 api:
 # => REST API service
 image: my-api
 # => Pre-built API image
 networks:
 # => Multi-network membership
 - external-network
 # => Same external network as database
 - dmz-network
 # => Also connects to DMZ for public access
 environment:
 # => API database connection
 DATABASE_URL: postgresql://postgres:secret@database:5432/mydb
 # => Full connection string using Docker DNS
 
networks:
 # => External network declarations
 external-network:
 # => Declare external-network reference
 external: true
 # => Uses existing network (doesn't create new one)
 dmz-network:
 # => Declare dmz-network reference
 external: true
 # => Reuses existing DMZ network
 # => Network created by Project A
 
volumes:
 db-data:
 # => Named volume for database persistence

# File: docker-compose-frontend.yml (Project B - Frontend Services)
 
version: "3.8"
# => Project B frontend services compose file
 
services:
 web:
 # => Nginx reverse proxy service
 image: nginx:alpine
 # => Minimal Nginx Alpine image
 networks:
 # => Network membership
 - dmz-network
 # => Connects to same DMZ network
 ports:
 # => Port mapping for external access
 - "8080:80"
 # => Nginx accessible at localhost:8080
 volumes:
 # => Nginx configuration mount
 - ./nginx.conf:/etc/nginx/nginx.conf:ro
 # => Read-only Nginx config bind mount
 
 frontend-app:
 # => Frontend application service
 image: my-frontend
 # => Pre-built frontend image
 networks:
 # => DMZ network membership
 - dmz-network
 # => Can communicate with api from Project A
 
networks:
 # => External network declarations
 dmz-network:
 # => Declare dmz-network reference
 external: true
 # => Reuses network created for Project A

# Start Project A (backend services)
docker compose up -d
# => Starts database and api, joins external networks
# => Database and API join external-network and dmz-network
 
# Start Project B (frontend services)
docker compose -f docker-compose-frontend.yml up -d
# => Web and frontend-app join dmz-network
 
# Verify cross-project communication
docker exec -it projecta-api-1 ping -c 1 projectb-web-1
# => PING projectb-web-1 (172.26.0.3): 56 data bytes
# => 64 bytes from 172.26.0.3: seq=0 ttl=64 time=0.089 ms
# => Containers from different projects can communicate via external network
 
# List containers on external network
docker network inspect external-network --format='{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}'
# => projecta-database-1 172.25.0.2/16
# => projecta-api-1 172.25.0.3/16
 
# List containers on DMZ network
docker network inspect dmz-network --format='{{range .Containers}}{{.Name}} {{.IPv4Address}}{{"\n"}}{{end}}'
# => projecta-api-1 172.26.0.2/16
# => projectb-web-1 172.26.0.3/16
# => projectb-frontend-app-1 172.26.0.4/16
 
# Connect standalone container to external network
docker run -d --name redis \
 --network external-network \
 redis:7-alpine
# => Redis joins external network, accessible from both projects
 
# Network isolation verification
docker exec projecta-database-1 ping -c 1 projectb-web-1
# => ping: bad address 'projectb-web-1'
# => Database NOT on dmz-network (isolated as intended)
 
# Use external network for legacy system integration
docker run -d --name legacy-service \
 --network external-network \
 --ip 172.25.0.100 \
 my-legacy-app
# => Assigns specific IP for compatibility with hardcoded configurations
 
# Disconnect project from external network
docker compose down
# => Removes containers but external network persists
 
docker network ls | grep external
# => external-network (still exists after Compose down)
 
# Clean up external networks (manual)
docker network rm external-network dmz-network
# => ERROR: network has active endpoints
# => Must stop all containers first
 
docker stop $(docker ps -q --filter network=external-network)
docker network rm external-network
# => external-network removed