Advanced

# Initialize swarm on manager node (first node)
docker swarm init --advertise-addr 192.168.1.10
# => Swarm initialized: current node (abc123) is now a manager.
# => To add a worker to this swarm, run the following command:
# =>     docker swarm join --token SWMTKN-1-xxx 192.168.1.10:2377
# => To add a manager to this swarm, run 'docker swarm join-token manager'

# Get join token for worker nodes
docker swarm join-token worker
# => docker swarm join --token SWMTKN-1-worker-xxx 192.168.1.10:2377

# Get join token for manager nodes
docker swarm join-token manager
# => docker swarm join --token SWMTKN-1-manager-xxx 192.168.1.10:2377

# On worker nodes: Join swarm as worker
docker swarm join --token SWMTKN-1-worker-xxx 192.168.1.10:2377
# => This node joined a swarm as a worker.

# On additional manager nodes: Join swarm as manager
docker swarm join --token SWMTKN-1-manager-xxx 192.168.1.10:2377
# => This node joined a swarm as a manager.

# List swarm nodes (from manager)
docker node ls
# => ID              HOSTNAME   STATUS  AVAILABILITY  MANAGER STATUS
# => abc123def456 *  manager1   Ready   Active        Leader
# => def456ghi789    manager2   Ready   Active        Reachable
# => ghi789jkl012    manager3   Ready   Active        Reachable
# => jkl012mno345    worker1    Ready   Active
# => mno345pqr678    worker2    Ready   Active
# => pqr678stu901    worker3    Ready   Active
# => * indicates current node
# => Leader = primary manager (Raft consensus)

# Inspect swarm cluster
docker info | grep -A 10 Swarm
# => Swarm: active
# =>  NodeID: abc123def456
# =>  Is Manager: true
# =>  ClusterID: xyz789uvw012
# =>  Managers: 3
# =>  Nodes: 6
# =>  Default Address Pool: 10.0.0.0/8
# =>  SubnetSize: 24
# =>  Orchestration:
# =>   Task History Retention Limit: 5

# Promote worker to manager
docker node promote worker1
# => Node worker1 promoted to a manager in the swarm.

# Demote manager to worker
docker node demote manager3
# => Manager manager3 demoted in the swarm.

# Drain node (stop scheduling new tasks)
docker node update --availability drain worker1
# => worker1 availability set to drain
# => Existing tasks are moved to other nodes

# Activate drained node
docker node update --availability active worker1
# => worker1 availability set to active

# Add label to node (for task placement constraints)
docker node update --label-add environment=production worker1
docker node update --label-add datacenter=us-east worker1
# => Labels added to worker1

# Inspect node details
docker node inspect worker1 --pretty
# => ID:                  jkl012mno345
# => Hostname:            worker1
# => Status:
# =>  State:              Ready
# =>  Availability:       Active
# => Platform:
# =>  OS:                 linux
# =>  Architecture:       x86_64
# => Labels:
# =>  environment=production
# =>  datacenter=us-east

# Leave swarm (from worker node)
docker swarm leave
# => Node left the swarm.

# Leave swarm (from manager node - requires force)
docker swarm leave --force
# => Manager left the swarm. Cluster may be unstable.

# Remove node from swarm (from active manager)
docker node rm worker1
# => worker1 removed from swarm

# File: docker-compose.yml

version: "3.8"

services:
  web:
    image: nginx:alpine
    deploy:
      replicas: 3
      # => Maintains 3 running replicas across swarm
      update_config:
        parallelism: 1
        # => Update 1 replica at a time
        delay: 10s
        # => Wait 10 seconds between updates
        failure_action: rollback
        # => Rollback on update failure
      restart_policy:
        condition: on-failure
        delay: 5s
        max_attempts: 3
      placement:
        constraints:
          - node.role == worker
          # => Only schedule on worker nodes
        preferences:
          - spread: node.labels.datacenter
          # => Spread across different datacenters
    ports:
      - "8080:80"
      # => Published port accessible on all nodes (ingress routing mesh)
    networks:
      - frontend

  api:
    image: my-api:latest
    deploy:
      replicas: 5
      resources:
        limits:
          cpus: "0.5"
          memory: 512M
        reservations:
          cpus: "0.25"
          memory: 256M
      update_config:
        parallelism: 2
        delay: 5s
        order: start-first
        # => Start new task before stopping old (zero-downtime)
      placement:
        constraints:
          - node.labels.environment == production
    networks:
      - frontend
      - backend

  database:
    image: postgres:15-alpine
    deploy:
      replicas: 1
      # => Stateful service: single replica
      placement:
        constraints:
          - node.hostname == worker1
          # => Pin to specific node (stateful data)
    environment:
      POSTGRES_PASSWORD: secret
    volumes:
      - db-data:/var/lib/postgresql/data
    networks:
      - backend

networks:
  frontend:
    driver: overlay
    # => Overlay network spans all swarm nodes
  backend:
    driver: overlay
    internal: true
    # => Internal-only (no external access)

volumes:
  db-data:
    driver: local

# Deploy stack to swarm
docker stack deploy -c docker-compose.yml myapp
# => Creating network myapp_frontend
# => Creating network myapp_backend
# => Creating service myapp_web
# => Creating service myapp_api
# => Creating service myapp_database

# List services
docker service ls
# => ID            NAME            MODE        REPLICAS  IMAGE
# => abc123def456  myapp_web       replicated  3/3       nginx:alpine
# => def456ghi789  myapp_api       replicated  5/5       my-api:latest
# => ghi789jkl012  myapp_database  replicated  1/1       postgres:15-alpine

# Inspect service
docker service ps myapp_web
# => ID        NAME          NODE      DESIRED STATE  CURRENT STATE
# => abc123    myapp_web.1   worker1   Running        Running 2 minutes ago
# => def456    myapp_web.2   worker2   Running        Running 2 minutes ago
# => ghi789    myapp_web.3   worker3   Running        Running 2 minutes ago

# Scale service manually
docker service scale myapp_api=10
# => myapp_api scaled to 10
# => Swarm creates 5 additional replicas across nodes

docker service ps myapp_api --filter desired-state=running
# => Shows 10 running replicas distributed across worker nodes

# Update service image (rolling update)
docker service update --image my-api:v2 myapp_api
# => myapp_api
# => overall progress: 5 out of 10 tasks
# => 1/10: running   [====================>]
# => 2/10: running   [====================>]
# => Updates 2 replicas at a time (parallelism: 2)
# => Zero-downtime deployment (order: start-first)

# Rollback service update
docker service rollback myapp_api
# => myapp_api
# => rollback: manually requested rollback
# => overall progress: rolling back update

# View service logs (aggregated from all replicas)
docker service logs -f myapp_web
# => myapp_web.1.abc123 | 192.168.1.50 - - [29/Dec/2025:11:20:00] "GET / HTTP/1.1" 200
# => myapp_web.2.def456 | 192.168.1.51 - - [29/Dec/2025:11:20:01] "GET / HTTP/1.1" 200
# => myapp_web.3.ghi789 | 192.168.1.52 - - [29/Dec/2025:11:20:02] "GET / HTTP/1.1" 200

# Test ingress routing mesh (access service from any node)
curl http://worker1:8080
# => <!DOCTYPE html>... (nginx welcome page from any replica)

curl http://worker2:8080
# => <!DOCTYPE html>... (load balanced across all replicas)

# Remove stack
docker stack rm myapp
# => Removing service myapp_database
# => Removing service myapp_api
# => Removing service myapp_web
# => Removing network myapp_backend
# => Removing network myapp_frontend

# Create secret from file
echo "db_password_value" | docker secret create db_password -
# => Secret created: db_password

# Create secret from existing file
echo "api_key_12345" > api_key.txt
docker secret create api_key api_key.txt
rm api_key.txt  # Remove plaintext file
# => Secret created: api_key

# List secrets
docker secret ls
# => ID             NAME         CREATED         UPDATED
# => abc123def456   db_password  2 minutes ago   2 minutes ago
# => def456ghi789   api_key      1 minute ago    1 minute ago

# Inspect secret (data NOT shown)
docker secret inspect db_password
# => [
# =>   {
# =>     "ID": "abc123def456",
# =>     "Version": { "Index": 10 },
# =>     "CreatedAt": "2025-12-29T11:25:00Z",
# =>     "UpdatedAt": "2025-12-29T11:25:00Z",
# =>     "Spec": {
# =>       "Name": "db_password",
# =>       "Labels": {}
# =>     }
# =>   }
# => ]
# => Secret value NOT visible (encrypted)

# Create service using secrets
docker service create \
  --name postgres \
  --secret db_password \
  --env POSTGRES_PASSWORD_FILE=/run/secrets/db_password \
  postgres:15-alpine
# => Mounts secret at /run/secrets/db_password
# => Read-only, in-memory filesystem (tmpfs)

# Verify secret mount inside container
docker exec $(docker ps -q -f name=postgres) ls -l /run/secrets/
# => total 0
# => -r--r--r-- 1 root root 18 Dec 29 11:26 db_password

docker exec $(docker ps -q -f name=postgres) cat /run/secrets/db_password
# => db_password_value

# Use secrets in docker-compose (swarm mode)
cat > docker-compose.yml << 'EOF'
version: '3.8'

services:
  database:
    image: postgres:15-alpine
    secrets:
      - db_password
      # => Mounts secret at /run/secrets/db_password
    environment:
      POSTGRES_PASSWORD_FILE: /run/secrets/db_password
      # => PostgreSQL reads password from file
    deploy:
      replicas: 1

  api:
    image: my-api
    secrets:
      - source: api_key
        target: /run/secrets/api_key
        # => Custom mount path
        uid: '1000'
        gid: '1000'
        mode: 0400
        # => File permissions: read-only for uid 1000
    environment:
      API_KEY_FILE: /run/secrets/api_key
    deploy:
      replicas: 3

secrets:
  db_password:
    external: true
    # => Uses existing secret (created with docker secret create)
  api_key:
    external: true
EOF

# Deploy stack with secrets
docker stack deploy -c docker-compose.yml myapp
# => Services automatically get secrets mounted

# Update secret (requires service recreation)
docker service update \
  --secret-rm db_password \
  --secret-add source=db_password_new,target=db_password \
  myapp_database
# => Removes old secret, adds new secret
# => Service recreated with new secret

# Rotate secret safely
echo "new_password" | docker secret create db_password_v2 -
docker service update \
  --secret-rm db_password \
  --secret-add source=db_password_v2,target=db_password \
  myapp_database
# => Zero-downtime secret rotation
# => Old tasks use old secret until replaced

# Remove secret (must not be in use)
docker secret rm api_key
# => Error: secret 'api_key' is in use by service 'myapp_api'

# Remove service first, then secret
docker service rm myapp_api
docker secret rm api_key
# => api_key removed

# File: docker-compose.yml

version: "3.8"

services:
  api:
    image: my-api:latest
    read_only: true
    # => Root filesystem is read-only
    # => Prevents malicious code from modifying system files
    tmpfs:
      - /tmp:size=100M,mode=1777
      # => Writable temporary directory in memory
      - /var/run:size=10M,mode=755
      # => Writable runtime directory
    volumes:
      - logs:/var/log/app:rw
      # => Writable volume for application logs
    environment:
      NODE_ENV: production
    deploy:
      replicas: 3

  nginx:
    image: nginx:alpine
    read_only: true
    tmpfs:
      - /var/cache/nginx:size=50M
      # => Nginx cache directory (writable)
      - /var/run:size=5M
      # => PID file location
    volumes:
      - ./nginx.conf:/etc/nginx/nginx.conf:ro
      # => Configuration (read-only)
      - static-content:/usr/share/nginx/html:ro
      # => Static files (read-only)
    ports:
      - "8080:80"

volumes:
  logs:
  static-content:

# Run container with read-only root filesystem
docker run -d --name readonly-test \
  --read-only \
  --tmpfs /tmp:size=100M \
  nginx:alpine
# => Root filesystem is read-only
# => /tmp is writable (in memory)

# Try to modify root filesystem (fails)
docker exec readonly-test sh -c 'echo "test" > /test.txt'
# => sh: can't create /test.txt: Read-only file system

# Writable tmpfs works
docker exec readonly-test sh -c 'echo "test" > /tmp/test.txt'
# => Success (tmpfs is writable)

docker exec readonly-test cat /tmp/test.txt
# => test

# Verify read-only setting
docker inspect readonly-test --format='{{.HostConfig.ReadonlyRootfs}}'
# => true

# Test application functionality
curl http://localhost:8080
# => <!DOCTYPE html>... (works normally)
# => Read-only filesystem doesn't affect normal operation

# Deploy to swarm with read-only root
docker stack deploy -c docker-compose.yml myapp

# Verify security improvement (simulate intrusion attempt)
# Attacker gains access to container shell
docker exec -it $(docker ps -q -f name=myapp_api) sh

# Inside container: Try to install malware (fails)
/ # apk add --no-cache curl
# => ERROR: Unable to lock database: Read-only file system

# Try to modify system files (fails)
/ # echo "malicious" >> /etc/passwd
# => sh: can't create /etc/passwd: Read-only file system

# Try to write to writable tmpfs (succeeds but data lost on restart)
/ # echo "temp data" > /tmp/file.txt
/ # exit

# Restart container (tmpfs data lost)
docker restart $(docker ps -q -f name=myapp_api.1)

docker exec $(docker ps -q -f name=myapp_api.1) cat /tmp/file.txt
# => cat: can't open '/tmp/file.txt': No such file or directory
# => Temporary data not persisted

# Default capabilities (root user in container)
docker run --rm alpine sh -c 'apk add --no-cache libcap && capsh --print'
# => Current: = cap_chown,cap_dac_override,cap_fowner,cap_fsetid,cap_kill,cap_setgid,cap_setuid,cap_setpcap,cap_net_bind_service,cap_net_raw,cap_sys_chroot,cap_mknod,cap_audit_write,cap_setfcap+eip
# => Many dangerous capabilities enabled by default

# Drop all capabilities, add only necessary ones
docker run --rm \
  --cap-drop=ALL \
  --cap-add=NET_BIND_SERVICE \
  alpine sh -c 'apk add --no-cache libcap && capsh --print'
# => Current: = cap_net_bind_service+eip
# => Only NET_BIND_SERVICE capability (bind to ports < 1024)

# Example: Web server needs only NET_BIND_SERVICE
docker run -d --name web \
  --cap-drop=ALL \
  --cap-add=NET_BIND_SERVICE \
  -p 80:80 \
  nginx:alpine
# => Runs with minimal capabilities

# Try to use dropped capability (fails)
docker exec web sh -c 'apk add --no-cache libcap && capsh --print | grep cap_sys_admin'
# => (no output - CAP_SYS_ADMIN not available)

# Example: Application needs no special capabilities
docker run -d --name api \
  --cap-drop=ALL \
  --user 1000:1000 \
  -p 3000:3000 \
  my-api
# => No capabilities at all (most secure)

# Compose example with dropped capabilities
cat > docker-compose.yml << 'EOF'
version: '3.8'

services:
  web:
    image: nginx:alpine
    cap_drop:
      - ALL
    cap_add:
      - NET_BIND_SERVICE
      # => Only capability needed for binding to port 80
    ports:
      - "80:80"

  api:
    image: my-api
    cap_drop:
      - ALL
      # => No capabilities needed (runs on port > 1024)
    user: "1000:1000"
    ports:
      - "3000:3000"

  database:
    image: postgres:15-alpine
    cap_drop:
      - ALL
    cap_add:
      - CHOWN
      - DAC_OVERRIDE
      - FOWNER
      - SETGID
      - SETUID
      # => Minimal set for PostgreSQL operation
    environment:
      POSTGRES_PASSWORD: secret
EOF

# Common capability uses:
# CAP_NET_BIND_SERVICE: Bind to ports < 1024
# CAP_CHOWN: Change file ownership
# CAP_DAC_OVERRIDE: Bypass file permission checks
# CAP_FOWNER: Bypass permission checks on file operations
# CAP_SETGID: Set GID
# CAP_SETUID: Set UID
# CAP_SYS_ADMIN: Mount filesystems, admin operations (DANGEROUS - never use)
# CAP_NET_RAW: Use raw sockets (DANGEROUS - enables network attacks)

# Dangerous capabilities to NEVER add:
# CAP_SYS_ADMIN: Full system administration (container escape risk)
# CAP_SYS_MODULE: Load kernel modules (container escape risk)
# CAP_SYS_RAWIO: Direct hardware access
# CAP_SYS_PTRACE: Trace processes (can dump secrets from memory)

# Check container capabilities at runtime
docker inspect web --format='{{.HostConfig.CapDrop}}'
# => [ALL]

docker inspect web --format='{{.HostConfig.CapAdd}}'
# => [NET_BIND_SERVICE]

# Install Trivy scanner
curl -sfL https://raw.githubusercontent.com/aquasecurity/trivy/main/contrib/install.sh | sh -s -- -b /usr/local/bin
# => Trivy scanner installed

# Scan image for vulnerabilities
trivy image nginx:alpine
# => 2025-12-29T11:30:00.000Z  INFO    Vulnerability scanning is enabled
# => 2025-12-29T11:30:05.000Z  INFO    Detected OS: alpine
# => 2025-12-29T11:30:05.000Z  INFO    Detected vulnerabilities: 15
# =>
# => nginx:alpine (alpine 3.19.0)
# => Total: 15 (UNKNOWN: 0, LOW: 5, MEDIUM: 8, HIGH: 2, CRITICAL: 0)
# =>
# => +------------------+------------------+----------+-------------------+
# => | LIBRARY          | VULNERABILITY ID | SEVERITY | INSTALLED VERSION |
# => +------------------+------------------+----------+-------------------+
# => | openssl          | CVE-2023-12345   | HIGH     | 3.1.0-r1          |
# => | libcurl          | CVE-2023-67890   | HIGH     | 8.4.0-r0          |
# => | zlib             | CVE-2023-11111   | MEDIUM   | 1.3-r0            |
# => +------------------+------------------+----------+-------------------+

# Scan only for HIGH and CRITICAL vulnerabilities
trivy image --severity HIGH,CRITICAL nginx:alpine
# => Shows only serious vulnerabilities

# Scan and fail build if vulnerabilities found
trivy image --exit-code 1 --severity CRITICAL nginx:alpine
# => Exit code 0: No critical vulnerabilities
# => Exit code 1: Critical vulnerabilities found (fails CI/CD)

# Scan local Dockerfile before building
trivy config Dockerfile
# => Scans Dockerfile for misconfigurations
# => Detects: Running as root, missing health checks, etc.

# Scan built image with detailed output
trivy image --format json my-app:latest > scan-results.json
# => JSON output for automated processing

# Extract vulnerability summary
cat scan-results.json | jq '.Results[0].Vulnerabilities | group_by(.Severity) | map({Severity: .[0].Severity, Count: length})'
# => [
# =>   { "Severity": "CRITICAL", "Count": 2 },
# =>   { "Severity": "HIGH", "Count": 5 },
# =>   { "Severity": "MEDIUM", "Count": 12 }
# => ]

# CI/CD integration (GitHub Actions example)
cat > .github/workflows/scan.yml << 'EOF'
name: Container Scan

on: [push]

jobs:
  scan:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Build image
        run: docker build -t my-app:${{ github.sha }} .

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: my-app:${{ github.sha }}
          format: 'sarif'
          output: 'trivy-results.sarif'
          severity: 'CRITICAL,HIGH'
          exit-code: '1'
          # => Fails build if CRITICAL or HIGH vulnerabilities found

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: 'trivy-results.sarif'
EOF

# Scan multi-stage build (scan final stage only)
trivy image --image-src my-app:latest
# => Scans only production stage, not build artifacts

# Ignore specific vulnerabilities (use with caution)
cat > .trivyignore << 'EOF'
# False positive - not applicable to our use case
CVE-2023-12345

# Accepted risk - no patch available, mitigated at network level
CVE-2023-67890
EOF

trivy image --ignorefile .trivyignore nginx:alpine
# => Skips ignored CVEs

# Scan running container
trivy container $(docker ps -q -f name=web)
# => Scans running container filesystem

# Regular scanning schedule (cron job)
cat > scan-images.sh << 'EOF'
#!/bin/bash
# Scan all running containers daily

for container in $(docker ps --format '{{.Names}}'); do
  echo "Scanning $container..."
  trivy container $container --severity HIGH,CRITICAL
done
EOF

chmod +x scan-images.sh
# => Add to cron: 0 2 * * * /path/to/scan-images.sh

# File: Dockerfile.distroless (Go application)

# Build stage
FROM golang:1.21-alpine AS builder

WORKDIR /app

COPY go.mod go.sum ./
RUN go mod download

COPY . .

RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o server .
# => Static binary (no dynamic linking)

# Production stage with distroless
FROM gcr.io/distroless/static-debian12
# => No shell, no package manager, no utilities
# => Only contains: /etc/passwd, /etc/group, tzdata, ca-certificates

COPY --from=builder /app/server /server

EXPOSE 8080

USER nonroot:nonroot
# => Runs as non-root user (UID 65532)

CMD ["/server"]
# => Array syntax (no shell needed)

# File: Dockerfile.distroless-node (Node.js application)

# Build stage
FROM node:18-alpine AS builder

WORKDIR /app

COPY package*.json ./
RUN npm ci --only=production

COPY . .
RUN npm run build

# Production stage with distroless
FROM gcr.io/distroless/nodejs18-debian12
# => Contains Node.js runtime only

COPY --from=builder /app/package.json /app/package-lock.json ./
COPY --from=builder /app/node_modules ./node_modules
COPY --from=builder /app/dist ./dist

EXPOSE 3000

CMD ["dist/main.js"]
# => Runs node dist/main.js automatically

# Build distroless image
docker build -f Dockerfile.distroless -t my-app:distroless .

# Compare image sizes
docker images | grep my-app
# => my-app  alpine      15MB   (alpine base)
# => my-app  distroless  10MB   (distroless base - 33% smaller)
# => my-app  debian      80MB   (debian base - 8x larger!)

# Try to access shell (fails - no shell in distroless)
docker run --rm my-app:distroless sh
# => docker: Error response: OCI runtime create failed: exec: "sh": executable file not found

# Debugging distroless containers (use debug variant)
FROM gcr.io/distroless/static-debian12:debug
# => Includes busybox shell for debugging

docker run --rm -it my-app:distroless-debug sh
# => / # (busybox shell available)

# Inspect distroless container filesystem
docker run --rm my-app:distroless ls /
# => docker: Error response: executable file not found
# => ls command doesn't exist!

# Use multi-stage to inspect
docker build -t my-app:inspect --target builder -f Dockerfile.distroless .
docker run --rm my-app:inspect ls -la /
# => Inspects builder stage (has shell and tools)

# Scan distroless image (minimal vulnerabilities)
trivy image my-app:distroless
# => Total: 0 (UNKNOWN: 0, LOW: 0, MEDIUM: 0, HIGH: 0, CRITICAL: 0)
# => No vulnerabilities! (minimal attack surface)

# Compare with alpine
trivy image my-app:alpine
# => Total: 8 (LOW: 3, MEDIUM: 4, HIGH: 1)
# => Alpine has more packages = more vulnerabilities

# Available distroless base images:
# gcr.io/distroless/static-debian12       - Static binaries only (Go, Rust)
# gcr.io/distroless/base-debian12         - glibc, libssl, ca-certificates
# gcr.io/distroless/cc-debian12           - libc, libssl (C/C++ apps)
# gcr.io/distroless/java17-debian12       - Java 17 runtime
# gcr.io/distroless/nodejs18-debian12     - Node.js 18 runtime
# gcr.io/distroless/python3-debian12      - Python 3 runtime

# Health check in distroless (use HEALTHCHECK in Dockerfile)
FROM gcr.io/distroless/static-debian12

COPY --from=builder /app/server /server
COPY --from=builder /app/healthcheck /healthcheck

HEALTHCHECK --interval=30s --timeout=3s \
  CMD ["/healthcheck"]
# => Custom healthcheck binary (no curl/wget available)

# Logging from distroless (stdout only)
# Ensure application logs to stdout (no need for log files)

# Enable user namespace remapping in Docker daemon
cat > /etc/docker/daemon.json << 'EOF'
{
  "userns-remap": "default"
}
EOF
# => Creates dockremap user/group automatically
# => Container UID 0 (root) maps to unprivileged UID on host

# Restart Docker daemon
sudo systemctl restart docker

# Check user namespace mapping
cat /etc/subuid | grep dockremap
# => dockremap:100000:65536
# => Container UIDs 0-65535 map to host UIDs 100000-165535

cat /etc/subgid | grep dockremap
# => dockremap:100000:65536
# => Container GIDs mapped similarly

# Run container with user namespace
docker run -d --name test-userns alpine sleep 3600

# Check process on host (remapped UID)
ps aux | grep sleep | grep -v grep
# => 100000   12345  0.0  0.0  1234  567  ?  Ss  11:35  0:00  sleep 3600
# => Process runs as UID 100000 on host (not 0!)

# Inside container: Check UID (appears as root)
docker exec test-userns id
# => uid=0(root) gid=0(root) groups=0(root),1(bin),2(daemon),...
# => Container sees UID 0 (root)

# On host: Verify actual UID
ps -o user,uid,pid,cmd -p $(pgrep -f "sleep 3600")
# => USER    UID   PID  CMD
# => 100000  100000  12345  sleep 3600
# => Host sees unprivileged UID

# Try to access host resources from container root
docker exec test-userns sh -c 'ls -la /hostdata'
# => ls: can't open '/hostdata': Permission denied
# => Even as "root" in container, no access to host directories

# Test privilege escalation attempt
docker exec test-userns sh -c 'echo "malicious" > /host-etc-passwd'
# => sh: can't create /host-etc-passwd: Permission denied
# => User namespace prevents host file access

# Disable user namespace for specific container (requires privilege)
docker run --userns=host -d alpine sleep 3600
# => Runs with host user namespace (dangerous - only for trusted containers)

# Compose with user namespace
cat > docker-compose.yml << 'EOF'
version: '3.8'

services:
  app:
    image: my-app
    # User namespace enabled automatically if daemon configured
    # OR explicitly disable for specific service:
    # userns_mode: "host"  # Dangerous - bypasses isolation
EOF

# Check files created by container on host
docker run --rm -v /tmp/test:/data alpine sh -c 'touch /data/file.txt'

ls -ln /tmp/test/
# => total 0
# => -rw-r--r-- 1 100000 100000 0 Dec 29 11:40 file.txt
# => File owned by remapped UID (100000), not root (0)

# Security benefit: Root exploit in container
# Attacker gains root in container
docker exec test-userns sh -c 'whoami'
# => root

# On host: Still unprivileged user
ps aux | grep test-userns | head -1
# => 100000  12345  ...
# => Host sees unprivileged UID (defense in depth)

# File: .github/workflows/security.yml (GitHub Actions)

name: Security Scan

on:
  push:
    branches: [main, develop]
  pull_request:
    branches: [main]

jobs:
  security-scan:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build Docker image
        run: |
          docker build -t ${{ github.repository }}:${{ github.sha }} .

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ github.repository }}:${{ github.sha }}
          format: "sarif"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          exit-code: "1"
          # => Fails build if CRITICAL or HIGH vulnerabilities found

      - name: Upload Trivy results to GitHub Security
        uses: github/codeql-action/upload-sarif@v2
        if: always()
        with:
          sarif_file: "trivy-results.sarif"

      - name: Run Hadolint (Dockerfile linter)
        uses: hadolint/hadolint-action@v3.1.0
        with:
          dockerfile: Dockerfile
          failure-threshold: error
          # => Fails on Dockerfile errors (not warnings)

      - name: Scan for secrets in code
        uses: trufflesecurity/trufflehog@main
        with:
          path: ./
          base: ${{ github.event.repository.default_branch }}
          head: HEAD

      - name: Docker Scout CVEs
        uses: docker/scout-action@v1
        with:
          command: cves
          image: ${{ github.repository }}:${{ github.sha }}
          only-severities: critical,high
          exit-code: true
          # => Alternative scanner using Docker Scout

      - name: Generate SBOM (Software Bill of Materials)
        run: |
          docker sbom ${{ github.repository }}:${{ github.sha }} > sbom.json

      - name: Upload SBOM artifact
        uses: actions/upload-artifact@v3
        with:
          name: sbom
          path: sbom.json

# File: .gitlab-ci.yml (GitLab CI/CD)

stages:
  - build
  - security
  - deploy

variables:
  IMAGE_NAME: $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA

build:
  stage: build
  script:
    - docker build -t $IMAGE_NAME .
    - docker push $IMAGE_NAME

trivy-scan:
  stage: security
  image: aquasec/trivy:latest
  script:
    - trivy image --exit-code 1 --severity CRITICAL,HIGH $IMAGE_NAME
  allow_failure: false
  # => Blocks pipeline if vulnerabilities found

hadolint:
  stage: security
  image: hadolint/hadolint:latest
  script:
    - hadolint Dockerfile
  allow_failure: false

grype-scan:
  stage: security
  image: anchore/grype:latest
  script:
    - grype $IMAGE_NAME --fail-on high
  # => Alternative vulnerability scanner

deploy:
  stage: deploy
  script:
    - kubectl set image deployment/myapp app=$IMAGE_NAME
  only:
    - main
  when: on_success
  # => Only deploys if security scans pass

# Local pre-commit hook for security
cat > .git/hooks/pre-commit << 'EOF'
#!/bin/bash

echo "Running security checks..."

# Scan Dockerfile
echo "Checking Dockerfile..."
docker run --rm -i hadolint/hadolint < Dockerfile
if [ $? -ne 0 ]; then
  echo "Dockerfile has issues. Fix them before committing."
  exit 1
fi

# Check for secrets
echo "Scanning for secrets..."
docker run --rm -v $(pwd):/src trufflesecurity/trufflehog:latest filesystem /src
if [ $? -ne 0 ]; then
  echo "Potential secrets found. Remove them before committing."
  exit 1
fi

echo "Security checks passed."
EOF

chmod +x .git/hooks/pre-commit
# => Runs security checks before each commit

# Manual security scan before pushing
docker build -t myapp:latest .

# Multiple scanners for comprehensive coverage
trivy image myapp:latest
grype myapp:latest
docker scout cves myapp:latest

# Generate compliance report
trivy image --format json --output report.json myapp:latest
cat report.json | jq '.Results[0].Vulnerabilities | length'
# => Total vulnerability count

# Track vulnerabilities over time
git add scan-results/$(date +%Y-%m-%d).json
# => Version-controlled vulnerability tracking

# File: docker-compose.yml (Private registry with authentication)

version: "3.8"

services:
  registry:
    image: registry:2
    ports:
      - "5000:5000"
    environment:
      REGISTRY_AUTH: htpasswd
      REGISTRY_AUTH_HTPASSWD_PATH: /auth/htpasswd
      REGISTRY_AUTH_HTPASSWD_REALM: Registry Realm
      REGISTRY_STORAGE_FILESYSTEM_ROOTDIRECTORY: /data
      # TLS configuration
      REGISTRY_HTTP_TLS_CERTIFICATE: /certs/domain.crt
      REGISTRY_HTTP_TLS_KEY: /certs/domain.key
    volumes:
      - registry-data:/data
      - ./auth:/auth:ro
      - ./certs:/certs:ro
    restart: unless-stopped

  registry-ui:
    image: joxit/docker-registry-ui:latest
    ports:
      - "8080:80"
    environment:
      REGISTRY_TITLE: My Private Registry
      REGISTRY_URL: https://registry.example.com:5000
      DELETE_IMAGES: "true"
      SINGLE_REGISTRY: "true"
    depends_on:
      - registry

volumes:
  registry-data:

# Create authentication file
mkdir -p auth certs

# Generate htpasswd file
docker run --rm \
  --entrypoint htpasswd \
  httpd:2 -Bbn admin secretpassword > auth/htpasswd
# => Creates htpasswd file with user: admin, password: secretpassword

# Generate self-signed TLS certificate (production: use Let's Encrypt)
openssl req -newkey rsa:4096 -nodes -sha256 \
  -keyout certs/domain.key -x509 -days 365 \
  -out certs/domain.crt \
  -subj "/CN=registry.example.com"
# => Creates TLS certificate for HTTPS

# Start private registry
docker compose up -d

# Trust self-signed certificate (Linux)
sudo cp certs/domain.crt /usr/local/share/ca-certificates/registry.crt
sudo update-ca-certificates
# => Adds certificate to system trust store

# Login to private registry
docker login registry.example.com:5000
# => Username: admin
# => Password: secretpassword
# => Login Succeeded

# Tag image for private registry
docker tag my-app:latest registry.example.com:5000/my-app:latest
docker tag my-app:latest registry.example.com:5000/my-app:1.0.0

# Push to private registry
docker push registry.example.com:5000/my-app:latest
docker push registry.example.com:5000/my-app:1.0.0
# => Uploads image to private registry

# List images in registry (API)
curl -u admin:secretpassword https://registry.example.com:5000/v2/_catalog
# => {"repositories":["my-app"]}

# List tags for image
curl -u admin:secretpassword https://registry.example.com:5000/v2/my-app/tags/list
# => {"name":"my-app","tags":["latest","1.0.0"]}

# Pull from private registry (on another machine)
docker login registry.example.com:5000
docker pull registry.example.com:5000/my-app:1.0.0
# => Downloads image from private registry

# Delete image tag from registry
curl -X DELETE -u admin:secretpassword \
  https://registry.example.com:5000/v2/my-app/manifests/sha256:<digest>
# => Deletes specific image manifest

# Run garbage collection to free space
docker exec registry bin/registry garbage-collect /etc/docker/registry/config.yml
# => Removes unreferenced layers

# Registry with S3 storage (production)
cat > registry-s3.yml << 'EOF'
version: '3.8'

services:
  registry:
    image: registry:2
    ports:
      - "5000:5000"
    environment:
      REGISTRY_STORAGE: s3
      REGISTRY_STORAGE_S3_REGION: us-east-1
      REGISTRY_STORAGE_S3_BUCKET: my-docker-registry
      REGISTRY_STORAGE_S3_ACCESSKEY: ${AWS_ACCESS_KEY_ID}
      REGISTRY_STORAGE_S3_SECRETKEY: ${AWS_SECRET_ACCESS_KEY}
EOF
# => Uses S3 for scalable, durable storage

# Registry with Redis cache (performance)
cat >> docker-compose.yml << 'EOF'
  redis:
    image: redis:7-alpine
    restart: unless-stopped

  registry:
    environment:
      REGISTRY_STORAGE_CACHE_BLOBDESCRIPTOR: redis
      REGISTRY_REDIS_ADDR: redis:6379
      REGISTRY_REDIS_DB: 0
EOF
# => Caches blob descriptors in Redis for faster pulls

# File: .github/workflows/ci-cd.yml

name: CI/CD Pipeline

on:
  push:
    branches: [main, develop]
    tags:
      - "v*"
  pull_request:
    branches: [main]

env:
  REGISTRY: ghcr.io
  IMAGE_NAME: ${{ github.repository }}

jobs:
  # Job 1: Build and test
  build-test:
    runs-on: ubuntu-latest
    permissions:
      contents: read
      packages: write

    steps:
      - name: Checkout code
        uses: actions/checkout@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
        # => Enables advanced build features (caching, multi-platform)

      - name: Log in to GitHub Container Registry
        uses: docker/login-action@v2
        with:
          registry: ${{ env.REGISTRY }}
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Extract metadata
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}
          tags: |
            type=ref,event=branch
            type=ref,event=pr
            type=semver,pattern={{version}}
            type=semver,pattern={{major}}.{{minor}}
            type=sha
          # => Generates tags: main, pr-123, v1.0.0, 1.0, sha-abc123

      - name: Build and push Docker image
        uses: docker/build-push-action@v4
        with:
          context: .
          push: ${{ github.event_name != 'pull_request' }}
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache
          cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:buildcache,mode=max
          # => Registry cache for faster subsequent builds

      - name: Run tests in container
        run: |
          docker run --rm ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }} npm test

  # Job 2: Security scanning
  security-scan:
    runs-on: ubuntu-latest
    needs: build-test
    permissions:
      security-events: write

    steps:
      - name: Run Trivy scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:sha-${{ github.sha }}
          format: "sarif"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"

      - name: Upload Trivy results
        uses: github/codeql-action/upload-sarif@v2
        with:
          sarif_file: "trivy-results.sarif"

  # Job 3: Deploy to staging
  deploy-staging:
    runs-on: ubuntu-latest
    needs: [build-test, security-scan]
    if: github.ref == 'refs/heads/develop'
    environment:
      name: staging
      url: https://staging.example.com

    steps:
      - name: Deploy to staging
        run: |
          # SSH to staging server and update deployment
          echo "${{ secrets.STAGING_SSH_KEY }}" > staging_key
          chmod 600 staging_key
          ssh -i staging_key -o StrictHostKeyChecking=no deploy@staging.example.com << 'EOF'
            docker pull ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:develop
            docker stop myapp || true
            docker rm myapp || true
            docker run -d --name myapp -p 80:3000 \
              ${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:develop
          EOF

  # Job 4: Deploy to production
  deploy-production:
    runs-on: ubuntu-latest
    needs: [build-test, security-scan]
    if: startsWith(github.ref, 'refs/tags/v')
    environment:
      name: production
      url: https://example.com

    steps:
      - name: Deploy to production Kubernetes
        run: |
          echo "${{ secrets.KUBECONFIG }}" > kubeconfig
          kubectl --kubeconfig=kubeconfig set image \
            deployment/myapp \
            app=${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}:${{ github.ref_name }}
          kubectl --kubeconfig=kubeconfig rollout status deployment/myapp

      - name: Create GitHub release
        uses: actions/create-release@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          tag_name: ${{ github.ref }}
          release_name: Release ${{ github.ref }}
          draft: false
          prerelease: false

# Trigger workflow by pushing to main
git add .
git commit -m "feat: add new feature"
git push origin main
# => Triggers CI/CD pipeline
# => Builds, tests, scans, deploys to staging

# Create release tag (triggers production deployment)
git tag -a v1.0.0 -m "Release version 1.0.0"
git push origin v1.0.0
# => Triggers production deployment workflow

# View workflow runs
# GitHub UI: Actions tab shows all workflow runs

# Pull image built by GitHub Actions
docker pull ghcr.io/username/repo:v1.0.0
# => Downloads image from GitHub Container Registry

# File: docker-compose-stack.yml
# Stack deployment with production features

version: "3.8"

services:
  web:
    image: nginx:alpine
    # => Uses official Nginx Alpine image
    deploy:
      replicas: 3
      # => Creates 3 web service replicas
      update_config:
        parallelism: 1
        # => Updates 1 replica at a time
        delay: 10s
        # => Waits 10s between updates
        failure_action: rollback
        # => Rolls back if update fails
      restart_policy:
        condition: on-failure
        # => Restarts only on failure
        delay: 5s
        # => Waits 5s before restart
        max_attempts: 3
        # => Maximum 3 restart attempts
      placement:
        constraints:
          - node.role == worker
          # => Deploys only on worker nodes
    ports:
      - "8080:80"
      # => Maps port 8080 on host to port 80 in container
    networks:
      - frontend
      # => Connects to frontend overlay network

  app:
    image: myapp:latest
    # => Uses custom application image
    deploy:
      replicas: 5
      # => Creates 5 app service replicas
      resources:
        limits:
          cpus: "0.5"
          # => Limits each replica to 0.5 CPU
          memory: 512M
          # => Limits each replica to 512MB RAM
        reservations:
          cpus: "0.25"
          # => Reserves minimum 0.25 CPU
          memory: 256M
          # => Reserves minimum 256MB RAM
      placement:
        preferences:
          - spread: node.labels.datacenter
          # => Spreads replicas across datacenters
    environment:
      DATABASE_URL: postgresql://db:5432/prod
      # => Database connection string
    networks:
      - frontend
      - backend
      # => Connects to both networks
    depends_on:
      - db
      # => Starts after db service (ordering hint)

  db:
    image: postgres:15-alpine
    # => Uses PostgreSQL 15 Alpine
    deploy:
      replicas: 1
      # => Single database instance (stateful)
      placement:
        constraints:
          - node.labels.database == true
          # => Deploys only on nodes labeled for database
      restart_policy:
        condition: any
        # => Always restart database
    volumes:
      - db-data:/var/lib/postgresql/data
      # => Persistent database storage
    networks:
      - backend
      # => Only accessible from backend network
    secrets:
      - db_password
      # => Mounts secret as file

networks:
  frontend:
    driver: overlay
    # => Creates overlay network for frontend
  backend:
    driver: overlay
    # => Creates overlay network for backend

volumes:
  db-data:
    driver: local
    # => Creates local volume for database

secrets:
  db_password:
    external: true
    # => References externally created secret

# Create secret before stack deployment
echo "super_secret_password" | docker secret create db_password -
# => Secret created: db_password
# => Secret is encrypted at rest and in transit

# Label node for database placement
docker node update --label-add database=true worker1
# => Node worker1 labeled with database=true

# Deploy stack
docker stack deploy -c docker-compose-stack.yml myapp
# => Creating network myapp_frontend
# => Creating network myapp_backend
# => Creating service myapp_web
# => Creating service myapp_app
# => Creating service myapp_db
# => Stack deployed successfully

# List stacks
docker stack ls
# => NAME     SERVICES   ORCHESTRATOR
# => myapp    3          Swarm

# List stack services
docker stack services myapp
# => ID       NAME        MODE        REPLICAS  IMAGE
# => abc123   myapp_web   replicated  3/3       nginx:alpine
# => def456   myapp_app   replicated  5/5       myapp:latest
# => ghi789   myapp_db    replicated  1/1       postgres:15-alpine

# Inspect service
docker service inspect myapp_app --pretty
# => ID:           def456ghi789
# => Name:         myapp_app
# => Mode:         Replicated
# =>  Replicas:    5
# => Placement:
# =>  Preferences: spread=node.labels.datacenter
# => UpdateConfig:
# =>  Parallelism: 1
# =>  Delay:       10s
# =>  Failure action: rollback
# => Resources:
# =>  Limits:      0.5 CPUs, 512MB Memory
# =>  Reservations: 0.25 CPUs, 256MB Memory

# View service logs
docker service logs myapp_app --tail 10
# => Shows last 10 log lines from all app replicas
# => [myapp_app.1] Application started on port 3000
# => [myapp_app.2] Application started on port 3000
# => [myapp_app.3] Application started on port 3000

# Scale service
docker service scale myapp_app=10
# => myapp_app scaled to 10 replicas
# => Swarm schedules 5 additional tasks

# Update service image
docker service update --image myapp:v2 myapp_app
# => myapp_app updated
# => Rolling update: 1 replica at a time
# => Waits 10s between updates
# => Automatically rolls back on failure

# Rollback service update
docker service rollback myapp_app
# => myapp_app rolled back to previous version

# View stack networks
docker stack ps myapp --filter "desired-state=running"
# => Shows all running tasks in stack
# => ID      NAME           NODE      CURRENT STATE
# => abc123  myapp_web.1    worker1   Running 5 minutes ago
# => def456  myapp_web.2    worker2   Running 5 minutes ago
# => ghi789  myapp_app.1    worker1   Running 5 minutes ago

# Remove stack
docker stack rm myapp
# => Removing service myapp_web
# => Removing service myapp_app
# => Removing service myapp_db
# => Removing network myapp_frontend
# => Removing network myapp_backend
# => Stack removed successfully

# Remove secret
docker secret rm db_password
# => Secret removed: db_password

# Label nodes for specialized workloads
docker node update --label-add environment=production worker1
# => Node worker1 labeled: environment=production

docker node update --label-add gpu=true worker2
# => Node worker2 labeled: gpu=true

docker node update --label-add storage=ssd worker3
# => Node worker3 labeled: storage=ssd

docker node update --label-add datacenter=us-west worker4
# => Node worker4 labeled: datacenter=us-west

docker node update --label-add datacenter=us-east worker5
# => Node worker5 labeled: datacenter=us-east

# Create service with role constraint
docker service create \
  --name manager-only \
  --constraint 'node.role == manager' \
  nginx:alpine
# => Service created: manager-only
# => Tasks scheduled only on manager nodes

# Create service with label constraint
docker service create \
  --name production-app \
  --constraint 'node.labels.environment == production' \
  --replicas 3 \
  myapp:latest
# => Service created: production-app
# => All 3 replicas scheduled on worker1 (only production node)

# Create GPU service
docker service create \
  --name ml-training \
  --constraint 'node.labels.gpu == true' \
  --reserve-memory 4G \
  --reserve-cpu 4.0 \
  tensorflow/tensorflow:latest-gpu
# => Service created: ml-training
# => Task scheduled on worker2 (GPU node)
# => Reserves 4GB RAM and 4 CPUs

# Create database service with SSD constraint
docker service create \
  --name postgres-db \
  --constraint 'node.labels.storage == ssd' \
  --mount type=volume,source=db-data,target=/var/lib/postgresql/data \
  postgres:15-alpine
# => Service created: postgres-db
# => Task scheduled on worker3 (SSD node)

# Multiple constraints (AND logic)
docker service create \
  --name secure-api \
  --constraint 'node.role == worker' \
  --constraint 'node.labels.environment == production' \
  --constraint 'node.labels.datacenter == us-west' \
  api:latest
# => Service created: secure-api
# => Task scheduled only on worker nodes in production environment in us-west datacenter

# Constraint with node ID
NODE_ID=$(docker node ls -q -f name=worker1)
docker service create \
  --name pinned-service \
  --constraint "node.id == $NODE_ID" \
  myapp:latest
# => Service created: pinned-service
# => Task pinned to specific node (worker1)

# OS constraint
docker service create \
  --name linux-service \
  --constraint 'node.platform.os == linux' \
  alpine:latest
# => Service created: linux-service
# => Runs only on Linux nodes

# Architecture constraint
docker service create \
  --name arm-service \
  --constraint 'node.platform.arch == aarch64' \
  arm64v8/alpine:latest
# => Service created: arm-service
# => Runs only on ARM64 nodes

# Hostname constraint
docker service create \
  --name hostname-service \
  --constraint 'node.hostname == worker1' \
  nginx:alpine
# => Service created: hostname-service
# => Scheduled on node with hostname "worker1"

# Spread preference (soft constraint, not hard)
docker service create \
  --name distributed-app \
  --replicas 6 \
  --placement-pref 'spread=node.labels.datacenter' \
  myapp:latest
# => Service created: distributed-app
# => 3 replicas on us-west nodes, 3 on us-east nodes
# => Spreads evenly across datacenter labels

# Combine constraints and preferences
docker service create \
  --name ha-database \
  --replicas 3 \
  --constraint 'node.labels.storage == ssd' \
  --placement-pref 'spread=node.labels.datacenter' \
  postgres:15-alpine
# => Service created: ha-database
# => Scheduled only on SSD nodes
# => Spread across datacenters for high availability

# Inspect service placement
docker service ps distributed-app
# => ID      NAME                 NODE      CURRENT STATE
# => abc123  distributed-app.1    worker4   Running
# => def456  distributed-app.2    worker5   Running
# => ghi789  distributed-app.3    worker4   Running
# => jkl012  distributed-app.4    worker5   Running
# => Shows task distribution across nodes

# Update service constraints
docker service update \
  --constraint-add 'node.labels.environment == staging' \
  production-app
# => production-app updated
# => Adds additional constraint (now must match both production AND staging)

# Remove constraint
docker service update \
  --constraint-rm 'node.labels.environment == staging' \
  production-app
# => production-app updated
# => Removes staging constraint

# View node labels
docker node inspect worker1 --format '{{.Spec.Labels}}'
# => map[datacenter:us-west environment:production]

# Remove node label
docker node update --label-rm environment worker1
# => Node worker1: label environment removed

# Create service with health check
docker service create \
  --name web-app \
  --replicas 5 \
  --health-cmd "curl -f http://localhost/health || exit 1" \
  --health-interval 10s \
  --health-timeout 5s \
  --health-retries 3 \
  --update-parallelism 1 \
  --update-delay 10s \
  --update-failure-action rollback \
  --rollback-parallelism 2 \
  --rollback-delay 5s \
  myapp:v1
# => Service created: web-app
# => Health check: curl /health every 10s
# => Update parallelism: 1 task at a time
# => Update delay: 10s between tasks
# => Failure action: automatic rollback
# => Rollback parallelism: 2 tasks at a time (faster recovery)

# Verify service is running
docker service ps web-app
# => ID      NAME         IMAGE       NODE      CURRENT STATE
# => abc123  web-app.1    myapp:v1    worker1   Running (healthy)
# => def456  web-app.2    myapp:v1    worker2   Running (healthy)
# => ghi789  web-app.3    myapp:v1    worker3   Running (healthy)
# => jkl012  web-app.4    myapp:v1    worker1   Running (healthy)
# => mno345  web-app.5    myapp:v1    worker2   Running (healthy)

# Perform rolling update to v2
docker service update --image myapp:v2 web-app
# => web-app updated
# => Update in progress...
# => Stopping web-app.1
# => Starting web-app.1 with image myapp:v2
# => Health check passed for web-app.1
# => Waiting 10s before next update
# => Stopping web-app.2
# => Starting web-app.2 with image myapp:v2
# => (continues for all replicas)

# Monitor update progress
docker service ps web-app
# => ID      NAME         IMAGE       NODE      CURRENT STATE
# => pqr678  web-app.1    myapp:v2    worker1   Running (healthy)
# => abc123   \_ web-app.1 myapp:v1    worker1   Shutdown 2 minutes ago
# => stu901  web-app.2    myapp:v2    worker2   Running (healthy)
# => def456   \_ web-app.2 myapp:v1    worker2   Shutdown 1 minute ago
# => Shows current and previous tasks

# Simulate failed update (v3 has failing health check)
docker service update --image myapp:v3-broken web-app
# => web-app updated
# => Update in progress...
# => Stopping web-app.1
# => Starting web-app.1 with image myapp:v3-broken
# => Health check failed for web-app.1 (3 consecutive failures)
# => Automatic rollback triggered
# => Stopping web-app.1 (v3-broken)
# => Starting web-app.1 with image myapp:v2
# => Rollback in progress (2 tasks at a time)
# => Service rolled back to myapp:v2

# Check service after failed update
docker service ps web-app --filter "desired-state=running"
# => All replicas running myapp:v2
# => Failed v3-broken tasks in shutdown state

# Manual rollback to previous version
docker service rollback web-app
# => web-app rolled back
# => Reverting to previous image myapp:v1
# => Rolling back 2 tasks at a time
# => Rollback complete

# Update with custom settings
docker service update \
  --image myapp:v4 \
  --update-parallelism 2 \
  --update-delay 30s \
  --update-max-failure-ratio 0.2 \
  --update-monitor 60s \
  web-app
# => Updates 2 tasks at a time
# => Waits 30s between batches
# => Rolls back if >20% of tasks fail
# => Monitors each task for 60s before considering it stable

# Pause update mid-rollout
docker service update --update-parallelism 0 web-app
# => Update paused (parallelism set to 0)
# => Current tasks remain running
# => No new tasks started

# Resume update
docker service update \
  --image myapp:v4 \
  --update-parallelism 1 \
  web-app
# => Update resumed with parallelism 1

# Force update (recreate all tasks even with same image)
docker service update --force web-app
# => Forces recreation of all tasks
# => Useful for picking up configuration changes

# Update with order (start-first vs stop-first)
docker service update \
  --image myapp:v5 \
  --update-order start-first \
  web-app
# => Starts new task before stopping old task
# => Ensures capacity is maintained (useful for stateless services)
# => Alternative: --update-order stop-first (default, for stateful services)

# Inspect update config
docker service inspect web-app --format '{{.Spec.UpdateConfig}}'
# => {1 10s 0s 0.2 pause 60s start-first}
# => Parallelism: 1
# => Delay: 10s
# => Max failure ratio: 0.2 (20%)
# => Monitor period: 60s
# => Order: start-first

# View service update history
docker service inspect web-app --format '{{.PreviousSpec.TaskTemplate.ContainerSpec.Image}}'
# => Shows previous image version

# Configure global update defaults for all services
docker swarm update \
  --task-history-limit 10 \
  --dispatcher-heartbeat 10s
# => Sets task history retention to 10 previous versions
# => Sets dispatcher heartbeat to 10s

# File: docker-compose.yml

version: "3.8"

services:
  # Jaeger all-in-one (dev/testing)
  jaeger:
    image: jaegertracing/all-in-one:latest
    environment:
      COLLECTOR_ZIPKIN_HOST_PORT: :9411
    ports:
      - "5775:5775/udp"   # Compact thrift (deprecated)
      - "6831:6831/udp"   # Jaeger thrift (binary)
      - "6832:6832/udp"   # Jaeger thrift (compact)
      - "5778:5778"       # Serve configs
      - "16686:16686"     # Jaeger UI
      - "14250:14250"     # Jaeger gRPC
      - "14268:14268"     # Jaeger HTTP
      - "9411:9411"       # Zipkin compatible endpoint

  # Application with tracing
  web:
    build: ./web
    environment:
      JAEGER_AGENT_HOST: jaeger
      JAEGER_AGENT_PORT: 6831
    depends_on:
      - jaeger
    ports:
      - "3000:3000"

  auth:
    build: ./auth
    environment:
      JAEGER_AGENT_HOST: jaeger
      JAEGER_AGENT_PORT: 6831
    depends_on:
      - jaeger

  user-service:
    build: ./user-service
    environment:
      JAEGER_AGENT_HOST: jaeger
      JAEGER_AGENT_PORT: 6831
    depends_on:
      - jaeger

// File: web/tracing.js (Node.js/OpenTelemetry)
const { NodeTracerProvider } = require('@opentelemetry/sdk-trace-node');
const { JaegerExporter } = require('@opentelemetry/exporter-jaeger');
const { Resource } = require('@opentelemetry/resources');
const { SemanticResourceAttributes } = require('@opentelemetry/semantic-conventions');
const { registerInstrumentations } = require('@opentelemetry/instrumentation');
const { HttpInstrumentation } = require('@opentelemetry/instrumentation-http');
const { ExpressInstrumentation } = require('@opentelemetry/instrumentation-express');

// Create tracer provider
const provider = new NodeTracerProvider({
  resource: new Resource({
    [SemanticResourceAttributes.SERVICE_NAME]: 'web-service',
    [SemanticResourceAttributes.SERVICE_VERSION]: '1.0.0',
  }),
});

// Configure Jaeger exporter
const jaegerExporter = new JaegerExporter({
  endpoint: `http://${process.env.JAEGER_AGENT_HOST}:14268/api/traces`,
});

provider.addSpanProcessor(new SimpleSpanProcessor(jaegerExporter));
provider.register();

// Auto-instrument HTTP and Express
registerInstrumentations({
  instrumentations: [
    new HttpInstrumentation(),
    new ExpressInstrumentation(),
  ],
});

// Manual span creation
const tracer = provider.getTracer('web-service');

app.get('/api/users/:id', async (req, res) => {
  const span = tracer.startSpan('get-user');
  span.setAttribute('user.id', req.params.id);

  try {
    // Call auth service (auto-traced)
    const authResult = await fetch('http://auth/verify', {
      headers: {
        authorization: req.headers.authorization,
      },
    });

    // Call user service
    const userResult = await fetch(`http://user-service/users/${req.params.id}`);

    span.setStatus({ code: SpanStatusCode.OK });
    res.json(await userResult.json());
  } catch (error) {
    span.setStatus({
      code: SpanStatusCode.ERROR,
      message: error.message,
    });
    span.recordException(error);
    res.status(500).json({ error: error.message });
  } finally {
    span.end();
  }
});