Intermediate

# === ON pve01 (first node): Create the cluster ===
 
# Create a new cluster named 'mycluster'
pvecm create mycluster
# => mycluster created; Corosync listens on UDP 5405-5412 (open in firewall)
 
# Verify cluster is running on pve01
pvecm status
# => Name: mycluster | Quorate: Yes | Votes: 1
 
# === ON pve02 and pve03: Join the cluster ===
# Prerequisites: joining node must have no existing VMs (joining wipes local cluster state)
pvecm add 192.168.1.100
# => Node joined; /etc/pve/ content synchronized from pve01
 
pvecm add 192.168.1.100
# => Node pve03 joined cluster 'mycluster'
 
# Verify cluster membership from any node
pvecm nodes
# => Node 1 (pve01): 1 vote | Node 2 (pve02): 1 vote | Node 3 (pve03): 1 vote (quorum=2)

# Check complete cluster status including quorum
pvecm status
# => Name: mycluster | Nodes: 3 | Quorate: Yes | Expected votes: 3 | Quorum: 2
 
# Check Corosync ring status (network connectivity between nodes)
corosync-quorumtool -s
# => Quorate: Yes | pve01: 1 vote online | pve02: 1 vote online | pve03: 1 vote online
 
# Check if cluster remains quorate after simulated node failure (disconnect pve03)
pvecm status | grep Quorate
# => Quorate: Yes  (2 of 3 nodes sufficient for majority)
 
# View cluster node health in JSON (useful for monitoring scripts)
pvesh get /cluster/status
# => [{"type":"cluster","quorate":1,"nodes":3}, {"type":"node","name":"pve01","online":1}, ...]

# === ON a separate Linux machine (not a PVE node): Set up QDevice server ===
# Requirements: Debian/Ubuntu with corosync-qnetd; provides tie-breaking vote
apt install corosync-qnetd
# => QNet daemon installed
 
# Start and enable the QNet daemon (listens on port 5403)
systemctl enable --now corosync-qnetd
# => corosync-qnetd.service started
 
# === ON pve01 (one of the two cluster nodes): Configure QDevice ===
apt install corosync-qdevice
# => corosync-qdevice installed
 
# Add QDevice (copies SSH keys, reconfigures Corosync on all cluster nodes)
pvecm qdevice setup 192.168.1.200
# => QDevice configured; Corosync restarted on all nodes
 
# Verify QDevice is active (Expected votes: 3 = 2 nodes + 1 QDevice)
pvecm status | grep -A5 "Quorum information"
# => Quorate: Yes | Expected votes: 3 | Flags: Quorate Qdevice
 
# Check QDevice connection status (State: connected = voting)
corosync-quorumtool -s | grep -i qdevice
# => State: connected | Votes: 1 | Algo: ffsplit

# Check where VM 100 is currently running
pvesh get /cluster/resources --type vm | python3 -c "
import sys, json
# => GET /cluster/resources?type=vm returns all VMs across all nodes
vms = json.load(sys.stdin)['data']
# => vms: list of dicts with vmid, node, status, cpu, mem fields
for vm in vms:
    # => iterate all VMs in the cluster
    if vm.get('vmid') == 100:
        # => filter to VM 100 only
        print(f\"VM 100 is on node: {vm['node']}, status: {vm['status']}\")
"
# => VM 100 is on node: pve01, status: running
 
# Live migrate VM 100 from pve01 to pve02 (zero downtime)
# --online 1: iterative pre-copy RAM pages; <100ms final pause for sync
qm migrate 100 pve02 --online 1
# => Precondition: shared storage (Ceph/NFS) required; local-lvm triggers disk copy
# => Migration complete: VM 100 now running on pve02
# => Without --online: VM stopped, disk copied, VM started on target (planned downtime)
 
# Verify VM is now on pve02
pvesh get /cluster/resources --type vm | python3 -c "
import sys, json
vms = json.load(sys.stdin)['data']
# => same API call; re-fetches current placement after migration
for vm in vms:
    # => iterate all VMs; check each for vmid match
    if vm.get('vmid') == 100:
        print(f\"VM 100 is now on node: {vm['node']}\")
        # => prints current node after migration confirms placement change
"
# => VM 100 is now on node: pve02
 
# Drain pve01: migrate all running VMs to pve02 before maintenance
# Python one-liner filters running VMs and prints their VMIDs
for vmid in $(pvesh get /nodes/pve01/qemu --output-format json | python3 -c "
import sys, json
vms = json.load(sys.stdin)['data']
# => vms: list of {vmid, name, status, mem, cpu, ...} dicts
for vm in vms:
    # => check each VM's status field
    if vm['status'] == 'running':
        # => only migrate running VMs; stopped VMs stay on pve01
        print(vm['vmid'])
"); do
  # => loop iterates over each running VMID: 100, 105, ...
  qm migrate $vmid pve02 --online 1
  # => live migration: VM stays running on pve02 immediately after
  echo "Migrated VM $vmid to pve02"
  # => prints confirmation per VM
done
# => pve01 drained; safe to reboot for kernel update or hardware maintenance

# Check container location
pvesh get /cluster/resources --type lxc | python3 -c "
import sys, json
cts = json.load(sys.stdin)['data']
for ct in cts:
    if ct.get('vmid') == 200:
        print(f\"CT 200 ({ct['name']}) is on node: {ct['node']}, status: {ct['status']}\")
"
# => CT 200 (web-server-01) is on node: pve01, status: running
 
# Migrate running container (online migration, container stays up)
pct migrate 200 pve02 \
  --target-storage local-lvm \
  --online 1
# => Migrating LXC container 200 from pve01 to pve02
# => --target-storage local-lvm: destination storage for rootfs
# => --online 1: container continues serving requests during migration
# => Syncing container filesystem (rsync initial pass)...
# => Pausing container for final sync (<1 second for most workloads)...
# => Resuming container on pve02...
# => CT 200 migration complete
 
# For offline migration (container stopped during transfer)
pct shutdown 200
# => CT 200 stopped
 
pct migrate 200 pve03 --target-storage local-lvm
# => Migrating stopped CT 200 from pve01 to pve03...
# => Copying rootfs (8 GB)... (takes seconds-to-minutes depending on size)
# => Container config updated: pve03 now manages CT 200
# => Starting CT 200 on pve03...
pct start 200
# => CT 200 running on pve03
 
# Verify container is healthy after migration
pct exec 200 -- systemctl status nginx
# => ● nginx.service - A high performance web server
# =>      Active: active (running) ...

# Enable VLAN awareness on the main bridge vmbr0
# bridge-vlan-aware yes: enables 802.1Q tagging; bridge-vids 2-4094: full range
sed -i '/^auto vmbr0/,/^auto/ { /bridge-stp/a\    bridge-vlan-aware yes\n    bridge-vids 2-4094 }' \
  /etc/network/interfaces
# => vmbr0 now supports 802.1Q VLAN tagging for all VMs
 
ifreload -a
# => vmbr0 reconfigured with VLAN awareness enabled
 
# Assign VM to VLAN 100 (web tier); switch port must be a trunk port
qm set 100 --net0 virtio,bridge=vmbr0,tag=100
# => VM 100 sends/receives VLAN 100 traffic; guest sees untagged frames
 
# Assign VM to VLAN 200 (database tier) — isolated from VLAN 100
qm set 105 --net0 virtio,bridge=vmbr0,tag=200
# => VM 105 NIC on VLAN 200 (no direct L2 path to VLAN 100)
 
# Trunk port: VM receives tagged traffic (for virtual routers like VyOS/OPNsense)
qm set 150 --net0 virtio,bridge=vmbr0,trunks=100;200;300
# => VM 150 receives tagged frames for VLANs 100, 200, and 300
 
# Verify VLAN configuration on bridge
bridge vlan show dev vmbr0
# => tap100i0: VLAN 100 PVID | tap105i0: VLAN 200 PVID  (per-VM tap isolation)

# Configure active-backup bonding (failover: one NIC active, second standby)
cat >> /etc/network/interfaces << 'EOF'
 
auto bond0
iface bond0 inet manual
    bond-slaves eno1 eno2     # Two physical NICs forming the bond
    bond-mode active-backup   # Only one NIC active at a time; failover on link failure
    bond-miimon 100           # Check link status every 100ms
    bond-primary eno1         # Prefer eno1 as active NIC when both are up
 
auto vmbr0
iface vmbr0 inet static
    address 192.168.1.100/24
    gateway 192.168.1.1
    bridge-ports bond0        # Bridge uses bonded interface instead of single NIC
    bridge-stp off
    bridge-fd 0
EOF
# => bond0 provides redundancy: if eno1 fails, eno2 becomes active in <200ms
 
ifreload -a
# => Network interfaces reloaded with bonding configuration
 
# Verify bonding status
cat /proc/net/bonding/bond0
# => Bonding Mode: fault-tolerance (active-backup)
# => Primary Slave: eno1 (primary_reselect failure)
# => Currently Active Slave: eno1
# => MII Status: up
# => MII Polling Interval (ms): 100
# => Slave Interface: eno1
# =>   MII Status: up
# =>   Speed: 10000 Mbps    (10 GbE NIC)
# =>   Duplex: full
# => Slave Interface: eno2
# =>   MII Status: up
# =>   Speed: 10000 Mbps
 
# Test failover by disabling the active NIC
ip link set eno1 down
# => bond0 detects eno1 link failure within 100ms
# => eno2 becomes active slave automatically
cat /proc/net/bonding/bond0 | grep "Active Slave"
# => Currently Active Slave: eno2   (failover successful)
 
ip link set eno1 up
# => eno1 returns; remains standby since eno2 is now active

# Add NFS storage to Proxmox (NFS server already configured separately)
pvesh create /storage \
  --storage nfs-share \
  --type nfs \
  --server 192.168.1.50 \
  --export /mnt/proxmox-storage \
  --content images,iso,backup \
  --options vers=4.2,hard,timeo=600 \
  --maxfiles 10
# => NFS storage 'nfs-share' added
# => server: NFS server IP or hostname
# => export: NFS export path on the server
# => content=images: VM disk images stored here (enables shared storage for live migration)
# => content=iso: ISOs available cluster-wide
# => options vers=4.2: NFSv4.2 for best performance (pNFS, sparse file support)
# => options hard: retries I/O indefinitely on timeout (vs soft which returns errors)
# => timeo=600: 60-second timeout before NFS retries (prevents fast-fail on transient network hiccups)
# => maxfiles 10: keep at most 10 backup archives per guest on this storage
 
# Verify NFS is mounted and accessible
pvesh get /nodes/pve01/storage/nfs-share/status
# => {"total": 1099511627776, "used": 214748364800, "avail": 884763262976, "active": 1}
 
# Test that NFS storage appears on all cluster nodes
for node in pve01 pve02 pve03; do
  echo -n "$node: "
  # => print node name prefix before each status line
  pvesh get /nodes/$node/storage/nfs-share/status | python3 -c "
import sys, json
d = json.load(sys.stdin)['data']
print(f\"active={'yes' if d.get('active') else 'no'}, avail={d.get('avail', 0)//1024//1024//1024}GB\")
  "
done
# => pve01: active=yes, avail=823GB | pve02: active=yes | pve03: active=yes
# => all nodes can access NFS; live VM migration is now possible
 
# List VM disk images on NFS storage
pvesh get /nodes/pve01/storage/nfs-share/content --content images
# => Returns list of .qcow2/.raw VM disk images stored on the NFS share

# Get the iSCSI initiator IQN (unique ID for this node)
cat /etc/iscsi/initiatorname.iscsi
# => InitiatorName=iqn.1993-08.org.debian:01:pve01-iscsi-initiator
 
# Add iSCSI storage (content=none: use LVM on top of raw block device)
pvesh create /storage \
  --storage iscsi-san \
  --type iscsi \
  --portal 192.168.1.60 \
  --target iqn.2025-01.com.company:storage-target-01 \
  --content none
# => iSCSI storage 'iscsi-san' registered; LUNs appear as raw block devices
# => --portal: IP:port of iSCSI target (default port 3260)
# => --target: iSCSI Qualified Name (IQN) of the storage target
# => --content none: raw block device; add LVM layer on top for VM disk provisioning
 
# Scan for available LUNs on the iSCSI target
pvesh get /nodes/pve01/storage/iscsi-san/content
# => [{"volid": "iscsi-san:0.0.0.0.0.0.0.0", "size": 1099511627776}]  (1 TB LUN 0)
 
# Add LVM thin pool on top of the iSCSI LUN (--nodes restricts to specific nodes)
pvesh create /storage \
  --storage iscsi-lvm \
  --type lvmthin \
  --vgname iscsi-vg \
  --thinpool data \
  --content images,rootdir \
  --nodes pve01,pve02,pve03
# => iscsi-lvm registered; all three nodes can provision VM disks here
# => --vgname: LVM volume group name created on the iSCSI LUN
# => --thinpool: thin pool name within the VG (enables thin provisioning / snapshots)
# => --nodes: restrict availability to listed nodes (prevents accidental use from others)
 
# Verify iSCSI connection and LVM physical volume
iscsiadm -m session
# => tcp: [1] 192.168.1.60:3260 iqn.2025-01.com.company:storage-target-01
 
pvs
# => /dev/sdb  iscsi-vg  1024.00g (iSCSI LUN) | /dev/sda3  pve  (local)

# List available disks for ZFS pool creation
lsblk -d -o NAME,SIZE,MODEL | grep -v "loop\|sr"
# => sda   500G   Samsung SSD 870
# => sdb   500G   WD Red Plus
# => sdc   500G   WD Red Plus
# => sdd   500G   WD Red Plus
 
# Create ZFS mirror pool (RAID-1: 2 drives, 1 drive can fail)
zpool create -f \
  -o ashift=12 \
  -O compression=lz4 \
  -O atime=off \
  -O recordsize=64K \
  tank mirror /dev/sdb /dev/sdc
# => ZFS pool 'tank' created (mirror: RAID-1 equivalent)
# => ashift=12: 4K sector alignment (required for modern SSDs and HDDs)
# => compression=lz4: fast transparent compression (30-50% space savings typical)
# => atime=off: disable access time updates (significant I/O reduction)
# => recordsize=64K: optimal for mixed workloads (databases may prefer 8K)
 
# Create RAIDZ pool (RAID-5 equivalent: 3 drives, 1 drive can fail)
zpool create -f \
  -o ashift=12 \
  -O compression=lz4 \
  -O atime=off \
  tank-raidz raidz /dev/sdb /dev/sdc /dev/sdd
# => ZFS RAIDZ pool 'tank-raidz' created
# => 3 drives, 1 parity: 66% usable capacity (vs mirror's 50%)
# => RAIDZ2 uses -o raidz2 for 2-parity (survives 2 simultaneous drive failures)
 
# Check pool health and statistics
zpool status tank
# => pool: tank | state: ONLINE | mirror-0: sdb ONLINE, sdc ONLINE
# => errors: No known data errors
 
# Register ZFS pool in Proxmox storage
pvesh create /storage \
  --storage tank \
  --type zfspool \
  --pool tank \
  --content images,rootdir \
  --sparse 1
# => ZFS pool 'tank' registered as Proxmox storage

# Initialize Ceph on the cluster (run on pve01)
pveceph init \
  --network 10.0.0.0/24 \
  --cluster-network 10.0.1.0/24
# => Initializing Ceph cluster...
# => --network: Ceph public network (client-to-OSD traffic)
# => --cluster-network: Ceph cluster network (OSD replication traffic, separate NIC recommended)
# => Ceph Squid (19.2.3) installed on all nodes
 
# Add Ceph monitors (MONs) — need 3+ for quorum
pveceph mon create --node pve01
# => Created Ceph MON on pve01
 
pveceph mon create --node pve02
# => Created Ceph MON on pve02
 
pveceph mon create --node pve03
# => Created Ceph MON on pve03
 
# Add Ceph managers (MGRs) — provide dashboards and metrics
pveceph mgr create --node pve01
# => Created Ceph MGR on pve01
 
pveceph mgr create --node pve02
# => Created Ceph MGR on pve02 (standby)
 
# Add OSDs (Object Storage Daemons) — one per disk per node
# Each OSD serves one physical disk; OSD manages data distribution
pveceph osd create /dev/sdb --node pve01
# => Created OSD on /dev/sdb (pve01): OSD ID 0
 
pveceph osd create /dev/sdb --node pve02
# => Created OSD on /dev/sdb (pve02): OSD ID 1
 
pveceph osd create /dev/sdb --node pve03
# => Created OSD on /dev/sdb (pve03): OSD ID 2
 
# Check Ceph cluster health
ceph status
# => health: HEALTH_OK
# => mon: 3 daemons quorum pve01,pve02,pve03 | mgr: pve01(active) pve02(standby)
# => osd: 3 up, 3 in | usage: 1.5 GiB used / 1.5 TiB avail

# Create a Ceph pool for VM images with 3x replication
# size=3: 3 replicas; min_size=2: writes succeed with 2 copies (1 OSD can fail)
pveceph pool create vm-images \
  --size 3 \
  --min_size 2 \
  --pg_autoscale_mode on \
  --application rbd
# => pool 'vm-images' created; pg_autoscale manages placement group count automatically
# => --application rbd: tag pool for RADOS Block Device use (enables RBD-specific optimizations)
# => --pg_autoscale_mode on: Ceph automatically adjusts placement group count as pool grows
 
# Register the Ceph pool as Proxmox storage (krbd=0: use QEMU librbd for better perf)
pvesh create /storage \
  --storage ceph-vm \
  --type rbd \
  --pool vm-images \
  --monhost "pve01,pve02,pve03" \
  --content images,rootdir \
  --krbd 0
# => ceph-vm registered; krbd=1 required for LXC containers on Ceph
# => --monhost: comma-separated list of Ceph monitor nodes (for initial connection)
# => --krbd 0: use QEMU in-process librbd (faster); krbd=1 uses kernel RBD module
 
# Verify pool health and replication
ceph osd pool ls detail | grep vm-images
# => pool 1 'vm-images' replicated size 3 min_size 2 pg_num 32
 
# Check pool usage and available capacity
ceph df | grep vm-images
# => vm-images  256M stored | 768M used | 0.05% | 1.5TiB available
# => 'used' = raw bytes across all replicas; 'stored' = logical data before replication

# Create an erasure coding profile (k=2 data chunks, m=1 parity chunk)
ceph osd erasure-code-profile set ec-2-1 \
  k=2 \
  m=1 \
  plugin=jerasure \
  technique=reed_sol_van
# => Erasure code profile 'ec-2-1' created
# => k=2: data is split into 2 chunks; 2 OSDs store data
# => m=1: 1 parity chunk stored on a third OSD
# => Total OSDs needed: k+m = 3 (minimum)
# => Usable capacity: k/(k+m) = 2/3 = 66.7% (vs 33.3% with 3x replication)
# => Survives: up to m=1 OSD failure simultaneously
 
# Create an erasure-coded pool using the profile
pveceph pool create ec-backup \
  --erasure-coding k=2,m=1 \
  --add-storages 1
# => Pool 'ec-backup' created with erasure coding k=2, m=1
# => --add-storages 1: automatically register as Proxmox storage
# => Note: EC pools require a companion replicated pool for metadata (auto-created)
 
# Alternatively, create EC pool directly with ceph commands
ceph osd pool create ec-pool-2-1 erasure ec-2-1
# => pool 'ec-pool-2-1' created
 
# Enable RBD on the EC pool
ceph osd pool application enable ec-pool-2-1 rbd
# => enabled application 'rbd' on pool 'ec-pool-2-1'
 
# Check pool info to verify EC configuration
ceph osd pool get ec-backup erasure_code_profile
# => erasure_code_profile: ec-2-1    (profile applied to pool)
 
ceph df | grep ec-backup
# => POOL       STORED  OBJECTS  USED    %USED  MAX AVAIL
# => ec-backup  0 B     0        0 B     0      987 GiB   (66% of 1.5TiB raw)

# Overall cluster health summary
ceph health detail
# => HEALTH_WARN: osd.2 down | PG_DEGRADED: 256 objects 5.0% degraded
# => writes continue (min_size=2 met with 2 remaining OSDs)
 
# Detailed OSD status and placement
ceph osd tree
# => osd.0 pve01 up | osd.1 pve02 up | osd.2 pve03 down  (failed OSD identified)
 
# Monitor OSD I/O performance
ceph osd perf
# => osd 0: commit 2ms | osd 1: commit 3ms | osd 2: - (down, no I/O)
 
# Watch cluster recovery progress in real-time (updates every 5s)
watch -n 5 'ceph status'
# => recovery: 51200 kB/s, 25 objects/s | 156 active+clean, 20 active+recovering
 
# Check disk usage per OSD (identify overloaded OSDs by high WEIGHT vs disk size)
ceph osd df tree
# => Shows per-OSD usage, PG count, and REWEIGHT factor
 
# Set OSD back in service after replacement
ceph osd in osd.2
# => osd.2 marked in; cluster begins rebalancing data to restored OSD

# Create a Simple zone (VLAN-based SDN zone using existing bridge)
pvesh create /cluster/sdn/zones \
  --zone simple-zone \
  --type simple \
  --bridge vmbr0 \
  --mtu 1500
# => SDN zone 'simple-zone' created
# => --type simple: VLAN-based zone; uses existing bridge (simplest SDN type)
# => --bridge vmbr0: physical bridge SDN uses for VLAN tagging
# => --mtu 1500: standard Ethernet MTU; lower if VLANs transit a VXLAN underlay
 
# Create a VNet in the zone with VLAN tag 100
pvesh create /cluster/sdn/vnets \
  --vnet web-vnet \
  --zone simple-zone \
  --tag 100 \
  --comment "Web application network (VLAN 100)"
# => VNet 'web-vnet' created with VLAN tag 100
# => --tag 100: 802.1Q VLAN ID; VMs on this VNet send/receive tagged frames for VLAN 100
 
# Create a subnet (snat=1: outbound NAT; dnszoneprefix: DNS registration)
pvesh create /cluster/sdn/vnets/web-vnet/subnets \
  --subnet 10.100.0.0/24 \
  --type subnet \
  --gateway 10.100.0.1 \
  --dnszoneprefix web \
  --snat 1
# => Subnet 10.100.0.0/24 added to web-vnet
# => --dnszoneprefix web: VM hostnames registered as web.<hostname>.internal in DNS
# => --snat 1: masquerade outbound traffic (VMs can reach internet without public IPs)
 
# Apply SDN configuration (propagates Linux interfaces to all cluster nodes)
pvesh set /cluster/sdn
# => SDN applied; all nodes have matching network interfaces
# => 'pvesh set /cluster/sdn' is the single apply command for all pending SDN changes
 
# Verify and assign VM to SDN VNet
pvesh get /cluster/sdn/vnets
# => [{"vnet":"web-vnet","zone":"simple-zone","tag":100}]
# => lists all configured VNets with zone and VLAN tag associations
qm set 100 --net0 virtio,bridge=web-vnet
# => VM 100 connected; SDN handles VLAN tagging automatically

# Create a VXLAN SDN zone (mtu=1450: physical 1500 minus 50-byte VXLAN header)
# CRITICAL: wrong MTU causes large packets to silently fragment/drop
pvesh create /cluster/sdn/zones \
  --zone vxlan-overlay \
  --type vxlan \
  --peers 192.168.1.100,192.168.1.101,192.168.1.102 \
  --mtu 1450
# => VXLAN zone 'vxlan-overlay' created (all node IPs must be listed as peers)
# => --type vxlan: encapsulates L2 frames in UDP (no switch VLAN configuration needed)
# => --peers: all participating node IPs; each node creates a VXLAN FDB entry per peer
 
# Create VNet with VNI 1001 (VNI = VXLAN equivalent of VLAN ID, 24-bit range)
pvesh create /cluster/sdn/vnets \
  --vnet vxlan-app-net \
  --zone vxlan-overlay \
  --tag 1001 \
  --comment "Application network via VXLAN (VNI 1001)"
# => VNet 'vxlan-app-net' with VNI 1001 created
# => --tag 1001: VXLAN Network Identifier (VNI); 24-bit range supports up to 16 million segments
 
pvesh create /cluster/sdn/vnets/vxlan-app-net/subnets \
  --subnet 172.16.0.0/24 \
  --gateway 172.16.0.1
# => Subnet 172.16.0.0/24 added
# => --gateway: L3 gateway IP created on each node (distributed anycast routing)
 
pvesh set /cluster/sdn
# => VXLAN interfaces created on all nodes at mtu 1450
# => applies the pending SDN config; creates vxlan<VNI> interfaces on every cluster node
 
# Verify VXLAN interfaces and test cross-node L2 connectivity
ip link show type vxlan
# => vxlan1001: mtu 1450 | vxlan id 1001 local 192.168.1.100 port 4789
qm agent 100 exec -- bash -c "ping -c3 172.16.0.20"
# => 64 bytes from 172.16.0.20: ttl=64 time=0.8 ms (traverses VXLAN tunnel)
# => uses QEMU guest agent to run ping inside VM without console access

# BGP-EVPN requires FRRouting installed on all cluster nodes
apt install frr frr-pythontools
# => FRRouting installed (BGP, OSPF, VXLAN/EVPN control plane)
systemctl enable frr
# => FRR enabled; provides the BGP control plane for EVPN
 
# Create BGP-EVPN SDN zone (vrf-vxlan: L3 VRF; exitnodes: border nodes for external traffic)
pvesh create /cluster/sdn/zones \
  --zone evpn-fabric \
  --type evpn \
  --controller evpn-controller \
  --vrf-vxlan 4000 \
  --mac-prefix "42:00:00:" \
  --exitnodes "pve01,pve02" \
  --peers 192.168.1.100,192.168.1.101,192.168.1.102
# => EVPN zone 'evpn-fabric' created with anycast GW MAC prefix
# => --type evpn: BGP-EVPN zone using VXLAN data plane with BGP control plane
# => --controller: references the SDN controller that manages BGP sessions
# => --vrf-vxlan 4000: VNI used for the L3 VRF (inter-VNet routing traffic)
# => --mac-prefix "42:00:00:": anycast gateway MAC prefix; same MAC on all nodes = local routing
# => --exitnodes: nodes acting as border gateways for north-south traffic to external networks
 
# Create SDN controller (BGP ASN 65000; all nodes are IBGP peers)
pvesh create /cluster/sdn/controllers \
  --controller evpn-controller \
  --type evpn \
  --asn 65000 \
  --peers 192.168.1.100,192.168.1.101,192.168.1.102
# => BGP ASN 65000 configured; nodes act as route reflectors to each other
# => --asn 65000: private BGP AS number (64512-65534 range for private use)
# => --peers: all node IPs form a full IBGP mesh for EVPN route exchange
 
# Create VNet and subnet in EVPN zone (automatically routable between VNets)
pvesh create /cluster/sdn/vnets \
  --vnet evpn-app \
  --zone evpn-fabric \
  --tag 2001
# => VNet 'evpn-app' created; --tag 2001 is the VXLAN VNI for this segment
# => EVPN VNets are automatically routable to other VNets in the same zone (no gateway VM needed)
 
pvesh create /cluster/sdn/vnets/evpn-app/subnets \
  --subnet 10.200.0.0/24 \
  --gateway 10.200.0.1 \
  --dhcp-range start-address=10.200.0.100,end-address=10.200.0.200
# => Subnet configured with DHCP range .100-.200; gateway is anycast across all nodes
# => --dhcp-range: built-in DHCP server allocates IPs in this range to VMs on the VNet
 
pvesh set /cluster/sdn
# => EVPN configuration applied; FRR BGP sessions establishing
# => FRR on each node generates BGP OPEN messages; sessions go Idle → Connect → Established
 
# Verify BGP sessions (Established = routing working between nodes)
vtysh -c "show bgp summary"
# => 192.168.1.101 AS65000 Established | 192.168.1.102 AS65000 Established
# => vtysh: FRRouting CLI; "show bgp summary" lists all BGP peer states
# => Established state = BGP UPDATE messages exchanging EVPN routes between nodes

# Create an OpenFabric underlay fabric (spine-leaf topology)
pvesh create /cluster/sdn/controllers \
  --controller fabric-01 \
  --type openfabric \
  --fabric-id 1 \
  --loopback 10.255.0.0/24
# => OpenFabric controller 'fabric-01' created (fabric-id range: 1-65535)
# => --type openfabric: IS-IS-based link-state routing protocol optimized for data centers
# => --fabric-id 1: unique fabric identifier; nodes with same fabric-id form one routing domain
# => --loopback 10.255.0.0/24: /24 pool; each node gets one /32 loopback from this range
 
# Add spine node and leaf nodes to the fabric
pvesh create /cluster/sdn/controllers/fabric-01/nodes \
  --node pve01 \
  --role spine \
  --loopback 10.255.0.1
# => pve01 = spine with loopback 10.255.0.1
# => /cluster/sdn/controllers/fabric-01/nodes: API path adds a node to the named fabric
# => spine role: central routing node; connects to all leaf nodes (hub in hub-and-spoke)
 
pvesh create /cluster/sdn/controllers/fabric-01/nodes \
  --node pve02 \
  --role leaf \
  --loopback 10.255.0.2 \
  --uplink-interface eno2
# => pve02 = leaf; uplinks to spine via eno2
# => --role leaf: edge node; connects to servers/VMs and upstream to spine
# => --uplink-interface eno2: physical NIC connecting this leaf to the spine node
 
pvesh create /cluster/sdn/controllers/fabric-01/nodes \
  --node pve03 \
  --role leaf \
  --loopback 10.255.0.3 \
  --uplink-interface eno2
# => pve03 = leaf; uplinks to spine via eno2
# => loopback 10.255.0.3: unique /32 address from the 10.255.0.0/24 pool assigned to pve03
 
pvesh set /cluster/sdn
# => OpenFabric routing sessions initializing on all nodes
# => SDN writes FRRouting config files and restarts frr service on each node
 
# Verify fabric routing sessions (spine should see both leaf neighbors)
vtysh -c "show openfabric summary"
# => Area: backbone | Neighbors: pve02, pve03 (spine sees both leaves)
# => backbone: OpenFabric default routing area (all nodes in same fabric share one area)
# => each neighbor entry confirms bidirectional routing adjacency is established
# => "2-Way" or "Full" state = routing tables exchanged; traffic can flow between nodes

# Create an IP set for application servers
pvesh create /cluster/firewall/ipset \
  --name app-servers \
  --comment "Application server IP addresses"
# => IP set 'app-servers' created; use +app-servers as source/dest in firewall rules
# => /cluster/firewall/ipset: cluster-wide IP set visible to all nodes and VMs
 
pvesh create /cluster/firewall/ipset/app-servers --cidr 10.100.0.10
# => 10.100.0.10 added to set (web-server-01)
pvesh create /cluster/firewall/ipset/app-servers --cidr 10.100.0.11
# => 10.100.0.11 added to set (web-server-02)
pvesh create /cluster/firewall/ipset/app-servers --cidr 10.100.0.12
# => Three app server IPs added to set 'app-servers'
# => CIDR notation supported: --cidr 10.100.0.0/24 adds entire subnet to the set
 
# Create a security group (reusable rule set)
pvesh create /cluster/firewall/groups \
  --group web-tier \
  --comment "Rules for public web tier VMs"
# => Security group 'web-tier' created
# => /cluster/firewall/groups: cluster-scoped; groups visible on all nodes and VMs
# => groups are reusable rule sets; apply to multiple VMs with a single reference
 
pvesh create /cluster/firewall/groups/web-tier \
  --type in --action ACCEPT --proto tcp --dport 80 --comment "Allow HTTP"
# => rule 1: inbound TCP port 80 accepted (web traffic)
pvesh create /cluster/firewall/groups/web-tier \
  --type in --action ACCEPT --proto tcp --dport 443 --comment "Allow HTTPS"
# => rule 2: inbound TCP port 443 accepted (TLS web traffic)
pvesh create /cluster/firewall/groups/web-tier \
  --type in --action ACCEPT --proto tcp --dport 22 \
  --source '+management-nets' --comment "SSH from management only"
# => rule 3: SSH allowed only from +management-nets IP set (limits attack surface)
# => +management-nets: references an IP set named 'management-nets' (+ prefix = IP set ref)
pvesh create /cluster/firewall/groups/web-tier \
  --type in --action DROP --comment "Default deny all other inbound"
# => rule 4: catch-all DROP; rules evaluated in order; unmatched traffic denied
# => DROP silently discards packets (vs REJECT which sends back an error to the sender)
# => Four rules added to security group 'web-tier'
 
# Apply security group to a VM (updating 'web-tier' propagates to all VMs using it)
pvesh create /nodes/pve01/qemu/100/firewall/rules \
  --type group \
  --action web-tier \
  --comment "Apply web-tier security group"
# => VM 100 uses all rules from 'web-tier'; group changes apply immediately cluster-wide
# => --type group: inserts the entire security group's rule set at this position in VM's chain
# => changes to 'web-tier' group automatically propagate to all VMs referencing it
 
# Create alias for cleaner rule definitions
pvesh create /cluster/firewall/aliases \
  --name db-cluster \
  --cidr 10.200.0.0/24 \
  --comment "Database cluster subnet"
# => Alias 'db-cluster' created; use in rules as source/dest
# => /cluster/firewall/aliases: cluster-scoped; alias resolves on all nodes at rule evaluation
# => aliases make rules readable: "source=db-cluster" vs "source=10.200.0.0/24"

# PBS 4.2 required for PVE 9; access PBS web UI at https://<pbs-server>:8007
 
# Add PBS as a storage backend (fingerprint from PBS dashboard)
pvesh create /storage \
  --storage pbs-main \
  --type pbs \
  --server 192.168.1.80 \
  --datastore main \
  --username backup@pbs \
  --password 'PBSBackupPassword!' \
  --fingerprint "XX:XX:XX:XX:..." \
  --content backup
# => pbs-main added; content=backup means VM disk images not stored here
# => --datastore main: PBS datastore name (configured on the PBS server)
# => --username backup@pbs: PBS realm user with Datastore.Backup privilege
# => --fingerprint: TLS certificate fingerprint; prevents MITM attacks on backup traffic
# => --content backup: storage type is backup-only (no VM disk images stored directly)
 
# Verify PBS connection (active=1 = PVE reaches PBS and authenticates successfully)
pvesh get /nodes/pve01/storage/pbs-main/status
# => total: 10 TB | used: 512 GB | avail: 9.5 TB | active: 1
# => active: 1 confirms TLS connection and authentication succeeded; 0 = connection failure
 
# Create daily backup job (first backup = full; subsequent = incremental changed-blocks only)
pvesh create /cluster/backup \
  --storage pbs-main \
  --schedule "0 1 * * *" \
  --mode snapshot \
  --prune-backups 'keep-daily=14,keep-weekly=8,keep-monthly=6' \
  --vmid all
# => backup job at 01:00; subsequent backups typically 5-20% of VM size
# => --schedule "0 1 * * *": cron syntax; runs at 01:00 every day
# => --mode snapshot: live snapshot (no VM downtime); needs thin-provisioned or ZFS storage
# => --prune-backups: automatic retention; keep 14 daily, 8 weekly, 6 monthly backups
# => --vmid all: back up all VMs and containers on the cluster

# Manual backup with all VMs and containers on this node
vzdump --all \
  --storage local \
  --mode snapshot \
  --compress zstd \
  --notes-template "Weekly backup - {{guestname}} - {{node}}"
# => backup archive per guest: vzdump-qemu-100-2026_04_29-02_00_00.vma.zst (5.2 GB → 3.1 GB compressed)
# => --all: backs up all running and stopped guests; excludes templates
# => --storage local: write archives to local storage (/var/lib/vz/dump/)
# => --compress zstd: zstd compression (better ratio than lzo, faster than gzip)
# => --notes-template: backup description embedded in archive metadata for identification
 
# Backup modes: snapshot=VM runs (needs thin-pool/ZFS); suspend=brief pause; stop=safest for DB
# snapshot (default): VM stays running; disk snapshot taken | suspend: 1-30s pause
# stop: VM stopped then restarted; maximum consistency; use for critical databases
 
# Backup a specific VM with retention policy
vzdump 100 \
  --storage pbs-main \
  --mode snapshot \
  --compress zstd \
  --remove 0
# => --remove 0: skip pruning after this job; manage retention via separate prune policy
# => vzdump 100: back up only VM 100; replaces --all for targeted backup
 
# View backup job status and history
pvesh get /cluster/backup --output-format json | python3 -c "
import sys, json
jobs = json.load(sys.stdin)['data']
# => jobs: list of backup job configs including schedule, storage, mode
for job in jobs:
    # => extract key fields: id, storage, schedule, mode
    print(f\"Job {job['id']}: storage={job['storage']}, schedule={job.get('schedule','manual')}, mode={job['mode']}\")
"
# => Job backup-abc123: storage=pbs-main, schedule=0 1 * * *, mode=snapshot

# List backups available in PBS for VM 100
pvesh get /nodes/pve01/storage/pbs-main/content --vmid 100
# => [{"volid":"pbs-main:backup/vm/100/2026-04-29T01:00:00Z","format":"pbs-vm","size":5497558138880},
# =>  {"volid":"pbs-main:backup/vm/100/2026-04-28T01:00:00Z", ...}]
 
# Restore VM with live-restore: VM starts immediately; remaining disk pages load on-demand
# --live-restore 1: VM boots in seconds; full disk present locally after ~20 min per 100 GB
qmrestore pbs-main:backup/vm/100/2026-04-29T01:00:00Z 100 \
  --storage local-lvm \
  --live-restore 1 \
  --force 1
# => VM 100 RUNNING immediately; disk restore proceeds in background from PBS
# => vs traditional restore: VM unavailable for the full 20-minute transfer duration
 
# Monitor live restore progress (poll tasks API for restore task status)
pvesh get /nodes/pve01/tasks --limit 5 | python3 -c "
import sys, json
# => GET /nodes/pve01/tasks returns last 5 tasks across this node
tasks = json.load(sys.stdin)['data']
# => tasks: list of {type, status, upid, starttime} dicts
for t in tasks:
    if 'restore' in t.get('type', ''):
        # => filter to restore-type tasks only
        print(f\"Restore task: {t['status']} progress: {t.get('upid', '')}\")
        # => status: 'running' while in progress; 'OK' when complete
"
# => Restore task: running progress: UPID:pve01:...
 
# Verify VM is running while restore proceeds in background
qm status 100
# => status: running  (VM serving requests; disk restore completes asynchronously)

# Generate AES-256 encryption key (CRITICAL: store separately — lost key = permanent data loss)
proxmox-backup-client key create \
  --master-pubkey /etc/proxmox-backup/master.pem
# => key saved to /etc/proxmox-backup/encryption-key.pem
# => --master-pubkey: wraps the encryption key with an RSA master key (enables key recovery)
 
# Configure PVE node to encrypt all backups to pbs-main
pvesh set /storage/pbs-main \
  --encryption-key /etc/proxmox-backup/encryption-key.pem
# => all new backups to pbs-main encrypted with AES-256
# => existing backups remain unencrypted; only new backups use the key
 
# Dry-run prune to preview retention (--dry-run: shows decisions without deleting)
proxmox-backup-client prune \
  --repository backup@pbs@192.168.1.80:main \
  --ns vm/100 \
  --keep-last 3 --keep-daily 14 --keep-weekly 8 --keep-monthly 6 --keep-yearly 2 \
  --dry-run
# => 33 kept | expired entries shown but NOT deleted
# => --ns vm/100: prune only the namespace for VM 100's backups
 
# Apply pruning (remove --dry-run)
proxmox-backup-client prune \
  --repository backup@pbs@192.168.1.80:main \
  --ns vm/100 \
  --keep-last 3 --keep-daily 14 --keep-weekly 8 --keep-monthly 6
# => 12 expired snapshots pruned | 45.2 GB freed
# => --repository format: user@realm@host:datastore (same as PBS web UI connection string)
 
# Run garbage collection to reclaim pruned space
proxmox-backup-client garbage-collect \
  --repository backup@pbs@192.168.1.80:main
# => 23,456 orphaned chunks removed | 38.7 GB freed
# => garbage-collect removes chunks no longer referenced by any snapshot (pruning only marks expired)

# View current resource limits for container 200
pct config 200
# => cores: 1 | cpulimit: 0 (unlimited) | memory: 512 | swap: 512
 
# Set CPU limit (0.5 = 50% of one core; enforced via cgroup v2 cpu.max)
pct set 200 --cpulimit 0.5
# => cgroup v2 cpu.max: 50000/100000 (50ms of each 100ms period)
 
# Set CPU units (lower = lower priority under contention; default 1024)
pct set 200 --cpuunits 512
# => container with 2048 units gets 4x CPU time vs this container during saturation
 
# Set memory hard limit (OOM-killed if exceeded)
pct set 200 --memory 1024 --swap 256
# => 1 GB RAM + 256 MB swap = 1280 MB total before OOM fires
 
# Enable memory ballooning (container gives back unused pages to host)
pct set 200 --memory 2048
# => dynamic: up to 2 GB; host reclaims unused pages from idle containers
 
# Add disk mount point with quota
pct set 200 --mp0 /var/data,disk=/dev/vg/data,quota=1,size=20G
# => mount /var/data with 20 GB size limit
 
# Verify effective cgroup limits inside the container
pct exec 200 -- cat /sys/fs/cgroup/cpu.max
# => 50000 100000  (50% of one core confirmed)
pct exec 200 -- cat /sys/fs/cgroup/memory.max
# => 1073741824  (1 GB = 2^30 bytes)
 
# Monitor container resource usage in real-time
pct monitor 200
# => CPU: 12.3%  MEM: 456/1024 MB  NET in: 1.2 MB/s out: 0.3 MB/s

# Attach cloud-init drive to a VM (stores cloud-init configuration as a small ISO)
qm set 100 \
  --ide2 local-lvm:cloudinit \
  --boot order=scsi0 \
  --serial0 socket \
  --vga serial0 \
  --ipconfig0 ip=dhcp \
  --nameserver 8.8.8.8 \
  --searchdomain lab.internal \
  --ciuser ubuntu \
  --cipassword 'CloudInitPass123!' \
  --sshkeys ~/.ssh/id_ed25519.pub
# => Cloud-init drive created at local-lvm:vm-100-cloudinit
# => On first boot: VM reads cloud-init config, sets root password and SSH key
# => --ide2 local-lvm:cloudinit: creates a small ISO on local-lvm with the cloud-init data
# => --boot order=scsi0: ensure VM boots from disk, not cloud-init ISO
# => --serial0 socket: enables serial console (required for cloud-init console output)
# => --vga serial0: redirects video to serial (works with cloud images that lack VGA drivers)
# => --ipconfig0 ip=dhcp: configure first NIC via DHCP
# => --nameserver 8.8.8.8: DNS resolver injected into guest via cloud-init
# => --searchdomain lab.internal: DNS search domain appended to short hostnames
# => --ciuser: default user account created in guest
# => --sshkeys: public key authorized for SSH login (no password needed)
 
# For static IP configuration:
qm set 100 \
  --ipconfig0 ip=192.168.1.150/24,gw=192.168.1.1
# => Static IP 192.168.1.150 configured via cloud-init
# => format: ip=<address>/<prefix>,gw=<gateway> (cloud-init network config v1)
 
# Custom cloud-init configuration (advanced: override with cicustom)
# Create custom user-data file
cat > /var/lib/vz/snippets/user-data-webserver.yml << 'EOF'
#cloud-config
packages:
  - nginx
  # => install nginx web server on first boot
  - fail2ban
  # => install fail2ban for SSH brute-force protection
  - unattended-upgrades
  # => enable automatic security updates
runcmd:
  - systemctl enable nginx
  # => enable nginx service to start on reboot
  - systemctl start nginx
  # => start nginx immediately after cloud-init runs
  - echo "Server configured by cloud-init" > /var/www/html/index.html
  # => write test page to confirm cloud-init ran successfully
users:
  - name: devops
    groups: sudo
    # => add user to sudo group for administrative access
    sudo: ALL=(ALL) NOPASSWD:ALL
    # => passwordless sudo (appropriate for automation; restrict in production)
    ssh_authorized_keys:
      - ssh-ed25519 AAAA... devops@company.com
      # => public key allows SSH login without password
EOF
# => Custom user-data YAML saved to /var/lib/vz/snippets/
 
# Apply custom cloud-init to VM (overrides web UI cloud-init settings)
qm set 100 --cicustom "user=local:snippets/user-data-webserver.yml"
# => VM 100 will execute custom cloud-init on next boot
# => local:snippets/ maps to /var/lib/vz/snippets/ on the PVE node
 
# Regenerate cloud-init ISO after changes
qm cloudinit update 100
# => Cloud-init ISO regenerated with current configuration
 
# Clone template and set unique cloud-init config per VM
qm clone 100 200 --name production-web-02 --full 1 --storage local-lvm
# => full clone: independent copy (--full 1); linked clone (--full 0) shares base disk
qm set 200 \
  --ipconfig0 ip=192.168.1.151/24,gw=192.168.1.1 \
  --ciuser ubuntu \
  --sshkeys ~/.ssh/id_ed25519.pub
# => VM 200 ready with unique IP; starts configured on first boot

# Get comprehensive cluster resource inventory (--output-format json: raw JSON for piping)
pvesh get /cluster/resources --type vm --output-format json | python3 -c "
import sys, json
# => GET /cluster/resources?type=vm returns all VMs across all nodes
data = json.load(sys.stdin)['data']
# => data: list of dicts with node, vmid, name, status, cpu (0.0-1.0), mem, maxmem
for vm in data:
    # => iterate each VM dict in the list
    cpu_pct = vm.get('cpu', 0) * 100
    # => cpu field is 0.0-1.0 fraction; multiply by 100 for percentage
    mem_mb = vm.get('mem', 0) // 1024 // 1024
    # => mem is in bytes; divide twice to get MB
    max_mem_mb = vm.get('maxmem', 0) // 1024 // 1024
    # => maxmem is also in bytes; convert to MB for the ratio display
    print(f\"{vm['node']}/{vm['vmid']:5d} {vm['name']:<25} {vm['status']:<10} CPU:{cpu_pct:5.1f}% MEM:{mem_mb:5d}/{max_mem_mb:5d}MB\")
    # => formatted output: node/vmid name status CPU% MEM used/max
"
# => pve01/100   ubuntu-24-server    running    CPU:  2.3% MEM: 1024/ 2048MB
# => pve02/105   db-postgres         running    CPU: 15.2% MEM: 4096/ 8192MB
 
# Modify a VM configuration using pvesh (memory takes effect on next reboot; CPU immediate)
pvesh set /nodes/pve01/qemu/100/config \
  --memory 4096 \
  --cores 4 \
  --description "Upgraded: 4 vCPU, 4GB RAM for increased web traffic"
# => VM 100 config updated; same as web UI config tab
# => --memory 4096: set RAM to 4 GB (takes effect on next reboot)
# => --cores 4: hot-plug vCPUs to 4 (takes effect immediately for running VMs)
 
# Trigger an action via pvesh (returns UPID to track async task progress)
pvesh create /nodes/pve01/qemu/100/status/start
# => {"data": "UPID:pve01:...:qmstart:100:root@pam:"}  (poll with /tasks/UPID/status)
# => /status/start: POST triggers VM start; other verbs: stop, shutdown, reboot, suspend
 
# Query storage usage across all nodes (filter nodes with maxdisk set)
pvesh get /cluster/resources --type storage --output-format json | python3 -c "
import sys, json
# => GET /cluster/resources?type=storage returns per-node storage metrics
stores = json.load(sys.stdin)['data']
for s in stores:
    # => iterate each storage entry (may have multiple per node)
    if s.get('maxdisk'):
        # => skip entries without capacity info (e.g., PBS remotes)
        used_pct = s.get('disk', 0) * 100 // s['maxdisk']
        # => disk field = bytes used; maxdisk = total bytes; integer division for percent
        print(f\"{s['node']}/{s['storage']:<15} {used_pct:3d}% used ({s.get('disk',0)//1024**3}GB/{s['maxdisk']//1024**3}GB)\")
"
# => pve01/local  12% | pve01/local-lvm  53% | pve02/ceph-vm  31%
 
# List recent cluster tasks with timestamp (useful for CI/CD audit)
pvesh get /cluster/tasks --limit 20 --output-format json | python3 -c "
import sys, json, datetime
# => GET /cluster/tasks returns last N tasks across all nodes
tasks = json.load(sys.stdin)['data']
for t in tasks:
    # => iterate each task dict; convert epoch starttime to human-readable
    st = datetime.datetime.fromtimestamp(t['starttime']).strftime('%H:%M:%S')
    # => format epoch starttime to HH:MM:SS for readability
    print(f\"[{st}] {t['node']}/{t['type']:<15} {t.get('id',''):<5} {t['status']}\")
    # => formatted: [time] node/type  id  status (OK/FAIL)
"
# => [02:00:01] pve01/vzdump  100  OK | [01:59:58] pve02/vzdump  200  OK

# Create a pool for the frontend team's VMs
pvesh create /pools \
  --poolid frontend-team \
  --comment "Frontend development team VM pool"
# => Pool 'frontend-team' created
# => pools are cluster-wide; visible from all nodes via the web UI left panel
 
# Add VMs to the pool
pvesh set /pools/frontend-team --vms 100,101,102
# => VMs 100, 101, 102 added to pool 'frontend-team'
 
# Add storage to the pool (restricts team to their storage quota)
pvesh set /pools/frontend-team --storage local-lvm
# => local-lvm storage added to pool (team can create VMs using this storage)
 
# Grant a user access to the pool only (not the entire cluster)
pvesh create /access/acl \
  --path /pool/frontend-team \
  --users frontend-dev@pve \
  --roles PVEVMUser \
  --propagate 1
# => frontend-dev@pve has PVEVMUser rights on all VMs in frontend-team pool
# => --propagate 1: ACL applies recursively to all VMs/storage within the pool
# => Users can: console, start, stop, reboot VMs in their pool
# => Users cannot: access other pools, create VMs, modify config, delete VMs
 
# Grant pool admin access to a team lead (can configure VMs in pool)
pvesh create /access/acl \
  --path /pool/frontend-team \
  --users frontend-lead@pve \
  --roles PVEVMAdmin
# => frontend-lead@pve has PVEVMAdmin rights on frontend-team pool
# => PVEVMAdmin role: full VM lifecycle management (create/modify/delete/migrate)
# => Team lead can: create, modify, delete, migrate VMs within the pool
 
# Verify pool ACL configuration
pvesh get /access/acl | python3 -c "
import sys, json
acls = json.load(sys.stdin)['data']
# => acls: list of all ACL entries across the entire cluster (path, ugid, roleid, propagate)
for acl in acls:
    # => filter to entries whose path contains 'frontend'
    if 'frontend' in acl.get('path', ''):
        print(f\"{acl['path']}: {acl['ugid']} -> {acl['roleid']}\")
"
# => /pool/frontend-team: frontend-dev@pve -> PVEVMUser
# => /pool/frontend-team: frontend-lead@pve -> PVEVMAdmin

# Install the assistant tool (run on a workstation, not on PVE)
apt install proxmox-auto-install-assistant
# => installed on Debian/Ubuntu tools machine
 
# Create a comprehensive answer.toml for a production node
cat > node-pve01-answer.toml << 'EOF'
[global]
# => [global]: installer-wide settings applied before disk and network setup
keyboard = "en-us"
# => keyboard layout (matches installer screen selection)
country = "us"
# => country code sets locale and NTP server defaults
fqdn = "pve01.prod.company.com"
# => fully qualified domain name written to /etc/hostname
mailto = "infra-alerts@company.com"
# => alert email; used for smartmontools and cron job failure emails
timezone = "UTC"
# => use UTC for servers; avoids daylight-saving ambiguity in log timestamps
root_password = "ProductionRootP@ss!"
# => root password set during unattended install
 
[disk-setup]
# => [disk-setup]: storage configuration for the Proxmox OS installation
filesystem = "zfs"
# => filesystem: zfs provides checksumming, compression, and snapshots for the OS disk
disk_list = ["sda", "sdb"]
# => ZFS mirror across two disks (RAID-1 equivalent)
zfs.raid = "mirror"
zfs.ashift = 12
# => ashift=12: 4K sector alignment (required for modern SSDs and HDDs)
zfs.compress = "lz4"
# => lz4 compression: fast with 30-50% space savings on typical workloads
zfs.checksum = "sha256"
# => sha256: stronger checksum (default fletcher4); use for critical data
zfs.copies = 2
# => copies=2: store two copies of data on each disk (extra redundancy within mirror)
 
[network]
# => [network]: NIC selection and addressing for the management interface
source = "from-answer"
# => source=from-answer: use the NIC specified in filter (vs auto-detect)
filter.ID_NET_NAME = "eno1"
# => select NIC by stable interface name (not bus position)
 
[network.network-settings]
cidr = "192.168.1.100/24"
# => static IP assigned during installation
gateway = "192.168.1.1"
# => default gateway for the management network
dns = "192.168.1.1"
# => DNS resolver; use router or internal DNS server
EOF
# => complete answer file: ZFS mirror on sda+sdb, static IP 192.168.1.100, hostname pve01
 
# Validate the answer file before embedding in ISO
proxmox-auto-install-assistant validate-answer node-pve01-answer.toml
# => Answer file is valid!  (catches TOML syntax errors and missing required fields)
 
# Embed answer file into PVE ISO for zero-touch installation
proxmox-auto-install-assistant prepare-iso proxmox-ve_9.2-1.iso \
  --fetch-from iso \
  --answer-file node-pve01-answer.toml \
  --output proxmox-pve01-auto.iso
# => proxmox-pve01-auto.iso (1.2 GB); boot this ISO to install without any interaction
# => --fetch-from iso: answer file embedded in ISO (single bootable image per node)
# => --answer-file: path to the validated TOML answer file
 
# PXE-based deployment: fetch unique per-node answer file from HTTP server by MAC
proxmox-auto-install-assistant prepare-iso proxmox-ve_9.2-1.iso \
  --fetch-from http \
  --url http://192.168.1.50/pxe/answers/pve01-answer.toml \
  --output proxmox-pve01-pxe.iso
# => ISO fetches answer file at boot; serve unique files per MAC for fleet provisioning
# => --fetch-from http: ISO downloads answer at boot time (one ISO for all nodes)
# => --url: HTTP endpoint serving per-MAC answer files (named by MAC address)

# Configure SMTP email (uses system sendmail; configure postfix for external relay)
pvesh create /cluster/notifications/endpoints/sendmail \
  --name smtp-alerts \
  --type sendmail \
  --mailto admin@company.com \
  --author "Proxmox Cluster" \
  --comment "Primary alert email endpoint"
# => smtp-alerts endpoint created
# => /cluster/notifications/endpoints/sendmail: API path creates a sendmail-type endpoint
# => --type sendmail: uses local MTA (postfix/exim); configure relay in /etc/postfix/main.cf
# => --mailto: destination address for all notifications sent to this endpoint
# => --author: From: display name in email headers
 
# Configure Gotify push (token from Gotify dashboard; sends push to mobile app)
pvesh create /cluster/notifications/endpoints/gotify \
  --name gotify-push \
  --url https://gotify.company.com \
  --token "AbCdEfGhIjKlMnOp" \
  --comment "Gotify push notification server"
# => gotify-push endpoint created
# => Gotify is an open-source self-hosted push notification server (alternative to FCM/APNs)
# => --url: Gotify server base URL (self-hosted push notification server)
# => --token: application token from Gotify dashboard (authenticates PVE to push)
 
# Configure Webhook for Slack ({{title}} and {{message}} substituted by PVE)
pvesh create /cluster/notifications/endpoints/webhook \
  --name slack-infra \
  --url "https://hooks.slack.com/services/T.../B.../..." \
  --method POST \
  --header '{"Content-Type": "application/json"}' \
  --body '{"text": "Proxmox Alert: {{title}}\n{{message}}"}' \
  --comment "Slack #infra-alerts channel webhook"
# => slack-infra webhook endpoint created
# => Slack incoming webhook URL generated in Slack App settings (Incoming Webhooks feature)
# => --method POST: Slack incoming webhooks require HTTP POST
# => --header: Content-Type header required by Slack webhook API
# => --body: JSON payload; {{title}} and {{message}} replaced by PVE at send time
 
# Route critical alerts to Slack and Gotify (match-field filters by severity)
pvesh create /cluster/notifications/matchers \
  --name critical-matcher \
  --match-field "severity:critical" \
  --target slack-infra,gotify-push \
  --comment "Route critical alerts to Slack and Gotify"
# => critical alerts → Slack + Gotify; non-critical → email only
# => matchers evaluate in order; first match wins; unmatched alerts go to default endpoint
# => --match-field "severity:critical": matches PVE alerts with severity=critical
# => --target: comma-separated list of endpoint names to notify
 
# Test endpoint and attach to backup job
pvesh create /cluster/notifications/endpoints/gotify/test \
  --target gotify-push
# => test notification sent; verify receipt in Gotify mobile app
# => /endpoints/gotify/test: built-in test action; sends a test message with mock title/body
pvesh set /cluster/backup/backup-1234abcd \
  --notification-mode notification-system \
  --notification-target smtp-alerts
# => backup job notifies via smtp-alerts on completion and failure
# => --notification-target smtp-alerts: override default routing for this specific backup job
# => --notification-mode notification-system: use the configured notification router (vs legacy email only)