#!/usr/bin/env bash
# verify-vm-stack.sh — prove the VM reference stack's controls are in force.
#
# ═══════════════════════════════════════════════════════════════════════════
#  HOW TO RUN THIS (start here)
# ═══════════════════════════════════════════════════════════════════════════
#
# This script only READS. It never changes a configuration, never restarts a
# service, and never bans anything, so it is safe to run as often as you like —
# including right before you submit, to catch a control you broke without
# noticing.
#
# WHAT YOU NEED FIRST
#   1. All three VMs running: kali, ubuntu-server, wazuh.
#   2. SSH working to ubuntu-server and wazuh *without a password prompt* — the
#      script runs many commands and will hang or fail on every one if it has
#      to ask. Test with:  ssh <your-ubuntu-alias> hostname
#   3. Passwordless sudo on both, since it inspects root-owned config.
#      Test with:  ssh <your-ubuntu-alias> 'sudo -n true && echo ok'
#   4. Your OpenBao service token at ../secrets/dvwa-service-token.txt
#      (section 3 checks the token can read but not write). Without it that
#      section reports a failure — which is a real finding, not a script bug.
#
#      The token is issued PER DEPLOYMENT. If you have more than one checkout,
#      or you rebuilt the VMs, the token sitting next to the script may belong
#      to a lab that no longer exists, and section 3 will report
#      "service token read failed" while everything else passes. Either run the
#      checkout whose secrets/ matches the VMs, or point SECRETS at it:
#
#        SECRETS=~/path/to/the/right/secrets ./verify-vm-stack.sh
#
#      Re-running the openbao_config role issues a fresh token and rewrites
#      secrets/ — do that if you have lost the old one.
#
# RUNNING IT
#   Default host aliases are ppubuntu / ppwazuh. If yours are named differently,
#   override them — you do NOT need to edit this file:
#
#     ./verify-vm-stack.sh                              # uses ppubuntu/ppwazuh
#     UBU=myubuntu WAZ=mywazuh ./verify-vm-stack.sh     # your own aliases
#     make verify                                       # same thing via the Makefile
#
#   The aliases must be real entries in ~/.ssh/config (with HostName, User and
#   any ProxyJump). Passing a raw "user@10.10.10.20" works only if you can
#   reach that address directly.
#
# READING THE OUTPUT
#   Each line is PASS or FAIL, grouped into ten sections. The last line is the
#   tally, and the exit status is 0 only if NOTHING failed — so you can use it
#   in CI or a pre-submit hook:
#
#     ./verify-vm-stack.sh || echo "fix these before submitting"
#
#   To see only what is broken:   ./verify-vm-stack.sh | grep FAIL
#
# WHEN SOMETHING FAILS
#   Read the section heading first — it tells you which lab the control came
#   from. Then go and look at the control itself rather than at this script.
#   Two failures that confuse people every term:
#     * "OpenBao is sealed" after a reboot is EXPECTED. OpenBao does not
#       auto-unseal; supply three key shares with `bao operator unseal`.
#     * "agent not Active" usually means the agent is running but never enrolled.
#       Check /var/ossec/etc/client.keys is non-empty.
#
#   A failure here is information, not a grade. Fix the control, re-run, and
#   say what happened in your write-up — an honestly documented gap earns more
#   than a hidden one.
#
# WHY SECTIONS 8-10 EXIST
#   Sections 1-7 check a control is PRESENT. That is not the same as working.
#   During the capstone this script reported a perfect score over a fail2ban
#   jail that had never banned anything, a dead FIM daemon, and a secrets store
#   that would vanish on reboot. Sections 8-10 exercise the controls instead of
#   describing them. Bear that in mind when you write your own checks: proving
#   a service is running is the easy half.
#
# ═══════════════════════════════════════════════════════════════════════════
#
# Runs read-only checks over SSH against the three lab VMs (aliases ppubuntu,
# ppwazuh, ppkali from ~/.ssh/config). It changes nothing. Exit 0 iff every
# check passes.
#
# ── Presence vs function ──────────────────────────────────────────────────────
# Sections 1-7 largely check that a control is PRESENT: a jail is listed, a
# ruleset is loaded, a config key exists. The 2026-07-31 capstone run showed
# that is not enough — this script returned 28/28 PASS while:
#
#   * the fail2ban DVWA jail had never banned anything (wrong backend, and a
#     failregex matching a timestamp fail2ban strips before matching),
#   * wazuh-syscheckd was dead, so FIM had no baseline and could not alert,
#   * openbao was disabled at boot, so it vanished on the first reboot.
#
# Every one of those is invisible to a presence check and obvious to a
# functional one. Section 8 exercises the controls instead of describing them,
# and section 9 checks they survive a reboot. Both stay read-only: fail2ban-regex
# takes the sample line as a string (no temp file), and wazuh-logtest reads
# stdin.
set -uo pipefail

UBU=${UBU:-ppubuntu}
WAZ=${WAZ:-ppwazuh}
# Defaults to this checkout's own secrets/, but is overridable — the OpenBao
# token is per-deployment, so running one checkout's verifier against another
# lab's VMs needs the matching token or section 3 fails confusingly.
SECRETS="${SECRETS:-$(cd "$(dirname "$0")/.." && pwd)/secrets}"

pass=0 fail=0
ok()  { printf '  \033[32mPASS\033[0m %s\n' "$1"; pass=$((pass+1)); }
no()  { printf '  \033[31mFAIL\033[0m %s\n' "$1"; fail=$((fail+1)); }
say() { printf '\n=== %s ===\n' "$1"; }
# run <host> <remote command>  → stdout of the remote command
r()   { ssh -o BatchMode=yes -o ConnectTimeout=10 "$1" "$2" 2>/dev/null; }

# rtok <host> <remote command>  → same, but with the OpenBao token delivered on
# stdin instead of embedded in the command string.
#
# Why this exists — and it is NOT the reason you would first guess. Measured,
# not assumed:
#
#   * On the REMOTE host, `ssh host "BAO_TOKEN=$tok bao kv ..."` does NOT leak.
#     The login shell applies the assignment and then execs, replacing itself,
#     so the remote cmdline reads `bao kv get ...` and the token exists only in
#     /proc/<pid>/environ, which is readable by the owner and root alone. A
#     400-iteration tight poll on the far side never caught it in `ps`.
#
#   * On the LOCAL machine it DOES leak. The token is an argument to ssh, so it
#     sits in ssh's own argv:
#         ssh -o BatchMode=yes ssubuntu BAO_TOKEN=<token> bao kv get ...
#     and /proc/<pid>/cmdline is WORLD-READABLE. Any other user on the box you
#     run this from can read the token out of `ps` for the life of the command.
#     That matters because students run this from shared CS machines.
#
# Passing it on stdin keeps it out of ssh's argv entirely. Verified: with the
# old form, one ssh process carried the token in argv; with this form, zero.
#
# It still ends up in the remote process's environment, which is the same
# exposure as before — this change closes the local, world-readable hole, not
# the remote, owner-readable one.
#
# $tok is used from the enclosing scope deliberately rather than passed as an
# argument: an argument would put the token in this script's OWN argv on the
# machine you are running it from, which is the bug we are fixing, just local.
rtok() {
  printf '%s\n' "$tok" | ssh -o BatchMode=yes -o ConnectTimeout=10 "$1" \
    "read -r BAO_TOKEN; export BAO_TOKEN BAO_ADDR=http://127.0.0.1:8200; $2" 2>/dev/null
}

say "1. Wazuh manager — rules, agent, active response"
rules="$(r "$WAZ" 'sudo grep -oE "rule id=\"100[0-9]+\"" /var/ossec/etc/rules/local_rules.xml')"
for id in 100001 100002 100100 100101 100102; do
  grep -q "$id" <<<"$rules" && ok "rule $id present" || no "rule $id missing"
done
r "$WAZ" 'sudo grep -q "<rules_id>100002</rules_id>" /var/ossec/etc/ossec.conf' \
  && ok "firewall-drop active response bound to 100002" \
  || no "active response not bound to 100002"
agents="$(r "$WAZ" 'sudo /var/ossec/bin/agent_control -l')"
grep -q 'ubuntu-server.*Active' <<<"$agents" \
  && ok "agent 'ubuntu-server' is Active" || no "agent not Active"
# rule 100001 fires in logtest
lt="$(r "$WAZ" 'printf "10.10.10.10 - - [01/Jan/2026:00:00:00 +0000] \"POST /dvwa/login.php HTTP/1.1\" 302 0 \"-\" \"c\"\n" | sudo /var/ossec/bin/wazuh-logtest 2>&1')"
grep -q "id: '100001'" <<<"$lt" && ok "logtest: rule 100001 fires on a DVWA login line" \
  || no "logtest: rule 100001 did not fire"

say "2. Workload — hardened container stack on :80"
ps="$(r "$UBU" 'cd /opt/dvwa-stack && sudo docker compose ps --format "{{.Service}}={{.Health}}"')"
for svc in nginx dvwa mariadb; do
  grep -q "$svc=healthy" <<<"$ps" && ok "$svc container healthy" || no "$svc not healthy ($ps)"
done
code="$(r "$UBU" 'curl -s -o /dev/null -w "%{http_code}" http://127.0.0.1/dvwa/login.php')"
[ "$code" = 200 ] && ok "DVWA served through proxy (/dvwa/login.php 200)" || no "proxy returned $code"

say "3. OpenBao — unsealed, least-privilege service token"
r "$UBU" 'BAO_ADDR=http://127.0.0.1:8200 bao status -format=json' | grep -q '"sealed": false' \
  && ok "OpenBao is unsealed" || no "OpenBao is sealed"
if [ -f "$SECRETS/dvwa-service-token.txt" ]; then
  tok="$(cat "$SECRETS/dvwa-service-token.txt")"
  rd="$(rtok "$UBU" 'bao kv get -field=username secret/dvwa/db')"
  if [ "$rd" = dvwauser ]; then
    ok "service token can READ secret/dvwa/db"
    # NB: bao writes the denial to stderr; fold it into stdout inside the remote
    # command since rtok() drops remote stderr. Capture to a var (not a pipe to
    # grep -q) so pipefail's SIGPIPE doesn't mask a match.
    wr="$(rtok "$UBU" 'bao kv put secret/dvwa/db x=y 2>&1')"
    grep -qi 'permission denied' <<<"$wr" && ok "service token WRITE denied (403)" || no "write was not denied"
  else
    # The write test is only meaningful once the read has succeeded. An invalid,
    # expired or WRONG-DEPLOYMENT token is denied EVERYTHING, so it would sail
    # through a bare "is the write denied?" check and report least-privilege as
    # working when in fact nothing was proven. Do not award that PASS.
    no "service token read failed — token is invalid, expired, or from another deployment"
    no "WRITE-denied check skipped: inconclusive while the token cannot read"
    echo "       (token file: $SECRETS/dvwa-service-token.txt)"
    echo "       secrets/ is resolved relative to this script. Running the copy"
    echo "       under courses/secdevops/solutions/ against a DIFFERENT lab will"
    echo "       use that copy's stale token. Point SECRETS= at the deployment"
    echo "       you actually provisioned, or re-run the openbao_config role."
  fi
else
  no "local service token missing at $SECRETS/dvwa-service-token.txt (run the openbao_config role)"
fi

say "4. auditd — watches loaded"
al="$(r "$UBU" 'sudo auditctl -l')"
for k in shadow_access docker_sock docker_exec; do
  grep -q "$k" <<<"$al" && ok "audit key $k loaded" || no "audit key $k missing"
done

say "5. Suricata — running, custom rule, EVE flowing"
r "$UBU" 'systemctl is-active suricata' | grep -q '^active' && ok "suricata active" || no "suricata not active"
for sid in 9000001 9000002 9000003; do
  r "$UBU" "sudo grep -q 'sid:$sid' /var/lib/suricata/rules/local.rules" \
    && ok "custom rule sid $sid installed" || no "custom rule sid $sid missing"
done
r "$UBU" 'sudo test -s /var/log/suricata/eve.json' && ok "eve.json is being written" || no "no eve.json"
# Per-flow eve records are what saturate the Wazuh agent queue: a -p- scan opens
# 65,535 flows, and the agent tails the whole file. With `- flow` enabled a scan
# ships ~65k records and Wazuh reports rule 203, "events may be lost".
r "$UBU" 'sudo grep -qE "^\s*#\s*- flow\s*($|#)" /etc/suricata/suricata.yaml' \
  && ok "eve-log per-flow records disabled (agent queue stays healthy)" \
  || no "eve-log still emits per-flow records — a port scan will drop events"

say "6. Wazuh agent — collection + FIM"
oc="$(r "$UBU" 'sudo cat /var/ossec/etc/ossec.conf')"
grep -q 'logs/nginx/access.log' <<<"$oc" && ok "agent reads the nginx access log" || no "nginx localfile missing"
grep -q '/var/log/suricata/eve.json' <<<"$oc" && ok "agent reads Suricata EVE" || no "suricata localfile missing"
grep -q '/var/log/audit/audit.log' <<<"$oc" && ok "agent reads the audit log" || no "audit localfile missing"
grep -q '<syscheck>' <<<"$oc" && ok "FIM (syscheck) configured" || no "syscheck missing"
r "$UBU" 'sudo /var/ossec/bin/wazuh-control status' | grep -q 'wazuh-agentd is running' \
  && ok "wazuh-agent running" || no "wazuh-agent not running"

say "7. Host firewall — nftables drop policy + fail2ban"
nt="$(r "$UBU" 'sudo nft list table inet lab_filter')"
grep -q 'policy drop' <<<"$nt" && ok "lab_filter input policy is drop" || no "no drop policy"
grep -q "saddr 10.10.10.10 tcp dport 22 accept" <<<"$nt" && ok "SSH allowed from Kali only" || no "kali SSH rule missing"
grep -q "dport 3306 accept" <<<"$nt" && grep -q "127.0.0.1 tcp dport 3306" <<<"$nt" \
  && ok "MariaDB 3306 localhost-only" || no "3306 rule missing/over-broad"
jl="$(r "$UBU" 'sudo fail2ban-client status')"
grep -q 'sshd' <<<"$jl" && grep -q 'nginx-dvwa-login' <<<"$jl" \
  && ok "fail2ban jails sshd + nginx-dvwa-login active" || no "fail2ban jails missing"

say "8. Functional checks — do the controls actually fire?"

# 8a. The fail2ban filter must MATCH a real attack line, not merely exist.
# fail2ban-regex accepts the sample as a string, so this writes nothing. The
# bracket pair is deliberate: fail2ban strips the matched date but leaves the
# surrounding "[]", so a filter still carrying \[[^\]]+\] silently matches
# nothing. That is exactly the bug this check was written to catch.
f2r="$(r "$UBU" 'sudo fail2ban-regex "10.10.10.10 - - [01/Jan/2026:00:00:00 +0000] \"POST /dvwa/login.php HTTP/1.0\" 302 0 \"-\" \"x\"" /etc/fail2ban/filter.d/nginx-dvwa-login.conf 2>&1')"
grep -qE '^Failregex: [1-9]' <<<"$f2r" \
  && ok "fail2ban filter matches a real DVWA login line" \
  || no "fail2ban filter matches NOTHING (jail is inert)"

# 8b. The jail must be tailing the nginx access FILE. With backend=systemd it
# reads the journal instead and ignores logpath entirely — the container never
# writes to the journal, so the jail can never see the evidence.
js="$(r "$UBU" 'sudo fail2ban-client status nginx-dvwa-login 2>/dev/null')"
grep -q 'File list:.*access\.log' <<<"$js" \
  && ok "fail2ban jail reads the nginx access log file" \
  || no "jail is not reading the access log (backend=systemd?)"

# 8c. FIM is a separate daemon from the agent. wazuh-agentd can be running
# while wazuh-syscheckd is dead, in which case there is no FIM at all.
wc_status="$(r "$UBU" 'sudo /var/ossec/bin/wazuh-control status')"
grep -q 'wazuh-syscheckd is running' <<<"$wc_status" \
  && ok "wazuh-syscheckd running (FIM has a baseline)" \
  || no "wazuh-syscheckd NOT running — FIM cannot alert"
grep -q 'wazuh-logcollector is running' <<<"$wc_status" \
  && ok "wazuh-logcollector running (localfiles are read)" \
  || no "wazuh-logcollector not running"

# 8d. The authored web-attack rules must fire on a representative Suricata
# event. Feeds wazuh-logtest a synthetic EVE alert on stdin — read-only.
for pair in 9000002:100101 9000003:100102; do
  sid="${pair%%:*}"; want="${pair##*:}"
  lt="$(r "$WAZ" 'printf "{\"timestamp\":\"2026-01-01T00:00:00.000000+0000\",\"event_type\":\"alert\",\"src_ip\":\"10.10.10.10\",\"dest_ip\":\"10.10.10.20\",\"proto\":\"TCP\",\"alert\":{\"gid\":1,\"signature_id\":'"$sid"',\"rev\":1,\"signature\":\"t\",\"category\":\"Web Application Attack\",\"severity\":1}}\n" | sudo /var/ossec/bin/wazuh-logtest 2>&1')"
  grep -q "id: '$want'" <<<"$lt" \
    && ok "logtest: sid $sid escalates to rule $want" \
    || no "logtest: sid $sid did not reach rule $want"
done

say "9. Durability — will the controls survive a reboot?"

# Every control here was applied at run time by Ansible. `enabled` is what
# decides whether it comes back. OpenBao was found `disabled` after a reboot:
# not sealed, not running, with the verifier reporting three unrelated-looking
# failures.
for svc in nftables openbao wazuh-agent suricata fail2ban auditd; do
  st="$(r "$UBU" "systemctl is-enabled $svc 2>&1")"
  case "$st" in
    enabled|enabled-runtime|static|indirect|alias|generated) ok "$svc enabled at boot ($st)" ;;
    *) no "$svc NOT enabled at boot ($st)" ;;
  esac
done

# The firewall's persistence lives in the include, not the entrypoint — grep the
# entrypoint alone and you will wrongly conclude the rules are not persisted.
r "$UBU" 'sudo nft -c -f /etc/nftables.conf' \
  && ok "/etc/nftables.conf (incl. nftables.d) parses — rules reload at boot" \
  || no "/etc/nftables.conf does not parse; firewall will not come back"
r "$UBU" 'sudo grep -q "lab_filter" /etc/nftables.d/lab.nft' \
  && ok "lab_filter is defined in /etc/nftables.d/lab.nft" \
  || no "lab_filter not persisted to /etc/nftables.d/lab.nft"

say "10. Container hardening — the runtime controls, not just liveness"

# Section 2 only proves the stack is UP. These are the Week 3/4 controls that
# actually stopped the capstone: the read-only rootfs refused a webshell written
# to the DVWA webroot, and the absent docker socket closed the standard escape
# route — both while the attacker had full RCE as www-data inside the container.
# ../reference-stack/scripts/verify-stack.sh tests these on a single Docker
# host; nothing tested them here, so an unhardened stack passed 28/28.
ins="$(r "$UBU" 'cd /opt/dvwa-stack && for s in nginx dvwa mariadb; do
  id=$(sudo docker compose ps -q $s)
  printf "%s ro=%s caps=%s secopt=%s user=[%s] nets=[%s] env=[%s]\n" "$s" \
    "$(sudo docker inspect -f "{{.HostConfig.ReadonlyRootfs}}" $id)" \
    "$(sudo docker inspect -f "{{json .HostConfig.CapDrop}}" $id)" \
    "$(sudo docker inspect -f "{{json .HostConfig.SecurityOpt}}" $id)" \
    "$(sudo docker inspect -f "{{.Config.User}}" $id)" \
    "$(sudo docker inspect -f "{{range \$k,\$v := .NetworkSettings.Networks}}{{\$k}} {{end}}" $id)" \
    "$(sudo docker inspect -f "{{range .Config.Env}}{{println .}}{{end}}" $id | grep -iE "password|secret|token" | grep -vi "_FILE=" | tr "\n" ",")"
done')"
svcline() { grep "^$1 " <<<"$ins"; }

# Read-only rootfs — nginx and dvwa only. mariadb is writable BY DESIGN: its
# entrypoint fixes data-dir ownership before dropping to the mysql user.
for svc in nginx dvwa; do
  grep -q 'ro=true' <<<"$(svcline $svc)" \
    && ok "$svc rootfs read-only (blocks webshell drop)" \
    || no "$svc rootfs is WRITABLE — persistence would succeed"
done

for svc in nginx dvwa mariadb; do
  grep -qi 'caps=\["ALL"\]' <<<"$(svcline $svc)" \
    && ok "$svc drops ALL capabilities" || no "$svc did not cap_drop ALL"
  grep -q 'no-new-privileges:true' <<<"$(svcline $svc)" \
    && ok "$svc has no-new-privileges" || no "$svc missing no-new-privileges"
done

grep -q 'seccomp' <<<"$(svcline dvwa)" \
  && ok "dvwa has a seccomp profile" || no "dvwa running seccomp=unconfined"

# nginx and dvwa must not run as root inside the container.
for svc in nginx dvwa; do
  u="$(sed -n 's/.*user=\[\([^]]*\)\].*/\1/p' <<<"$(svcline $svc)")"
  [ -n "$u" ] && [ "$u" != "root" ] && [ "$u" != "0" ] \
    && ok "$svc runs as non-root ($u)" || no "$svc runs as root ('$u')"
done

# Secrets must be *_FILE pointers, never literal values in the environment —
# `docker inspect` is readable by anyone in the docker group.
for svc in nginx dvwa mariadb; do
  e="$(sed -n 's/.*env=\[\([^]]*\)\].*/\1/p' <<<"$(svcline $svc)")"
  [ -z "$e" ] && ok "$svc environment has no literal secret values" \
              || no "$svc exposes a secret in its environment ($e)"
done

# Segmentation: nginx must share no network with the database. Topology is
# nginx=[frontend], dvwa=[frontend,backend], mariadb=[backend] — dvwa bridges.
nnets="$(sed -n 's/.*nets=\[\([^]]*\)\].*/\1/p' <<<"$(svcline nginx)")"
dnets="$(sed -n 's/.*nets=\[\([^]]*\)\].*/\1/p' <<<"$(svcline mariadb)")"
shared=0
for n in $dnets; do case " $nnets " in *" $n "*) shared=1;; esac; done
[ "$shared" -eq 0 ] && ok "nginx shares no network with mariadb (segmented)" \
                    || no "nginx and mariadb share a network — segmentation broken"

# The docker socket must not be mounted into any container: that is a direct
# host-takeover path, and the capstone confirmed it was absent.
sock="$(r "$UBU" 'sudo docker ps -q | while read i; do sudo docker inspect -f "{{range .Mounts}}{{.Source}} {{end}}" $i; done | grep -c docker.sock')"
[ "${sock:-1}" = 0 ] && ok "no container mounts /var/run/docker.sock" \
                     || no "docker.sock is mounted into a container (escape path)"

printf '\n===========================================\n'
printf 'RESULT: %d passed, %d failed\n' "$pass" "$fail"
[ "$fail" -eq 0 ] && { echo "ALL CHECKS PASSED"; exit 0; } || exit 1