obmp-portability-test/scripts/run-test.sh

#!/usr/bin/env bash
#
# run-test.sh -- assertion harness for the two-VM portability test.
#
# Assumes `terraform apply` has already run in ../terraform/. Pulls the
# store VM's IP from terraform output, waits for cloud-init to finish
# (poll /var/lib/cloud/portability-test.done), then runs the six
# assertions from docs/design.md. Fails on the first failing assertion.

set -euo pipefail
cd "$(dirname "$0")/../terraform"

# --- config ---------------------------------------------------------------
: "${WAIT_TIMEOUT_SEC:=900}"     # 15 min for cloud-init + deploy.sh
: "${SSH_OPTS:=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10}"

# --- pull terraform outputs ----------------------------------------------
if ! command -v jq >/dev/null; then
    echo "jq not installed; needed to parse terraform outputs" >&2
    exit 2
fi
if ! command -v terraform >/dev/null; then
    echo "terraform not installed on this host" >&2
    exit 2
fi

TF_JSON="$(terraform output -json)"
STORE_IP=$(echo "$TF_JSON"  | jq -r .store_vm_ip.value)
BMPGEN_IP=$(echo "$TF_JSON" | jq -r .bmpgen_vm_ip.value)
SSH_USER=$(echo "$TF_JSON"  | jq -r .ssh_username.value)

if [ -z "$STORE_IP" ] || [ "$STORE_IP" = "null" ]; then
    echo "Could not resolve store_vm_ip from terraform outputs. Is terraform apply complete?" >&2
    exit 2
fi

echo "store VM:  $STORE_IP"
echo "bmpgen VM: $BMPGEN_IP"
echo "ssh user:  $SSH_USER"
echo

# --- helper --------------------------------------------------------------
ssh_store() { ssh $SSH_OPTS "$SSH_USER@$STORE_IP" "$@" ; }

# --- wait for cloud-init + deploy.sh completion --------------------------
echo "== waiting for cloud-init/deploy.sh to finish on the store VM (timeout ${WAIT_TIMEOUT_SEC}s) =="
start=$(date +%s)
while true; do
    if ssh_store "test -f /var/lib/cloud/portability-test.done" 2>/dev/null; then
        marker="$(ssh_store cat /var/lib/cloud/portability-test.done)"
        echo "marker:"
        echo "$marker" | sed 's/^/  /'
        rc=$(echo "$marker" | grep -E '^deploy_exit=' | cut -d= -f2)
        if [ "$rc" != "0" ]; then
            echo "FAIL: deploy.sh exited $rc on the store VM"
            echo "  tail /var/log/obmp-deploy.log:"
            ssh_store tail -40 /var/log/obmp-deploy.log | sed 's/^/    /'
            exit 1
        fi
        break
    fi
    now=$(date +%s)
    if [ $((now - start)) -gt "$WAIT_TIMEOUT_SEC" ]; then
        echo "FAIL: cloud-init did not finish within ${WAIT_TIMEOUT_SEC}s"
        exit 1
    fi
    sleep 15
    echo -n "."
done
echo

# --- inject synthetic routes on bmpgen so ip_rib has rows to assert ------
if [ -n "$BMPGEN_IP" ] && [ "$BMPGEN_IP" != "null" ]; then
    echo "== injecting synthetic routes on bmpgen VM =="
    ssh $SSH_OPTS "$SSH_USER@$BMPGEN_IP" sudo /usr/local/bin/inject-synthetic-routes.sh \
        || echo "  (warning: route injection failed; assertion 5 may not pass)"
    echo
fi

# --- assertions ----------------------------------------------------------
echo "== running assertions =="

pass=0
fail=0
assert() {
    local name="$1" ; shift
    echo -n "  [$((pass + fail + 1))] $name ... "
    if "$@" >/dev/null 2>&1; then
        echo "PASS"
        pass=$((pass + 1))
    else
        echo "FAIL"
        fail=$((fail + 1))
        # re-run visibly so failure output ends up in the log
        echo "    (retrying with output for diagnosis)"
        "$@" 2>&1 | sed 's/^/      /'
    fi
}

# 1: all obmp containers healthy
assert_all_containers_healthy() {
    local unhealthy
    unhealthy=$(ssh_store "docker ps --filter 'name=obmp-' --format '{{.Status}}' | grep -c 'unhealthy\\|Restarting' || true")
    [ "$unhealthy" = "0" ]
}
assert "all obmp containers healthy" assert_all_containers_healthy

# 2: collector logs show a BMP peer up event
assert_collector_peer_up() {
    ssh_store "docker logs obmp-collector 2>&1 | grep -qE 'peer[[:space:]]+up|Received BMP INIT'"
}
assert "collector accepted a BMP session" assert_collector_peer_up

# 3: routers table has at least 1 row (the bmpgen VM's gobgp)
assert_routers_row() {
    local n
    n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM routers'")
    [ "${n:-0}" -ge 1 ]
}
assert "routers table has >= 1 row" assert_routers_row

# 4: at least 1 UP BGP peer
assert_bgp_peer_up() {
    local n
    n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc \"SELECT count(*) FROM bgp_peers WHERE state='UP'\"")
    [ "${n:-0}" -ge 1 ]
}
assert "bgp_peers has >= 1 UP session" assert_bgp_peer_up

# 5: ip_rib has rows (from the synthetic route injection)
assert_ip_rib_rows() {
    local n
    n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM ip_rib'")
    [ "${n:-0}" -ge 1 ]
}
assert "ip_rib has >= 1 row" assert_ip_rib_rows

# 6: Grafana healthcheck
assert_grafana_healthy() {
    curl -sf "http://$STORE_IP:3000/api/health" >/dev/null
}
assert "Grafana /api/health returns 200" assert_grafana_healthy

# --- summary -------------------------------------------------------------
echo
echo "== summary: $pass passed, $fail failed =="
if [ "$fail" -gt 0 ]; then
    echo "One or more assertions failed. Diagnostic pointers:"
    echo "  - deploy log:      ssh $SSH_USER@$STORE_IP tail -200 /var/log/obmp-deploy.log"
    echo "  - collector logs:  ssh $SSH_USER@$STORE_IP docker logs obmp-collector --tail 200"
    echo "  - container state: ssh $SSH_USER@$STORE_IP docker ps -a"
    exit 1
fi

echo "All assertions passed. Grafana: http://$STORE_IP:3000/grafana/  (admin / openbmp)"