#!/usr/bin/env bash # # run-test.sh -- assertion harness for the two-VM portability test. # # Assumes `terraform apply` has already run in ../terraform/. Pulls the # store VM's IP from terraform output, waits for cloud-init to finish # (poll /var/lib/cloud/portability-test.done), then runs the six # assertions from docs/design.md. Fails on the first failing assertion. set -euo pipefail cd "$(dirname "$0")/../terraform" # --- config --------------------------------------------------------------- : "${WAIT_TIMEOUT_SEC:=900}" # 15 min for cloud-init + deploy.sh : "${SSH_OPTS:=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10}" # --- pull terraform outputs ---------------------------------------------- if ! command -v jq >/dev/null; then echo "jq not installed; needed to parse terraform outputs" >&2 exit 2 fi if ! command -v terraform >/dev/null; then echo "terraform not installed on this host" >&2 exit 2 fi TF_JSON="$(terraform output -json)" STORE_IP=$(echo "$TF_JSON" | jq -r .store_vm_ip.value) BMPGEN_IP=$(echo "$TF_JSON" | jq -r .bmpgen_vm_ip.value) SSH_USER=$(echo "$TF_JSON" | jq -r .ssh_username.value) if [ -z "$STORE_IP" ] || [ "$STORE_IP" = "null" ]; then echo "Could not resolve store_vm_ip from terraform outputs. Is terraform apply complete?" >&2 exit 2 fi echo "store VM: $STORE_IP" echo "bmpgen VM: $BMPGEN_IP" echo "ssh user: $SSH_USER" echo # --- helper -------------------------------------------------------------- ssh_store() { ssh $SSH_OPTS "$SSH_USER@$STORE_IP" "$@" ; } # --- wait for cloud-init + deploy.sh completion -------------------------- echo "== waiting for cloud-init/deploy.sh to finish on the store VM (timeout ${WAIT_TIMEOUT_SEC}s) ==" start=$(date +%s) while true; do if ssh_store "test -f /var/lib/cloud/portability-test.done" 2>/dev/null; then marker="$(ssh_store cat /var/lib/cloud/portability-test.done)" echo "marker:" echo "$marker" | sed 's/^/ /' rc=$(echo "$marker" | grep -E '^deploy_exit=' | cut -d= -f2) if [ "$rc" != "0" ]; then echo "FAIL: deploy.sh exited $rc on the store VM" echo " tail /var/log/obmp-deploy.log:" ssh_store tail -40 /var/log/obmp-deploy.log | sed 's/^/ /' exit 1 fi break fi now=$(date +%s) if [ $((now - start)) -gt "$WAIT_TIMEOUT_SEC" ]; then echo "FAIL: cloud-init did not finish within ${WAIT_TIMEOUT_SEC}s" exit 1 fi sleep 15 echo -n "." done echo # --- inject synthetic routes on bmpgen so ip_rib has rows to assert ------ if [ -n "$BMPGEN_IP" ] && [ "$BMPGEN_IP" != "null" ]; then echo "== injecting synthetic routes on bmpgen VM ==" ssh $SSH_OPTS "$SSH_USER@$BMPGEN_IP" sudo /usr/local/bin/inject-synthetic-routes.sh \ || echo " (warning: route injection failed; assertion 5 may not pass)" echo fi # --- assertions ---------------------------------------------------------- echo "== running assertions ==" pass=0 fail=0 assert() { local name="$1" ; shift echo -n " [$((pass + fail + 1))] $name ... " if "$@" >/dev/null 2>&1; then echo "PASS" pass=$((pass + 1)) else echo "FAIL" fail=$((fail + 1)) # re-run visibly so failure output ends up in the log echo " (retrying with output for diagnosis)" "$@" 2>&1 | sed 's/^/ /' fi } # 1: all obmp containers healthy assert_all_containers_healthy() { local unhealthy unhealthy=$(ssh_store "docker ps --filter 'name=obmp-' --format '{{.Status}}' | grep -c 'unhealthy\\|Restarting' || true") [ "$unhealthy" = "0" ] } assert "all obmp containers healthy" assert_all_containers_healthy # 2: collector logs show a BMP peer up event assert_collector_peer_up() { ssh_store "docker logs obmp-collector 2>&1 | grep -qE 'peer[[:space:]]+up|Received BMP INIT'" } assert "collector accepted a BMP session" assert_collector_peer_up # 3: routers table has at least 1 row (the bmpgen VM's gobgp) assert_routers_row() { local n n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM routers'") [ "${n:-0}" -ge 1 ] } assert "routers table has >= 1 row" assert_routers_row # 4: at least 1 UP BGP peer. bgp_peers.state uses lowercase 'up' (opstate enum). assert_bgp_peer_up() { local n n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc \"SELECT count(*) FROM bgp_peers WHERE state='up'\"") [ "${n:-0}" -ge 1 ] } assert "bgp_peers has >= 1 UP session" assert_bgp_peer_up # 5: ip_rib has rows (from the synthetic route injection) assert_ip_rib_rows() { local n n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM ip_rib'") [ "${n:-0}" -ge 1 ] } assert "ip_rib has >= 1 row" assert_ip_rib_rows # 6: Grafana healthcheck assert_grafana_healthy() { curl -sf "http://$STORE_IP:3000/api/health" >/dev/null } assert "Grafana /api/health returns 200" assert_grafana_healthy # --- summary ------------------------------------------------------------- echo echo "== summary: $pass passed, $fail failed ==" if [ "$fail" -gt 0 ]; then echo "One or more assertions failed. Diagnostic pointers:" echo " - deploy log: ssh $SSH_USER@$STORE_IP tail -200 /var/log/obmp-deploy.log" echo " - collector logs: ssh $SSH_USER@$STORE_IP docker logs obmp-collector --tail 200" echo " - container state: ssh $SSH_USER@$STORE_IP docker ps -a" exit 1 fi echo "All assertions passed. Grafana: http://$STORE_IP:3000/grafana/ (admin / openbmp)"