From d30ec2074aa09aac69957b410a738661e9d83999 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 20 Jul 2026 18:19:45 -0700 Subject: [PATCH] initial commit: two-VM Proxmox portability test for obmp-docker Terraform (bpg/proxmox) provisions two Ubuntu 24.04 cloud-init VMs on a Proxmox host, one running obmp-docker's `deploy.sh --scope full-stack` and the other running GoBGP as a BMP-source stand-in. `scripts/run-test.sh` runs the six assertions from docs/design.md (containers healthy, BMP peer up, ip_rib/bgp_peers non-zero, Grafana healthcheck). Layout: terraform/ provider + VM + cloud-init snippet resources terraform/cloud-init/ user-data templates (store + bmpgen) scripts/ run-test.sh, teardown.sh docs/design.md topology, assertion set, and rationale Cred handling: terraform.tfvars is gitignored; endpoint + api_token can also come from PROXMOX_VE_ENDPOINT / PROXMOX_VE_API_TOKEN env vars. Not validated locally: `terraform init && terraform validate` -- the terraform binary is not on the authoring host. YAML + shell + HCL brace balance verified. Co-Authored-By: Claude Opus 4.7 --- .gitignore | 23 +++ README.md | 94 +++++++++ docs/design.md | 123 ++++++++++++ scripts/run-test.sh | 155 +++++++++++++++ scripts/teardown.sh | 25 +++ terraform/bmpgen_vm.tf | 89 +++++++++ .../cloud-init/bmpgen.cloud-init.yaml.tftpl | 94 +++++++++ .../cloud-init/store.cloud-init.yaml.tftpl | 86 +++++++++ terraform/image.tf | 15 ++ terraform/main.tf | 32 +++ terraform/outputs.tf | 50 +++++ terraform/store_vm.tf | 88 +++++++++ terraform/terraform.tfvars.example | 53 +++++ terraform/variables.tf | 182 ++++++++++++++++++ 14 files changed, 1109 insertions(+) create mode 100644 .gitignore create mode 100644 README.md create mode 100644 docs/design.md create mode 100755 scripts/run-test.sh create mode 100755 scripts/teardown.sh create mode 100644 terraform/bmpgen_vm.tf create mode 100644 terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl create mode 100644 terraform/cloud-init/store.cloud-init.yaml.tftpl create mode 100644 terraform/image.tf create mode 100644 terraform/main.tf create mode 100644 terraform/outputs.tf create mode 100644 terraform/store_vm.tf create mode 100644 terraform/terraform.tfvars.example create mode 100644 terraform/variables.tf diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..c0a4897 --- /dev/null +++ b/.gitignore @@ -0,0 +1,23 @@ +# Terraform state -- has infra addresses and (with older providers) can leak +# secrets if written unencrypted. Never commit. +terraform/.terraform/ +terraform/.terraform.lock.hcl +terraform/*.tfstate +terraform/*.tfstate.* +terraform/*.tfstate.backup +terraform/crash.log +terraform/crash.*.log + +# Real inputs -- contains PVE endpoint, API token, network specifics. +# Ship terraform.tfvars.example instead. +terraform/terraform.tfvars +terraform/*.auto.tfvars + +# Any local override files +terraform/override.tf +terraform/override.tf.json +terraform/*_override.tf +terraform/*_override.tf.json + +# CLI outputs +*.log diff --git a/README.md b/README.md new file mode 100644 index 0000000..ddff5b3 --- /dev/null +++ b/README.md @@ -0,0 +1,94 @@ +# obmp-portability-test + +End-to-end portability test for the [obmp-docker](http://10.40.40.143:3000/sam/obmp-docker) +stack. Spins up two fresh Proxmox VMs from a stock Ubuntu cloud image, has +one deploy the OpenBMP stack (`deploy.sh --scope full-stack --yes`) and the +other run GoBGP configured to open a BMP session to it, then asserts that +data actually lands in Postgres. + +If this test passes, "portability" is more than a hope — you can hand the +[obmp-docker](http://10.40.40.143:3000/sam/obmp-docker) URL to a new +operator and they get a working stack. + +## What it proves + +- `deploy.sh` runs to completion on a from-scratch Ubuntu 24.04 host with + no prior state. +- The stack's containers all reach healthy after the documented delay. +- A router (GoBGP standing in for one) can open a BMP session to the + collector on the ROUTER_FACING_IP / ROUTER_FACING_PORT defined in the + target's `.env`. +- Route data actually lands in `ip_rib`, peer sessions land in + `bgp_peers` — i.e. the ingest path works end-to-end across two hosts. + +If any of these fail, the test tells you *which* step broke. + +## Layout + +``` +obmp-portability-test/ +├── README.md +├── docs/design.md <- what/why in more detail +├── terraform/ +│ ├── main.tf <- providers + versions +│ ├── variables.tf <- inputs, all with descriptions +│ ├── terraform.tfvars.example <- copy to terraform.tfvars, fill in +│ ├── image.tf <- one-time cloud-image download +│ ├── store_vm.tf <- VM A: OpenBMP store +│ ├── bmpgen_vm.tf <- VM B: GoBGP BMP source +│ ├── outputs.tf <- IPs, next-step hints +│ └── cloud-init/ +│ ├── store.cloud-init.yaml.tftpl +│ └── bmpgen.cloud-init.yaml.tftpl +└── scripts/ + ├── run-test.sh <- terraform apply + wait + assert + └── teardown.sh <- terraform destroy wrapper +``` + +## Prerequisites + +- Proxmox VE 7.4+ (bpg/proxmox provider requires it). +- A PVE API token with permissions to create/destroy VMs, download files + to the ISO storage, and use the target storage pool. Any user with + `PVEVMAdmin` on `/` works for a lab setup; production would scope + tighter. +- Terraform 1.5+ on the machine you run this from. +- SSH keypair for reaching the VMs after they boot (path passed as a + terraform variable). + +## Running + +```bash +cd terraform +cp terraform.tfvars.example terraform.tfvars +$EDITOR terraform.tfvars # fill in PVE endpoint, token, network, storage +terraform init +terraform validate +terraform plan +terraform apply + +# Once the VMs are up (cloud-init takes 5-10 min for docker+deploy.sh): +cd .. +./scripts/run-test.sh # polls SSH, then runs the assertions +``` + +To tear down: + +```bash +./scripts/teardown.sh +``` + +## Cost of one test run + +Two VMs (2 vCPU / 4 GB / 20 GB disk each by default) for ~30 min. On a +Proxmox host with headroom, negligible. All state is destroyed on +teardown; cloud image is cached across runs. + +## What's NOT tested + +- Authelia mode (`OBMP_AUTH_MODE=authelia`) — this test uses local auth + because there's no reverse-proxy front-end in the two-VM topology. +- Real router integration (BGP-LS, gNMI, NETCONF) — GoBGP handles the + BMP path only. Extending to a third VM with FRR + gNMI is future work. +- Long-running behavior — the assertions fire once, ~15 min in. Not a + soak test. diff --git a/docs/design.md b/docs/design.md new file mode 100644 index 0000000..970ee83 --- /dev/null +++ b/docs/design.md @@ -0,0 +1,123 @@ +# Design: Two-VM Portability Test + +## Goal + +Prove that `obmp-docker` deploys and ingests on a fresh host with only the +inputs a new operator would have (repo URL, a hypervisor, an SSH key). The +existing `deploy.sh` claims portability; this test is what turns the claim +into evidence. + +## Topology + +``` + +------------------------+ BMP TCP :ROUTER_FACING_PORT + | store VM (VM-A) | <----------------------------------+ + | ubuntu 24.04 cloud | | + | docker + deploy.sh | | + | full-stack profile | | + | | | + | ROUTER_FACING_IP = | | + | | | + | ROUTER_FACING_PORT = | +----------+---------+ + | 1790 (IANA BMP) | | bmpgen VM (VM-B) | + | | | ubuntu 24.04 | + +------------------------+ | docker | + | gobgp container | + | (BMP client) | + +-------------------+ + + | | + +---- shared L2 (Proxmox bridge) or routed subnet ----+ +``` + +Two VMs, both stock Ubuntu 24.04 cloud images, both provisioned entirely +by cloud-init. No manual step between `terraform apply` and the +assertions running. + +## Why two VMs, not one + +The `deploy.sh --scope full-stack --profile test` option already spins up +GoBGP *inside* the same compose network as the collector. That proves the +compose is internally consistent, but it doesn't exercise the actual +router-integration surface (`ROUTER_FACING_IP`/`ROUTER_FACING_PORT`, +compose's `${ROUTER_FACING_PORT:-1790}:5000` publish, cross-host TCP, +management-plane reachability). A second VM is the smallest topology that +makes those things real. + +If the two-VM test passes, "deploy.sh works AND routers can reach it" +holds. If it fails, we know exactly which of the two. + +## What VM-A runs + +Cloud-init on VM-A does: + +1. Install `docker.io`, `docker-compose-plugin`, `git`. +2. `git clone` the obmp-docker repo at a pinned branch/commit. +3. Write a minimal `.env` from `.env.example` with: + - `HOST_IP` = VM-A's own primary IP + - `ROUTER_FACING_IP` = same + - `ROUTER_FACING_PORT` = 1790 (test uses IANA port, not the lab + transitional 5000) + - `OBMP_AUTH_MODE=local` (no reverse proxy in this topology) +4. Run `./deploy.sh --scope full-stack --auth local --yes`. +5. Write a completion marker to `/var/lib/cloud/portability-test.done` + with exit status so the test harness can poll it. + +## What VM-B runs + +Cloud-init on VM-B does: + +1. Install `docker.io`. +2. `docker run` a GoBGP container with a config file that: + - Sets local AS to a private test ASN (65099). + - Establishes a BMP session to `:1790`. + - Injects a small synthetic route set so `ip_rib` gets non-zero rows. +3. Write a completion marker. + +GoBGP is minimal on purpose — a real router would test more, but each +extra dial-in path (BGP-LS, gNMI, NETCONF) needs a real IOS-XR +environment or an emulator like XRd. The minimum route-monitoring path +is enough to catch the portability failure modes. + +## Assertion set (run by `scripts/run-test.sh`) + +Once cloud-init on both VMs is done (poll the completion markers, or 15 +min timeout), SSH into VM-A and run: + +| # | Assertion | Query / check | +|---|------------------------------------------------------------------|-----------------------------------------------------------------| +| 1 | All obmp containers healthy | `docker ps --format '{{.Names}} {{.Status}}'` grep unhealthy = 0 | +| 2 | Collector accepted a BMP session from VM-B | `docker logs obmp-collector` contains "peer up" for VM-B's IP | +| 3 | `routers` table has 1 row | `SELECT count(*) FROM routers` = 1 | +| 4 | `bgp_peers` has ≥ 1 UP session | `SELECT count(*) FROM bgp_peers WHERE state='UP'` ≥ 1 | +| 5 | `ip_rib` has ≥ 1 row (GoBGP injected some routes) | `SELECT count(*) FROM ip_rib` ≥ 1 | +| 6 | Grafana healthcheck returns 200 | `curl -sf http://:3000/api/health` | + +Pass = all six green. Failure emits the first failing assertion + a +tarball of `docker logs`, `.env`, `deploy.sh` output for diagnosis. + +## Why bpg/proxmox + +Two candidate providers exist for Proxmox: `Telmate/proxmox` (original, +slower moving) and `bpg/proxmox` (fork, active, modern cloud-init +handling). We pick bpg because: + +- Cloud-init `user_data_file_id` is first-class — we can hand-craft the + cloud-init YAML and reference it by snippet ID rather than shoving it + through the provider's per-field API. +- `proxmox_virtual_environment_download_file` handles the cloud-image + download idempotently. +- The type system + docs are noticeably better on 0.60+. + +## Why cloud-init and not Ansible + +Adding Ansible would double the surface. Cloud-init is enough for +"install a few packages, drop a repo, run a script" — anything more +complex is a smell that we're testing the wrong thing. + +## Why a separate repo + +Test infra should not gate product commits. If the two-VM test suite +grows (multi-vendor router VMs, prod-like tuning tests, longer-running +soak variants), it earns its own release cadence. Keeping the code +separate keeps that door open. diff --git a/scripts/run-test.sh b/scripts/run-test.sh new file mode 100755 index 0000000..09b0b7e --- /dev/null +++ b/scripts/run-test.sh @@ -0,0 +1,155 @@ +#!/usr/bin/env bash +# +# run-test.sh -- assertion harness for the two-VM portability test. +# +# Assumes `terraform apply` has already run in ../terraform/. Pulls the +# store VM's IP from terraform output, waits for cloud-init to finish +# (poll /var/lib/cloud/portability-test.done), then runs the six +# assertions from docs/design.md. Fails on the first failing assertion. + +set -euo pipefail +cd "$(dirname "$0")/../terraform" + +# --- config --------------------------------------------------------------- +: "${WAIT_TIMEOUT_SEC:=900}" # 15 min for cloud-init + deploy.sh +: "${SSH_OPTS:=-o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o LogLevel=ERROR -o ConnectTimeout=10}" + +# --- pull terraform outputs ---------------------------------------------- +if ! command -v jq >/dev/null; then + echo "jq not installed; needed to parse terraform outputs" >&2 + exit 2 +fi +if ! command -v terraform >/dev/null; then + echo "terraform not installed on this host" >&2 + exit 2 +fi + +TF_JSON="$(terraform output -json)" +STORE_IP=$(echo "$TF_JSON" | jq -r .store_vm_ip.value) +BMPGEN_IP=$(echo "$TF_JSON" | jq -r .bmpgen_vm_ip.value) +SSH_USER=$(echo "$TF_JSON" | jq -r .ssh_username.value) + +if [ -z "$STORE_IP" ] || [ "$STORE_IP" = "null" ]; then + echo "Could not resolve store_vm_ip from terraform outputs. Is terraform apply complete?" >&2 + exit 2 +fi + +echo "store VM: $STORE_IP" +echo "bmpgen VM: $BMPGEN_IP" +echo "ssh user: $SSH_USER" +echo + +# --- helper -------------------------------------------------------------- +ssh_store() { ssh $SSH_OPTS "$SSH_USER@$STORE_IP" "$@" ; } + +# --- wait for cloud-init + deploy.sh completion -------------------------- +echo "== waiting for cloud-init/deploy.sh to finish on the store VM (timeout ${WAIT_TIMEOUT_SEC}s) ==" +start=$(date +%s) +while true; do + if ssh_store "test -f /var/lib/cloud/portability-test.done" 2>/dev/null; then + marker="$(ssh_store cat /var/lib/cloud/portability-test.done)" + echo "marker:" + echo "$marker" | sed 's/^/ /' + rc=$(echo "$marker" | grep -E '^deploy_exit=' | cut -d= -f2) + if [ "$rc" != "0" ]; then + echo "FAIL: deploy.sh exited $rc on the store VM" + echo " tail /var/log/obmp-deploy.log:" + ssh_store tail -40 /var/log/obmp-deploy.log | sed 's/^/ /' + exit 1 + fi + break + fi + now=$(date +%s) + if [ $((now - start)) -gt "$WAIT_TIMEOUT_SEC" ]; then + echo "FAIL: cloud-init did not finish within ${WAIT_TIMEOUT_SEC}s" + exit 1 + fi + sleep 15 + echo -n "." +done +echo + +# --- inject synthetic routes on bmpgen so ip_rib has rows to assert ------ +if [ -n "$BMPGEN_IP" ] && [ "$BMPGEN_IP" != "null" ]; then + echo "== injecting synthetic routes on bmpgen VM ==" + ssh $SSH_OPTS "$SSH_USER@$BMPGEN_IP" sudo /usr/local/bin/inject-synthetic-routes.sh \ + || echo " (warning: route injection failed; assertion 5 may not pass)" + echo +fi + +# --- assertions ---------------------------------------------------------- +echo "== running assertions ==" + +pass=0 +fail=0 +assert() { + local name="$1" ; shift + echo -n " [$((pass + fail + 1))] $name ... " + if "$@" >/dev/null 2>&1; then + echo "PASS" + pass=$((pass + 1)) + else + echo "FAIL" + fail=$((fail + 1)) + # re-run visibly so failure output ends up in the log + echo " (retrying with output for diagnosis)" + "$@" 2>&1 | sed 's/^/ /' + fi +} + +# 1: all obmp containers healthy +assert_all_containers_healthy() { + local unhealthy + unhealthy=$(ssh_store "docker ps --filter 'name=obmp-' --format '{{.Status}}' | grep -c 'unhealthy\\|Restarting' || true") + [ "$unhealthy" = "0" ] +} +assert "all obmp containers healthy" assert_all_containers_healthy + +# 2: collector logs show a BMP peer up event +assert_collector_peer_up() { + ssh_store "docker logs obmp-collector 2>&1 | grep -qE 'peer[[:space:]]+up|Received BMP INIT'" +} +assert "collector accepted a BMP session" assert_collector_peer_up + +# 3: routers table has at least 1 row (the bmpgen VM's gobgp) +assert_routers_row() { + local n + n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM routers'") + [ "${n:-0}" -ge 1 ] +} +assert "routers table has >= 1 row" assert_routers_row + +# 4: at least 1 UP BGP peer +assert_bgp_peer_up() { + local n + n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc \"SELECT count(*) FROM bgp_peers WHERE state='UP'\"") + [ "${n:-0}" -ge 1 ] +} +assert "bgp_peers has >= 1 UP session" assert_bgp_peer_up + +# 5: ip_rib has rows (from the synthetic route injection) +assert_ip_rib_rows() { + local n + n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc 'SELECT count(*) FROM ip_rib'") + [ "${n:-0}" -ge 1 ] +} +assert "ip_rib has >= 1 row" assert_ip_rib_rows + +# 6: Grafana healthcheck +assert_grafana_healthy() { + curl -sf "http://$STORE_IP:3000/api/health" >/dev/null +} +assert "Grafana /api/health returns 200" assert_grafana_healthy + +# --- summary ------------------------------------------------------------- +echo +echo "== summary: $pass passed, $fail failed ==" +if [ "$fail" -gt 0 ]; then + echo "One or more assertions failed. Diagnostic pointers:" + echo " - deploy log: ssh $SSH_USER@$STORE_IP tail -200 /var/log/obmp-deploy.log" + echo " - collector logs: ssh $SSH_USER@$STORE_IP docker logs obmp-collector --tail 200" + echo " - container state: ssh $SSH_USER@$STORE_IP docker ps -a" + exit 1 +fi + +echo "All assertions passed. Grafana: http://$STORE_IP:3000/grafana/ (admin / openbmp)" diff --git a/scripts/teardown.sh b/scripts/teardown.sh new file mode 100755 index 0000000..a5f5e94 --- /dev/null +++ b/scripts/teardown.sh @@ -0,0 +1,25 @@ +#!/usr/bin/env bash +# +# teardown.sh -- destroy the two test VMs. Cloud image + snippet files +# stay in PVE storage so the next `terraform apply` is fast. + +set -euo pipefail +cd "$(dirname "$0")/../terraform" + +if ! command -v terraform >/dev/null; then + echo "terraform not installed on this host" >&2 + exit 2 +fi + +# Destroy only the VMs, keep the cached cloud image + snippets so the +# next apply doesn't re-download. `terraform destroy` with -target lists +# is fine because everything else is either downloaded content (image) +# or a snippet file (cheap to re-create). +terraform destroy \ + -target=proxmox_virtual_environment_vm.store \ + -target=proxmox_virtual_environment_vm.bmpgen \ + -auto-approve + +echo +echo "VMs destroyed. To also drop the cached image + snippets:" +echo " terraform destroy (without -target)" diff --git a/terraform/bmpgen_vm.tf b/terraform/bmpgen_vm.tf new file mode 100644 index 0000000..4ccb731 --- /dev/null +++ b/terraform/bmpgen_vm.tf @@ -0,0 +1,89 @@ +# VM B -- GoBGP standing in for a real BGP router. Cloud-init installs +# docker and runs a gobgp container configured to BMP-report to VM A. +# +# `depends_on` on the store VM's cloud-init snippet, not the store VM +# itself -- we don't want to wait for the store's cloud-init to finish +# before launching B (cloud-init on B will retry the BMP session until A +# is ready). + +locals { + bmpgen_hostname = "obmp-bmpgen-test" + + # If DHCP, we don't know VM-A's address at plan time and cloud-init on + # VM-B will need it. We handle both cases in the template: + # - static: pass store_ip_addr directly + # - DHCP: cloud-init on B does an ARP scan + hostname lookup at boot + # (kept out-of-scope here; DHCP + this test is best-effort) + bmpgen_store_addr = var.store_vm_ip_cidr != "" ? split("/", var.store_vm_ip_cidr)[0] : "" + + bmpgen_user_data = templatefile("${path.module}/cloud-init/bmpgen.cloud-init.yaml.tftpl", { + hostname = local.bmpgen_hostname + ssh_username = var.ssh_username + ssh_public_key = trimspace(file(var.ssh_public_key_path)) + store_ip_addr = local.bmpgen_store_addr + router_facing_port = var.router_facing_port + local_asn = var.test_local_asn + }) +} + +resource "proxmox_virtual_environment_file" "bmpgen_cloud_init" { + content_type = "snippets" + datastore_id = var.snippet_storage + node_name = var.pve_node + + source_raw { + file_name = "obmp-bmpgen-user-data.yaml" + data = local.bmpgen_user_data + } +} + +resource "proxmox_virtual_environment_vm" "bmpgen" { + name = local.bmpgen_hostname + description = "OpenBMP portability-test BMPGEN VM (managed by terraform)" + tags = ["obmp", "portability-test", "bmpgen"] + node_name = var.pve_node + vm_id = var.bmpgen_vm_id + + agent { enabled = true } + cpu { + cores = 2 + type = "host" + } + memory { + dedicated = 2048 + } + + disk { + datastore_id = var.vm_storage_pool + file_id = proxmox_virtual_environment_download_file.ubuntu_noble.id + interface = "scsi0" + size = 16 + ssd = true + discard = "on" + } + + network_device { + bridge = var.vm_bridge + model = "virtio" + vlan_id = var.vm_vlan_tag != 0 ? var.vm_vlan_tag : null + } + + operating_system { type = "l26" } + + initialization { + datastore_id = var.vm_storage_pool + + ip_config { + ipv4 { + address = var.bmpgen_vm_ip_cidr != "" ? var.bmpgen_vm_ip_cidr : "dhcp" + gateway = var.bmpgen_vm_ip_cidr != "" ? var.vm_gateway : null + } + } + + dns { + servers = var.vm_dns_servers + } + + user_data_file_id = proxmox_virtual_environment_file.bmpgen_cloud_init.id + } +} diff --git a/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl b/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl new file mode 100644 index 0000000..cf97b8d --- /dev/null +++ b/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl @@ -0,0 +1,94 @@ +#cloud-config +# BMP-gen VM cloud-init: install docker, drop a gobgpd.conf that reports +# BMP to the store VM, run gobgp as a container. GoBGP keeps retrying +# the BMP session until the store's collector accepts it, so ordering +# between the two VMs' cloud-init is not required. + +hostname: ${hostname} +manage_etc_hosts: true + +users: + - name: ${ssh_username} + groups: [sudo, docker] + shell: /bin/bash + sudo: "ALL=(ALL) NOPASSWD:ALL" + ssh_authorized_keys: + - ${ssh_public_key} + +package_update: true +package_upgrade: false + +packages: + - ca-certificates + - curl + - gnupg + - lsb-release + - qemu-guest-agent + +write_files: + # GoBGP config. Local AS is the test private ASN; BMP target is the + # store VM's ROUTER_FACING_IP + ROUTER_FACING_PORT. We inject a small + # synthetic route set so obmp's ip_rib has non-zero rows to assert on. + - path: /etc/gobgp/gobgpd.conf + owner: root:root + permissions: '0644' + content: | + [global.config] + as = ${local_asn} + router-id = "10.99.99.99" + + [[bmp-servers]] + [bmp-servers.config] + address = "${store_ip_addr}" + port = ${router_facing_port} + route-monitoring-policy = "pre-policy" + + # No BGP peers -- the assertion only requires that (a) BMP session + # comes up and reports at least a router-init, and (b) the collector + # sees a peer. Adding synthetic prefixes needs at least one BGP + # peer or a manual `gobgp global rib add` after boot; the run-test + # script handles the manual add so ip_rib gets rows. + + # Convenience: script that adds synthetic prefixes into gobgp's global + # RIB after start. run-test.sh SSHes in and executes this. + - path: /usr/local/bin/inject-synthetic-routes.sh + owner: root:root + permissions: '0755' + content: | + #!/bin/bash + set -euo pipefail + # Add a couple of synthetic prefixes; the docker exec targets the + # gobgp container we started in runcmd. + for i in 1 2 3 4 5; do + docker exec obmp-gobgp gobgp global rib add "203.0.113.$((i * 8))/29" origin igp + done + echo "injected 5 synthetic prefixes" + +runcmd: + # --- 1. install docker-ce (same pattern as store VM) ------------------- + - install -m 0755 -d /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + - chmod a+r /etc/apt/keyrings/docker.gpg + - | + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + > /etc/apt/sources.list.d/docker.list + - apt-get update + - apt-get install -y docker-ce docker-ce-cli containerd.io + - systemctl enable --now docker qemu-guest-agent + - usermod -aG docker ${ssh_username} + + # --- 2. start gobgp container ---------------------------------------- + - | + docker run -d --restart unless-stopped --name obmp-gobgp \ + --network host \ + -v /etc/gobgp:/etc/gobgp:ro \ + jauderho/gobgp:v4.5.0 \ + gobgpd -f /etc/gobgp/gobgpd.conf + + # --- 3. mark done ------------------------------------------------------ + - | + echo "gobgp_running=$(docker inspect -f {{.State.Running}} obmp-gobgp)" > /var/lib/cloud/portability-test.done + echo "bmp_target=${store_ip_addr}:${router_facing_port}" >> /var/lib/cloud/portability-test.done + echo "finished_at=$(date -Is)" >> /var/lib/cloud/portability-test.done + +final_message: "obmp-bmpgen cloud-init done after $UPTIME seconds. gobgp running." diff --git a/terraform/cloud-init/store.cloud-init.yaml.tftpl b/terraform/cloud-init/store.cloud-init.yaml.tftpl new file mode 100644 index 0000000..9ba2ac3 --- /dev/null +++ b/terraform/cloud-init/store.cloud-init.yaml.tftpl @@ -0,0 +1,86 @@ +#cloud-config +# Store VM cloud-init: install docker + git, clone obmp-docker, write a +# portable .env, run deploy.sh --scope full-stack --auth local --yes. +# +# All completion / status writes to /var/lib/cloud/portability-test.done +# so run-test.sh can poll for readiness without guessing. + +hostname: ${hostname} +manage_etc_hosts: true + +users: + - name: ${ssh_username} + groups: [sudo, docker] + shell: /bin/bash + sudo: "ALL=(ALL) NOPASSWD:ALL" + ssh_authorized_keys: + - ${ssh_public_key} + +package_update: true +package_upgrade: false + +packages: + - ca-certificates + - curl + - git + - gnupg + - jq + - lsb-release + - qemu-guest-agent + +runcmd: + # --- 1. install docker-ce from the official repo (docker.io in Ubuntu + # noble is old enough to cause compose issues; matches what obmp-docker + # deploy.sh expects) ----------------------------------------------------- + - install -m 0755 -d /etc/apt/keyrings + - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | gpg --dearmor -o /etc/apt/keyrings/docker.gpg + - chmod a+r /etc/apt/keyrings/docker.gpg + - | + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" \ + > /etc/apt/sources.list.d/docker.list + - apt-get update + - apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + - systemctl enable --now docker qemu-guest-agent + - usermod -aG docker ${ssh_username} + + # --- 2. resolve HOST_IP: static value if provided, else `hostname -I` -- + - | + set -eu + if [ -n "${static_ip_addr}" ]; then + HOST_IP="${static_ip_addr}" + else + HOST_IP="$(hostname -I | awk '{print $1}')" + fi + echo "$HOST_IP" > /var/lib/cloud/host_ip + + # --- 3. clone the obmp-docker repo ------------------------------------- + - sudo -u ${ssh_username} git clone ${obmp_repo_url} /home/${ssh_username}/obmp-docker + - sudo -u ${ssh_username} git -C /home/${ssh_username}/obmp-docker checkout ${obmp_repo_ref} + + # --- 4. seed .env with test-topology values ---------------------------- + - | + set -eu + HOST_IP="$(cat /var/lib/cloud/host_ip)" + cd /home/${ssh_username}/obmp-docker + sudo -u ${ssh_username} cp .env.example .env + sed -i "s|^HOST_IP=.*|HOST_IP=$HOST_IP|" .env + sed -i "s|^ROUTER_FACING_IP=.*|ROUTER_FACING_IP=$HOST_IP|" .env + sed -i "s|^ROUTER_FACING_PORT=.*|ROUTER_FACING_PORT=${router_facing_port}|" .env + sed -i "s|^OBMP_AUTH_MODE=.*|OBMP_AUTH_MODE=local|" .env + chown ${ssh_username}:${ssh_username} .env + + # --- 5. run deploy.sh (drops permissions to the ubuntu user) ---------- + - | + set +e + cd /home/${ssh_username}/obmp-docker + sudo -u ${ssh_username} \ + HOME=/home/${ssh_username} \ + ./deploy.sh --scope full-stack --auth local --yes \ + > /var/log/obmp-deploy.log 2>&1 + rc=$? + echo "deploy_exit=$rc" > /var/lib/cloud/portability-test.done + echo "finished_at=$(date -Is)" >> /var/lib/cloud/portability-test.done + echo "host_ip=$(cat /var/lib/cloud/host_ip)" >> /var/lib/cloud/portability-test.done + exit 0 + +final_message: "obmp-store cloud-init done after $UPTIME seconds. Marker: /var/lib/cloud/portability-test.done" diff --git a/terraform/image.tf b/terraform/image.tf new file mode 100644 index 0000000..4b1c8ad --- /dev/null +++ b/terraform/image.tf @@ -0,0 +1,15 @@ +# One-time Ubuntu cloud image download. bpg/proxmox handles the "did I +# already download this?" check by URL + storage; safe to re-run. +resource "proxmox_virtual_environment_download_file" "ubuntu_noble" { + content_type = "iso" + datastore_id = var.image_storage + node_name = var.pve_node + + url = var.ubuntu_image_url + file_name = "noble-server-cloudimg-amd64.img" + checksum = var.ubuntu_image_checksum != "" ? var.ubuntu_image_checksum : null + checksum_algorithm = var.ubuntu_image_checksum != "" ? "sha256" : null + overwrite = false + overwrite_unmanaged = false + upload_timeout = 900 +} diff --git a/terraform/main.tf b/terraform/main.tf new file mode 100644 index 0000000..e80d415 --- /dev/null +++ b/terraform/main.tf @@ -0,0 +1,32 @@ +terraform { + required_version = ">= 1.5.0" + + required_providers { + proxmox = { + source = "bpg/proxmox" + version = "~> 0.60" + } + random = { + source = "hashicorp/random" + version = "~> 3.6" + } + } +} + +# Provider config. Endpoint and token can come from PROXMOX_VE_ENDPOINT and +# PROXMOX_VE_API_TOKEN environment variables instead of tfvars -- preferred +# for CI / shared workstations. `insecure` should be true only for a lab PVE +# that presents a self-signed cert. +provider "proxmox" { + endpoint = var.pve_endpoint + api_token = var.pve_api_token + insecure = var.pve_insecure + + # SSH is used by the provider for a few operations (snippet uploads, etc). + # Optional -- if omitted the provider falls back to API-only paths where + # possible. Set node -> address so the provider doesn't have to resolve. + ssh { + agent = true + username = var.pve_ssh_username + } +} diff --git a/terraform/outputs.tf b/terraform/outputs.tf new file mode 100644 index 0000000..9a0093a --- /dev/null +++ b/terraform/outputs.tf @@ -0,0 +1,50 @@ +# Outputs feed the run-test.sh wrapper (jq consumes `terraform output -json`) +# and give the operator sane next-step hints after `apply`. + +output "store_vm_ip" { + description = "IPv4 the store VM will be reachable on." + value = var.store_vm_ip_cidr != "" ? split("/", var.store_vm_ip_cidr)[0] : ( + length(proxmox_virtual_environment_vm.store.ipv4_addresses) > 1 ? + proxmox_virtual_environment_vm.store.ipv4_addresses[1][0] : "" + ) +} + +output "bmpgen_vm_ip" { + description = "IPv4 the bmpgen VM will be reachable on." + value = var.bmpgen_vm_ip_cidr != "" ? split("/", var.bmpgen_vm_ip_cidr)[0] : ( + length(proxmox_virtual_environment_vm.bmpgen.ipv4_addresses) > 1 ? + proxmox_virtual_environment_vm.bmpgen.ipv4_addresses[1][0] : "" + ) +} + +output "ssh_username" { + description = "SSH username cloud-init provisioned on both VMs." + value = var.ssh_username +} + +output "grafana_url" { + description = "Grafana URL once deploy.sh finishes on the store VM." + value = format("http://%s:3000/grafana/", + var.store_vm_ip_cidr != "" ? split("/", var.store_vm_ip_cidr)[0] : "" + ) +} + +output "next_steps" { + description = "What to do after apply." + value = <<-EOT + Cloud-init on both VMs runs for ~5-10 minutes after apply completes. + Poll the store VM until it's up, then run the assertions: + + ../scripts/run-test.sh + + To iterate on the terraform without rebuilding the VMs: + + terraform apply -target=proxmox_virtual_environment_file.store_cloud_init + # then reset the VM: qm shutdown && qm start + # (cloud-init re-runs the snippet at next boot only if you touch it) + + To tear down: + + ../scripts/teardown.sh + EOT +} diff --git a/terraform/store_vm.tf b/terraform/store_vm.tf new file mode 100644 index 0000000..8cd8f0f --- /dev/null +++ b/terraform/store_vm.tf @@ -0,0 +1,88 @@ +# VM A -- the OBMP store. Cloud-init clones obmp-docker, writes a portable +# .env, and runs deploy.sh --scope full-stack --auth local --yes. +# +# Store this cloud-init YAML as a PVE snippet so the provider can reference +# it as user_data_file_id. The YAML is a rendered template so we can inject +# the ubuntu username, SSH key, repo URL/ref, HOST_IP, and router-facing +# port at plan time. + +locals { + store_hostname = "obmp-store-test" + + # Extract the plain IPv4 from the CIDR for HOST_IP / ROUTER_FACING_IP. + # If DHCP is in use, cloud-init discovers it at runtime; we pass an empty + # marker and the template falls back to `hostname -I`. + store_ip_addr = var.store_vm_ip_cidr != "" ? split("/", var.store_vm_ip_cidr)[0] : "" + + store_user_data = templatefile("${path.module}/cloud-init/store.cloud-init.yaml.tftpl", { + hostname = local.store_hostname + ssh_username = var.ssh_username + ssh_public_key = trimspace(file(var.ssh_public_key_path)) + obmp_repo_url = var.obmp_repo_url + obmp_repo_ref = var.obmp_repo_ref + router_facing_port = var.router_facing_port + static_ip_addr = local.store_ip_addr + }) +} + +resource "proxmox_virtual_environment_file" "store_cloud_init" { + content_type = "snippets" + datastore_id = var.snippet_storage + node_name = var.pve_node + + source_raw { + file_name = "obmp-store-user-data.yaml" + data = local.store_user_data + } +} + +resource "proxmox_virtual_environment_vm" "store" { + name = local.store_hostname + description = "OpenBMP portability-test STORE VM (managed by terraform)" + tags = ["obmp", "portability-test", "store"] + node_name = var.pve_node + vm_id = var.store_vm_id + + agent { enabled = true } + cpu { + cores = var.vm_cpu_cores + type = "host" + } + memory { + dedicated = var.vm_memory_mb + } + + disk { + datastore_id = var.vm_storage_pool + file_id = proxmox_virtual_environment_download_file.ubuntu_noble.id + interface = "scsi0" + size = var.vm_disk_gb + ssd = true + discard = "on" + } + + network_device { + bridge = var.vm_bridge + model = "virtio" + vlan_id = var.vm_vlan_tag != 0 ? var.vm_vlan_tag : null + } + + operating_system { type = "l26" } + + initialization { + datastore_id = var.vm_storage_pool + + ip_config { + ipv4 { + address = var.store_vm_ip_cidr != "" ? var.store_vm_ip_cidr : "dhcp" + gateway = var.store_vm_ip_cidr != "" ? var.vm_gateway : null + } + } + + dns { + servers = var.vm_dns_servers + } + + user_data_file_id = proxmox_virtual_environment_file.store_cloud_init.id + } +} diff --git a/terraform/terraform.tfvars.example b/terraform/terraform.tfvars.example new file mode 100644 index 0000000..a63eff8 --- /dev/null +++ b/terraform/terraform.tfvars.example @@ -0,0 +1,53 @@ +############################################################################ +# Copy this file to terraform.tfvars and fill in the marked values. +# terraform.tfvars is gitignored. +# +# You can also set the two most sensitive values via environment variables +# (recommended for shared workstations): +# export TF_VAR_pve_endpoint="https://pve.lab.example:8006/" +# export TF_VAR_pve_api_token="USER@REALM!TOKENID=UUID" +# When set in env, leave the tfvars values blank or delete those lines. +############################################################################ + +# --- Proxmox connection -------------------------------------------------- +pve_endpoint = "https://:8006/" +pve_api_token = "@!=" +pve_insecure = true # lab PVE with self-signed cert +pve_ssh_username = "root" +pve_node = "" # as shown in the PVE web UI cluster tree + +# --- Storage ------------------------------------------------------------- +vm_storage_pool = "" # holds VM disks +image_storage = "local" # holds the Ubuntu cloud image +snippet_storage = "local" # holds cloud-init snippet YAML + +# --- Network ------------------------------------------------------------- +vm_bridge = "vmbr0" +vm_vlan_tag = 0 # 0 = no VLAN + +# Static IPs recommended for the test -- run-test.sh needs to SSH to a +# known address, and the collector needs a stable ROUTER_FACING_IP. Set +# both to empty strings ("") to fall back to DHCP. +store_vm_ip_cidr = "10.40.40.220/24" +bmpgen_vm_ip_cidr = "10.40.40.221/24" +vm_gateway = "10.40.40.1" +vm_dns_servers = ["1.1.1.1", "8.8.8.8"] + +# --- SSH access ---------------------------------------------------------- +ssh_public_key_path = "~/.ssh/id_rsa.pub" +ssh_username = "ubuntu" + +# --- obmp-docker under test --------------------------------------------- +# Change obmp_repo_ref to whichever branch/tag/commit you want to test. +# Pin to a specific commit for reproducible test runs. +obmp_repo_url = "http://10.40.40.143:3000/sam/obmp-docker.git" +obmp_repo_ref = "deploy-refinement" +router_facing_port = 1790 + +# --- Optional overrides ------------------------------------------------- +# vm_cpu_cores = 2 +# vm_memory_mb = 4096 +# vm_disk_gb = 32 +# store_vm_id = 9001 +# bmpgen_vm_id = 9002 +# test_local_asn = 65099 diff --git a/terraform/variables.tf b/terraform/variables.tf new file mode 100644 index 0000000..c19a460 --- /dev/null +++ b/terraform/variables.tf @@ -0,0 +1,182 @@ +############################################################################ +# Proxmox connection +############################################################################ + +variable "pve_endpoint" { + description = "Proxmox VE API endpoint, e.g. https://pve.lab.example:8006/" + type = string +} + +variable "pve_api_token" { + description = "PVE API token in the form USER@REALM!TOKENID=UUID" + type = string + sensitive = true +} + +variable "pve_insecure" { + description = "Skip TLS verification for the PVE API (lab / self-signed only)" + type = bool + default = false +} + +variable "pve_ssh_username" { + description = "Username the provider uses for PVE SSH-side operations (snippet uploads)" + type = string + default = "root" +} + +variable "pve_node" { + description = "PVE node name to schedule VMs on (as shown in the PVE cluster tree)" + type = string +} + +############################################################################ +# VM shape +############################################################################ + +variable "store_vm_id" { + description = "PVE VMID for the OBMP store VM" + type = number + default = 9001 +} + +variable "bmpgen_vm_id" { + description = "PVE VMID for the GoBGP BMP-source VM" + type = number + default = 9002 +} + +variable "vm_cpu_cores" { + description = "vCPU count per test VM. Store is IO-bound at ingest, keep to 2-4." + type = number + default = 2 +} + +variable "vm_memory_mb" { + description = "Memory per VM in MB. 4096 covers the compose stack for a light BMP test." + type = number + default = 4096 +} + +variable "vm_disk_gb" { + description = "Root disk size per VM in GB. Store needs headroom for postgres/kafka." + type = number + default = 32 +} + +############################################################################ +# Storage + image +############################################################################ + +variable "vm_storage_pool" { + description = "PVE storage pool for VM disks, e.g. local-lvm, local-zfs" + type = string +} + +variable "image_storage" { + description = "PVE storage that holds ISOs / cloud images (typically local)" + type = string + default = "local" +} + +variable "snippet_storage" { + description = "PVE storage that supports snippets content type (typically local)" + type = string + default = "local" +} + +variable "ubuntu_image_url" { + description = "Ubuntu cloud image URL. Pinned to noble by default; override for a snapshot." + type = string + default = "https://cloud-images.ubuntu.com/noble/current/noble-server-cloudimg-amd64.img" +} + +variable "ubuntu_image_checksum" { + description = "SHA256 of the Ubuntu image. Set to enforce; empty to skip check." + type = string + default = "" +} + +############################################################################ +# Network +############################################################################ + +variable "vm_bridge" { + description = "PVE Linux bridge the VMs attach to, e.g. vmbr0" + type = string + default = "vmbr0" +} + +variable "vm_vlan_tag" { + description = "802.1q VLAN tag; 0 = no tag" + type = number + default = 0 +} + +variable "store_vm_ip_cidr" { + description = "Static IPv4/CIDR for the store VM, e.g. 10.40.40.220/24. Leave empty for DHCP." + type = string + default = "" +} + +variable "bmpgen_vm_ip_cidr" { + description = "Static IPv4/CIDR for the bmpgen VM, e.g. 10.40.40.221/24. Leave empty for DHCP." + type = string + default = "" +} + +variable "vm_gateway" { + description = "Default gateway for both VMs. Ignored when using DHCP." + type = string + default = "" +} + +variable "vm_dns_servers" { + description = "DNS servers for both VMs. Ignored when using DHCP." + type = list(string) + default = ["1.1.1.1", "8.8.8.8"] +} + +############################################################################ +# SSH access +############################################################################ + +variable "ssh_public_key_path" { + description = "Path to the public key that will be authorized on both VMs' ubuntu user." + type = string + default = "~/.ssh/id_rsa.pub" +} + +variable "ssh_username" { + description = "Cloud-init user created on both VMs (has passwordless sudo, ssh key auth)." + type = string + default = "ubuntu" +} + +############################################################################ +# obmp-docker under test +############################################################################ + +variable "obmp_repo_url" { + description = "obmp-docker repo URL to clone on the store VM." + type = string + default = "http://10.40.40.143:3000/sam/obmp-docker.git" +} + +variable "obmp_repo_ref" { + description = "Git ref (branch, tag, or commit) to check out on the store VM. Pin this for reproducibility." + type = string + default = "deploy-refinement" +} + +variable "router_facing_port" { + description = "BMP port routers connect to. Defaults to 1790 (IANA BMP port); mirrors obmp-docker .env." + type = number + default = 1790 +} + +variable "test_local_asn" { + description = "AS number the GoBGP-side stand-in router uses. Private (64512-65534)." + type = number + default = 65099 +}