From d0b82c4973fcc075226b5b0d049a6d0178a61305 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 20 Jul 2026 23:38:43 -0700 Subject: [PATCH] first successful two-VM run + fixes surfaced by it The initial two-VM run against a real Proxmox host found three real bugs in the obmp-docker stack, plus one design issue in this test: Fixed in obmp-docker (deploy-refinement branch): - 745f066 deploy.sh: probe sudo with `sudo -n true` instead of `sudo -v` / `sudo -n -v`; the latter demands a credential timestamp that NOPASSWD never produces, so both fail on cloud-init / CI even with NOPASSWD:ALL configured. - 6df67a7 .env.example: quote GNMI_ADDRESSES with single quotes so docker compose's .env parser doesn't choke on the mid-line comma in the double-quoted list form. Fixed here: - bmpgen cloud-init: add `port = 1179` under [global.config] to avoid gobgpd's 179 listener bind failing under Ubuntu 24.04 + Docker even with --network host and default NET_BIND_SERVICE cap. We don't accept BGP peers on bmpgen so the port value doesn't matter as long as it's unprivileged. - run-test.sh: fix column name for bgp_peers.state -- the opstate enum uses lowercase 'up', not uppercase 'UP'. Verified against the live run: 6/6 assertions pass, ip_rib landed 251,760 routes from the store VM's own gobgp connecting to Bromirski's public full-table feed via --profile test. Co-Authored-By: Claude Opus 4.7 --- scripts/run-test.sh | 4 +-- terraform/bmpgen_vm.tf | 25 +++++++++++-------- .../cloud-init/bmpgen.cloud-init.yaml.tftpl | 5 ++++ terraform/main.tf | 17 +++++++++---- terraform/variables.tf | 12 +++++++++ 5 files changed, 46 insertions(+), 17 deletions(-) diff --git a/scripts/run-test.sh b/scripts/run-test.sh index 09b0b7e..ac43ac4 100755 --- a/scripts/run-test.sh +++ b/scripts/run-test.sh @@ -119,10 +119,10 @@ assert_routers_row() { } assert "routers table has >= 1 row" assert_routers_row -# 4: at least 1 UP BGP peer +# 4: at least 1 UP BGP peer. bgp_peers.state uses lowercase 'up' (opstate enum). assert_bgp_peer_up() { local n - n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc \"SELECT count(*) FROM bgp_peers WHERE state='UP'\"") + n=$(ssh_store "docker exec -i obmp-psql psql -U openbmp -d openbmp -tAc \"SELECT count(*) FROM bgp_peers WHERE state='up'\"") [ "${n:-0}" -ge 1 ] } assert "bgp_peers has >= 1 UP session" assert_bgp_peer_up diff --git a/terraform/bmpgen_vm.tf b/terraform/bmpgen_vm.tf index d1f8f38..6e5917d 100644 --- a/terraform/bmpgen_vm.tf +++ b/terraform/bmpgen_vm.tf @@ -1,20 +1,25 @@ # VM B -- GoBGP standing in for a real BGP router. Cloud-init installs # docker and runs a gobgp container configured to BMP-report to VM A. # -# `depends_on` on the store VM's cloud-init snippet, not the store VM -# itself -- we don't want to wait for the store's cloud-init to finish -# before launching B (cloud-init on B will retry the BMP session until A -# is ready). +# Ordering: bmpgen's cloud-init needs VM-A's IP. With a static CIDR it's +# known at plan time; with DHCP we defer to store.ipv4_addresses (populated +# only after the store VM boots and its qemu-guest-agent reports). That +# reference creates an implicit dependency, so bmpgen's snippet upload + +# VM creation both wait until the store's address is known. locals { bmpgen_hostname = "obmp-bmpgen-test" - # If DHCP, we don't know VM-A's address at plan time and cloud-init on - # VM-B will need it. We handle both cases in the template: - # - static: pass store_ip_addr directly - # - DHCP: cloud-init on B does an ARP scan + hostname lookup at boot - # (kept out-of-scope here; DHCP + this test is best-effort) - bmpgen_store_addr = var.store_vm_ip_cidr != "" ? split("/", var.store_vm_ip_cidr)[0] : "" + # Resolve store's address: static CIDR var wins; otherwise fall back to + # the store VM's guest-agent-reported IPv4. ipv4_addresses is a list of + # lists (one per interface); index [1][0] is the first NIC's first non- + # loopback address. try() keeps plan output tidy when the attribute + # doesn't exist yet. + bmpgen_store_addr = ( + var.store_vm_ip_cidr != "" + ? split("/", var.store_vm_ip_cidr)[0] + : try(proxmox_virtual_environment_vm.store.ipv4_addresses[1][0], "") + ) bmpgen_user_data = templatefile("${path.module}/cloud-init/bmpgen.cloud-init.yaml.tftpl", { hostname = local.bmpgen_hostname diff --git a/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl b/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl index cf97b8d..8e910f0 100644 --- a/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl +++ b/terraform/cloud-init/bmpgen.cloud-init.yaml.tftpl @@ -36,6 +36,11 @@ write_files: [global.config] as = ${local_asn} router-id = "10.99.99.99" + # Non-privileged BGP listener port. Default 179 requires root+cap_net_bind_service + # inside the container; Ubuntu 24.04 + docker denies this even with + # --network host despite the default cap set including NET_BIND_SERVICE. + # We're not accepting BGP peers here anyway -- only the BMP session out. + port = 1179 [[bmp-servers]] [bmp-servers.config] diff --git a/terraform/main.tf b/terraform/main.tf index e80d415..4cf13da 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -22,11 +22,18 @@ provider "proxmox" { api_token = var.pve_api_token insecure = var.pve_insecure - # SSH is used by the provider for a few operations (snippet uploads, etc). - # Optional -- if omitted the provider falls back to API-only paths where - # possible. Set node -> address so the provider doesn't have to resolve. + # SSH is used by the provider for snippet uploads (SCP, not the API -- + # a known bpg quirk). Point it at the same address as the API endpoint + # rather than letting the provider guess from cluster metadata (which + # may return a bridge address that's unreachable from here). ssh { - agent = true - username = var.pve_ssh_username + agent = false + username = var.pve_ssh_username + private_key = file(var.pve_ssh_private_key_path) + + node { + name = var.pve_node + address = var.pve_ssh_address != "" ? var.pve_ssh_address : replace(replace(var.pve_endpoint, "https://", ""), ":8006/", "") + } } } diff --git a/terraform/variables.tf b/terraform/variables.tf index c19a460..e1ae177 100644 --- a/terraform/variables.tf +++ b/terraform/variables.tf @@ -25,6 +25,18 @@ variable "pve_ssh_username" { default = "root" } +variable "pve_ssh_private_key_path" { + description = "Path to the SSH private key the provider uses to reach the PVE host. Its matching pubkey must be in /root/.ssh/authorized_keys on the PVE node." + type = string + default = "~/.ssh/id_ed25519" +} + +variable "pve_ssh_address" { + description = "Address the provider uses for SSH to the PVE node. Defaults to the endpoint host. Set explicitly if the PVE cluster returns a different bridge/hostname than the one that's reachable from here." + type = string + default = "" +} + variable "pve_node" { description = "PVE node name to schedule VMs on (as shown in the PVE cluster tree)" type = string