From cf4e5b07c631f6a45fbac5a2835f0e0ad0978066 Mon Sep 17 00:00:00 2001 From: sam Date: Mon, 18 May 2026 19:21:04 -0700 Subject: [PATCH] Add Compose profiles, setup.sh bootstrap, and config templates for portable deployment Pins the Compose project name and splits services into core / test / auth profiles so the BMP collector core can deploy standalone. Adds setup.sh (idempotent bootstrap), .env.example, and repo-resident Authelia config templates so a fresh host deploys without manual steps. Parameterizes hardcoded host IP and domain; points the Grafana InfluxDB datasource at the container name. Co-Authored-By: Claude Opus 4.7 (1M context) --- .env.example | 56 ++++++++ DOCS.md | 47 +++++++ README.md | 9 ++ authelia/configuration.yml.template | 51 +++++++ authelia/users_database.yml.template | 15 +++ docker-compose.yml | 32 +++-- .../provisioning/datasources/influxdb-ds.yml | 2 +- setup.sh | 127 ++++++++++++++++++ 8 files changed, 326 insertions(+), 13 deletions(-) create mode 100644 .env.example create mode 100644 authelia/configuration.yml.template create mode 100644 authelia/users_database.yml.template create mode 100755 setup.sh diff --git a/.env.example b/.env.example new file mode 100644 index 0000000..ce01e88 --- /dev/null +++ b/.env.example @@ -0,0 +1,56 @@ +# OpenBMP stack configuration — copy to .env and fill in. +# cp .env.example .env && $EDITOR .env && ./setup.sh +# The real .env is git-ignored and never committed. + +# --------------------------------------------------------------------------- +# Core deployment +# --------------------------------------------------------------------------- +# Host path for all persistent data (postgres, kafka, grafana, authelia, ...). +OBMP_DATA_ROOT=/var/openbmp + +# IP of this host that routers and external clients connect to +# (Kafka external listener, BMP source, ExaBGP peering). +HOST_IP=changeme + +# Public domain fronting Grafana / Authelia / portal (TLS terminates upstream). +OBMP_DOMAIN=changeme.example.com + +# Authelia session-cookie domain — the parent domain of OBMP_DOMAIN so the +# cookie is valid across subpaths/subdomains. +OBMP_COOKIE_DOMAIN=example.com + +# --------------------------------------------------------------------------- +# ExaBGP route injector (test profile) +# --------------------------------------------------------------------------- +EXABGP_LOCAL_IP=changeme +EXABGP_LOCAL_AS=65100 +EXABGP_API_PORT=5050 +# Semicolon-separated peer list, each entry "ip:peer_as:description". +EXABGP_PEERS=10.100.0.100:65020:CML-R9K-CORE-01;10.100.0.200:65020:CML-R9K-CORE-02 + +# --------------------------------------------------------------------------- +# CML lab API + IOS-XR NETCONF (used by cml/ automation scripts) +# --------------------------------------------------------------------------- +PROX-CML_URL=http://changeme +PROX-CML_USERNAME=changeme +PROX-CML_PASSWORD=changeme + +# Default IOS-XR NETCONF credentials, plus the admin-tier override for routers +# that use a separate account. +IOSXR_NETCONF_USER=changeme +IOSXR_NETCONF_PASS=changeme +IOSXR_NETCONF_ADMIN_USER=changeme +IOSXR_NETCONF_ADMIN_PASS=changeme + +# --------------------------------------------------------------------------- +# Integrations +# --------------------------------------------------------------------------- +GITEA_API_KEY=changeme + +# --------------------------------------------------------------------------- +# Authelia secrets — leave BLANK; setup.sh generates them with openssl on a +# fresh host and appends them here. Existing values are never overwritten. +# --------------------------------------------------------------------------- +AUTHELIA_SESSION_SECRET= +AUTHELIA_JWT_SECRET= +AUTHELIA_STORAGE_ENCRYPTION_KEY= diff --git a/DOCS.md b/DOCS.md index a76eaba..3e73d40 100644 --- a/DOCS.md +++ b/DOCS.md @@ -127,6 +127,37 @@ Traffic Generator (Phase 4): ## 4. Initial Setup (First Time) +### 4.0 Quick deploy (recommended) + +`setup.sh` bootstraps a fresh host — it creates the data directories, syncs +Grafana provisioning, generates Authelia secrets, and renders config. It is +idempotent and safe to re-run. + +```bash +git clone +cd obmp-docker +cp .env.example .env +$EDITOR .env # set HOST_IP, OBMP_DOMAIN, OBMP_COOKIE_DOMAIN, credentials +./setup.sh +docker compose up -d # BMP collector core only +docker compose --profile test --profile auth up -d # full stack (lab tools + auth) +``` + +The stack uses Docker Compose **profiles**: + +| Command | Brings up | +|---------|-----------| +| `docker compose up -d` | Collector core only — zookeeper, kafka, collector, psql, psql-app, grafana, whois | +| `docker compose --profile test up -d` | Core **+** ExaBGP, traffic generator, telegraf, influxdb | +| `docker compose --profile auth up -d` | Core **+** Authelia gateway and portal | +| `docker compose --profile test --profile auth up -d` | Everything | + +The bare `docker compose up` is the shippable standalone BMP collector — it has +no dependency on the lab/test tooling. + +The sections below (4.1–4.6) document the equivalent **manual** steps if you +prefer not to use `setup.sh`. + ### 4.1 Clone the repository ```bash @@ -248,6 +279,22 @@ See `exabgp/iosxr_bgp_config.md` for a Python/ncclient script that pushes all of Credentials: `username=webui`, `password=cisco`, port 830. +### 5.6 Bulk BMP config (`cml/proxmox_bmp_config.py`) + +To point a whole lab of IOS-XR routers at the BMP collector at once, +`cml/proxmox_bmp_config.py` applies the `bmp server 1` block over SSH (IOS-XR +BMP config is not exposed via NETCONF YANG on current releases). It is +idempotent. + +```bash +pip install paramiko +python3 cml/proxmox_bmp_config.py # all routers in the inventory +python3 cml/proxmox_bmp_config.py r9k-05 # a single router (smoke test) +``` + +Edit the `ROUTERS` list at the top of the script for your inventory and the +`COLLECTOR_HOST` constant for the collector address. + --- ## 6. Starting and Stopping diff --git a/README.md b/README.md index 6e10bec..6889f31 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,15 @@ Each docker file contains a readme file, see below: ## Using Docker Compose to run everything +> **Quick start (recommended):** copy `.env.example` to `.env`, fill it in, and +> run `./setup.sh` — it creates the data directories, syncs Grafana +> provisioning, and generates Authelia secrets. Then: +> ``` +> docker compose up -d # BMP collector core +> docker compose --profile test --profile auth up -d # full stack +> ``` +> See [DOCS.md](DOCS.md) section 4 for details and the manual alternative below. + ### Install Docker Compose You will need docker-compose. You can install that via [Docker Compose](https://docs.docker.com/compose/install/) instructions. Docker compose will run everything, including handling restarts of containers. diff --git a/authelia/configuration.yml.template b/authelia/configuration.yml.template new file mode 100644 index 0000000..8af2268 --- /dev/null +++ b/authelia/configuration.yml.template @@ -0,0 +1,51 @@ +--- +# Authelia configuration template. +# setup.sh renders this to ${OBMP_DATA_ROOT}/authelia/configuration.yml, +# substituting the ${...} values from .env. Only rendered if the target +# file does not already exist — an existing deployment is never overwritten. +theme: dark + +server: + address: 'tcp://0.0.0.0:9091/authelia' + endpoints: + authz: + forward-auth: + implementation: ForwardAuth + +log: + level: info + +totp: + issuer: openbmp + +authentication_backend: + file: + path: /config/users_database.yml + password: + algorithm: bcrypt + iterations: 12 + +session: + name: authelia_session + secret: ${AUTHELIA_SESSION_SECRET} + expiration: 12h + inactivity: 6h + cookies: + - domain: ${OBMP_COOKIE_DOMAIN} + authelia_url: https://${OBMP_DOMAIN}/authelia + +identity_validation: + reset_password: + jwt_secret: ${AUTHELIA_JWT_SECRET} + +storage: + local: + path: /config/db.sqlite3 + encryption_key: ${AUTHELIA_STORAGE_ENCRYPTION_KEY} + +access_control: + default_policy: one_factor + +notifier: + filesystem: + filename: /config/notification.txt diff --git a/authelia/users_database.yml.template b/authelia/users_database.yml.template new file mode 100644 index 0000000..cedb65d --- /dev/null +++ b/authelia/users_database.yml.template @@ -0,0 +1,15 @@ +--- +# Authelia user database template. +# setup.sh copies this to ${OBMP_DATA_ROOT}/authelia/users_database.yml only +# if that file does not already exist. The bcrypt hash below is the default +# demo account (username: openbmp). Change it after first login, or generate +# a new hash with: +# docker run --rm authelia/authelia:4.38 \ +# authelia crypto hash generate bcrypt --password '' +users: + openbmp: + displayname: "OpenBMP Demo" + password: "$2b$12$KQiQo1bYWqadD51HlgfgO.M1JfVlA5qP2YVRoBMTPmWq6osPljUTW" + email: demo@apodacalab.com + groups: + - admins diff --git a/docker-compose.yml b/docker-compose.yml index 9cdf826..b9d9c17 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,5 +1,5 @@ --- -version: '3' +name: obmp volumes: data-volume: driver_opts: @@ -45,7 +45,7 @@ services: # Change/add listeners based on your FQDN that the host and other containers can access. You can use # an IP address as well. By default, only within the compose/containers can Kafka be accesssed # using port 29092. Outside access can be enabled, but you should use an FQDN listener. - KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://obmp-kafka:29092,PLAINTEXT_HOST://10.40.40.202:9092 + KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://obmp-kafka:29092,PLAINTEXT_HOST://${HOST_IP:-10.40.40.202}:9092 KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1 @@ -93,7 +93,7 @@ services: environment: - GF_SECURITY_ADMIN_PASSWORD=openbmp - GF_AUTH_ANONYMOUS_ENABLED=false - - GF_SERVER_ROOT_URL=https://bmp.apodacalab.com/grafana/ + - GF_SERVER_ROOT_URL=https://${OBMP_DOMAIN:-bmp.apodacalab.com}/grafana/ - GF_SERVER_SERVE_FROM_SUB_PATH=true - GF_AUTH_PROXY_ENABLED=true - GF_AUTH_PROXY_HEADER_NAME=Remote-User @@ -206,22 +206,22 @@ services: exabgp: restart: unless-stopped container_name: obmp-exabgp + profiles: ["test"] build: context: ./exabgp dockerfile: Dockerfile # Host networking so ExaBGP can reach CML routers directly on port 179 network_mode: host environment: - # IP on the host that CML routers can reach (matches Kafka external listener) - - EXABGP_LOCAL_IP=10.40.40.202 - # ExaBGP presents as AS 65100 (eBGP peer to your AS 65020 lab) - - EXABGP_LOCAL_AS=65100 - - EXABGP_PEER_AS=65020 - # CORE routers to peer with — these propagate routes into the iBGP mesh - - EXABGP_PEER_1=10.100.0.100 - - EXABGP_PEER_2=10.100.0.200 + # IP on the host that CML routers reach (BGP peering source) + - EXABGP_LOCAL_IP=${HOST_IP:-10.40.40.202} + # ExaBGP presents as AS 65100 (eBGP peer to the lab route reflectors) + - EXABGP_LOCAL_AS=${EXABGP_LOCAL_AS:-65100} + # Peer list — ";"-separated entries of "ip:peer_as:description". + # Default covers both labs: AS 65020 (ESXi) and AS 65021 (Proxmox). + - EXABGP_PEERS=${EXABGP_PEERS:-10.100.0.100:65020:CML-R9K-CORE-01;10.100.0.200:65020:CML-R9K-CORE-02;10.100.1.100:65021:PROX-R9K-CORE-01;10.100.1.200:65021:PROX-R9K-CORE-02} # Flask API port (also on host network) - - EXABGP_API_PORT=5050 + - EXABGP_API_PORT=${EXABGP_API_PORT:-5050} volumes: # Mount scenarios dir so you can edit/add scenarios without rebuilding - ./exabgp/scenarios:/exabgp/scenarios @@ -230,6 +230,7 @@ services: exabgp-ui: restart: unless-stopped container_name: obmp-exabgp-ui + profiles: ["test"] build: context: ./exabgp-ui dockerfile: Dockerfile @@ -242,6 +243,7 @@ services: influxdb: restart: unless-stopped container_name: obmp-influxdb + profiles: ["test"] image: influxdb:2.7 ports: - "8086:8086" @@ -259,6 +261,7 @@ services: telegraf: restart: unless-stopped container_name: obmp-telegraf + profiles: ["test"] build: context: ./telegraf dockerfile: Dockerfile @@ -273,6 +276,7 @@ services: traffic-gen: restart: unless-stopped container_name: obmp-traffic-gen + profiles: ["test"] build: context: ./traffic-gen dockerfile: Dockerfile @@ -288,6 +292,7 @@ services: traffic-gen-ui: restart: unless-stopped container_name: obmp-traffic-gen-ui + profiles: ["test"] build: context: ./traffic-gen-ui dockerfile: Dockerfile @@ -297,6 +302,7 @@ services: traffic-gen-responder: restart: unless-stopped container_name: obmp-traffic-gen-responder + profiles: ["test"] build: context: ./traffic-gen dockerfile: Dockerfile @@ -336,6 +342,7 @@ services: authelia: restart: unless-stopped container_name: obmp-authelia + profiles: ["auth"] image: authelia/authelia:4.38 ports: - "9091:9091" @@ -347,6 +354,7 @@ services: portal: restart: unless-stopped container_name: obmp-portal + profiles: ["auth"] image: nginx:alpine ports: - "8080:80" diff --git a/obmp-grafana/provisioning/datasources/influxdb-ds.yml b/obmp-grafana/provisioning/datasources/influxdb-ds.yml index dbbac77..f09088a 100644 --- a/obmp-grafana/provisioning/datasources/influxdb-ds.yml +++ b/obmp-grafana/provisioning/datasources/influxdb-ds.yml @@ -5,7 +5,7 @@ datasources: uid: obmp_influxdb type: influxdb access: proxy - url: http://10.40.40.202:8086 + url: http://obmp-influxdb:8086 jsonData: version: Flux organization: openbmp diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..796eaf4 --- /dev/null +++ b/setup.sh @@ -0,0 +1,127 @@ +#!/usr/bin/env bash +# +# OpenBMP stack bootstrap — idempotent. Safe to run against an existing +# deployment: every step is guarded and never overwrites live config. +# +# cp .env.example .env # first run only +# $EDITOR .env # fill in HOST_IP, OBMP_DOMAIN, ... +# ./setup.sh +# docker compose up -d # collector core +# docker compose --profile test --profile auth up -d # full stack +# +set -euo pipefail +cd "$(dirname "$0")" + +AUTHELIA_IMAGE="authelia/authelia:4.38" + +# --- .env ------------------------------------------------------------------- +if [ ! -f .env ]; then + cp .env.example .env + echo "Created .env from .env.example." + echo "Edit it (HOST_IP, OBMP_DOMAIN, OBMP_COOKIE_DOMAIN, credentials), then re-run ./setup.sh" + exit 1 +fi + +# Read a single KEY=value from .env without sourcing it — .env contains keys +# with hyphens (PROX-CML_*) that a shell `source` would choke on. +get_env() { grep -E "^$1=" .env | head -1 | cut -d= -f2- || true; } + +# Set KEY=value in .env: replace the line if present, else append. +set_env() { + local key="$1" val="$2" + if grep -qE "^${key}=" .env; then + # `|` delimiter — values are hex, no `|`. + sed -i "s|^${key}=.*|${key}=${val}|" .env + else + printf '%s=%s\n' "$key" "$val" >> .env + fi +} + +OBMP_DATA_ROOT="$(get_env OBMP_DATA_ROOT)" +OBMP_DATA_ROOT="${OBMP_DATA_ROOT:-/var/openbmp}" +OBMP_DOMAIN="$(get_env OBMP_DOMAIN)" +OBMP_COOKIE_DOMAIN="$(get_env OBMP_COOKIE_DOMAIN)" +HOST_IP="$(get_env HOST_IP)" + +# --- validate --------------------------------------------------------------- +fail=0 +for var in HOST_IP OBMP_DOMAIN OBMP_COOKIE_DOMAIN; do + val="$(get_env "$var")" + if [ -z "$val" ] || [[ "$val" == changeme* ]]; then + echo "ERROR: $var is unset or still 'changeme' in .env" >&2 + fail=1 + fi +done +[ "$fail" -eq 0 ] || { echo "Fix .env and re-run." >&2; exit 1; } + +# --- privilege helper ------------------------------------------------------- +# $OBMP_DATA_ROOT is often root-owned (e.g. /var/openbmp). Use sudo only if the +# current user cannot write the parent directory. +parent="$(dirname "$OBMP_DATA_ROOT")" +SUDO="" +if [ ! -w "$parent" ] || { [ -d "$OBMP_DATA_ROOT" ] && [ ! -w "$OBMP_DATA_ROOT" ]; }; then + SUDO="sudo" + echo "Note: using sudo for filesystem setup under $OBMP_DATA_ROOT" +fi + +# --- data-root directory tree ----------------------------------------------- +echo "Creating data tree under $OBMP_DATA_ROOT ..." +for d in config grafana grafana/provisioning kafka-data zk-data zk-log \ + postgres/data postgres/ts influxdb authelia; do + $SUDO mkdir -p "$OBMP_DATA_ROOT/$d" +done +# Container processes run as assorted UIDs; lab-permissive perms. +$SUDO chmod -R 777 "$OBMP_DATA_ROOT" 2>/dev/null || true + +# --- Grafana provisioning --------------------------------------------------- +echo "Syncing Grafana provisioning ..." +$SUDO cp -r obmp-grafana/provisioning/. "$OBMP_DATA_ROOT/grafana/provisioning/" + +# --- Authelia secrets ------------------------------------------------------- +# Generate only if blank/absent; never overwrite an existing value. +for key in AUTHELIA_SESSION_SECRET AUTHELIA_JWT_SECRET AUTHELIA_STORAGE_ENCRYPTION_KEY; do + cur="$(get_env "$key")" + if [ -z "$cur" ]; then + set_env "$key" "$(openssl rand -hex 32)" + echo "Generated $key" + fi +done +AUTHELIA_SESSION_SECRET="$(get_env AUTHELIA_SESSION_SECRET)" +AUTHELIA_JWT_SECRET="$(get_env AUTHELIA_JWT_SECRET)" +AUTHELIA_STORAGE_ENCRYPTION_KEY="$(get_env AUTHELIA_STORAGE_ENCRYPTION_KEY)" + +# --- Authelia config (fresh-deploy only — never clobber a live config) ------ +export AUTHELIA_SESSION_SECRET AUTHELIA_JWT_SECRET AUTHELIA_STORAGE_ENCRYPTION_KEY \ + OBMP_DOMAIN OBMP_COOKIE_DOMAIN +SUBST='${AUTHELIA_SESSION_SECRET} ${AUTHELIA_JWT_SECRET} ${AUTHELIA_STORAGE_ENCRYPTION_KEY} ${OBMP_DOMAIN} ${OBMP_COOKIE_DOMAIN}' + +if [ ! -f "$OBMP_DATA_ROOT/authelia/configuration.yml" ]; then + envsubst "$SUBST" < authelia/configuration.yml.template \ + | $SUDO tee "$OBMP_DATA_ROOT/authelia/configuration.yml" > /dev/null + echo "Rendered authelia/configuration.yml" +else + echo "authelia/configuration.yml exists — left untouched" +fi + +if [ ! -f "$OBMP_DATA_ROOT/authelia/users_database.yml" ]; then + $SUDO cp authelia/users_database.yml.template \ + "$OBMP_DATA_ROOT/authelia/users_database.yml" + echo "Rendered authelia/users_database.yml (demo user: openbmp)" +else + echo "authelia/users_database.yml exists — left untouched" +fi + +# --- images ----------------------------------------------------------------- +echo "Pulling and building images ..." +docker compose pull --quiet +docker compose --profile test --profile auth build + +# --- done ------------------------------------------------------------------- +cat <