From c5a0245dd2acca226d1e866cf6da3e0da4363d3b Mon Sep 17 00:00:00 2001 From: sam Date: Fri, 27 Feb 2026 20:46:59 -0700 Subject: [PATCH] Add project infrastructure and configuration files Docker Compose stack, nginx config, OAuth2 client bootstrap, Hydra DB init, setup script, and gitignore for secrets. Co-Authored-By: Claude Opus 4.6 --- .claude/settings.json | 9 + .gitignore | 8 + docker-compose.yml | 173 +++++++++++++++ init-hydra-db.sh | 18 ++ nginx/nginx.conf | 51 +++++ oauth2/client/bootstrap-clients.sh | 40 ++++ setup.sh | 325 +++++++++++++++++++++++++++++ 7 files changed, 624 insertions(+) create mode 100644 .claude/settings.json create mode 100644 .gitignore create mode 100644 docker-compose.yml create mode 100755 init-hydra-db.sh create mode 100644 nginx/nginx.conf create mode 100755 oauth2/client/bootstrap-clients.sh create mode 100755 setup.sh diff --git a/.claude/settings.json b/.claude/settings.json new file mode 100644 index 0000000..74a641a --- /dev/null +++ b/.claude/settings.json @@ -0,0 +1,9 @@ +{ + "permissions": { + "allow": [ + "Bash(/home/user/netbox-diode-project/.venv/bin/python --version)", + "Bash(/home/user/netbox-diode-project/.venv/bin/pip --version)", + "Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)" + ] + } +} diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..5619072 --- /dev/null +++ b/.gitignore @@ -0,0 +1,8 @@ +.venv/ +__pycache__/ +*.pyc +.env + +# Generated by setup.sh (contain secrets) +oauth2/client/client-credentials.json +orb-agent/agent.yaml diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..13dd3a2 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,173 @@ +services: + # ========================================================================= + # Ingress — routes gRPC traffic to Diode services + # ========================================================================= + ingress-nginx: + image: nginx:1.27-alpine + ports: + - "8080:8080" + volumes: + - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro + depends_on: + diode-ingester: + condition: service_started + diode-reconciler: + condition: service_started + diode-auth: + condition: service_started + restart: unless-stopped + + # ========================================================================= + # Diode Ingester — accepts gRPC, publishes to Redis streams + # ========================================================================= + diode-ingester: + image: netboxlabs/diode-ingester:latest + environment: + DIODE_REDIS_HOST: redis + DIODE_REDIS_PORT: "6379" + DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}" + DIODE_GRPC_PORT: "8081" + DIODE_AUTH_GRPC_TARGET: diode-auth:8081 + depends_on: + redis: + condition: service_healthy + restart: unless-stopped + + # ========================================================================= + # Diode Reconciler — consumes Redis streams, reconciles with NetBox + # ========================================================================= + diode-reconciler: + image: netboxlabs/diode-reconciler:latest + environment: + DIODE_REDIS_HOST: redis + DIODE_REDIS_PORT: "6379" + DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}" + DIODE_GRPC_PORT: "8081" + DIODE_NETBOX_API_URL: "${NETBOX_API_URL}" + DIODE_NETBOX_API_TOKEN: "${NETBOX_API_TOKEN}" + DIODE_RECONCILER_CLIENT_ID: "${RECONCILER_CLIENT_ID}" + DIODE_RECONCILER_CLIENT_SECRET: "${RECONCILER_CLIENT_SECRET}" + DIODE_AUTH_GRPC_TARGET: diode-auth:8081 + DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}" + DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}" + depends_on: + redis: + condition: service_healthy + diode-auth-bootstrap: + condition: service_completed_successfully + restart: unless-stopped + + # ========================================================================= + # Diode Auth — validates OAuth2 tokens via Hydra + # ========================================================================= + diode-auth: + image: netboxlabs/diode-auth:latest + environment: + DIODE_GRPC_PORT: "8081" + DIODE_HYDRA_ADMIN_URL: http://hydra:4445 + DIODE_HYDRA_PUBLIC_URL: http://hydra:4444 + depends_on: + hydra: + condition: service_started + restart: unless-stopped + + # ========================================================================= + # Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot) + # ========================================================================= + diode-auth-bootstrap: + image: python:3.12-alpine + command: /client-credentials/bootstrap-clients.sh + environment: + HYDRA_ADMIN_URL: http://hydra:4445 + volumes: + - ./oauth2/client:/client-credentials:ro + depends_on: + hydra: + condition: service_started + restart: "no" + + # ========================================================================= + # Hydra — OAuth2 / OpenID Connect server + # ========================================================================= + hydra: + image: oryd/hydra:v2.2 + command: serve all --dev --config /etc/hydra/hydra.yml + environment: + DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable" + SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}" + URLS_SELF_ISSUER: http://hydra:4444/ + URLS_SELF_PUBLIC: http://hydra:4444/ + depends_on: + hydra-migrate: + condition: service_completed_successfully + restart: unless-stopped + + # ========================================================================= + # Hydra Migrate — runs Hydra database migrations (one-shot) + # ========================================================================= + hydra-migrate: + image: oryd/hydra:v2.2 + command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable" + depends_on: + postgres: + condition: service_healthy + restart: "no" + + # ========================================================================= + # Redis — message streams for Diode ingester ↔ reconciler + # ========================================================================= + redis: + image: redis/redis-stack-server:latest + command: redis-server --requirepass "${REDIS_PASSWORD}" --appendonly yes + ports: + - "6378:6379" + healthcheck: + test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"] + start_period: 5s + interval: 5s + timeout: 3s + retries: 5 + volumes: + - diode-redis-data:/data + restart: unless-stopped + + # ========================================================================= + # PostgreSQL — stores Diode + Hydra databases + # ========================================================================= + postgres: + image: postgres:16-alpine + environment: + POSTGRES_USER: "${DIODE_DB_USER}" + POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}" + POSTGRES_DB: "${DIODE_DB_NAME}" + healthcheck: + test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"] + start_period: 10s + interval: 10s + timeout: 5s + retries: 5 + volumes: + - diode-postgres-data:/var/lib/postgresql/data + - ./init-hydra-db.sh:/docker-entrypoint-initdb.d/init-hydra-db.sh:ro + restart: unless-stopped + + # ========================================================================= + # Orb Agent — network discovery (NMAP, SNMP, NAPALM) + # ========================================================================= + orb-agent: + image: netboxlabs/orb-agent:latest + network_mode: host + volumes: + - ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro + depends_on: + diode-ingester: + condition: service_started + diode-auth-bootstrap: + condition: service_completed_successfully + restart: unless-stopped + +volumes: + diode-redis-data: + driver: local + diode-postgres-data: + driver: local diff --git a/init-hydra-db.sh b/init-hydra-db.sh new file mode 100755 index 0000000..9adf6ef --- /dev/null +++ b/init-hydra-db.sh @@ -0,0 +1,18 @@ +#!/bin/bash +set -e + +# Create the Hydra database and user if they don't already exist. +# This runs as part of PostgreSQL's docker-entrypoint-initdb.d. + +psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL + DO \$\$ + BEGIN + IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${HYDRA_DB_USER}') THEN + CREATE ROLE ${HYDRA_DB_USER} WITH LOGIN PASSWORD '${HYDRA_DB_PASSWORD}'; + END IF; + END + \$\$; + + SELECT 'CREATE DATABASE ${HYDRA_DB_NAME} OWNER ${HYDRA_DB_USER}' + WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${HYDRA_DB_NAME}')\gexec +EOSQL diff --git a/nginx/nginx.conf b/nginx/nginx.conf new file mode 100644 index 0000000..c6b2dcc --- /dev/null +++ b/nginx/nginx.conf @@ -0,0 +1,51 @@ +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent"'; + + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; + + upstream ingester_grpc { + server diode-ingester:8081; + } + + upstream reconciler_grpc { + server diode-reconciler:8081; + } + + upstream auth_grpc { + server diode-auth:8081; + } + + server { + listen 8080 http2; + + # Diode Ingester gRPC + location /diode.v1.IngesterService/ { + grpc_pass grpc://ingester_grpc; + } + + # Diode Reconciler gRPC + location /diode.v1.ReconcilerService/ { + grpc_pass grpc://reconciler_grpc; + } + + # Diode Auth gRPC + location /diode.v1.AuthService/ { + grpc_pass grpc://auth_grpc; + } + + # Health check + location /health { + return 200 'OK'; + add_header Content-Type text/plain; + } + } +} diff --git a/oauth2/client/bootstrap-clients.sh b/oauth2/client/bootstrap-clients.sh new file mode 100755 index 0000000..af1d2f0 --- /dev/null +++ b/oauth2/client/bootstrap-clients.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env sh +set -e + +HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}" +CREDENTIALS_FILE="/client-credentials/client-credentials.json" + +echo "Waiting for Hydra to be ready..." +until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do + echo " Hydra not ready yet, retrying in 3s..." + sleep 3 +done +echo "Hydra is ready." + +CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))") + +for i in $(seq 0 $((CLIENT_COUNT - 1))); do + CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))") + CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])") + + echo "Checking client: ${CLIENT_ID}" + + # Check if client already exists + HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}') + if [ "${HTTP_CODE}" = "200" ]; then + echo " Client '${CLIENT_ID}' already exists, skipping." + continue + fi + + echo " Registering client '${CLIENT_ID}'..." + wget -qO- --header="Content-Type: application/json" \ + --post-data="${CLIENT_JSON}" \ + "${HYDRA_ADMIN_URL}/admin/clients" || { + echo " ERROR: Failed to register client '${CLIENT_ID}'" + exit 1 + } + echo "" + echo " Client '${CLIENT_ID}' registered successfully." +done + +echo "All OAuth2 clients registered." diff --git a/setup.sh b/setup.sh new file mode 100755 index 0000000..9be2782 --- /dev/null +++ b/setup.sh @@ -0,0 +1,325 @@ +#!/usr/bin/env bash +set -euo pipefail + +# ============================================================================= +# NetBox Diode Project - Bootstrap Script +# Generates secrets, writes configs, and prepares all services for startup. +# ============================================================================= + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +NETBOX_DOCKER_DIR="/home/user/netbox-docker" +HOST_IP="172.19.77.160" + +echo "==> Generating secrets..." + +gen_secret() { + openssl rand -hex 32 +} + +REDIS_PASSWORD="$(gen_secret)" +DIODE_DB_PASSWORD="$(gen_secret)" +HYDRA_DB_PASSWORD="$(gen_secret)" +HYDRA_SYSTEM_SECRET="$(gen_secret)" + +# OAuth2 client secrets (used by Diode services and Orb Agent) +INGESTER_CLIENT_SECRET="$(gen_secret)" +RECONCILER_CLIENT_SECRET="$(gen_secret)" +NETBOX_TO_DIODE_CLIENT_SECRET="$(gen_secret)" + +echo "==> Writing .env file..." +cat > "${SCRIPT_DIR}/.env" < Writing OAuth2 client credentials..." +cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" < Writing OAuth2 bootstrap script..." +cat > "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" <<'BOOTSTRAP_EOF' +#!/usr/bin/env sh +set -e + +HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}" +CREDENTIALS_FILE="/client-credentials/client-credentials.json" + +echo "Waiting for Hydra to be ready..." +until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do + echo " Hydra not ready yet, retrying in 3s..." + sleep 3 +done +echo "Hydra is ready." + +CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))") + +for i in $(seq 0 $((CLIENT_COUNT - 1))); do + CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))") + CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])") + + echo "Checking client: ${CLIENT_ID}" + + # Check if client already exists + HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}') + if [ "${HTTP_CODE}" = "200" ]; then + echo " Client '${CLIENT_ID}' already exists, skipping." + continue + fi + + echo " Registering client '${CLIENT_ID}'..." + wget -qO- --header="Content-Type: application/json" \ + --post-data="${CLIENT_JSON}" \ + "${HYDRA_ADMIN_URL}/admin/clients" || { + echo " ERROR: Failed to register client '${CLIENT_ID}'" + exit 1 + } + echo "" + echo " Client '${CLIENT_ID}' registered successfully." +done + +echo "All OAuth2 clients registered." +BOOTSTRAP_EOF +chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" + +echo "==> Writing Orb Agent config..." +cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" < Writing nginx.conf..." +mkdir -p "${SCRIPT_DIR}/nginx" +cat > "${SCRIPT_DIR}/nginx/nginx.conf" <<'NGINX_EOF' +worker_processes 1; + +events { + worker_connections 1024; +} + +http { + log_format main '$remote_addr - $remote_user [$time_local] "$request" ' + '$status $body_bytes_sent "$http_referer" ' + '"$http_user_agent"'; + + access_log /var/log/nginx/access.log main; + error_log /var/log/nginx/error.log warn; + + upstream ingester_grpc { + server diode-ingester:8081; + } + + upstream reconciler_grpc { + server diode-reconciler:8081; + } + + upstream auth_grpc { + server diode-auth:8081; + } + + server { + listen 8080 http2; + + # Diode Ingester gRPC + location /diode.v1.IngesterService/ { + grpc_pass grpc://ingester_grpc; + } + + # Diode Reconciler gRPC + location /diode.v1.ReconcilerService/ { + grpc_pass grpc://reconciler_grpc; + } + + # Diode Auth gRPC + location /diode.v1.AuthService/ { + grpc_pass grpc://auth_grpc; + } + + # Health check + location /health { + return 200 'OK'; + add_header Content-Type text/plain; + } + } +} +NGINX_EOF + +echo "==> Updating NetBox plugins.py..." +PLUGINS_FILE="${NETBOX_DOCKER_DIR}/configuration/plugins.py" +cat > "${PLUGINS_FILE}" < Creating NetBox Diode plugin Dockerfile..." +cat > "${NETBOX_DOCKER_DIR}/Dockerfile.diode-plugin" <<'DOCKERFILE_EOF' +FROM netboxcommunity/netbox:v4.5-4.0.1 + +# Install the Diode NetBox plugin +RUN /usr/local/bin/uv pip install --python /opt/netbox/venv/bin/python netboxlabs-diode-netbox-plugin==1.7.0 +DOCKERFILE_EOF + +echo "==> Updating NetBox docker-compose.override.yml..." +cat > "${NETBOX_DOCKER_DIR}/docker-compose.override.yml" <<'OVERRIDE_EOF' +services: + netbox: &netbox-override + build: + context: . + dockerfile: Dockerfile.diode-plugin + ports: + - "8000:8080" + environment: + SKIP_SUPERUSER: "false" + SUPERUSER_API_TOKEN: "0123456789abcdef0123456789abcdef01234567" + SUPERUSER_EMAIL: "admin@example.com" + SUPERUSER_NAME: "admin" + SUPERUSER_PASSWORD: "admin" + ALLOWED_HOSTS: "*" + extra_hosts: + - "host.docker.internal:host-gateway" + netbox-worker: + <<: *netbox-override + ports: [] +OVERRIDE_EOF + +echo "" +echo "============================================================" +echo " Setup complete!" +echo "============================================================" +echo "" +echo "Generated files:" +echo " ${SCRIPT_DIR}/.env" +echo " ${SCRIPT_DIR}/nginx/nginx.conf" +echo " ${SCRIPT_DIR}/oauth2/client/client-credentials.json" +echo " ${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" +echo " ${SCRIPT_DIR}/orb-agent/agent.yaml" +echo " ${NETBOX_DOCKER_DIR}/Dockerfile.diode-plugin" +echo " ${NETBOX_DOCKER_DIR}/docker-compose.override.yml" +echo " ${NETBOX_DOCKER_DIR}/configuration/plugins.py" +echo "" +echo "Next steps:" +echo " 1. cd ${NETBOX_DOCKER_DIR} && docker compose build --no-cache && docker compose up -d" +echo " 2. docker compose exec netbox python /opt/netbox/netbox/manage.py migrate netbox_diode_plugin" +echo " 3. cd ${SCRIPT_DIR} && docker compose up -d" +echo ""