Add project infrastructure and configuration files

Docker Compose stack, nginx config, OAuth2 client bootstrap,
Hydra DB init, setup script, and gitignore for secrets.

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
sam 2026-02-27 20:46:59 -07:00
parent 955618a44a
commit c5a0245dd2
7 changed files with 624 additions and 0 deletions

9
.claude/settings.json Normal file
View File

@ -0,0 +1,9 @@
{
"permissions": {
"allow": [
"Bash(/home/user/netbox-diode-project/.venv/bin/python --version)",
"Bash(/home/user/netbox-diode-project/.venv/bin/pip --version)",
"Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)"
]
}
}

8
.gitignore vendored Normal file
View File

@ -0,0 +1,8 @@
.venv/
__pycache__/
*.pyc
.env
# Generated by setup.sh (contain secrets)
oauth2/client/client-credentials.json
orb-agent/agent.yaml

173
docker-compose.yml Normal file
View File

@ -0,0 +1,173 @@
services:
# =========================================================================
# Ingress — routes gRPC traffic to Diode services
# =========================================================================
ingress-nginx:
image: nginx:1.27-alpine
ports:
- "8080:8080"
volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
depends_on:
diode-ingester:
condition: service_started
diode-reconciler:
condition: service_started
diode-auth:
condition: service_started
restart: unless-stopped
# =========================================================================
# Diode Ingester — accepts gRPC, publishes to Redis streams
# =========================================================================
diode-ingester:
image: netboxlabs/diode-ingester:latest
environment:
DIODE_REDIS_HOST: redis
DIODE_REDIS_PORT: "6379"
DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
DIODE_GRPC_PORT: "8081"
DIODE_AUTH_GRPC_TARGET: diode-auth:8081
depends_on:
redis:
condition: service_healthy
restart: unless-stopped
# =========================================================================
# Diode Reconciler — consumes Redis streams, reconciles with NetBox
# =========================================================================
diode-reconciler:
image: netboxlabs/diode-reconciler:latest
environment:
DIODE_REDIS_HOST: redis
DIODE_REDIS_PORT: "6379"
DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
DIODE_GRPC_PORT: "8081"
DIODE_NETBOX_API_URL: "${NETBOX_API_URL}"
DIODE_NETBOX_API_TOKEN: "${NETBOX_API_TOKEN}"
DIODE_RECONCILER_CLIENT_ID: "${RECONCILER_CLIENT_ID}"
DIODE_RECONCILER_CLIENT_SECRET: "${RECONCILER_CLIENT_SECRET}"
DIODE_AUTH_GRPC_TARGET: diode-auth:8081
DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}"
DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}"
depends_on:
redis:
condition: service_healthy
diode-auth-bootstrap:
condition: service_completed_successfully
restart: unless-stopped
# =========================================================================
# Diode Auth — validates OAuth2 tokens via Hydra
# =========================================================================
diode-auth:
image: netboxlabs/diode-auth:latest
environment:
DIODE_GRPC_PORT: "8081"
DIODE_HYDRA_ADMIN_URL: http://hydra:4445
DIODE_HYDRA_PUBLIC_URL: http://hydra:4444
depends_on:
hydra:
condition: service_started
restart: unless-stopped
# =========================================================================
# Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot)
# =========================================================================
diode-auth-bootstrap:
image: python:3.12-alpine
command: /client-credentials/bootstrap-clients.sh
environment:
HYDRA_ADMIN_URL: http://hydra:4445
volumes:
- ./oauth2/client:/client-credentials:ro
depends_on:
hydra:
condition: service_started
restart: "no"
# =========================================================================
# Hydra — OAuth2 / OpenID Connect server
# =========================================================================
hydra:
image: oryd/hydra:v2.2
command: serve all --dev --config /etc/hydra/hydra.yml
environment:
DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}"
URLS_SELF_ISSUER: http://hydra:4444/
URLS_SELF_PUBLIC: http://hydra:4444/
depends_on:
hydra-migrate:
condition: service_completed_successfully
restart: unless-stopped
# =========================================================================
# Hydra Migrate — runs Hydra database migrations (one-shot)
# =========================================================================
hydra-migrate:
image: oryd/hydra:v2.2
command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
depends_on:
postgres:
condition: service_healthy
restart: "no"
# =========================================================================
# Redis — message streams for Diode ingester ↔ reconciler
# =========================================================================
redis:
image: redis/redis-stack-server:latest
command: redis-server --requirepass "${REDIS_PASSWORD}" --appendonly yes
ports:
- "6378:6379"
healthcheck:
test: ["CMD", "redis-cli", "-a", "${REDIS_PASSWORD}", "ping"]
start_period: 5s
interval: 5s
timeout: 3s
retries: 5
volumes:
- diode-redis-data:/data
restart: unless-stopped
# =========================================================================
# PostgreSQL — stores Diode + Hydra databases
# =========================================================================
postgres:
image: postgres:16-alpine
environment:
POSTGRES_USER: "${DIODE_DB_USER}"
POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
POSTGRES_DB: "${DIODE_DB_NAME}"
healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"]
start_period: 10s
interval: 10s
timeout: 5s
retries: 5
volumes:
- diode-postgres-data:/var/lib/postgresql/data
- ./init-hydra-db.sh:/docker-entrypoint-initdb.d/init-hydra-db.sh:ro
restart: unless-stopped
# =========================================================================
# Orb Agent — network discovery (NMAP, SNMP, NAPALM)
# =========================================================================
orb-agent:
image: netboxlabs/orb-agent:latest
network_mode: host
volumes:
- ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro
depends_on:
diode-ingester:
condition: service_started
diode-auth-bootstrap:
condition: service_completed_successfully
restart: unless-stopped
volumes:
diode-redis-data:
driver: local
diode-postgres-data:
driver: local

18
init-hydra-db.sh Executable file
View File

@ -0,0 +1,18 @@
#!/bin/bash
set -e
# Create the Hydra database and user if they don't already exist.
# This runs as part of PostgreSQL's docker-entrypoint-initdb.d.
psql -v ON_ERROR_STOP=1 --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" <<-EOSQL
DO \$\$
BEGIN
IF NOT EXISTS (SELECT FROM pg_catalog.pg_roles WHERE rolname = '${HYDRA_DB_USER}') THEN
CREATE ROLE ${HYDRA_DB_USER} WITH LOGIN PASSWORD '${HYDRA_DB_PASSWORD}';
END IF;
END
\$\$;
SELECT 'CREATE DATABASE ${HYDRA_DB_NAME} OWNER ${HYDRA_DB_USER}'
WHERE NOT EXISTS (SELECT FROM pg_database WHERE datname = '${HYDRA_DB_NAME}')\gexec
EOSQL

51
nginx/nginx.conf Normal file
View File

@ -0,0 +1,51 @@
worker_processes 1;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log warn;
upstream ingester_grpc {
server diode-ingester:8081;
}
upstream reconciler_grpc {
server diode-reconciler:8081;
}
upstream auth_grpc {
server diode-auth:8081;
}
server {
listen 8080 http2;
# Diode Ingester gRPC
location /diode.v1.IngesterService/ {
grpc_pass grpc://ingester_grpc;
}
# Diode Reconciler gRPC
location /diode.v1.ReconcilerService/ {
grpc_pass grpc://reconciler_grpc;
}
# Diode Auth gRPC
location /diode.v1.AuthService/ {
grpc_pass grpc://auth_grpc;
}
# Health check
location /health {
return 200 'OK';
add_header Content-Type text/plain;
}
}
}

View File

@ -0,0 +1,40 @@
#!/usr/bin/env sh
set -e
HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
CREDENTIALS_FILE="/client-credentials/client-credentials.json"
echo "Waiting for Hydra to be ready..."
until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
echo " Hydra not ready yet, retrying in 3s..."
sleep 3
done
echo "Hydra is ready."
CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
for i in $(seq 0 $((CLIENT_COUNT - 1))); do
CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
echo "Checking client: ${CLIENT_ID}"
# Check if client already exists
HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
if [ "${HTTP_CODE}" = "200" ]; then
echo " Client '${CLIENT_ID}' already exists, skipping."
continue
fi
echo " Registering client '${CLIENT_ID}'..."
wget -qO- --header="Content-Type: application/json" \
--post-data="${CLIENT_JSON}" \
"${HYDRA_ADMIN_URL}/admin/clients" || {
echo " ERROR: Failed to register client '${CLIENT_ID}'"
exit 1
}
echo ""
echo " Client '${CLIENT_ID}' registered successfully."
done
echo "All OAuth2 clients registered."

325
setup.sh Executable file
View File

@ -0,0 +1,325 @@
#!/usr/bin/env bash
set -euo pipefail
# =============================================================================
# NetBox Diode Project - Bootstrap Script
# Generates secrets, writes configs, and prepares all services for startup.
# =============================================================================
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
NETBOX_DOCKER_DIR="/home/user/netbox-docker"
HOST_IP="172.19.77.160"
echo "==> Generating secrets..."
gen_secret() {
openssl rand -hex 32
}
REDIS_PASSWORD="$(gen_secret)"
DIODE_DB_PASSWORD="$(gen_secret)"
HYDRA_DB_PASSWORD="$(gen_secret)"
HYDRA_SYSTEM_SECRET="$(gen_secret)"
# OAuth2 client secrets (used by Diode services and Orb Agent)
INGESTER_CLIENT_SECRET="$(gen_secret)"
RECONCILER_CLIENT_SECRET="$(gen_secret)"
NETBOX_TO_DIODE_CLIENT_SECRET="$(gen_secret)"
echo "==> Writing .env file..."
cat > "${SCRIPT_DIR}/.env" <<EOF
# =============================================================================
# Generated by setup.sh — do not edit manually
# =============================================================================
# Host networking
HOST_IP=${HOST_IP}
# Redis
REDIS_PASSWORD=${REDIS_PASSWORD}
# Postgres (Diode)
DIODE_DB_NAME=diode
DIODE_DB_USER=diode
DIODE_DB_PASSWORD=${DIODE_DB_PASSWORD}
# Postgres (Hydra)
HYDRA_DB_NAME=hydra
HYDRA_DB_USER=hydra
HYDRA_DB_PASSWORD=${HYDRA_DB_PASSWORD}
# Hydra
HYDRA_SYSTEM_SECRET=${HYDRA_SYSTEM_SECRET}
# OAuth2 client IDs (fixed)
INGESTER_CLIENT_ID=diode-ingester
RECONCILER_CLIENT_ID=diode-reconciler
NETBOX_TO_DIODE_CLIENT_ID=netbox-to-diode
# OAuth2 client secrets
INGESTER_CLIENT_SECRET=${INGESTER_CLIENT_SECRET}
RECONCILER_CLIENT_SECRET=${RECONCILER_CLIENT_SECRET}
NETBOX_TO_DIODE_CLIENT_SECRET=${NETBOX_TO_DIODE_CLIENT_SECRET}
# NetBox connection
NETBOX_API_URL=http://${HOST_IP}:8000
NETBOX_API_TOKEN=0123456789abcdef0123456789abcdef01234567
NETBOX_DIODE_PLUGIN_VERSION=1.7.0
EOF
echo "==> Writing OAuth2 client credentials..."
cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
[
{
"client_id": "diode-ingester",
"client_secret": "${INGESTER_CLIENT_SECRET}",
"client_name": "Diode Ingester",
"grant_types": ["client_credentials"],
"token_endpoint_auth_method": "client_secret_post",
"scope": "diode:ingester",
"audience": ["diode-ingester"]
},
{
"client_id": "diode-reconciler",
"client_secret": "${RECONCILER_CLIENT_SECRET}",
"client_name": "Diode Reconciler",
"grant_types": ["client_credentials"],
"token_endpoint_auth_method": "client_secret_post",
"scope": "diode:reconciler",
"audience": ["diode-reconciler"]
},
{
"client_id": "netbox-to-diode",
"client_secret": "${NETBOX_TO_DIODE_CLIENT_SECRET}",
"client_name": "NetBox to Diode",
"grant_types": ["client_credentials"],
"token_endpoint_auth_method": "client_secret_post",
"scope": "diode:netbox-to-diode",
"audience": ["netbox-to-diode"]
}
]
EOF
echo "==> Writing OAuth2 bootstrap script..."
cat > "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" <<'BOOTSTRAP_EOF'
#!/usr/bin/env sh
set -e
HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
CREDENTIALS_FILE="/client-credentials/client-credentials.json"
echo "Waiting for Hydra to be ready..."
until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
echo " Hydra not ready yet, retrying in 3s..."
sleep 3
done
echo "Hydra is ready."
CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
for i in $(seq 0 $((CLIENT_COUNT - 1))); do
CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
echo "Checking client: ${CLIENT_ID}"
# Check if client already exists
HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
if [ "${HTTP_CODE}" = "200" ]; then
echo " Client '${CLIENT_ID}' already exists, skipping."
continue
fi
echo " Registering client '${CLIENT_ID}'..."
wget -qO- --header="Content-Type: application/json" \
--post-data="${CLIENT_JSON}" \
"${HYDRA_ADMIN_URL}/admin/clients" || {
echo " ERROR: Failed to register client '${CLIENT_ID}'"
exit 1
}
echo ""
echo " Client '${CLIENT_ID}' registered successfully."
done
echo "All OAuth2 clients registered."
BOOTSTRAP_EOF
chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
echo "==> Writing Orb Agent config..."
cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" <<EOF
orb:
backends:
network_discovery:
# NMAP-based network scan of the target subnet
scan_policy:
targets:
- "10.0.0.0/24"
interval: "1800" # every 30 minutes
config:
top_ports: 100
os_detection: true
service_detection: true
diode:
target: "grpc://${HOST_IP}:8080"
client_id: "diode-ingester"
client_secret: "${INGESTER_CLIENT_SECRET}"
site: "main"
snmp_discovery:
# SNMPv2c discovery across the subnet
scan_policy:
targets:
- "10.0.0.0/24"
interval: "21600" # every 6 hours
config:
communities:
- "public"
versions:
- "2c"
ports:
- 161
diode:
target: "grpc://${HOST_IP}:8080"
client_id: "diode-ingester"
client_secret: "${INGESTER_CLIENT_SECRET}"
site: "main"
device_discovery:
# NAPALM SSH-based device interrogation for known devices
scan_policy:
targets:
- "10.0.0.1" # gateway / core router (update as needed)
interval: "43200" # every 12 hours
config:
driver: "ios"
credentials:
username: "admin"
password: "admin"
diode:
target: "grpc://${HOST_IP}:8080"
client_id: "diode-ingester"
client_secret: "${INGESTER_CLIENT_SECRET}"
site: "main"
EOF
echo "==> Writing nginx.conf..."
mkdir -p "${SCRIPT_DIR}/nginx"
cat > "${SCRIPT_DIR}/nginx/nginx.conf" <<'NGINX_EOF'
worker_processes 1;
events {
worker_connections 1024;
}
http {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log warn;
upstream ingester_grpc {
server diode-ingester:8081;
}
upstream reconciler_grpc {
server diode-reconciler:8081;
}
upstream auth_grpc {
server diode-auth:8081;
}
server {
listen 8080 http2;
# Diode Ingester gRPC
location /diode.v1.IngesterService/ {
grpc_pass grpc://ingester_grpc;
}
# Diode Reconciler gRPC
location /diode.v1.ReconcilerService/ {
grpc_pass grpc://reconciler_grpc;
}
# Diode Auth gRPC
location /diode.v1.AuthService/ {
grpc_pass grpc://auth_grpc;
}
# Health check
location /health {
return 200 'OK';
add_header Content-Type text/plain;
}
}
}
NGINX_EOF
echo "==> Updating NetBox plugins.py..."
PLUGINS_FILE="${NETBOX_DOCKER_DIR}/configuration/plugins.py"
cat > "${PLUGINS_FILE}" <<EOF
PLUGINS = ["netbox_diode_plugin"]
PLUGINS_CONFIG = {
"netbox_diode_plugin": {
"diode_target_override": "grpc://${HOST_IP}:8080",
"diode_username": "admin",
"netbox_to_diode_client_secret": "${NETBOX_TO_DIODE_CLIENT_SECRET}",
}
}
EOF
echo "==> Creating NetBox Diode plugin Dockerfile..."
cat > "${NETBOX_DOCKER_DIR}/Dockerfile.diode-plugin" <<'DOCKERFILE_EOF'
FROM netboxcommunity/netbox:v4.5-4.0.1
# Install the Diode NetBox plugin
RUN /usr/local/bin/uv pip install --python /opt/netbox/venv/bin/python netboxlabs-diode-netbox-plugin==1.7.0
DOCKERFILE_EOF
echo "==> Updating NetBox docker-compose.override.yml..."
cat > "${NETBOX_DOCKER_DIR}/docker-compose.override.yml" <<'OVERRIDE_EOF'
services:
netbox: &netbox-override
build:
context: .
dockerfile: Dockerfile.diode-plugin
ports:
- "8000:8080"
environment:
SKIP_SUPERUSER: "false"
SUPERUSER_API_TOKEN: "0123456789abcdef0123456789abcdef01234567"
SUPERUSER_EMAIL: "admin@example.com"
SUPERUSER_NAME: "admin"
SUPERUSER_PASSWORD: "admin"
ALLOWED_HOSTS: "*"
extra_hosts:
- "host.docker.internal:host-gateway"
netbox-worker:
<<: *netbox-override
ports: []
OVERRIDE_EOF
echo ""
echo "============================================================"
echo " Setup complete!"
echo "============================================================"
echo ""
echo "Generated files:"
echo " ${SCRIPT_DIR}/.env"
echo " ${SCRIPT_DIR}/nginx/nginx.conf"
echo " ${SCRIPT_DIR}/oauth2/client/client-credentials.json"
echo " ${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
echo " ${SCRIPT_DIR}/orb-agent/agent.yaml"
echo " ${NETBOX_DOCKER_DIR}/Dockerfile.diode-plugin"
echo " ${NETBOX_DOCKER_DIR}/docker-compose.override.yml"
echo " ${NETBOX_DOCKER_DIR}/configuration/plugins.py"
echo ""
echo "Next steps:"
echo " 1. cd ${NETBOX_DOCKER_DIR} && docker compose build --no-cache && docker compose up -d"
echo " 2. docker compose exec netbox python /opt/netbox/netbox/manage.py migrate netbox_diode_plugin"
echo " 3. cd ${SCRIPT_DIR} && docker compose up -d"
echo ""