sam/netbox-diode-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Diode stack configuration to match upstream architecture

Browse Source 
Major changes to align with the official netboxlabs/diode docker-compose:

docker-compose.yml:
- Upgrade Hydra from v2.2 to v25.4.0 (latest stable)
- Fix env var names: drop DIODE_ prefix on ingester/reconciler/auth
  (DIODE_REDIS_HOST -> REDIS_HOST, DIODE_GRPC_PORT -> removed, etc.)
- Remove AUTH_GRPC_TARGET from ingester — auth is handled by nginx
  via HTTP subrequests to diode-auth, not by the ingester directly
- Point DIODE_AUTH_TOKEN_URL to diode-auth:8080/token (not Hydra)
- Add Postgres connection vars to reconciler (MIGRATION_ENABLED, etc.)
- Mount nginx.conf as /etc/nginx/conf.d/default.conf (not nginx.conf)
- Use netboxlabs/diode-auth image for bootstrap (has hydra CLI + jq)
- Add Hydra JWT strategy config (STRATEGIES_ACCESS_TOKEN: jwt)
- Add orb-agent run command with explicit config path
- Expose Hydra ports 4444/4445 for external token requests
- Add Hydra DB env vars to Postgres for init script

nginx/nginx.conf:
- Rewrite to match official Diode nginx architecture
- Auth via auth_request subrequests to diode-auth:8080 (HTTP)
- gRPC pass to ingester/reconciler on :8081 after auth
- Add /diode/auth proxy for token endpoint access
- Add error handlers for 401/403

oauth2/client/bootstrap-clients.sh:
- Rewrite to use hydra CLI (create/get oauth2-client) instead of
  raw wget calls to Hydra admin API
- Use jq for JSON parsing instead of python3
- Idempotent: checks if client exists before creating

setup.sh:
- Fix OAuth2 scope: diode:ingester -> diode:ingest
- Rewrite orb-agent config to match current agent.yaml schema
  (config_manager, policies with cron schedules, scope-based targets)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
sam 2026-02-27 21:51:40 -07:00
parent c5a0245dd2
commit 2f459e6f4a
4 changed files with 206 additions and 153 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

65
docker-compose.yml
View File

@ -7,7 +7,7 @@ services:
ports: ports:
- "8080:8080" - "8080:8080"
volumes: volumes:
- ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
depends_on: depends_on:
diode-ingester: diode-ingester:
condition: service_started condition: service_started
@ -23,11 +23,9 @@ services:
diode-ingester: diode-ingester:
image: netboxlabs/diode-ingester:latest image: netboxlabs/diode-ingester:latest
environment: environment:
DIODE_REDIS_HOST: redis REDIS_HOST: redis
DIODE_REDIS_PORT: "6379" REDIS_PORT: "6379"
DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}" REDIS_PASSWORD: "${REDIS_PASSWORD}"
DIODE_GRPC_PORT: "8081"
DIODE_AUTH_GRPC_TARGET: diode-auth:8081
depends_on: depends_on:
redis: redis:
condition: service_healthy condition: service_healthy
@ -39,20 +37,26 @@ services:
diode-reconciler: diode-reconciler:
image: netboxlabs/diode-reconciler:latest image: netboxlabs/diode-reconciler:latest
environment: environment:
DIODE_REDIS_HOST: redis REDIS_HOST: redis
DIODE_REDIS_PORT: "6379" REDIS_PORT: "6379"
DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}" REDIS_PASSWORD: "${REDIS_PASSWORD}"
DIODE_GRPC_PORT: "8081" POSTGRES_HOST: postgres
DIODE_NETBOX_API_URL: "${NETBOX_API_URL}" POSTGRES_PORT: "5432"
DIODE_NETBOX_API_TOKEN: "${NETBOX_API_TOKEN}" POSTGRES_DB_NAME: "${DIODE_DB_NAME}"
DIODE_RECONCILER_CLIENT_ID: "${RECONCILER_CLIENT_ID}" POSTGRES_USER: "${DIODE_DB_USER}"
DIODE_RECONCILER_CLIENT_SECRET: "${RECONCILER_CLIENT_SECRET}" POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
DIODE_AUTH_GRPC_TARGET: diode-auth:8081 MIGRATION_ENABLED: "true"
NETBOX_DIODE_PLUGIN_API_BASE_URL: "${NETBOX_API_URL}/api/plugins/diode"
DIODE_AUTH_TOKEN_URL: http://diode-auth:8080/token
DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}" DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}"
DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}" DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}"
volumes:
- ./oauth2/client:/etc/config/oauth2/client:ro
depends_on: depends_on:
redis: redis:
condition: service_healthy condition: service_healthy
postgres:
condition: service_healthy
diode-auth-bootstrap: diode-auth-bootstrap:
condition: service_completed_successfully condition: service_completed_successfully
restart: unless-stopped restart: unless-stopped
@ -63,9 +67,9 @@ services:
diode-auth: diode-auth:
image: netboxlabs/diode-auth:latest image: netboxlabs/diode-auth:latest
environment: environment:
DIODE_GRPC_PORT: "8081" HTTP_PORT: "8080"
DIODE_HYDRA_ADMIN_URL: http://hydra:4445 OAUTH2_PUBLIC_SERVER_URL: http://hydra:4444
DIODE_HYDRA_PUBLIC_URL: http://hydra:4444 OAUTH2_ADMIN_SERVER_URL: http://hydra:4445
depends_on: depends_on:
hydra: hydra:
condition: service_started condition: service_started
@ -75,12 +79,12 @@ services:
# Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot) # Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot)
# ========================================================================= # =========================================================================
diode-auth-bootstrap: diode-auth-bootstrap:
image: python:3.12-alpine image: netboxlabs/diode-auth:latest
command: /client-credentials/bootstrap-clients.sh command: ["/bin/sh", "/etc/config/oauth2/client/bootstrap-clients.sh"]
environment: environment:
HYDRA_ADMIN_URL: http://hydra:4445 HYDRA_ADMIN_URL: http://hydra:4445
volumes: volumes:
- ./oauth2/client:/client-credentials:ro - ./oauth2/client:/etc/config/oauth2/client:ro
depends_on: depends_on:
hydra: hydra:
condition: service_started condition: service_started
@ -90,13 +94,18 @@ services:
# Hydra — OAuth2 / OpenID Connect server # Hydra — OAuth2 / OpenID Connect server
# ========================================================================= # =========================================================================
hydra: hydra:
image: oryd/hydra:v2.2 image: oryd/hydra:v25.4.0
command: serve all --dev --config /etc/hydra/hydra.yml command: serve all --dev
ports:
- "4444:4444"
- "4445:4445"
environment: environment:
DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable" DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}" SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}"
URLS_SELF_ISSUER: http://hydra:4444/ URLS_SELF_ISSUER: http://${HOST_IP}:4444/
URLS_SELF_PUBLIC: http://hydra:4444/ URLS_SELF_PUBLIC: http://${HOST_IP}:4444/
STRATEGIES_ACCESS_TOKEN: jwt
STRATEGIES_JWT_SCOPE_CLAIM: both
depends_on: depends_on:
hydra-migrate: hydra-migrate:
condition: service_completed_successfully condition: service_completed_successfully
@ -106,7 +115,7 @@ services:
# Hydra Migrate — runs Hydra database migrations (one-shot) # Hydra Migrate — runs Hydra database migrations (one-shot)
# ========================================================================= # =========================================================================
hydra-migrate: hydra-migrate:
image: oryd/hydra:v2.2 image: oryd/hydra:v25.4.0
command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable" command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
depends_on: depends_on:
postgres: postgres:
@ -140,6 +149,9 @@ services:
POSTGRES_USER: "${DIODE_DB_USER}" POSTGRES_USER: "${DIODE_DB_USER}"
POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}" POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
POSTGRES_DB: "${DIODE_DB_NAME}" POSTGRES_DB: "${DIODE_DB_NAME}"
HYDRA_DB_USER: "${HYDRA_DB_USER}"
HYDRA_DB_PASSWORD: "${HYDRA_DB_PASSWORD}"
HYDRA_DB_NAME: "${HYDRA_DB_NAME}"
healthcheck: healthcheck:
test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"] test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"]
start_period: 10s start_period: 10s
@ -157,6 +169,7 @@ services:
orb-agent: orb-agent:
image: netboxlabs/orb-agent:latest image: netboxlabs/orb-agent:latest
network_mode: host network_mode: host
command: ["run", "-c", "/opt/orb/agent.yaml"]
volumes: volumes:
- ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro - ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro
depends_on: depends_on:

94
nginx/nginx.conf
View File

@ -1,51 +1,79 @@
worker_processes 1; upstream diode-ingester {
server diode-ingester:8081;
events {
worker_connections 1024;
} }
http { upstream diode-reconciler {
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
'$status $body_bytes_sent "$http_referer" '
'"$http_user_agent"';
access_log /var/log/nginx/access.log main;
error_log /var/log/nginx/error.log warn;
upstream ingester_grpc {
server diode-ingester:8081;
}
upstream reconciler_grpc {
server diode-reconciler:8081; server diode-reconciler:8081;
}
upstream diode-auth {
server diode-auth:8080;
}
server {
listen 8080;
listen [::]:8080;
http2 on;
server_name localhost;
client_max_body_size 25m;
location /auth/introspect {
internal;
proxy_method POST;
proxy_pass http://diode-auth/introspect;
proxy_pass_request_body off;
proxy_set_header Content-Length "";
proxy_set_header X-Original-URI $request_uri;
} }
upstream auth_grpc { location /diode/auth {
server diode-auth:8081; rewrite /diode/auth/(.*) /$1 break;
proxy_pass http://diode-auth;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $remote_addr;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header X-Forwarded-Proto $scheme;
} }
server { location /diode/diode.v1.IngesterService {
listen 8080 http2; auth_request /auth/introspect;
auth_request_set $auth_status $upstream_status;
error_page 401 = @error401;
error_page 403 = @error403;
# Diode Ingester gRPC rewrite /diode/(.*) /$1 break;
location /diode.v1.IngesterService/ { grpc_pass grpc://diode-ingester;
grpc_pass grpc://ingester_grpc; grpc_set_header Host $host;
grpc_set_header X-Real-IP $remote_addr;
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
grpc_set_header X-Forwarded-Proto $scheme;
} }
# Diode Reconciler gRPC location /diode/diode.v1.ReconcilerService {
location /diode.v1.ReconcilerService/ { auth_request /auth/introspect;
grpc_pass grpc://reconciler_grpc; auth_request_set $auth_status $upstream_status;
error_page 401 = @error401;
error_page 403 = @error403;
rewrite /diode/(.*) /$1 break;
grpc_pass grpc://diode-reconciler;
grpc_set_header Host $host;
grpc_set_header X-Real-IP $remote_addr;
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
grpc_set_header X-Forwarded-Proto $scheme;
} }
# Diode Auth gRPC
location /diode.v1.AuthService/ {
grpc_pass grpc://auth_grpc;
}
# Health check
location /health { location /health {
return 200 'OK'; return 200 'OK';
add_header Content-Type text/plain; add_header Content-Type text/plain;
} }
location @error401 {
return 401 '{"error":"unauthorized","error_description":"Authentication required"}';
}
location @error403 {
return 403 '{"error":"forbidden","error_description":"Access denied"}';
} }
} }

78
oauth2/client/bootstrap-clients.sh
View File

@ -1,40 +1,56 @@
#!/usr/bin/env sh #!/usr/bin/env bash
set -e
HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}" set -euo pipefail
CREDENTIALS_FILE="/client-credentials/client-credentials.json"
echo "Waiting for Hydra to be ready..." # Constants
until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json"
echo " Hydra not ready yet, retrying in 3s..."
sleep 3
done
echo "Hydra is ready."
CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))") # Create the credentials file if it doesn't exist
if [ ! -f "$CREDENTIALS_FILE" ]; then
echo "ERROR: credentials file $CREDENTIALS_FILE not found"
exit 1
fi
for i in $(seq 0 $((CLIENT_COUNT - 1))); do # Wait for Hydra to be ready
CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))") sleep 3
CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
echo "Checking client: ${CLIENT_ID}" # Function to create client
create_client() {
local client_id=$1
local client_secret=$2
local scope=$3
local exists_in_hydra=false
# Check if client already exists # Check if client exists in Hydra
HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}') if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then
if [ "${HTTP_CODE}" = "200" ]; then exists_in_hydra=true
echo " Client '${CLIENT_ID}' already exists, skipping."
continue
fi fi
echo " Registering client '${CLIENT_ID}'..." # Log client existence status
wget -qO- --header="Content-Type: application/json" \ if [ "$exists_in_hydra" = true ]; then
--post-data="${CLIENT_JSON}" \ echo "INFO: client $client_id exists in Hydra"
"${HYDRA_ADMIN_URL}/admin/clients" || { return 0
echo " ERROR: Failed to register client '${CLIENT_ID}'" fi
exit 1
}
echo ""
echo " Client '${CLIENT_ID}' registered successfully."
done
echo "All OAuth2 clients registered." # Create new client if it doesn't exist in Hydra
if [ "$exists_in_hydra" = false ]; then
client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \
--id $client_id \
--secret $client_secret \
--grant-type "client_credentials" \
--response-type "token" \
--scope "$scope" \
--token-endpoint-auth-method "client_secret_post" \
--format json)
echo "INFO: client $client_id created"
fi
}
# Load client credentials
jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do
client_id=$(echo "$client" | jq -r '.client_id')
client_secret=$(echo "$client" | jq -r '.client_secret')
scope=$(echo "$client" | jq -r '.scope')
create_client "$client_id" "$client_secret" "$scope"
done

86
setup.sh
View File

@ -76,7 +76,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
"client_name": "Diode Ingester", "client_name": "Diode Ingester",
"grant_types": ["client_credentials"], "grant_types": ["client_credentials"],
"token_endpoint_auth_method": "client_secret_post", "token_endpoint_auth_method": "client_secret_post",
"scope": "diode:ingester", "scope": "diode:ingest",
"audience": ["diode-ingester"] "audience": ["diode-ingester"]
}, },
{ {
@ -148,58 +148,54 @@ chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
echo "==> Writing Orb Agent config..." echo "==> Writing Orb Agent config..."
cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" <<EOF cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" <<EOF
orb: orb:
config_manager:
active: local
backends: backends:
network_discovery: network_discovery:
# NMAP-based network scan of the target subnet
scan_policy:
targets:
- "10.0.0.0/24"
interval: "1800" # every 30 minutes
config:
top_ports: 100
os_detection: true
service_detection: true
diode:
target: "grpc://${HOST_IP}:8080"
client_id: "diode-ingester"
client_secret: "${INGESTER_CLIENT_SECRET}"
site: "main"
snmp_discovery: snmp_discovery:
# SNMPv2c discovery across the subnet common:
scan_policy: diode:
target: grpc://${HOST_IP}:8080/diode
client_id: diode-ingester
client_secret: ${INGESTER_CLIENT_SECRET}
agent_name: orb-agent-01
policies:
network_discovery:
nmap_scan:
config:
schedule: "*/30 * * * *"
timeout: 300
defaults:
site: "main"
scope:
targets: targets:
- "10.0.0.0/24" - "10.0.0.0/24"
interval: "21600" # every 6 hours snmp_discovery:
snmp_scan:
config: config:
communities: schedule: "0 */6 * * *"
- "public" timeout: 300
versions: snmp_timeout: 10
- "2c" retries: 3
ports: defaults:
- 161
diode:
target: "grpc://${HOST_IP}:8080"
client_id: "diode-ingester"
client_secret: "${INGESTER_CLIENT_SECRET}"
site: "main" site: "main"
scope:
device_discovery:
# NAPALM SSH-based device interrogation for known devices
scan_policy:
targets: targets:
- "10.0.0.1" # gateway / core router (update as needed) - host: "10.0.0.0/24"
interval: "43200" # every 12 hours authentication:
config: protocol_version: "SNMPv2c"
driver: "ios" community: "public"
credentials: # device_discovery:
username: "admin" # napalm_scan:
password: "admin" # config:
diode: # schedule: "0 */12 * * *"
target: "grpc://${HOST_IP}:8080" # defaults:
client_id: "diode-ingester" # site: "main"
client_secret: "${INGESTER_CLIENT_SECRET}" # scope:
site: "main" # - driver: ios
# hostname: 10.0.0.1
# username: admin
# password: admin
EOF EOF
echo "==> Writing nginx.conf..." echo "==> Writing nginx.conf..."