Fix Diode stack configuration to match upstream architecture

Major changes to align with the official netboxlabs/diode docker-compose:

docker-compose.yml:
- Upgrade Hydra from v2.2 to v25.4.0 (latest stable)
- Fix env var names: drop DIODE_ prefix on ingester/reconciler/auth
  (DIODE_REDIS_HOST -> REDIS_HOST, DIODE_GRPC_PORT -> removed, etc.)
- Remove AUTH_GRPC_TARGET from ingester — auth is handled by nginx
  via HTTP subrequests to diode-auth, not by the ingester directly
- Point DIODE_AUTH_TOKEN_URL to diode-auth:8080/token (not Hydra)
- Add Postgres connection vars to reconciler (MIGRATION_ENABLED, etc.)
- Mount nginx.conf as /etc/nginx/conf.d/default.conf (not nginx.conf)
- Use netboxlabs/diode-auth image for bootstrap (has hydra CLI + jq)
- Add Hydra JWT strategy config (STRATEGIES_ACCESS_TOKEN: jwt)
- Add orb-agent run command with explicit config path
- Expose Hydra ports 4444/4445 for external token requests
- Add Hydra DB env vars to Postgres for init script

nginx/nginx.conf:
- Rewrite to match official Diode nginx architecture
- Auth via auth_request subrequests to diode-auth:8080 (HTTP)
- gRPC pass to ingester/reconciler on :8081 after auth
- Add /diode/auth proxy for token endpoint access
- Add error handlers for 401/403

oauth2/client/bootstrap-clients.sh:
- Rewrite to use hydra CLI (create/get oauth2-client) instead of
  raw wget calls to Hydra admin API
- Use jq for JSON parsing instead of python3
- Idempotent: checks if client exists before creating

setup.sh:
- Fix OAuth2 scope: diode:ingester -> diode:ingest
- Rewrite orb-agent config to match current agent.yaml schema
  (config_manager, policies with cron schedules, scope-based targets)

Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>

docker-compose.yml

@@ -7,7 +7,7 @@ services:
     ports:
       - "8080:8080"
     volumes:
-      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
     depends_on:
       diode-ingester:
         condition: service_started
@@ -23,11 +23,9 @@ services:
   diode-ingester:
     image: netboxlabs/diode-ingester:latest
     environment:
-      DIODE_REDIS_HOST: redis
+      REDIS_HOST: redis
-      DIODE_REDIS_PORT: "6379"
+      REDIS_PORT: "6379"
-      DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      DIODE_GRPC_PORT: "8081"
-      DIODE_AUTH_GRPC_TARGET: diode-auth:8081
     depends_on:
       redis:
         condition: service_healthy
@@ -39,20 +37,26 @@ services:
   diode-reconciler:
     image: netboxlabs/diode-reconciler:latest
     environment:
-      DIODE_REDIS_HOST: redis
+      REDIS_HOST: redis
-      DIODE_REDIS_PORT: "6379"
+      REDIS_PORT: "6379"
-      DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      DIODE_GRPC_PORT: "8081"
+      POSTGRES_HOST: postgres
-      DIODE_NETBOX_API_URL: "${NETBOX_API_URL}"
+      POSTGRES_PORT: "5432"
-      DIODE_NETBOX_API_TOKEN: "${NETBOX_API_TOKEN}"
+      POSTGRES_DB_NAME: "${DIODE_DB_NAME}"
-      DIODE_RECONCILER_CLIENT_ID: "${RECONCILER_CLIENT_ID}"
+      POSTGRES_USER: "${DIODE_DB_USER}"
-      DIODE_RECONCILER_CLIENT_SECRET: "${RECONCILER_CLIENT_SECRET}"
+      POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
-      DIODE_AUTH_GRPC_TARGET: diode-auth:8081
+      MIGRATION_ENABLED: "true"
+      NETBOX_DIODE_PLUGIN_API_BASE_URL: "${NETBOX_API_URL}/api/plugins/diode"
+      DIODE_AUTH_TOKEN_URL: http://diode-auth:8080/token
       DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}"
       DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}"
+    volumes:
+      - ./oauth2/client:/etc/config/oauth2/client:ro
     depends_on:
       redis:
         condition: service_healthy
+      postgres:
+        condition: service_healthy
       diode-auth-bootstrap:
         condition: service_completed_successfully
     restart: unless-stopped
@@ -63,9 +67,9 @@ services:
   diode-auth:
     image: netboxlabs/diode-auth:latest
     environment:
-      DIODE_GRPC_PORT: "8081"
+      HTTP_PORT: "8080"
-      DIODE_HYDRA_ADMIN_URL: http://hydra:4445
+      OAUTH2_PUBLIC_SERVER_URL: http://hydra:4444
-      DIODE_HYDRA_PUBLIC_URL: http://hydra:4444
+      OAUTH2_ADMIN_SERVER_URL: http://hydra:4445
     depends_on:
       hydra:
         condition: service_started
@@ -75,12 +79,12 @@ services:
   # Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot)
   # =========================================================================
   diode-auth-bootstrap:
-    image: python:3.12-alpine
+    image: netboxlabs/diode-auth:latest
-    command: /client-credentials/bootstrap-clients.sh
+    command: ["/bin/sh", "/etc/config/oauth2/client/bootstrap-clients.sh"]
     environment:
       HYDRA_ADMIN_URL: http://hydra:4445
     volumes:
-      - ./oauth2/client:/client-credentials:ro
+      - ./oauth2/client:/etc/config/oauth2/client:ro
     depends_on:
       hydra:
         condition: service_started
@@ -90,13 +94,18 @@ services:
   # Hydra — OAuth2 / OpenID Connect server
   # =========================================================================
   hydra:
-    image: oryd/hydra:v2.2
+    image: oryd/hydra:v25.4.0
-    command: serve all --dev --config /etc/hydra/hydra.yml
+    command: serve all --dev
+    ports:
+      - "4444:4444"
+      - "4445:4445"
     environment:
       DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
       SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}"
-      URLS_SELF_ISSUER: http://hydra:4444/
+      URLS_SELF_ISSUER: http://${HOST_IP}:4444/
-      URLS_SELF_PUBLIC: http://hydra:4444/
+      URLS_SELF_PUBLIC: http://${HOST_IP}:4444/
+      STRATEGIES_ACCESS_TOKEN: jwt
+      STRATEGIES_JWT_SCOPE_CLAIM: both
     depends_on:
       hydra-migrate:
         condition: service_completed_successfully
@@ -106,7 +115,7 @@ services:
   # Hydra Migrate — runs Hydra database migrations (one-shot)
   # =========================================================================
   hydra-migrate:
-    image: oryd/hydra:v2.2
+    image: oryd/hydra:v25.4.0
     command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
     depends_on:
       postgres:
@@ -140,6 +149,9 @@ services:
       POSTGRES_USER: "${DIODE_DB_USER}"
       POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
       POSTGRES_DB: "${DIODE_DB_NAME}"
+      HYDRA_DB_USER: "${HYDRA_DB_USER}"
+      HYDRA_DB_PASSWORD: "${HYDRA_DB_PASSWORD}"
+      HYDRA_DB_NAME: "${HYDRA_DB_NAME}"
     healthcheck:
       test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"]
       start_period: 10s
@@ -157,6 +169,7 @@ services:
   orb-agent:
     image: netboxlabs/orb-agent:latest
     network_mode: host
+    command: ["run", "-c", "/opt/orb/agent.yaml"]
     volumes:
       - ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro
     depends_on:

nginx/nginx.conf

@@ -1,51 +1,79 @@
-worker_processes 1;
+upstream diode-ingester {
-
-events {
-    worker_connections 1024;
-}
-
-http {
-    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
-                    '$status $body_bytes_sent "$http_referer" '
-                    '"$http_user_agent"';
-
-    access_log /var/log/nginx/access.log main;
-    error_log  /var/log/nginx/error.log warn;
-
-    upstream ingester_grpc {
   server diode-ingester:8081;
 }
 
-    upstream reconciler_grpc {
+upstream diode-reconciler {
   server diode-reconciler:8081;
 }
 
-    upstream auth_grpc {
+upstream diode-auth {
-        server diode-auth:8081;
+  server diode-auth:8080;
 }
 
 server {
-        listen 8080 http2;
+  listen 8080;
+  listen [::]:8080;
+  http2 on;
+  server_name localhost;
+  client_max_body_size 25m;
 
-        # Diode Ingester gRPC
+  location /auth/introspect {
-        location /diode.v1.IngesterService/ {
+    internal;
-            grpc_pass grpc://ingester_grpc;
+    proxy_method POST;
+    proxy_pass http://diode-auth/introspect;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header X-Original-URI $request_uri;
   }
 
-        # Diode Reconciler gRPC
+  location /diode/auth {
-        location /diode.v1.ReconcilerService/ {
+    rewrite /diode/auth/(.*) /$1 break;
-            grpc_pass grpc://reconciler_grpc;
+    proxy_pass http://diode-auth;
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
   }
 
-        # Diode Auth gRPC
+  location /diode/diode.v1.IngesterService {
-        location /diode.v1.AuthService/ {
+    auth_request /auth/introspect;
-            grpc_pass grpc://auth_grpc;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;
+
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-ingester;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
+  }
+
+  location /diode/diode.v1.ReconcilerService {
+    auth_request /auth/introspect;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;
+
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-reconciler;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
   }
 
-        # Health check
   location /health {
     return 200 'OK';
     add_header Content-Type text/plain;
   }
+
+  location @error401 {
+    return 401 '{"error":"unauthorized","error_description":"Authentication required"}';
+  }
+
+  location @error403 {
+    return 403 '{"error":"forbidden","error_description":"Access denied"}';
   }
 }

oauth2/client/bootstrap-clients.sh

@@ -1,40 +1,56 @@
-#!/usr/bin/env sh
+#!/usr/bin/env bash
-set -e
 
-HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
+set -euo pipefail
-CREDENTIALS_FILE="/client-credentials/client-credentials.json"
 
-echo "Waiting for Hydra to be ready..."
+# Constants
-until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
+CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json"
-  echo "  Hydra not ready yet, retrying in 3s..."
-  sleep 3
-done
-echo "Hydra is ready."
 
-CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
+# Create the credentials file if it doesn't exist
-
+if [ ! -f "$CREDENTIALS_FILE" ]; then
-for i in $(seq 0 $((CLIENT_COUNT - 1))); do
+  echo "ERROR: credentials file $CREDENTIALS_FILE not found"
-  CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
+  exit 1
-  CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
-
-  echo "Checking client: ${CLIENT_ID}"
-
-  # Check if client already exists
-  HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
-  if [ "${HTTP_CODE}" = "200" ]; then
-    echo "  Client '${CLIENT_ID}' already exists, skipping."
-    continue
 fi
 
-  echo "  Registering client '${CLIENT_ID}'..."
+# Wait for Hydra to be ready
-  wget -qO- --header="Content-Type: application/json" \
+sleep 3
-    --post-data="${CLIENT_JSON}" \
-    "${HYDRA_ADMIN_URL}/admin/clients" || {
-      echo "  ERROR: Failed to register client '${CLIENT_ID}'"
-      exit 1
-    }
-  echo ""
-  echo "  Client '${CLIENT_ID}' registered successfully."
-done
 
-echo "All OAuth2 clients registered."
+# Function to create client
+create_client() {
+  local client_id=$1
+  local client_secret=$2
+  local scope=$3
+  local exists_in_hydra=false
+
+  # Check if client exists in Hydra
+  if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then
+    exists_in_hydra=true
+  fi
+
+  # Log client existence status
+  if [ "$exists_in_hydra" = true ]; then
+    echo "INFO: client $client_id exists in Hydra"
+    return 0
+  fi
+
+  # Create new client if it doesn't exist in Hydra
+  if [ "$exists_in_hydra" = false ]; then
+    client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \
+      --id $client_id \
+      --secret $client_secret \
+      --grant-type "client_credentials" \
+      --response-type "token" \
+      --scope "$scope" \
+      --token-endpoint-auth-method "client_secret_post" \
+      --format json)
+
+    echo "INFO: client $client_id created"
+  fi
+}
+
+# Load client credentials
+jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do
+  client_id=$(echo "$client" | jq -r '.client_id')
+  client_secret=$(echo "$client" | jq -r '.client_secret')
+  scope=$(echo "$client" | jq -r '.scope')
+  create_client "$client_id" "$client_secret" "$scope"
+done

setup.sh

@@ -76,7 +76,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
     "client_name": "Diode Ingester",
     "grant_types": ["client_credentials"],
     "token_endpoint_auth_method": "client_secret_post",
-    "scope": "diode:ingester",
+    "scope": "diode:ingest",
     "audience": ["diode-ingester"]
   },
   {
@@ -148,58 +148,54 @@ chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
 echo "==> Writing Orb Agent config..."
 cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" <<EOF
 orb:
+  config_manager:
+    active: local
   backends:
     network_discovery:
-      # NMAP-based network scan of the target subnet
-      scan_policy:
-        targets:
-          - "10.0.0.0/24"
-        interval: "1800"   # every 30 minutes
-        config:
-          top_ports: 100
-          os_detection: true
-          service_detection: true
-      diode:
-        target: "grpc://${HOST_IP}:8080"
-        client_id: "diode-ingester"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
-        site: "main"
-
     snmp_discovery:
-      # SNMPv2c discovery across the subnet
+    common:
-      scan_policy:
+      diode:
+        target: grpc://${HOST_IP}:8080/diode
+        client_id: diode-ingester
+        client_secret: ${INGESTER_CLIENT_SECRET}
+        agent_name: orb-agent-01
+  policies:
+    network_discovery:
+      nmap_scan:
+        config:
+          schedule: "*/30 * * * *"
+          timeout: 300
+          defaults:
+            site: "main"
+        scope:
           targets:
             - "10.0.0.0/24"
-        interval: "21600"  # every 6 hours
+    snmp_discovery:
+      snmp_scan:
         config:
-          communities:
+          schedule: "0 */6 * * *"
-            - "public"
+          timeout: 300
-          versions:
+          snmp_timeout: 10
-            - "2c"
+          retries: 3
-          ports:
+          defaults:
-            - 161
-      diode:
-        target: "grpc://${HOST_IP}:8080"
-        client_id: "diode-ingester"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
             site: "main"
-
+        scope:
-    device_discovery:
-      # NAPALM SSH-based device interrogation for known devices
-      scan_policy:
           targets:
-          - "10.0.0.1"     # gateway / core router (update as needed)
+            - host: "10.0.0.0/24"
-        interval: "43200"  # every 12 hours
+          authentication:
-        config:
+            protocol_version: "SNMPv2c"
-          driver: "ios"
+            community: "public"
-          credentials:
+    # device_discovery:
-            username: "admin"
+    #   napalm_scan:
-            password: "admin"
+    #     config:
-      diode:
+    #       schedule: "0 */12 * * *"
-        target: "grpc://${HOST_IP}:8080"
+    #       defaults:
-        client_id: "diode-ingester"
+    #         site: "main"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
+    #     scope:
-        site: "main"
+    #       - driver: ios
+    #         hostname: 10.0.0.1
+    #         username: admin
+    #         password: admin
 EOF
 
 echo "==> Writing nginx.conf..."