Fix Diode stack configuration to match upstream architecture

Major changes to align with the official netboxlabs/diode docker-compose: docker-compose.yml: - Upgrade Hydra from v2.2 to v25.4.0 (latest stable) - Fix env var names: drop DIODE_ prefix on ingester/reconciler/auth (DIODE_REDIS_HOST -> REDIS_HOST, DIODE_GRPC_PORT -> removed, etc.) - Remove AUTH_GRPC_TARGET from ingester — auth is handled by nginx via HTTP subrequests to diode-auth, not by the ingester directly - Point DIODE_AUTH_TOKEN_URL to diode-auth:8080/token (not Hydra) - Add Postgres connection vars to reconciler (MIGRATION_ENABLED, etc.) - Mount nginx.conf as /etc/nginx/conf.d/default.conf (not nginx.conf) - Use netboxlabs/diode-auth image for bootstrap (has hydra CLI + jq) - Add Hydra JWT strategy config (STRATEGIES_ACCESS_TOKEN: jwt) - Add orb-agent run command with explicit config path - Expose Hydra ports 4444/4445 for external token requests - Add Hydra DB env vars to Postgres for init script nginx/nginx.conf: - Rewrite to match official Diode nginx architecture - Auth via auth_request subrequests to diode-auth:8080 (HTTP) - gRPC pass to ingester/reconciler on :8081 after auth - Add /diode/auth proxy for token endpoint access - Add error handlers for 401/403 oauth2/client/bootstrap-clients.sh: - Rewrite to use hydra CLI (create/get oauth2-client) instead of raw wget calls to Hydra admin API - Use jq for JSON parsing instead of python3 - Idempotent: checks if client exists before creating setup.sh: - Fix OAuth2 scope: diode:ingester -> diode:ingest - Rewrite orb-agent config to match current agent.yaml schema (config_manager, policies with cron schedules, scope-based targets) Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-27 21:51:40 -07:00 · 2026-02-27 21:51:40 -07:00 · 2f459e6f4a
commit 2f459e6f4a
parent c5a0245dd2
4 changed files with 206 additions and 153 deletions
--- a/docker-compose.yml
+++ b/docker-compose.yml
@ -7,7 +7,7 @@ services:
    ports:
      - "8080:8080"
    volumes:
-      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
+      - ./nginx/nginx.conf:/etc/nginx/conf.d/default.conf:ro
    depends_on:
      diode-ingester:
        condition: service_started
@ -23,11 +23,9 @@ services:
  diode-ingester:
    image: netboxlabs/diode-ingester:latest
    environment:
-      DIODE_REDIS_HOST: redis
-      DIODE_REDIS_PORT: "6379"
-      DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      DIODE_GRPC_PORT: "8081"
-      DIODE_AUTH_GRPC_TARGET: diode-auth:8081
+      REDIS_HOST: redis
+      REDIS_PORT: "6379"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
    depends_on:
      redis:
        condition: service_healthy
@ -39,20 +37,26 @@ services:
  diode-reconciler:
    image: netboxlabs/diode-reconciler:latest
    environment:
-      DIODE_REDIS_HOST: redis
-      DIODE_REDIS_PORT: "6379"
-      DIODE_REDIS_PASSWORD: "${REDIS_PASSWORD}"
-      DIODE_GRPC_PORT: "8081"
-      DIODE_NETBOX_API_URL: "${NETBOX_API_URL}"
-      DIODE_NETBOX_API_TOKEN: "${NETBOX_API_TOKEN}"
-      DIODE_RECONCILER_CLIENT_ID: "${RECONCILER_CLIENT_ID}"
-      DIODE_RECONCILER_CLIENT_SECRET: "${RECONCILER_CLIENT_SECRET}"
-      DIODE_AUTH_GRPC_TARGET: diode-auth:8081
+      REDIS_HOST: redis
+      REDIS_PORT: "6379"
+      REDIS_PASSWORD: "${REDIS_PASSWORD}"
+      POSTGRES_HOST: postgres
+      POSTGRES_PORT: "5432"
+      POSTGRES_DB_NAME: "${DIODE_DB_NAME}"
+      POSTGRES_USER: "${DIODE_DB_USER}"
+      POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
+      MIGRATION_ENABLED: "true"
+      NETBOX_DIODE_PLUGIN_API_BASE_URL: "${NETBOX_API_URL}/api/plugins/diode"
+      DIODE_AUTH_TOKEN_URL: http://diode-auth:8080/token
      DIODE_TO_NETBOX_CLIENT_ID: "${NETBOX_TO_DIODE_CLIENT_ID}"
      DIODE_TO_NETBOX_CLIENT_SECRET: "${NETBOX_TO_DIODE_CLIENT_SECRET}"
+    volumes:
+      - ./oauth2/client:/etc/config/oauth2/client:ro
    depends_on:
      redis:
        condition: service_healthy
+      postgres:
+        condition: service_healthy
      diode-auth-bootstrap:
        condition: service_completed_successfully
    restart: unless-stopped
@ -63,9 +67,9 @@ services:
  diode-auth:
    image: netboxlabs/diode-auth:latest
    environment:
-      DIODE_GRPC_PORT: "8081"
-      DIODE_HYDRA_ADMIN_URL: http://hydra:4445
-      DIODE_HYDRA_PUBLIC_URL: http://hydra:4444
+      HTTP_PORT: "8080"
+      OAUTH2_PUBLIC_SERVER_URL: http://hydra:4444
+      OAUTH2_ADMIN_SERVER_URL: http://hydra:4445
    depends_on:
      hydra:
        condition: service_started
@ -75,12 +79,12 @@ services:
  # Diode Auth Bootstrap — registers OAuth2 clients in Hydra (one-shot)
  # =========================================================================
  diode-auth-bootstrap:
-    image: python:3.12-alpine
-    command: /client-credentials/bootstrap-clients.sh
+    image: netboxlabs/diode-auth:latest
+    command: ["/bin/sh", "/etc/config/oauth2/client/bootstrap-clients.sh"]
    environment:
      HYDRA_ADMIN_URL: http://hydra:4445
    volumes:
-      - ./oauth2/client:/client-credentials:ro
+      - ./oauth2/client:/etc/config/oauth2/client:ro
    depends_on:
      hydra:
        condition: service_started
@ -90,13 +94,18 @@ services:
  # Hydra — OAuth2 / OpenID Connect server
  # =========================================================================
  hydra:
-    image: oryd/hydra:v2.2
-    command: serve all --dev --config /etc/hydra/hydra.yml
+    image: oryd/hydra:v25.4.0
+    command: serve all --dev
+    ports:
+      - "4444:4444"
+      - "4445:4445"
    environment:
      DSN: "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
      SECRETS_SYSTEM: "${HYDRA_SYSTEM_SECRET}"
-      URLS_SELF_ISSUER: http://hydra:4444/
-      URLS_SELF_PUBLIC: http://hydra:4444/
+      URLS_SELF_ISSUER: http://${HOST_IP}:4444/
+      URLS_SELF_PUBLIC: http://${HOST_IP}:4444/
+      STRATEGIES_ACCESS_TOKEN: jwt
+      STRATEGIES_JWT_SCOPE_CLAIM: both
    depends_on:
      hydra-migrate:
        condition: service_completed_successfully
@ -106,7 +115,7 @@ services:
  # Hydra Migrate — runs Hydra database migrations (one-shot)
  # =========================================================================
  hydra-migrate:
-    image: oryd/hydra:v2.2
+    image: oryd/hydra:v25.4.0
    command: migrate sql --yes "postgres://${HYDRA_DB_USER}:${HYDRA_DB_PASSWORD}@postgres:5432/${HYDRA_DB_NAME}?sslmode=disable"
    depends_on:
      postgres:
@ -140,6 +149,9 @@ services:
      POSTGRES_USER: "${DIODE_DB_USER}"
      POSTGRES_PASSWORD: "${DIODE_DB_PASSWORD}"
      POSTGRES_DB: "${DIODE_DB_NAME}"
+      HYDRA_DB_USER: "${HYDRA_DB_USER}"
+      HYDRA_DB_PASSWORD: "${HYDRA_DB_PASSWORD}"
+      HYDRA_DB_NAME: "${HYDRA_DB_NAME}"
    healthcheck:
      test: ["CMD-SHELL", "pg_isready -U ${DIODE_DB_USER} -d ${DIODE_DB_NAME}"]
      start_period: 10s
@ -157,6 +169,7 @@ services:
  orb-agent:
    image: netboxlabs/orb-agent:latest
    network_mode: host
+    command: ["run", "-c", "/opt/orb/agent.yaml"]
    volumes:
      - ./orb-agent/agent.yaml:/opt/orb/agent.yaml:ro
    depends_on:
--- a/nginx/nginx.conf
+++ b/nginx/nginx.conf
@ -1,51 +1,79 @@
-worker_processes 1;
-
-events {
-    worker_connections 1024;
+upstream diode-ingester {
+  server diode-ingester:8081;
 }

-http {
-    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
-                    '$status $body_bytes_sent "$http_referer" '
-                    '"$http_user_agent"';
-
-    access_log /var/log/nginx/access.log main;
-    error_log  /var/log/nginx/error.log warn;
-
-    upstream ingester_grpc {
-        server diode-ingester:8081;
-    }
-
-    upstream reconciler_grpc {
+upstream diode-reconciler {
  server diode-reconciler:8081;
+}
+
+upstream diode-auth {
+  server diode-auth:8080;
+}
+
+server {
+  listen 8080;
+  listen [::]:8080;
+  http2 on;
+  server_name localhost;
+  client_max_body_size 25m;
+
+  location /auth/introspect {
+    internal;
+    proxy_method POST;
+    proxy_pass http://diode-auth/introspect;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header X-Original-URI $request_uri;
  }

-    upstream auth_grpc {
-        server diode-auth:8081;
+  location /diode/auth {
+    rewrite /diode/auth/(.*) /$1 break;
+    proxy_pass http://diode-auth;
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
  }

-    server {
-        listen 8080 http2;
+  location /diode/diode.v1.IngesterService {
+    auth_request /auth/introspect;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;

-        # Diode Ingester gRPC
-        location /diode.v1.IngesterService/ {
-            grpc_pass grpc://ingester_grpc;
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-ingester;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
  }

-        # Diode Reconciler gRPC
-        location /diode.v1.ReconcilerService/ {
-            grpc_pass grpc://reconciler_grpc;
+  location /diode/diode.v1.ReconcilerService {
+    auth_request /auth/introspect;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;
+
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-reconciler;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
  }

-        # Diode Auth gRPC
-        location /diode.v1.AuthService/ {
-            grpc_pass grpc://auth_grpc;
-        }
-
-        # Health check
  location /health {
    return 200 'OK';
    add_header Content-Type text/plain;
  }
+
+  location @error401 {
+    return 401 '{"error":"unauthorized","error_description":"Authentication required"}';
+  }
+
+  location @error403 {
+    return 403 '{"error":"forbidden","error_description":"Access denied"}';
  }
 }
--- a/oauth2/client/bootstrap-clients.sh
+++ b/oauth2/client/bootstrap-clients.sh
@ -1,40 +1,56 @@
-#!/usr/bin/env sh
-set -e
+#!/usr/bin/env bash

-HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
-CREDENTIALS_FILE="/client-credentials/client-credentials.json"
+set -euo pipefail

-echo "Waiting for Hydra to be ready..."
-until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
-  echo "  Hydra not ready yet, retrying in 3s..."
-  sleep 3
-done
-echo "Hydra is ready."
+# Constants
+CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json"

-CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
+# Create the credentials file if it doesn't exist
+if [ ! -f "$CREDENTIALS_FILE" ]; then
+  echo "ERROR: credentials file $CREDENTIALS_FILE not found"
+  exit 1
+fi

-for i in $(seq 0 $((CLIENT_COUNT - 1))); do
-  CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
-  CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
+# Wait for Hydra to be ready
+sleep 3

-  echo "Checking client: ${CLIENT_ID}"
+# Function to create client
+create_client() {
+  local client_id=$1
+  local client_secret=$2
+  local scope=$3
+  local exists_in_hydra=false

-  # Check if client already exists
-  HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
-  if [ "${HTTP_CODE}" = "200" ]; then
-    echo "  Client '${CLIENT_ID}' already exists, skipping."
-    continue
+  # Check if client exists in Hydra
+  if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then
+    exists_in_hydra=true
  fi

-  echo "  Registering client '${CLIENT_ID}'..."
-  wget -qO- --header="Content-Type: application/json" \
-    --post-data="${CLIENT_JSON}" \
-    "${HYDRA_ADMIN_URL}/admin/clients" || {
-      echo "  ERROR: Failed to register client '${CLIENT_ID}'"
-      exit 1
-    }
-  echo ""
-  echo "  Client '${CLIENT_ID}' registered successfully."
-done
+  # Log client existence status
+  if [ "$exists_in_hydra" = true ]; then
+    echo "INFO: client $client_id exists in Hydra"
+    return 0
+  fi

-echo "All OAuth2 clients registered."
+  # Create new client if it doesn't exist in Hydra
+  if [ "$exists_in_hydra" = false ]; then
+    client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \
+      --id $client_id \
+      --secret $client_secret \
+      --grant-type "client_credentials" \
+      --response-type "token" \
+      --scope "$scope" \
+      --token-endpoint-auth-method "client_secret_post" \
+      --format json)
+
+    echo "INFO: client $client_id created"
+  fi
+}
+
+# Load client credentials
+jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do
+  client_id=$(echo "$client" | jq -r '.client_id')
+  client_secret=$(echo "$client" | jq -r '.client_secret')
+  scope=$(echo "$client" | jq -r '.scope')
+  create_client "$client_id" "$client_secret" "$scope"
+done
--- a/setup.sh
+++ b/setup.sh
@ -76,7 +76,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
    "client_name": "Diode Ingester",
    "grant_types": ["client_credentials"],
    "token_endpoint_auth_method": "client_secret_post",
-    "scope": "diode:ingester",
+    "scope": "diode:ingest",
    "audience": ["diode-ingester"]
  },
  {
@ -148,58 +148,54 @@ chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
 echo "==> Writing Orb Agent config..."
 cat > "${SCRIPT_DIR}/orb-agent/agent.yaml" <<EOF
 orb:
+  config_manager:
+    active: local
  backends:
    network_discovery:
-      # NMAP-based network scan of the target subnet
-      scan_policy:
-        targets:
-          - "10.0.0.0/24"
-        interval: "1800"   # every 30 minutes
-        config:
-          top_ports: 100
-          os_detection: true
-          service_detection: true
-      diode:
-        target: "grpc://${HOST_IP}:8080"
-        client_id: "diode-ingester"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
-        site: "main"
-
    snmp_discovery:
-      # SNMPv2c discovery across the subnet
-      scan_policy:
+    common:
+      diode:
+        target: grpc://${HOST_IP}:8080/diode
+        client_id: diode-ingester
+        client_secret: ${INGESTER_CLIENT_SECRET}
+        agent_name: orb-agent-01
+  policies:
+    network_discovery:
+      nmap_scan:
+        config:
+          schedule: "*/30 * * * *"
+          timeout: 300
+          defaults:
+            site: "main"
+        scope:
          targets:
            - "10.0.0.0/24"
-        interval: "21600"  # every 6 hours
+    snmp_discovery:
+      snmp_scan:
        config:
-          communities:
-            - "public"
-          versions:
-            - "2c"
-          ports:
-            - 161
-      diode:
-        target: "grpc://${HOST_IP}:8080"
-        client_id: "diode-ingester"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
+          schedule: "0 */6 * * *"
+          timeout: 300
+          snmp_timeout: 10
+          retries: 3
+          defaults:
            site: "main"
-
-    device_discovery:
-      # NAPALM SSH-based device interrogation for known devices
-      scan_policy:
+        scope:
          targets:
-          - "10.0.0.1"     # gateway / core router (update as needed)
-        interval: "43200"  # every 12 hours
-        config:
-          driver: "ios"
-          credentials:
-            username: "admin"
-            password: "admin"
-      diode:
-        target: "grpc://${HOST_IP}:8080"
-        client_id: "diode-ingester"
-        client_secret: "${INGESTER_CLIENT_SECRET}"
-        site: "main"
+            - host: "10.0.0.0/24"
+          authentication:
+            protocol_version: "SNMPv2c"
+            community: "public"
+    # device_discovery:
+    #   napalm_scan:
+    #     config:
+    #       schedule: "0 */12 * * *"
+    #       defaults:
+    #         site: "main"
+    #     scope:
+    #       - driver: ios
+    #         hostname: 10.0.0.1
+    #         username: admin
+    #         password: admin
 EOF

 echo "==> Writing nginx.conf..."