Add ingestion test and fix OAuth2 scopes and bootstrap logic

- Add tests/test_ingestion.py for end-to-end Diode pipeline verification - Fix OAuth2 client scopes: reconciler uses diode:reconcile, netbox-to-diode needs diode:read diode:write netbox:read netbox:write - Rewrite bootstrap-clients.sh with upsert behavior (delete+recreate) so scope and secret changes are applied on restart - Rewrite nginx.conf in setup.sh to match upstream auth_request architecture - Update .claude/settings.json with expanded tool permissions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
2026-02-28 01:55:37 -07:00 · 2026-02-28 01:55:37 -07:00 · fbde598be3
commit fbde598be3
parent 2f459e6f4a
4 changed files with 241 additions and 93 deletions
--- a/.claude/settings.json
+++ b/.claude/settings.json
@ -3,7 +3,26 @@
    "allow": [
      "Bash(/home/user/netbox-diode-project/.venv/bin/python --version)",
      "Bash(/home/user/netbox-diode-project/.venv/bin/pip --version)",
-      "Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)"
+      "Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)",
+      "Bash(git commit:*)",
+      "Bash(git config user.name \"sam\")",
+      "Bash(git config user.email \"info@apodacalab.com\")",
+      "Bash(git push origin main)",
+      "Bash(docker compose down)",
+      "Bash(docker compose up -d)",
+      "Bash(docker ps --filter \"name=netbox-diode-project\" --format \"table {{.Names}}\\\\t{{.Status}}\")",
+      "WebFetch(domain:github.com)",
+      "WebFetch(domain:pypi.org)",
+      "WebFetch(domain:docs.netboxlabs.com)",
+      "WebFetch(domain:netboxlabs.com)",
+      "WebFetch(domain:gist.github.com)",
+      "WebFetch(domain:raw.githubusercontent.com)",
+      "WebFetch(domain:virtualwires.wordpress.com)",
+      "WebFetch(domain:python-ipmi.readthedocs.io)",
+      "WebFetch(domain:proxmoxer.github.io)",
+      "WebFetch(domain:docs.openstack.org)",
+      "WebFetch(domain:deepwiki.com)",
+      "WebFetch(domain:docs.ansible.com)"
    ]
  }
 }
--- a/oauth2/client/bootstrap-clients.sh
+++ b/oauth2/client/bootstrap-clients.sh
@ -26,25 +26,22 @@ create_client() {
    exists_in_hydra=true
  fi

-  # Log client existence status
+  # Upsert behavior: remove stale client definition so scope/secret updates are applied.
  if [ "$exists_in_hydra" = true ]; then
-    echo "INFO: client $client_id exists in Hydra"
-    return 0
+    echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret"
+    hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null
  fi

-  # Create new client if it doesn't exist in Hydra
-  if [ "$exists_in_hydra" = false ]; then
-    client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \
-      --id $client_id \
-      --secret $client_secret \
+  hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \
+    --id "$client_id" \
+    --secret "$client_secret" \
    --grant-type "client_credentials" \
    --response-type "token" \
    --scope "$scope" \
    --token-endpoint-auth-method "client_secret_post" \
-      --format json)
+    --format json >/dev/null

-    echo "INFO: client $client_id created"
-  fi
+  echo "INFO: client $client_id created/updated"
 }

 # Load client credentials
--- a/setup.sh
+++ b/setup.sh
@ -85,7 +85,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
    "client_name": "Diode Reconciler",
    "grant_types": ["client_credentials"],
    "token_endpoint_auth_method": "client_secret_post",
-    "scope": "diode:reconciler",
+    "scope": "diode:reconcile",
    "audience": ["diode-reconciler"]
  },
  {
@ -94,7 +94,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
    "client_name": "NetBox to Diode",
    "grant_types": ["client_credentials"],
    "token_endpoint_auth_method": "client_secret_post",
-    "scope": "diode:netbox-to-diode",
+    "scope": "diode:read diode:write netbox:read netbox:write",
    "audience": ["netbox-to-diode"]
  }
 ]
@ -102,46 +102,59 @@ EOF

 echo "==> Writing OAuth2 bootstrap script..."
 cat > "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" <<'BOOTSTRAP_EOF'
-#!/usr/bin/env sh
-set -e
+#!/usr/bin/env bash

-HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
-CREDENTIALS_FILE="/client-credentials/client-credentials.json"
+set -euo pipefail

-echo "Waiting for Hydra to be ready..."
-until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
-  echo "  Hydra not ready yet, retrying in 3s..."
-  sleep 3
-done
-echo "Hydra is ready."
+# Constants
+CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json"

-CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
+# Create the credentials file if it doesn't exist
+if [ ! -f "$CREDENTIALS_FILE" ]; then
+  echo "ERROR: credentials file $CREDENTIALS_FILE not found"
+  exit 1
+fi

-for i in $(seq 0 $((CLIENT_COUNT - 1))); do
-  CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
-  CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
+# Wait for Hydra to be ready
+sleep 3

-  echo "Checking client: ${CLIENT_ID}"
+# Function to create client
+create_client() {
+  local client_id=$1
+  local client_secret=$2
+  local scope=$3
+  local exists_in_hydra=false

-  # Check if client already exists
-  HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
-  if [ "${HTTP_CODE}" = "200" ]; then
-    echo "  Client '${CLIENT_ID}' already exists, skipping."
-    continue
+  # Check if client exists in Hydra
+  if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then
+    exists_in_hydra=true
  fi

-  echo "  Registering client '${CLIENT_ID}'..."
-  wget -qO- --header="Content-Type: application/json" \
-    --post-data="${CLIENT_JSON}" \
-    "${HYDRA_ADMIN_URL}/admin/clients" || {
-      echo "  ERROR: Failed to register client '${CLIENT_ID}'"
-      exit 1
-    }
-  echo ""
-  echo "  Client '${CLIENT_ID}' registered successfully."
-done
+  # Upsert behavior: remove stale client definition so scope/secret updates are applied.
+  if [ "$exists_in_hydra" = true ]; then
+    echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret"
+    hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null
+  fi

-echo "All OAuth2 clients registered."
+  hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \
+    --id "$client_id" \
+    --secret "$client_secret" \
+    --grant-type "client_credentials" \
+    --response-type "token" \
+    --scope "$scope" \
+    --token-endpoint-auth-method "client_secret_post" \
+    --format json >/dev/null
+
+  echo "INFO: client $client_id created/updated"
+}
+
+# Load client credentials
+jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do
+  client_id=$(echo "$client" | jq -r '.client_id')
+  client_secret=$(echo "$client" | jq -r '.client_secret')
+  scope=$(echo "$client" | jq -r '.scope')
+  create_client "$client_id" "$client_secret" "$scope"
+done
 BOOTSTRAP_EOF
 chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"

@ -173,15 +186,27 @@ orb:
    snmp_discovery:
      snmp_scan:
        config:
-          schedule: "0 */6 * * *"
-          timeout: 300
-          snmp_timeout: 10
-          retries: 3
+          # Staged SNMP rollout: crawl known hosts with more tolerant timing to avoid partial walks.
+          schedule: "15 */6 * * *"
+          timeout: 180
+          snmp_timeout: 15
+          retries: 2
          defaults:
            site: "main"
+            role: "Network Device"
+            if_type: "other"
+            device:
+              manufacturer: "Unknown SNMP Manufacturer"
+              model: "Unknown SNMP Model"
+            interface_patterns:
+              - match: ".*"
+                type: "other"
+            interface:
+              description: "Discovered by orb snmp_discovery"
        scope:
          targets:
-            - host: "10.0.0.0/24"
+            - host: "10.10.20.182"
+            - host: "10.10.20.55"
          authentication:
            protocol_version: "SNMPv2c"
            community: "public"
@ -201,55 +226,83 @@ EOF
 echo "==> Writing nginx.conf..."
 mkdir -p "${SCRIPT_DIR}/nginx"
 cat > "${SCRIPT_DIR}/nginx/nginx.conf" <<'NGINX_EOF'
-worker_processes 1;
-
-events {
-    worker_connections 1024;
+upstream diode-ingester {
+  server diode-ingester:8081;
 }

-http {
-    log_format main '$remote_addr - $remote_user [$time_local] "$request" '
-                    '$status $body_bytes_sent "$http_referer" '
-                    '"$http_user_agent"';
-
-    access_log /var/log/nginx/access.log main;
-    error_log  /var/log/nginx/error.log warn;
-
-    upstream ingester_grpc {
-        server diode-ingester:8081;
-    }
-
-    upstream reconciler_grpc {
+upstream diode-reconciler {
  server diode-reconciler:8081;
+}
+
+upstream diode-auth {
+  server diode-auth:8080;
+}
+
+server {
+  listen 8080;
+  listen [::]:8080;
+  http2 on;
+  server_name localhost;
+  client_max_body_size 25m;
+
+  location /auth/introspect {
+    internal;
+    proxy_method POST;
+    proxy_pass http://diode-auth/introspect;
+    proxy_pass_request_body off;
+    proxy_set_header Content-Length "";
+    proxy_set_header X-Original-URI $request_uri;
  }

-    upstream auth_grpc {
-        server diode-auth:8081;
+  location /diode/auth {
+    rewrite /diode/auth/(.*) /$1 break;
+    proxy_pass http://diode-auth;
+
+    proxy_set_header Host $host;
+    proxy_set_header X-Real-IP $remote_addr;
+    proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    proxy_set_header X-Forwarded-Proto $scheme;
  }

-    server {
-        listen 8080 http2;
+  location /diode/diode.v1.IngesterService {
+    auth_request /auth/introspect;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;

-        # Diode Ingester gRPC
-        location /diode.v1.IngesterService/ {
-            grpc_pass grpc://ingester_grpc;
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-ingester;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
  }

-        # Diode Reconciler gRPC
-        location /diode.v1.ReconcilerService/ {
-            grpc_pass grpc://reconciler_grpc;
+  location /diode/diode.v1.ReconcilerService {
+    auth_request /auth/introspect;
+    auth_request_set $auth_status $upstream_status;
+    error_page 401 = @error401;
+    error_page 403 = @error403;
+
+    rewrite /diode/(.*) /$1 break;
+    grpc_pass grpc://diode-reconciler;
+    grpc_set_header Host $host;
+    grpc_set_header X-Real-IP $remote_addr;
+    grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+    grpc_set_header X-Forwarded-Proto $scheme;
  }

-        # Diode Auth gRPC
-        location /diode.v1.AuthService/ {
-            grpc_pass grpc://auth_grpc;
-        }
-
-        # Health check
  location /health {
    return 200 'OK';
    add_header Content-Type text/plain;
  }
+
+  location @error401 {
+    return 401 '{"error":"unauthorized","error_description":"Authentication required"}';
+  }
+
+  location @error403 {
+    return 403 '{"error":"forbidden","error_description":"Access denied"}';
  }
 }
 NGINX_EOF
--- a/tests/test_ingestion.py
+++ b/tests/test_ingestion.py
@ -0,0 +1,79 @@
+#!/usr/bin/env python3
+"""
+Minimal Diode ingestion test.
+
+Pushes a test Site and Device through the Diode pipeline to verify
+the full flow: SDK -> nginx -> ingester -> Redis -> reconciler -> NetBox
+"""
+
+import os
+import sys
+
+from netboxlabs.diode.sdk import DiodeClient
+from netboxlabs.diode.sdk.ingester import (
+    Device,
+    DeviceRole,
+    DeviceType,
+    Entity,
+    Manufacturer,
+    Platform,
+    Site,
+)
+
+DIODE_TARGET = os.getenv("DIODE_TARGET", "grpc://localhost:8080/diode")
+DIODE_CLIENT_ID = os.getenv("DIODE_CLIENT_ID", os.getenv("INGESTER_CLIENT_ID", "diode-ingester"))
+DIODE_CLIENT_SECRET = os.getenv("DIODE_CLIENT_SECRET", os.getenv("INGESTER_CLIENT_SECRET"))
+
+if not DIODE_CLIENT_SECRET:
+    print("ERROR: DIODE_CLIENT_SECRET must be set")
+    sys.exit(1)
+
+
+def main():
+    print(f"Connecting to Diode at {DIODE_TARGET}...")
+
+    with DiodeClient(
+        target=DIODE_TARGET,
+        client_id=DIODE_CLIENT_ID,
+        client_secret=DIODE_CLIENT_SECRET,
+        app_name="diode-test",
+        app_version="0.0.1",
+    ) as client:
+        entities = [
+            # Test Site
+            Entity(site=Site(
+                name="Test Site - Diode",
+                status="active",
+                description="Created by Diode ingestion test",
+            )),
+
+            # Test Device
+            Entity(device=Device(
+                name="test-device-diode-01",
+                device_type=DeviceType(
+                    model="Test Model",
+                    manufacturer=Manufacturer(name="Test Manufacturer"),
+                ),
+                platform=Platform(name="Linux"),
+                site=Site(name="Test Site - Diode"),
+                role=DeviceRole(name="Test Role"),
+                status="active",
+                serial="DIODE-TEST-001",
+                comments="Created by Diode ingestion test",
+            )),
+        ]
+
+        print(f"Ingesting {len(entities)} entities...")
+        response = client.ingest(entities=entities)
+
+        if response.errors:
+            print(f"ERRORS: {response.errors}")
+            sys.exit(1)
+        else:
+            print("Ingestion successful!")
+            print("Check NetBox for 'Test Site - Diode' and 'test-device-diode-01'")
+            print(f"  -> The reconciler will process these shortly.")
+
+
+if __name__ == "__main__":
+    main()