From fbde598be3517ebb8a1b6d142f2bafe33d641b8a Mon Sep 17 00:00:00 2001 From: sam Date: Sat, 28 Feb 2026 01:55:37 -0700 Subject: [PATCH] Add ingestion test and fix OAuth2 scopes and bootstrap logic - Add tests/test_ingestion.py for end-to-end Diode pipeline verification - Fix OAuth2 client scopes: reconciler uses diode:reconcile, netbox-to-diode needs diode:read diode:write netbox:read netbox:write - Rewrite bootstrap-clients.sh with upsert behavior (delete+recreate) so scope and secret changes are applied on restart - Rewrite nginx.conf in setup.sh to match upstream auth_request architecture - Update .claude/settings.json with expanded tool permissions Co-Authored-By: Claude Opus 4.6 --- .claude/settings.json | 21 ++- oauth2/client/bootstrap-clients.sh | 27 ++-- setup.sh | 207 ++++++++++++++++++----------- tests/test_ingestion.py | 79 +++++++++++ 4 files changed, 241 insertions(+), 93 deletions(-) create mode 100644 tests/test_ingestion.py diff --git a/.claude/settings.json b/.claude/settings.json index 74a641a..ca3c6cc 100644 --- a/.claude/settings.json +++ b/.claude/settings.json @@ -3,7 +3,26 @@ "allow": [ "Bash(/home/user/netbox-diode-project/.venv/bin/python --version)", "Bash(/home/user/netbox-diode-project/.venv/bin/pip --version)", - "Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)" + "Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)", + "Bash(git commit:*)", + "Bash(git config user.name \"sam\")", + "Bash(git config user.email \"info@apodacalab.com\")", + "Bash(git push origin main)", + "Bash(docker compose down)", + "Bash(docker compose up -d)", + "Bash(docker ps --filter \"name=netbox-diode-project\" --format \"table {{.Names}}\\\\t{{.Status}}\")", + "WebFetch(domain:github.com)", + "WebFetch(domain:pypi.org)", + "WebFetch(domain:docs.netboxlabs.com)", + "WebFetch(domain:netboxlabs.com)", + "WebFetch(domain:gist.github.com)", + "WebFetch(domain:raw.githubusercontent.com)", + "WebFetch(domain:virtualwires.wordpress.com)", + "WebFetch(domain:python-ipmi.readthedocs.io)", + "WebFetch(domain:proxmoxer.github.io)", + "WebFetch(domain:docs.openstack.org)", + "WebFetch(domain:deepwiki.com)", + "WebFetch(domain:docs.ansible.com)" ] } } diff --git a/oauth2/client/bootstrap-clients.sh b/oauth2/client/bootstrap-clients.sh index 0563c8b..0ab5b84 100755 --- a/oauth2/client/bootstrap-clients.sh +++ b/oauth2/client/bootstrap-clients.sh @@ -26,25 +26,22 @@ create_client() { exists_in_hydra=true fi - # Log client existence status + # Upsert behavior: remove stale client definition so scope/secret updates are applied. if [ "$exists_in_hydra" = true ]; then - echo "INFO: client $client_id exists in Hydra" - return 0 + echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret" + hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null fi - # Create new client if it doesn't exist in Hydra - if [ "$exists_in_hydra" = false ]; then - client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \ - --id $client_id \ - --secret $client_secret \ - --grant-type "client_credentials" \ - --response-type "token" \ - --scope "$scope" \ - --token-endpoint-auth-method "client_secret_post" \ - --format json) + hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \ + --id "$client_id" \ + --secret "$client_secret" \ + --grant-type "client_credentials" \ + --response-type "token" \ + --scope "$scope" \ + --token-endpoint-auth-method "client_secret_post" \ + --format json >/dev/null - echo "INFO: client $client_id created" - fi + echo "INFO: client $client_id created/updated" } # Load client credentials diff --git a/setup.sh b/setup.sh index 16ec6ee..7a84fd0 100755 --- a/setup.sh +++ b/setup.sh @@ -85,7 +85,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" < "${SCRIPT_DIR}/oauth2/client/client-credentials.json" < Writing OAuth2 bootstrap script..." cat > "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" <<'BOOTSTRAP_EOF' -#!/usr/bin/env sh -set -e +#!/usr/bin/env bash -HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}" -CREDENTIALS_FILE="/client-credentials/client-credentials.json" +set -euo pipefail -echo "Waiting for Hydra to be ready..." -until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do - echo " Hydra not ready yet, retrying in 3s..." - sleep 3 -done -echo "Hydra is ready." +# Constants +CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json" -CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))") +# Create the credentials file if it doesn't exist +if [ ! -f "$CREDENTIALS_FILE" ]; then + echo "ERROR: credentials file $CREDENTIALS_FILE not found" + exit 1 +fi -for i in $(seq 0 $((CLIENT_COUNT - 1))); do - CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))") - CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])") +# Wait for Hydra to be ready +sleep 3 - echo "Checking client: ${CLIENT_ID}" +# Function to create client +create_client() { + local client_id=$1 + local client_secret=$2 + local scope=$3 + local exists_in_hydra=false - # Check if client already exists - HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}') - if [ "${HTTP_CODE}" = "200" ]; then - echo " Client '${CLIENT_ID}' already exists, skipping." - continue + # Check if client exists in Hydra + if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then + exists_in_hydra=true fi - echo " Registering client '${CLIENT_ID}'..." - wget -qO- --header="Content-Type: application/json" \ - --post-data="${CLIENT_JSON}" \ - "${HYDRA_ADMIN_URL}/admin/clients" || { - echo " ERROR: Failed to register client '${CLIENT_ID}'" - exit 1 - } - echo "" - echo " Client '${CLIENT_ID}' registered successfully." -done + # Upsert behavior: remove stale client definition so scope/secret updates are applied. + if [ "$exists_in_hydra" = true ]; then + echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret" + hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null + fi -echo "All OAuth2 clients registered." + hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \ + --id "$client_id" \ + --secret "$client_secret" \ + --grant-type "client_credentials" \ + --response-type "token" \ + --scope "$scope" \ + --token-endpoint-auth-method "client_secret_post" \ + --format json >/dev/null + + echo "INFO: client $client_id created/updated" +} + +# Load client credentials +jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do + client_id=$(echo "$client" | jq -r '.client_id') + client_secret=$(echo "$client" | jq -r '.client_secret') + scope=$(echo "$client" | jq -r '.scope') + create_client "$client_id" "$client_secret" "$scope" +done BOOTSTRAP_EOF chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" @@ -173,15 +186,27 @@ orb: snmp_discovery: snmp_scan: config: - schedule: "0 */6 * * *" - timeout: 300 - snmp_timeout: 10 - retries: 3 + # Staged SNMP rollout: crawl known hosts with more tolerant timing to avoid partial walks. + schedule: "15 */6 * * *" + timeout: 180 + snmp_timeout: 15 + retries: 2 defaults: site: "main" + role: "Network Device" + if_type: "other" + device: + manufacturer: "Unknown SNMP Manufacturer" + model: "Unknown SNMP Model" + interface_patterns: + - match: ".*" + type: "other" + interface: + description: "Discovered by orb snmp_discovery" scope: targets: - - host: "10.0.0.0/24" + - host: "10.10.20.182" + - host: "10.10.20.55" authentication: protocol_version: "SNMPv2c" community: "public" @@ -201,56 +226,84 @@ EOF echo "==> Writing nginx.conf..." mkdir -p "${SCRIPT_DIR}/nginx" cat > "${SCRIPT_DIR}/nginx/nginx.conf" <<'NGINX_EOF' -worker_processes 1; - -events { - worker_connections 1024; +upstream diode-ingester { + server diode-ingester:8081; } -http { - log_format main '$remote_addr - $remote_user [$time_local] "$request" ' - '$status $body_bytes_sent "$http_referer" ' - '"$http_user_agent"'; +upstream diode-reconciler { + server diode-reconciler:8081; +} - access_log /var/log/nginx/access.log main; - error_log /var/log/nginx/error.log warn; +upstream diode-auth { + server diode-auth:8080; +} - upstream ingester_grpc { - server diode-ingester:8081; - } +server { + listen 8080; + listen [::]:8080; + http2 on; + server_name localhost; + client_max_body_size 25m; - upstream reconciler_grpc { - server diode-reconciler:8081; - } + location /auth/introspect { + internal; + proxy_method POST; + proxy_pass http://diode-auth/introspect; + proxy_pass_request_body off; + proxy_set_header Content-Length ""; + proxy_set_header X-Original-URI $request_uri; + } - upstream auth_grpc { - server diode-auth:8081; - } + location /diode/auth { + rewrite /diode/auth/(.*) /$1 break; + proxy_pass http://diode-auth; - server { - listen 8080 http2; + proxy_set_header Host $host; + proxy_set_header X-Real-IP $remote_addr; + proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + proxy_set_header X-Forwarded-Proto $scheme; + } - # Diode Ingester gRPC - location /diode.v1.IngesterService/ { - grpc_pass grpc://ingester_grpc; - } + location /diode/diode.v1.IngesterService { + auth_request /auth/introspect; + auth_request_set $auth_status $upstream_status; + error_page 401 = @error401; + error_page 403 = @error403; - # Diode Reconciler gRPC - location /diode.v1.ReconcilerService/ { - grpc_pass grpc://reconciler_grpc; - } + rewrite /diode/(.*) /$1 break; + grpc_pass grpc://diode-ingester; + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + } - # Diode Auth gRPC - location /diode.v1.AuthService/ { - grpc_pass grpc://auth_grpc; - } + location /diode/diode.v1.ReconcilerService { + auth_request /auth/introspect; + auth_request_set $auth_status $upstream_status; + error_page 401 = @error401; + error_page 403 = @error403; - # Health check - location /health { - return 200 'OK'; - add_header Content-Type text/plain; - } - } + rewrite /diode/(.*) /$1 break; + grpc_pass grpc://diode-reconciler; + grpc_set_header Host $host; + grpc_set_header X-Real-IP $remote_addr; + grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for; + grpc_set_header X-Forwarded-Proto $scheme; + } + + location /health { + return 200 'OK'; + add_header Content-Type text/plain; + } + + location @error401 { + return 401 '{"error":"unauthorized","error_description":"Authentication required"}'; + } + + location @error403 { + return 403 '{"error":"forbidden","error_description":"Access denied"}'; + } } NGINX_EOF diff --git a/tests/test_ingestion.py b/tests/test_ingestion.py new file mode 100644 index 0000000..e47c4be --- /dev/null +++ b/tests/test_ingestion.py @@ -0,0 +1,79 @@ +#!/usr/bin/env python3 +""" +Minimal Diode ingestion test. + +Pushes a test Site and Device through the Diode pipeline to verify +the full flow: SDK -> nginx -> ingester -> Redis -> reconciler -> NetBox +""" + +import os +import sys + +from netboxlabs.diode.sdk import DiodeClient +from netboxlabs.diode.sdk.ingester import ( + Device, + DeviceRole, + DeviceType, + Entity, + Manufacturer, + Platform, + Site, +) + +DIODE_TARGET = os.getenv("DIODE_TARGET", "grpc://localhost:8080/diode") +DIODE_CLIENT_ID = os.getenv("DIODE_CLIENT_ID", os.getenv("INGESTER_CLIENT_ID", "diode-ingester")) +DIODE_CLIENT_SECRET = os.getenv("DIODE_CLIENT_SECRET", os.getenv("INGESTER_CLIENT_SECRET")) + +if not DIODE_CLIENT_SECRET: + print("ERROR: DIODE_CLIENT_SECRET must be set") + sys.exit(1) + + +def main(): + print(f"Connecting to Diode at {DIODE_TARGET}...") + + with DiodeClient( + target=DIODE_TARGET, + client_id=DIODE_CLIENT_ID, + client_secret=DIODE_CLIENT_SECRET, + app_name="diode-test", + app_version="0.0.1", + ) as client: + entities = [ + # Test Site + Entity(site=Site( + name="Test Site - Diode", + status="active", + description="Created by Diode ingestion test", + )), + + # Test Device + Entity(device=Device( + name="test-device-diode-01", + device_type=DeviceType( + model="Test Model", + manufacturer=Manufacturer(name="Test Manufacturer"), + ), + platform=Platform(name="Linux"), + site=Site(name="Test Site - Diode"), + role=DeviceRole(name="Test Role"), + status="active", + serial="DIODE-TEST-001", + comments="Created by Diode ingestion test", + )), + ] + + print(f"Ingesting {len(entities)} entities...") + response = client.ingest(entities=entities) + + if response.errors: + print(f"ERRORS: {response.errors}") + sys.exit(1) + else: + print("Ingestion successful!") + print("Check NetBox for 'Test Site - Diode' and 'test-device-diode-01'") + print(f" -> The reconciler will process these shortly.") + + +if __name__ == "__main__": + main()