Add ingestion test and fix OAuth2 scopes and bootstrap logic
- Add tests/test_ingestion.py for end-to-end Diode pipeline verification - Fix OAuth2 client scopes: reconciler uses diode:reconcile, netbox-to-diode needs diode:read diode:write netbox:read netbox:write - Rewrite bootstrap-clients.sh with upsert behavior (delete+recreate) so scope and secret changes are applied on restart - Rewrite nginx.conf in setup.sh to match upstream auth_request architecture - Update .claude/settings.json with expanded tool permissions Co-Authored-By: Claude Opus 4.6 <noreply@anthropic.com>
This commit is contained in:
parent
2f459e6f4a
commit
fbde598be3
@ -3,7 +3,26 @@
|
||||
"allow": [
|
||||
"Bash(/home/user/netbox-diode-project/.venv/bin/python --version)",
|
||||
"Bash(/home/user/netbox-diode-project/.venv/bin/pip --version)",
|
||||
"Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)"
|
||||
"Bash(git add .gitignore .claude/settings.json docker-compose.yml init-hydra-db.sh nginx/nginx.conf oauth2/client/bootstrap-clients.sh setup.sh)",
|
||||
"Bash(git commit:*)",
|
||||
"Bash(git config user.name \"sam\")",
|
||||
"Bash(git config user.email \"info@apodacalab.com\")",
|
||||
"Bash(git push origin main)",
|
||||
"Bash(docker compose down)",
|
||||
"Bash(docker compose up -d)",
|
||||
"Bash(docker ps --filter \"name=netbox-diode-project\" --format \"table {{.Names}}\\\\t{{.Status}}\")",
|
||||
"WebFetch(domain:github.com)",
|
||||
"WebFetch(domain:pypi.org)",
|
||||
"WebFetch(domain:docs.netboxlabs.com)",
|
||||
"WebFetch(domain:netboxlabs.com)",
|
||||
"WebFetch(domain:gist.github.com)",
|
||||
"WebFetch(domain:raw.githubusercontent.com)",
|
||||
"WebFetch(domain:virtualwires.wordpress.com)",
|
||||
"WebFetch(domain:python-ipmi.readthedocs.io)",
|
||||
"WebFetch(domain:proxmoxer.github.io)",
|
||||
"WebFetch(domain:docs.openstack.org)",
|
||||
"WebFetch(domain:deepwiki.com)",
|
||||
"WebFetch(domain:docs.ansible.com)"
|
||||
]
|
||||
}
|
||||
}
|
||||
|
||||
@ -26,25 +26,22 @@ create_client() {
|
||||
exists_in_hydra=true
|
||||
fi
|
||||
|
||||
# Log client existence status
|
||||
# Upsert behavior: remove stale client definition so scope/secret updates are applied.
|
||||
if [ "$exists_in_hydra" = true ]; then
|
||||
echo "INFO: client $client_id exists in Hydra"
|
||||
return 0
|
||||
echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret"
|
||||
hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null
|
||||
fi
|
||||
|
||||
# Create new client if it doesn't exist in Hydra
|
||||
if [ "$exists_in_hydra" = false ]; then
|
||||
client_output=$(hydra create oauth2-client --endpoint $HYDRA_ADMIN_URL \
|
||||
--id $client_id \
|
||||
--secret $client_secret \
|
||||
--grant-type "client_credentials" \
|
||||
--response-type "token" \
|
||||
--scope "$scope" \
|
||||
--token-endpoint-auth-method "client_secret_post" \
|
||||
--format json)
|
||||
hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \
|
||||
--id "$client_id" \
|
||||
--secret "$client_secret" \
|
||||
--grant-type "client_credentials" \
|
||||
--response-type "token" \
|
||||
--scope "$scope" \
|
||||
--token-endpoint-auth-method "client_secret_post" \
|
||||
--format json >/dev/null
|
||||
|
||||
echo "INFO: client $client_id created"
|
||||
fi
|
||||
echo "INFO: client $client_id created/updated"
|
||||
}
|
||||
|
||||
# Load client credentials
|
||||
|
||||
207
setup.sh
207
setup.sh
@ -85,7 +85,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
|
||||
"client_name": "Diode Reconciler",
|
||||
"grant_types": ["client_credentials"],
|
||||
"token_endpoint_auth_method": "client_secret_post",
|
||||
"scope": "diode:reconciler",
|
||||
"scope": "diode:reconcile",
|
||||
"audience": ["diode-reconciler"]
|
||||
},
|
||||
{
|
||||
@ -94,7 +94,7 @@ cat > "${SCRIPT_DIR}/oauth2/client/client-credentials.json" <<EOF
|
||||
"client_name": "NetBox to Diode",
|
||||
"grant_types": ["client_credentials"],
|
||||
"token_endpoint_auth_method": "client_secret_post",
|
||||
"scope": "diode:netbox-to-diode",
|
||||
"scope": "diode:read diode:write netbox:read netbox:write",
|
||||
"audience": ["netbox-to-diode"]
|
||||
}
|
||||
]
|
||||
@ -102,46 +102,59 @@ EOF
|
||||
|
||||
echo "==> Writing OAuth2 bootstrap script..."
|
||||
cat > "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh" <<'BOOTSTRAP_EOF'
|
||||
#!/usr/bin/env sh
|
||||
set -e
|
||||
#!/usr/bin/env bash
|
||||
|
||||
HYDRA_ADMIN_URL="${HYDRA_ADMIN_URL:-http://hydra:4445}"
|
||||
CREDENTIALS_FILE="/client-credentials/client-credentials.json"
|
||||
set -euo pipefail
|
||||
|
||||
echo "Waiting for Hydra to be ready..."
|
||||
until wget -qO- "${HYDRA_ADMIN_URL}/health/ready" 2>/dev/null | grep -q '"status":"ok"'; do
|
||||
echo " Hydra not ready yet, retrying in 3s..."
|
||||
sleep 3
|
||||
done
|
||||
echo "Hydra is ready."
|
||||
# Constants
|
||||
CREDENTIALS_FILE="/etc/config/oauth2/client/client-credentials.json"
|
||||
|
||||
CLIENT_COUNT=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(len(json.load(sys.stdin)))")
|
||||
# Create the credentials file if it doesn't exist
|
||||
if [ ! -f "$CREDENTIALS_FILE" ]; then
|
||||
echo "ERROR: credentials file $CREDENTIALS_FILE not found"
|
||||
exit 1
|
||||
fi
|
||||
|
||||
for i in $(seq 0 $((CLIENT_COUNT - 1))); do
|
||||
CLIENT_JSON=$(cat "${CREDENTIALS_FILE}" | python3 -c "import sys,json; print(json.dumps(json.load(sys.stdin)[$i]))")
|
||||
CLIENT_ID=$(echo "${CLIENT_JSON}" | python3 -c "import sys,json; print(json.load(sys.stdin)['client_id'])")
|
||||
# Wait for Hydra to be ready
|
||||
sleep 3
|
||||
|
||||
echo "Checking client: ${CLIENT_ID}"
|
||||
# Function to create client
|
||||
create_client() {
|
||||
local client_id=$1
|
||||
local client_secret=$2
|
||||
local scope=$3
|
||||
local exists_in_hydra=false
|
||||
|
||||
# Check if client already exists
|
||||
HTTP_CODE=$(wget -qO/dev/null -S "${HYDRA_ADMIN_URL}/admin/clients/${CLIENT_ID}" 2>&1 | grep "HTTP/" | tail -1 | awk '{print $2}')
|
||||
if [ "${HTTP_CODE}" = "200" ]; then
|
||||
echo " Client '${CLIENT_ID}' already exists, skipping."
|
||||
continue
|
||||
# Check if client exists in Hydra
|
||||
if hydra get oauth2-client $client_id --endpoint $HYDRA_ADMIN_URL >/dev/null 2>&1; then
|
||||
exists_in_hydra=true
|
||||
fi
|
||||
|
||||
echo " Registering client '${CLIENT_ID}'..."
|
||||
wget -qO- --header="Content-Type: application/json" \
|
||||
--post-data="${CLIENT_JSON}" \
|
||||
"${HYDRA_ADMIN_URL}/admin/clients" || {
|
||||
echo " ERROR: Failed to register client '${CLIENT_ID}'"
|
||||
exit 1
|
||||
}
|
||||
echo ""
|
||||
echo " Client '${CLIENT_ID}' registered successfully."
|
||||
done
|
||||
# Upsert behavior: remove stale client definition so scope/secret updates are applied.
|
||||
if [ "$exists_in_hydra" = true ]; then
|
||||
echo "INFO: client $client_id exists in Hydra, replacing to refresh scope/secret"
|
||||
hydra delete oauth2-client "$client_id" --endpoint "$HYDRA_ADMIN_URL" >/dev/null
|
||||
fi
|
||||
|
||||
echo "All OAuth2 clients registered."
|
||||
hydra create oauth2-client --endpoint "$HYDRA_ADMIN_URL" \
|
||||
--id "$client_id" \
|
||||
--secret "$client_secret" \
|
||||
--grant-type "client_credentials" \
|
||||
--response-type "token" \
|
||||
--scope "$scope" \
|
||||
--token-endpoint-auth-method "client_secret_post" \
|
||||
--format json >/dev/null
|
||||
|
||||
echo "INFO: client $client_id created/updated"
|
||||
}
|
||||
|
||||
# Load client credentials
|
||||
jq -c '.[]' "$CREDENTIALS_FILE" | while read -r client; do
|
||||
client_id=$(echo "$client" | jq -r '.client_id')
|
||||
client_secret=$(echo "$client" | jq -r '.client_secret')
|
||||
scope=$(echo "$client" | jq -r '.scope')
|
||||
create_client "$client_id" "$client_secret" "$scope"
|
||||
done
|
||||
BOOTSTRAP_EOF
|
||||
chmod +x "${SCRIPT_DIR}/oauth2/client/bootstrap-clients.sh"
|
||||
|
||||
@ -173,15 +186,27 @@ orb:
|
||||
snmp_discovery:
|
||||
snmp_scan:
|
||||
config:
|
||||
schedule: "0 */6 * * *"
|
||||
timeout: 300
|
||||
snmp_timeout: 10
|
||||
retries: 3
|
||||
# Staged SNMP rollout: crawl known hosts with more tolerant timing to avoid partial walks.
|
||||
schedule: "15 */6 * * *"
|
||||
timeout: 180
|
||||
snmp_timeout: 15
|
||||
retries: 2
|
||||
defaults:
|
||||
site: "main"
|
||||
role: "Network Device"
|
||||
if_type: "other"
|
||||
device:
|
||||
manufacturer: "Unknown SNMP Manufacturer"
|
||||
model: "Unknown SNMP Model"
|
||||
interface_patterns:
|
||||
- match: ".*"
|
||||
type: "other"
|
||||
interface:
|
||||
description: "Discovered by orb snmp_discovery"
|
||||
scope:
|
||||
targets:
|
||||
- host: "10.0.0.0/24"
|
||||
- host: "10.10.20.182"
|
||||
- host: "10.10.20.55"
|
||||
authentication:
|
||||
protocol_version: "SNMPv2c"
|
||||
community: "public"
|
||||
@ -201,56 +226,84 @@ EOF
|
||||
echo "==> Writing nginx.conf..."
|
||||
mkdir -p "${SCRIPT_DIR}/nginx"
|
||||
cat > "${SCRIPT_DIR}/nginx/nginx.conf" <<'NGINX_EOF'
|
||||
worker_processes 1;
|
||||
|
||||
events {
|
||||
worker_connections 1024;
|
||||
upstream diode-ingester {
|
||||
server diode-ingester:8081;
|
||||
}
|
||||
|
||||
http {
|
||||
log_format main '$remote_addr - $remote_user [$time_local] "$request" '
|
||||
'$status $body_bytes_sent "$http_referer" '
|
||||
'"$http_user_agent"';
|
||||
upstream diode-reconciler {
|
||||
server diode-reconciler:8081;
|
||||
}
|
||||
|
||||
access_log /var/log/nginx/access.log main;
|
||||
error_log /var/log/nginx/error.log warn;
|
||||
upstream diode-auth {
|
||||
server diode-auth:8080;
|
||||
}
|
||||
|
||||
upstream ingester_grpc {
|
||||
server diode-ingester:8081;
|
||||
}
|
||||
server {
|
||||
listen 8080;
|
||||
listen [::]:8080;
|
||||
http2 on;
|
||||
server_name localhost;
|
||||
client_max_body_size 25m;
|
||||
|
||||
upstream reconciler_grpc {
|
||||
server diode-reconciler:8081;
|
||||
}
|
||||
location /auth/introspect {
|
||||
internal;
|
||||
proxy_method POST;
|
||||
proxy_pass http://diode-auth/introspect;
|
||||
proxy_pass_request_body off;
|
||||
proxy_set_header Content-Length "";
|
||||
proxy_set_header X-Original-URI $request_uri;
|
||||
}
|
||||
|
||||
upstream auth_grpc {
|
||||
server diode-auth:8081;
|
||||
}
|
||||
location /diode/auth {
|
||||
rewrite /diode/auth/(.*) /$1 break;
|
||||
proxy_pass http://diode-auth;
|
||||
|
||||
server {
|
||||
listen 8080 http2;
|
||||
proxy_set_header Host $host;
|
||||
proxy_set_header X-Real-IP $remote_addr;
|
||||
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
proxy_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Diode Ingester gRPC
|
||||
location /diode.v1.IngesterService/ {
|
||||
grpc_pass grpc://ingester_grpc;
|
||||
}
|
||||
location /diode/diode.v1.IngesterService {
|
||||
auth_request /auth/introspect;
|
||||
auth_request_set $auth_status $upstream_status;
|
||||
error_page 401 = @error401;
|
||||
error_page 403 = @error403;
|
||||
|
||||
# Diode Reconciler gRPC
|
||||
location /diode.v1.ReconcilerService/ {
|
||||
grpc_pass grpc://reconciler_grpc;
|
||||
}
|
||||
rewrite /diode/(.*) /$1 break;
|
||||
grpc_pass grpc://diode-ingester;
|
||||
grpc_set_header Host $host;
|
||||
grpc_set_header X-Real-IP $remote_addr;
|
||||
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
grpc_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
# Diode Auth gRPC
|
||||
location /diode.v1.AuthService/ {
|
||||
grpc_pass grpc://auth_grpc;
|
||||
}
|
||||
location /diode/diode.v1.ReconcilerService {
|
||||
auth_request /auth/introspect;
|
||||
auth_request_set $auth_status $upstream_status;
|
||||
error_page 401 = @error401;
|
||||
error_page 403 = @error403;
|
||||
|
||||
# Health check
|
||||
location /health {
|
||||
return 200 'OK';
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
}
|
||||
rewrite /diode/(.*) /$1 break;
|
||||
grpc_pass grpc://diode-reconciler;
|
||||
grpc_set_header Host $host;
|
||||
grpc_set_header X-Real-IP $remote_addr;
|
||||
grpc_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
|
||||
grpc_set_header X-Forwarded-Proto $scheme;
|
||||
}
|
||||
|
||||
location /health {
|
||||
return 200 'OK';
|
||||
add_header Content-Type text/plain;
|
||||
}
|
||||
|
||||
location @error401 {
|
||||
return 401 '{"error":"unauthorized","error_description":"Authentication required"}';
|
||||
}
|
||||
|
||||
location @error403 {
|
||||
return 403 '{"error":"forbidden","error_description":"Access denied"}';
|
||||
}
|
||||
}
|
||||
NGINX_EOF
|
||||
|
||||
|
||||
79
tests/test_ingestion.py
Normal file
79
tests/test_ingestion.py
Normal file
@ -0,0 +1,79 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Minimal Diode ingestion test.
|
||||
|
||||
Pushes a test Site and Device through the Diode pipeline to verify
|
||||
the full flow: SDK -> nginx -> ingester -> Redis -> reconciler -> NetBox
|
||||
"""
|
||||
|
||||
import os
|
||||
import sys
|
||||
|
||||
from netboxlabs.diode.sdk import DiodeClient
|
||||
from netboxlabs.diode.sdk.ingester import (
|
||||
Device,
|
||||
DeviceRole,
|
||||
DeviceType,
|
||||
Entity,
|
||||
Manufacturer,
|
||||
Platform,
|
||||
Site,
|
||||
)
|
||||
|
||||
DIODE_TARGET = os.getenv("DIODE_TARGET", "grpc://localhost:8080/diode")
|
||||
DIODE_CLIENT_ID = os.getenv("DIODE_CLIENT_ID", os.getenv("INGESTER_CLIENT_ID", "diode-ingester"))
|
||||
DIODE_CLIENT_SECRET = os.getenv("DIODE_CLIENT_SECRET", os.getenv("INGESTER_CLIENT_SECRET"))
|
||||
|
||||
if not DIODE_CLIENT_SECRET:
|
||||
print("ERROR: DIODE_CLIENT_SECRET must be set")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def main():
|
||||
print(f"Connecting to Diode at {DIODE_TARGET}...")
|
||||
|
||||
with DiodeClient(
|
||||
target=DIODE_TARGET,
|
||||
client_id=DIODE_CLIENT_ID,
|
||||
client_secret=DIODE_CLIENT_SECRET,
|
||||
app_name="diode-test",
|
||||
app_version="0.0.1",
|
||||
) as client:
|
||||
entities = [
|
||||
# Test Site
|
||||
Entity(site=Site(
|
||||
name="Test Site - Diode",
|
||||
status="active",
|
||||
description="Created by Diode ingestion test",
|
||||
)),
|
||||
|
||||
# Test Device
|
||||
Entity(device=Device(
|
||||
name="test-device-diode-01",
|
||||
device_type=DeviceType(
|
||||
model="Test Model",
|
||||
manufacturer=Manufacturer(name="Test Manufacturer"),
|
||||
),
|
||||
platform=Platform(name="Linux"),
|
||||
site=Site(name="Test Site - Diode"),
|
||||
role=DeviceRole(name="Test Role"),
|
||||
status="active",
|
||||
serial="DIODE-TEST-001",
|
||||
comments="Created by Diode ingestion test",
|
||||
)),
|
||||
]
|
||||
|
||||
print(f"Ingesting {len(entities)} entities...")
|
||||
response = client.ingest(entities=entities)
|
||||
|
||||
if response.errors:
|
||||
print(f"ERRORS: {response.errors}")
|
||||
sys.exit(1)
|
||||
else:
|
||||
print("Ingestion successful!")
|
||||
print("Check NetBox for 'Test Site - Diode' and 'test-device-diode-01'")
|
||||
print(f" -> The reconciler will process these shortly.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Loading…
x
Reference in New Issue
Block a user