From cc0d20bf9efdabb214342194e19034d1e7aa39dc Mon Sep 17 00:00:00 2001 From: sam Date: Tue, 19 May 2026 07:04:38 -0700 Subject: [PATCH] Back AS Relationship Map with a materialized view MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The AS map previously exploded ~4.4M base_attrs AS_PATH rows live, three times per load (one per panel), ~1.8s each — slow enough that navigating away cancelled the queries mid-flight. Add mv_as_adjacency: undirected consecutive-AS pairs with occurrence counts over the full RIB (17k rows), refreshed hourly by pg_cron via REFRESH ... CONCURRENTLY. The dashboard panels now read the view in ~1ms. Min-occurrence options rescaled for full-RIB counts (2000/5000/10000/50000, default 2000 -> ~63-node graph). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../obmp/Maps-1006/as_relationship_map.json | 14 +++++----- postgres/scripts/006_obmp_matviews.sql | 26 +++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) create mode 100644 postgres/scripts/006_obmp_matviews.sql diff --git a/obmp-grafana/dashboards/obmp/Maps-1006/as_relationship_map.json b/obmp-grafana/dashboards/obmp/Maps-1006/as_relationship_map.json index 9d775e2..2ff9b83 100644 --- a/obmp-grafana/dashboards/obmp/Maps-1006/as_relationship_map.json +++ b/obmp-grafana/dashboards/obmp/Maps-1006/as_relationship_map.json @@ -1,6 +1,6 @@ { "annotations": {"list": [{"builtIn": 1,"datasource": {"type": "datasource","uid": "grafana"},"enable": true,"hide": true,"iconColor": "rgba(0, 211, 255, 1)","name": "Annotations & Alerts","type": "dashboard"}]}, - "description": "AS adjacency graph derived from consecutive AS pairs in observed AS_PATHs. Edge label = how many times that adjacency appears in a 200k-route sample. Raise 'Min occurrences' to thin the graph; set 'Focus AS' to a 1-hop view around one AS. Manual refresh — the query explodes ~200k AS_PATH arrays.", + "description": "AS adjacency graph derived from consecutive AS pairs in observed AS_PATHs. Reads the mv_as_adjacency materialized view (full RIB, refreshed hourly by pg_cron) so panels load instantly. Edge label = how many times that adjacency appears. Raise 'Min occurrences' to thin the graph; set 'Focus AS' for a 1-hop view around one AS.", "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, @@ -10,7 +10,7 @@ "panels": [ { "datasource": {"type": "postgres","uid": "obmp_postgres"}, - "description": "Each node is an AS (enriched from info_asn whois data); each edge is an adjacency seen in the AS_PATH sample. Edge label is the occurrence count.", + "description": "Each node is an AS (enriched from info_asn whois data); each edge is an adjacency seen in the AS_PATH data. Edge label is the occurrence count.", "fieldConfig": {"defaults": {},"overrides": []}, "gridPos": {"h": 24,"w": 24,"x": 0,"y": 0}, "id": 1, @@ -19,13 +19,13 @@ { "datasource": {"type": "postgres","uid": "obmp_postgres"}, "format": "table", - "rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n),\nfedges AS (\n SELECT * FROM edges\n WHERE '$focus_as' = '' OR x::text = '$focus_as' OR y::text = '$focus_as'\n),\nnlist AS (SELECT x AS asn FROM fedges UNION SELECT y FROM fedges),\ndeg AS (SELECT asn, COUNT(*) AS d FROM (SELECT x AS asn FROM fedges UNION ALL SELECT y FROM fedges) z GROUP BY asn)\nSELECT n.asn::text AS id,\n COALESCE(NULLIF(ia.as_name,''),'AS'||n.asn) AS title,\n 'AS ' || n.asn AS mainstat,\n COALESCE(NULLIF(ia.country,''),'?') || ' · ' || dg.d::text || ' links' AS secondarystat,\n COALESCE(NULLIF(ia.org_name,''),'—') AS detail__org,\n COALESCE(NULLIF(ia.country,''),'—') AS detail__country,\n dg.d::text AS detail__degree\nFROM nlist n\nLEFT JOIN info_asn ia ON ia.asn = n.asn\nLEFT JOIN deg dg ON dg.asn = n.asn\nORDER BY dg.d DESC", + "rawSql": "WITH edges AS (\n SELECT asn_a, asn_b, occ FROM mv_as_adjacency\n WHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\n ORDER BY occ DESC LIMIT 300\n),\nnlist AS (SELECT asn_a AS asn FROM edges UNION SELECT asn_b FROM edges),\ndeg AS (SELECT asn, COUNT(*) AS d FROM (SELECT asn_a AS asn FROM edges UNION ALL SELECT asn_b FROM edges) z GROUP BY asn)\nSELECT n.asn::text AS id,\n COALESCE(NULLIF(ia.as_name,''),'AS'||n.asn) AS title,\n 'AS ' || n.asn AS mainstat,\n COALESCE(NULLIF(ia.country,''),'?') || ' · ' || dg.d::text || ' links' AS secondarystat,\n COALESCE(NULLIF(ia.org_name,''),'—') AS detail__org,\n COALESCE(NULLIF(ia.country,''),'—') AS detail__country,\n dg.d::text AS detail__degree\nFROM nlist n\nLEFT JOIN info_asn ia ON ia.asn = n.asn\nLEFT JOIN deg dg ON dg.asn = n.asn\nORDER BY dg.d DESC", "refId": "nodes" }, { "datasource": {"type": "postgres","uid": "obmp_postgres"}, "format": "table", - "rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n)\nSELECT x::text || '-' || y::text AS id,\n x::text AS source, y::text AS target,\n occ AS mainstat,\n occ::text || ' paths' AS detail__occurrences\nFROM edges\nWHERE '$focus_as' = '' OR x::text = '$focus_as' OR y::text = '$focus_as'", + "rawSql": "SELECT asn_a::text || '-' || asn_b::text AS id,\n asn_a::text AS source, asn_b::text AS target,\n occ AS mainstat,\n occ::text || ' paths' AS detail__occurrences\nFROM mv_as_adjacency\nWHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\nORDER BY occ DESC LIMIT 300", "refId": "edges" } ], @@ -34,7 +34,7 @@ }, { "datasource": {"type": "postgres","uid": "obmp_postgres"}, - "description": "The strongest AS adjacencies in the sample, with whois names for both endpoints.", + "description": "The strongest AS adjacencies, with whois names for both endpoints.", "fieldConfig": {"defaults": {"custom": {"align": "auto","displayMode": "auto"}},"overrides": [{"matcher": {"id": "byName","options": "Occurrences"},"properties": [{"id": "custom.displayMode","value": "gradient-gauge"},{"id": "color","value": {"mode": "continuous-BlPu"}}]}]}, "gridPos": {"h": 10,"w": 24,"x": 0,"y": 24}, "id": 2, @@ -43,7 +43,7 @@ { "datasource": {"type": "postgres","uid": "obmp_postgres"}, "format": "table", - "rawSql": "WITH sample AS (SELECT as_path FROM base_attrs WHERE as_path_count >= 2 LIMIT 200000),\npairs AS (\n SELECT a.asn AS src, b.asn AS dst\n FROM sample ba\n CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn,ord)\n JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn,ord) ON b.ord = a.ord + 1\n WHERE a.asn <> b.asn\n),\nedges AS (\n SELECT LEAST(src,dst) AS x, GREATEST(src,dst) AS y, COUNT(*) AS occ\n FROM pairs GROUP BY 1,2 HAVING COUNT(*) >= $min_occ ORDER BY occ DESC LIMIT 300\n)\nSELECT e.x AS \"AS A\",\n COALESCE(NULLIF(ax.as_name,''),'—') AS \"Name A\",\n e.y AS \"AS B\",\n COALESCE(NULLIF(ay.as_name,''),'—') AS \"Name B\",\n e.occ AS \"Occurrences\"\nFROM edges e\nLEFT JOIN info_asn ax ON ax.asn = e.x\nLEFT JOIN info_asn ay ON ay.asn = e.y\nWHERE '$focus_as' = '' OR e.x::text = '$focus_as' OR e.y::text = '$focus_as'\nORDER BY e.occ DESC", + "rawSql": "SELECT e.asn_a AS \"AS A\",\n COALESCE(NULLIF(ax.as_name,''),'—') AS \"Name A\",\n e.asn_b AS \"AS B\",\n COALESCE(NULLIF(ay.as_name,''),'—') AS \"Name B\",\n e.occ AS \"Occurrences\"\nFROM (\n SELECT asn_a, asn_b, occ FROM mv_as_adjacency\n WHERE occ >= $min_occ\n AND ('$focus_as' = '' OR asn_a::text = '$focus_as' OR asn_b::text = '$focus_as')\n ORDER BY occ DESC LIMIT 300\n) e\nLEFT JOIN info_asn ax ON ax.asn = e.asn_a\nLEFT JOIN info_asn ay ON ay.asn = e.asn_b\nORDER BY e.occ DESC", "refId": "A" } ], @@ -56,7 +56,7 @@ "tags": ["obmp", "obmp-nav", "obmp-maps", "asn"], "templating": { "list": [ - {"name": "min_occ","type": "custom","label": "Min occurrences","description": "Only draw adjacencies seen at least this many times in the 200k-route sample. Raise it to thin a cluttered graph.","query": "5,20,100,500","current": {"text": "20","value": "20"},"options": [{"text": "5","value": "5","selected": false},{"text": "20","value": "20","selected": true},{"text": "100","value": "100","selected": false},{"text": "500","value": "500","selected": false}],"hide": 0}, + {"name": "min_occ","type": "custom","label": "Min occurrences","description": "Only draw adjacencies seen at least this many times across the RIB. Raise it to thin a cluttered graph.","query": "2000,5000,10000,50000","current": {"text": "2000","value": "2000"},"options": [{"text": "2000","value": "2000","selected": true},{"text": "5000","value": "5000","selected": false},{"text": "10000","value": "10000","selected": false},{"text": "50000","value": "50000","selected": false}],"hide": 0}, {"name": "focus_as","type": "textbox","label": "Focus AS","description": "Optional. Enter an ASN to show only adjacencies touching that AS (1-hop view). Leave blank for the full graph.","query": "","current": {"text": "","value": ""},"options": [{"text": "","value": "","selected": true}],"hide": 0} ] }, diff --git a/postgres/scripts/006_obmp_matviews.sql b/postgres/scripts/006_obmp_matviews.sql new file mode 100644 index 0000000..31e7e1a --- /dev/null +++ b/postgres/scripts/006_obmp_matviews.sql @@ -0,0 +1,26 @@ +-- OBMP analytics materialized views +-- +-- Applied to the live database and version-controlled here for reproducibility. +-- Safe to run against an empty database at init time: the view is created empty +-- and the pg_cron job below populates/refreshes it hourly. + +-- AS adjacency: every undirected pair of consecutive ASes in an observed AS_PATH, +-- with an occurrence count. Backs the AS Relationship Map dashboard so its panels +-- read a 17k-row view instead of exploding ~4.4M base_attrs rows live. +CREATE MATERIALIZED VIEW IF NOT EXISTS mv_as_adjacency AS +SELECT LEAST(a.asn, b.asn) AS asn_a, + GREATEST(a.asn, b.asn) AS asn_b, + COUNT(*) AS occ +FROM base_attrs ba +CROSS JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY a(asn, ord) +JOIN LATERAL unnest(ba.as_path) WITH ORDINALITY b(asn, ord) ON b.ord = a.ord + 1 +WHERE ba.as_path_count >= 2 AND a.asn <> b.asn +GROUP BY 1, 2; + +-- Unique index required for REFRESH ... CONCURRENTLY (no read lock during refresh). +CREATE UNIQUE INDEX IF NOT EXISTS mv_as_adjacency_pair_idx ON mv_as_adjacency (asn_a, asn_b); +CREATE INDEX IF NOT EXISTS mv_as_adjacency_occ_idx ON mv_as_adjacency (occ DESC); + +-- Refresh hourly at minute 23. +SELECT cron.schedule('refresh-mv-as-adjacency', '23 * * * *', + 'REFRESH MATERIALIZED VIEW CONCURRENTLY mv_as_adjacency');